This dashboard describes the operational status and configuration of an ITSI instance. Because searches on this page access sensitive indexes (e.g. _internal) and REST endpoints, reports will be incomplete if not run by an Admin user.
table | seriesByName(\"cpu_cores\") | formatByType(cpu_coresColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"cpu_cores\") | pick(cpu_coresRowColorsEditorConfig)",
"rowBackgroundColors": "> table | seriesByName(\"cpu_cores\") | rangeValue(cpu_coresRowBackgroundColorsEditorConfig)"
},
"virtual_cpu_cores": {
"data": "> table | seriesByName(\"virtual_cpu_cores\") | formatByType(virtual_cpu_coresColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"virtual_cpu_cores\") | pick(virtual_cpu_coresRowColorsEditorConfig)",
"rowBackgroundColors": "> table | seriesByName(\"virtual_cpu_cores\") | rangeValue(virtual_cpu_coresRowBackgroundColorsEditorConfig)"
},
"physical_mem_MB": {
"data": "> table | seriesByName(\"physical_mem_MB\") | formatByType(physical_mem_MBColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"physical_mem_MB\") | pick(physical_mem_MBRowColorsEditorConfig)",
"rowBackgroundColors": "> table | seriesByName(\"physical_mem_MB\") | rangeValue(physical_mem_MBRowBackgroundColorsEditorConfig)"
},
"THP_kernel_settings": {
"data": "> table | seriesByName(\"THP_kernel_settings\") | formatByType(THP_kernel_settingsColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"THP_kernel_settings\") | pick(THP_kernel_settingsRowColorsEditorConfig)",
"rowBackgroundColors": "> table | seriesByName(\"THP_kernel_settings\") | matchValue(THP_kernel_settingsRowBackgroundColorsEditorConfig)"
}
}
},
"context": {
"cpu_coresColumnFormatEditorConfig": {
"number": {
"thousandSeparated": false,
"unitPosition": "after"
}
},
"cpu_coresRowColorsEditorConfig": [
"#ffffff"
],
"cpu_coresRowBackgroundColorsEditorConfig": [
{
"value": "#d93f3c",
"to": 12
},
{
"value": "#f7bc38",
"from": 12,
"to": 16
},
{
"value": "#65a637",
"from": 16
}
],
"virtual_cpu_coresColumnFormatEditorConfig": {
"number": {
"thousandSeparated": false,
"unitPosition": "after"
}
},
"physical_mem_MBColumnFormatEditorConfig": {
"number": {
"thousandSeparated": false,
"unitPosition": "after"
}
},
"THP_kernel_settingsColumnFormatEditorConfig": {
"string": {
"unitPosition": "after"
}
},
"THP_kernel_settingsRowColorsEditorConfig": [
"#ffffff"
],
"THP_kernel_settingsRowBackgroundColorsEditorConfig": [
{
"match": "ok",
"value": "#65a637"
},
{
"match": "not ok",
"value": "#d93f3c"
}
],
"virtual_cpu_coresRowColorsEditorConfig": [
"#ffffff"
],
"virtual_cpu_coresRowBackgroundColorsEditorConfig": [
{
"value": "#d93f3c",
"to": 24
},
{
"value": "#f7bc38",
"from": 24,
"to": 32
},
{
"value": "#65a637",
"from": 32
}
],
"physical_mem_MBRowColorsEditorConfig": [
"#ffffff"
],
"physical_mem_MBRowBackgroundColorsEditorConfig": [
{
"value": "#d93f3c",
"to": 12288
},
{
"value": "#f7bc38",
"from": 12288,
"to": 16384
},
{
"value": "#65a637",
"from": 16384
}
]
}
},
"viz_itsi_migration_status": {
"type": "splunk.table",
"dataSources": {
"primary": "ds_NTcJUPg0"
},
"title": "ITSI Migration Status",
"options": {
"showInternalFields": false
}
},
"viz_itsi_upgrade_readiness": {
"type": "splunk.table",
"dataSources": {
"primary": "ds_DreUmgu8"
},
"title": "ITSI Upgrade Readiness",
"eventHandlers": [
{
"type": "drilldown.customUrl",
"options": {
"url": "/app/itsi/upgrade_readiness",
"newTab": true
}
}
],
"description": "ITSI is ready for upgrade if all major prechecks succeed and no service template syncs are in progress"
},
"viz_basic_information": {
"type": "splunk.table",
"dataSources": {
"primary": "ds_FRTTZ2ac"
},
"title": "Basic ITSI Information",
"options": {
"columnFormat": {
"kvstore_data_not_ok": {
"data": "> table | seriesByName(\"kvstore_data_not_ok\") | formatByType(kvstore_data_not_okColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"kvstore_data_not_ok\") | pick(kvstore_data_not_okRowColorsEditorConfig)",
"rowBackgroundColors": "> table | seriesByName(\"kvstore_data_not_ok\") | rangeValue(kvstore_data_not_okRowBackgroundColorsEditorConfig)"
},
"HEC_bytes_indexed": {
"data": "> table | seriesByName(\"HEC_bytes_indexed\") | formatByType(HEC_bytes_indexedColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"HEC_bytes_indexed\") | pick(HEC_bytes_indexedRowColorsEditorConfig)",
"rowBackgroundColors": "> table | seriesByName(\"HEC_bytes_indexed\") | rangeValue(HEC_bytes_indexedRowBackgroundColorsEditorConfig)"
},
"service_count": {
"data": "> table | seriesByName(\"service_count\") | formatByType(service_countColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"service_count\") | pick(service_countRowColorsEditorConfig)",
"rowBackgroundColors": "> table | seriesByName(\"service_count\") | rangeValue(service_countRowBackgroundColorsEditorConfig)"
},
"kpi_base_searches": {
"data": "> table | seriesByName(\"kpi_base_searches\") | formatByType(kpi_base_searchesColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"kpi_base_searches\") | pick(kpi_base_searchesRowColorsEditorConfig)",
"rowBackgroundColors": "> table | seriesByName(\"kpi_base_searches\") | rangeValue(kpi_base_searchesRowBackgroundColorsEditorConfig)"
},
"kpi_adhoc_searches": {
"data": "> table | seriesByName(\"kpi_adhoc_searches\") | formatByType(kpi_adhoc_searchesColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"kpi_adhoc_searches\") | pick(kpi_adhoc_searchesRowColorsEditorConfig)",
"rowBackgroundColors": "> table | seriesByName(\"kpi_adhoc_searches\") | rangeValue(kpi_adhoc_searchesRowBackgroundColorsEditorConfig)"
},
"entity_count": {
"data": "> table | seriesByName(\"entity_count\") | formatByType(entity_countColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"entity_count\") | pick(entity_countRowColorsEditorConfig)",
"rowBackgroundColors": "> table | seriesByName(\"entity_count\") | rangeValue(entity_countRowBackgroundColorsEditorConfig)"
},
"kvstore_status": {
"data": "> table | seriesByName(\"kvstore_status\") | formatByType(kvstore_statusColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"kvstore_status\") | pick(kvstore_statusRowColorsEditorConfig)",
"rowBackgroundColors": "> table | seriesByName(\"kvstore_status\") | matchValue(kvstore_statusRowBackgroundColorsEditorConfig)"
},
"HEC_errors": {
"data": "> table | seriesByName(\"HEC_errors\") | formatByType(HEC_errorsColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"HEC_errors\") | pick(HEC_errorsRowColorsEditorConfig)",
"rowBackgroundColors": "> table | seriesByName(\"HEC_errors\") | rangeValue(HEC_errorsRowBackgroundColorsEditorConfig)"
},
"HEC_parser_errors": {
"data": "> table | seriesByName(\"HEC_parser_errors\") | formatByType(HEC_parser_errorsColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"HEC_parser_errors\") | pick(HEC_parser_errorsRowColorsEditorConfig)",
"rowBackgroundColors": "> table | seriesByName(\"HEC_parser_errors\") | rangeValue(HEC_parser_errorsRowBackgroundColorsEditorConfig)"
}
}
},
"context": {
"kvstore_data_not_okColumnFormatEditorConfig": {
"number": {
"thousandSeparated": false,
"unitPosition": "after"
}
},
"HEC_bytes_indexedColumnFormatEditorConfig": {
"number": {
"thousandSeparated": false,
"unitPosition": "after"
}
},
"service_countColumnFormatEditorConfig": {
"number": {
"thousandSeparated": false,
"unitPosition": "after"
}
},
"kpi_base_searchesColumnFormatEditorConfig": {
"number": {
"thousandSeparated": false,
"unitPosition": "after"
}
},
"kpi_adhoc_searchesColumnFormatEditorConfig": {
"number": {
"thousandSeparated": false,
"unitPosition": "after"
}
},
"entity_countColumnFormatEditorConfig": {
"number": {
"thousandSeparated": false,
"unitPosition": "after"
}
},
"kvstore_statusColumnFormatEditorConfig": {
"string": {
"unitPosition": "after"
}
},
"HEC_errorsColumnFormatEditorConfig": {
"number": {
"thousandSeparated": false,
"unitPosition": "after"
}
},
"HEC_bytes_indexedRowColorsEditorConfig": [
"#ffffff"
],
"HEC_bytes_indexedRowBackgroundColorsEditorConfig": [
{
"value": "#f7bc38",
"to": 1
},
{
"value": "#65A637",
"from": 1
}
],
"kvstore_data_not_okRowColorsEditorConfig": [
"#ffffff"
],
"kvstore_data_not_okRowBackgroundColorsEditorConfig": [
{
"value": "#65A637",
"to": 1
},
{
"value": "#F7BC38",
"from": 1
}
],
"HEC_errorsRowColorsEditorConfig": [
"#ffffff"
],
"HEC_errorsRowBackgroundColorsEditorConfig": [
{
"value": "#65A637",
"to": 1
},
{
"value": "#F7BC38",
"from": 1
}
],
"HEC_parser_errorsColumnFormatEditorConfig": {
"number": {
"thousandSeparated": false,
"unitPosition": "after"
}
},
"HEC_parser_errorsRowColorsEditorConfig": [
"#ffffff"
],
"HEC_parser_errorsRowBackgroundColorsEditorConfig": [
{
"value": "#65A637",
"to": 1
},
{
"value": "#F7BC38",
"from": 1
}
],
"service_countRowColorsEditorConfig": [
"#ffffff"
],
"service_countRowBackgroundColorsEditorConfig": [
{
"value": "#F7BC38",
"to": 1
},
{
"value": "#65A637",
"from": 1,
"to": 2000
},
{
"value": "#F7BC38",
"from": 2000
}
],
"kpi_base_searchesRowColorsEditorConfig": [
"#ffffff"
],
"kpi_base_searchesRowBackgroundColorsEditorConfig": [
{
"value": "#F7BC38",
"to": 1
},
{
"value": "#65A637",
"from": 1,
"to": 300
},
{
"value": "#F7BC38",
"from": 300
}
],
"kpi_adhoc_searchesRowColorsEditorConfig": [
"#ffffff"
],
"kpi_adhoc_searchesRowBackgroundColorsEditorConfig": [
{
"value": "#F7BC38",
"to": 1
},
{
"value": "#65A637",
"from": 1,
"to": 750
},
{
"value": "#F7BC38",
"from": 750
}
],
"entity_countRowColorsEditorConfig": [
"#ffffff"
],
"entity_countRowBackgroundColorsEditorConfig": [
{
"value": "#F7BC38",
"to": 1
},
{
"value": "#65A637",
"from": 1,
"to": 40000
},
{
"value": "#F7BC38",
"from": 40000,
"to": 75000
},
{
"value": "#D93F3C",
"from": 75000
}
],
"kvstore_statusRowColorsEditorConfig": [
"#ffffff"
],
"kvstore_statusRowBackgroundColorsEditorConfig": [
{
"match": "ready",
"value": "#65A637"
}
]
}
},
"viz_kpi_base_search_usage_summary": {
"type": "splunk.table",
"dataSources": {
"primary": "ds_iH15XtmZ"
},
"eventHandlers": [
{
"type": "drilldown.customUrl",
"options": {
"url": "/app/itsi/base_search_configuration?savedBaseSearchId=$row.kpi_base_search_id.value$",
"newTab": true
}
}
],
"title": "KPI Base Search Usage Summary",
"options": {
"showInternalFields": false,
"columnFormat": {
"count": {
"data": "> table | seriesByName(\"count\") | formatByType(countColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"count\") | pick(countRowColorsEditorConfig)",
"rowBackgroundColors": "> table | seriesByName(\"count\") | rangeValue(countRowBackgroundColorsEditorConfig)"
}
}
},
"context": {
"countColumnFormatEditorConfig": {
"number": {
"thousandSeparated": false,
"unitPosition": "after"
}
},
"countRowColorsEditorConfig": [
"#ffffff"
],
"countRowBackgroundColorsEditorConfig": [
{
"value": "#f7bc38",
"to": 1
},
{
"value": "#65a637",
"from": 1,
"to": 600
},
{
"value": "#f7bc38",
"from": 600
}
]
}
},
"viz_kvstore_collections": {
"type": "splunk.table",
"dataSources": {
"primary": "ds_0eshUJji"
},
"title": "KV Store Collections",
"description": "",
"options": {
"columnFormat": {
"Number of Objects": {
"data": "> table | seriesByName(\"Number of Objects\") | formatByType(Number_of_ObjectsColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"Number of Objects\") | pick(Number_of_ObjectsRowColorsEditorConfig)",
"rowBackgroundColors": "> table | seriesByName(\"Number of Objects\") | rangeValue(Number_of_ObjectsRowBackgroundColorsEditorConfig)"
},
"Collection Size (MB)": {
"data": "> table | seriesByName(\"Collection Size (MB)\") | formatByType(Collection_Size__MB_ColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"Collection Size (MB)\") | pick(Collection_Size__MB_RowColorsEditorConfig)",
"rowBackgroundColors": "> table | seriesByName(\"Collection Size (MB)\") | rangeValue(Collection_Size__MB_RowBackgroundColorsEditorConfig)"
}
},
"showInternalFields": false
},
"context": {
"Number_of_ObjectsColumnFormatEditorConfig": {
"number": {
"thousandSeparated": false,
"unitPosition": "after"
}
},
"Number_of_ObjectsRowColorsEditorConfig": [
"#ffffff"
],
"Number_of_ObjectsRowBackgroundColorsEditorConfig": [
{
"value": "#65A637",
"to": 430000
},
{
"value": "#cba700",
"from": 430000,
"to": 500000
},
{
"value": "#D93F3C",
"from": 500000
}
],
"Collection_Size__MB_ColumnFormatEditorConfig": {
"number": {
"thousandSeparated": false,
"unitPosition": "after"
}
},
"Collection_Size__MB_RowColorsEditorConfig": [
"#ffffff"
],
"Collection_Size__MB_RowBackgroundColorsEditorConfig": [
{
"value": "#65a637",
"to": 1024
},
{
"value": "#f7bc38",
"from": 1024
}
]
}
},
"viz_maximum_collection_objects_message": {
"type": "splunk.markdown",
"options": {
"markdown": "## The maximum number of objects in each collection is 500,000. You may notice performance degradation as a collection approaches its limit."
}
},
"viz_concurrent_searches": {
"type": "splunk.table",
"dataSources": {
"primary": "ds_WGEOEG7e"
},
"title": "Concurrent Searches",
"options": {
"columnFormat": {
"max_skipped": {
"data": "> table | seriesByName(\"max_skipped\") | formatByType(max_skippedColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"max_skipped\") | pick(max_skippedRowColorsEditorConfig)",
"rowBackgroundColors": "> table | seriesByName(\"max_skipped\") | rangeValue(max_skippedRowBackgroundColorsEditorConfig)"
}
},
"showInternalFields": false
},
"context": {
"max_skippedColumnFormatEditorConfig": {
"number": {
"thousandSeparated": false,
"unitPosition": "after"
}
},
"max_skippedRowColorsEditorConfig": [
"#ffffff"
],
"max_skippedRowBackgroundColorsEditorConfig": [
{
"value": "#65A637",
"to": 1
},
{
"value": "#F7BC38",
"from": 1
}
]
}
},
"viz_interesting_searches": {
"type": "splunk.table",
"dataSources": {
"primary": "ds_vPrg8GSd"
},
"title": "Interesting Searches (If the real-time searches are not running, this could indicate a Java problem)",
"options": {
"tableFormat": {
"rowBackgroundColors": "> table | seriesByIndex(0) | pick(tableAltRowBackgroundColorsByTheme)"
},
"columnFormat": {
"isFailed": {
"data": "> table | seriesByName(\"isFailed\") | formatByType(isFailedColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"isFailed\") | pick(isFailedRowColorsEditorConfig)",
"rowBackgroundColors": "> table | seriesByName(\"isFailed\") | rangeValue(isFailedRowBackgroundColorsEditorConfig)"
}
},
"showInternalFields": false,
"count": 6
},
"context": {
"isFailedColumnFormatEditorConfig": {
"number": {
"thousandSeparated": false,
"unitPosition": "after"
}
},
"isFailedRowColorsEditorConfig": [
"#ffffff"
],
"isFailedRowBackgroundColorsEditorConfig": [
{
"value": "#65A637",
"to": 1
},
{
"value": "#F7BC38",
"from": 1
}
]
},
"eventHandlers": [
{
"type": "drilldown.customUrl",
"options": {
"url": "/manager/itsi/saved/searches?owner=nobody&search=$row.search_name.value$",
"newTab": true
}
}
]
},
"viz_kpi_performance": {
"type": "splunk.table",
"dataSources": {
"primary": "ds_uTDF0c9D"
},
"title": "",
"description": "",
"options": {
"columnFormat": {
"failed_count": {
"data": "> table | seriesByName(\"failed_count\") | formatByType(failed_countColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"failed_count\") | pick(failed_countRowColorsEditorConfig)",
"rowBackgroundColors": "> table | seriesByName(\"failed_count\") | rangeValue(failed_countRowBackgroundColorsEditorConfig)"
},
"suppressed_count": {
"data": "> table | seriesByName(\"suppressed_count\") | formatByType(suppressed_countColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"suppressed_count\") | pick(suppressed_countRowColorsEditorConfig)",
"rowBackgroundColors": "> table | seriesByName(\"suppressed_count\") | rangeValue(suppressed_countRowBackgroundColorsEditorConfig)"
},
"runtime_headroom_pct": {
"data": "> table | seriesByName(\"runtime_headroom_pct\") | formatByType(runtime_headroom_pctColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"runtime_headroom_pct\") | pick(runtime_headroom_pctRowColorsEditorConfig)",
"rowBackgroundColors": "> table | seriesByName(\"runtime_headroom_pct\") | rangeValue(runtime_headroom_pctRowBackgroundColorsEditorConfig)"
},
"max_result_count": {
"data": "> table | seriesByName(\"max_result_count\") | formatByType(max_result_countColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"max_result_count\") | pick(max_result_countRowColorsEditorConfig)",
"rowBackgroundColors": "> table | seriesByName(\"max_result_count\") | rangeValue(max_result_countRowBackgroundColorsEditorConfig)"
},
"avg_result_count": {
"data": "> table | seriesByName(\"avg_result_count\") | formatByType(avg_result_countColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"avg_result_count\") | pick(avg_result_countRowColorsEditorConfig)",
"rowBackgroundColors": "> table | seriesByName(\"avg_result_count\") | rangeValue(avg_result_countRowBackgroundColorsEditorConfig)"
}
},
"showInternalFields": false
},
"context": {
"failed_countColumnFormatEditorConfig": {
"number": {
"thousandSeparated": false,
"unitPosition": "after"
}
},
"suppressed_countColumnFormatEditorConfig": {
"number": {
"thousandSeparated": false,
"unitPosition": "after"
}
},
"runtime_headroom_pctColumnFormatEditorConfig": {
"number": {
"thousandSeparated": false,
"unitPosition": "after"
}
},
"max_result_countColumnFormatEditorConfig": {
"number": {
"thousandSeparated": false,
"unitPosition": "after"
}
},
"avg_result_countColumnFormatEditorConfig": {
"number": {
"thousandSeparated": false,
"unitPosition": "after"
}
},
"failed_countRowColorsEditorConfig": [
"#ffffff"
],
"failed_countRowBackgroundColorsEditorConfig": [
{
"value": "#65A637",
"to": 1
},
{
"value": "#D93F3C",
"from": 1
}
],
"suppressed_countRowColorsEditorConfig": [
"#ffffff"
],
"suppressed_countRowBackgroundColorsEditorConfig": [
{
"value": "#65A637",
"to": 1
},
{
"value": "#F7BC38",
"from": 1
}
],
"runtime_headroom_pctRowColorsEditorConfig": [
"#ffffff"
],
"runtime_headroom_pctRowBackgroundColorsEditorConfig": [
{
"value": "#D93F3C",
"to": 25
},
{
"value": "#F7BC38",
"from": 25,
"to": 50
},
{
"value": "#65A637",
"from": 50
}
],
"avg_result_countRowColorsEditorConfig": [
"#ffffff"
],
"avg_result_countRowBackgroundColorsEditorConfig": [
{
"value": "#F7BC38",
"to": 1
},
{
"value": "#65A637",
"from": 1,
"to": 35000
},
{
"value": "#F7BC38",
"from": 35000,
"to": 50000
},
{
"value": "#D93F3C",
"from": 50000
}
],
"max_result_countRowColorsEditorConfig": [
"#ffffff"
],
"max_result_countRowBackgroundColorsEditorConfig": [
{
"value": "#F7BC38",
"to": 1
},
{
"value": "#65A637",
"from": 1,
"to": 35000
},
{
"value": "#F7BC38",
"from": 35000,
"to": 50000
},
{
"value": "#D93F3C",
"from": 50000
}
]
},
"eventHandlers": [
{
"type": "drilldown.customUrl",
"options": {
"url": "/manager/itsi/saved/searches?owner=nobody&search=$row.savedsearch_name.value$",
"newTab": true
}
}
]
},
"viz_kpi_performance_message": {
"type": "splunk.markdown",
"options": {
"markdown": "### KPI Performance\n\n*(\"runtime_headroom\" is (100 - runtime / scheduled interval). For a search scheduled to run every 60sec, with a runtime of 45sec, runtime_headroom_pct = 25. 100 is good, 0 is bad). Your avg_result_count or max_result_count should not exceed the max_action_results for scheduler in limits.conf (default: 50k)*\n\n**limit = (number of KPIs * number of entities associated with KPIs) + (number of services * 2). Exceeding the limit may lead to inconsistent results for KPI aggregation. Increasing the limit can impact system performance because more memory must be allocated to support increased search results.**"
}
},
"viz_saved_search_error_message": {
"type": "splunk.table",
"dataSources": {
"primary": "ds_3XNunSOu"
},
"title": "Savedsearch Error Messages",
"options": {
"showInternalFields": false
}
},
"viz_not_executed_searches": {
"type": "splunk.table",
"dataSources": {
"primary": "ds_IczyACq6"
},
"title": "Not Executed Searches (In last 1 hour)",
"options": {
"showInternalFields": false
}
},
"viz_refresh_queue_stats_message": {
"type": "splunk.markdown",
"options": {
"markdown": "### Refresh Queue Statistics\n\n\n#### The refresh queue ensures data integrity and eventual consistency of your ITSI configuration. It runs as a single instance."
}
},
"viz_refresh_queue_runtimes": {
"type": "splunk.table",
"dataSources": {
"primary": "ds_vjT4ohMh"
},
"title": "Refresh Queue Runtimes",
"options": {
"columnFormat": {
"Average Job Time": {
"data": "> table | seriesByName(\"Average Job Time\") | formatByType(Average_Job_TimeColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"Average Job Time\") | pick(Average_Job_TimeRowColorsEditorConfig)",
"rowBackgroundColors": "> table | seriesByName(\"Average Job Time\") | rangeValue(Average_Job_TimeRowBackgroundColorsEditorConfig)"
},
"Average Queue Time": {
"data": "> table | seriesByName(\"Average Queue Time\") | formatByType(Average_Queue_TimeColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"Average Queue Time\") | pick(Average_Queue_TimeRowColorsEditorConfig)",
"rowBackgroundColors": "> table | seriesByName(\"Average Queue Time\") | rangeValue(Average_Queue_TimeRowBackgroundColorsEditorConfig)"
}
}
},
"context": {
"Average_Job_TimeColumnFormatEditorConfig": {
"number": {
"thousandSeparated": false,
"unitPosition": "after"
}
},
"Average_Job_TimeRowColorsEditorConfig": [
"#ffffff"
],
"Average_Job_TimeRowBackgroundColorsEditorConfig": [
{
"value": "#65a637",
"to": 180
},
{
"value": "#f7bc38",
"from": 180,
"to": 360
},
{
"value": "#d93f3c",
"from": 360
}
],
"Average_Queue_TimeColumnFormatEditorConfig": {
"number": {
"thousandSeparated": false,
"unitPosition": "after"
}
},
"Average_Queue_TimeRowColorsEditorConfig": [
"#ffffff"
],
"Average_Queue_TimeRowBackgroundColorsEditorConfig": [
{
"value": "#65a637",
"to": 60
},
{
"value": "#f7bc38",
"from": 60,
"to": 120
},
{
"value": "#d93f3c",
"from": 120
}
]
}
},
"viz_refresh_queue_jobs_stats": {
"type": "splunk.table",
"dataSources": {
"primary": "ds_ZNKAgTXv"
},
"title": "Refresh Queue Jobs Stats",
"options": {
"columnFormat": {
"Failed Jobs": {
"data": "> table | seriesByName(\"Failed Jobs\") | formatByType(Failed_JobsColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"Failed Jobs\") | pick(Failed_JobsRowColorsEditorConfig)",
"rowBackgroundColors": "> table | seriesByName(\"Failed Jobs\") | rangeValue(Failed_JobsRowBackgroundColorsEditorConfig)"
}
}
},
"context": {
"Failed_JobsColumnFormatEditorConfig": {
"number": {
"thousandSeparated": false,
"unitPosition": "after"
}
},
"Failed_JobsRowColorsEditorConfig": [
"#ffffff"
],
"Failed_JobsRowBackgroundColorsEditorConfig": [
{
"value": "#65A637",
"to": 1
},
{
"value": "#D93F3C",
"from": 1
}
]
}
},
"viz_itsi_logs_messages_panel": {
"type": "splunk.table",
"dataSources": {
"primary": "ds_xLmhUySn"
},
"title": "ITSI Log Messages (deduplicated)",
"description": "Use the \"ITSI Log Levels\" dropdown to filter results",
"options": {
"showInternalFields": false,
"tableFormat": {
"rowBackgroundColors": "> table | seriesByIndex(0) | pick(tableAltRowBackgroundColorsByTheme)"
},
"count": 8
}
},
"viz_refresh_queue_recent_jobs": {
"type": "splunk.table",
"dataSources": {
"primary": "ds_PaEXs4Gt"
},
"title": "Recent Refresh Queue Jobs",
"options": {
"showInternalFields": false,
"columnFormat": {
"Total Time": {
"data": "> table | seriesByName(\"Total Time\") | formatByType(Total_TimeColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"Total Time\") | pick(Total_TimeRowColorsEditorConfig)",
"rowBackgroundColors": "> table | seriesByName(\"Total Time\") | rangeValue(Total_TimeRowBackgroundColorsEditorConfig)"
},
"Queue Time": {
"data": "> table | seriesByName(\"Queue Time\") | formatByType(Queue_TimeColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"Queue Time\") | pick(Queue_TimeRowColorsEditorConfig)",
"rowBackgroundColors": "> table | seriesByName(\"Queue Time\") | rangeValue(Queue_TimeRowBackgroundColorsEditorConfig)"
}
}
},
"context": {
"Total_TimeColumnFormatEditorConfig": {
"number": {
"thousandSeparated": false,
"unitPosition": "after"
}
},
"Total_TimeRowColorsEditorConfig": [
"#ffffff"
],
"Total_TimeRowBackgroundColorsEditorConfig": [
{
"value": "#65a637",
"to": 180
},
{
"value": "#f7bc38",
"from": 180,
"to": 360
},
{
"value": "#d93f3c",
"from": 360
}
],
"Queue_TimeColumnFormatEditorConfig": {
"number": {
"thousandSeparated": false,
"unitPosition": "after"
}
},
"Queue_TimeRowColorsEditorConfig": [
"#ffffff"
],
"Queue_TimeRowBackgroundColorsEditorConfig": [
{
"value": "#65a637",
"to": 60
},
{
"value": "#f7bc38",
"from": 60,
"to": 120
},
{
"value": "#d93f3c",
"from": 120
}
]
}
},
"viz_cpu_utilization": {
"type": "splunk.line",
"dataSources": {
"primary": "ds_mH7FmIE9"
},
"showProgressBar": false,
"showLastUpdated": false,
"title": "Average CPU Utilization per Host"
},
"viz_entitites_by_shared_base_search": {
"type": "splunk.table",
"options": {
"columnFormat": {
"entries": {
"data": "> table | seriesByName(\"entries\") | formatByType(entriesColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"entries\") | pick(entriesRowColorsEditorConfig)",
"rowBackgroundColors": "> table | seriesByName(\"entries\") | rangeValue(entriesRowBackgroundColorsEditorConfig)"
},
"Associated Entities": {
"data": "> table | seriesByName(\"Associated Entities\") | formatByType(Associated_EntitiesColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"Associated Entities\") | pick(Associated_EntitiesRowColorsEditorConfig)",
"rowBackgroundColors": "> table | seriesByName(\"Associated Entities\") | rangeValue(Associated_EntitiesRowBackgroundColorsEditorConfig)"
}
},
"count": 8,
"showInternalFields": false
},
"dataSources": {
"primary": "ds_1v0Ozclm_ds_A96VXarg"
},
"context": {
"entriesColumnFormatEditorConfig": {
"number": {
"thousandSeparated": false,
"unitPosition": "after"
}
},
"entriesRowColorsEditorConfig": [
"#ffffff"
],
"entriesRowBackgroundColorsEditorConfig": [
{
"value": "#F7BC38",
"to": 1
},
{
"value": "#65A637",
"from": 1
}
],
"Associated_EntitiesColumnFormatEditorConfig": {
"number": {
"thousandSeparated": false,
"unitPosition": "after"
}
},
"Associated_EntitiesRowColorsEditorConfig": [
"#ffffff"
],
"Associated_EntitiesRowBackgroundColorsEditorConfig": [
{
"value": "#f7bc38",
"to": 1
},
{
"value": "#65a637",
"from": 1,
"to": 15000
},
{
"value": "#f7bc38",
"from": 15000,
"to": 30000
},
{
"value": "#d93f3c",
"from": 30000
}
]
},
"title": "Entity Count by Shared Base Search",
"description": "KPIs created from Shared Base Searches",
"eventHandlers": [
{
"type": "drilldown.customUrl",
"options": {
"url": "/app/itsi/base_search_configuration?savedBaseSearchId=$row.kpi_base_search_id.value$",
"newTab": true
}
}
]
},
"viz_JYhq0M7j": {
"type": "splunk.table",
"options": {
"columnFormat": {
"entries": {
"data": "> table | seriesByName(\"entries\") | formatByType(entriesColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"entries\") | pick(entriesRowColorsEditorConfig)",
"rowBackgroundColors": "> table | seriesByName(\"entries\") | rangeValue(entriesRowBackgroundColorsEditorConfig)"
},
"Associated Entities": {
"data": "> table | seriesByName(\"Associated Entities\") | formatByType(Associated_EntitiesColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"Associated Entities\") | pick(Associated_EntitiesRowColorsEditorConfig)",
"rowBackgroundColors": "> table | seriesByName(\"Associated Entities\") | rangeValue(Associated_EntitiesRowBackgroundColorsEditorConfig)"
}
},
"count": 8,
"showInternalFields": false
},
"dataSources": {
"primary": "ds_ebkdTNtz"
},
"context": {
"entriesColumnFormatEditorConfig": {
"number": {
"thousandSeparated": false,
"unitPosition": "after"
}
},
"entriesRowColorsEditorConfig": [
"#ffffff"
],
"entriesRowBackgroundColorsEditorConfig": [
{
"value": "#F7BC38",
"to": 1
},
{
"value": "#65A637",
"from": 1
}
],
"Associated_EntitiesColumnFormatEditorConfig": {
"number": {
"thousandSeparated": false,
"unitPosition": "after"
}
},
"Associated_EntitiesRowColorsEditorConfig": [
"#ffffff"
],
"Associated_EntitiesRowBackgroundColorsEditorConfig": [
{
"value": "#f7bc38",
"to": 1
},
{
"value": "#65a637",
"from": 1,
"to": 15000
},
{
"value": "#f7bc38",
"from": 15000,
"to": 30000
},
{
"value": "#d93f3c",
"from": 30000
}
]
},
"title": "Entity Count by KPI",
"description": "Adhoc KPIs, Metrics KPIs, or Datamodels",
"eventHandlers": [
{
"type": "drilldown.customUrl",
"options": {
"url": "/app/itsi/service_definition?serviceId=$row.service_key.value$&tab=kpi&kpiId=$row.kpi_id.value$",
"newTab": true
}
}
]
},
"viz_kpi_entity_count_message": {
"type": "splunk.markdown",
"options": {
"markdown": "**KPIs that filter to entities will be associated with entities matching the entity rules defined in the services.\nWhen using shared base searches, all of the associated entities come from each of the linked KPI's entity rules**\n\nToo many KPIs linked to a single shared base search (that filters entities) may lead to a single KPI shared base search filtering on too many entities. This may lead to long refresh queue job updates or even potentially slower search performance."
}
},
"viz_avg_cpu_util": {
"type": "splunk.column",
"dataSources": {
"primary": "ds_0t1NTiBc"
},
"showProgressBar": false,
"showLastUpdated": false,
"title": "Average CPU Utilization by Process (%)",
"options": {
"stackMode": "stacked"
}
},
"viz_avg_memory_util": {
"type": "splunk.column",
"dataSources": {
"primary": "ds_5e9eV1ng"
},
"title": "Average Memory Utilization by Process (MB)",
"options": {
"stackMode": "stacked"
},
"showProgressBar": false,
"showLastUpdated": false
},
"viz_roles_changed": {
"type": "splunk.table",
"options": {
"showInternalFields": false
},
"dataSources": {
"primary": "ds_N6k4trnp"
},
"title": "ITSI role/capability modifications"
},
"viz_deprecation_notice": {
"type": "splunk.markdown",
"options": {
"markdown": "**NOTE:** You're now using the latest ITSI Health Check Dashboard experience. The old version is deprecated and no longer receiving updates."
}
},
"viz_duplicate_entities": {
"type": "splunk.table",
"dataSources": {
"primary": "ds_8BGAeDhq"
},
"title": "",
"description": "",
"eventHandlers": [
{
"type": "drilldown.customUrl",
"options": {
"url": "/app/itsi/duplicate_entities_management?duplicate_type=$row._page_param.value$",
"newTab": true
}
}
],
"options": {
"showInternalFields": false,
"columnFormat": {
"Entity_configuration_issues": {
"data": "> table | seriesByName(\"Entity_configuration_issues\") | formatByType(Entity_configuration_issuesColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"Entity_configuration_issues\") | matchValue(Entity_configuration_issuesRowColorsEditorConfig)"
},
"total_affected": {
"data": "> table | seriesByName(\"total_affected\") | formatByType(total_affectedColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"total_affected\") | rangeValue(total_affectedRowColorsEditorConfig)"
},
"percent_affected": {
"data": "> table | seriesByName(\"percent_affected\") | formatByType(percent_affectedColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"percent_affected\") | matchValue(percent_affectedRowColorsEditorConfig)"
},
"Total_affected": {
"data": "> table | seriesByName(\"Total_affected\") | formatByType(Total_affectedColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"Total_affected\") | matchValue(Total_affectedRowColorsEditorConfig)"
},
"Percent_affected": {
"data": "> table | seriesByName(\"Percent_affected\") | formatByType(Percent_affectedColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"Percent_affected\") | matchValue(Percent_affectedRowColorsEditorConfig)"
},
"Entity configuration issues": {
"data": "> table | seriesByName(\"Entity configuration issues\") | formatByType(Entity_configuration_issuesColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"Entity configuration issues\") | matchValue(Entity_configuration_issuesRowColorsEditorConfig)"
},
"Total affected": {
"data": "> table | seriesByName(\"Total affected\") | formatByType(Total_affectedColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"Total affected\") | matchValue(Total_affectedRowColorsEditorConfig)"
},
"Percent affected": {
"data": "> table | seriesByName(\"Percent affected\") | formatByType(Percent_affectedColumnFormatEditorConfig)",
"rowColors": "> table | seriesByName(\"Percent affected\") | matchValue(Percent_affectedRowColorsEditorConfig)"
}
}
},
"context": {
"Entity_configuration_issuesColumnFormatEditorConfig": {
"string": {
"unitPosition": "after"
}
},
"Entity_configuration_issuesRowColorsEditorConfig": [
{
"match": -1,
"value": "#ffffff"
}
],
"total_affectedColumnFormatEditorConfig": {
"number": {
"thousandSeparated": false,
"unitPosition": "after"
}
},
"total_affectedRowColorsEditorConfig": [
{
"value": "#000000",
"to": 20
},
{
"value": "#000000",
"from": 20
}
],
"percent_affectedColumnFormatEditorConfig": {
"string": {
"unitPosition": "after"
}
},
"percent_affectedRowColorsEditorConfig": [
{
"match": "",
"value": "#000000"
}
],
"Total_affectedColumnFormatEditorConfig": {
"number": {
"thousandSeparated": false,
"unitPosition": "after"
}
},
"Total_affectedRowColorsEditorConfig": [
{
"match": -1,
"value": "#ffffff"
}
],
"Percent_affectedColumnFormatEditorConfig": {
"string": {
"unitPosition": "after"
}
},
"Percent_affectedRowColorsEditorConfig": [
{
"match": -1,
"value": "#000000"
}
]
}
},
"viz_duplicate_entities_job": {
"type": "splunk.markdown",
"options": {
"markdown": "### Check for Duplicate Entities \n\nReload panel with latest duplicate entities data: [Execute command ](/app/itsi/search?q=%7C%20enqueueduplicateentitiesgeneratejob)"
}
},
"viz_unstable_entities": {
"type": "splunk.table",
"dataSources": {
"primary": "ds_9ydvJT9I"
},
"title": "Unstable entities count"
}
},
"dataSources": {
"ds_cqNQBdKm": {
"type": "ds.search",
"options": {
"query": "| rest splunk_server=local /services/server/info\n | stats values(version) as splunk_version, values(server_roles) as server_roles, values(os_name) as os, values(numberOfCores) as cpu_cores, values(numberOfVirtualCores) as virtual_cpu_cores, values(physicalMemoryMB) as physical_mem_MB by splunk_server\n | rename splunk_server as host\n | join type=left host [| rest splunk_server=* /services/server/status/resource-usage/hostwide | eval \"%_memory_used\"=round(mem_used/mem,4)*100, host=splunk_server | table host %_memory_used]\n | join type=left host [| rest splunk_server=local /services/apps/local/itsi\n | stats values(version) as itsi_version by splunk_server\n | rename splunk_server as host]\n | join type=left host [ search index=_introspection sourcetype=splunk_disk_objects component=Indexes data.name=\"*itsi*\"\n | stats dc(data.name) as index_count, values(data.name) as indexes by host]\n | join type=left host [ search index=_internal splunk_server=local sourcetype=splunkd \"Linux transparent hugepage support\" latest=now() | head 1 | rex field=event_message \"enabled= (?\\S+)\" | eval THP_kernel_settings=if(enabled=\"always\", \"not ok\", \"ok\")]\n | table host splunk_version itsi_version os cpu_cores virtual_cpu_cores physical_mem_MB %_memory_used THP_kernel_settings server_roles index_count indexes",
"queryParameters": {
"earliest": "$global_time.earliest$",
"latest": "$global_time.latest$"
}
},
"name": "Splunk Server Information"
},
"ds_NTcJUPg0": {
"type": "ds.search",
"options": {
"query": "| rest splunk_server=local /services/apps/local/itsi\n | stats values(version) as \"Current ITSI version\" | join\n [ | rest splunk_server=local /services/apps/local/SA-ITOA | stats values(version) as \"Current SA-ITOA version\" | join\n [|inputlookup itsi_migration_check | eval \"Current KV Store version\"=itsi_latest_version | fields - itsi_old_version, itsi_latest_version, is_migration_done]]"
},
"name": "ITSI Migration Status"
},
"ds_DreUmgu8": {
"type": "ds.search",
"options": {
"query": "index=_internal source=*itsi_upgrade_readiness.log* OR source=*itsi_migration_utility.log OR source=*itsi_appserver*.log* sourcetype=itsi_internal_log severity=*\n| stats count(severity) as failures\n| eval upgrade_readiness_conditions=if(failures > 0, \"Precheck failures detected (see more details)\", \"No precheck failures\")\n| fields upgrade_readiness_conditions\n| append [| inputlookup itsi_service_template_sync_status_lookup \n| stats count(eval(sync_status==\"syncing\" OR (sync_status==\"sync scheduled\" AND isnull(scheduled_time)))) as my_count \n| eval upgrade_readiness_conditions=if(my_count > 0, \"Service template syncs scheduled or in progress\", \"No service template syncs scheduled or in progress\") \n| fields - my_count]",
"queryParameters": {
"earliest": "-48h@h",
"latest": "now"
}
},
"name": "ITSI Upgrade Readiness"
},
"ds_FRTTZ2ac": {
"type": "ds.search",
"options": {
"query": "| rest splunk_server=local /services/server/info\n | stats values(kvStoreStatus) as kvstore_status by splunk_server\n | rename splunk_server as host\n | join type=left host [ search index=_introspection sourcetype=kvstore component=KVStoreCollectionStats data.ns=\"*itsi*\"\n | stats dc(data.ns) as kvstore_collections, count(eval(data.ok=\"0\")) as kvstore_data_not_ok by host]\n | join type=left host [ search index=_introspection sourcetype=http_event_collector_metrics data.token_name=\"Auto Generated ITSI Event Management Token\"\n | stats sum(data.num_of_errors) as HEC_errors, sum(data.num_of_parser_errors) as HEC_parser_errors, sum(data.total_bytes_indexed) as HEC_bytes_indexed by host]\n | join type=left host [ | rest splunk_server=local /servicesNS/nobody/SA-ITOA/itoa_interface/vLatest/service/count report_as=text\n | spath input=value\n | rename splunk_server as host, count as service_count\n | table host service_count]\n | join type=left host [ | rest splunk_server=local /servicesNS/nobody/SA-ITOA/itoa_interface/vLatest/entity/count report_as=text\n | spath input=value\n | rename splunk_server as host, count as entity_count\n | table host entity_count]\n | join type=left host [ search index=_internal sourcetype=scheduler savedsearch_name=\"Indicator*\"\n | stats count as run_count, count(eval(status=\"delegated_remote_error\" OR status=\"skipped\")) as failed_count, count(eval(suppressed!=\"0\")) as suppressed_count,\n avg(run_time) as avg_runtime, max(run_time) as max_runtime, earliest(_time) as first, latest(_time) as last\n by host, savedsearch_name\n | eval KPI_search_type=if(savedsearch_name like \"%Shared%\", \"base\", \"ad hoc\")\n | stats count(eval(KPI_search_type=\"base\")) as kpi_base_searches, count(eval(KPI_search_type=\"ad hoc\")) as kpi_adhoc_searches by host]\n | table host service_count kpi_base_searches kpi_adhoc_searches entity_count kvstore_status kvstore_collections kvstore_data_not_ok HEC_bytes_indexed HEC_errors HEC_parser_errors"
},
"name": "Basic ITSI Information"
},
"ds_iH15XtmZ": {
"type": "ds.search",
"options": {
"query": "| inputlookup service_kpi_sbs_lookup\n| eval zipped = mvzip(mvzip('kpis.base_search', 'kpis.search_type', \"==@@==\"), 'kpis.title', \"==@@==\")\n| fields - kpis._key, kpis.base_search, kpis.search_type, kpis.title, sec_grp, title, kpis.base_search\n| eval sharedBaseZipped=mvfilter(match(zipped, \"shared_base\"))\n| rename kpis.base_search_id as base_search_id | fields - zipped\n| eval t=mvzip(base_search_id, sharedBaseZipped, \"==@@==\") | fields - sharedBaseZipped, base_search_id\n| mvexpand t | eval x=split(t, \"==@@==\") | eval search_id = mvindex(x, 0) | eval search_str = mvindex(x, -3)\n| eval search_type = mvindex(x, -2) | eval kpi_title = mvindex(x, -1) | search search_type = shared_base\n| table search_str, search_id | stats count by search_id, search_str | rename search_id as key\n| join [| inputlookup kpi_base_search_title_lookup | eval key=_key]\n| rename title as kpi_base_search_title, key as kpi_base_search_id | table kpi_base_search_title, search_str, count, kpi_base_search_id\n| sort -count",
"enableSmartSources": true
},
"name": "KPI Base Search Usage Summary"
},
"ds_0eshUJji": {
"type": "ds.search",
"options": {
"query": "| rest splunk_server=local /services/server/introspection/kvstore/collectionstats\n | mvexpand data\n | spath input=data\n | search ns IN (itsi, *itsi*, SA-ITOA, SA-*)\n | rex field=ns \"(?.*)\\.(?.*)\"\n | eval dbsize=size/1024/1024\n | eval indexsize=totalIndexSize/1024/1024\n | stats first(count) AS \"Number of Objects\" first(nindexes) AS Accelerations first(indexsize) AS \"Acceleration Size (MB)\" first(dbsize) AS \"Collection Size (MB)\" by App,Collection\n | sort - \"Number of Objects\""
},
"name": "KV Store Collections"
},
"ds_WGEOEG7e": {
"type": "ds.search",
"options": {
"query": "source=\"*/metrics.log\" sourcetype=splunkd index=_internal active_hist_searches group=search_concurrency \"system total\"\n | stats max(active_hist_searches) as max_historical_searches, avg(active_hist_searches) as avg_historical_searches, max(active_realtime_searches) as max_realtime_searches, avg(active_realtime_searches) as avg_realtime_searches by splunk_server\n | rename splunk_server as host\n | eval avg_historical_searches=round(avg_historical_searches,0)\n | eval avg_realtime_searches=round(avg_realtime_searches,0)\n | join type=left host [ search source=\"*/metrics.log\" sourcetype=splunkd index=_internal group=searchscheduler\n | stats max(skipped) as max_skipped, max(max_running) as max_running, max(total_runtime) as max_total_runtime, avg(total_runtime) as avg_total_runtime by splunk_server\n | rename splunk_server as host\n | eval max_total_runtime=round(max_total_runtime,0)\n | eval avg_total_runtime=round(avg_total_runtime,0)]"
},
"name": "Concurrent Searches"
},
"ds_A96VXarg": {
"type": "ds.search",
"options": {
"query": "| tstats count as entries latest(_time) as most_recent where index=itsi* OR index=_internal by index, splunk_server\n | stats sum(entries) as entries, max(most_recent) as most_recent, values(splunk_server) as indexers by index\n | eval most_recent=strftime(most_recent,\"%F %T\")"
},
"name": "Interesting Indexes"
},
"ds_vPrg8GSd": {
"type": "ds.search",
"options": {
"query": "| rest splunk_server=local /services/search/jobs/\n | search label=itsi*\n | fields label dispatchState isFailed isRealTimeSearch runDuration\n | rename label as search_name\n | sort isFailed",
"enableSmartSources": true
},
"name": "Interesting Searches"
},
"ds_uTDF0c9D": {
"type": "ds.search",
"options": {
"query": "index=_internal sourcetype=scheduler savedsearch_name=\"Indicator*\"\n | stats dc(sid) as run_count, count(eval(status=\"delegated_remote_error\" OR status=\"skipped\")) as failed_count, count(eval(suppressed!=\"0\")) as suppressed_count,\n avg(run_time) as avg_runtime, max(run_time) as max_runtime, earliest(_time) as first, latest(_time) as last,\n max(result_count) as max_result_count, avg(result_count) as avg_result_count\n by savedsearch_name\n | eval KPI_search_type=if(savedsearch_name like \"%Shared%\", \"base\", \"ad hoc\")\n | eval runtime_headroom_pct=round((100-(max_runtime/((last-first)/(run_count-1))*100)),1)\n | eval avg_runtime=round(avg_runtime, 2)\n | eval max_runtime=round(max_runtime, 2)\n | eval avg_result_count=round(avg_result_count, 2)\n | eval max_result_count=round(max_result_count, 2)\n | table savedsearch_name KPI_search_type failed_count avg_result_count max_result_count suppressed_count runtime_headroom_pct avg_runtime\n max_runtime run_count\n | sort +runtime_headroom_pct, -max_result_count",
"enableSmartSources": true
},
"name": "KPI Performance"
},
"ds_3XNunSOu": {
"type": "ds.search",
"options": {
"query": "index=_internal sourcetype=scheduler savedsearch_name=\"Indicator*\"\n | join sid\n [ search index=_internal sourcetype=splunk_search_messages app=\"itsi\" log_level=ERROR]\n | stats count(savedsearch_name) as \"count\" avg(run_time) as \"Avg Runtime(sec)\" values(message_key) as \"Message Key\" values(message) as \"Error Message\" by savedsearch_name\n | eval Avg Runtime(sec)=round('Avg Runtime(sec)', 3)\n | rename savedsearch_name AS \"Savedsearch Name\""
},
"name": "Savedsearch Error Messages"
},
"ds_IczyACq6": {
"type": "ds.search",
"options": {
"query": "index=_internal source=*splunkd.log \"search not executed\" user=\"splunk-system-user\" | timechart count span=1h"
},
"name": "Not Executed Searches"
},
"ds_vjT4ohMh": {
"type": "ds.search",
"options": {
"query": "index=_internal sourcetype=itsi_internal_log source=*itsi_consumer* \"Job Successful\" |stats avg(transaction_time) as \"Average Job Time\", avg(queue_time) as \"Average Queue Time\", max(transaction_time) as \"Maximum Job Time\", max(queue_time) as \"Maximum Queue Time\"",
"queryParameters": {
"earliest": "$rq_time.earliest$",
"latest": "$rq_time.latest$"
}
},
"name": "Refresh Queue Runtimes"
},
"ds_o3zoA1iF": {
"type": "ds.search",
"options": {
"query": "index=_internal sourcetype=itsi_internal_log source=*itsi_consumer* \"Job Failed\" |stats count as \"Failed Jobs\"",
"queryParameters": {
"earliest": "$rq_time.earliest$",
"latest": "$rq_time.latest$"
}
},
"name": "Refresh Queue Failed Jobs"
},
"ds_xLmhUySn": {
"type": "ds.search",
"options": {
"query": "index=_internal sourcetype=itsi_internal_log log_level=$LoggingLevel$\n | rex max_match=3 \"\\[(?[^\\]]+)\"\n | eval comp1=mvindex(itsi_components,0), comp2=mvindex(itsi_components,1), comp3=mvindex(itsi_components,2)\n | fillnull value=\"none\" comp3\n | dedup comp1 comp2 comp3",
"queryParameters": {}
},
"name": "ITSI Log Messages"
},
"ds_HbT2Pv5Z": {
"type": "ds.search",
"options": {
"query": "| inputlookup itsi_entities\n | eval identical_alias = _itsi_identifier_lookups\n | mvexpand \"identical_alias\"\n | eval entity_key=_key\n | stats count AS duplicate_occurrences values(title) AS entity_name values(services._key) AS service_keys values(entity_key) AS entity_keys by identical_alias | where duplicate_occurrences>1"
},
"name": "Duplicate Entities"
},
"ds_PaEXs4Gt": {
"type": "ds.search",
"options": {
"query": "index=_internal source=*itsi* *itsi_consumer* *tid* transaction_time=* job_change_type!=\"\" \n| table _time, tid, queue_time, job_time, transaction_time, job_change_type \n| rename tid AS \"Transaction ID\", queue_time AS \"Queue Time\", job_time AS \"Job Time\", transaction_time AS \"Total Time\", job_change_type AS \"Job Name\" \n| table _time, \"Job Name\", \"Total Time\", \"Job Time\", \"Queue Time\", \"Transaction ID\"\n| sort -_time",
"queryParameters": {
"earliest": "$rq_time.earliest$",
"latest": "$rq_time.latest$"
}
},
"name": "Recent Refresh Queue Jobs"
},
"ds_mH7FmIE9": {
"type": "ds.search",
"options": {
"query": "index=_introspection sourcetype=splunk_resource_usage component=Hostwide | rename data.* as * | eval pct_cpu=100-cpu_idle_pct | timechart span=5m avg(pct_cpu) as avg_pct_cpu by host",
"queryParameters": {
"earliest": "$rq_time.earliest$",
"latest": "$rq_time.latest$"
}
},
"name": "CPU Utilization"
},
"ds_1v0Ozclm_ds_A96VXarg": {
"type": "ds.search",
"options": {
"query": "| inputlookup itsi_entity_filter_rules_lookup \n| rename entity_info.entity_key AS entity_key, base_search_id AS kpi_base_search_id\n| stats count(entity_key) AS \"Associated Entities\" by kpi_base_search_id\n| sort -\"Associated Entities\"",
"enableSmartSources": true
},
"name": "Entity Filter Shared Base Search"
},
"ds_ebkdTNtz": {
"type": "ds.search",
"options": {
"query": "| inputlookup itsi_entity_filter_rules_lookup where kpi_id=* \n| rename entity_info.entity_key AS entity_key \n| stats count(entity_key) AS \"Associated Entities\" by kpi_id \n| join kpi_id \n [| inputlookup service_kpi_lookup WHERE \n [| inputlookup itsi_entity_filter_rules_lookup where kpi_id=* \n | fields kpi_id \n | rename kpi_id AS kpis._key] \n | rename _* as hidden_*, hidden_key AS service_key, kpis._key AS kpi_id \n | mvexpand kpi_id \n | fields kpi_id, service_key]",
"enableSmartSources": true
},
"name": "Entity Count Single Search"
},
"ds_b7LxeCU0": {
"type": "ds.search",
"options": {
"query": "| inputlookup itsi_entities \n| lookup itsi_entity_types _key AS entity_type_ids OUTPUT title AS entity_type_title \n| fields - _status.breakdown* \n| rename _* AS hidden_* \n| eval entity_status_report=mvappend(entity_status_report,if('hidden_status.combined'=\"N/A\",\"potentially_manually_imported_entity\",null())) \n| eval entity_status_report=mvappend(entity_status_report,if(mvcount(hidden_itsi_entity_status_lookups)=2 AND 'hidden_status.combined'=\"active\" AND match(hidden_itsi_entity_status_lookups,\"itsi import objects - itsi_entity_name_normalizer\"),\"only_active_because_of_normalizer_search\",null())) \n| eval entity_status_report=mvappend(entity_status_report,if('hidden_status.combined'=\"unstable\",\"unstable_entity\",null())) \n| eval entity_status_report=mvappend(entity_status_report,if('hidden_status.combined'=\"active\",\"active_entity\",null())) \n| eventstats count AS total_entity_count \n| stats count, max(total_entity_count) AS total_entity_count by entity_status_report \n| eval percentage=round(count/total_entity_count*100,2)\n| sort - percentage"
},
"name": "Entity Status"
},
"ds_0t1NTiBc": {
"type": "ds.search",
"options": {
"query": "index=_introspection host=* source=*/resource_usage.log* component=PerProcess data.process=python* \n| rename data.* as * \n| rex field=args \"(?i).*/(?[^/]+)$\"| timechart span=5m avg(normalized_pct_cpu) as cpu_utilization by filename",
"queryParameters": {
"earliest": "$rq_time.earliest$",
"latest": "$rq_time.latest$"
}
},
"name": "Avg Cpu Utilization by Process"
},
"ds_5e9eV1ng": {
"type": "ds.search",
"options": {
"query": "index=_introspection host=* source=*/resource_usage.log* component=PerProcess data.process=python* \n| rename data.* as * \n| eval mem_used=mem_used / 1024\n| rex field=args \"(?i).*/(?[^/]+)$\"\n| timechart span=5m sum(mem_used) as memory_utilization by filename",
"queryParameters": {
"earliest": "$rq_time.earliest$",
"latest": "$rq_time.latest$"
}
},
"name": "Search_1"
},
"ds_N6k4trnp": {
"type": "ds.search",
"options": {
"query": "index=_audit action=edit* role IN (itoa_admin, itoa_team_admin, itoa_analyst, itoa_user) \n| table _time, user, role, old_capabilities, new_capabilities\n| eval old_capabilities=split(old_capabilities,\",\"), new_capabilities=split(new_capabilities,\",\")",
"queryParameters": {
"earliest": "$global_time.earliest$",
"latest": "$global_time.latest$"
}
},
"name": "ITSI role edits"
},
"ds_8BGAeDhq": {
"type": "ds.search",
"options": {
"query": "| inputlookup itsi_duplicate_entities_job_queue\n| search job_type=\"GENERATE\" AND (status=\"SUCCESS\" OR status=\"ERROR\")\n| sort -create_time\n| head 1\n|where status=\"SUCCESS\"\n| eval \"Entity configuration issues\" = \"Duplicate Entity Alias\",\ntotal_entities='job_result.total_entities',\nTotal_affected='job_result.duplicate_entity_count.duplicate_alias',\n \"Total affected\" = 'job_result.duplicate_entity_count.duplicate_alias',\n \"Percent affected\" = if(total_entities==0,\"0%\",round((Total_affected * 100) / 'job_result.total_entities',2). \"%\"),\n Actions=\"Inspect configuration\",\n _page_param=\"duplicate_aliases\"\n| table \"Entity configuration issues\", \"Total affected\", \"Percent affected\" ,Actions, _page_param\n| append[\n| inputlookup itsi_duplicate_entities_job_queue\n| search job_type=\"GENERATE\" AND (status=\"SUCCESS\" OR status=\"ERROR\")\n| sort -create_time\n| head 1\n| where status=\"SUCCESS\"\n| eval \"Entity configuration issues\" = \"Multiple Duplicate Entities Due to Case Sensitivity and FQDN\",\ntotal_entities='job_result.total_entities', \n\"Total affected\" = 'job_result.duplicate_entity_count.merge_key',\nTotal_affected='job_result.duplicate_entity_count.merge_key',\n \"Percent affected\" = if(total_entities==0,\"0%\",round((Total_affected * 100) / 'job_result.total_entities',2). \"%\"),\n Actions=\"Inspect configuration\",\n _page_param=\"merge_field\"\n | table \"Entity configuration issues\", \"Total affected\", \"Percent affected\" ,Actions, _page_param\n]| append[\n| inputlookup itsi_duplicate_entities_job_queue\n| search job_type=\"GENERATE\" AND (status = \"SUCCESS\" OR status=\"ERROR\")\n| sort -create_time\n| head 1\n| where status=\"ERROR\"\n| eval \"Entity configuration issues\" = \"Job failed\",\ntotal_entities=\"N/A\", \n\"Total affected\" = \"N/A\",\nTotal_affected='N/A',\n \"Percent affected\" = \"N/A\",\n Actions=\"N/A\"| table \"Entity configuration issues\", \"Total affected\", \"Percent affected\" ,Actions, _page_param\n]"
},
"name": "Entity Normalization"
},
"ds_9ydvJT9I": {
"type": "ds.search",
"options": {
"query": "| inputlookup itsi_entities \n| rename _* as hidden.* \n| search hidden.status.combined=\"unstable\" | stats count AS Total_affected\n| join [ | inputlookup itsi_entities | stats count as total_entities]\n| eval \"Entity status\"=\"Unstable\",\n\"Total affected\"=Total_affected,\n\"Percent affected\"=if(total_entities==0,\"0%\",round((Total_affected*100)/total_entities,2).\"%\")\n| table \"Entity status\",\"Total affected\",\"Percent affected\""
},
"name": "Count of Unstable Entities"
},
"ds_ZNKAgTXv": {
"type": "ds.search",
"options": {
"query": "index=_internal sourcetype=itsi_internal_log source=*itsi_consumer* (\"Job Failed\" OR \"Job Successful\") \n| stats count(eval(match(_raw, \"Job Failed\"))) as \"Failed Jobs\", count(eval(number_of_failures>0 AND match(_raw, \"Job Successful\"))) as \"Successful Jobs on Retry\", count(eval(number_of_failures=0 AND match(_raw, \"Job Successful\"))) as \"Successful Jobs\"",
"queryParameters": {
"earliest": "$rq_time.earliest$",
"latest": "$rq_time.latest$"
}
},
"name": "Refresh Queue Job Stats"
}
},
"defaults": {
"dataSources": {
"ds.search": {
"options": {
"queryParameters": {
"latest": "$global_time.latest$",
"earliest": "$global_time.earliest$"
}
}
}
}
},
"inputs": {
"input_global_trp": {
"type": "input.timerange",
"options": {
"token": "global_time",
"defaultValue": "-24h@h,now"
},
"title": "Global Time Range"
},
"input_refresh_queue_lookback": {
"options": {
"defaultValue": "-4h@m,now",
"token": "rq_time"
},
"title": "Refresh Queue Lookback",
"type": "input.timerange"
},
"input_itsi_log_levels": {
"options": {
"items": [
{
"label": "All",
"value": "*"
},
{
"label": "Info",
"value": "INFO"
},
{
"label": "Error",
"value": "ERROR"
},
{
"label": "Debug",
"value": "DEBUG"
},
{
"label": "Warning",
"value": "WAR*"
}
],
"defaultValue": "*",
"token": "LoggingLevel"
},
"title": "ITSI Log Levels",
"type": "input.dropdown"
}
},
"layout": {
"type": "grid",
"options": {
"gutterSize": 11
},
"structure": [
{
"item": "viz_deprecation_notice",
"type": "block",
"position": {
"x": 0,
"y": 0,
"w": 1198,
"h": 64
}
},
{
"item": "viz_splunk_server_information",
"type": "block",
"position": {
"x": 0,
"y": 64,
"w": 1198,
"h": 223
}
},
{
"item": "viz_itsi_migration_status",
"type": "block",
"position": {
"x": 0,
"y": 287,
"w": 752,
"h": 186
}
},
{
"item": "viz_basic_information",
"type": "block",
"position": {
"x": 0,
"y": 473,
"w": 1198,
"h": 126
}
},
{
"item": "viz_maximum_collection_objects_message",
"type": "block",
"position": {
"x": 0,
"y": 599,
"w": 1198,
"h": 64
}
},
{
"item": "viz_kvstore_collections",
"type": "block",
"position": {
"x": 0,
"y": 663,
"w": 1198,
"h": 481
}
},
{
"item": "viz_kpi_performance_message",
"type": "block",
"position": {
"x": 0,
"y": 1144,
"w": 1198,
"h": 98
}
},
{
"item": "viz_kpi_performance",
"type": "block",
"position": {
"x": 0,
"y": 1242,
"w": 1198,
"h": 432
}
},
{
"item": "viz_kpi_entity_count_message",
"type": "block",
"position": {
"x": 0,
"y": 1674,
"w": 1198,
"h": 76
}
},
{
"item": "viz_entitites_by_shared_base_search",
"type": "block",
"position": {
"x": 0,
"y": 1750,
"w": 595,
"h": 409
}
},
{
"item": "viz_kpi_base_search_usage_summary",
"type": "block",
"position": {
"x": 0,
"y": 2159,
"w": 1198,
"h": 428
}
},
{
"item": "viz_interesting_searches",
"type": "block",
"position": {
"x": 0,
"y": 2587,
"w": 1198,
"h": 211
}
},
{
"item": "viz_refresh_queue_stats_message",
"type": "block",
"position": {
"x": 0,
"y": 2798,
"w": 1200,
"h": 91
}
},
{
"item": "viz_refresh_queue_jobs_stats",
"type": "block",
"position": {
"x": 0,
"y": 2889,
"w": 600,
"h": 156
}
},
{
"item": "viz_refresh_queue_recent_jobs",
"type": "block",
"position": {
"x": 0,
"y": 3045,
"w": 600,
"h": 369
}
},
{
"item": "viz_avg_cpu_util",
"type": "block",
"position": {
"x": 0,
"y": 3414,
"w": 600,
"h": 378
}
},
{
"item": "viz_concurrent_searches",
"type": "block",
"position": {
"x": 0,
"y": 3792,
"w": 1200,
"h": 141
}
},
{
"item": "viz_saved_search_error_message",
"type": "block",
"position": {
"x": 0,
"y": 3933,
"w": 600,
"h": 258
}
},
{
"item": "viz_itsi_logs_messages_panel",
"type": "block",
"position": {
"x": 0,
"y": 4191,
"w": 1200,
"h": 546
}
},
{
"item": "viz_roles_changed",
"type": "block",
"position": {
"x": 0,
"y": 4737,
"w": 1200,
"h": 400
}
},
{
"item": "viz_duplicate_entities_job",
"type": "block",
"position": {
"x": 0,
"y": 5137,
"w": 1200,
"h": 64
}
},
{
"item": "viz_duplicate_entities",
"type": "block",
"position": {
"x": 0,
"y": 5201,
"w": 1200,
"h": 169
}
},
{
"item": "viz_unstable_entities",
"type": "block",
"position": {
"x": 0,
"y": 5370,
"w": 1200,
"h": 137
}
},
{
"item": "viz_refresh_queue_runtimes",
"type": "block",
"position": {
"x": 600,
"y": 2889,
"w": 600,
"h": 156
}
},
{
"item": "viz_JYhq0M7j",
"type": "block",
"position": {
"x": 595,
"y": 1750,
"w": 603,
"h": 409
}
},
{
"item": "viz_cpu_utilization",
"type": "block",
"position": {
"x": 600,
"y": 3045,
"w": 600,
"h": 369
}
},
{
"item": "viz_avg_memory_util",
"type": "block",
"position": {
"x": 600,
"y": 3414,
"w": 600,
"h": 378
}
},
{
"item": "viz_not_executed_searches",
"type": "block",
"position": {
"x": 600,
"y": 3933,
"w": 600,
"h": 258
}
},
{
"item": "viz_itsi_upgrade_readiness",
"type": "block",
"position": {
"x": 752,
"y": 287,
"w": 446,
"h": 186
}
}
],
"globalInputs": [
"input_global_trp",
"input_itsi_log_levels",
"input_refresh_queue_lookback"
]
},
"description": "This dashboard describes the operational status and configuration of an ITSI instance. Because searches on this page access sensitive indexes (e.g. _internal) and REST endpoints, reports will be incomplete if not run by an Admin user.",
"title": "ITSI Health Check"
}
]]>