<form version="1.1">
  <label>ITSI Health Check (Deprecated)</label>
  <description>NOTE: This version of the ITSI health check dashboard has been deprecated. This dashboard describes the operational status and configuration of an ITSI instance.  Because it accesses sensitive indexes (_internal) and REST endpoints, reports will be incomplete if not run by an Admin user.</description>
  <search id="base_errors">
    <query>
      <![CDATA[
           index=_internal source=*itsi*.log*
           | rex "\s(?<log_level>(INFO|ER\w+|WAR\w+|FAT\w+|DEBUG|CRI\w+))\s+\["
           | fillnull log_level value="UNKNOWN"
           | eval log_level=case(log_level="WAR","WARNING",1=1,log_level)
          | rex mode=sed field=component "s/(.*)-\d+/\1/"
          | bucket _time span=5m
          | stats count by _time log_level component host
    ]]>
    </query>
    <earliest>$field1.earliest$</earliest>
    <latest>$field1.latest$</latest>
  </search>
  <fieldset submitButton="false" autoRun="true">
    <input type="time" token="field1">
      <label></label>
      <default>
        <earliest>-24h@h</earliest>
        <latest>now</latest>
      </default>
    </input>
    <html>
      <style>
         div[data-test="real-time-column"] {
             display: none;
         }
         div[data-test-panel-id="realTime"] {
             display: none;
         }
      </style>
    </html>
  </fieldset>
  <row>
    <panel>
      <table>
        <title>Splunk Server Information</title>
        <search>
          <query>| rest splunk_server=local /services/server/info
            | stats values(version) as splunk_version, values(server_roles) as server_roles, values(os_name) as os, values(numberOfCores) as cpu_cores, values(numberOfVirtualCores) as virtual_cpu_cores, values(physicalMemoryMB) as physical_mem_MB by splunk_server
            | rename splunk_server as host
            | join type=left host [| rest splunk_server=local /services/apps/local/itsi
            | stats values(version) as itsi_version by splunk_server
            | rename splunk_server as host]
            | join type=left host [ search index=_introspection sourcetype=splunk_disk_objects component=Indexes data.name="*itsi*"
            | stats dc(data.name) as index_count, values(data.name) as indexes by host]
            | join type=left host [ search index=_internal splunk_server=local sourcetype=splunkd "Linux transparent hugepage support" latest=now()
            | head 1 | rex field=event_message "enabled= (?&lt;enabled&gt;\S+)"
            | eval THP_kernel_settings=if(enabled="always", "not ok", "ok") ]
            | table host splunk_version itsi_version os cpu_cores virtual_cpu_cores physical_mem_MB THP_kernel_settings server_roles index_count indexes</query>
          <earliest>$field1.earliest$</earliest>
          <latest>$field1.latest$</latest>
        </search>
        <option name="count">10</option>
        <option name="drilldown">none</option>
        <option name="refresh.display">progressbar</option>
        <format type="color" field="THP_kernel_settings">
          <colorPalette type="map">{"bad":#F7BC38,"ok":#65A637}</colorPalette>
        </format>
        <format type="color" field="cpu_cores">
          <colorPalette type="list">[#D93F3C,#F7BC38,#65A637]</colorPalette>
          <scale type="threshold">12,16</scale>
        </format>
        <format type="color" field="virtual_cpu_cores">
          <colorPalette type="list">[#D93F3C,#F7BC38,#65A637]</colorPalette>
          <scale type="threshold">24,32</scale>
        </format>
        <format type="color" field="physical_mem_MB">
          <colorPalette type="list">[#D93F3C,#F7BC38,#65A637]</colorPalette>
          <scale type="threshold">12288,16384</scale>
        </format>
      </table>
    </panel>
  </row>
  <row>
    <panel>
      <table>
        <title>ITSI Migration Status</title>
        <search>
          <query>| rest splunk_server=local /services/apps/local/itsi
            | stats values(version) as "Current ITSI version" | join
            [ | rest splunk_server=local /services/apps/local/SA-ITOA | stats values(version) as "Current SA-ITOA version"  | join
            [|inputlookup itsi_migration_check | eval "Current KV Store version"=itsi_latest_version | fields - itsi_old_version, itsi_latest_version, is_migration_done]]</query>
          <earliest>-24h@h</earliest>
          <latest>now</latest>
        </search>
        <option name="drilldown">none</option>
      </table>
    </panel>
  </row>
  <row>
    <panel>
      <table>
        <title>ITSI Upgrade Readiness</title>
        <search>
          <query>|inputlookup itsi_service_template_sync_status_lookup | stats count(eval(sync_status=="syncing" OR (sync_status=="sync scheduled" AND isnull(scheduled_time)))) as my_count | eval upgrade_ready=if(my_count &gt; 0, "False", "True") | fields - my_count</query>
          <earliest>-24h@h</earliest>
          <latest>now</latest>
        </search>
        <option name="drilldown">none</option>
      </table>
    </panel>
  </row>
  <row>
    <panel>
      <table>
        <title>Basic ITSI Information</title>
        <search>
          <query>| rest splunk_server=local /services/server/info
            | stats values(kvStoreStatus) as kvstore_status by splunk_server
            | rename splunk_server as host
            | join type=left host [ search index=_introspection sourcetype=kvstore component=KVStoreCollectionStats data.ns="*itsi*"
            | stats dc(data.ns) as kvstore_collections, count(eval(data.ok="0")) as kvstore_data_not_ok by host]
            | join type=left host [ search index=_introspection sourcetype=http_event_collector_metrics data.token_name="Auto Generated ITSI Event Management Token"
            | stats sum(data.num_of_errors) as HEC_errors, sum(data.num_of_parser_errors) as HEC_parser_errors, sum(data.total_bytes_indexed) as HEC_bytes_indexed by host]
            | join type=left host [ | rest splunk_server=local /servicesNS/nobody/SA-ITOA/itoa_interface/vLatest/service/count report_as=text
            | spath input=value
            | rename splunk_server as host, count as service_count
            | table host service_count]
            | join type=left host [ | rest splunk_server=local /servicesNS/nobody/SA-ITOA/itoa_interface/vLatest/entity/count report_as=text
            | spath input=value
            | rename splunk_server as host, count as entity_count
            | table host entity_count]
            | join type=left host [ search index=_internal sourcetype=scheduler savedsearch_name="Indicator*"
            | stats count as run_count, count(eval(status="delegated_remote_error" OR status="skipped")) as failed_count, count(eval(suppressed!="0")) as suppressed_count,
            avg(run_time) as avg_runtime, max(run_time) as max_runtime, earliest(_time) as first, latest(_time) as last
            by host, savedsearch_name
            | eval KPI_search_type=if(savedsearch_name like "%Shared%", "base", "ad hoc")
            | stats count(eval(KPI_search_type="base")) as kpi_base_searches, count(eval(KPI_search_type="ad hoc")) as kpi_adhoc_searches by host]
            | table host service_count kpi_base_searches kpi_adhoc_searches entity_count kvstore_status kvstore_collections kvstore_data_not_ok HEC_bytes_indexed HEC_errors HEC_parser_errors</query>
          <earliest>$field1.earliest$</earliest>
          <latest>$field1.latest$</latest>
        </search>
        <option name="count">10</option>
        <option name="drilldown">none</option>
        <option name="refresh.display">progressbar</option>
        <format type="color" field="kvstore_data_not_ok">
          <colorPalette type="list">[#65A637,#F7BC38]</colorPalette>
          <scale type="threshold">1</scale>
        </format>
        <format type="color" field="HEC_bytes_indexed">
          <colorPalette type="list">[#D93F3C,#65A637]</colorPalette>
          <scale type="threshold">1</scale>
        </format>
        <format type="color" field="HEC_errors">
          <colorPalette type="list">[#65A637,#F7BC38]</colorPalette>
          <scale type="threshold">1</scale>
        </format>
        <format type="color" field="HEC_parser_errors">
          <colorPalette type="list">[#65A637,#F7BC38]</colorPalette>
          <scale type="threshold">1</scale>
        </format>
        <format type="color" field="kvstore_collections">
          <colorPalette type="list">[#F7BC38,#65A637]</colorPalette>
          <scale type="threshold">29</scale>
        </format>
        <format type="color" field="service_count">
          <colorPalette type="list">[#F7BC38,#65A637,#F7BC38]</colorPalette>
          <scale type="threshold">1,1000</scale>
        </format>
        <format type="color" field="kpi_base_searches">
          <colorPalette type="list">[#F7BC38,#65A637,#F7BC38]</colorPalette>
          <scale type="threshold">1,100</scale>
        </format>
        <format type="color" field="kpi_adhoc_searches">
          <colorPalette type="list">[#F7BC38,#65A637,#F7BC38]</colorPalette>
          <scale type="threshold">1,100</scale>
        </format>
        <format type="color" field="entity_count">
          <colorPalette type="list">[#F7BC38,#65A637,#F7BC38,#D93F3C]</colorPalette>
          <scale type="threshold">1,1000,30000</scale>
        </format>
        <format type="color" field="kvstore_status">
          <colorPalette type="map">{"ready":#65A637}</colorPalette>
        </format>
      </table>
    </panel>
  </row>
  <row>
    <panel>
      <table>
        <title>KPI Base Search Usage Summary</title>
        <search>
          <query>| inputlookup service_kpi_sbs_lookup
            | eval zipped = mvzip(mvzip('kpis.base_search', 'kpis.search_type', "==@@=="), 'kpis.title', "==@@==")
            | fields - kpis._key, kpis.base_search, kpis.search_type, kpis.title, sec_grp, title, kpis.base_search
            | eval sharedBaseZipped=mvfilter(match(zipped, "shared_base"))
            | rename kpis.base_search_id as base_search_id | fields - zipped
            | eval t=mvzip(base_search_id, sharedBaseZipped, "==@@==") | fields - sharedBaseZipped, base_search_id
            | mvexpand t | eval x=split(t, "==@@==") | eval search_id = mvindex(x, 0) | eval search_str = mvindex(x, -3)
            | eval search_type = mvindex(x, -2) | eval kpi_title = mvindex(x, -1) | search search_type = shared_base
            | table search_str, search_id | stats count by search_id, search_str | rename search_id as key
            | join [| inputlookup kpi_base_search_title_lookup | eval key=_key]
            | rename title as kpi_base_search_title | table kpi_base_search_title, search_str, count</query>
          <earliest>-24h@h</earliest>
          <latest>now</latest>
        </search>
        <option name="drilldown">none</option>
      </table>
    </panel>
  </row>
  <row>
    <panel>
      <html>
        <h2>The maximum number of objects in each collection is 500,000. You may notice performance degradation as a collection approaches its limit.
        </h2>
      </html>
      <table>
        <title>KV Store Collections</title>
        <search>
          <query>| rest splunk_server=local /services/server/introspection/kvstore/collectionstats
            | mvexpand data
            | spath input=data
            | rex field=ns "(?&lt;App&gt;.*)\.(?&lt;Collection&gt;.*)"
            | eval dbsize=size/1024/1024
            | eval indexsize=totalIndexSize/1024/1024
            | stats first(count) AS "Number of Objects" first(nindexes) AS Accelerations first(indexsize) AS "Acceleration Size (MB)" first(dbsize) AS "Collection Size (MB)" by App,Collection
            | sort - "Number of Objects"</query>
          <earliest>$field1.earliest$</earliest>
          <latest>$field1.latest$</latest>
        </search>
        <option name="count">10</option>
        <option name="drilldown">none</option>
        <option name="refresh.display">progressbar</option>
        <format type="color" field="Number of Objects">
          <colorPalette type="list">[#65A637,#F7BC38,#D93F3C]</colorPalette>
          <scale type="threshold">430000,500000</scale>
        </format>
      </table>
    </panel>
  </row>
  <row>
    <panel>
      <table>
        <title>Concurrent Searches</title>
        <search>
          <query>source="*/metrics.log" sourcetype=splunkd index=_internal active_hist_searches  group=search_concurrency "system total"
            | stats max(active_hist_searches) as max_historical_searches, avg(active_hist_searches) as avg_historical_searches, max(active_realtime_searches) as max_realtime_searches, avg(active_realtime_searches) as avg_realtime_searches by splunk_server
            | rename splunk_server as host
            | eval avg_historical_searches=round(avg_historical_searches,0)
            | eval avg_realtime_searches=round(avg_realtime_searches,0)
            | join type=left host [ search source="*/metrics.log" sourcetype=splunkd index=_internal group=searchscheduler
            | stats max(skipped) as max_skipped, max(max_running) as max_running, max(total_runtime) as max_total_runtime, avg(total_runtime) as avg_total_runtime by splunk_server
            | rename splunk_server as host
            | eval max_total_runtime=round(max_total_runtime,0)
            | eval avg_total_runtime=round(avg_total_runtime,0)]</query>
          <earliest>$field1.earliest$</earliest>
          <latest>$field1.latest$</latest>
        </search>
        <option name="count">10</option>
        <option name="drilldown">none</option>
        <option name="refresh.display">progressbar</option>
        <format type="color" field="max_skipped">
          <colorPalette type="list">[#65A637,#F7BC38]</colorPalette>
          <scale type="threshold">1</scale>
        </format>
      </table>
    </panel>
  </row>
  <row>
    <panel>
      <table>
        <title>Interesting Indexes</title>
        <search>
          <query>| tstats count as entries latest(_time) as most_recent where index=itsi* OR index=_internal by index, splunk_server
              | stats sum(entries) as entries, max(most_recent) as most_recent, values(splunk_server) as indexers by index
              | eval most_recent=strftime(most_recent,"%F %T")</query>
          <earliest>$field1.earliest$</earliest>
          <latest>$field1.latest$</latest>
        </search>
        <option name="count">10</option>
        <option name="drilldown">none</option>
        <option name="refresh.display">progressbar</option>
        <format type="color" field="entries">
          <colorPalette type="list">[#F7BC38,#65A637]</colorPalette>
          <scale type="threshold">1</scale>
        </format>
      </table>
    </panel>
    <panel>
      <table>
        <title>Interesting Searches (If the real-time searches are not running, this could indicate a Java problem)</title>
        <search>
          <query>| rest splunk_server=local /services/search/jobs/
            | search label=itsi*
            | fields label dispatchState isFailed isRealTimeSearch runDuration
            | rename label as search_name</query>
          <earliest>$field1.earliest$</earliest>
          <latest>$field1.latest$</latest>
        </search>
        <option name="count">10</option>
        <option name="drilldown">none</option>
        <format type="color" field="isFailed">
          <colorPalette type="list">[#65A637,#F7BC38]</colorPalette>
          <scale type="threshold">1</scale>
        </format>
      </table>
    </panel>
  </row>
  <row>
    <panel>
      <html>
        <h3>KPI Performance ("runtime_headroom" is (100 - runtime / scheduled interval). For a search scheduled to run every 60sec, with a runtime of 45sec, runtime_headroom_pct = 25.  100 is good, 0 is bad). Your avg_result_count or max_result_count should not exceed the max_action_results for scheduler in limits.conf (default: 50k)</h3>
      </html>
      <html>
        <h2>limit = (number of KPIs * number of entities associated with KPIs) + (number of services * 2). Exceeding the limit may lead to inconsistent results for KPI aggregation. Increasing the limit can impact system performance because more memory must be allocated to support increased search results.</h2>
      </html>
      <table>
        <search>
          <query>index=_internal sourcetype=scheduler savedsearch_name="Indicator*"
            | stats dc(sid) as run_count, count(eval(status="delegated_remote_error" OR status="skipped")) as failed_count, count(eval(suppressed!="0")) as suppressed_count,
            avg(run_time) as avg_runtime, max(run_time) as max_runtime, earliest(_time) as first, latest(_time) as last,
            max(result_count) as max_result_count, avg(result_count) as avg_result_count
            by savedsearch_name
            | eval KPI_search_type=if(savedsearch_name like "%Shared%", "base", "ad hoc")
            | eval runtime_headroom_pct=round((100-(max_runtime/((last-first)/(run_count-1))*100)),1)
            | eval avg_runtime=round(avg_runtime, 2)
            | eval max_runtime=round(max_runtime, 2)
            | eval avg_result_count=round(avg_result_count, 2)
            | eval max_result_count=round(max_result_count, 2)
            | table savedsearch_name KPI_search_type failed_count suppressed_count runtime_headroom_pct avg_runtime
            max_runtime avg_result_count max_result_count run_count
            | sort +runtime_headroom_pct</query>
          <earliest>$field1.earliest$</earliest>
          <latest>$field1.latest$</latest>
        </search>
        <option name="count">10</option>
        <option name="drilldown">none</option>
        <format type="color" field="runtime_headroom_pct">
          <colorPalette type="list">[#D93F3C,#F7BC38,#65A637]</colorPalette>
          <scale type="threshold">25,50</scale>
        </format>
        <format type="color" field="failed_count">
          <colorPalette type="list">[#65A637,#D93F3C]</colorPalette>
          <scale type="threshold">1</scale>
        </format>
        <format type="color" field="suppressed_count">
          <colorPalette type="list">[#65A637,#F7BC38]</colorPalette>
          <scale type="threshold">1</scale>
        </format>
      </table>
    </panel>
  </row>
  <row>
    <panel>
      <table>
        <title>Savedsearch Error Messages</title>
        <search>
          <query>index=_internal sourcetype=scheduler savedsearch_name="Indicator*"
            | join sid
            [ search index=_internal sourcetype=splunk_search_messages app="itsi" log_level=ERROR]
            | stats count(savedsearch_name) as "count" avg(run_time) as "Avg Runtime(sec)" values(message_key) as "Message Key" values(message) as "Error Message" by savedsearch_name
            | eval Avg Runtime(sec)=round('Avg Runtime(sec)', 3)
            | rename savedsearch_name AS "Savedsearch Name"</query>
          <earliest>$field1.earliest$</earliest>
          <latest>$field1.latest$</latest>
        </search>
        <option name="count">10</option>
        <option name="drilldown">none</option>
        <option name="refresh.display">progressbar</option>
        <format type="color" field="message">
          <colorPalette type="minMidMax" maxColor="#53A051" minColor="#FFFFFF"></colorPalette>
          <scale type="minMidMax"></scale>
        </format>
        <format type="color" field="savedsearch_name">
          <colorPalette type="minMidMax" maxColor="#53A051" minColor="#FFFFFF"></colorPalette>
          <scale type="minMidMax"></scale>
        </format>
      </table>
    </panel>
  </row>
  <row>
    <panel>
      <single>
        <title>Not Executed Searches (In last 1 hour)</title>
        <search>
          <query>index=_internal source=*splunkd.log "search not executed" user="splunk-system-user" | timechart count span=1h</query>
          <earliest>$field1.earliest$</earliest>
          <latest>$field1.latest$</latest>
        </search>
        <option name="colorBy">value</option>
        <option name="colorMode">none</option>
        <option name="drilldown">all</option>
        <option name="rangeColors">["0x53a051","0x53a051","0xf8be34","0xf1813f","0xdc4e41"]</option>
        <option name="refresh.display">progressbar</option>
        <option name="trendColorInterpretation">inverse</option>
        <option name="useColors">1</option>
      </single>
    </panel>
  </row>
  <row>
    <panel>
      <title>Refresh Queue Statistics</title>
      <html>
        <h2>The refresh queue ensures data integrity and eventual consistency of your ITSI configuration. It runs as a single instance.
        </h2>
      </html>
      <table>
        <title>Refresh Queue Runtimes</title>
        <search>
          <query>index=_internal sourcetype=itsi_internal_log source=*itsi_consumer* "Job Successful" |stats avg(transaction_time) as "Average Job Time", avg(queue_time) as "Average Queue Time", max(transaction_time) as "Maximum Job Time", max(queue_time) as "Maximum Queue Time"</query>
          <earliest>$field1.earliest$</earliest>
          <latest>$field1.latest$</latest>
        </search>
        <option name="count">10</option>
        <option name="drilldown">none</option>
      </table>
      <table>
        <title>Refresh Queue Failed Jobs</title>
        <search>
          <query>index=_internal sourcetype=itsi_internal_log source=*itsi_consumer* "Job Failed" |stats count as "Failed Jobs"</query>
          <earliest>$field1.earliest$</earliest>
          <latest>$field1.latest$</latest>
        </search>
        <option name="count">10</option>
        <option name="drilldown">cell</option>
        <format type="color" field="Failed Jobs">
          <colorPalette type="list">[#65A637,#D93F3C]</colorPalette>
          <scale type="threshold">1</scale>
        </format>
        <drilldown>
          <link target="_blank">search?q=index=_internal sourcetype=itsi_internal_log source=*itsi_consumer* "Job Failed"&amp;earliest=$field1.earliest$&amp;latest=$field1.latest$</link>
        </drilldown>
      </table>
    </panel>
  </row>
  <row>
    <panel>
      <title>ITSI Log Messages (deduplicated)</title>
      <input type="time" token="field2">
        <label>ITSI Logs</label>
        <default>
          <earliest>-60m@m</earliest>
          <latest>now</latest>
        </default>
      </input>
      <input type="multiselect" token="LogLevel">
        <label>Log Level</label>
        <choice value="DEBUG">Debug</choice>
        <choice value="INFO">Info</choice>
        <choice value="WAR*">Warning</choice>
        <choice value="ERROR">Error</choice>
        <choice value="*">All</choice>
        <initialValue>WAR*,ERROR</initialValue>
        <valuePrefix>log_level="</valuePrefix>
        <valueSuffix>"</valueSuffix>
        <delimiter> OR </delimiter>
      </input>
      <event>
        <search>
          <query>index=_internal sourcetype=itsi_internal_log $LogLevel$
            | rex max_match=3 "\[(?&lt;itsi_components&gt;[^\]]+)"
            | eval comp1=mvindex(itsi_components,0), comp2=mvindex(itsi_components,1), comp3=mvindex(itsi_components,2)
            | fillnull value="none" comp3
            | dedup comp1 comp2 comp3</query>
          <earliest>$field2.earliest$</earliest>
          <latest>$field2.latest$</latest>
        </search>
        <option name="list.drilldown">none</option>
        <option name="refresh.display">progressbar</option>
      </event>
    </panel>
  </row>
  <row>
    <panel>
      <html>
        <h2>If more than one entity is using the same alias field value, KPI base searches might have incorrect statistical aggregation results. To remedy duplicate entity alias values, click <b>Configure</b> &gt; <b>Entities</b> and edit the entity definitions for the entities with duplicate aliases. Keep the alias value for one of the entities and edit the others to remove the duplicate alias value. <a href=" http://docs.splunk.com/Documentation/ITSI/latest/Configure/Installationandconfigurationconsiderationsandissues#Duplicate_entity_aliases" target="_blank">Learn More </a>
        </h2>
      </html>
      <table>
        <title>Check for Duplicate Entity Aliases</title>
        <search>
          <query>
            <![CDATA[
                | inputlookup itsi_entities
                | eval identical_alias = _itsi_identifier_lookups
                | mvexpand "identical_alias"
                | eval entity_key=_key
                | stats count AS duplicate_occurrences  values(title) AS entity_name values(services._key) AS service_keys  values(entity_key) AS entity_keys  by identical_alias | where duplicate_occurrences>1
            ]]>
          </query>
          <earliest>$field1.earliest$</earliest>
          <latest>$field1.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="count">20</option>
        <option name="dataOverlayMode">none</option>
        <option name="drilldown">none</option>
        <option name="percentagesRow">false</option>
        <option name="rowNumbers">false</option>
        <option name="totalsRow">false</option>
        <option name="wrap">true</option>
      </table>
    </panel>
  </row>
</form>