<form script="dashboard/EaFeatureFlagRedirect.js" stylesheet="css/dashboard/event_analytics_monitoring.css" version="1.1">
  <label>Event Analytics Monitoring</label>
  <description>This dashboard provides statistics and troubleshooting information related to ITSI's Event Analytics functionality.</description>
  <init>
    <set token="reModeSearch">reMode IN ("RealTime", "Adhoc", "Queue")</set>
  </init>
  <search>
    <query>
      | rest splunk_server=local /services/configs/conf-app_common_flags/itsi-rulesengine-adhoc
      | rename disabled as reMode
      | eval reMode = if(reMode == "0", "Adhoc", "RealTime")
      | append [
          | rest splunk_server=local /services/configs/conf-app_common_flags/itsi-rulesengine-queue
          | rename disabled as reMode
          | eval reMode = if(reMode == "0", "Queue", "RealTime")
      ]
      | stats count(eval(reMode="Adhoc")) as AdhocCount, count(eval(reMode="RealTime")) as RealTimeCount, count(eval(reMode="Queue")) as QueueCount
      | eval reMode = case(
          AdhocCount > 0, "Adhoc",
          QueueCount > 0, "Queue",
          RealTimeCount > 0, "RealTime",
          true(), "Unknown"
      )
      | table reMode
    </query>
    <earliest>-1d</earliest>
    <latest>now</latest>
    <done>
      <condition match="'result.reMode'==&quot;Queue&quot;">
        <set token="reModeQueue">true</set>
        <unset token="reModeRealtime"></unset>
        <unset token="reModeAdhoc"></unset>
        <set token="reMode">$result.reMode$</set>
      </condition>
      <condition match="'result.reMode'==&quot;Adhoc&quot;">
        <set token="reModeAdhoc">true</set>
        <unset token="reModeRealtime"></unset>
        <unset token="reModeQueue"></unset>
        <set token="reMode">$result.reMode$</set>
      </condition>
      <condition match="'result.reMode'==&quot;RealTime&quot;">
        <unset token="reModeAdhoc"></unset>
        <unset token="reModeQueue"></unset>
        <set token="reModeRealtime">true</set>
        <set token="reMode">$result.reMode$</set>
      </condition>
    </done>
  </search>
  <fieldset submitButton="false">
    <input id="itsi_ea_link" type="link" token="contents">
      <label>Select a panel:</label>
      <choice value="rules_engine_statistics">Rules Engine Statistics</choice>
      <choice value="skipped_events">Skipped Events</choice>
      <choice value="episode_processing_time">Episode Processing Time</choice>
      <choice value="realtime_search_status">Real-time Search Status</choice>
      <choice value="hec_tokens">HEC Tokens</choice>
      <choice value="kv_store_lookups">KV Store Lookups</choice>
      <choice value="action_processing">Action Processing</choice>
      <choice value="event_size_check">Event Size Check</choice>
      <choice value="correlation_search">Correlation Search</choice>
      <choice value="neap">Aggregation Policy</choice>
      <choice value="smartmode">Smart Mode</choice>
      <default>rules_engine_statistics</default>
      <change>
        <condition value="rules_engine_statistics">
          <set token="rules_engine_statistics">true</set>
          <unset token="skipped_events"/>
          <unset token="episode_processing_time"/>
          <unset token="realtime_search_status"/>
          <unset token="hec_tokens"/>
          <unset token="kv_store_lookups"/>
          <unset token="action_processing"/>
          <unset token="event_size_check"/>
          <set token="show_time_token">true</set>
          <unset token="correlation_search"/>
          <unset token="neap"/>
          <unset token="smartmode"/>
        </condition>
        <condition value="skipped_events">
          <unset token="rules_engine_statistics"/>
          <set token="skipped_events">true</set>
          <unset token="episode_processing_time"/>
          <unset token="realtime_search_status"/>
          <unset token="hec_tokens"/>
          <unset token="kv_store_lookups"/>
          <unset token="action_processing"/>
          <unset token="event_size_check"/>
          <set token="show_time_token">true</set>
          <unset token="correlation_search"/>
          <unset token="neap"/>
          <unset token="smartmode"/>
        </condition>
        <condition value="episode_processing_time">
          <unset token="rules_engine_statistics"/>
          <unset token="skipped_events"/>
          <set token="episode_processing_time">true</set>
          <unset token="realtime_search_status"/>
          <unset token="hec_tokens"/>
          <unset token="kv_store_lookups"/>
          <unset token="action_processing"/>
          <unset token="event_size_check"/>
          <set token="show_time_token">true</set>
          <unset token="correlation_search"/>
          <unset token="neap"/>
          <unset token="smartmode"/>
        </condition>
        <condition value="realtime_search_status">
          <unset token="rules_engine_statistics"/>
          <unset token="skipped_events"/>
          <unset token="episode_processing_time"/>
          <set token="realtime_search_status">true</set>
          <unset token="hec_tokens"/>
          <unset token="kv_store_lookups"/>
          <unset token="action_processing"/>
          <unset token="event_size_check"/>
          <unset token="show_time_token"/>
          <unset token="correlation_search"/>
          <unset token="neap"/>
          <unset token="smartmode"/>
        </condition>
        <condition value="hec_tokens">
          <unset token="rules_engine_statistics"/>
          <unset token="skipped_events"/>
          <unset token="episode_processing_time"/>
          <unset token="realtime_search_status"/>
          <set token="hec_tokens">true</set>
          <unset token="kv_store_lookups"/>
          <unset token="action_processing"/>
          <unset token="event_size_check"/>
          <unset token="show_time_token"/>
          <unset token="correlation_search"/>
          <unset token="neap"/>
          <unset token="smartmode"/>
        </condition>
        <condition value="kv_store_lookups">
          <unset token="rules_engine_statistics"/>
          <unset token="skipped_events"/>
          <unset token="episode_processing_time"/>
          <unset token="realtime_search_status"/>
          <unset token="hec_tokens"/>
          <set token="kv_store_lookups">true</set>
          <unset token="action_processing"/>
          <unset token="event_size_check"/>
          <unset token="show_time_token"/>
          <unset token="correlation_search"/>
          <unset token="neap"/>
          <unset token="smartmode"/>
        </condition>
        <condition value="action_processing">
          <unset token="rules_engine_statistics"/>
          <unset token="skipped_events"/>
          <unset token="episode_processing_time"/>
          <unset token="realtime_search_status"/>
          <unset token="hec_tokens"/>
          <unset token="kv_store_lookups"/>
          <set token="action_processing">true</set>
          <unset token="event_size_check"/>
          <set token="show_time_token">true</set>
          <unset token="correlation_search"/>
          <unset token="neap"/>
          <unset token="smartmode"/>
        </condition>
        <condition value="event_size_check">
          <unset token="rules_engine_statistics"/>
          <unset token="skipped_events"/>
          <unset token="episode_processing_time"/>
          <unset token="realtime_search_status"/>
          <unset token="hec_tokens"/>
          <unset token="kv_store_lookups"/>
          <unset token="action_processing"/>
          <set token="event_size_check">true</set>
          <set token="show_time_token">true</set>
          <unset token="correlation_search"/>
          <unset token="neap"/>
          <unset token="smartmode"/>
        </condition>
        <condition value="correlation_search">
          <unset token="rules_engine_statistics"/>
          <unset token="skipped_events"/>
          <unset token="episode_processing_time"/>
          <unset token="realtime_search_status"/>
          <unset token="hec_tokens"/>
          <unset token="kv_store_lookups"/>
          <unset token="action_processing"/>
          <unset token="event_size_check"/>
          <set token="show_time_token">true</set>
          <set token="correlation_search">true</set>
          <unset token="neap"/>
          <unset token="smartmode"/>
        </condition>
        <condition value="neap">
          <unset token="rules_engine_statistics"/>
          <unset token="skipped_events"/>
          <unset token="episode_processing_time"/>
          <unset token="realtime_search_status"/>
          <unset token="hec_tokens"/>
          <unset token="kv_store_lookups"/>
          <unset token="action_processing"/>
          <unset token="event_size_check"/>
          <set token="show_time_token">true</set>
          <unset token="correlation_search"/>
          <set token="neap">true</set>
          <unset token="smartmode"/>
        </condition>
        <condition value="smartmode">
          <unset token="rules_engine_statistics"/>
          <unset token="skipped_events"/>
          <unset token="episode_processing_time"/>
          <unset token="realtime_search_status"/>
          <unset token="hec_tokens"/>
          <unset token="kv_store_lookups"/>
          <unset token="action_processing"/>
          <unset token="event_size_check"/>
          <set token="show_time_token">true</set>
          <unset token="correlation_search"/>
          <unset token="neap"/>
          <set token="smartmode">true</set>
        </condition>
      </change>
    </input>
    <input depends="$show_time_token$" type="time" token="time_token" searchWhenChanged="true">
      <label>Time range:</label>
      <default>
        <earliest>-24h@h</earliest>
        <latest>now</latest>
      </default>
    </input>
    <input depends="$correlation_search$" type="dropdown" token="cs" searchWhenChanged="true">
      <label>Correlation Search</label>
      <fieldForLabel>Correlation Search</fieldForLabel>
      <fieldForValue>search_name</fieldForValue>
      <search>
        <query>`itsi_tracked_alerts_index` | stats first(_time) by search_name | fields search_name</query>
        <earliest>-24h@h</earliest>
        <latest>now</latest>
      </search>
      <choice value="*">All</choice>
      <default>*</default>
      <initialValue>*</initialValue>
    </input>
    <input depends="$action_processing$" type="dropdown" token="at" searchWhenChanged="true">
      <label>Action Type</label>
      <fieldForLabel>Action Type</fieldForLabel>
      <fieldForValue>actionInternalName</fieldForValue>
      <search>
        <query>index=_internal sourcetype=itsi_internal_log source="*itsi_rules_engine*" actionInternalName | stats first(_time) by actionInternalName | fields actionInternalName</query>
        <earliest>-24h@h</earliest>
        <latest>now</latest>
      </search>
      <choice value="*">All</choice>
      <default>*</default>
      <initialValue>*</initialValue>
    </input>
    <input depends="$neap$" type="multiselect" token="policy" searchWhenChanged="true">
      <label>Aggregation Policy:</label>
      <fieldForLabel>policy_title</fieldForLabel>
      <fieldForValue>itsi_policy_id</fieldForValue>
      <search>
        <query>| rest servicesNS/nobody/SA-ITOA/event_management_interface/notable_event_aggregation_policy splunk_server=local report_as=text
  | spath input=value path={}.title | spath input=value path={}._key
  | eval zipped=mvzip('{}._key', '{}.title') | mvexpand zipped
  | eval zipped=split(zipped, ",") | eval itsi_policy_id=mvindex(zipped,0), policy_title=mvindex(zipped, 1)</query>
        <earliest>-24h@h</earliest>
        <latest>now</latest>
      </search>
      <delimiter> OR </delimiter>
      <valuePrefix>itsi_policy_id="</valuePrefix>
      <valueSuffix>"</valueSuffix>
    </input>
  </fieldset>
  <row depends="$rules_engine_statistics$">
    <panel>
      <title>Event Analytics &amp; Rules Engine Statistics</title>
      <input type="time" token="info_panel_time_token">
        <label>Rules Engine Information Time Range</label>
        <default>
          <earliest>-30d@d</earliest>
          <latest>now</latest>
        </default>
      </input>
      <html>
        <h2>The Event Analytics and Rules Engine components are responsible for creating aggregation policies and generating episodes.</h2>
      </html>
      <table depends="$reModeRealtime$">
        <title>Rules Engine Information</title>
        <search>
          <query>
            index="_internal" source="*itsi_rules_engine*" Rulesenginesearch JavaSpecificationVersion JavaVersion| head 1
            | table JavaVersion,JavaSpecificationVersion
            | rename JavaVersion as "Java Version" JavaSpecificationVersion as "Java Specification Version"
            | appendcols [search index=_internal source="*itsi_rules_engine*" earliest=-30s reMode=RealTime | stats dc(reId) as "Number of Rules Engine Processes"]
            | appendcols [search index=_internal source="*itsi_rules_engine*" earliest=-30s reMode=Preview | stats dc(reId) as "Number of Rules Engine Preview Processes"]</query>
          <earliest>$info_panel_time_token.earliest$</earliest>
          <latest>$info_panel_time_token.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="count">20</option>
        <option name="dataOverlayMode">none</option>
        <option name="drilldown">none</option>
        <option name="percentagesRow">false</option>
        <option name="refresh.display">progressbar</option>
        <option name="rowNumbers">false</option>
        <option name="totalsRow">false</option>
        <option name="wrap">true</option>
        <format type="color" field="Java Specification Version">
          <colorPalette type="map">{"N.A.":#DC4E41,"1.7":#F8BE34,"1.8":#53A051,"1.9":#DC4E41,"1.10":#DC4E41,"1.6":#DC4E41}</colorPalette>
        </format>
        <format type="color" field="Number of Rules Engine Processes">
          <colorPalette type="list">[#DC4E41,#53A051,#F8BE34]</colorPalette>
          <scale type="threshold">1,2</scale>
        </format>
        <format type="color" field="Number of Rules Engine Preview Processes">
          <colorPalette type="list">[#53A051,#F8BE34,#DC4E41]</colorPalette>
          <scale type="threshold">4,24</scale>
        </format>
      </table>
      <table depends="$reModeAdhoc$">
        <title>Rules Engine Information</title>
        <search>
          <query>
            index="_internal" source="*itsi_rules_engine*" Rulesenginesearch
            JavaSpecificationVersion JavaVersion| head 1
            | table JavaVersion,JavaSpecificationVersion
            | rename JavaVersion as "Java Version" JavaSpecificationVersion as "Java Specification
            Version"
            | appendcols [search index=_internal source="*itsi_rules_engine*" earliest=-30s
            reMode=Adhoc | stats dc(reId) as "Number of Rules Engine Adhoc Processes"]
            | appendcols [search index=_internal source="*itsi_rules_engine*" earliest=-30s
            reMode=RealTime | stats dc(reId) as "Number of Rules Engine Realtime Processes"]
            | appendcols [search index=_internal source="*itsi_rules_engine*" earliest=-30s
            reMode=Preview | stats dc(reId) as "Number of Rules Engine Preview Processes"]</query>
          <earliest>$info_panel_time_token.earliest$</earliest>
          <latest>$info_panel_time_token.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="count">20</option>
        <option name="dataOverlayMode">none</option>
        <option name="drilldown">none</option>
        <option name="percentagesRow">false</option>
        <option name="rowNumbers">false</option>
        <option name="totalsRow">false</option>
        <option name="wrap">true</option>
        <format type="color" field="Java Specification Version">
          <colorPalette type="map">
            {"N.A.":#DC4E41,"1.7":#F8BE34,"1.8":#53A051,"1.9":#DC4E41,"1.10":#DC4E41,"1.6":#DC4E41}</colorPalette>
        </format>
        <format type="color" field="Number of Rules Engine Adhoc Processes">
          <colorPalette type="list">[#DC4E41,#53A051,#F8BE34]</colorPalette>
          <scale type="threshold">1,2</scale>
        </format>
        <format type="color" field="Number of Rules Engine Realtime Processes">
          <colorPalette type="list">[#53A051,#DC4E41]</colorPalette>
          <scale type="threshold">1</scale>
        </format>
        <format type="color" field="Number of Rules Engine Preview Processes">
          <colorPalette type="list">[#53A051,#F8BE34,#DC4E41]</colorPalette>
          <scale type="threshold">4,24</scale>
        </format>
      </table>
      <table depends="$reModeQueue$">
        <title>Rules Engine Information</title>
        <search>
          <query>
            index="_internal" source="*itsi_rules_engine*" Rulesenginesearch
            JavaSpecificationVersion JavaVersion| head 1
            | table JavaVersion,JavaSpecificationVersion
            | rename JavaVersion as "Java Version" JavaSpecificationVersion as "Java Specification
            Version"
            | appendcols [search index=_internal source="*itsi_rules_engine*" earliest=-30s
            reMode=Queue | stats dc(reId) as "Number of Rules Engine Queue Processes"]
            | appendcols [search index=_internal source="*itsi_rules_engine*" earliest=-30s
            reMode=RealTime | stats dc(reId) as "Number of Rules Engine Realtime Processes"]
            | appendcols [search index=_internal source="*itsi_rules_engine*" earliest=-30s
            reMode=Preview | stats dc(reId) as "Number of Rules Engine Preview Processes"]</query>
          <earliest>$info_panel_time_token.earliest$</earliest>
          <latest>$info_panel_time_token.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="count">20</option>
        <option name="dataOverlayMode">none</option>
        <option name="drilldown">none</option>
        <option name="percentagesRow">false</option>
        <option name="rowNumbers">false</option>
        <option name="totalsRow">false</option>
        <option name="wrap">true</option>
        <format type="color" field="Java Specification Version">
          <colorPalette type="map">
            {"N.A.":#DC4E41,"1.7":#F8BE34,"1.8":#53A051,"1.9":#DC4E41,"1.10":#DC4E41,"1.6":#DC4E41}</colorPalette>
        </format>
        <format type="color" field="Number of Rules Engine Queue Processes">
          <colorPalette type="list">[#DC4E41,#53A051,#F8BE34]</colorPalette>
          <scale type="threshold">1,2</scale>
        </format>
        <format type="color" field="Number of Rules Engine Realtime Processes">
          <colorPalette type="list">[#53A051,#DC4E41]</colorPalette>
          <scale type="threshold">1</scale>
        </format>
        <format type="color" field="Number of Rules Engine Preview Processes">
          <colorPalette type="list">[#53A051,#F8BE34,#DC4E41]</colorPalette>
          <scale type="threshold">4,24</scale>
        </format>
      </table>
      <table>
        <title>Rules Engine Real-Time Search Configuration</title>
        <search>
          <query>
            | rest splunk_server=local /services/configs/conf-savedsearches/itsi_event_grouping
            | rename dispatch.indexedRealtimeOffset as indexedRealtimeOffset
            | appendcols [| rest splunk_server=local /services/configs/conf-limits/realtime]
            | eval indexedRealtimeOffset=if(indexedRealtimeOffset=="", indexed_realtime_disk_sync_delay, indexedRealtimeOffset)
            | table dispatch.indexedRealtime,indexedRealtimeOffset,dispatch.rt_backfill,cron_schedule
            | rename dispatch.indexedRealtime as "Indexed Realtime Enabled" indexedRealtimeOffset as "Indexed Realtime Offset (In Sec.)" dispatch.rt_backfill as "Backfill ALL data in Index flag" cron_schedule as "Search Schedule"
          </query>
          <earliest>0</earliest>
          <latest>now</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="count">20</option>
        <option name="dataOverlayMode">none</option>
        <option name="drilldown">none</option>
        <option name="percentagesRow">false</option>
        <option name="refresh.display">progressbar</option>
        <option name="rowNumbers">false</option>
        <option name="totalsRow">false</option>
        <option name="wrap">true</option>
        <format type="color" field="Indexed Real-Time Enabled">
          <colorPalette type="map">{"":#DC4E41,"0":#DC4E41,"1":#53A051}</colorPalette>
        </format>
        <format type="color" field="Indexed Realtime Offset (in seconds)">
          <colorPalette type="list">[#DC4E41,#53A051,#F8BE34]</colorPalette>
          <scale type="threshold">60,120</scale>
        </format>
        <format type="color" field="Backfill ALL data in Index flag">
          <colorPalette type="map">{"":#53A051,"0":#53A051,"1":#DC4E41}</colorPalette>
        </format>
      </table>
    </panel>
  </row>
  <row depends="$rules_engine_statistics$">
      <panel>
      <title>Rules Engine Event Processing Volume</title>
      <html>
        <h2>The number of events processed by each Rules Engine activity every 10 minutes.</h2>
      </html>
      <chart>
        <search>
          <query>
index="_internal" source="*itsi_rules_engine*" $reModeSearch$ EventId Status
| timechart span=10m count(eval(searchmatch("Status=EventReceived"))) AS "Events Recived"
count(eval(searchmatch("Status=EventBackfilled*"))) AS "Events Backfilled"
count(eval(searchmatch("Status=EventSkipped"))) AS "Events Skipped"
count(eval(searchmatch("Status=EventRouted"))) AS "Events Routed"
count(eval(searchmatch("Status=EventMatchedPolicy"))) AS "Events Matched Policy"
count(eval(searchmatch("Status=Received PolicyExecutor"))) AS "Events Recived by Policy"
count(eval(searchmatch("Status=EventGroupedPushedToHEC"))) AS "Events Pushed To HEC"
count(eval(searchmatch("Status=Failed"))) AS "Events Failed"
          </query>
          <earliest>$time_token.earliest$</earliest>
          <latest>$time_token.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.abbreviation">none</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.abbreviation">none</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.abbreviation">none</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">line</option>
        <option name="charting.chart.bubbleMaximumSize">50</option>
        <option name="charting.chart.bubbleMinimumSize">10</option>
        <option name="charting.chart.bubbleSizeBy">area</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.showDataLabels">none</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">default</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">none</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.mode">standard</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.lineWidth">2</option>
        <option name="refresh.display">progressbar</option>
        <option name="trellis.enabled">0</option>
        <option name="trellis.scales.shared">1</option>
        <option name="trellis.size">medium</option>
      </chart>
    </panel>
  </row>
  <row depends="$rules_engine_statistics$">
      <panel>
      <title>Rules Engine Event Processing Times</title>
      <html>
        <h2>The median time taken by each Rules Engine activity to process an event.</h2>
      </html>
      <chart>
        <search>
          <query>
index=_internal source=*itsi_rules_engine* $reModeSearch$ EventId (EventReceived OR EventRouted OR EventMatchedPolicy OR Received OR Processed)
| stats latest(_time) as etime by EventId,Status
| xyseries EventId Status etime
| eval _time=EventReceived, router_queue_time=EventRouted-EventReceived, router_processing_time=EventMatchedPolicy-EventRouted, pe_queue_time=Received-EventMatchedPolicy, pe_processing_time=if(((Processed-Received) &lt; 0), NULL, Processed-Received), total_time=Processed-EventReceived
```Corner case: pe_processing_time may be negative in scenarios where an already processed event is restored however it gets skipped. Treat the negative values as NULL so they don't get accounted when calculating the overall pe_processing_time.```
| fields _time,router_queue_time,router_processing_time,pe_queue_time,pe_processing_time,total_time
| timechart span=10m median(total_time) as "Total Processing Time" median(router_queue_time) as "Router Queue Time" median(router_processing_time) as "Router Processing Time" median(pe_queue_time) as "PolicyExecutor Queue Time" median(pe_processing_time) as "PolicyExecutor Processing Time"
          </query>
          <earliest>$time_token.earliest$</earliest>
          <latest>$time_token.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.abbreviation">none</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.abbreviation">none</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.abbreviation">none</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">line</option>
        <option name="charting.chart.bubbleMaximumSize">50</option>
        <option name="charting.chart.bubbleMinimumSize">10</option>
        <option name="charting.chart.bubbleSizeBy">area</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.showDataLabels">none</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">default</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">none</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.mode">standard</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.lineWidth">2</option>
        <option name="refresh.display">progressbar</option>
        <option name="trellis.enabled">0</option>
        <option name="trellis.scales.shared">1</option>
        <option name="trellis.size">medium</option>
      </chart>
    </panel>
  </row>
    <row depends="$rules_engine_statistics$">
    <panel>
      <title>Rules Engine Schedulers Health Check</title>
      <html>
        <h2>The number of times various Rules Engine schedulers run per 12-minute time frame. These statistics are used internally to ensure the schedulers are running as expected. By default, the Event Periodic Backfill Scheduler runs every 12 minutes, the Policy Rules Check Scheduler runs every 1 minute for each aggregation policy, the Policy Group Updates KV Store Sync Scheduler runs every 10 seconds for each aggregation policy, and the Refresh Sub-Groups Scheduler runs every 3 hours to remove closed sub-groups for each aggregation policy.</h2>
      </html>
      <chart>
      <search>
          <query>
index="_internal" source="*itsi_rules_engine*" $reModeSearch$ Status=Scheduled
| timechart span=12m count(eval(searchmatch("ActorMsg=PolicyGroupUpdatesKVStoreSyncScheduleMessage*"))) AS "Policy GroupUpdates KVStore Sync Scheduler"
count(eval(searchmatch("ActorMsg=PolicyRulesCheckScheduleMessage*"))) AS "Policy Rules Check Scheduler"
count(eval(searchmatch("ActorMsg=EventBackfillScheduleMessage*"))) AS "Event Periodic Backfill Scheduler"
count(eval(searchmatch("ActorMsg=RefreshSubGroupsScheduleMessage*"))) AS "Refresh SubGroups Scheduler"
          </query>
          <earliest>$time_token.earliest$</earliest>
          <latest>$time_token.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.abbreviation">none</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.abbreviation">none</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.abbreviation">none</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">line</option>
        <option name="charting.chart.bubbleMaximumSize">50</option>
        <option name="charting.chart.bubbleMinimumSize">10</option>
        <option name="charting.chart.bubbleSizeBy">area</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.showDataLabels">none</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">default</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">none</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.mode">standard</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.lineWidth">2</option>
        <option name="refresh.display">progressbar</option>
        <option name="trellis.enabled">0</option>
        <option name="trellis.scales.shared">1</option>
        <option name="trellis.size">medium</option>
      </chart>
    </panel>
  </row>
  <row depends="$rules_engine_statistics$">
    <panel>
      <title>Rules Engine Starts and Stops</title>
      <html>
        <h2>The number of times the Rules Engine starts and stops each hour. </h2>
      </html>
      <input type="dropdown" token="mode" searchWhenChanged="true">
        <label>Running Mode</label>
        <choice value="*">All</choice>
        <choice value="RealTime">Real-time</choice>
        <choice value="Preview">Preview</choice>
        <choice value="Adhoc">Adhoc</choice>
        <choice value="Queue">Queue</choice>
        <search>
          <query/>
          <earliest>$time_token.earliest$</earliest>
          <latest>$time_token.latest$</latest>
        </search>
        <initialValue>$reMode$</initialValue>
        <default>$reMode$</default>
      </input>
      <chart>
        <search>
          <query>
            index="_internal" source="*itsi_rules_engine*"  RulesEngineTask reMode
            | timechart span=60m count(eval(searchmatch("RulesEngineTask=RealTimeSearch Status=Initialized"))) AS "Rules engine Initialized"
            count(eval(searchmatch("RulesEngineTask=RealTimeSearch Status=ValidationFailed"))) AS "Rules engine Validation Failed"
            count(eval(searchmatch("RulesEngineTask=RealTimeSearch Status=Started"))) AS "Rules engine Started"
            count(eval(searchmatch("RulesEngineTask=RealTimeSearch Status=Stopped"))) AS "Rules engine Stopped" by reMode
            | table _time, "Rules engine *: $mode$"
          </query>
          <earliest>$time_token.earliest$</earliest>
          <latest>$time_token.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.abbreviation">none</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.abbreviation">none</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.abbreviation">none</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">column</option>
        <option name="charting.chart.bubbleMaximumSize">50</option>
        <option name="charting.chart.bubbleMinimumSize">10</option>
        <option name="charting.chart.bubbleSizeBy">area</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.showDataLabels">none</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">default</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">none</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.mode">standard</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.lineWidth">2</option>
        <option name="trellis.enabled">0</option>
        <option name="trellis.scales.shared">1</option>
        <option name="trellis.size">medium</option>
      </chart>
    </panel>
  </row>
  <row depends="$rules_engine_statistics$">
    <panel>
      <title>Rules Engine Activity</title>
      <html>
        <h2>The number of states for each completed Rules Engine instance. You can use the instance ID to search the ITSI logs and troubleshoot issues. </h2>
      </html>
      <chart>
        <search>
          <query>
index="_internal" source="*itsi_rules_engine*" | stats latest(_time) as latestTime by reId
| join type=left reId [search earliest=-30d index="_internal" source="*itsi_rules_engine*" RulesEngineTask | stats count(Status) as ActivityCount, latest(reMode) as RuleEngineMode by reId]
| sort latestTime |eval RulesEngineId=reId." - ".RuleEngineMode | table RulesEngineId, ActivityCount
          </query>
          <earliest>$time_token.earliest$</earliest>
          <latest>$time_token.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.abbreviation">none</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.abbreviation">none</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.abbreviation">none</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">column</option>
        <option name="charting.chart.bubbleMaximumSize">50</option>
        <option name="charting.chart.bubbleMinimumSize">10</option>
        <option name="charting.chart.bubbleSizeBy">area</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.showDataLabels">all</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">default</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">none</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.mode">standard</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.lineWidth">2</option>
        <option name="trellis.enabled">0</option>
        <option name="trellis.scales.shared">1</option>
        <option name="trellis.size">medium</option>
      </chart>
    </panel>
  </row>
  <row depends="$rules_engine_statistics$">
    <panel>
        <title>Rules Engine Activity Details</title>
        <html>
        <h2>The details of activities of each Rules Engine instance completed. You can use the instance ID to search the ITSI logs and troubleshoot issues.</h2>
        </html>
      <table>
        <search>
          <query>
index="_internal" source="*itsi_rules_engine*" | stats latest(_time) as latestTime first(host) as RulesEngineHost by reId
| join type=left reId [search earliest=-30d index="_internal" source="*itsi_rules_engine*" RulesEngineTask | eval activity=strftime(_time,"%Y-%m-%d %H:%M:%S.%3N")." - ".RulesEngineTask." - ".Status.", "
| stats values(activity) as Activities,count(RulesEngineTask) as ActivityCount, earliest(_time) as earliestTime,latest(reMode) as RulesEngineMode by reId]
| sort -latestTime | eval RunTime= tostring(latestTime-earliestTime, "duration"), StartTime=strftime(earliestTime,"%Y-%m-%d %H:%M:%S.%3N")
| rename reId as RulesEngineId | table RulesEngineId, RulesEngineMode, Activities, ActivityCount, StartTime, RulesEngineHost, RunTime
          </query>
          <earliest>$time_token.earliest$</earliest>
          <latest>$time_token.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
      </table>
    </panel>
  </row>
  <row depends="$skipped_events$">
    <panel>
      <title>Skipped Events Count</title>
      <html>
        <h2>A raw count of skipped events (events that are not included in any episodes). Under normal conditions, this number should be zero although some skipped events may be observed in the latest time range because the Rules Engine is yet to process those events.
        </h2>
      </html>
      <chart>
        <search>
          <query>
            (`itsi_event_management_index_with_close_events` event_id!=null) OR (`itsi_event_management_group_index_with_close_events` event_id!=null) NOT orig_sourcetype=snow:incident NOT source=itsi@internal@group_closing_event
            | stats first(_time) AS _time dc(itsi_group_id) AS c_grouped by event_id | eval c_tracked = 1
            | where c_grouped=0
            | timechart span=10m sum(c_tracked) as "Skipped Alert Count"
          </query>
          <earliest>$time_token.earliest$</earliest>
          <latest>$time_token.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.abbreviation">none</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.abbreviation">none</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.abbreviation">none</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">column</option>
        <option name="charting.chart.bubbleMaximumSize">50</option>
        <option name="charting.chart.bubbleMinimumSize">10</option>
        <option name="charting.chart.bubbleSizeBy">area</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.showDataLabels">none</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">default</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">none</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.mode">standard</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.lineWidth">2</option>
        <option name="trellis.enabled">0</option>
        <option name="trellis.scales.shared">1</option>
        <option name="trellis.size">medium</option>
      </chart>
    </panel>
    <panel>
      <title>Skipped Events Percentage</title>
      <html>
        <h2>The percentage of ungrouped events versus grouped events. Under normal conditions, this percentage should be zero although some skipped events may be observed in the latest time range because the Rules Engine is yet to process those events.
        </h2>
      </html>
      <chart>
        <search>
          <query>
            (`itsi_event_management_index_with_close_events` event_id!=null) OR (`itsi_event_management_group_index_with_close_events` event_id!=null) NOT orig_sourcetype=snow:incident NOT source=itsi@internal@group_closing_event
            | stats first(_time) AS _time dc(itsi_group_id) AS c_grouped by event_id
            | eval c_tracked = 1
            | timechart span=10m sum(c_tracked) as total sum(eval( if(c_tracked-c_grouped &gt; 0, c_tracked-c_grouped, 0))) as lost
            | eval lost_percent=lost/total*100
            | fields _time, lost_percent
            | rename lost_percent AS "Skipped Alerts Percentage"
          </query>
          <earliest>$time_token.earliest$</earliest>
          <latest>$time_token.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.abbreviation">none</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.abbreviation">none</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.abbreviation">none</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">column</option>
        <option name="charting.chart.bubbleMaximumSize">50</option>
        <option name="charting.chart.bubbleMinimumSize">10</option>
        <option name="charting.chart.bubbleSizeBy">area</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.showDataLabels">none</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">default</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">none</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.mode">standard</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.lineWidth">2</option>
        <option name="refresh.display">progressbar</option>
        <option name="trellis.enabled">0</option>
        <option name="trellis.scales.shared">1</option>
        <option name="trellis.size">medium</option>
      </chart>
    </panel>
  </row>
  <row depends="$skipped_events$">
    <panel>
      <title>Backfilled Events Count</title>
      <html>
        <h2>A raw count of backfilled events. Under normal conditions, this number should be small.
        </h2>
      </html>
      <chart>
        <search>
          <query>
            index="_internal" source="*itsi_rules_engine*" EventId $reModeSearch$ Status=EventBackfilledPeriodically
            | stats first(_time) AS _time  dc(reMode) as c_backfilled by EventId
            | timechart span=1h sum(c_backfilled) as "Backfilled Event Count"
          </query>
          <earliest>$time_token.earliest$</earliest>
          <latest>$time_token.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.abbreviation">none</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.abbreviation">none</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.abbreviation">none</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">column</option>
        <option name="charting.chart.bubbleMaximumSize">50</option>
        <option name="charting.chart.bubbleMinimumSize">10</option>
        <option name="charting.chart.bubbleSizeBy">area</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.showDataLabels">none</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">default</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">none</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.mode">standard</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.lineWidth">2</option>
        <option name="trellis.enabled">0</option>
        <option name="trellis.scales.shared">1</option>
        <option name="trellis.size">medium</option>
      </chart>
    </panel>
    <panel>
      <title>Backfilled Events Percentage</title>
      <html>
        <h2>The percentage of backfilled events versus tracked events. Under normal conditions, this percentage should be less than 1%.
        </h2>
      </html>
      <chart>
        <search>
          <query>
            (`itsi_tracked_alerts_index` event_id!=null ) OR (index="_internal" source="*itsi_rules_engine*" EventId $reModeSearch$ Status=EventBackfilledPeriodically)
            | eval eId=if(isnull(reId),event_id,EventId)
              ``` Need to sort by _time because we need to make sure that we get itsi_tracked_alerts events before _internal index events ```
            | sort _time
            | stats first(_time) AS _time last(sourcetype) as stype by eId
            | eval f_tracked = 1, f_backfilled=if(stype=="itsi_internal_log",1,0)
            | timechart span=1h sum(f_tracked) as c_tracked  sum(f_backfilled) as c_backfilled
            | eval backfilled_percent=c_backfilled/c_tracked*100
            | fields _time, backfilled_percent
            | rename backfilled_percent AS "Backfilled Event Percentage"
          </query>
          <earliest>$time_token.earliest$</earliest>
          <latest>$time_token.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.abbreviation">none</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.abbreviation">none</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.abbreviation">none</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">column</option>
        <option name="charting.chart.bubbleMaximumSize">50</option>
        <option name="charting.chart.bubbleMinimumSize">10</option>
        <option name="charting.chart.bubbleSizeBy">area</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.showDataLabels">none</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">default</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">none</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.mode">standard</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.lineWidth">2</option>
        <option name="refresh.display">progressbar</option>
        <option name="trellis.enabled">0</option>
        <option name="trellis.scales.shared">1</option>
        <option name="trellis.size">medium</option>
      </chart>
    </panel>
  </row>
  <row depends="$episode_processing_time$">
    <panel>
      <title>Episode Processing Times</title>
      <html>
        <h2>The amount of time it takes to convert tracked alerts (active raw notable events) to grouped alerts (active grouped notable events). This time includes the real-time search offset (default index offset is 60 seconds).
        </h2>
      </html>
      <chart>
        <search>
          <query>
              `itsi_grouped_alerts_index` OR `itsi_tracked_alerts_index` | rename _indextime as it
              | stats earliest(it) as it by index event_id | xyseries event_id index it
              | search itsi_grouped_alerts=* AND itsi_tracked_alerts=* | eval latency=itsi_grouped_alerts-itsi_tracked_alerts
              | fields itsi_tracked_alerts latency | bin itsi_tracked_alerts span=10m
              | stats p99(latency) as "99th Percentile Time" min(latency) as "Min Elapsed Time" median(latency) as "Median Elapsed Time" max(latency) as "Max Elapsed Time" by itsi_tracked_alerts
              | rename itsi_tracked_alerts as _time
          </query>
          <earliest>$time_token.earliest$</earliest>
          <latest>$time_token.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.abbreviation">none</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.abbreviation">none</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.abbreviation">none</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">line</option>
        <option name="charting.chart.bubbleMaximumSize">50</option>
        <option name="charting.chart.bubbleMinimumSize">10</option>
        <option name="charting.chart.bubbleSizeBy">area</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.showDataLabels">none</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">default</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">none</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.mode">standard</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.lineWidth">2</option>
        <option name="refresh.display">progressbar</option>
        <option name="trellis.enabled">0</option>
        <option name="trellis.scales.shared">1</option>
        <option name="trellis.size">medium</option>
      </chart>
    </panel>
  </row>
  <row depends="$episode_processing_time$">
    <panel>
      <title>Event Processing Volume</title>
      <html>
        <h2>The number of events tracked in its_tracked_alerts, processed by the Rules Engine, and ingested into its_grouped_alerts per 10-minute time frame. Use this panel to troubleshoot grouping issues.
        </h2>
      </html>
      <chart>
        <search>
          <query>
(`itsi_tracked_alerts_index` event_id!=null ) OR (`itsi_grouped_alerts_index` event_id!=null) OR (index="_internal" source="*itsi_rules_engine*" EventId $reModeSearch$ Status=EventReceived OR Status=EventBackfilled*)
| eval event_id=EventId,IsEventBackfilled=if(Status=="EventBackfilledPeriodically",1,0),IsGrouped=if(isnull(itsi_group_id),1,0)
| stats first(_indextime) AS _time sum(IsGrouped) AS GroupedCount first(reId) AS rId sum(IsEventBackfilled) AS EventBackfilledCount by event_id
| eval  EventTracked = 1, EventReachedRE = if(isnull(rId), 0, 1), EventMissedBySearch = if(isnull(rId), 1, 0), NotGrouped=if(GroupedCount==0, 1, 0)
| table _time, EventTracked, GroupedCount, EventReachedRE, EventMissedBySearch, EventBackfilledCount,NotGrouped
| timechart span=10m  sum(EventTracked) as "Events Tracked", sum(EventReachedRE) as "Events Processed (RE)", sum(GroupedCount) as "Events Grouped", , sum(EventBackfilledCount) as "Event Backfilled Periodically", sum(NotGrouped) as "Events Not Grouped", sum(EventMissedBySearch) as "Event Missed By Search"
          </query>
          <earliest>$time_token.earliest$</earliest>
          <latest>$time_token.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.abbreviation">none</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.abbreviation">none</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.abbreviation">none</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">line</option>
        <option name="charting.chart.bubbleMaximumSize">50</option>
        <option name="charting.chart.bubbleMinimumSize">10</option>
        <option name="charting.chart.bubbleSizeBy">area</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.showDataLabels">none</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">default</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">none</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.mode">standard</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.lineWidth">2</option>
        <option name="refresh.display">progressbar</option>
        <option name="trellis.enabled">0</option>
        <option name="trellis.scales.shared">1</option>
        <option name="trellis.size">medium</option>
      </chart>
    </panel>
  </row>
  <row depends="$episode_processing_time$">
    <panel>
      <title>Event Processing Times</title>
      <html>
        <h2>The median time each Rules Engine component takes to process events. This time does not include the real-time search offset (default index offset is 60 seconds) and is calculated from the point at which an event is received by the Rules Engine.
        </h2>
      </html>
      <chart>
        <search>
          <query>
index=_internal source=*itsi_rules_engine* $reModeSearch$ EventId Status=EventReceived OR Status=EventBackfilled* | stats latest(_time) as recivedLTime by EventId
| join type=left EventId [search index=_internal source=*itsi_rules_engine* $reModeSearch$ EventId Status=EventRouted OR Status=EventSkipped | stats latest(_time) as routerLTime by EventId]
| join type=left EventId [search index=_internal source=*itsi_rules_engine* $reModeSearch$ EventId Status=Received PolicyExecutor | stats latest(_time) as perLtime,earliest(_time) as perEtime by EventId]
| join type=left EventId [search index=_internal source=*itsi_rules_engine* $reModeSearch$ EventId Status=EventGroupedPushedToHEC | stats latest(_time) as hecPushLTime,earliest(_time) as hecPushETime by EventId]
| join type=left EventId [search index=_internal source=*itsi_rules_engine* $reModeSearch$ EventId Status=Processed PolicyExecutor | stats latest(_time) as pepLtime by EventId]
| join type=left EventId [search `itsi_grouped_alerts_index` |  stats latest(_indextime) as giLTime by event_id | rename event_id as EventId]
| join type=left EventId [search `itsi_tracked_alerts_index` |  stats latest(_indextime) as tiLTime by event_id | rename event_id as EventId]
| eval rqTime=routerLTime-recivedLTime, peqTime=perLtime-routerLTime, pepTime=pepLtime-perEtime, heciTime=giLTime+1-hecPushLTime, retpTime=pepLtime-recivedLTime, _time=recivedLTime, rsTime=recivedLTime-tiLTime-60
| eval heciTime=if(heciTime &lt; 0,0,heciTime)
| fields _time, recivedLTime EventId retpTime rqTime peqTime pepTime heciTime rsTime
| timechart span=10m median(rsTime) as "$reMode$ Search Time(-IndexOffset)" median(retpTime) as "RulesEngine Processing Time" median(rqTime) as "Router Processing Time" median(peqTime) as "Policy Queue Wait Time" median(pepTime) as "Policy Processing Time" median(heciTime) as "HEC to Index Time"          </query>
          <earliest>$time_token.earliest$</earliest>
          <latest>$time_token.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.abbreviation">none</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.abbreviation">none</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.abbreviation">none</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">line</option>
        <option name="charting.chart.bubbleMaximumSize">50</option>
        <option name="charting.chart.bubbleMinimumSize">10</option>
        <option name="charting.chart.bubbleSizeBy">area</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.showDataLabels">none</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">default</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">none</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.mode">standard</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.lineWidth">2</option>
        <option name="refresh.display">progressbar</option>
        <option name="trellis.enabled">0</option>
        <option name="trellis.scales.shared">1</option>
        <option name="trellis.size">medium</option>
      </chart>
    </panel>
  </row>
    <row depends="$episode_processing_time$">
    <panel>
      <title>Event Processing Time By Policy</title>
      <chart>
        <search>
          <query>
index=_internal source=*itsi_rules_engine* $reModeSearch$ (PolicyExecutor OR Router) EventMessage EventId!=null PolicyTitle!=null (Received OR EventMatchedPolicy OR Processed)
| stats latest(_time) as perLtime by EventId,PolicyTitle,Status
| eval peid= PolicyTitle."-$-".EventId
| xyseries peid Status perLtime
| eval peids=split(peid,"-$-")
| eval pId=mvindex(peids,0), _time=Received, pe_queue_time=Received-EventMatchedPolicy, pe_process_time=Processed-Received
| fields _time, pId,pe_queue_time,pe_process_time
| timechart span=10m  median(pe_queue_time) as "Policy Queue Wait Time" median(pe_process_time) as "Policy Processing Time" by pId
          </query>
          <earliest>$time_token.earliest$</earliest>
          <latest>$time_token.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.abbreviation">none</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.abbreviation">none</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.abbreviation">none</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">line</option>
        <option name="charting.chart.bubbleMaximumSize">50</option>
        <option name="charting.chart.bubbleMinimumSize">10</option>
        <option name="charting.chart.bubbleSizeBy">area</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.showDataLabels">none</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">default</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">none</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.mode">standard</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.lineWidth">2</option>
        <option name="refresh.display">progressbar</option>
        <option name="trellis.enabled">0</option>
        <option name="trellis.scales.shared">1</option>
        <option name="trellis.size">medium</option>
      </chart>
    </panel>
  </row>
  <row depends="$realtime_search_status$">
    <panel>
      <table>
        <title>Event Analytics Real-Time Search Status</title>
        <search>
          <query>| rest /services/search/jobs
                | search label="itsi_event_grouping"
                | table label dispatchState diskUsage runDuration published
                | appendcols [|rest /servicesNS/-/SA-ITOA/saved/searches splunk_server=local | search title="itsi_event_grouping" | table is_scheduled]
                | eval diskUsage = round(diskUsage/1024/1024, 2)
                | rename label as Label, dispatchState as State, diskUsage as "Disk Usage (MB)", runDuration as "Duration (Second)", published as "Creation Time"
                | filldown
                | eval Scheduled=if(is_scheduled&gt;0, "True", "False")
                | fields - is_scheduled
            </query>
          <earliest>-24h</earliest>
          <latest>now</latest>
        </search>
        <format type="color" field="State">
          <colorPalette type="map">{"RUNNING":#65A637,"ERROR":#D93F3C}</colorPalette>
        </format>
      </table>
    </panel>
  </row>
  <row depends="$hec_tokens$">
    <panel>
      <table>
        <title>Event Analytics HEC Tokens</title>
        <search>
          <query>| rest /services/data/inputs/http
              | search * "eai:acl.app"=*
              | eval title=replace (title, "http://", "")
              | stats values(title) as v by host
              | search host="sh*"
              | eval Auto Generated ITSI Event Management Token=if(v = "Auto Generated ITSI Event Management Token", "Available", "Not Found"),
                Auto Generated ITSI Notable Event Retention Policy Token=if(v = "Auto Generated ITSI Notable Event Retention Policy Token", "Available", "Not Found"),
                Auto Generated ITSI Notable Index Audit Token=if(v = "Auto Generated ITSI Notable Index Audit Token", "Available", "Not Found"),
                itsi_group_alerts_sync_token=if(v = "itsi_group_alerts_sync_token", "Available", "Not Found"),
                itsi_group_alerts_token=if(v = "itsi_group_alerts_token", "Available", "Not Found"),
                itsi_group_comments_token=if(v = "itsi_group_comments_token", "Available", "Not Found")
              | fields - v
          </query>
          <earliest>-24h</earliest>
          <latest>now</latest>
        </search>
      </table>
    </panel>
  </row>
  <row depends="$kv_store_lookups$">
    <panel>
      <table>
        <title>Event Analytics KV Store Lookups</title>
        <search>
          <query>| rest splunk_server=local servicesNS/nobody/SA-ITOA/configs/conf-transforms
              | search * "eai:acl.app"="SA-ITOA"
              | append [| rest splunk_server=local servicesNS/admin/search/data/transforms/lookups | search external_type=kvstore "eai:acl.app"="SA-ITOA"]
              | stats count by title
              | eval Created=if(count&gt;1, "Yes", "No")
              | rename title as Title
              | table Title, Created
          </query>
          <earliest>-24h</earliest>
          <latest>now</latest>
        </search>
        <format type="color" field="Created">
          <colorPalette type="map">{"Yes":#65A637,"No":#D93F3C}</colorPalette>
        </format>
      </table>
    </panel>
  </row>
  <row depends="$action_processing$">
    <panel>
      <title>Actions Processing Volume</title>
      <html>
        <h2>The number of actions created, queued, and processed every 10 minutes.
        </h2>
      </html>
      <chart>
        <search>
          <query>
            index=_internal sourcetype=itsi_internal_log actionId Status=Completed actionInternalName="$at$"
            | timechart span=10m count(eval(searchmatch("FunctionName=actionAddingToQueue"))) AS "Action Added to Queue" count(eval(searchmatch("FunctionName=actionExecution"))) AS "Action Processed"
          </query>
          <earliest>$time_token.earliest$</earliest>
          <latest>$time_token.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.abbreviation">none</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.abbreviation">none</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.abbreviation">none</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">line</option>
        <option name="charting.chart.nullValueMode">zero</option>
        <option name="charting.chart.showDataLabels">none</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">default</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">none</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.mode">standard</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.lineWidth">2</option>
        <option name="refresh.display">progressbar</option>
        <option name="trellis.enabled">0</option>
        <option name="trellis.scales.shared">1</option>
        <option name="trellis.size">medium</option>
      </chart>
    </panel>
    <panel>
      <title>Action Processing Times</title>
      <html>
        <h2>The total time taken in seconds to process a single action.
        </h2>
      </html>
      <chart>
        <search>
          <query>
            index=_internal sourcetype=itsi_internal_log actionId (FunctionName="actionAddingToQueue" OR FunctionName="actionExecution") actionInternalName="$at$"
            | eval startTime=if((Status == "Started" AND FunctionName == "actionAddingToQueue"),'_time',null), endTime=if((Status == "Completed" AND FunctionName="actionExecution"),'_time',null)
            | stats earliest(startTime) as _time latest(endTime) as endTime by actionId
            | eval duration_in_secs=if((abs(endTime - '_time') &lt; 0),0,abs(endTime - '_time'))
            | fields + _time, duration_in_secs
            | timechart span=10m p99(duration_in_secs) as "99th Percentile Processing Time" min(duration_in_secs) as "Min Processing Time" median(duration_in_secs) as "Median Processing Time" max(duration_in_secs) as "Max Processing Time"
          </query>
          <earliest>$time_token.earliest$</earliest>
          <latest>$time_token.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.abbreviation">none</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.abbreviation">none</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.abbreviation">none</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">line</option>
        <option name="charting.chart.nullValueMode">zero</option>
        <option name="charting.chart.showDataLabels">none</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">default</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">none</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.mode">standard</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.lineWidth">2</option>
        <option name="refresh.display">progressbar</option>
        <option name="trellis.enabled">0</option>
        <option name="trellis.scales.shared">1</option>
        <option name="trellis.size">medium</option>
      </chart>
    </panel>
  </row>
  <row depends="$action_processing$">
    <panel>
      <title>Action Queue Times</title>
      <html>
        <h2>The amount of time in seconds that an action spends on the action queue.
        </h2>
      </html>
      <chart>
        <search>
          <query>
          index=_internal sourcetype=itsi_internal_log actionId (FunctionName="actionAddingToQueue" OR FunctionName="actionExecution") actionInternalName="$at$"
          | eval startTime=if((Status == "Completed" AND FunctionName == "actionAddingToQueue"),'_time',null), endTime=if((Status == "Started" AND FunctionName="actionExecution"),'_time',null)
          | stats latest(startTime) as _time earliest(endTime) as endTime by actionId
          | eval duration_in_secs=if((abs(endTime - '_time') &lt; 0),0,abs(endTime - '_time'))
          | fields + _time, duration_in_secs
          | timechart span=10m min(duration_in_secs) as "Min Queue Time" median(duration_in_secs) as "Median Queue Time" max(duration_in_secs) as "Max Queue Time"
          </query>
          <earliest>$time_token.earliest$</earliest>
          <latest>$time_token.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.abbreviation">none</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.abbreviation">none</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.abbreviation">none</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">line</option>
        <option name="charting.chart.nullValueMode">zero</option>
        <option name="charting.chart.showDataLabels">none</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">default</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">none</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.mode">standard</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.lineWidth">2</option>
        <option name="refresh.display">progressbar</option>
        <option name="trellis.enabled">0</option>
        <option name="trellis.scales.shared">1</option>
        <option name="trellis.size">medium</option>
      </chart>
    </panel>
    <panel>
      <title>Action Execution Times</title>
      <html>
        <h2>The time taken in seconds to execute an action after it is pulled from the action queue.
        </h2>
      </html>
      <chart>
        <search>
          <query>
            FunctionName="actionExecution" index=_internal sourcetype=itsi_internal_log actionId actionInternalName="$at$"
            | eval startTime=if((Status == "Started"),'_time',null), endTime=if((Status == "Completed"),'_time',null)
            | stats earliest(startTime) as _time latest(endTime) as endTime by actionId
            | eval duration_in_secs=if((abs(endTime - '_time') &lt; 0),0,abs(endTime - '_time'))
            | fields + _time, duration_in_secs
            | timechart span=10m min(duration_in_secs) as "Min Execution Time" median(duration_in_secs) as "Median Execution Time" max(duration_in_secs) as "Max Execution Time"
          </query>
          <earliest>$time_token.earliest$</earliest>
          <latest>$time_token.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.abbreviation">none</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.abbreviation">none</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.abbreviation">none</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">line</option>
        <option name="charting.chart.nullValueMode">zero</option>
        <option name="charting.chart.showDataLabels">none</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">default</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">none</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.mode">standard</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.lineWidth">2</option>
        <option name="refresh.display">progressbar</option>
        <option name="trellis.enabled">0</option>
        <option name="trellis.scales.shared">1</option>
        <option name="trellis.size">medium</option>
      </chart>
    </panel>
  </row>
  <row depends="$action_processing$">
    <panel>
      <title>Action Queue Consumer Errors</title>
      <html>
        <h2>Count of ERROR logs generated by action queue consumers.
        </h2>
      </html>
      <chart>
        <search>
          <query>index="_internal" sourcetype="itsi_internal_log" source="*itsi_notable_event_actions_queue_consumer*" log_level=ERROR
              | timechart span=5m count as error_count
          </query>
          <earliest>$time_token.earliest$</earliest>
          <latest>$time_token.latest$</latest>
        </search>
        <option name="charting.chart">area</option>
        <option name="charting.drilldown">none</option>
      </chart>
    </panel>
  </row>
  <row depends="$action_processing$">
    <panel>
        <title>Action Failures</title>
        <html>
        <h2>Details of action failures by episode</h2>
        </html>
      <table>
        <search>
          <query>`itsi_notable_audit_index` sourcetype=itsi_notable:audit activity_type="Action failed for episode"
          | eval mod_time=strftime(_time, "%Y-%m-%d %H:%M:%S") | table event_id activity_type mod_time user activity
          | rename event_id AS itsi_group_id activity_type AS "Activity Type" mod_time AS Time user AS User activity AS Activity
          </query>
          <earliest>$time_token.earliest$</earliest>
          <latest>$time_token.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
      </table>
    </panel>
  </row>
  <row depends="$action_processing$">
    <panel>
      <title>Queue Consumer Object Count</title>
      <html>
        <h2>Count of Queue Consumer Object by Action type.
        </h2>
      </html>
      <table>
        <search>
          <query>| inputlookup itsi_notable_event_actions_queue_lookup
| eval uri_info=uri
| rex field=uri_info "(?&lt;action_info&gt;([^/]+)$)"
| stats count as "Action Count" by action_info
| rename action_info as "Action Name"</query>
          <earliest>$time_token.earliest$</earliest>
          <latest>$time_token.latest$</latest>
        </search>
        <option name="drilldown">none</option>
        <option name="refresh.display">progressbar</option>
      </table>
    </panel>
  </row>
  <row depends="$event_size_check$">
    <panel>
      <title>Notable Event Size Check</title>
      <chart>
        <search>
          <query>index=itsi_*_alerts sourcetype="itsi_notable:event" OR sourcetype="itsi_notable:group"
              | eval invalid_ending=if(substr(_raw, len(_raw), 1)=="}", 0, 1), raw_len=len(_raw)
              | timechart span=30m  max(raw_len) as "Maximum Event Size" avg(raw_len) as "Average Event Size"
                min(raw_len) as "Minimum Event Size" count as "Total Event Count" sum(invalid_ending) as "Invalid Event Count"
          </query>
          <earliest>$time_token.earliest$</earliest>
          <latest>$time_token.latest$</latest>
        </search>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">collapsed</option>
        <option name="charting.axisTitleY.text">Size in Byte</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.text">Event Count</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.abbreviation">none</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.abbreviation">none</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.abbreviation">none</option>
        <option name="charting.axisY2.enabled">1</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">column</option>
        <option name="charting.chart.bubbleMaximumSize">50</option>
        <option name="charting.chart.bubbleMinimumSize">10</option>
        <option name="charting.chart.bubbleSizeBy">area</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.overlayFields">"Invalid Event Count","Total Event Count"</option>
        <option name="charting.chart.showDataLabels">none</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">default</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">none</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.mode">standard</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.lineWidth">2</option>
        <option name="charting.seriesColors">[99e6e6,adebeb,c2f0f0,009900,ff0000]</option>
        <option name="refresh.display">progressbar</option>
        <option name="trellis.enabled">0</option>
        <option name="trellis.scales.shared">1</option>
        <option name="trellis.size">medium</option>
      </chart>
    </panel>
  </row>
  <row depends="$correlation_search$">
    <panel>
      <title>Events By Correlation Searches and Indexes</title>
      <chart>
        <search>
          <query>index="itsi_*_alerts" search_name="$cs$" | bin span=10m _time  | stats count by _time index search_name | eval index_search_name=index." : ".search_name | xyseries  _time index_search_name count</query>
          <earliest>$time_token.earliest$</earliest>
          <latest>$time_token.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.abbreviation">none</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.abbreviation">none</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.abbreviation">none</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">column</option>
        <option name="charting.chart.bubbleMaximumSize">50</option>
        <option name="charting.chart.bubbleMinimumSize">10</option>
        <option name="charting.chart.bubbleSizeBy">area</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.showDataLabels">none</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">default</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">none</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.mode">standard</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.lineWidth">2</option>
        <option name="trellis.enabled">0</option>
        <option name="trellis.scales.shared">1</option>
        <option name="trellis.size">medium</option>
      </chart>
    </panel>
  </row>
  <row depends="$neap$">
    <panel>
      <title>Events by Aggregation Policy</title>
      <chart>
        <search>
          <query>`itsi_grouped_alerts_index` $policy$
| join type=left itsi_policy_id [
  | rest servicesNS/nobody/SA-ITOA/event_management_interface/notable_event_aggregation_policy splunk_server=local report_as=text
  | spath input=value path={}.title | spath input=value path={}._key
  | eval zipped=mvzip('{}._key', '{}.title') | mvexpand zipped
  | eval zipped=split(zipped, ",") | eval itsi_policy_id=mvindex(zipped,0), policy_title=mvindex(zipped, 1)]
| timechart span=10m count by policy_title</query>
          <earliest>$time_token.earliest$</earliest>
          <latest>$time_token.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.abbreviation">none</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.abbreviation">none</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.abbreviation">none</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">line</option>
        <option name="charting.chart.bubbleMaximumSize">50</option>
        <option name="charting.chart.bubbleMinimumSize">10</option>
        <option name="charting.chart.bubbleSizeBy">area</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.showDataLabels">none</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">default</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">none</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.mode">standard</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.lineWidth">2</option>
        <option name="refresh.display">progressbar</option>
        <option name="trellis.enabled">0</option>
        <option name="trellis.scales.shared">1</option>
        <option name="trellis.size">medium</option>
      </chart>
    </panel>
  </row>
  <row depends="$smartmode$">
    <panel>
      <title>Smart Mode Policies</title>
      <html>
        <h2>The number of seed groups and active groups in each aggregation policy with Smart Mode enabled every 10 minutes.</h2>
      </html>
      <chart>
        <search>
          <query>index="_internal" source=*rules_engine* FunctionName=handleEventMetaData Status=Started
| timechart span=10m max(SeedGroupMapSize) as "Seed Groups", max(ActiveGroupIdMapSize) as "Active Groups" by PolicyTitle
          </query>
          <earliest>$time_token.earliest$</earliest>
          <latest>$time_token.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.abbreviation">none</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.abbreviation">none</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.abbreviation">none</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">line</option>
        <option name="charting.chart.bubbleMaximumSize">50</option>
        <option name="charting.chart.bubbleMinimumSize">10</option>
        <option name="charting.chart.bubbleSizeBy">area</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.showDataLabels">none</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">default</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">none</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.mode">standard</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.lineWidth">2</option>
        <option name="refresh.display">progressbar</option>
        <option name="trellis.enabled">0</option>
        <option name="trellis.scales.shared">1</option>
        <option name="trellis.size">medium</option>
      </chart>
    </panel>
  </row>
  <row depends="$smartmode$">
    <panel>
      <title>Rules Engine Smart Mode Event Processing Volume</title>
      <html>
        <h2>The number of events processed by each Rules Engine activity for Smart Mode policies every 10 minutes.</h2>
      </html>
      <chart>
        <search>
          <query>
index="_internal" source="*itsi_rules_engine*" $reModeSearch$ EventId SmartMode
| timechart span=10m count(eval(searchmatch("SmartMode=EventsSentToEventDataExtractor"))) AS "Events Received"
count(eval(searchmatch("SmartMode=EventsMetadataExtracted"))) AS "Events Metadata Extracted"
count(eval(searchmatch("SmartMode=EventsSentToServiceEventCorrelator"))) AS "Events Sent To Service EventCorrelator"
count(eval(searchmatch("SmartMode=EventsSentToEntityEventCorrelator"))) AS "Events Sent To Entity EventCorrelator"
count(eval(searchmatch("SmartMode=EventsSentToDefaultEventCorrelator"))) AS "Events Sent To Default EventCorrelator"
count(eval(searchmatch("SmartMode=EventsForTicketingActions"))) AS "Events For Ticketing Actions"
count(eval(searchmatch("SmartMode=EventsForBreakingGroup"))) AS "Events For Breaking Group"
count(eval(searchmatch("SmartMode=EventsSentToEventMatcher"))) AS "Events Sent To EventMatcher"
count(eval(searchmatch("SmartMode=ProcessedWinningGroupsForEvents"))) AS "Processed Winning Groups For Events"
count(eval(searchmatch("SmartMode=EventsSentToHEC"))) AS "Events Sent To HEC"
          </query>
          <earliest>$time_token.earliest$</earliest>
          <latest>$time_token.latest$</latest>
          <sampleRatio>1</sampleRatio>
        </search>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.abbreviation">none</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.abbreviation">none</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.abbreviation">none</option>
        <option name="charting.axisY2.enabled">0</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">line</option>
        <option name="charting.chart.bubbleMaximumSize">50</option>
        <option name="charting.chart.bubbleMinimumSize">10</option>
        <option name="charting.chart.bubbleSizeBy">area</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.showDataLabels">none</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">default</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">none</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.mode">standard</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.lineWidth">2</option>
        <option name="refresh.display">progressbar</option>
        <option name="trellis.enabled">0</option>
        <option name="trellis.scales.shared">1</option>
        <option name="trellis.size">medium</option>
      </chart>
    </panel>
  </row>
</form>
