<form stylesheet="search_health.css" script="search_health.js">
  <label>Search Activity</label>
  <description></description>
  <fieldset submitButton="false" autoRun="false">
    <input type="dropdown" token="host" searchWhenChanged="true">
      <label>Select a Search Head</label>
      <selectFirstChoice>true</selectFirstChoice>
      <populatingSearch fieldForLabel="host" fieldForValue="host">|inputlookup all_servers.csv | search role=*search* | dedup host| sort + host</populatingSearch>
      <prefix>host=</prefix>
      <fieldForLabel>host</fieldForLabel>
      <fieldForValue>host</fieldForValue>
    </input>
    <input type="time" token="time" searchWhenChanged="true">
      <label></label>
      <default>
        <earliestTime>@d</earliestTime>
        <latestTime>now</latestTime>
      </default>
    </input>
  </fieldset>
  <row>
    <panel>
      <html>
      <div id="usage_help">
        <p/>This dashboard is intended to illustrate search activity and usage on the Splunk Search Head(s) in your environment.
        <p/>Additionally, you can select a range on the CPU utilization graph to narrow down a time range to filter the other panels in the dashboard to identify correlations.
        <p/>The Search Window Statistics table below illustrates different searches ran for a selected period of time. I.e a user selected a search to run for the last 15 Minutes or 30 Days and the panel displays how many searches were executed or cumulative/average execution time during that time frame by index or user.
        </div>
        </html>
    </panel>
  </row>
  <row>
    <panel>
      <chart>
        <title>Peak Historical Search Concurrency by Host - Adhoc and Scheduled</title>
        <searchString>index=_internal $host$ source=*metrics.log group=search_concurrency "system total" NOT user=* | timechart max(active_hist_searches) by host</searchString>
        <earliestTime>$time.earliest$</earliestTime>
        <latestTime>$time.latest$</latestTime>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.enabled">false</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">column</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">stacked</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">all</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.axisTitleX.text">Time</option>
        <option name="charting.axisTitleY.text">Count</option>
        <selection>
          <set token="selection.earliest">$start$</set>
          <set token="selection.latest">$end$</set>
        </selection>
        <option name="charting.chart.bubbleMaximumSize">50</option>
        <option name="charting.chart.bubbleMinimumSize">10</option>
        <option name="charting.chart.bubbleSizeBy">area</option>
      </chart>
    </panel>
    <panel>
      <chart>
        <title>Peak Real-Time Search Concurrency by Host  - Adhoc and Scheduled</title>
        <searchString>index=_internal $host$ source=*metrics.log group=search_concurrency "system total" NOT user=* | timechart max(active_realtime_searches) by host</searchString>
        <earliestTime>$time.earliest$</earliestTime>
        <latestTime>$time.latest$</latestTime>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.text">Time</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.text">Count</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.enabled">false</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">column</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">stacked</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">all</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.placement">right</option>
        <selection>
          <set token="selection.earliest">$start$</set>
          <set token="selection.latest">$end$</set>
        </selection>
        <option name="charting.chart.bubbleMaximumSize">50</option>
        <option name="charting.chart.bubbleMinimumSize">10</option>
        <option name="charting.chart.bubbleSizeBy">area</option>
      </chart>
    </panel>
  </row>
  <row>
    <panel>
      <chart>
        <title>Avg % CPU Utilization by Process</title>
        <searchString>| tstats max(Introspection.data.pct_cpu) AS pct_cpu FROM datamodel=Introspection_Usage WHERE nodename=Introspection.PerProcess_Resource_Usage $host$ GROUPBY _time Introspection.data.process | rename Introspection.data.process AS process | timechart avg(pct_cpu) by process</searchString>
        <earliestTime>$time.earliest$</earliestTime>
        <latestTime>$time.latest$</latestTime>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.enabled">false</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">line</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">default</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">none</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.placement">right</option>
        <selection>
          <set token="selection.earliest">$start$</set>
          <set token="selection.latest">$end$</set>
        </selection>
        <option name="charting.axisTitleX.text">Time</option>
        <option name="charting.axisTitleY.text">% CPU</option>
        <option name="charting.chart.bubbleMaximumSize">50</option>
        <option name="charting.chart.bubbleMinimumSize">10</option>
        <option name="charting.chart.bubbleSizeBy">area</option>
      </chart>
    </panel>
    <panel>
      <chart>
        <title>Avg % Memory Utilization by Process</title>
        <searchString>| tstats max(Introspection.data.pct_memory) AS pct_memory FROM datamodel=Introspection_Usage WHERE nodename=Introspection.PerProcess_Resource_Usage $host$ GROUPBY _time Introspection.data.process | rename Introspection.data.process AS process | timechart avg(pct_memory) by process</searchString>
        <earliestTime>$time.earliest$</earliestTime>
        <latestTime>$time.latest$</latestTime>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.text">Time</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.text">MB</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.enabled">false</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">line</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">default</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">all</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.placement">right</option>
        <selection>
          <set token="selection.earliest">$start$</set>
          <set token="selection.latest">$end$</set>
        </selection>
        <option name="charting.chart.bubbleMaximumSize">50</option>
        <option name="charting.chart.bubbleMinimumSize">10</option>
        <option name="charting.chart.bubbleSizeBy">area</option>
      </chart>
    </panel>
  </row>
  <row>
    <panel>
      <chart>
        <title>Search Duration Histogram by Search Head</title>
        <searchString>index=_audit $host$ action=search user!=splunk-system-user search_id=* (info=granted OR info=completed) | eval run_time_min=round(total_run_time/60,2)  | stats earliest(_time) AS "Start Time" latest(_time) AS "End Time" count values(search) AS Search values(user) AS User max(run_time_min) AS run_time_min by search_id host | convert ctime(*Time) | where count&gt;1 | fields - count  | rename search_id AS SID | bucket run_time_min span=5 | chart count over run_time_min by host | rename run_time_min AS "Run Time (Minutes)"</searchString>
        <earliestTime>$selection.earliest$</earliestTime>
        <latestTime>$selection.latest$</latestTime>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.enabled">false</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">bar</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">stacked</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">all</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.axisTitleY.text">Search Execution Count</option>
        <option name="charting.chart.bubbleMaximumSize">50</option>
        <option name="charting.chart.bubbleMinimumSize">10</option>
        <option name="charting.chart.bubbleSizeBy">area</option>
      </chart>
    </panel>
    <panel>
      <chart>
        <title>Search Count and Average Run Time</title>
        <searchString>index=_audit $host$ action=search user!=splunk-system-user search_id=* info=completed | eval run_time_min=round(total_run_time/60,2) | timechart span=1h count AS "Search Count" avg(run_time_min) AS "Run Time Min"</searchString>
	<earliestTime>$selection.earliest$</earliestTime>
        <latestTime>$selection.latest$</latestTime>
        <option name="charting.chart">column</option>
        <option name="charting.axisY2.enabled">true</option>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart.bubbleMaximumSize">50</option>
        <option name="charting.chart.bubbleMinimumSize">10</option>
        <option name="charting.chart.bubbleSizeBy">area</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">default</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">all</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.chart.overlayFields">"Run Time Min"</option>
        <option name="charting.axisTitleX.text">Time</option>
      </chart>
    </panel>
  </row>
  <row>
    <panel>
      <table id="table_status">
        <title>Searches and Run Time</title>
        <searchString>index=_audit $host$ action=search user!=splunk-system-user search_id=* (info=granted OR info=completed) |rex field=apiStartTime "'(?&lt;start_time&gt;[^']+)'" |rex field=apiEndTime "'(?&lt;end_time&gt;[^']+)'" | eval search_id = trim(if(isnull(search_id), id, search_id), "'") | eval run_time_min=round(total_run_time/60,2) |eval range=if(start_time=="ZERO_TIME","All Time", tostring(strptime(end_time, "%a %b %d %H:%M:%S %Y") - strptime(start_time, "%a %b %d %H:%M:%S %Y"),"duration")) | stats earliest(_time) AS "Start Time" latest(_time) AS "End Time" values(start_time) AS "Search Earliest" values(end_time) AS "Search Latest" count values(range) AS range values(search) AS Search values(user) AS User max(run_time_min) AS "Run Time (Min)" by search_id | convert ctime(*Time) | where count&gt;1 | rename search_id AS SID range AS "Search Range" | table "Start Time" "End Time" SID Search User "Run Time (Min)" "Search Range" "Search Earliest" "Search Latest"| sort - "Run Time (Min)"</searchString>
        <earliestTime>$selection.earliest$</earliestTime>
        <latestTime>$selection.latest$</latestTime>
        <option name="wrap">true</option>
        <option name="rowNumbers">false</option>
        <option name="dataOverlayMode">none</option>
        <drilldown>
          <link>
            <![CDATA[
/app/splunk_health_overview/search_detail?form.sid=$row.SID$]]>
          </link>
        </drilldown>
        <option name="count">10</option>
        <option name="drilldown">cell</option>
      </table>
    </panel>
  </row>
  <row>
    <panel>
      <input type="dropdown" token="split" searchWhenChanged="true">
        <label>Split By:</label>
        <choice value="Index">Index</choice>
        <choice value="user">User</choice>
        <default>Index</default>
      </input>
      <input type="radio" token="value" searchWhenChanged="true">
        <label>Values:</label>
        <choice value="sum(run_time_min)">Sum of Duration (Minutes)</choice>
        <choice value="avg(run_time_min)">Average Duration (Minutes)</choice>
        <choice value="count">Count of Searches</choice>
        <default>count</default>
      </input>
      <table>
        <title>Search Window Statistics by $split$</title>
        <searchString>index=_audit $host$ action=search search=* |rex field=apiStartTime "'(?&lt;start_time&gt;[^']+)'" |rex field=_raw "[^\_]index=\"?(?&lt;Index&gt;[\_a-zA-Z\-\:]{2,})\"?" |eval Index=lower(Index) |search Index!="|history" Index!="|metadata" Index!="|dbinspect" |eval ctime = _time | join search_id [search index=_audit action=search info=completed total_run_time=* | eval run_time_min=total_run_time/60 | fields run_time_min search_id user] |eval range=case( start_time=="ZERO_TIME","All Time", start_time!="ZERO_TIME" AND ctime - strptime(start_time, "%a %b %d %H:%M:%S %Y") &lt;= 300, "5 Minutes", start_time!="ZERO_TIME" AND ctime - strptime(start_time, "%a %b %d %H:%M:%S %Y") &lt;= 900 AND ctime - strptime(start_time, "%a %b %d %H:%M:%S %Y") &gt; 300, "15 Minutes", start_time!="ZERO_TIME" AND ctime - strptime(start_time, "%a %b %d %H:%M:%S %Y") &lt;= 3600 AND ctime - strptime(start_time, "%a %b %d %H:%M:%S %Y") &gt; 900, "1 Hour", start_time!="ZERO_TIME" AND ctime - strptime(start_time, "%a %b %d %H:%M:%S %Y") &lt;= 14400 AND ctime - strptime(start_time, "%a %b %d %H:%M:%S %Y") &gt; 3600, "4 Hours", start_time!="ZERO_TIME" AND ctime - strptime(start_time, "%a %b %d %H:%M:%S %Y") &lt;= 86400 AND ctime - strptime(start_time, "%a %b %d %H:%M:%S %Y") &gt; 14400, "1 Day", start_time!="ZERO_TIME" AND ctime - strptime(start_time, "%a %b %d %H:%M:%S %Y") &lt;= 610000 AND ctime - strptime(start_time, "%a %b %d %H:%M:%S %Y") &gt; 86400, "7 Days", start_time!="ZERO_TIME" AND ctime - strptime(start_time, "%a %b %d %H:%M:%S %Y") &lt;= 2592000 AND ctime - strptime(start_time, "%a %b %d %H:%M:%S %Y") &gt; 604800, "30 Days", start_time!="ZERO_TIME" AND ctime - strptime(start_time, "%a %b %d %H:%M:%S %Y") &lt;= 3888000 AND ctime - strptime(start_time, "%a %b %d %H:%M:%S %Y") &gt; 2592000, "45 Days", start_time!="ZERO_TIME" AND ctime - strptime(start_time, "%a %b %d %H:%M:%S %Y") &gt; 3888000, "45 Days +", start_time==start_time, "Other")  | eval Index=if(Index="","No Index Used in Search",Index) |eval user=if(user="","N/A",user) |chart limit=0 $value$ over $split$ by range | fields $split$ "5 Minutes" "15 Minutes" "1 Hour" "4 Hours" "1 Day" "7 Days" "30 Days" "45 Days" "45 Days +" "All Time" | fillnull $split$ "5 Minutes" "15 Minutes" "1 Hour" "4 Hours" "1 Day" "7 Days" "30 Days" "45 Days" "45 Days +" "All Time" | table $split$ "5 Minutes" "15 Minutes" "1 Hour" "4 Hours" "1 Day" "7 Days" "30 Days" "45 Days" "45 Days +" "All Time" | addtotals col=true | fillnull value=Total</searchString>
        <earliestTime>$selection.earliest$</earliestTime>
        <latestTime>$selection.latest$</latestTime>
        <option name="list.drilldown">full</option>
        <option name="list.wrap">1</option>
        <option name="maxLines">5</option>
        <option name="raw.drilldown">full</option>
        <option name="rowNumbers">false</option>
        <option name="table.drilldown">all</option>
        <option name="table.wrap">1</option>
        <option name="type">list</option>
        <option name="wrap">true</option>
        <option name="dataOverlayMode">heatmap</option>
        <option name="drilldown">cell</option>
        <option name="count">10</option>
      </table>
    </panel>
    <panel>
      <table>
        <title>Usage by User</title>
        <searchString>index=_audit $host$ action=search (id=* OR search_id=*)   | eval search_id = if(isnull(search_id), id, search_id) | replace '*' with * in search_id |  rex "search='(?&lt;search&gt;.*?)', autojoin" | search search_id!=scheduler_* | convert num(total_run_time) | rex "\,\s+user=(?&lt;user&gt;\S+)," | eval user = if(user="n/a", null(), user) | stats min(_time) as _time first(user) as user max(total_run_time) as total_run_time first(search) as search by search_id host  | search search=search* search!=*_internal* search!=*_audit* | stats median(total_run_time) as "Median search time" perc95(total_run_time) as "95th Percentile search time" sum(total_run_time) as "Total search time" count as "Search count" max(_time) as "Last use" by user host  | fieldformat "Last use" = strftime('Last use', "%F %T.%Q %:z") | rename host AS "Search Head" | sort - "Median search time"</searchString>
        <earliestTime>$selection.earliest$</earliestTime>
        <latestTime>$selection.latest$</latestTime>
        <option name="wrap">true</option>
        <option name="rowNumbers">false</option>
        <option name="drilldown">cell</option>
        <option name="dataOverlayMode">none</option>
        <option name="count">10</option>
      </table>
    </panel>
  </row>
</form>
