<form>
  <label>Scheduler Activity</label>
  <description></description>
  <fieldset submitButton="false">
    <input type="dropdown" token="host" searchWhenChanged="true">
      <label>Select a Search Head</label>
      <selectFirstChoice>true</selectFirstChoice>
     
              <populatingSearch fieldForLabel="host" fieldForValue="host">|inputlookup all_servers.csv | search role=*search* | dedup host | fields host  | sort + host</populatingSearch>
     
      <prefix>host=</prefix>
      <fieldForLabel>host</fieldForLabel>
      <fieldForValue>host</fieldForValue>
    </input>
    <input type="time" token="time" searchWhenChanged="true">
      <label></label>
      <default>
        <earliestTime>@d</earliestTime>
        <latestTime>now</latestTime>
      </default>
    </input>
  </fieldset>
  <row>
    <panel>
      <html>
      <div id="usage_help">
        <p/>This dashboard is intended to illustrate scheduled search activity and usage across your different Splunk Search Head(s).
        <p/>Use the input to choose the Search Head you would like to include results from and the timerange picker to select the dates.
        </div>
        </html>
    </panel>
  </row>
  <row>
    <panel>
      <table>
        <title>Scheduler Activity</title>
        <searchString>index=_internal $host$ source=*scheduler.log app="*" savedsearch_name="*"                     | stats min(run_time) as "Min runtime (seconds)", median(run_time) as median_runtime, max(run_time) as max_runtime, count(eval(status!="continued")) AS total_exec, count(eval(status=="success")) as "Successful executions", count(eval(status=="skipped")) AS "Skipped executions" by app, savedsearch_name, user host                | stats first(*) as * by savedsearch_name                                              | eval interval_usage_ratio=round((median_runtime/schedule_period),2)                       | search total_exec&gt;0                       | rename app AS App median_runtime AS "Median runtime (seconds)"  max_runtime AS "Max runtime (seconds)" savedsearch_name AS "Scheduled search name" user AS User total_exec AS "Total executions"                      | table host  App "Scheduled search name" User "Min runtime (seconds)" "Median runtime (seconds)" "Max runtime (seconds)" "Total executions" "Successful executions" "Skipped executions"                        | sort - "Median runtime (seconds)" "Total executions"</searchString>
        <earliestTime>$time.earliest$</earliestTime>
        <latestTime>$time.latest$</latestTime>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.enabled">false</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">column</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">default</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">all</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.placement">right</option>
        <option name="wrap">true</option>
        <option name="rowNumbers">false</option>
        <option name="dataOverlayMode">none</option>
        <option name="drilldown">cell</option>
        <option name="count">10</option>
      </table>
    </panel>
  </row>
  <row>
    <panel>
      <chart>
        <title>Scheduler Execution</title>
        <searchString>index=_internal $host$ source=*metrics.log group=searchscheduler             | timechart partial=false sum(dispatched) AS Started, sum(skipped) AS Skipped             | appendcols [search index=_internal $host$ sourcetype=scheduler status=continued                          | eval savedsearch_id_scheduled_time=savedsearch_id."-".scheduled_time                          | timechart dc(savedsearch_id_scheduled_time) AS Deferred]</searchString>
        <earliestTime>$time.earliest$</earliestTime>
        <latestTime>$time.latest$</latestTime>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.enabled">false</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">column</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">stacked</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">all</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.axisTitleX.text">Time</option>
        <option name="charting.chart.bubbleMaximumSize">50</option>
        <option name="charting.chart.bubbleMinimumSize">10</option>
        <option name="charting.chart.bubbleSizeBy">area</option>
      </chart>
    </panel>
    <panel>
      <table>
        <title>Skipped Scheduled Searches</title>
        <searchString>index=_internal $host$ source=*scheduler.log savedsplunker status=skipped  | stats count by host savedsearch_name, app, user | sort - SkipCount | rename savedsearch_name AS "Scheduled search name" count AS "Skip count" host AS Server</searchString>
        <earliestTime>$time.earliest$</earliestTime>
        <latestTime>$time.latest$</latestTime>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.enabled">false</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">column</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">default</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">all</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.placement">right</option>
        <option name="wrap">true</option>
        <option name="rowNumbers">false</option>
        <option name="dataOverlayMode">none</option>
        <option name="drilldown">cell</option>
      </table>
    </panel>
  </row>
  <row>
    <panel>
      <chart>
        <title>Execution Count by Saved Search</title>
        <searchString>index=_internal $host$ source=*scheduler.log status="*" AND status!="continued" | timechart count by savedsearch_name</searchString>
        <earliestTime>$time.earliest$</earliestTime>
        <latestTime>$time.latest$</latestTime>
        <option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
        <option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
        <option name="charting.axisTitleX.visibility">visible</option>
        <option name="charting.axisTitleY.visibility">visible</option>
        <option name="charting.axisTitleY2.visibility">visible</option>
        <option name="charting.axisX.scale">linear</option>
        <option name="charting.axisY.scale">linear</option>
        <option name="charting.axisY2.enabled">false</option>
        <option name="charting.axisY2.scale">inherit</option>
        <option name="charting.chart">column</option>
        <option name="charting.chart.nullValueMode">gaps</option>
        <option name="charting.chart.sliceCollapsingThreshold">0.01</option>
        <option name="charting.chart.stackMode">stacked</option>
        <option name="charting.chart.style">shiny</option>
        <option name="charting.drilldown">all</option>
        <option name="charting.layout.splitSeries">0</option>
        <option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
        <option name="charting.legend.placement">right</option>
        <option name="charting.axisTitleX.text">Time</option>
      </chart>
    </panel>
    <panel>
      <table>
        <title>Most Active Searches - Top 10</title>
        <searchString>| rest splunk_server=* /servicesNS/-/-/admin/savedsearch/ earliest_time=-0s@s latest_time=+2d@d  search="is_scheduled=1" search="disabled=0" | table splunk_server eai:acl.app eai:acl.owner cron_schedule title scheduled_times  | mvexpand scheduled_times  | rename scheduled_times as _time eai:acl.app as app eai:acl.owner as user title as search |  stats count(search) as count by search cron_schedule app | sort 10 -count</searchString>
        <earliestTime>0</earliestTime>
        <latestTime></latestTime>
        <option name="wrap">undefined</option>
        <option name="rowNumbers">undefined</option>
        <option name="drilldown">row</option>
      </table>
    </panel>
  </row>
  <row>
    <panel>
      <event>
        <title>Scheduler Errors</title>
        <searchString>index=_internal $host$ source=*scheduler.log error OR warn</searchString>
        <earliestTime>$time.earliest$</earliestTime>
        <latestTime>$time.latest$</latestTime>
        <option name="wrap">true</option>
        <option name="rowNumbers">0</option>
        <option name="dataOverlayMode">none</option>
        <option name="list.drilldown">full</option>
        <option name="list.wrap">1</option>
        <option name="maxLines">5</option>
        <option name="raw.drilldown">full</option>
        <option name="table.drilldown">all</option>
        <option name="table.wrap">1</option>
        <option name="type">list</option>
        <fields>["host","source","sourcetype"]</fields>
      </event>
    </panel>
  </row>
</form>
