You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
462 lines
27 KiB
462 lines
27 KiB
<form version="1.1">
|
|
<label>Dashboard - Troubleshooting Indexer CPU</label>
|
|
<fieldset submitButton="false">
|
|
<input type="time" token="time_tok">
|
|
<label>General Time Picker</label>
|
|
<default>
|
|
<earliest>-4h@h</earliest>
|
|
<latest>@h</latest>
|
|
</default>
|
|
</input>
|
|
<input type="text" token="user">
|
|
<label>User</label>
|
|
</input>
|
|
<input type="dropdown" searchWhenChanged="true" token="interval">
|
|
<label></label>
|
|
<choice value="10m">10m</choice>
|
|
<choice value="30m">30m</choice>
|
|
<choice value="60m">60m</choice>
|
|
<choice value="120m">120m</choice>
|
|
<choice value="240m">4h</choice>
|
|
<default>60m</default>
|
|
</input>
|
|
<input type="time" token="CPUtimetoken">
|
|
<label>CPU Based Time Picker (Pie charts near top)</label>
|
|
<default>
|
|
<earliest>-1h@h</earliest>
|
|
<latest>@h</latest>
|
|
</default>
|
|
</input>
|
|
</fieldset>
|
|
<row>
|
|
<panel>
|
|
<title>Search Count Per Application</title>
|
|
<chart>
|
|
<search>
|
|
<query>index=_introspection `indexerhosts` sourcetype=splunk_resource_usage data.search_props.sid::* | eval app = 'data.search_props.app'
|
|
| chart count by app</query>
|
|
<earliest>$CPUtimetoken.earliest$</earliest>
|
|
<latest>$CPUtimetoken.latest$</latest>
|
|
</search>
|
|
<option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
|
|
<option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
|
|
<option name="charting.axisTitleX.visibility">visible</option>
|
|
<option name="charting.axisTitleY.visibility">visible</option>
|
|
<option name="charting.axisTitleY2.visibility">visible</option>
|
|
<option name="charting.axisX.scale">linear</option>
|
|
<option name="charting.axisY.scale">linear</option>
|
|
<option name="charting.axisY2.enabled">0</option>
|
|
<option name="charting.axisY2.scale">inherit</option>
|
|
<option name="charting.chart">pie</option>
|
|
<option name="charting.chart.bubbleMaximumSize">50</option>
|
|
<option name="charting.chart.bubbleMinimumSize">10</option>
|
|
<option name="charting.chart.bubbleSizeBy">area</option>
|
|
<option name="charting.chart.nullValueMode">gaps</option>
|
|
<option name="charting.chart.showDataLabels">none</option>
|
|
<option name="charting.chart.sliceCollapsingThreshold">0.01</option>
|
|
<option name="charting.chart.stackMode">default</option>
|
|
<option name="charting.chart.style">shiny</option>
|
|
<option name="charting.drilldown">all</option>
|
|
<option name="charting.layout.splitSeries">0</option>
|
|
<option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
|
|
<option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
|
|
<option name="charting.legend.placement">right</option>
|
|
</chart>
|
|
</panel>
|
|
<panel>
|
|
<title>CPU Usage By Application (point in time across all indexers)</title>
|
|
<chart>
|
|
<title>CPU is approx CPU% at any point in time</title>
|
|
<search>
|
|
<query>index=_introspection `indexerhosts` sourcetype=splunk_resource_usage data.search_props.sid::* | eval app = 'data.search_props.app' | eval cpuperc = 'data.pct_cpu' | bin _time span=1m | stats sum(cpuperc) AS totalCPU, avg(cpuperc) AS avgCPU by data.pid, host, _time, app | stats sum(totalCPU) AS totalCPU, sum(avgCPU) AS avgCPU by app | addinfo | eval overThisManyMinutes = round((info_max_time-info_min_time)/60) | eval CPUPercUsed = round(avgCPU/overThisManyMinutes) | fields - totalCPU, info* overThisManyMinutes, avgCPU</query>
|
|
<earliest>$CPUtimetoken.earliest$</earliest>
|
|
<latest>$CPUtimetoken.latest$</latest>
|
|
<sampleRatio>1</sampleRatio>
|
|
</search>
|
|
<option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
|
|
<option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
|
|
<option name="charting.axisTitleX.visibility">visible</option>
|
|
<option name="charting.axisTitleY.visibility">visible</option>
|
|
<option name="charting.axisTitleY2.visibility">visible</option>
|
|
<option name="charting.axisX.scale">linear</option>
|
|
<option name="charting.axisY.scale">linear</option>
|
|
<option name="charting.axisY2.enabled">0</option>
|
|
<option name="charting.axisY2.scale">inherit</option>
|
|
<option name="charting.chart">pie</option>
|
|
<option name="charting.chart.bubbleMaximumSize">50</option>
|
|
<option name="charting.chart.bubbleMinimumSize">10</option>
|
|
<option name="charting.chart.bubbleSizeBy">area</option>
|
|
<option name="charting.chart.nullValueMode">gaps</option>
|
|
<option name="charting.chart.showDataLabels">none</option>
|
|
<option name="charting.chart.sliceCollapsingThreshold">0.01</option>
|
|
<option name="charting.chart.stackMode">default</option>
|
|
<option name="charting.chart.style">shiny</option>
|
|
<option name="charting.drilldown">all</option>
|
|
<option name="charting.layout.splitSeries">0</option>
|
|
<option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
|
|
<option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
|
|
<option name="charting.legend.placement">right</option>
|
|
</chart>
|
|
</panel>
|
|
<panel>
|
|
<title>Searches Running Per Indexer</title>
|
|
<chart>
|
|
<search>
|
|
<query>index=_introspection `indexerhosts` sourcetype=splunk_resource_usage data.search_props.sid::* | chart count by host</query>
|
|
<earliest>$CPUtimetoken.earliest$</earliest>
|
|
<latest>$CPUtimetoken.latest$</latest>
|
|
<sampleRatio>1</sampleRatio>
|
|
</search>
|
|
<option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
|
|
<option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
|
|
<option name="charting.axisTitleX.visibility">visible</option>
|
|
<option name="charting.axisTitleY.visibility">visible</option>
|
|
<option name="charting.axisTitleY2.visibility">visible</option>
|
|
<option name="charting.axisX.scale">linear</option>
|
|
<option name="charting.axisY.scale">linear</option>
|
|
<option name="charting.axisY2.enabled">0</option>
|
|
<option name="charting.axisY2.scale">inherit</option>
|
|
<option name="charting.chart">pie</option>
|
|
<option name="charting.chart.bubbleMaximumSize">50</option>
|
|
<option name="charting.chart.bubbleMinimumSize">10</option>
|
|
<option name="charting.chart.bubbleSizeBy">area</option>
|
|
<option name="charting.chart.nullValueMode">gaps</option>
|
|
<option name="charting.chart.showDataLabels">none</option>
|
|
<option name="charting.chart.sliceCollapsingThreshold">0.01</option>
|
|
<option name="charting.chart.stackMode">default</option>
|
|
<option name="charting.chart.style">shiny</option>
|
|
<option name="charting.drilldown">all</option>
|
|
<option name="charting.layout.splitSeries">0</option>
|
|
<option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
|
|
<option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
|
|
<option name="charting.legend.placement">right</option>
|
|
</chart>
|
|
</panel>
|
|
<panel>
|
|
<title>Search Related CPU By Indexer</title>
|
|
<chart>
|
|
<title>CPU is approx CPU% at any point in time</title>
|
|
<search>
|
|
<query>index=_introspection `indexerhosts` sourcetype=splunk_resource_usage data.search_props.sid::* | eval cpuperc = 'data.pct_cpu' | bin _time span=1m | stats sum(cpuperc) AS totalCPU, avg(cpuperc) AS avgCPU by data.pid, host, _time| stats sum(totalCPU) AS totalCPU, sum(avgCPU) AS avgCPUTotal by host | addinfo | eval overThisManyMinutes = round((info_max_time-info_min_time)/60) | eval CPUPercUsed = round(avgCPUTotal/overThisManyMinutes) | fields - info* overThisManyMinutes, totalCPU, avgCPUTotal</query>
|
|
<earliest>$CPUtimetoken.earliest$</earliest>
|
|
<latest>$CPUtimetoken.latest$</latest>
|
|
<sampleRatio>1</sampleRatio>
|
|
</search>
|
|
<option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
|
|
<option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
|
|
<option name="charting.axisTitleX.visibility">visible</option>
|
|
<option name="charting.axisTitleY.visibility">visible</option>
|
|
<option name="charting.axisTitleY2.visibility">visible</option>
|
|
<option name="charting.axisX.scale">linear</option>
|
|
<option name="charting.axisY.scale">linear</option>
|
|
<option name="charting.axisY2.enabled">0</option>
|
|
<option name="charting.axisY2.scale">inherit</option>
|
|
<option name="charting.chart">pie</option>
|
|
<option name="charting.chart.bubbleMaximumSize">50</option>
|
|
<option name="charting.chart.bubbleMinimumSize">10</option>
|
|
<option name="charting.chart.bubbleSizeBy">area</option>
|
|
<option name="charting.chart.nullValueMode">gaps</option>
|
|
<option name="charting.chart.showDataLabels">none</option>
|
|
<option name="charting.chart.sliceCollapsingThreshold">0.01</option>
|
|
<option name="charting.chart.stackMode">default</option>
|
|
<option name="charting.chart.style">shiny</option>
|
|
<option name="charting.drilldown">all</option>
|
|
<option name="charting.layout.splitSeries">0</option>
|
|
<option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
|
|
<option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
|
|
<option name="charting.legend.placement">right</option>
|
|
</chart>
|
|
</panel>
|
|
</row>
|
|
<row>
|
|
<panel>
|
|
<title>TotalCPU By Indexer And Application</title>
|
|
<chart>
|
|
<title>This is not % CPU, a rough guide only</title>
|
|
<search>
|
|
<query>index=_introspection `indexerhosts` sourcetype=splunk_resource_usage data.search_props.sid::* | eval app = 'data.search_props.app' | eval cpuperc = 'data.pct_cpu' | chart sum(cpuperc) AS totalCPU by host, app</query>
|
|
<earliest>$time_tok.earliest$</earliest>
|
|
<latest>$time_tok.latest$</latest>
|
|
<sampleRatio>1</sampleRatio>
|
|
</search>
|
|
<option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
|
|
<option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
|
|
<option name="charting.axisTitleX.visibility">visible</option>
|
|
<option name="charting.axisTitleY.visibility">visible</option>
|
|
<option name="charting.axisTitleY2.visibility">visible</option>
|
|
<option name="charting.axisX.scale">linear</option>
|
|
<option name="charting.axisY.scale">linear</option>
|
|
<option name="charting.axisY2.enabled">0</option>
|
|
<option name="charting.axisY2.scale">inherit</option>
|
|
<option name="charting.chart">area</option>
|
|
<option name="charting.chart.bubbleMaximumSize">50</option>
|
|
<option name="charting.chart.bubbleMinimumSize">10</option>
|
|
<option name="charting.chart.bubbleSizeBy">area</option>
|
|
<option name="charting.chart.nullValueMode">gaps</option>
|
|
<option name="charting.chart.showDataLabels">none</option>
|
|
<option name="charting.chart.sliceCollapsingThreshold">0.01</option>
|
|
<option name="charting.chart.stackMode">stacked</option>
|
|
<option name="charting.chart.style">shiny</option>
|
|
<option name="charting.drilldown">all</option>
|
|
<option name="charting.layout.splitSeries">0</option>
|
|
<option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
|
|
<option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
|
|
<option name="charting.legend.placement">right</option>
|
|
</chart>
|
|
</panel>
|
|
</row>
|
|
<row>
|
|
<panel>
|
|
<title>Search count by app, indexer</title>
|
|
<chart>
|
|
<search>
|
|
<query>index=_introspection `indexerhosts` sourcetype=splunk_resource_usage data.search_props.sid::* | eval app = 'data.search_props.app'
|
|
| chart count by app, host</query>
|
|
<earliest>$time_tok.earliest$</earliest>
|
|
<latest>$time_tok.latest$</latest>
|
|
</search>
|
|
<option name="charting.chart">line</option>
|
|
</chart>
|
|
</panel>
|
|
</row>
|
|
<row>
|
|
<panel>
|
|
<title>Usage by non system users - per $interval$ block of time</title>
|
|
<table>
|
|
<title>CPU is total measured amount, memory is maximum memory usage by process, 100 is 1 CPU core</title>
|
|
<search>
|
|
<query>index=_introspection `indexerhosts` sourcetype=splunk_resource_usage data.search_props.sid::* "data.search_props.user"!=admin "data.search_props.user"!=splunk-system-user
|
|
| eval mem_used = 'data.mem_used' | eval app = 'data.search_props.app' | eval elapsed = 'data.elapsed' | eval label = 'data.search_props.label'
|
|
| eval type = 'data.search_props.type' | eval mode = 'data.search_props.mode' | eval user = 'data.search_props.user' | eval cpuperc = 'data.pct_cpu'
|
|
| eval search_head = if(isnull('data.search_props.search_head'),"N/A",'data.search_props.search_head') | eval read_mb = 'data.read_mb'
|
|
| eval provenance='data.search_props.provenance' | eval label=coalesce(label, provenance)
|
|
| stats max(elapsed) as runtime max(mem_used) as mem_used earliest(_time) as searchStartTime, sum(cpuperc) AS totalCPU, avg(cpuperc) AS avgCPU, max(read_mb) AS read_mb by type, mode, app, user, label, host, search_head, data.pid
|
|
| bin searchStartTime span=$interval$
|
|
| stats sum(totalCPU) AS totalCPU, sum(mem_used) AS totalMemUsed, sum(runtime) AS totalRuntime, avg(runtime) AS avgRuntime, sum(avgCPU) AS avgCPUAcrossAllIndexers, sum(read_mb) AS totalReadMB by searchStartTime, type, mode, app, user
|
|
| eval totalduration = tostring(totalRuntime, "duration"), averageduration = tostring(avgRuntime, "duration")
|
|
| eval Started = strftime(searchStartTime,"%+")
|
|
| eval avgCPUAcrossAllIndexers = round(avgCPUAcrossAllIndexers)
|
|
| sort - totalCPU, totalMemUsed
|
|
| eval totalCPU=tostring(totalCPU,"commas"), avgCPUAcrossAllIndexers=tostring(avgCPUAcrossAllIndexers,"commas")
|
|
| fields Started, totalMemUsed, user, app, mode, type, averageduration, totalduration, totalCPU, avgCPUAcrossAllIndexers, totalReadMB</query>
|
|
<earliest>$time_tok.earliest$</earliest>
|
|
<latest>$time_tok.latest$</latest>
|
|
<sampleRatio>1</sampleRatio>
|
|
</search>
|
|
<option name="count">20</option>
|
|
<option name="dataOverlayMode">none</option>
|
|
<option name="drilldown">cell</option>
|
|
<option name="percentagesRow">false</option>
|
|
<option name="rowNumbers">false</option>
|
|
<option name="totalsRow">false</option>
|
|
<option name="wrap">true</option>
|
|
<drilldown>
|
|
<eval token="app">if($click.name2$="app", $click.value2$, "*"</eval>
|
|
<eval token="user">if($click.name2$="user", $click.value2$, ""</eval>
|
|
<link target="_blank">/app/SplunkAdmins/troubleshooting_indexer_cpu_drilldown?form.app=$app$&form.user=$user$</link>
|
|
</drilldown>
|
|
</table>
|
|
</panel>
|
|
</row>
|
|
<row>
|
|
<panel>
|
|
<title>Usage by system users per $interval$ block of time</title>
|
|
<table>
|
|
<title>CPU is totalMeasuredAmount, memory is maximum memory usage by process, 100 is 1 CPU core</title>
|
|
<search>
|
|
<query>index=_introspection `indexerhosts` sourcetype=splunk_resource_usage data.search_props.sid::* "data.search_props.user"=admin OR "data.search_props.user"=splunk-system-user
|
|
| eval mem_used = 'data.mem_used' | eval app = 'data.search_props.app' | eval elapsed = 'data.elapsed' | eval label = 'data.search_props.label'
|
|
| eval type = 'data.search_props.type' | eval mode = 'data.search_props.mode' | eval user = 'data.search_props.user' | eval cpuperc = 'data.pct_cpu'
|
|
| eval search_head = if(isnull('data.search_props.search_head'),"N/A",'data.search_props.search_head') | eval read_mb = 'data.read_mb'
|
|
| eval provenance='data.search_props.provenance' | eval label=coalesce(label, provenance)
|
|
| stats max(elapsed) as runtime max(mem_used) as mem_used earliest(_time) as searchStartTime, sum(cpuperc) AS totalCPU, avg(cpuperc) AS avgCPU, max(read_mb) AS read_mb by type, mode, app, user, label, host, search_head, data.pid
|
|
| bin searchStartTime span=$interval$
|
|
| stats sum(totalCPU) AS totalCPU, sum(mem_used) AS totalMemUsed, sum(runtime) AS totalRuntime, avg(runtime) AS avgRuntime, sum(avgCPU) AS avgCPUAcrossAllIndexers, sum(read_mb) AS totalReadMB by searchStartTime, type, mode, app, user
|
|
| eval totalduration = tostring(totalRuntime, "duration"), averageduration = tostring(avgRuntime, "duration")
|
|
| eval Started = strftime(searchStartTime,"%+")
|
|
| eval avgCPUAcrossAllIndexers = round(avgCPUAcrossAllIndexers)
|
|
| sort - totalCPU, totalMemUsed
|
|
| eval totalCPU=tostring(totalCPU,"commas"), avgCPUAcrossAllIndexers=tostring(avgCPUAcrossAllIndexers,"commas")
|
|
| fields Started, totalMemUsed, user, app, mode, type, averageduration, totalduration, totalCPU, avgCPUAcrossAllIndexers, totalReadMB</query>
|
|
<earliest>$time_tok.earliest$</earliest>
|
|
<latest>$time_tok.latest$</latest>
|
|
<sampleRatio>1</sampleRatio>
|
|
</search>
|
|
<option name="count">20</option>
|
|
<option name="dataOverlayMode">none</option>
|
|
<option name="drilldown">cell</option>
|
|
<option name="percentagesRow">false</option>
|
|
<option name="rowNumbers">false</option>
|
|
<option name="totalsRow">false</option>
|
|
<option name="wrap">true</option>
|
|
<drilldown>
|
|
<eval token="app">if($click.name2$="app", $click.value2$, "*"</eval>
|
|
<eval token="user">if($click.name2$="user", $click.value2$, ""</eval>
|
|
<link target="_blank">/app/SplunkAdmins/troubleshooting_indexer_cpu_drilldown?form.app=$app$&form.user=$user$</link>
|
|
</drilldown>
|
|
</table>
|
|
</panel>
|
|
</row>
|
|
<row>
|
|
<panel>
|
|
<title>CPU used per indexer per search label, CPU measured at point in time</title>
|
|
<input type="dropdown" token="labelExclusion" searchWhenChanged="false">
|
|
<label>Exclude</label>
|
|
<choice value="_ACCELERATE*">_ACCELERATE*</choice>
|
|
<choice value="__DONTEXCLUDE__">No Exclusion</choice>
|
|
<default>__DONTEXCLUDE__</default>
|
|
</input>
|
|
<input type="dropdown" token="sort">
|
|
<label>Sort By</label>
|
|
<choice value="totalAVGCPU, totalMemUsed">avgCPU, memory</choice>
|
|
<choice value="totalCPU, totalMemUsed">totalCPU, memory</choice>
|
|
<choice value="totalRuntime, totalCPU">duration, totalCPU</choice>
|
|
<choice value="totalRuntime, totalAVGCPU">duration, avgCPU</choice>
|
|
<default>totalAVGCPU, totalMemUsed</default>
|
|
<initialValue>totalAVGCPU, totalMemUsed</initialValue>
|
|
</input>
|
|
<table>
|
|
<title>CPU is approx CPU% at any point in time, memory is maximum memory usage by process, 100 is 1 CPU core</title>
|
|
<search>
|
|
<query>index=_introspection `indexerhosts` sourcetype=splunk_resource_usage data.search_props.sid::* NOT ("data.search_props.label"=$labelExclusion$)
|
|
| eval mem_used = 'data.mem_used' | eval app = 'data.search_props.app' | eval elapsed = 'data.elapsed' | eval label = 'data.search_props.label'
|
|
| eval type = 'data.search_props.type' | eval mode = 'data.search_props.mode' | eval user = 'data.search_props.user' | eval cpuperc = 'data.pct_cpu'
|
|
| eval read_mb = 'data.read_mb'
|
|
| eval provenance='data.search_props.provenance'
|
|
| eval label=coalesce(label, provenance)
|
|
| eval search_head = if(isnull('data.search_props.search_head'),"N/A",'data.search_props.search_head')
|
|
| stats max(elapsed) as runtime max(mem_used) as mem_used earliest(_time) as Started, sum(cpuperc) AS totalCPU, max(read_mb) AS read_mb, avg(cpuperc) AS avgCPU by type, mode, app, user, label, host, data.pid
|
|
| stats sum(avgCPU) AS totalAVGCPU, sum(mem_used) AS totalMemUsed, sum(runtime) AS totalRuntime, sum(read_mb) AS totalReadMB, sum(totalCPU) AS totalCPU by Started, type, "mode", app, user, label, host
|
|
| eval totalMemUsed = round(totalMemUsed, 2)
|
|
| eval Started=strftime(Started,"%+")
|
|
| eval duration = tostring(totalRuntime, "duration")
|
|
| eval avgCPU = round(totalAVGCPU)
|
|
| sort - $sort$
|
|
| eval totalCPU=tostring(totalCPU,"commas"), avgCPU=tostring(avgCPU,"commas")
|
|
| fields - totalRuntime, totalAVGCPU</query>
|
|
<earliest>$time_tok.earliest$</earliest>
|
|
<latest>$time_tok.latest$</latest>
|
|
<sampleRatio>1</sampleRatio>
|
|
</search>
|
|
<option name="count">20</option>
|
|
<option name="dataOverlayMode">none</option>
|
|
<option name="drilldown">cell</option>
|
|
<option name="percentagesRow">false</option>
|
|
<option name="rowNumbers">false</option>
|
|
<option name="totalsRow">false</option>
|
|
<option name="wrap">true</option>
|
|
<drilldown>
|
|
<eval token="app">if($click.name2$="app", $click.value2$, "*"</eval>
|
|
<eval token="user">if($click.name2$="user", $click.value2$, ""</eval>
|
|
<link target="_blank">/app/SplunkAdmins/troubleshooting_indexer_cpu_drilldown?form.app=$app$&form.user=$user$</link>
|
|
</drilldown>
|
|
</table>
|
|
</panel>
|
|
</row>
|
|
<row>
|
|
<panel>
|
|
<title>Most Expensive Non System Queries with CPU measured at point in time</title>
|
|
<input type="dropdown" token="sort2">
|
|
<label>Sort By</label>
|
|
<choice value="totalAVGCPU, totalMemUsed">avgCPU, memory</choice>
|
|
<choice value="totalCPU, totalMemUsed">totalCPU, memory</choice>
|
|
<choice value="totalRuntime, totalCPU">duration, totalCPU</choice>
|
|
<choice value="totalRuntime, totalAVGCPU">duration, avgCPU</choice>
|
|
<default>totalAVGCPU, totalMemUsed</default>
|
|
<initialValue>totalAVGCPU, totalMemUsed</initialValue>
|
|
</input>
|
|
<table>
|
|
<title>CPU is approx CPU% at any point in time, memory is maximum memory usage by process, 100 is 1 CPU core</title>
|
|
<search>
|
|
<query>index=_introspection `indexerhosts` sourcetype=splunk_resource_usage data.search_props.sid::* "data.search_props.user"!=admin "data.search_props.user"!=splunk-system-user
|
|
| eval mem_used = 'data.mem_used' | eval app = 'data.search_props.app' | eval elapsed = 'data.elapsed' | eval label = 'data.search_props.label'
|
|
| eval type = 'data.search_props.type' | eval mode = 'data.search_props.mode' | eval user = 'data.search_props.user' | eval cpuperc = 'data.pct_cpu'
|
|
| eval read_mb = 'data.read_mb'
|
|
| eval provenance='data.search_props.provenance'
|
|
| eval label=coalesce(label, provenance)
|
|
| eval search_head = if(isnull('data.search_props.search_head'),"N/A",'data.search_props.search_head')
|
|
| stats max(elapsed) as runtime max(mem_used) as mem_used earliest(_time) as Started, sum(cpuperc) AS totalCPU, max(read_mb) AS read_mb, avg(cpuperc) AS avgCPU by type, mode, app, user, label, host, data.pid
|
|
| stats sum(avgCPU) AS totalAVGCPU, sum(mem_used) AS totalMemUsed, sum(runtime) AS totalRuntime, sum(read_mb) AS totalReadMB, sum(totalCPU) AS totalCPU by Started, type, "mode", app, user, label, host
|
|
| eval totalMemUsed = round(totalMemUsed, 2)
|
|
| eval Started=strftime(Started,"%+")
|
|
| eval duration = tostring(totalRuntime, "duration")
|
|
| eval avgCPU = round(totalAVGCPU)
|
|
| sort - $sort2$
|
|
| eval totalCPU=tostring(totalCPU,"commas"), avgCPU=tostring(avgCPU,"commas")
|
|
| fields - totalRuntime, totalAVGCPU</query>
|
|
<earliest>$time_tok.earliest$</earliest>
|
|
<latest>$time_tok.latest$</latest>
|
|
<sampleRatio>1</sampleRatio>
|
|
</search>
|
|
<option name="count">20</option>
|
|
<option name="dataOverlayMode">none</option>
|
|
<option name="drilldown">cell</option>
|
|
<option name="percentagesRow">false</option>
|
|
<option name="rowNumbers">false</option>
|
|
<option name="totalsRow">false</option>
|
|
<option name="wrap">true</option>
|
|
<drilldown>
|
|
<eval token="app">if($click.name2$="app", $click.value2$, "*"</eval>
|
|
<eval token="user">if($click.name2$="user", $click.value2$, ""</eval>
|
|
<link target="_blank">/app/SplunkAdmins/troubleshooting_indexer_cpu_drilldown?form.app=$app$&form.user=$user$</link>
|
|
</drilldown>
|
|
</table>
|
|
</panel>
|
|
</row>
|
|
<row>
|
|
<panel>
|
|
<title>CPU used on a per SID basis</title>
|
|
<input type="dropdown" token="sort3">
|
|
<label>Sort By</label>
|
|
<choice value="totalAVGCPUPerMinute, totalMemUsed">avgCPU, memory</choice>
|
|
<choice value="totalCPU, totalMemUsed">totalCPU, memory</choice>
|
|
<choice value="totalRuntime, totalCPU">duration, totalCPU</choice>
|
|
<choice value="totalRuntime, totalAVGCPUPerMinute">duration, avgCPU</choice>
|
|
<initialValue>totalRuntime, totalCPU</initialValue>
|
|
</input>
|
|
<input type="dropdown" token="labelExclusion2">
|
|
<label>Exclude</label>
|
|
<choice value="_ACCELERATE*">_ACCELERATE*</choice>
|
|
<choice value="__DONTEXCLUDE__">No Exclusion</choice>
|
|
<default>__DONTEXCLUDE__</default>
|
|
</input>
|
|
<table>
|
|
<search>
|
|
<query>index=_introspection `indexerhosts` sourcetype=splunk_resource_usage data.search_props.sid::* NOT ("data.search_props.label"=$labelExclusion2$)
|
|
| eval mem_used = 'data.mem_used' | eval app = 'data.search_props.app' | eval elapsed = 'data.elapsed' | eval label = 'data.search_props.label'
|
|
| eval type = 'data.search_props.type' | eval mode = 'data.search_props.mode' | eval user = 'data.search_props.user' | eval cpuperc = 'data.pct_cpu'
|
|
| eval read_mb = 'data.read_mb'
|
|
| eval sid='data.search_props.sid'
|
|
| eval provenance='data.search_props.provenance'
|
|
| eval label=coalesce(label, provenance)
|
|
| eval search_head = if(isnull('data.search_props.search_head'),"N/A",'data.search_props.search_head')
|
|
| stats max(elapsed) as runtime max(mem_used) as mem_used earliest(_time) as Started, sum(cpuperc) AS totalCPU, max(read_mb) AS read_mb, avg(cpuperc) AS avgCPUPerMinute by type, mode, app, user, label, host, data.pid, sid
|
|
| stats sum(avgCPUPerMinute) AS totalAVGCPUPerMinute, sum(mem_used) AS totalMemUsed, sum(runtime) AS totalRuntime, sum(read_mb) AS totalReadMB, sum(totalCPU) AS totalCPU by Started, type, "mode", app, user, label, host, sid, data.pid
|
|
| eval totalMemUsed = round(totalMemUsed, 2)
|
|
| eval Started=strftime(Started,"%+")
|
|
| eval duration = tostring(totalRuntime, "duration")
|
|
| eval avgCPU = round(totalAVGCPUPerMinute)
|
|
| sort - $sort3$
|
|
| eval totalCPU=tostring(totalCPU,"commas"), avgCPU=tostring(avgCPU,"commas")
|
|
| fields - totalRuntime, totalAVGCPUPerMinute, sid</query>
|
|
<earliest>$time_tok.earliest$</earliest>
|
|
<latest>$time_tok.latest$</latest>
|
|
<sampleRatio>1</sampleRatio>
|
|
</search>
|
|
<option name="count">20</option>
|
|
<option name="dataOverlayMode">none</option>
|
|
<option name="drilldown">cell</option>
|
|
<option name="percentagesRow">false</option>
|
|
<option name="rowNumbers">false</option>
|
|
<option name="totalsRow">false</option>
|
|
<option name="wrap">true</option>
|
|
<drilldown>
|
|
<eval token="app">if($click.name2$="app", $click.value2$, "*"</eval>
|
|
<eval token="user">if($click.name2$="user", $click.value2$, ""</eval>
|
|
<link target="_blank">/app/SplunkAdmins/troubleshooting_indexer_cpu_drilldown?form.app=$app$&form.user=$user$</link>
|
|
</drilldown>
|
|
</table>
|
|
</panel>
|
|
</row>
|
|
</form>
|