Dashboard - Troubleshooting Indexer CPU

General Time Picker -4h@h @h User 10m 30m 60m 120m 4h 60m CPU Based Time Picker (Pie charts near top) -1h@h @h

index=_introspection `indexerhosts` sourcetype=splunk_resource_usage data.search_props.sid::* | eval app = 'data.search_props.app' | chart count by app $CPUtimetoken.earliest$ $CPUtimetoken.latest$ ellipsisNone 0 visible visible visible linear linear 0 inherit pie 50 10 area gaps none 0.01 default shiny all 0 0 ellipsisMiddle right index=_introspection `indexerhosts` sourcetype=splunk_resource_usage data.search_props.sid::* | eval app = 'data.search_props.app' | eval cpuperc = 'data.pct_cpu' | bin _time span=1m | stats sum(cpuperc) AS totalCPU, avg(cpuperc) AS avgCPU by data.pid, host, _time, app | stats sum(totalCPU) AS totalCPU, sum(avgCPU) AS avgCPU by app | addinfo | eval overThisManyMinutes = round((info_max_time-info_min_time)/60) | eval CPUPercUsed = round(avgCPU/overThisManyMinutes) | fields - totalCPU, info* overThisManyMinutes, avgCPU $CPUtimetoken.earliest$ $CPUtimetoken.latest$ 1 ellipsisNone 0 visible visible visible linear linear 0 inherit pie 50 10 area gaps none 0.01 default shiny all 0 0 ellipsisMiddle right index=_introspection `indexerhosts` sourcetype=splunk_resource_usage data.search_props.sid::* | chart count by host $CPUtimetoken.earliest$ $CPUtimetoken.latest$ 1 ellipsisNone 0 visible visible visible linear linear 0 inherit pie 50 10 area gaps none 0.01 default shiny all 0 0 ellipsisMiddle right index=_introspection `indexerhosts` sourcetype=splunk_resource_usage data.search_props.sid::* | eval cpuperc = 'data.pct_cpu' | bin _time span=1m | stats sum(cpuperc) AS totalCPU, avg(cpuperc) AS avgCPU by data.pid, host, _time| stats sum(totalCPU) AS totalCPU, sum(avgCPU) AS avgCPUTotal by host | addinfo | eval overThisManyMinutes = round((info_max_time-info_min_time)/60) | eval CPUPercUsed = round(avgCPUTotal/overThisManyMinutes) | fields - info* overThisManyMinutes, totalCPU, avgCPUTotal $CPUtimetoken.earliest$ $CPUtimetoken.latest$ 1 ellipsisNone 0 visible visible visible linear linear 0 inherit pie 50 10 area gaps none 0.01 default shiny all 0 0 ellipsisMiddle right index=_introspection `indexerhosts` sourcetype=splunk_resource_usage data.search_props.sid::* | eval app = 'data.search_props.app' | eval cpuperc = 'data.pct_cpu' | chart sum(cpuperc) AS totalCPU by host, app $time_tok.earliest$ $time_tok.latest$ 1 ellipsisNone 0 visible visible visible linear linear 0 inherit area 50 10 area gaps none 0.01 stacked shiny all 0 0 ellipsisMiddle right index=_introspection `indexerhosts` sourcetype=splunk_resource_usage data.search_props.sid::* | eval app = 'data.search_props.app' | chart count by app, host $time_tok.earliest$ $time_tok.latest$ line index=_introspection `indexerhosts` sourcetype=splunk_resource_usage data.search_props.sid::* "data.search_props.user"!=admin "data.search_props.user"!=splunk-system-user | eval mem_used = 'data.mem_used' | eval app = 'data.search_props.app' | eval elapsed = 'data.elapsed' | eval label = 'data.search_props.label' | eval type = 'data.search_props.type' | eval mode = 'data.search_props.mode' | eval user = 'data.search_props.user' | eval cpuperc = 'data.pct_cpu' | eval search_head = if(isnull('data.search_props.search_head'),"N/A",'data.search_props.search_head') | eval read_mb = 'data.read_mb' | eval provenance='data.search_props.provenance' | eval label=coalesce(label, provenance) | stats max(elapsed) as runtime max(mem_used) as mem_used earliest(_time) as searchStartTime, sum(cpuperc) AS totalCPU, avg(cpuperc) AS avgCPU, max(read_mb) AS read_mb by type, mode, app, user, label, host, search_head, data.pid | bin searchStartTime span=$interval$ | stats sum(totalCPU) AS totalCPU, sum(mem_used) AS totalMemUsed, sum(runtime) AS totalRuntime, avg(runtime) AS avgRuntime, sum(avgCPU) AS avgCPUAcrossAllIndexers, sum(read_mb) AS totalReadMB by searchStartTime, type, mode, app, user | eval totalduration = tostring(totalRuntime, "duration"), averageduration = tostring(avgRuntime, "duration") | eval Started = strftime(searchStartTime,"%+") | eval avgCPUAcrossAllIndexers = round(avgCPUAcrossAllIndexers) | sort - totalCPU, totalMemUsed | eval totalCPU=tostring(totalCPU,"commas"), avgCPUAcrossAllIndexers=tostring(avgCPUAcrossAllIndexers,"commas") | fields Started, totalMemUsed, user, app, mode, type, averageduration, totalduration, totalCPU, avgCPUAcrossAllIndexers, totalReadMB $time_tok.earliest$ $time_tok.latest$ 1 20nonecellfalsefalsefalsetrue if($click.name2$="app", $click.value2$, "*" if($click.name2$="user", $click.value2$, "" /app/SplunkAdmins/troubleshooting_indexer_cpu_drilldown?form.app=$app$&form.user=$user$ index=_introspection `indexerhosts` sourcetype=splunk_resource_usage data.search_props.sid::* "data.search_props.user"=admin OR "data.search_props.user"=splunk-system-user | eval mem_used = 'data.mem_used' | eval app = 'data.search_props.app' | eval elapsed = 'data.elapsed' | eval label = 'data.search_props.label' | eval type = 'data.search_props.type' | eval mode = 'data.search_props.mode' | eval user = 'data.search_props.user' | eval cpuperc = 'data.pct_cpu' | eval search_head = if(isnull('data.search_props.search_head'),"N/A",'data.search_props.search_head') | eval read_mb = 'data.read_mb' | eval provenance='data.search_props.provenance' | eval label=coalesce(label, provenance) | stats max(elapsed) as runtime max(mem_used) as mem_used earliest(_time) as searchStartTime, sum(cpuperc) AS totalCPU, avg(cpuperc) AS avgCPU, max(read_mb) AS read_mb by type, mode, app, user, label, host, search_head, data.pid | bin searchStartTime span=$interval$ | stats sum(totalCPU) AS totalCPU, sum(mem_used) AS totalMemUsed, sum(runtime) AS totalRuntime, avg(runtime) AS avgRuntime, sum(avgCPU) AS avgCPUAcrossAllIndexers, sum(read_mb) AS totalReadMB by searchStartTime, type, mode, app, user | eval totalduration = tostring(totalRuntime, "duration"), averageduration = tostring(avgRuntime, "duration") | eval Started = strftime(searchStartTime,"%+") | eval avgCPUAcrossAllIndexers = round(avgCPUAcrossAllIndexers) | sort - totalCPU, totalMemUsed | eval totalCPU=tostring(totalCPU,"commas"), avgCPUAcrossAllIndexers=tostring(avgCPUAcrossAllIndexers,"commas") | fields Started, totalMemUsed, user, app, mode, type, averageduration, totalduration, totalCPU, avgCPUAcrossAllIndexers, totalReadMB $time_tok.earliest$ $time_tok.latest$ 1 20nonecellfalsefalsefalsetrue if($click.name2$="app", $click.value2$, "*" if($click.name2$="user", $click.value2$, "" /app/SplunkAdmins/troubleshooting_indexer_cpu_drilldown?form.app=$app$&form.user=$user$ Exclude _ACCELERATE* No Exclusion __DONTEXCLUDE__ Sort By avgCPU, memory totalCPU, memory duration, totalCPU duration, avgCPU totalAVGCPU, totalMemUsed totalAVGCPU, totalMemUsed index=_introspection `indexerhosts` sourcetype=splunk_resource_usage data.search_props.sid::* NOT ("data.search_props.label"=$labelExclusion$) | eval mem_used = 'data.mem_used' | eval app = 'data.search_props.app' | eval elapsed = 'data.elapsed' | eval label = 'data.search_props.label' | eval type = 'data.search_props.type' | eval mode = 'data.search_props.mode' | eval user = 'data.search_props.user' | eval cpuperc = 'data.pct_cpu' | eval read_mb = 'data.read_mb' | eval provenance='data.search_props.provenance' | eval label=coalesce(label, provenance) | eval search_head = if(isnull('data.search_props.search_head'),"N/A",'data.search_props.search_head') | stats max(elapsed) as runtime max(mem_used) as mem_used earliest(_time) as Started, sum(cpuperc) AS totalCPU, max(read_mb) AS read_mb, avg(cpuperc) AS avgCPU by type, mode, app, user, label, host, data.pid | stats sum(avgCPU) AS totalAVGCPU, sum(mem_used) AS totalMemUsed, sum(runtime) AS totalRuntime, sum(read_mb) AS totalReadMB, sum(totalCPU) AS totalCPU by Started, type, "mode", app, user, label, host | eval totalMemUsed = round(totalMemUsed, 2) | eval Started=strftime(Started,"%+") | eval duration = tostring(totalRuntime, "duration") | eval avgCPU = round(totalAVGCPU) | sort - $sort$ | eval totalCPU=tostring(totalCPU,"commas"), avgCPU=tostring(avgCPU,"commas") | fields - totalRuntime, totalAVGCPU $time_tok.earliest$ $time_tok.latest$ 1 20nonecellfalsefalsefalsetrue if($click.name2$="app", $click.value2$, "*" if($click.name2$="user", $click.value2$, "" /app/SplunkAdmins/troubleshooting_indexer_cpu_drilldown?form.app=$app$&form.user=$user$ Sort By avgCPU, memory totalCPU, memory duration, totalCPU duration, avgCPU totalAVGCPU, totalMemUsed totalAVGCPU, totalMemUsed index=_introspection `indexerhosts` sourcetype=splunk_resource_usage data.search_props.sid::* "data.search_props.user"!=admin "data.search_props.user"!=splunk-system-user | eval mem_used = 'data.mem_used' | eval app = 'data.search_props.app' | eval elapsed = 'data.elapsed' | eval label = 'data.search_props.label' | eval type = 'data.search_props.type' | eval mode = 'data.search_props.mode' | eval user = 'data.search_props.user' | eval cpuperc = 'data.pct_cpu' | eval read_mb = 'data.read_mb' | eval provenance='data.search_props.provenance' | eval label=coalesce(label, provenance) | eval search_head = if(isnull('data.search_props.search_head'),"N/A",'data.search_props.search_head') | stats max(elapsed) as runtime max(mem_used) as mem_used earliest(_time) as Started, sum(cpuperc) AS totalCPU, max(read_mb) AS read_mb, avg(cpuperc) AS avgCPU by type, mode, app, user, label, host, data.pid | stats sum(avgCPU) AS totalAVGCPU, sum(mem_used) AS totalMemUsed, sum(runtime) AS totalRuntime, sum(read_mb) AS totalReadMB, sum(totalCPU) AS totalCPU by Started, type, "mode", app, user, label, host | eval totalMemUsed = round(totalMemUsed, 2) | eval Started=strftime(Started,"%+") | eval duration = tostring(totalRuntime, "duration") | eval avgCPU = round(totalAVGCPU) | sort - $sort2$ | eval totalCPU=tostring(totalCPU,"commas"), avgCPU=tostring(avgCPU,"commas") | fields - totalRuntime, totalAVGCPU $time_tok.earliest$ $time_tok.latest$ 1 20nonecellfalsefalsefalsetrue if($click.name2$="app", $click.value2$, "*" if($click.name2$="user", $click.value2$, "" /app/SplunkAdmins/troubleshooting_indexer_cpu_drilldown?form.app=$app$&form.user=$user$ Sort By avgCPU, memory totalCPU, memory duration, totalCPU duration, avgCPU totalRuntime, totalCPU Exclude _ACCELERATE* No Exclusion __DONTEXCLUDE__ index=_introspection `indexerhosts` sourcetype=splunk_resource_usage data.search_props.sid::* NOT ("data.search_props.label"=$labelExclusion2$) | eval mem_used = 'data.mem_used' | eval app = 'data.search_props.app' | eval elapsed = 'data.elapsed' | eval label = 'data.search_props.label' | eval type = 'data.search_props.type' | eval mode = 'data.search_props.mode' | eval user = 'data.search_props.user' | eval cpuperc = 'data.pct_cpu' | eval read_mb = 'data.read_mb' | eval sid='data.search_props.sid' | eval provenance='data.search_props.provenance' | eval label=coalesce(label, provenance) | eval search_head = if(isnull('data.search_props.search_head'),"N/A",'data.search_props.search_head') | stats max(elapsed) as runtime max(mem_used) as mem_used earliest(_time) as Started, sum(cpuperc) AS totalCPU, max(read_mb) AS read_mb, avg(cpuperc) AS avgCPUPerMinute by type, mode, app, user, label, host, data.pid, sid | stats sum(avgCPUPerMinute) AS totalAVGCPUPerMinute, sum(mem_used) AS totalMemUsed, sum(runtime) AS totalRuntime, sum(read_mb) AS totalReadMB, sum(totalCPU) AS totalCPU by Started, type, "mode", app, user, label, host, sid, data.pid | eval totalMemUsed = round(totalMemUsed, 2) | eval Started=strftime(Started,"%+") | eval duration = tostring(totalRuntime, "duration") | eval avgCPU = round(totalAVGCPUPerMinute) | sort - $sort3$ | eval totalCPU=tostring(totalCPU,"commas"), avgCPU=tostring(avgCPU,"commas") | fields - totalRuntime, totalAVGCPUPerMinute, sid $time_tok.earliest$ $time_tok.latest$ 1 20nonecellfalsefalsefalsetrue if($click.name2$="app", $click.value2$, "*" if($click.name2$="user", $click.value2$, "" /app/SplunkAdmins/troubleshooting_indexer_cpu_drilldown?form.app=$app$&form.user=$user$