You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
218 lines
11 KiB
218 lines
11 KiB
<form version="1.1">
|
|
<label>Dashboard - HEC Performance</label>
|
|
<description>Based on the original version from https://github.com/camrunr/hec_perf_report/blob/master/hec_perf_report.xml</description>
|
|
<search id="by_token">
|
|
<query>index=_introspection (`indexerhosts`) OR (`heavyforwarderhosts`) `splunkadmins_hec_metrics_source` http_event_collector_token
|
|
| bucket _time span=$dd_span$
|
|
| stats sum(data.num_of_events) as Events sum(data.total_bytes_received) as Bytes by _time data.token_name</query>
|
|
<earliest>$timepicker.earliest$</earliest>
|
|
<latest>$timepicker.latest$</latest>
|
|
<sampleRatio>1</sampleRatio>
|
|
<refresh>$refreshinterval$</refresh>
|
|
</search>
|
|
<search id="by_host">
|
|
<query>index=_introspection (`indexerhosts`) OR (`heavyforwarderhosts`) `splunkadmins_hec_metrics_source` http_event_collector_token
|
|
| bucket _time span=$dd_span$
|
|
| stats sum(data.num_of_events) as Events sum(data.total_bytes_received) as Bytes by _time host
|
|
| eval host=replace(host,"\..*","")</query>
|
|
<earliest>$timepicker.earliest$</earliest>
|
|
<latest>$timepicker.latest$</latest>
|
|
<sampleRatio>1</sampleRatio>
|
|
<refresh>$refreshinterval$</refresh>
|
|
</search>
|
|
<fieldset submitButton="false">
|
|
<input type="time" token="timepicker">
|
|
<label>Time</label>
|
|
<default>
|
|
<earliest>-4h@m</earliest>
|
|
<latest>now</latest>
|
|
</default>
|
|
</input>
|
|
<input type="dropdown" token="dd_span">
|
|
<label>span</label>
|
|
<choice value="1min">1 minute</choice>
|
|
<choice value="5min">5 minutes</choice>
|
|
<choice value="30m">30 minutes</choice>
|
|
<choice value="1h">1 hour</choice>
|
|
<choice value="1d">1 day</choice>
|
|
<default>1min</default>
|
|
</input>
|
|
<input type="text" token="hostcount">
|
|
<label>Timechart host limit</label>
|
|
<default>15</default>
|
|
</input>
|
|
<input type="text" token="refreshinterval" searchWhenChanged="true">
|
|
<label>Refresh Interval</label>
|
|
<default>300</default>
|
|
</input>
|
|
</fieldset>
|
|
<row>
|
|
<panel>
|
|
<title>Events/sec by host</title>
|
|
<chart>
|
|
<search base="by_host">
|
|
<query>timechart limit=$hostcount$ span=$dd_span$ per_second(Events) as Events/sec by host</query>
|
|
</search>
|
|
<option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
|
|
<option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
|
|
<option name="charting.axisTitleX.visibility">collapsed</option>
|
|
<option name="charting.axisTitleY.visibility">visible</option>
|
|
<option name="charting.axisTitleY2.visibility">visible</option>
|
|
<option name="charting.axisX.abbreviation">none</option>
|
|
<option name="charting.axisX.scale">linear</option>
|
|
<option name="charting.axisY.abbreviation">auto</option>
|
|
<option name="charting.axisY.scale">linear</option>
|
|
<option name="charting.axisY2.abbreviation">none</option>
|
|
<option name="charting.axisY2.enabled">0</option>
|
|
<option name="charting.axisY2.scale">inherit</option>
|
|
<option name="charting.chart">column</option>
|
|
<option name="charting.chart.bubbleMaximumSize">50</option>
|
|
<option name="charting.chart.bubbleMinimumSize">10</option>
|
|
<option name="charting.chart.bubbleSizeBy">area</option>
|
|
<option name="charting.chart.nullValueMode">gaps</option>
|
|
<option name="charting.chart.showDataLabels">none</option>
|
|
<option name="charting.chart.sliceCollapsingThreshold">0.01</option>
|
|
<option name="charting.chart.stackMode">stacked</option>
|
|
<option name="charting.chart.style">shiny</option>
|
|
<option name="charting.drilldown">none</option>
|
|
<option name="charting.layout.splitSeries">0</option>
|
|
<option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
|
|
<option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
|
|
<option name="charting.legend.mode">standard</option>
|
|
<option name="charting.legend.placement">right</option>
|
|
<option name="charting.lineWidth">2</option>
|
|
<option name="refresh.display">progressbar</option>
|
|
<option name="trellis.enabled">0</option>
|
|
<option name="trellis.scales.shared">1</option>
|
|
<option name="trellis.size">medium</option>
|
|
</chart>
|
|
</panel>
|
|
<panel>
|
|
<title>Bytes/sec by host</title>
|
|
<chart>
|
|
<search base="by_host">
|
|
<query>timechart limit=$hostcount$ span=$dd_span$ per_second(Bytes) as Bytes/sec by host</query>
|
|
</search>
|
|
<option name="charting.axisTitleX.visibility">collapsed</option>
|
|
<option name="charting.axisY.abbreviation">auto</option>
|
|
<option name="charting.chart">column</option>
|
|
<option name="charting.chart.stackMode">stacked</option>
|
|
<option name="charting.drilldown">none</option>
|
|
<option name="refresh.display">progressbar</option>
|
|
</chart>
|
|
</panel>
|
|
</row>
|
|
<row>
|
|
<panel>
|
|
<title>Events/sec by input/group</title>
|
|
<chart>
|
|
<search base="by_token">
|
|
<query>timechart span=$dd_span$ per_second(Events) as Events/sec by data.token_name</query>
|
|
</search>
|
|
<option name="charting.axisLabelsX.majorLabelStyle.overflowMode">ellipsisNone</option>
|
|
<option name="charting.axisLabelsX.majorLabelStyle.rotation">0</option>
|
|
<option name="charting.axisTitleX.visibility">collapsed</option>
|
|
<option name="charting.axisTitleY.visibility">visible</option>
|
|
<option name="charting.axisTitleY2.visibility">visible</option>
|
|
<option name="charting.axisX.abbreviation">none</option>
|
|
<option name="charting.axisX.scale">linear</option>
|
|
<option name="charting.axisY.abbreviation">auto</option>
|
|
<option name="charting.axisY.scale">linear</option>
|
|
<option name="charting.axisY2.abbreviation">none</option>
|
|
<option name="charting.axisY2.enabled">0</option>
|
|
<option name="charting.axisY2.scale">inherit</option>
|
|
<option name="charting.chart">column</option>
|
|
<option name="charting.chart.bubbleMaximumSize">50</option>
|
|
<option name="charting.chart.bubbleMinimumSize">10</option>
|
|
<option name="charting.chart.bubbleSizeBy">area</option>
|
|
<option name="charting.chart.nullValueMode">gaps</option>
|
|
<option name="charting.chart.showDataLabels">none</option>
|
|
<option name="charting.chart.sliceCollapsingThreshold">0.01</option>
|
|
<option name="charting.chart.stackMode">stacked</option>
|
|
<option name="charting.chart.style">shiny</option>
|
|
<option name="charting.drilldown">none</option>
|
|
<option name="charting.layout.splitSeries">0</option>
|
|
<option name="charting.layout.splitSeries.allowIndependentYRanges">0</option>
|
|
<option name="charting.legend.labelStyle.overflowMode">ellipsisMiddle</option>
|
|
<option name="charting.legend.mode">standard</option>
|
|
<option name="charting.legend.placement">right</option>
|
|
<option name="charting.lineWidth">2</option>
|
|
<option name="refresh.display">progressbar</option>
|
|
<option name="trellis.enabled">0</option>
|
|
<option name="trellis.scales.shared">1</option>
|
|
<option name="trellis.size">medium</option>
|
|
</chart>
|
|
</panel>
|
|
<panel>
|
|
<title>Bytes/sec by input/group</title>
|
|
<chart>
|
|
<search base="by_token">
|
|
<query>timechart span=$dd_span$ per_second(Bytes) as Bytes/sec by data.token_name</query>
|
|
</search>
|
|
<option name="charting.axisTitleX.visibility">collapsed</option>
|
|
<option name="charting.axisY.abbreviation">auto</option>
|
|
<option name="charting.chart">column</option>
|
|
<option name="charting.chart.stackMode">stacked</option>
|
|
<option name="charting.drilldown">none</option>
|
|
<option name="refresh.display">progressbar</option>
|
|
</chart>
|
|
</panel>
|
|
</row>
|
|
<row>
|
|
<panel>
|
|
<title>HEC Batching Efficiency</title>
|
|
<table>
|
|
<search>
|
|
<refresh>$refreshinterval$</refresh>
|
|
<query>index=_introspection (`indexerhosts`) OR (`heavyforwarderhosts`) `splunkadmins_hec_metrics_source` http_event_collector_token
|
|
| eval EpR='data.num_of_events'/'data.num_of_requests'
|
|
| bucket _time span=5m
|
|
| stats sum(data.num_of_events) as events avg(EpR) as events_per_POST sum(data.num_of_requests) as reqs sum(data.total_bytes_received) as Bytes by _time data.token_name
|
|
| eval reqs_per_sec=reqs/300, bytes_per_post=Bytes/reqs
|
|
| rename data.token_name as token_name
|
|
| stats sum(eval(Bytes/1024/1024)) as MBytes sum(events) as Events p50(events_per_POST) as events_per_post p50(bytes_per_post) as bytes_per_post p90(reqs_per_sec) as posts_per_sec by token_name
|
|
| eval MBytes = round(MBytes, 2), events_per_post=round(events_per_post,2), bytes_per_post=round(bytes_per_post,2), posts_per_sec=round(posts_per_sec,2)
|
|
| sort - posts_per_sec</query>
|
|
<earliest>$timepicker.earliest$</earliest>
|
|
<latest>$timepicker.latest$</latest>
|
|
</search>
|
|
<option name="count">10</option>
|
|
<option name="drilldown">none</option>
|
|
<option name="refresh.display">progressbar</option>
|
|
<format type="color" field="events_per_POST">
|
|
<colorPalette type="list">[#DC4E41,#DC4E41,#F8BE34,#53A051]</colorPalette>
|
|
<scale type="threshold">0,5,10</scale>
|
|
</format>
|
|
<format type="color" field="reqs_per_sec">
|
|
<colorPalette type="list">[#53A051,#F8BE34,#DC4E41]</colorPalette>
|
|
<scale type="threshold">10,50</scale>
|
|
</format>
|
|
<format type="number" field="MBytes"></format>
|
|
<format type="number" field="events_per_post"></format>
|
|
<format type="number" field="bytes_per_post"></format>
|
|
<format type="number" field="posts_per_sec"></format>
|
|
<format type="number" field="Events">
|
|
<option name="precision">0</option>
|
|
</format>
|
|
</table>
|
|
</panel>
|
|
</row>
|
|
<row>
|
|
<panel>
|
|
<title>If useACK is in use num_of_requests_waiting_ack is high then this can be an issue (HEC tokens with useACK will stop allowing data through)</title>
|
|
<chart>
|
|
<search>
|
|
<refresh>$refreshinterval$</refresh>
|
|
<query>index=_introspection (`indexerhosts`) OR (`heavyforwarderhosts`) data.series=http_event_collector data.num_of_requests_waiting_ack=* sourcetype=http_event_collector_metrics
|
|
| timechart minspan=2m max(data.num_of_requests_waiting_ack) AS num_of_requests_waiting_ack</query>
|
|
<earliest>$timepicker.earliest$</earliest>
|
|
<latest>$timepicker.latest$</latest>
|
|
</search>
|
|
<option name="charting.chart">line</option>
|
|
<option name="charting.drilldown">none</option>
|
|
<option name="refresh.display">progressbar</option>
|
|
</chart>
|
|
</panel>
|
|
</row>
|
|
</form>
|