You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
936 lines
76 KiB
936 lines
76 KiB
{
|
|
"algorithms": {
|
|
"GradientBoostingRegressor": {
|
|
"RMSE": 0,
|
|
"modelId": "",
|
|
"rSquared": 0,
|
|
"recommended": false
|
|
},
|
|
"LinearRegression": {
|
|
"RMSE": 0,
|
|
"modelId": "",
|
|
"rSquared": 0,
|
|
"recommended": false
|
|
},
|
|
"LogisticRegression": {
|
|
"accuracy": 0,
|
|
"f1_score": 0,
|
|
"modelId": "",
|
|
"precision": 0,
|
|
"recall": 0,
|
|
"recommended": false
|
|
},
|
|
"RandomForestRegressor": {
|
|
"RMSE": 0,
|
|
"modelId": "",
|
|
"rSquared": 0,
|
|
"recommended": false
|
|
}
|
|
},
|
|
"description": "Service for monitoring Kubernetes Pods",
|
|
"enabled": true,
|
|
"entity_rules": [],
|
|
"key": "da-itsi-cp-splunk-observability-kubernetes-pods",
|
|
"kpis": [
|
|
{
|
|
"adaptive_thresholding_training_window": "-7d",
|
|
"adaptive_thresholds_is_enabled": false,
|
|
"aggregate_eval": "",
|
|
"aggregate_statop": "avg",
|
|
"aggregate_threshold_alert_enabled": false,
|
|
"aggregate_thresholds": {
|
|
"base_severity_color": "#99D18B",
|
|
"base_severity_color_light": "#DCEFD7",
|
|
"base_severity_label": "normal",
|
|
"base_severity_value": 2.0,
|
|
"gauge_max": 100,
|
|
"gauge_min": 0,
|
|
"is_max_static": false,
|
|
"is_min_static": true,
|
|
"metric_field": "count",
|
|
"render_boundary_max": 100.0,
|
|
"render_boundary_min": 0.0,
|
|
"threshold_levels": [
|
|
{
|
|
"dynamic_param": null,
|
|
"severity_color": "#B50101",
|
|
"severity_color_light": "#E5A6A6",
|
|
"severity_label": "critical",
|
|
"severity_label_localized": null,
|
|
"severity_value": 6.0,
|
|
"threshold_value": 0.0
|
|
},
|
|
{
|
|
"dynamic_param": null,
|
|
"severity_color": "#F26A35",
|
|
"severity_color_light": "#FBCBB9",
|
|
"severity_label": "high",
|
|
"severity_label_localized": null,
|
|
"severity_value": 5.0,
|
|
"threshold_value": 20.0
|
|
},
|
|
{
|
|
"dynamic_param": null,
|
|
"severity_color": "#FCB64E",
|
|
"severity_color_light": "#FEE6C1",
|
|
"severity_label": "medium",
|
|
"severity_label_localized": null,
|
|
"severity_value": 4.0,
|
|
"threshold_value": 40.0
|
|
},
|
|
{
|
|
"dynamic_param": null,
|
|
"severity_color": "#FFE98C",
|
|
"severity_color_light": "#FFF4C5",
|
|
"severity_label": "low",
|
|
"severity_label_localized": null,
|
|
"severity_value": 3.0,
|
|
"threshold_value": 60.0
|
|
},
|
|
{
|
|
"dynamic_param": null,
|
|
"severity_color": "#99D18B",
|
|
"severity_color_light": "#DCEFD7",
|
|
"severity_label": "normal",
|
|
"severity_label_localized": null,
|
|
"severity_value": 2.0,
|
|
"threshold_value": 80.0
|
|
}
|
|
]
|
|
},
|
|
"aggregate_thresholds_alert_enabled": false,
|
|
"aggregate_thresholds_custom_alert_enabled": false,
|
|
"aggregate_thresholds_custom_alert_rules": [],
|
|
"alert_eval": "",
|
|
"alert_lag": "30",
|
|
"alert_on": "both",
|
|
"alert_period": "1",
|
|
"anomaly_detection_alerting_enabled": false,
|
|
"anomaly_detection_is_enabled": false,
|
|
"anomaly_detection_sensitivity": 0.999,
|
|
"anomaly_detection_training_window": "-7d",
|
|
"backfill_earliest_time": "-7d",
|
|
"backfill_enabled": false,
|
|
"base_search": "`get_full_itsi_summary_service_health_events(da-itsi-cp-splunk-observability-kubernetes-pods)`",
|
|
"base_search_id": null,
|
|
"base_search_metric": null,
|
|
"cohesive_ad": {
|
|
"sensitivity": 8
|
|
},
|
|
"cohesive_anomaly_detection_is_enabled": false,
|
|
"datamodel": {
|
|
"datamodel": "",
|
|
"field": "",
|
|
"object": "",
|
|
"owner_field": ""
|
|
},
|
|
"datamodel_filter": [],
|
|
"datamodel_filter_clauses": null,
|
|
"description": "",
|
|
"enabled": false,
|
|
"entity_filter_field": "",
|
|
"entity_split_field": "",
|
|
"entity_statop": "avg",
|
|
"entity_thresholds": {
|
|
"base_severity_color": "#99D18B",
|
|
"base_severity_color_light": "#DCEFD7",
|
|
"base_severity_label": "normal",
|
|
"base_severity_value": 2.0,
|
|
"gauge_max": 100,
|
|
"gauge_min": 0,
|
|
"is_max_static": false,
|
|
"is_min_static": true,
|
|
"metric_field": "count",
|
|
"render_boundary_max": 100.0,
|
|
"render_boundary_min": 0.0,
|
|
"threshold_levels": [
|
|
{
|
|
"dynamic_param": null,
|
|
"severity_color": "#B50101",
|
|
"severity_color_light": "#E5A6A6",
|
|
"severity_label": "critical",
|
|
"severity_label_localized": null,
|
|
"severity_value": 6.0,
|
|
"threshold_value": 0.0
|
|
},
|
|
{
|
|
"dynamic_param": null,
|
|
"severity_color": "#F26A35",
|
|
"severity_color_light": "#FBCBB9",
|
|
"severity_label": "high",
|
|
"severity_label_localized": null,
|
|
"severity_value": 5.0,
|
|
"threshold_value": 20.0
|
|
},
|
|
{
|
|
"dynamic_param": null,
|
|
"severity_color": "#FCB64E",
|
|
"severity_color_light": "#FEE6C1",
|
|
"severity_label": "medium",
|
|
"severity_label_localized": null,
|
|
"severity_value": 4.0,
|
|
"threshold_value": 40.0
|
|
},
|
|
{
|
|
"dynamic_param": null,
|
|
"severity_color": "#FFE98C",
|
|
"severity_color_light": "#FFF4C5",
|
|
"severity_label": "low",
|
|
"severity_label_localized": null,
|
|
"severity_value": 3.0,
|
|
"threshold_value": 60.0
|
|
},
|
|
{
|
|
"dynamic_param": null,
|
|
"severity_color": "#99D18B",
|
|
"severity_color_light": "#DCEFD7",
|
|
"severity_label": "normal",
|
|
"severity_label_localized": null,
|
|
"severity_value": 2.0,
|
|
"threshold_value": 80.0
|
|
}
|
|
]
|
|
},
|
|
"fill_gaps": "null_value",
|
|
"gap_custom_alert_value": 0.0,
|
|
"gap_severity": "unknown",
|
|
"gap_severity_color": "#CCCCCC",
|
|
"gap_severity_color_light": "#EEEEEE",
|
|
"gap_severity_value": "-1",
|
|
"is_filter_entities_to_service": false,
|
|
"is_split_by_entity": false,
|
|
"key": "SHKPI-da-itsi-cp-splunk-observability-kubernetes-pods",
|
|
"kpi_base_search": "",
|
|
"kpi_template_kpi_id": "",
|
|
"kpi_threshold_template_id": "",
|
|
"metric_qualifier": null,
|
|
"metric_search_spec": {
|
|
"metric_index": "",
|
|
"metric_name": ""
|
|
},
|
|
"search": "`get_full_itsi_summary_service_health_events(da-itsi-cp-splunk-observability-kubernetes-pods)` | stats latest(health_score) AS aggregate",
|
|
"search_aggregate": "`get_full_itsi_summary_service_health_events(da-itsi-cp-splunk-observability-kubernetes-pods)` | stats latest(health_score) AS aggregate",
|
|
"search_alert": "",
|
|
"search_alert_earliest": "15",
|
|
"search_alert_entities": "",
|
|
"search_buckets": "",
|
|
"search_entities": null,
|
|
"search_occurrences": 1.0,
|
|
"search_time_compare": "`get_full_itsi_summary_service_health_events(da-itsi-cp-splunk-observability-kubernetes-pods)` [| stats count | addinfo | eval search= \"earliest=\" + tostring(info_min_time-(info_max_time-info_min_time))+ \" latest=\" + tostring(info_max_time) |fields search] | addinfo | eval bucket=if(_time<info_max_time-((info_max_time-info_min_time)/2), \"last_window\", \"current_window\") | stats avg(health_score) AS aggregate BY bucket | reverse | delta aggregate AS window_delta | search bucket=current_window | eval window_direction=if(window_delta >0, \"increase\", if(window_delta < 0, \"decrease\", \"none\"))",
|
|
"search_time_series": "`get_full_itsi_summary_service_health_events(da-itsi-cp-splunk-observability-kubernetes-pods)` | timechart avg(health_score) AS aggregate",
|
|
"search_time_series_aggregate": "`get_full_itsi_summary_service_health_events(da-itsi-cp-splunk-observability-kubernetes-pods)` | timechart avg(health_score) AS aggregate",
|
|
"search_time_series_entities": "",
|
|
"search_type": "adhoc",
|
|
"service_title": "Kubernetes Pods",
|
|
"threshold_eval": "",
|
|
"threshold_field": "aggregate",
|
|
"time_policies": {
|
|
"policies": {
|
|
"default_policy": {
|
|
"aggregate_thresholds": {
|
|
"base_severity_color": "#AED3E5",
|
|
"base_severity_color_light": "#E3F0F6",
|
|
"base_severity_label": "info",
|
|
"base_severity_value": 1.0,
|
|
"gauge_max": 100,
|
|
"gauge_min": 0,
|
|
"is_max_static": false,
|
|
"is_min_static": true,
|
|
"metric_field": "count",
|
|
"render_boundary_max": 100.0,
|
|
"render_boundary_min": 0.0,
|
|
"threshold_levels": []
|
|
},
|
|
"entity_thresholds": {
|
|
"base_severity_color": "#AED3E5",
|
|
"base_severity_color_light": "#E3F0F6",
|
|
"base_severity_label": "info",
|
|
"base_severity_value": 1.0,
|
|
"gauge_max": 100,
|
|
"gauge_min": 0,
|
|
"is_max_static": false,
|
|
"is_min_static": true,
|
|
"metric_field": "count",
|
|
"render_boundary_max": 100.0,
|
|
"render_boundary_min": 0.0,
|
|
"threshold_levels": []
|
|
},
|
|
"policy_type": "static",
|
|
"time_blocks": [],
|
|
"title": "Default"
|
|
}
|
|
}
|
|
},
|
|
"title": "ServiceHealthScore",
|
|
"trending_ad": {
|
|
"sensitivity": 8
|
|
},
|
|
"type": "service_health",
|
|
"tz_offset": null,
|
|
"unit": "",
|
|
"urgency": 11.0,
|
|
"use_time_policies": false
|
|
},
|
|
{
|
|
"adaptive_thresholding_training_window": "-7d",
|
|
"adaptive_thresholds_is_enabled": false,
|
|
"aggregate_eval": "",
|
|
"aggregate_statop": "avg",
|
|
"aggregate_threshold_alert_enabled": false,
|
|
"aggregate_thresholds": {
|
|
"base_severity_color": "#99D18B",
|
|
"base_severity_color_light": "#DCEFD7",
|
|
"base_severity_label": "normal",
|
|
"base_severity_value": 2.0,
|
|
"gauge_max": 99,
|
|
"gauge_min": 0,
|
|
"is_max_static": false,
|
|
"is_min_static": true,
|
|
"metric_field": "count",
|
|
"render_boundary_max": 99.0,
|
|
"render_boundary_min": 0.0,
|
|
"threshold_levels": [
|
|
{
|
|
"dynamic_param": 0.0,
|
|
"severity_color": "#FFE98C",
|
|
"severity_color_light": "#FFF4C5",
|
|
"severity_label": "low",
|
|
"severity_label_localized": "Low",
|
|
"severity_value": 3.0,
|
|
"threshold_value": 50.0
|
|
},
|
|
{
|
|
"dynamic_param": 0.0,
|
|
"severity_color": "#FCB64E",
|
|
"severity_color_light": "#FEE6C1",
|
|
"severity_label": "medium",
|
|
"severity_label_localized": "Medium",
|
|
"severity_value": 4.0,
|
|
"threshold_value": 80.0
|
|
},
|
|
{
|
|
"dynamic_param": 0.0,
|
|
"severity_color": "#F26A35",
|
|
"severity_color_light": "#FBCBB9",
|
|
"severity_label": "high",
|
|
"severity_label_localized": "High",
|
|
"severity_value": 5.0,
|
|
"threshold_value": 90.0
|
|
}
|
|
]
|
|
},
|
|
"aggregate_thresholds_alert_enabled": false,
|
|
"aggregate_thresholds_custom_alert_enabled": false,
|
|
"aggregate_thresholds_custom_alert_rules": [],
|
|
"alert_eval": "",
|
|
"alert_lag": "30",
|
|
"alert_on": "both",
|
|
"alert_period": "5",
|
|
"anomaly_detection_alerting_enabled": false,
|
|
"anomaly_detection_is_enabled": false,
|
|
"anomaly_detection_sensitivity": null,
|
|
"anomaly_detection_training_window": null,
|
|
"backfill_earliest_time": "-7d",
|
|
"backfill_enabled": false,
|
|
"base_search": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors",
|
|
"base_search_id": "da-itsi-cp-splunk-observability-sim-kubernetes",
|
|
"base_search_metric": "cpu-utilization",
|
|
"cohesive_ad": {
|
|
"sensitivity": 8
|
|
},
|
|
"cohesive_anomaly_detection_is_enabled": false,
|
|
"datamodel": {
|
|
"datamodel": "",
|
|
"field": "",
|
|
"object": "",
|
|
"owner_field": ""
|
|
},
|
|
"datamodel_filter": [],
|
|
"datamodel_filter_clauses": null,
|
|
"description": "Pod CPU Utilization by %",
|
|
"enabled": true,
|
|
"entity_filter_field": "host",
|
|
"entity_split_field": "k8s.pod.name",
|
|
"entity_statop": "avg",
|
|
"entity_thresholds": {
|
|
"base_severity_color": "#99D18B",
|
|
"base_severity_color_light": "#DCEFD7",
|
|
"base_severity_label": "normal",
|
|
"base_severity_value": 2.0,
|
|
"gauge_max": 99,
|
|
"gauge_min": 0,
|
|
"is_max_static": false,
|
|
"is_min_static": true,
|
|
"metric_field": "count",
|
|
"render_boundary_max": 99.0,
|
|
"render_boundary_min": 0.0,
|
|
"threshold_levels": [
|
|
{
|
|
"dynamic_param": 0.0,
|
|
"severity_color": "#FFE98C",
|
|
"severity_color_light": "#FFF4C5",
|
|
"severity_label": "low",
|
|
"severity_label_localized": "Low",
|
|
"severity_value": 3.0,
|
|
"threshold_value": 50.0
|
|
},
|
|
{
|
|
"dynamic_param": 0.0,
|
|
"severity_color": "#FCB64E",
|
|
"severity_color_light": "#FEE6C1",
|
|
"severity_label": "medium",
|
|
"severity_label_localized": "Medium",
|
|
"severity_value": 4.0,
|
|
"threshold_value": 80.0
|
|
},
|
|
{
|
|
"dynamic_param": 0.0,
|
|
"severity_color": "#F26A35",
|
|
"severity_color_light": "#FBCBB9",
|
|
"severity_label": "high",
|
|
"severity_label_localized": "High",
|
|
"severity_value": 5.0,
|
|
"threshold_value": 90.0
|
|
}
|
|
]
|
|
},
|
|
"fill_gaps": "null_value",
|
|
"gap_custom_alert_value": 0.0,
|
|
"gap_severity": "unknown",
|
|
"gap_severity_color": "#CCCCCC",
|
|
"gap_severity_color_light": "#EEEEEE",
|
|
"gap_severity_value": "-1",
|
|
"is_filter_entities_to_service": false,
|
|
"is_split_by_entity": true,
|
|
"key": "da-itsi-cp-splunk-observability-b9fba4990e500f27a55a4d31",
|
|
"kpi_base_search": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors",
|
|
"kpi_template_kpi_id": "",
|
|
"kpi_threshold_template_id": "",
|
|
"metric_qualifier": "",
|
|
"metric_search_spec": {
|
|
"metric_index": "",
|
|
"metric_name": ""
|
|
},
|
|
"search": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_entity(avg, k8s_cpu_utilization, \"k8s.pod.name\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(k8s.pod.name, sec_grp)` | eval serviceid = \"da-itsi-cp-splunk-observability-kubernetes-pods\" | `aggregate_entity_into_service(avg)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-b9fba4990e500f27a55a4d31, true, true, true)` | eval kpi=\"CPU Utilization\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-splunk-observability-kubernetes-pods\" | `assess_urgency` | `gettime`",
|
|
"search_aggregate": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_single_value(avg, avg, k8s_cpu_utilization, \"k8s.pod.name\", 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-b9fba4990e500f27a55a4d31)`",
|
|
"search_alert": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_entity(avg, k8s_cpu_utilization, \"k8s.pod.name\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(k8s.pod.name, sec_grp)` | eval serviceid = \"da-itsi-cp-splunk-observability-kubernetes-pods\" | `aggregate_entity_into_service(avg)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-b9fba4990e500f27a55a4d31, true, true, true)` | eval kpi=\"CPU Utilization\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-splunk-observability-kubernetes-pods\" | `assess_urgency` | `gettime`",
|
|
"search_alert_earliest": "15",
|
|
"search_alert_entities": "",
|
|
"search_buckets": "",
|
|
"search_entities": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_single_value(avg, avg, k8s_cpu_utilization, \"k8s.pod.name\", 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-b9fba4990e500f27a55a4d31)`",
|
|
"search_occurrences": 1.0,
|
|
"search_time_compare": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_and_compare(avg, avg, k8s_cpu_utilization, \"k8s.pod.name\", 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-b9fba4990e500f27a55a4d31)`",
|
|
"search_time_series": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_entity_time_series(avg, k8s_cpu_utilization, \"k8s.pod.name\", 15)` | `aggregate_entity_into_service_time_series(avg, 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-b9fba4990e500f27a55a4d31)`",
|
|
"search_time_series_aggregate": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_entity_time_series(avg, k8s_cpu_utilization, \"k8s.pod.name\", 15)` | `aggregate_entity_into_service_time_series(avg, 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-b9fba4990e500f27a55a4d31)`",
|
|
"search_time_series_entities": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_limited_entity_time_series(avg, k8s_cpu_utilization, \"k8s.pod.name\", 15)`",
|
|
"search_type": "shared_base",
|
|
"service_title": "Kubernetes Pods",
|
|
"threshold_eval": "",
|
|
"threshold_field": "k8s_cpu_utilization",
|
|
"time_policies": {
|
|
"policies": {
|
|
"default_policy": {
|
|
"aggregate_thresholds": {
|
|
"base_severity_color": "#99D18B",
|
|
"base_severity_color_light": "#DCEFD7",
|
|
"base_severity_label": "normal",
|
|
"base_severity_value": 2.0,
|
|
"gauge_max": 100,
|
|
"gauge_min": 0,
|
|
"is_max_static": false,
|
|
"is_min_static": true,
|
|
"metric_field": "count",
|
|
"render_boundary_max": 100.0,
|
|
"render_boundary_min": 0.0,
|
|
"threshold_levels": []
|
|
},
|
|
"entity_thresholds": {
|
|
"base_severity_color": "#99D18B",
|
|
"base_severity_color_light": "#DCEFD7",
|
|
"base_severity_label": "normal",
|
|
"base_severity_value": 2.0,
|
|
"gauge_max": 100,
|
|
"gauge_min": 0,
|
|
"is_max_static": false,
|
|
"is_min_static": true,
|
|
"metric_field": "count",
|
|
"render_boundary_max": 100.0,
|
|
"render_boundary_min": 0.0,
|
|
"threshold_levels": []
|
|
},
|
|
"policy_type": "static",
|
|
"time_blocks": [],
|
|
"title": "Default"
|
|
}
|
|
}
|
|
},
|
|
"title": "CPU Utilization",
|
|
"trending_ad": {
|
|
"sensitivity": 8
|
|
},
|
|
"type": "kpis_primary",
|
|
"tz_offset": null,
|
|
"unit": "%",
|
|
"urgency": 5.0,
|
|
"use_time_policies": false
|
|
},
|
|
{
|
|
"adaptive_thresholding_training_window": "-7d",
|
|
"adaptive_thresholds_is_enabled": false,
|
|
"aggregate_eval": null,
|
|
"aggregate_statop": "avg",
|
|
"aggregate_threshold_alert_enabled": false,
|
|
"aggregate_thresholds": {
|
|
"base_severity_color": "#99D18B",
|
|
"base_severity_color_light": "#DCEFD7",
|
|
"base_severity_label": "normal",
|
|
"base_severity_value": 2.0,
|
|
"gauge_max": 100,
|
|
"gauge_min": 0,
|
|
"is_max_static": false,
|
|
"is_min_static": true,
|
|
"metric_field": "count",
|
|
"render_boundary_max": 100.0,
|
|
"render_boundary_min": 0.0,
|
|
"threshold_levels": []
|
|
},
|
|
"aggregate_thresholds_alert_enabled": false,
|
|
"aggregate_thresholds_custom_alert_enabled": false,
|
|
"aggregate_thresholds_custom_alert_rules": [],
|
|
"alert_eval": null,
|
|
"alert_lag": "30",
|
|
"alert_on": "both",
|
|
"alert_period": "5",
|
|
"anomaly_detection_alerting_enabled": false,
|
|
"anomaly_detection_is_enabled": false,
|
|
"anomaly_detection_sensitivity": null,
|
|
"anomaly_detection_training_window": null,
|
|
"backfill_earliest_time": "-7d",
|
|
"backfill_enabled": false,
|
|
"base_search": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors",
|
|
"base_search_id": "da-itsi-cp-splunk-observability-sim-kubernetes",
|
|
"base_search_metric": "memory-utilization",
|
|
"cohesive_ad": {
|
|
"sensitivity": 8
|
|
},
|
|
"cohesive_anomaly_detection_is_enabled": false,
|
|
"datamodel": {
|
|
"datamodel": "",
|
|
"field": "",
|
|
"object": "",
|
|
"owner_field": ""
|
|
},
|
|
"datamodel_filter": [],
|
|
"datamodel_filter_clauses": null,
|
|
"description": "Pod Memory Utilization by %",
|
|
"enabled": true,
|
|
"entity_filter_field": "host",
|
|
"entity_split_field": "k8s.pod.name",
|
|
"entity_statop": "avg",
|
|
"entity_thresholds": {
|
|
"base_severity_color": "#99D18B",
|
|
"base_severity_color_light": "#DCEFD7",
|
|
"base_severity_label": "normal",
|
|
"base_severity_value": 2.0,
|
|
"gauge_max": 100,
|
|
"gauge_min": 0,
|
|
"is_max_static": false,
|
|
"is_min_static": true,
|
|
"metric_field": "count",
|
|
"render_boundary_max": 260.69268041204964,
|
|
"render_boundary_min": 0.0,
|
|
"threshold_levels": []
|
|
},
|
|
"fill_gaps": "null_value",
|
|
"gap_custom_alert_value": 0.0,
|
|
"gap_severity": "unknown",
|
|
"gap_severity_color": "#CCCCCC",
|
|
"gap_severity_color_light": "#EEEEEE",
|
|
"gap_severity_value": "-1",
|
|
"is_filter_entities_to_service": false,
|
|
"is_split_by_entity": true,
|
|
"key": "da-itsi-cp-splunk-observability-19dab576ab91ba1ffcc023c0",
|
|
"kpi_base_search": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors",
|
|
"kpi_template_kpi_id": "",
|
|
"kpi_threshold_template_id": "",
|
|
"metric_qualifier": "",
|
|
"metric_search_spec": {
|
|
"metric_index": "",
|
|
"metric_name": ""
|
|
},
|
|
"search": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_entity(avg, k8s_memory_utilization, \"k8s.pod.name\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(k8s.pod.name, sec_grp)` | eval serviceid = \"da-itsi-cp-splunk-observability-kubernetes-pods\" | `aggregate_entity_into_service(avg)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-19dab576ab91ba1ffcc023c0, true, true, true)` | eval kpi=\"Memory Utilization\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-splunk-observability-kubernetes-pods\" | `assess_urgency` | `gettime`",
|
|
"search_aggregate": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_single_value(avg, avg, k8s_memory_utilization, \"k8s.pod.name\", 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-19dab576ab91ba1ffcc023c0)`",
|
|
"search_alert": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_entity(avg, k8s_memory_utilization, \"k8s.pod.name\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(k8s.pod.name, sec_grp)` | eval serviceid = \"da-itsi-cp-splunk-observability-kubernetes-pods\" | `aggregate_entity_into_service(avg)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-19dab576ab91ba1ffcc023c0, true, true, true)` | eval kpi=\"Memory Utilization\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-splunk-observability-kubernetes-pods\" | `assess_urgency` | `gettime`",
|
|
"search_alert_earliest": "15",
|
|
"search_alert_entities": "",
|
|
"search_buckets": "",
|
|
"search_entities": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_single_value(avg, avg, k8s_memory_utilization, \"k8s.pod.name\", 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-19dab576ab91ba1ffcc023c0)`",
|
|
"search_occurrences": 1.0,
|
|
"search_time_compare": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_and_compare(avg, avg, k8s_memory_utilization, \"k8s.pod.name\", 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-19dab576ab91ba1ffcc023c0)`",
|
|
"search_time_series": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_entity_time_series(avg, k8s_memory_utilization, \"k8s.pod.name\", 15)` | `aggregate_entity_into_service_time_series(avg, 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-19dab576ab91ba1ffcc023c0)`",
|
|
"search_time_series_aggregate": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_entity_time_series(avg, k8s_memory_utilization, \"k8s.pod.name\", 15)` | `aggregate_entity_into_service_time_series(avg, 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-19dab576ab91ba1ffcc023c0)`",
|
|
"search_time_series_entities": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_limited_entity_time_series(avg, k8s_memory_utilization, \"k8s.pod.name\", 15)`",
|
|
"search_type": "shared_base",
|
|
"service_title": "Kubernetes Pods",
|
|
"threshold_eval": null,
|
|
"threshold_field": "k8s_memory_utilization",
|
|
"time_policies": {
|
|
"policies": {
|
|
"default_policy": {
|
|
"aggregate_thresholds": {
|
|
"base_severity_color": "#99D18B",
|
|
"base_severity_color_light": "#DCEFD7",
|
|
"base_severity_label": "normal",
|
|
"base_severity_value": 2.0,
|
|
"gauge_max": 100,
|
|
"gauge_min": 0,
|
|
"is_max_static": false,
|
|
"is_min_static": true,
|
|
"metric_field": "count",
|
|
"render_boundary_max": 100.0,
|
|
"render_boundary_min": 0.0,
|
|
"threshold_levels": []
|
|
},
|
|
"entity_thresholds": {
|
|
"base_severity_color": "#99D18B",
|
|
"base_severity_color_light": "#DCEFD7",
|
|
"base_severity_label": "normal",
|
|
"base_severity_value": 2.0,
|
|
"gauge_max": 100,
|
|
"gauge_min": 0,
|
|
"is_max_static": false,
|
|
"is_min_static": true,
|
|
"metric_field": "count",
|
|
"render_boundary_max": 100.0,
|
|
"render_boundary_min": 0.0,
|
|
"threshold_levels": []
|
|
},
|
|
"policy_type": "static",
|
|
"time_blocks": [],
|
|
"title": "Default"
|
|
}
|
|
}
|
|
},
|
|
"title": "Memory Utilization",
|
|
"trending_ad": {
|
|
"sensitivity": 8
|
|
},
|
|
"type": "kpis_primary",
|
|
"tz_offset": null,
|
|
"unit": "%",
|
|
"urgency": 5.0,
|
|
"use_time_policies": false
|
|
},
|
|
{
|
|
"adaptive_thresholding_training_window": "-7d",
|
|
"adaptive_thresholds_is_enabled": false,
|
|
"aggregate_eval": "",
|
|
"aggregate_statop": "avg",
|
|
"aggregate_threshold_alert_enabled": false,
|
|
"aggregate_thresholds": {
|
|
"base_severity_color": "#99D18B",
|
|
"base_severity_color_light": "#DCEFD7",
|
|
"base_severity_label": "normal",
|
|
"base_severity_value": 2.0,
|
|
"gauge_max": 100,
|
|
"gauge_min": 0,
|
|
"is_max_static": false,
|
|
"is_min_static": true,
|
|
"metric_field": "count",
|
|
"render_boundary_max": 100.0,
|
|
"render_boundary_min": 0.0,
|
|
"threshold_levels": []
|
|
},
|
|
"aggregate_thresholds_alert_enabled": false,
|
|
"aggregate_thresholds_custom_alert_enabled": false,
|
|
"aggregate_thresholds_custom_alert_rules": [],
|
|
"alert_eval": "",
|
|
"alert_lag": "30",
|
|
"alert_on": "both",
|
|
"alert_period": "5",
|
|
"anomaly_detection_alerting_enabled": false,
|
|
"anomaly_detection_is_enabled": false,
|
|
"anomaly_detection_sensitivity": null,
|
|
"anomaly_detection_training_window": null,
|
|
"backfill_earliest_time": "-7d",
|
|
"backfill_enabled": false,
|
|
"base_search": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors",
|
|
"base_search_id": "da-itsi-cp-splunk-observability-sim-kubernetes",
|
|
"base_search_metric": "network-rx-errors",
|
|
"cohesive_ad": {
|
|
"sensitivity": 8
|
|
},
|
|
"cohesive_anomaly_detection_is_enabled": false,
|
|
"datamodel": {
|
|
"datamodel": "",
|
|
"field": "",
|
|
"object": "",
|
|
"owner_field": ""
|
|
},
|
|
"datamodel_filter": [],
|
|
"datamodel_filter_clauses": null,
|
|
"description": "Pod Network Receive Errors",
|
|
"enabled": true,
|
|
"entity_filter_field": "host",
|
|
"entity_split_field": "k8s.pod.name",
|
|
"entity_statop": "avg",
|
|
"entity_thresholds": {
|
|
"base_severity_color": "#99D18B",
|
|
"base_severity_color_light": "#DCEFD7",
|
|
"base_severity_label": "normal",
|
|
"base_severity_value": 2.0,
|
|
"gauge_max": 100,
|
|
"gauge_min": 0,
|
|
"is_max_static": false,
|
|
"is_min_static": true,
|
|
"metric_field": "count",
|
|
"render_boundary_max": 100.0,
|
|
"render_boundary_min": 0.0,
|
|
"threshold_levels": []
|
|
},
|
|
"fill_gaps": "null_value",
|
|
"gap_custom_alert_value": 0.0,
|
|
"gap_severity": "unknown",
|
|
"gap_severity_color": "#CCCCCC",
|
|
"gap_severity_color_light": "#EEEEEE",
|
|
"gap_severity_value": "-1",
|
|
"is_filter_entities_to_service": false,
|
|
"is_split_by_entity": true,
|
|
"key": "da-itsi-cp-splunk-observability-1c175b66337825eab27f5fba",
|
|
"kpi_base_search": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors",
|
|
"kpi_template_kpi_id": "",
|
|
"kpi_threshold_template_id": "",
|
|
"metric_qualifier": "",
|
|
"metric_search_spec": {
|
|
"metric_index": "",
|
|
"metric_name": ""
|
|
},
|
|
"search": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_entity(avg, k8s_network_rx_errors, \"k8s.pod.name\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(k8s.pod.name, sec_grp)` | eval serviceid = \"da-itsi-cp-splunk-observability-kubernetes-pods\" | `aggregate_entity_into_service(avg)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-1c175b66337825eab27f5fba, true, true, true)` | eval kpi=\"Network Rx Errors\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-splunk-observability-kubernetes-pods\" | `assess_urgency` | `gettime`",
|
|
"search_aggregate": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_single_value(avg, avg, k8s_network_rx_errors, \"k8s.pod.name\", 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-1c175b66337825eab27f5fba)`",
|
|
"search_alert": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_entity(avg, k8s_network_rx_errors, \"k8s.pod.name\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(k8s.pod.name, sec_grp)` | eval serviceid = \"da-itsi-cp-splunk-observability-kubernetes-pods\" | `aggregate_entity_into_service(avg)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-1c175b66337825eab27f5fba, true, true, true)` | eval kpi=\"Network Rx Errors\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-splunk-observability-kubernetes-pods\" | `assess_urgency` | `gettime`",
|
|
"search_alert_earliest": "15",
|
|
"search_alert_entities": "",
|
|
"search_buckets": "",
|
|
"search_entities": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_single_value(avg, avg, k8s_network_rx_errors, \"k8s.pod.name\", 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-1c175b66337825eab27f5fba)`",
|
|
"search_occurrences": 1.0,
|
|
"search_time_compare": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_and_compare(avg, avg, k8s_network_rx_errors, \"k8s.pod.name\", 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-1c175b66337825eab27f5fba)`",
|
|
"search_time_series": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_entity_time_series(avg, k8s_network_rx_errors, \"k8s.pod.name\", 15)` | `aggregate_entity_into_service_time_series(avg, 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-1c175b66337825eab27f5fba)`",
|
|
"search_time_series_aggregate": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_entity_time_series(avg, k8s_network_rx_errors, \"k8s.pod.name\", 15)` | `aggregate_entity_into_service_time_series(avg, 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-1c175b66337825eab27f5fba)`",
|
|
"search_time_series_entities": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_limited_entity_time_series(avg, k8s_network_rx_errors, \"k8s.pod.name\", 15)`",
|
|
"search_type": "shared_base",
|
|
"service_title": "Kubernetes Pods",
|
|
"threshold_eval": "",
|
|
"threshold_field": "k8s_network_rx_errors",
|
|
"time_policies": {
|
|
"policies": {
|
|
"default_policy": {
|
|
"aggregate_thresholds": {
|
|
"base_severity_color": "#99D18B",
|
|
"base_severity_color_light": "#DCEFD7",
|
|
"base_severity_label": "normal",
|
|
"base_severity_value": 2.0,
|
|
"gauge_max": 100,
|
|
"gauge_min": 0,
|
|
"is_max_static": false,
|
|
"is_min_static": true,
|
|
"metric_field": "count",
|
|
"render_boundary_max": 100.0,
|
|
"render_boundary_min": 0.0,
|
|
"threshold_levels": []
|
|
},
|
|
"entity_thresholds": {
|
|
"base_severity_color": "#99D18B",
|
|
"base_severity_color_light": "#DCEFD7",
|
|
"base_severity_label": "normal",
|
|
"base_severity_value": 2.0,
|
|
"gauge_max": 100,
|
|
"gauge_min": 0,
|
|
"is_max_static": false,
|
|
"is_min_static": true,
|
|
"metric_field": "count",
|
|
"render_boundary_max": 100.0,
|
|
"render_boundary_min": 0.0,
|
|
"threshold_levels": []
|
|
},
|
|
"policy_type": "static",
|
|
"time_blocks": [],
|
|
"title": "Default"
|
|
}
|
|
}
|
|
},
|
|
"title": "Network Rx Errors",
|
|
"trending_ad": {
|
|
"sensitivity": 8
|
|
},
|
|
"type": "kpis_primary",
|
|
"tz_offset": null,
|
|
"unit": "",
|
|
"urgency": 5.0,
|
|
"use_time_policies": false
|
|
},
|
|
{
|
|
"adaptive_thresholding_training_window": "-7d",
|
|
"adaptive_thresholds_is_enabled": false,
|
|
"aggregate_eval": "",
|
|
"aggregate_statop": "avg",
|
|
"aggregate_threshold_alert_enabled": false,
|
|
"aggregate_thresholds": {
|
|
"base_severity_color": "#99D18B",
|
|
"base_severity_color_light": "#DCEFD7",
|
|
"base_severity_label": "normal",
|
|
"base_severity_value": 2.0,
|
|
"gauge_max": 100,
|
|
"gauge_min": 0,
|
|
"is_max_static": false,
|
|
"is_min_static": true,
|
|
"metric_field": "count",
|
|
"render_boundary_max": 100.0,
|
|
"render_boundary_min": 0.0,
|
|
"threshold_levels": []
|
|
},
|
|
"aggregate_thresholds_alert_enabled": false,
|
|
"aggregate_thresholds_custom_alert_enabled": false,
|
|
"aggregate_thresholds_custom_alert_rules": [],
|
|
"alert_eval": "",
|
|
"alert_lag": "30",
|
|
"alert_on": "both",
|
|
"alert_period": "5",
|
|
"anomaly_detection_alerting_enabled": false,
|
|
"anomaly_detection_is_enabled": false,
|
|
"anomaly_detection_sensitivity": null,
|
|
"anomaly_detection_training_window": null,
|
|
"backfill_earliest_time": "-7d",
|
|
"backfill_enabled": false,
|
|
"base_search": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors",
|
|
"base_search_id": "da-itsi-cp-splunk-observability-sim-kubernetes",
|
|
"base_search_metric": "network-tx-errors",
|
|
"cohesive_ad": {
|
|
"sensitivity": 8
|
|
},
|
|
"cohesive_anomaly_detection_is_enabled": false,
|
|
"datamodel": {
|
|
"datamodel": "",
|
|
"field": "",
|
|
"object": "",
|
|
"owner_field": ""
|
|
},
|
|
"datamodel_filter": [],
|
|
"datamodel_filter_clauses": null,
|
|
"description": "Pod Network Transmit Errors",
|
|
"enabled": true,
|
|
"entity_filter_field": "host",
|
|
"entity_split_field": "k8s.pod.name",
|
|
"entity_statop": "avg",
|
|
"entity_thresholds": {
|
|
"base_severity_color": "#99D18B",
|
|
"base_severity_color_light": "#DCEFD7",
|
|
"base_severity_label": "normal",
|
|
"base_severity_value": 2.0,
|
|
"gauge_max": 100,
|
|
"gauge_min": 0,
|
|
"is_max_static": false,
|
|
"is_min_static": true,
|
|
"metric_field": "count",
|
|
"render_boundary_max": 100.0,
|
|
"render_boundary_min": 0.0,
|
|
"threshold_levels": []
|
|
},
|
|
"fill_gaps": "null_value",
|
|
"gap_custom_alert_value": 0.0,
|
|
"gap_severity": "unknown",
|
|
"gap_severity_color": "#CCCCCC",
|
|
"gap_severity_color_light": "#EEEEEE",
|
|
"gap_severity_value": "-1",
|
|
"is_filter_entities_to_service": false,
|
|
"is_split_by_entity": true,
|
|
"key": "da-itsi-cp-splunk-observability-d81b5b3f3fca0bcf50733969",
|
|
"kpi_base_search": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors",
|
|
"kpi_template_kpi_id": "",
|
|
"kpi_threshold_template_id": "",
|
|
"metric_qualifier": "",
|
|
"metric_search_spec": {
|
|
"metric_index": "",
|
|
"metric_name": ""
|
|
},
|
|
"search": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_entity(avg, k8s_network_tx_errors, \"k8s.pod.name\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(k8s.pod.name, sec_grp)` | eval serviceid = \"da-itsi-cp-splunk-observability-kubernetes-pods\" | `aggregate_entity_into_service(avg)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-d81b5b3f3fca0bcf50733969, true, true, true)` | eval kpi=\"Network Tx Errors\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-splunk-observability-kubernetes-pods\" | `assess_urgency` | `gettime`",
|
|
"search_aggregate": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_single_value(avg, avg, k8s_network_tx_errors, \"k8s.pod.name\", 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-d81b5b3f3fca0bcf50733969)`",
|
|
"search_alert": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_entity(avg, k8s_network_tx_errors, \"k8s.pod.name\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(k8s.pod.name, sec_grp)` | eval serviceid = \"da-itsi-cp-splunk-observability-kubernetes-pods\" | `aggregate_entity_into_service(avg)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-d81b5b3f3fca0bcf50733969, true, true, true)` | eval kpi=\"Network Tx Errors\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-splunk-observability-kubernetes-pods\" | `assess_urgency` | `gettime`",
|
|
"search_alert_earliest": "15",
|
|
"search_alert_entities": "",
|
|
"search_buckets": "",
|
|
"search_entities": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_single_value(avg, avg, k8s_network_tx_errors, \"k8s.pod.name\", 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-d81b5b3f3fca0bcf50733969)`",
|
|
"search_occurrences": 1.0,
|
|
"search_time_compare": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_and_compare(avg, avg, k8s_network_tx_errors, \"k8s.pod.name\", 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-d81b5b3f3fca0bcf50733969)`",
|
|
"search_time_series": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_entity_time_series(avg, k8s_network_tx_errors, \"k8s.pod.name\", 15)` | `aggregate_entity_into_service_time_series(avg, 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-d81b5b3f3fca0bcf50733969)`",
|
|
"search_time_series_aggregate": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_entity_time_series(avg, k8s_network_tx_errors, \"k8s.pod.name\", 15)` | `aggregate_entity_into_service_time_series(avg, 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-d81b5b3f3fca0bcf50733969)`",
|
|
"search_time_series_entities": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_limited_entity_time_series(avg, k8s_network_tx_errors, \"k8s.pod.name\", 15)`",
|
|
"search_type": "shared_base",
|
|
"service_title": "Kubernetes Pods",
|
|
"threshold_eval": "",
|
|
"threshold_field": "k8s_network_tx_errors",
|
|
"time_policies": {
|
|
"policies": {
|
|
"default_policy": {
|
|
"aggregate_thresholds": {
|
|
"base_severity_color": "#99D18B",
|
|
"base_severity_color_light": "#DCEFD7",
|
|
"base_severity_label": "normal",
|
|
"base_severity_value": 2.0,
|
|
"gauge_max": 100,
|
|
"gauge_min": 0,
|
|
"is_max_static": false,
|
|
"is_min_static": true,
|
|
"metric_field": "count",
|
|
"render_boundary_max": 100.0,
|
|
"render_boundary_min": 0.0,
|
|
"threshold_levels": []
|
|
},
|
|
"entity_thresholds": {
|
|
"base_severity_color": "#99D18B",
|
|
"base_severity_color_light": "#DCEFD7",
|
|
"base_severity_label": "normal",
|
|
"base_severity_value": 2.0,
|
|
"gauge_max": 100,
|
|
"gauge_min": 0,
|
|
"is_max_static": false,
|
|
"is_min_static": true,
|
|
"metric_field": "count",
|
|
"render_boundary_max": 100.0,
|
|
"render_boundary_min": 0.0,
|
|
"threshold_levels": []
|
|
},
|
|
"policy_type": "static",
|
|
"time_blocks": [],
|
|
"title": "Default"
|
|
}
|
|
}
|
|
},
|
|
"title": "Network Tx Errors",
|
|
"trending_ad": {
|
|
"sensitivity": 8
|
|
},
|
|
"type": "kpis_primary",
|
|
"tz_offset": null,
|
|
"unit": "",
|
|
"urgency": 5.0,
|
|
"use_time_policies": false
|
|
}
|
|
],
|
|
"service_tags": {
|
|
"tags": [],
|
|
"template_tags": []
|
|
},
|
|
"service_template_id": "",
|
|
"services_depending_on_me": [
|
|
{
|
|
"kpis_depending_on": [
|
|
"SHKPI-da-itsi-cp-splunk-observability-kubernetes-pods"
|
|
],
|
|
"service_id": "da-itsi-cp-splunk-observability-my-data-center-hosts"
|
|
}
|
|
],
|
|
"services_depends_on": [],
|
|
"team_id": "default_itsi_security_group",
|
|
"title": "Kubernetes Pods",
|
|
"version": "0.0.38"
|
|
} |