{ "algorithms": { "GradientBoostingRegressor": { "RMSE": 0, "modelId": "", "rSquared": 0, "recommended": false }, "LinearRegression": { "RMSE": 0, "modelId": "", "rSquared": 0, "recommended": false }, "LogisticRegression": { "accuracy": 0, "f1_score": 0, "modelId": "", "precision": 0, "recall": 0, "recommended": false }, "RandomForestRegressor": { "RMSE": 0, "modelId": "", "rSquared": 0, "recommended": false } }, "description": "Service for monitoring Kubernetes Pods", "enabled": true, "entity_rules": [], "key": "da-itsi-cp-splunk-observability-kubernetes-pods", "kpis": [ { "adaptive_thresholding_training_window": "-7d", "adaptive_thresholds_is_enabled": false, "aggregate_eval": "", "aggregate_statop": "avg", "aggregate_threshold_alert_enabled": false, "aggregate_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [ { "dynamic_param": null, "severity_color": "#B50101", "severity_color_light": "#E5A6A6", "severity_label": "critical", "severity_label_localized": null, "severity_value": 6.0, "threshold_value": 0.0 }, { "dynamic_param": null, "severity_color": "#F26A35", "severity_color_light": "#FBCBB9", "severity_label": "high", "severity_label_localized": null, "severity_value": 5.0, "threshold_value": 20.0 }, { "dynamic_param": null, "severity_color": "#FCB64E", "severity_color_light": "#FEE6C1", "severity_label": "medium", "severity_label_localized": null, "severity_value": 4.0, "threshold_value": 40.0 }, { "dynamic_param": null, "severity_color": "#FFE98C", "severity_color_light": "#FFF4C5", "severity_label": "low", "severity_label_localized": null, "severity_value": 3.0, "threshold_value": 60.0 }, { "dynamic_param": null, "severity_color": "#99D18B", "severity_color_light": "#DCEFD7", "severity_label": "normal", "severity_label_localized": null, "severity_value": 2.0, "threshold_value": 80.0 } ] }, "aggregate_thresholds_alert_enabled": false, "aggregate_thresholds_custom_alert_enabled": false, "aggregate_thresholds_custom_alert_rules": [], "alert_eval": "", "alert_lag": "30", "alert_on": "both", "alert_period": "1", "anomaly_detection_alerting_enabled": false, "anomaly_detection_is_enabled": false, "anomaly_detection_sensitivity": 0.999, "anomaly_detection_training_window": "-7d", "backfill_earliest_time": "-7d", "backfill_enabled": false, "base_search": "`get_full_itsi_summary_service_health_events(da-itsi-cp-splunk-observability-kubernetes-pods)`", "base_search_id": null, "base_search_metric": null, "cohesive_ad": { "sensitivity": 8 }, "cohesive_anomaly_detection_is_enabled": false, "datamodel": { "datamodel": "", "field": "", "object": "", "owner_field": "" }, "datamodel_filter": [], "datamodel_filter_clauses": null, "description": "", "enabled": false, "entity_filter_field": "", "entity_split_field": "", "entity_statop": "avg", "entity_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [ { "dynamic_param": null, "severity_color": "#B50101", "severity_color_light": "#E5A6A6", "severity_label": "critical", "severity_label_localized": null, "severity_value": 6.0, "threshold_value": 0.0 }, { "dynamic_param": null, "severity_color": "#F26A35", "severity_color_light": "#FBCBB9", "severity_label": "high", "severity_label_localized": null, "severity_value": 5.0, "threshold_value": 20.0 }, { "dynamic_param": null, "severity_color": "#FCB64E", "severity_color_light": "#FEE6C1", "severity_label": "medium", "severity_label_localized": null, "severity_value": 4.0, "threshold_value": 40.0 }, { "dynamic_param": null, "severity_color": "#FFE98C", "severity_color_light": "#FFF4C5", "severity_label": "low", "severity_label_localized": null, "severity_value": 3.0, "threshold_value": 60.0 }, { "dynamic_param": null, "severity_color": "#99D18B", "severity_color_light": "#DCEFD7", "severity_label": "normal", "severity_label_localized": null, "severity_value": 2.0, "threshold_value": 80.0 } ] }, "fill_gaps": "null_value", "gap_custom_alert_value": 0.0, "gap_severity": "unknown", "gap_severity_color": "#CCCCCC", "gap_severity_color_light": "#EEEEEE", "gap_severity_value": "-1", "is_filter_entities_to_service": false, "is_split_by_entity": false, "key": "SHKPI-da-itsi-cp-splunk-observability-kubernetes-pods", "kpi_base_search": "", "kpi_template_kpi_id": "", "kpi_threshold_template_id": "", "metric_qualifier": null, "metric_search_spec": { "metric_index": "", "metric_name": "" }, "search": "`get_full_itsi_summary_service_health_events(da-itsi-cp-splunk-observability-kubernetes-pods)` | stats latest(health_score) AS aggregate", "search_aggregate": "`get_full_itsi_summary_service_health_events(da-itsi-cp-splunk-observability-kubernetes-pods)` | stats latest(health_score) AS aggregate", "search_alert": "", "search_alert_earliest": "15", "search_alert_entities": "", "search_buckets": "", "search_entities": null, "search_occurrences": 1.0, "search_time_compare": "`get_full_itsi_summary_service_health_events(da-itsi-cp-splunk-observability-kubernetes-pods)` [| stats count | addinfo | eval search= \"earliest=\" + tostring(info_min_time-(info_max_time-info_min_time))+ \" latest=\" + tostring(info_max_time) |fields search] | addinfo | eval bucket=if(_time0, \"increase\", if(window_delta < 0, \"decrease\", \"none\"))", "search_time_series": "`get_full_itsi_summary_service_health_events(da-itsi-cp-splunk-observability-kubernetes-pods)` | timechart avg(health_score) AS aggregate", "search_time_series_aggregate": "`get_full_itsi_summary_service_health_events(da-itsi-cp-splunk-observability-kubernetes-pods)` | timechart avg(health_score) AS aggregate", "search_time_series_entities": "", "search_type": "adhoc", "service_title": "Kubernetes Pods", "threshold_eval": "", "threshold_field": "aggregate", "time_policies": { "policies": { "default_policy": { "aggregate_thresholds": { "base_severity_color": "#AED3E5", "base_severity_color_light": "#E3F0F6", "base_severity_label": "info", "base_severity_value": 1.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "entity_thresholds": { "base_severity_color": "#AED3E5", "base_severity_color_light": "#E3F0F6", "base_severity_label": "info", "base_severity_value": 1.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "policy_type": "static", "time_blocks": [], "title": "Default" } } }, "title": "ServiceHealthScore", "trending_ad": { "sensitivity": 8 }, "type": "service_health", "tz_offset": null, "unit": "", "urgency": 11.0, "use_time_policies": false }, { "adaptive_thresholding_training_window": "-7d", "adaptive_thresholds_is_enabled": false, "aggregate_eval": "", "aggregate_statop": "avg", "aggregate_threshold_alert_enabled": false, "aggregate_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 99, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 99.0, "render_boundary_min": 0.0, "threshold_levels": [ { "dynamic_param": 0.0, "severity_color": "#FFE98C", "severity_color_light": "#FFF4C5", "severity_label": "low", "severity_label_localized": "Low", "severity_value": 3.0, "threshold_value": 50.0 }, { "dynamic_param": 0.0, "severity_color": "#FCB64E", "severity_color_light": "#FEE6C1", "severity_label": "medium", "severity_label_localized": "Medium", "severity_value": 4.0, "threshold_value": 80.0 }, { "dynamic_param": 0.0, "severity_color": "#F26A35", "severity_color_light": "#FBCBB9", "severity_label": "high", "severity_label_localized": "High", "severity_value": 5.0, "threshold_value": 90.0 } ] }, "aggregate_thresholds_alert_enabled": false, "aggregate_thresholds_custom_alert_enabled": false, "aggregate_thresholds_custom_alert_rules": [], "alert_eval": "", "alert_lag": "30", "alert_on": "both", "alert_period": "5", "anomaly_detection_alerting_enabled": false, "anomaly_detection_is_enabled": false, "anomaly_detection_sensitivity": null, "anomaly_detection_training_window": null, "backfill_earliest_time": "-7d", "backfill_enabled": false, "base_search": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors", "base_search_id": "da-itsi-cp-splunk-observability-sim-kubernetes", "base_search_metric": "cpu-utilization", "cohesive_ad": { "sensitivity": 8 }, "cohesive_anomaly_detection_is_enabled": false, "datamodel": { "datamodel": "", "field": "", "object": "", "owner_field": "" }, "datamodel_filter": [], "datamodel_filter_clauses": null, "description": "Pod CPU Utilization by %", "enabled": true, "entity_filter_field": "host", "entity_split_field": "k8s.pod.name", "entity_statop": "avg", "entity_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 99, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 99.0, "render_boundary_min": 0.0, "threshold_levels": [ { "dynamic_param": 0.0, "severity_color": "#FFE98C", "severity_color_light": "#FFF4C5", "severity_label": "low", "severity_label_localized": "Low", "severity_value": 3.0, "threshold_value": 50.0 }, { "dynamic_param": 0.0, "severity_color": "#FCB64E", "severity_color_light": "#FEE6C1", "severity_label": "medium", "severity_label_localized": "Medium", "severity_value": 4.0, "threshold_value": 80.0 }, { "dynamic_param": 0.0, "severity_color": "#F26A35", "severity_color_light": "#FBCBB9", "severity_label": "high", "severity_label_localized": "High", "severity_value": 5.0, "threshold_value": 90.0 } ] }, "fill_gaps": "null_value", "gap_custom_alert_value": 0.0, "gap_severity": "unknown", "gap_severity_color": "#CCCCCC", "gap_severity_color_light": "#EEEEEE", "gap_severity_value": "-1", "is_filter_entities_to_service": false, "is_split_by_entity": true, "key": "da-itsi-cp-splunk-observability-b9fba4990e500f27a55a4d31", "kpi_base_search": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors", "kpi_template_kpi_id": "", "kpi_threshold_template_id": "", "metric_qualifier": "", "metric_search_spec": { "metric_index": "", "metric_name": "" }, "search": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_entity(avg, k8s_cpu_utilization, \"k8s.pod.name\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(k8s.pod.name, sec_grp)` | eval serviceid = \"da-itsi-cp-splunk-observability-kubernetes-pods\" | `aggregate_entity_into_service(avg)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-b9fba4990e500f27a55a4d31, true, true, true)` | eval kpi=\"CPU Utilization\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-splunk-observability-kubernetes-pods\" | `assess_urgency` | `gettime`", "search_aggregate": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_single_value(avg, avg, k8s_cpu_utilization, \"k8s.pod.name\", 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-b9fba4990e500f27a55a4d31)`", "search_alert": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_entity(avg, k8s_cpu_utilization, \"k8s.pod.name\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(k8s.pod.name, sec_grp)` | eval serviceid = \"da-itsi-cp-splunk-observability-kubernetes-pods\" | `aggregate_entity_into_service(avg)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-b9fba4990e500f27a55a4d31, true, true, true)` | eval kpi=\"CPU Utilization\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-splunk-observability-kubernetes-pods\" | `assess_urgency` | `gettime`", "search_alert_earliest": "15", "search_alert_entities": "", "search_buckets": "", "search_entities": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_single_value(avg, avg, k8s_cpu_utilization, \"k8s.pod.name\", 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-b9fba4990e500f27a55a4d31)`", "search_occurrences": 1.0, "search_time_compare": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_and_compare(avg, avg, k8s_cpu_utilization, \"k8s.pod.name\", 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-b9fba4990e500f27a55a4d31)`", "search_time_series": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_entity_time_series(avg, k8s_cpu_utilization, \"k8s.pod.name\", 15)` | `aggregate_entity_into_service_time_series(avg, 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-b9fba4990e500f27a55a4d31)`", "search_time_series_aggregate": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_entity_time_series(avg, k8s_cpu_utilization, \"k8s.pod.name\", 15)` | `aggregate_entity_into_service_time_series(avg, 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-b9fba4990e500f27a55a4d31)`", "search_time_series_entities": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_limited_entity_time_series(avg, k8s_cpu_utilization, \"k8s.pod.name\", 15)`", "search_type": "shared_base", "service_title": "Kubernetes Pods", "threshold_eval": "", "threshold_field": "k8s_cpu_utilization", "time_policies": { "policies": { "default_policy": { "aggregate_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "entity_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "policy_type": "static", "time_blocks": [], "title": "Default" } } }, "title": "CPU Utilization", "trending_ad": { "sensitivity": 8 }, "type": "kpis_primary", "tz_offset": null, "unit": "%", "urgency": 5.0, "use_time_policies": false }, { "adaptive_thresholding_training_window": "-7d", "adaptive_thresholds_is_enabled": false, "aggregate_eval": null, "aggregate_statop": "avg", "aggregate_threshold_alert_enabled": false, "aggregate_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "aggregate_thresholds_alert_enabled": false, "aggregate_thresholds_custom_alert_enabled": false, "aggregate_thresholds_custom_alert_rules": [], "alert_eval": null, "alert_lag": "30", "alert_on": "both", "alert_period": "5", "anomaly_detection_alerting_enabled": false, "anomaly_detection_is_enabled": false, "anomaly_detection_sensitivity": null, "anomaly_detection_training_window": null, "backfill_earliest_time": "-7d", "backfill_enabled": false, "base_search": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors", "base_search_id": "da-itsi-cp-splunk-observability-sim-kubernetes", "base_search_metric": "memory-utilization", "cohesive_ad": { "sensitivity": 8 }, "cohesive_anomaly_detection_is_enabled": false, "datamodel": { "datamodel": "", "field": "", "object": "", "owner_field": "" }, "datamodel_filter": [], "datamodel_filter_clauses": null, "description": "Pod Memory Utilization by %", "enabled": true, "entity_filter_field": "host", "entity_split_field": "k8s.pod.name", "entity_statop": "avg", "entity_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 260.69268041204964, "render_boundary_min": 0.0, "threshold_levels": [] }, "fill_gaps": "null_value", "gap_custom_alert_value": 0.0, "gap_severity": "unknown", "gap_severity_color": "#CCCCCC", "gap_severity_color_light": "#EEEEEE", "gap_severity_value": "-1", "is_filter_entities_to_service": false, "is_split_by_entity": true, "key": "da-itsi-cp-splunk-observability-19dab576ab91ba1ffcc023c0", "kpi_base_search": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors", "kpi_template_kpi_id": "", "kpi_threshold_template_id": "", "metric_qualifier": "", "metric_search_spec": { "metric_index": "", "metric_name": "" }, "search": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_entity(avg, k8s_memory_utilization, \"k8s.pod.name\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(k8s.pod.name, sec_grp)` | eval serviceid = \"da-itsi-cp-splunk-observability-kubernetes-pods\" | `aggregate_entity_into_service(avg)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-19dab576ab91ba1ffcc023c0, true, true, true)` | eval kpi=\"Memory Utilization\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-splunk-observability-kubernetes-pods\" | `assess_urgency` | `gettime`", "search_aggregate": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_single_value(avg, avg, k8s_memory_utilization, \"k8s.pod.name\", 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-19dab576ab91ba1ffcc023c0)`", "search_alert": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_entity(avg, k8s_memory_utilization, \"k8s.pod.name\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(k8s.pod.name, sec_grp)` | eval serviceid = \"da-itsi-cp-splunk-observability-kubernetes-pods\" | `aggregate_entity_into_service(avg)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-19dab576ab91ba1ffcc023c0, true, true, true)` | eval kpi=\"Memory Utilization\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-splunk-observability-kubernetes-pods\" | `assess_urgency` | `gettime`", "search_alert_earliest": "15", "search_alert_entities": "", "search_buckets": "", "search_entities": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_single_value(avg, avg, k8s_memory_utilization, \"k8s.pod.name\", 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-19dab576ab91ba1ffcc023c0)`", "search_occurrences": 1.0, "search_time_compare": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_and_compare(avg, avg, k8s_memory_utilization, \"k8s.pod.name\", 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-19dab576ab91ba1ffcc023c0)`", "search_time_series": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_entity_time_series(avg, k8s_memory_utilization, \"k8s.pod.name\", 15)` | `aggregate_entity_into_service_time_series(avg, 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-19dab576ab91ba1ffcc023c0)`", "search_time_series_aggregate": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_entity_time_series(avg, k8s_memory_utilization, \"k8s.pod.name\", 15)` | `aggregate_entity_into_service_time_series(avg, 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-19dab576ab91ba1ffcc023c0)`", "search_time_series_entities": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_limited_entity_time_series(avg, k8s_memory_utilization, \"k8s.pod.name\", 15)`", "search_type": "shared_base", "service_title": "Kubernetes Pods", "threshold_eval": null, "threshold_field": "k8s_memory_utilization", "time_policies": { "policies": { "default_policy": { "aggregate_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "entity_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "policy_type": "static", "time_blocks": [], "title": "Default" } } }, "title": "Memory Utilization", "trending_ad": { "sensitivity": 8 }, "type": "kpis_primary", "tz_offset": null, "unit": "%", "urgency": 5.0, "use_time_policies": false }, { "adaptive_thresholding_training_window": "-7d", "adaptive_thresholds_is_enabled": false, "aggregate_eval": "", "aggregate_statop": "avg", "aggregate_threshold_alert_enabled": false, "aggregate_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "aggregate_thresholds_alert_enabled": false, "aggregate_thresholds_custom_alert_enabled": false, "aggregate_thresholds_custom_alert_rules": [], "alert_eval": "", "alert_lag": "30", "alert_on": "both", "alert_period": "5", "anomaly_detection_alerting_enabled": false, "anomaly_detection_is_enabled": false, "anomaly_detection_sensitivity": null, "anomaly_detection_training_window": null, "backfill_earliest_time": "-7d", "backfill_enabled": false, "base_search": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors", "base_search_id": "da-itsi-cp-splunk-observability-sim-kubernetes", "base_search_metric": "network-rx-errors", "cohesive_ad": { "sensitivity": 8 }, "cohesive_anomaly_detection_is_enabled": false, "datamodel": { "datamodel": "", "field": "", "object": "", "owner_field": "" }, "datamodel_filter": [], "datamodel_filter_clauses": null, "description": "Pod Network Receive Errors", "enabled": true, "entity_filter_field": "host", "entity_split_field": "k8s.pod.name", "entity_statop": "avg", "entity_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "fill_gaps": "null_value", "gap_custom_alert_value": 0.0, "gap_severity": "unknown", "gap_severity_color": "#CCCCCC", "gap_severity_color_light": "#EEEEEE", "gap_severity_value": "-1", "is_filter_entities_to_service": false, "is_split_by_entity": true, "key": "da-itsi-cp-splunk-observability-1c175b66337825eab27f5fba", "kpi_base_search": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors", "kpi_template_kpi_id": "", "kpi_threshold_template_id": "", "metric_qualifier": "", "metric_search_spec": { "metric_index": "", "metric_name": "" }, "search": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_entity(avg, k8s_network_rx_errors, \"k8s.pod.name\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(k8s.pod.name, sec_grp)` | eval serviceid = \"da-itsi-cp-splunk-observability-kubernetes-pods\" | `aggregate_entity_into_service(avg)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-1c175b66337825eab27f5fba, true, true, true)` | eval kpi=\"Network Rx Errors\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-splunk-observability-kubernetes-pods\" | `assess_urgency` | `gettime`", "search_aggregate": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_single_value(avg, avg, k8s_network_rx_errors, \"k8s.pod.name\", 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-1c175b66337825eab27f5fba)`", "search_alert": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_entity(avg, k8s_network_rx_errors, \"k8s.pod.name\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(k8s.pod.name, sec_grp)` | eval serviceid = \"da-itsi-cp-splunk-observability-kubernetes-pods\" | `aggregate_entity_into_service(avg)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-1c175b66337825eab27f5fba, true, true, true)` | eval kpi=\"Network Rx Errors\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-splunk-observability-kubernetes-pods\" | `assess_urgency` | `gettime`", "search_alert_earliest": "15", "search_alert_entities": "", "search_buckets": "", "search_entities": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_single_value(avg, avg, k8s_network_rx_errors, \"k8s.pod.name\", 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-1c175b66337825eab27f5fba)`", "search_occurrences": 1.0, "search_time_compare": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_and_compare(avg, avg, k8s_network_rx_errors, \"k8s.pod.name\", 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-1c175b66337825eab27f5fba)`", "search_time_series": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_entity_time_series(avg, k8s_network_rx_errors, \"k8s.pod.name\", 15)` | `aggregate_entity_into_service_time_series(avg, 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-1c175b66337825eab27f5fba)`", "search_time_series_aggregate": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_entity_time_series(avg, k8s_network_rx_errors, \"k8s.pod.name\", 15)` | `aggregate_entity_into_service_time_series(avg, 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-1c175b66337825eab27f5fba)`", "search_time_series_entities": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_limited_entity_time_series(avg, k8s_network_rx_errors, \"k8s.pod.name\", 15)`", "search_type": "shared_base", "service_title": "Kubernetes Pods", "threshold_eval": "", "threshold_field": "k8s_network_rx_errors", "time_policies": { "policies": { "default_policy": { "aggregate_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "entity_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "policy_type": "static", "time_blocks": [], "title": "Default" } } }, "title": "Network Rx Errors", "trending_ad": { "sensitivity": 8 }, "type": "kpis_primary", "tz_offset": null, "unit": "", "urgency": 5.0, "use_time_policies": false }, { "adaptive_thresholding_training_window": "-7d", "adaptive_thresholds_is_enabled": false, "aggregate_eval": "", "aggregate_statop": "avg", "aggregate_threshold_alert_enabled": false, "aggregate_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "aggregate_thresholds_alert_enabled": false, "aggregate_thresholds_custom_alert_enabled": false, "aggregate_thresholds_custom_alert_rules": [], "alert_eval": "", "alert_lag": "30", "alert_on": "both", "alert_period": "5", "anomaly_detection_alerting_enabled": false, "anomaly_detection_is_enabled": false, "anomaly_detection_sensitivity": null, "anomaly_detection_training_window": null, "backfill_earliest_time": "-7d", "backfill_enabled": false, "base_search": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors", "base_search_id": "da-itsi-cp-splunk-observability-sim-kubernetes", "base_search_metric": "network-tx-errors", "cohesive_ad": { "sensitivity": 8 }, "cohesive_anomaly_detection_is_enabled": false, "datamodel": { "datamodel": "", "field": "", "object": "", "owner_field": "" }, "datamodel_filter": [], "datamodel_filter_clauses": null, "description": "Pod Network Transmit Errors", "enabled": true, "entity_filter_field": "host", "entity_split_field": "k8s.pod.name", "entity_statop": "avg", "entity_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "fill_gaps": "null_value", "gap_custom_alert_value": 0.0, "gap_severity": "unknown", "gap_severity_color": "#CCCCCC", "gap_severity_color_light": "#EEEEEE", "gap_severity_value": "-1", "is_filter_entities_to_service": false, "is_split_by_entity": true, "key": "da-itsi-cp-splunk-observability-d81b5b3f3fca0bcf50733969", "kpi_base_search": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors", "kpi_template_kpi_id": "", "kpi_threshold_template_id": "", "metric_qualifier": "", "metric_search_spec": { "metric_index": "", "metric_name": "" }, "search": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_entity(avg, k8s_network_tx_errors, \"k8s.pod.name\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(k8s.pod.name, sec_grp)` | eval serviceid = \"da-itsi-cp-splunk-observability-kubernetes-pods\" | `aggregate_entity_into_service(avg)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-d81b5b3f3fca0bcf50733969, true, true, true)` | eval kpi=\"Network Tx Errors\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-splunk-observability-kubernetes-pods\" | `assess_urgency` | `gettime`", "search_aggregate": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_single_value(avg, avg, k8s_network_tx_errors, \"k8s.pod.name\", 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-d81b5b3f3fca0bcf50733969)`", "search_alert": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_entity(avg, k8s_network_tx_errors, \"k8s.pod.name\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(k8s.pod.name, sec_grp)` | eval serviceid = \"da-itsi-cp-splunk-observability-kubernetes-pods\" | `aggregate_entity_into_service(avg)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-d81b5b3f3fca0bcf50733969, true, true, true)` | eval kpi=\"Network Tx Errors\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-splunk-observability-kubernetes-pods\" | `assess_urgency` | `gettime`", "search_alert_earliest": "15", "search_alert_entities": "", "search_buckets": "", "search_entities": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_single_value(avg, avg, k8s_network_tx_errors, \"k8s.pod.name\", 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-d81b5b3f3fca0bcf50733969)`", "search_occurrences": 1.0, "search_time_compare": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_and_compare(avg, avg, k8s_network_tx_errors, \"k8s.pod.name\", 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-d81b5b3f3fca0bcf50733969)`", "search_time_series": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_entity_time_series(avg, k8s_network_tx_errors, \"k8s.pod.name\", 15)` | `aggregate_entity_into_service_time_series(avg, 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-d81b5b3f3fca0bcf50733969)`", "search_time_series_aggregate": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_entity_time_series(avg, k8s_network_tx_errors, \"k8s.pod.name\", 15)` | `aggregate_entity_into_service_time_series(avg, 15)` | `assess_severity(da-itsi-cp-splunk-observability-kubernetes-pods, da-itsi-cp-splunk-observability-d81b5b3f3fca0bcf50733969)`", "search_time_series_entities": "| mstats avg(\"container_cpu_utilization\") as k8s_cpu_utilization, avg(\"pod_network_receive_errors_total\") as k8s_network_rx_errors, avg(\"pod_network_transmit_errors_total\") as k8s_network_tx_errors, sum(\"container.memory.usage\") as k8s_memory_usage, sum(\"kubernetes.container_memory_limit\") as k8s_memory_limit, where `itsi-cp-observability-indexes` AND (cluster=* OR kubernetes_cluster=*) AND k8s.pod.name=* by kubernetes_cluster, k8s.pod.name span=15m | eval k8s_memory_utilization = (k8s_memory_usage / k8s_memory_limit)*100 | table _time, kubernetes_cluster, k8s.pod.name, k8s_cpu_utilization, k8s_memory_usage, k8s_memory_limit, k8s_memory_utilization, k8s_network_rx_errors, k8s_network_tx_errors | `aggregate_raw_into_limited_entity_time_series(avg, k8s_network_tx_errors, \"k8s.pod.name\", 15)`", "search_type": "shared_base", "service_title": "Kubernetes Pods", "threshold_eval": "", "threshold_field": "k8s_network_tx_errors", "time_policies": { "policies": { "default_policy": { "aggregate_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "entity_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "policy_type": "static", "time_blocks": [], "title": "Default" } } }, "title": "Network Tx Errors", "trending_ad": { "sensitivity": 8 }, "type": "kpis_primary", "tz_offset": null, "unit": "", "urgency": 5.0, "use_time_policies": false } ], "service_tags": { "tags": [], "template_tags": [] }, "service_template_id": "", "services_depending_on_me": [ { "kpis_depending_on": [ "SHKPI-da-itsi-cp-splunk-observability-kubernetes-pods" ], "service_id": "da-itsi-cp-splunk-observability-my-data-center-hosts" } ], "services_depends_on": [], "team_id": "default_itsi_security_group", "title": "Kubernetes Pods", "version": "0.0.38" }