{ "algorithms": { "GradientBoostingRegressor": { "RMSE": 0, "modelId": "", "rSquared": 0, "recommended": false }, "LinearRegression": { "RMSE": 0, "modelId": "", "rSquared": 0, "recommended": false }, "LogisticRegression": { "accuracy": 0, "f1_score": 0, "modelId": "", "precision": 0, "recall": 0, "recommended": false }, "RandomForestRegressor": { "RMSE": 0, "modelId": "", "rSquared": 0, "recommended": false } }, "description": "", "enabled": true, "entity_rules": [ { "rule_condition": "AND", "rule_items": [ { "field": "splunk_role", "field_type": "info", "rule_type": "matches", "value": "indexer" } ] } ], "key": "da-itsi-cp-monitoring-splunk-splunk-indexer-queues", "kpis": [ { "adaptive_thresholding_training_window": "-7d", "adaptive_thresholds_is_enabled": false, "aggregate_eval": "", "aggregate_statop": "avg", "aggregate_threshold_alert_enabled": false, "aggregate_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [ { "dynamic_param": null, "severity_color": "#B50101", "severity_color_light": "#E5A6A6", "severity_label": "critical", "severity_label_localized": null, "severity_value": 6.0, "threshold_value": 0.0 }, { "dynamic_param": null, "severity_color": "#F26A35", "severity_color_light": "#FBCBB9", "severity_label": "high", "severity_label_localized": null, "severity_value": 5.0, "threshold_value": 20.0 }, { "dynamic_param": null, "severity_color": "#FCB64E", "severity_color_light": "#FEE6C1", "severity_label": "medium", "severity_label_localized": null, "severity_value": 4.0, "threshold_value": 40.0 }, { "dynamic_param": null, "severity_color": "#FFE98C", "severity_color_light": "#FFF4C5", "severity_label": "low", "severity_label_localized": null, "severity_value": 3.0, "threshold_value": 60.0 }, { "dynamic_param": null, "severity_color": "#99D18B", "severity_color_light": "#DCEFD7", "severity_label": "normal", "severity_label_localized": null, "severity_value": 2.0, "threshold_value": 80.0 } ] }, "aggregate_thresholds_alert_enabled": false, "aggregate_thresholds_custom_alert_enabled": false, "aggregate_thresholds_custom_alert_rules": [], "alert_eval": "", "alert_lag": "30", "alert_on": "both", "alert_period": "1", "anomaly_detection_alerting_enabled": false, "anomaly_detection_is_enabled": false, "anomaly_detection_sensitivity": 0.999, "anomaly_detection_training_window": "-7d", "backfill_earliest_time": "-7d", "backfill_enabled": false, "base_search": "`get_full_itsi_summary_service_health_events(da-itsi-cp-monitoring-splunk-splunk-indexer-queues)`", "base_search_id": null, "base_search_metric": null, "cohesive_ad": { "sensitivity": 8 }, "cohesive_anomaly_detection_is_enabled": false, "datamodel": { "datamodel": "", "field": "", "object": "", "owner_field": "" }, "datamodel_filter": [], "datamodel_filter_clauses": null, "description": "", "enabled": false, "entity_filter_field": "", "entity_split_field": "", "entity_statop": "avg", "entity_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [ { "dynamic_param": null, "severity_color": "#B50101", "severity_color_light": "#E5A6A6", "severity_label": "critical", "severity_label_localized": null, "severity_value": 6.0, "threshold_value": 0.0 }, { "dynamic_param": null, "severity_color": "#F26A35", "severity_color_light": "#FBCBB9", "severity_label": "high", "severity_label_localized": null, "severity_value": 5.0, "threshold_value": 20.0 }, { "dynamic_param": null, "severity_color": "#FCB64E", "severity_color_light": "#FEE6C1", "severity_label": "medium", "severity_label_localized": null, "severity_value": 4.0, "threshold_value": 40.0 }, { "dynamic_param": null, "severity_color": "#FFE98C", "severity_color_light": "#FFF4C5", "severity_label": "low", "severity_label_localized": null, "severity_value": 3.0, "threshold_value": 60.0 }, { "dynamic_param": null, "severity_color": "#99D18B", "severity_color_light": "#DCEFD7", "severity_label": "normal", "severity_label_localized": null, "severity_value": 2.0, "threshold_value": 80.0 } ] }, "fill_gaps": "null_value", "gap_custom_alert_value": 0.0, "gap_severity": "unknown", "gap_severity_color": "#CCCCCC", "gap_severity_color_light": "#EEEEEE", "gap_severity_value": "-1", "is_filter_entities_to_service": false, "is_split_by_entity": false, "key": "SHKPI-da-itsi-cp-monitoring-splunk-splunk-indexer-queues", "kpi_base_search": "", "kpi_template_kpi_id": "", "kpi_threshold_template_id": "", "metric_qualifier": null, "metric_search_spec": { "metric_index": "", "metric_name": "" }, "search": "`get_full_itsi_summary_service_health_events(da-itsi-cp-monitoring-splunk-splunk-indexer-queues)` | stats latest(health_score) AS aggregate", "search_aggregate": "`get_full_itsi_summary_service_health_events(da-itsi-cp-monitoring-splunk-splunk-indexer-queues)` | stats latest(health_score) AS aggregate", "search_alert": "", "search_alert_earliest": "15", "search_alert_entities": "", "search_buckets": "", "search_entities": null, "search_occurrences": 1.0, "search_time_compare": "`get_full_itsi_summary_service_health_events(da-itsi-cp-monitoring-splunk-splunk-indexer-queues)` [| stats count | addinfo | eval search= \"earliest=\" + tostring(info_min_time-(info_max_time-info_min_time))+ \" latest=\" + tostring(info_max_time) |fields search] | addinfo | eval bucket=if(_time0, \"increase\", if(window_delta < 0, \"decrease\", \"none\"))", "search_time_series": "`get_full_itsi_summary_service_health_events(da-itsi-cp-monitoring-splunk-splunk-indexer-queues)` | timechart avg(health_score) AS aggregate", "search_time_series_aggregate": "`get_full_itsi_summary_service_health_events(da-itsi-cp-monitoring-splunk-splunk-indexer-queues)` | timechart avg(health_score) AS aggregate", "search_time_series_entities": "", "search_type": "adhoc", "service_title": "Splunk Indexer Queues", "threshold_eval": "", "threshold_field": "aggregate", "time_policies": { "policies": { "default_policy": { "aggregate_thresholds": { "base_severity_color": "#AED3E5", "base_severity_color_light": "#E3F0F6", "base_severity_label": "info", "base_severity_value": 1.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "entity_thresholds": { "base_severity_color": "#AED3E5", "base_severity_color_light": "#E3F0F6", "base_severity_label": "info", "base_severity_value": 1.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "policy_type": "static", "time_blocks": [], "title": "Default" } } }, "title": "ServiceHealthScore", "trending_ad": { "sensitivity": 8 }, "type": "service_health", "tz_offset": null, "unit": "", "urgency": 11.0, "use_time_policies": false }, { "adaptive_thresholding_training_window": "-7d", "adaptive_thresholds_is_enabled": false, "aggregate_eval": "", "aggregate_statop": "avg", "aggregate_threshold_alert_enabled": false, "aggregate_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 110, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 110.0, "render_boundary_min": 0.0, "threshold_levels": [ { "dynamic_param": 0.0, "severity_color": "#FFE98C", "severity_color_light": "#FFF4C5", "severity_label": "low", "severity_label_localized": null, "severity_value": 3.0, "threshold_value": 75.0 }, { "dynamic_param": 0.0, "severity_color": "#FCB64E", "severity_color_light": "#FEE6C1", "severity_label": "medium", "severity_label_localized": null, "severity_value": 4.0, "threshold_value": 90.0 }, { "dynamic_param": 0.0, "severity_color": "#F26A35", "severity_color_light": "#FBCBB9", "severity_label": "high", "severity_label_localized": null, "severity_value": 5.0, "threshold_value": 95.0 }, { "dynamic_param": 0.0, "severity_color": "#B50101", "severity_color_light": "#E5A6A6", "severity_label": "critical", "severity_label_localized": null, "severity_value": 6.0, "threshold_value": 100.0 } ] }, "aggregate_thresholds_alert_enabled": false, "aggregate_thresholds_custom_alert_enabled": false, "aggregate_thresholds_custom_alert_rules": [], "alert_eval": "", "alert_lag": "5", "alert_on": "both", "alert_period": "5", "anomaly_detection_alerting_enabled": false, "anomaly_detection_is_enabled": false, "anomaly_detection_sensitivity": null, "anomaly_detection_training_window": null, "backfill_earliest_time": "-7d", "backfill_enabled": false, "base_search": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc)", "base_search_id": "da-itsi-cp-monitoring-splunk-splk-indexer-queues", "base_search_metric": "aggregate-queue-fill", "cohesive_ad": { "sensitivity": 8 }, "cohesive_anomaly_detection_is_enabled": false, "datamodel": { "datamodel": "", "field": "", "object": "", "owner_field": "" }, "datamodel_filter": [], "datamodel_filter_clauses": null, "description": "", "enabled": true, "entity_filter_field": "host", "entity_split_field": "host", "entity_statop": "avg", "entity_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 110, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 110.0, "render_boundary_min": 0.0, "threshold_levels": [ { "dynamic_param": 0.0, "severity_color": "#FFE98C", "severity_color_light": "#FFF4C5", "severity_label": "low", "severity_label_localized": null, "severity_value": 3.0, "threshold_value": 75.0 }, { "dynamic_param": 0.0, "severity_color": "#FCB64E", "severity_color_light": "#FEE6C1", "severity_label": "medium", "severity_label_localized": null, "severity_value": 4.0, "threshold_value": 90.0 }, { "dynamic_param": 0.0, "severity_color": "#F26A35", "severity_color_light": "#FBCBB9", "severity_label": "high", "severity_label_localized": null, "severity_value": 5.0, "threshold_value": 95.0 }, { "dynamic_param": 0.0, "severity_color": "#B50101", "severity_color_light": "#E5A6A6", "severity_label": "critical", "severity_label_localized": null, "severity_value": 6.0, "threshold_value": 100.0 } ] }, "fill_gaps": "null_value", "gap_custom_alert_value": 0.0, "gap_severity": "unknown", "gap_severity_color": "#CCCCCC", "gap_severity_color_light": "#EEEEEE", "gap_severity_value": "-1", "is_filter_entities_to_service": true, "is_split_by_entity": true, "key": "da-itsi-cp-monitoring-splunk-b28b6b95aa2ca65c550b2a84", "kpi_base_search": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc) | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-b28b6b95aa2ca65c550b2a84 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ]", "kpi_template_kpi_id": "", "kpi_threshold_template_id": "", "metric_qualifier": "", "metric_search_spec": { "metric_index": "", "metric_name": "" }, "search": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc) | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-b28b6b95aa2ca65c550b2a84 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity(avg, agg_fill_perc, \"host\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(host, sec_grp)` | eval serviceid = \"da-itsi-cp-monitoring-splunk-splunk-indexer-queues\" | `aggregate_entity_into_service(avg)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-b28b6b95aa2ca65c550b2a84, true, true, true)` | eval kpi=\"Aggregate Queue Fill\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-monitoring-splunk-splunk-indexer-queues\" | `assess_urgency` | `gettime`", "search_aggregate": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc) | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-b28b6b95aa2ca65c550b2a84 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_single_value(avg, avg, agg_fill_perc, \"host\", 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-b28b6b95aa2ca65c550b2a84)`", "search_alert": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc) | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-b28b6b95aa2ca65c550b2a84 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity(avg, agg_fill_perc, \"host\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(host, sec_grp)` | eval serviceid = \"da-itsi-cp-monitoring-splunk-splunk-indexer-queues\" | `aggregate_entity_into_service(avg)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-b28b6b95aa2ca65c550b2a84, true, true, true)` | eval kpi=\"Aggregate Queue Fill\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-monitoring-splunk-splunk-indexer-queues\" | `assess_urgency` | `gettime`", "search_alert_earliest": "5", "search_alert_entities": "", "search_buckets": "", "search_entities": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc) | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-b28b6b95aa2ca65c550b2a84 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_single_value(avg, avg, agg_fill_perc, \"host\", 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-b28b6b95aa2ca65c550b2a84)`", "search_occurrences": 1.0, "search_time_compare": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc) | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-b28b6b95aa2ca65c550b2a84 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_and_compare(avg, avg, agg_fill_perc, \"host\", 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-b28b6b95aa2ca65c550b2a84)`", "search_time_series": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc) | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-b28b6b95aa2ca65c550b2a84 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity_time_series(avg, agg_fill_perc, \"host\", 5)` | `aggregate_entity_into_service_time_series(avg, 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-b28b6b95aa2ca65c550b2a84)`", "search_time_series_aggregate": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc) | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-b28b6b95aa2ca65c550b2a84 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity_time_series(avg, agg_fill_perc, \"host\", 5)` | `aggregate_entity_into_service_time_series(avg, 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-b28b6b95aa2ca65c550b2a84)`", "search_time_series_entities": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc) | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-b28b6b95aa2ca65c550b2a84 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_limited_entity_time_series(avg, agg_fill_perc, \"host\", 5)`", "search_type": "shared_base", "service_title": "Splunk Indexer Queues", "threshold_eval": "", "threshold_field": "agg_fill_perc", "time_policies": { "policies": { "default_policy": { "aggregate_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "entity_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "policy_type": "static", "time_blocks": [], "title": "Default" } } }, "title": "Aggregate Queue Fill", "trending_ad": { "sensitivity": 8 }, "type": "kpis_primary", "tz_offset": null, "unit": "%", "urgency": 5.0, "use_time_policies": false }, { "adaptive_thresholding_training_window": "-7d", "adaptive_thresholds_is_enabled": false, "aggregate_eval": "", "aggregate_statop": "latest", "aggregate_threshold_alert_enabled": false, "aggregate_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "aggregate_thresholds_alert_enabled": false, "aggregate_thresholds_custom_alert_enabled": false, "aggregate_thresholds_custom_alert_rules": [], "alert_eval": "", "alert_lag": "5", "alert_on": "both", "alert_period": "5", "anomaly_detection_alerting_enabled": false, "anomaly_detection_is_enabled": false, "anomaly_detection_sensitivity": null, "anomaly_detection_training_window": null, "backfill_earliest_time": "-7d", "backfill_enabled": false, "base_search": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host", "base_search_id": "da-itsi-cp-monitoring-splunk-splk-indexer-search-metrics", "base_search_metric": "queue-size-current", "cohesive_ad": { "sensitivity": 8 }, "cohesive_anomaly_detection_is_enabled": false, "datamodel": { "datamodel": "", "field": "", "object": "", "owner_field": "" }, "datamodel_filter": [], "datamodel_filter_clauses": null, "description": "", "enabled": true, "entity_filter_field": "host", "entity_split_field": "host", "entity_statop": "latest", "entity_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "fill_gaps": "null_value", "gap_custom_alert_value": 0.0, "gap_severity": "unknown", "gap_severity_color": "#CCCCCC", "gap_severity_color_light": "#EEEEEE", "gap_severity_value": "-1", "is_filter_entities_to_service": true, "is_split_by_entity": true, "key": "da-itsi-cp-monitoring-splunk-e690a0b213b5130c2fa1c375", "kpi_base_search": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-e690a0b213b5130c2fa1c375 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ]", "kpi_template_kpi_id": "", "kpi_threshold_template_id": "", "metric_qualifier": "", "metric_search_spec": { "metric_index": "", "metric_name": "" }, "search": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-e690a0b213b5130c2fa1c375 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity(latest, current_queue_size, \"host\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(host, sec_grp)` | eval serviceid = \"da-itsi-cp-monitoring-splunk-splunk-indexer-queues\" | `aggregate_entity_into_service(latest)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-e690a0b213b5130c2fa1c375, true, true, true)` | eval kpi=\"Current Queue Size\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-monitoring-splunk-splunk-indexer-queues\" | `assess_urgency` | `gettime`", "search_aggregate": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-e690a0b213b5130c2fa1c375 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_single_value(latest, latest, current_queue_size, \"host\", 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-e690a0b213b5130c2fa1c375)`", "search_alert": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-e690a0b213b5130c2fa1c375 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity(latest, current_queue_size, \"host\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(host, sec_grp)` | eval serviceid = \"da-itsi-cp-monitoring-splunk-splunk-indexer-queues\" | `aggregate_entity_into_service(latest)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-e690a0b213b5130c2fa1c375, true, true, true)` | eval kpi=\"Current Queue Size\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-monitoring-splunk-splunk-indexer-queues\" | `assess_urgency` | `gettime`", "search_alert_earliest": "5", "search_alert_entities": "", "search_buckets": "", "search_entities": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-e690a0b213b5130c2fa1c375 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_single_value(latest, latest, current_queue_size, \"host\", 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-e690a0b213b5130c2fa1c375)`", "search_occurrences": 1.0, "search_time_compare": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-e690a0b213b5130c2fa1c375 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_and_compare(latest, latest, current_queue_size, \"host\", 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-e690a0b213b5130c2fa1c375)`", "search_time_series": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-e690a0b213b5130c2fa1c375 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity_time_series(latest, current_queue_size, \"host\", 5)` | `aggregate_entity_into_service_time_series(latest, 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-e690a0b213b5130c2fa1c375)`", "search_time_series_aggregate": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-e690a0b213b5130c2fa1c375 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity_time_series(latest, current_queue_size, \"host\", 5)` | `aggregate_entity_into_service_time_series(latest, 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-e690a0b213b5130c2fa1c375)`", "search_time_series_entities": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-e690a0b213b5130c2fa1c375 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_limited_entity_time_series(latest, current_queue_size, \"host\", 5)`", "search_type": "shared_base", "service_title": "Splunk Indexer Queues", "threshold_eval": "", "threshold_field": "current_queue_size", "time_policies": { "policies": { "default_policy": { "aggregate_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "entity_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "policy_type": "static", "time_blocks": [], "title": "Default" } } }, "title": "Current Queue Size", "trending_ad": { "sensitivity": 8 }, "type": "kpis_primary", "tz_offset": null, "unit": "", "urgency": 5.0, "use_time_policies": false }, { "adaptive_thresholding_training_window": "-7d", "adaptive_thresholds_is_enabled": false, "aggregate_eval": "", "aggregate_statop": "avg", "aggregate_threshold_alert_enabled": false, "aggregate_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 110, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 110.0, "render_boundary_min": 0.0, "threshold_levels": [ { "dynamic_param": 0.0, "severity_color": "#FFE98C", "severity_color_light": "#FFF4C5", "severity_label": "low", "severity_label_localized": null, "severity_value": 3.0, "threshold_value": 75.0 }, { "dynamic_param": 0.0, "severity_color": "#FCB64E", "severity_color_light": "#FEE6C1", "severity_label": "medium", "severity_label_localized": null, "severity_value": 4.0, "threshold_value": 90.0 }, { "dynamic_param": 0.0, "severity_color": "#F26A35", "severity_color_light": "#FBCBB9", "severity_label": "high", "severity_label_localized": null, "severity_value": 5.0, "threshold_value": 95.0 }, { "dynamic_param": 0.0, "severity_color": "#B50101", "severity_color_light": "#E5A6A6", "severity_label": "critical", "severity_label_localized": null, "severity_value": 6.0, "threshold_value": 100.0 } ] }, "aggregate_thresholds_alert_enabled": false, "aggregate_thresholds_custom_alert_enabled": false, "aggregate_thresholds_custom_alert_rules": [], "alert_eval": "", "alert_lag": "5", "alert_on": "both", "alert_period": "5", "anomaly_detection_alerting_enabled": false, "anomaly_detection_is_enabled": false, "anomaly_detection_sensitivity": null, "anomaly_detection_training_window": null, "backfill_earliest_time": "-7d", "backfill_enabled": false, "base_search": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc)", "base_search_id": "da-itsi-cp-monitoring-splunk-splk-indexer-queues", "base_search_metric": "index-queue-fill", "cohesive_ad": { "sensitivity": 8 }, "cohesive_anomaly_detection_is_enabled": false, "datamodel": { "datamodel": "", "field": "", "object": "", "owner_field": "" }, "datamodel_filter": [], "datamodel_filter_clauses": null, "description": "", "enabled": true, "entity_filter_field": "host", "entity_split_field": "host", "entity_statop": "avg", "entity_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 110, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 110.0, "render_boundary_min": 0.0, "threshold_levels": [ { "dynamic_param": 0.0, "severity_color": "#FFE98C", "severity_color_light": "#FFF4C5", "severity_label": "low", "severity_label_localized": null, "severity_value": 3.0, "threshold_value": 75.0 }, { "dynamic_param": 0.0, "severity_color": "#FCB64E", "severity_color_light": "#FEE6C1", "severity_label": "medium", "severity_label_localized": null, "severity_value": 4.0, "threshold_value": 90.0 }, { "dynamic_param": 0.0, "severity_color": "#F26A35", "severity_color_light": "#FBCBB9", "severity_label": "high", "severity_label_localized": null, "severity_value": 5.0, "threshold_value": 95.0 }, { "dynamic_param": 0.0, "severity_color": "#B50101", "severity_color_light": "#E5A6A6", "severity_label": "critical", "severity_label_localized": null, "severity_value": 6.0, "threshold_value": 100.0 } ] }, "fill_gaps": "null_value", "gap_custom_alert_value": 0.0, "gap_severity": "unknown", "gap_severity_color": "#CCCCCC", "gap_severity_color_light": "#EEEEEE", "gap_severity_value": "-1", "is_filter_entities_to_service": true, "is_split_by_entity": true, "key": "da-itsi-cp-monitoring-splunk-f0ae5af9405e272ba61779c5", "kpi_base_search": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc) | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-f0ae5af9405e272ba61779c5 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ]", "kpi_template_kpi_id": "", "kpi_threshold_template_id": "", "metric_qualifier": "", "metric_search_spec": { "metric_index": "", "metric_name": "" }, "search": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc) | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-f0ae5af9405e272ba61779c5 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity(avg, index_fill_perc, \"host\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(host, sec_grp)` | eval serviceid = \"da-itsi-cp-monitoring-splunk-splunk-indexer-queues\" | `aggregate_entity_into_service(avg)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-f0ae5af9405e272ba61779c5, true, true, true)` | eval kpi=\"Indexing Queue Fill\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-monitoring-splunk-splunk-indexer-queues\" | `assess_urgency` | `gettime`", "search_aggregate": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc) | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-f0ae5af9405e272ba61779c5 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_single_value(avg, avg, index_fill_perc, \"host\", 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-f0ae5af9405e272ba61779c5)`", "search_alert": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc) | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-f0ae5af9405e272ba61779c5 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity(avg, index_fill_perc, \"host\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(host, sec_grp)` | eval serviceid = \"da-itsi-cp-monitoring-splunk-splunk-indexer-queues\" | `aggregate_entity_into_service(avg)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-f0ae5af9405e272ba61779c5, true, true, true)` | eval kpi=\"Indexing Queue Fill\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-monitoring-splunk-splunk-indexer-queues\" | `assess_urgency` | `gettime`", "search_alert_earliest": "5", "search_alert_entities": "", "search_buckets": "", "search_entities": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc) | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-f0ae5af9405e272ba61779c5 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_single_value(avg, avg, index_fill_perc, \"host\", 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-f0ae5af9405e272ba61779c5)`", "search_occurrences": 1.0, "search_time_compare": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc) | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-f0ae5af9405e272ba61779c5 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_and_compare(avg, avg, index_fill_perc, \"host\", 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-f0ae5af9405e272ba61779c5)`", "search_time_series": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc) | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-f0ae5af9405e272ba61779c5 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity_time_series(avg, index_fill_perc, \"host\", 5)` | `aggregate_entity_into_service_time_series(avg, 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-f0ae5af9405e272ba61779c5)`", "search_time_series_aggregate": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc) | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-f0ae5af9405e272ba61779c5 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity_time_series(avg, index_fill_perc, \"host\", 5)` | `aggregate_entity_into_service_time_series(avg, 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-f0ae5af9405e272ba61779c5)`", "search_time_series_entities": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc) | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-f0ae5af9405e272ba61779c5 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_limited_entity_time_series(avg, index_fill_perc, \"host\", 5)`", "search_type": "shared_base", "service_title": "Splunk Indexer Queues", "threshold_eval": "", "threshold_field": "index_fill_perc", "time_policies": { "policies": { "default_policy": { "aggregate_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "entity_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "policy_type": "static", "time_blocks": [], "title": "Default" } } }, "title": "Indexing Queue Fill", "trending_ad": { "sensitivity": 8 }, "type": "kpis_primary", "tz_offset": null, "unit": "%", "urgency": 5.0, "use_time_policies": false }, { "adaptive_thresholding_training_window": "-7d", "adaptive_thresholds_is_enabled": false, "aggregate_eval": "", "aggregate_statop": "max", "aggregate_threshold_alert_enabled": false, "aggregate_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "aggregate_thresholds_alert_enabled": false, "aggregate_thresholds_custom_alert_enabled": false, "aggregate_thresholds_custom_alert_rules": [], "alert_eval": "", "alert_lag": "5", "alert_on": "both", "alert_period": "5", "anomaly_detection_alerting_enabled": false, "anomaly_detection_is_enabled": false, "anomaly_detection_sensitivity": null, "anomaly_detection_training_window": null, "backfill_earliest_time": "-7d", "backfill_enabled": false, "base_search": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host", "base_search_id": "da-itsi-cp-monitoring-splunk-splk-indexer-search-metrics", "base_search_metric": "queue-size-max", "cohesive_ad": { "sensitivity": 8 }, "cohesive_anomaly_detection_is_enabled": false, "datamodel": { "datamodel": "", "field": "", "object": "", "owner_field": "" }, "datamodel_filter": [], "datamodel_filter_clauses": null, "description": "", "enabled": true, "entity_filter_field": "host", "entity_split_field": "host", "entity_statop": "max", "entity_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "fill_gaps": "null_value", "gap_custom_alert_value": 0.0, "gap_severity": "unknown", "gap_severity_color": "#CCCCCC", "gap_severity_color_light": "#EEEEEE", "gap_severity_value": "-1", "is_filter_entities_to_service": true, "is_split_by_entity": true, "key": "da-itsi-cp-monitoring-splunk-ec724c8c3bc2f095b7c1dde2", "kpi_base_search": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-ec724c8c3bc2f095b7c1dde2 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ]", "kpi_template_kpi_id": "", "kpi_threshold_template_id": "", "metric_qualifier": "", "metric_search_spec": { "metric_index": "", "metric_name": "" }, "search": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-ec724c8c3bc2f095b7c1dde2 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity(max, largest_queue_size, \"host\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(host, sec_grp)` | eval serviceid = \"da-itsi-cp-monitoring-splunk-splunk-indexer-queues\" | `aggregate_entity_into_service(max)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-ec724c8c3bc2f095b7c1dde2, true, true, true)` | eval kpi=\"Max Queue Size\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-monitoring-splunk-splunk-indexer-queues\" | `assess_urgency` | `gettime`", "search_aggregate": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-ec724c8c3bc2f095b7c1dde2 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_single_value(max, max, largest_queue_size, \"host\", 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-ec724c8c3bc2f095b7c1dde2)`", "search_alert": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-ec724c8c3bc2f095b7c1dde2 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity(max, largest_queue_size, \"host\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(host, sec_grp)` | eval serviceid = \"da-itsi-cp-monitoring-splunk-splunk-indexer-queues\" | `aggregate_entity_into_service(max)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-ec724c8c3bc2f095b7c1dde2, true, true, true)` | eval kpi=\"Max Queue Size\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-monitoring-splunk-splunk-indexer-queues\" | `assess_urgency` | `gettime`", "search_alert_earliest": "5", "search_alert_entities": "", "search_buckets": "", "search_entities": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-ec724c8c3bc2f095b7c1dde2 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_single_value(max, max, largest_queue_size, \"host\", 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-ec724c8c3bc2f095b7c1dde2)`", "search_occurrences": 1.0, "search_time_compare": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-ec724c8c3bc2f095b7c1dde2 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_and_compare(max, max, largest_queue_size, \"host\", 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-ec724c8c3bc2f095b7c1dde2)`", "search_time_series": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-ec724c8c3bc2f095b7c1dde2 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity_time_series(max, largest_queue_size, \"host\", 5)` | `aggregate_entity_into_service_time_series(max, 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-ec724c8c3bc2f095b7c1dde2)`", "search_time_series_aggregate": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-ec724c8c3bc2f095b7c1dde2 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity_time_series(max, largest_queue_size, \"host\", 5)` | `aggregate_entity_into_service_time_series(max, 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-ec724c8c3bc2f095b7c1dde2)`", "search_time_series_entities": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-ec724c8c3bc2f095b7c1dde2 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_limited_entity_time_series(max, largest_queue_size, \"host\", 5)`", "search_type": "shared_base", "service_title": "Splunk Indexer Queues", "threshold_eval": "", "threshold_field": "largest_queue_size", "time_policies": { "policies": { "default_policy": { "aggregate_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "entity_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "policy_type": "static", "time_blocks": [], "title": "Default" } } }, "title": "Max Queue Size", "trending_ad": { "sensitivity": 8 }, "type": "kpis_primary", "tz_offset": null, "unit": "", "urgency": 5.0, "use_time_policies": false }, { "adaptive_thresholding_training_window": "-7d", "adaptive_thresholds_is_enabled": false, "aggregate_eval": "", "aggregate_statop": "avg", "aggregate_threshold_alert_enabled": false, "aggregate_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 110, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 110.0, "render_boundary_min": 0.0, "threshold_levels": [ { "dynamic_param": 0.0, "severity_color": "#FFE98C", "severity_color_light": "#FFF4C5", "severity_label": "low", "severity_label_localized": null, "severity_value": 3.0, "threshold_value": 75.0 }, { "dynamic_param": 0.0, "severity_color": "#FCB64E", "severity_color_light": "#FEE6C1", "severity_label": "medium", "severity_label_localized": null, "severity_value": 4.0, "threshold_value": 90.0 }, { "dynamic_param": 0.0, "severity_color": "#F26A35", "severity_color_light": "#FBCBB9", "severity_label": "high", "severity_label_localized": null, "severity_value": 5.0, "threshold_value": 95.0 }, { "dynamic_param": 0.0, "severity_color": "#B50101", "severity_color_light": "#E5A6A6", "severity_label": "critical", "severity_label_localized": null, "severity_value": 6.0, "threshold_value": 100.0 } ] }, "aggregate_thresholds_alert_enabled": false, "aggregate_thresholds_custom_alert_enabled": false, "aggregate_thresholds_custom_alert_rules": [], "alert_eval": "", "alert_lag": "5", "alert_on": "both", "alert_period": "5", "anomaly_detection_alerting_enabled": false, "anomaly_detection_is_enabled": false, "anomaly_detection_sensitivity": null, "anomaly_detection_training_window": null, "backfill_earliest_time": "-7d", "backfill_enabled": false, "base_search": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc)", "base_search_id": "da-itsi-cp-monitoring-splunk-splk-indexer-queues", "base_search_metric": "parsing-queue-fill", "cohesive_ad": { "sensitivity": 8 }, "cohesive_anomaly_detection_is_enabled": false, "datamodel": { "datamodel": "", "field": "", "object": "", "owner_field": "" }, "datamodel_filter": [], "datamodel_filter_clauses": null, "description": "", "enabled": true, "entity_filter_field": "host", "entity_split_field": "host", "entity_statop": "avg", "entity_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 110, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 110.0, "render_boundary_min": 0.0, "threshold_levels": [ { "dynamic_param": 0.0, "severity_color": "#FFE98C", "severity_color_light": "#FFF4C5", "severity_label": "low", "severity_label_localized": null, "severity_value": 3.0, "threshold_value": 75.0 }, { "dynamic_param": 0.0, "severity_color": "#FCB64E", "severity_color_light": "#FEE6C1", "severity_label": "medium", "severity_label_localized": null, "severity_value": 4.0, "threshold_value": 90.0 }, { "dynamic_param": 0.0, "severity_color": "#F26A35", "severity_color_light": "#FBCBB9", "severity_label": "high", "severity_label_localized": null, "severity_value": 5.0, "threshold_value": 95.0 }, { "dynamic_param": 0.0, "severity_color": "#B50101", "severity_color_light": "#E5A6A6", "severity_label": "critical", "severity_label_localized": null, "severity_value": 6.0, "threshold_value": 100.0 } ] }, "fill_gaps": "null_value", "gap_custom_alert_value": 0.0, "gap_severity": "unknown", "gap_severity_color": "#CCCCCC", "gap_severity_color_light": "#EEEEEE", "gap_severity_value": "-1", "is_filter_entities_to_service": true, "is_split_by_entity": true, "key": "da-itsi-cp-monitoring-splunk-6f3799fc4e73d4b18e2a3f3f", "kpi_base_search": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc) | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-6f3799fc4e73d4b18e2a3f3f entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ]", "kpi_template_kpi_id": "", "kpi_threshold_template_id": "", "metric_qualifier": "", "metric_search_spec": { "metric_index": "", "metric_name": "" }, "search": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc) | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-6f3799fc4e73d4b18e2a3f3f entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity(avg, parsing_fill_perc, \"host\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(host, sec_grp)` | eval serviceid = \"da-itsi-cp-monitoring-splunk-splunk-indexer-queues\" | `aggregate_entity_into_service(avg)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-6f3799fc4e73d4b18e2a3f3f, true, true, true)` | eval kpi=\"Parsing Queue Fill\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-monitoring-splunk-splunk-indexer-queues\" | `assess_urgency` | `gettime`", "search_aggregate": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc) | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-6f3799fc4e73d4b18e2a3f3f entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_single_value(avg, avg, parsing_fill_perc, \"host\", 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-6f3799fc4e73d4b18e2a3f3f)`", "search_alert": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc) | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-6f3799fc4e73d4b18e2a3f3f entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity(avg, parsing_fill_perc, \"host\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(host, sec_grp)` | eval serviceid = \"da-itsi-cp-monitoring-splunk-splunk-indexer-queues\" | `aggregate_entity_into_service(avg)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-6f3799fc4e73d4b18e2a3f3f, true, true, true)` | eval kpi=\"Parsing Queue Fill\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-monitoring-splunk-splunk-indexer-queues\" | `assess_urgency` | `gettime`", "search_alert_earliest": "5", "search_alert_entities": "", "search_buckets": "", "search_entities": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc) | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-6f3799fc4e73d4b18e2a3f3f entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_single_value(avg, avg, parsing_fill_perc, \"host\", 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-6f3799fc4e73d4b18e2a3f3f)`", "search_occurrences": 1.0, "search_time_compare": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc) | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-6f3799fc4e73d4b18e2a3f3f entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_and_compare(avg, avg, parsing_fill_perc, \"host\", 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-6f3799fc4e73d4b18e2a3f3f)`", "search_time_series": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc) | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-6f3799fc4e73d4b18e2a3f3f entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity_time_series(avg, parsing_fill_perc, \"host\", 5)` | `aggregate_entity_into_service_time_series(avg, 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-6f3799fc4e73d4b18e2a3f3f)`", "search_time_series_aggregate": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc) | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-6f3799fc4e73d4b18e2a3f3f entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity_time_series(avg, parsing_fill_perc, \"host\", 5)` | `aggregate_entity_into_service_time_series(avg, 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-6f3799fc4e73d4b18e2a3f3f)`", "search_time_series_entities": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc) | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-6f3799fc4e73d4b18e2a3f3f entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_limited_entity_time_series(avg, parsing_fill_perc, \"host\", 5)`", "search_type": "shared_base", "service_title": "Splunk Indexer Queues", "threshold_eval": "", "threshold_field": "parsing_fill_perc", "time_policies": { "policies": { "default_policy": { "aggregate_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "entity_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "policy_type": "static", "time_blocks": [], "title": "Default" } } }, "title": "Parsing Queue Fill", "trending_ad": { "sensitivity": 8 }, "type": "kpis_primary", "tz_offset": null, "unit": "%", "urgency": 5.0, "use_time_policies": false }, { "adaptive_thresholding_training_window": "-7d", "adaptive_thresholds_is_enabled": false, "aggregate_eval": "", "aggregate_statop": "avg", "aggregate_threshold_alert_enabled": false, "aggregate_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "aggregate_thresholds_alert_enabled": false, "aggregate_thresholds_custom_alert_enabled": false, "aggregate_thresholds_custom_alert_rules": [], "alert_eval": "", "alert_lag": "5", "alert_on": "both", "alert_period": "5", "anomaly_detection_alerting_enabled": false, "anomaly_detection_is_enabled": false, "anomaly_detection_sensitivity": null, "anomaly_detection_training_window": null, "backfill_earliest_time": "-7d", "backfill_enabled": false, "base_search": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host", "base_search_id": "da-itsi-cp-monitoring-splunk-splk-indexer-search-metrics", "base_search_metric": "time-in-queue-avg", "cohesive_ad": { "sensitivity": 8 }, "cohesive_anomaly_detection_is_enabled": false, "datamodel": { "datamodel": "", "field": "", "object": "", "owner_field": "" }, "datamodel_filter": [], "datamodel_filter_clauses": null, "description": "", "enabled": true, "entity_filter_field": "host", "entity_split_field": "host", "entity_statop": "avg", "entity_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "fill_gaps": "null_value", "gap_custom_alert_value": 0.0, "gap_severity": "unknown", "gap_severity_color": "#CCCCCC", "gap_severity_color_light": "#EEEEEE", "gap_severity_value": "-1", "is_filter_entities_to_service": true, "is_split_by_entity": true, "key": "da-itsi-cp-monitoring-splunk-c6a11f0a42a4406e86ffc3c8", "kpi_base_search": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-c6a11f0a42a4406e86ffc3c8 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ]", "kpi_template_kpi_id": "", "kpi_threshold_template_id": "", "metric_qualifier": "", "metric_search_spec": { "metric_index": "", "metric_name": "" }, "search": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-c6a11f0a42a4406e86ffc3c8 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity(avg, avg_time_spent_in_queue, \"host\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(host, sec_grp)` | eval serviceid = \"da-itsi-cp-monitoring-splunk-splunk-indexer-queues\" | `aggregate_entity_into_service(avg)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-c6a11f0a42a4406e86ffc3c8, true, true, true)` | eval kpi=\"Time in Queue, Current\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-monitoring-splunk-splunk-indexer-queues\" | `assess_urgency` | `gettime`", "search_aggregate": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-c6a11f0a42a4406e86ffc3c8 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_single_value(avg, avg, avg_time_spent_in_queue, \"host\", 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-c6a11f0a42a4406e86ffc3c8)`", "search_alert": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-c6a11f0a42a4406e86ffc3c8 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity(avg, avg_time_spent_in_queue, \"host\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(host, sec_grp)` | eval serviceid = \"da-itsi-cp-monitoring-splunk-splunk-indexer-queues\" | `aggregate_entity_into_service(avg)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-c6a11f0a42a4406e86ffc3c8, true, true, true)` | eval kpi=\"Time in Queue, Current\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-monitoring-splunk-splunk-indexer-queues\" | `assess_urgency` | `gettime`", "search_alert_earliest": "5", "search_alert_entities": "", "search_buckets": "", "search_entities": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-c6a11f0a42a4406e86ffc3c8 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_single_value(avg, avg, avg_time_spent_in_queue, \"host\", 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-c6a11f0a42a4406e86ffc3c8)`", "search_occurrences": 1.0, "search_time_compare": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-c6a11f0a42a4406e86ffc3c8 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_and_compare(avg, avg, avg_time_spent_in_queue, \"host\", 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-c6a11f0a42a4406e86ffc3c8)`", "search_time_series": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-c6a11f0a42a4406e86ffc3c8 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity_time_series(avg, avg_time_spent_in_queue, \"host\", 5)` | `aggregate_entity_into_service_time_series(avg, 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-c6a11f0a42a4406e86ffc3c8)`", "search_time_series_aggregate": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-c6a11f0a42a4406e86ffc3c8 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity_time_series(avg, avg_time_spent_in_queue, \"host\", 5)` | `aggregate_entity_into_service_time_series(avg, 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-c6a11f0a42a4406e86ffc3c8)`", "search_time_series_entities": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-c6a11f0a42a4406e86ffc3c8 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_limited_entity_time_series(avg, avg_time_spent_in_queue, \"host\", 5)`", "search_type": "shared_base", "service_title": "Splunk Indexer Queues", "threshold_eval": "", "threshold_field": "avg_time_spent_in_queue", "time_policies": { "policies": { "default_policy": { "aggregate_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "entity_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "policy_type": "static", "time_blocks": [], "title": "Default" } } }, "title": "Time in Queue, Current", "trending_ad": { "sensitivity": 8 }, "type": "kpis_primary", "tz_offset": null, "unit": "", "urgency": 5.0, "use_time_policies": false }, { "adaptive_thresholding_training_window": "-7d", "adaptive_thresholds_is_enabled": false, "aggregate_eval": "", "aggregate_statop": "max", "aggregate_threshold_alert_enabled": false, "aggregate_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "aggregate_thresholds_alert_enabled": false, "aggregate_thresholds_custom_alert_enabled": false, "aggregate_thresholds_custom_alert_rules": [], "alert_eval": "", "alert_lag": "5", "alert_on": "both", "alert_period": "5", "anomaly_detection_alerting_enabled": false, "anomaly_detection_is_enabled": false, "anomaly_detection_sensitivity": null, "anomaly_detection_training_window": null, "backfill_earliest_time": "-7d", "backfill_enabled": false, "base_search": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host", "base_search_id": "da-itsi-cp-monitoring-splunk-splk-indexer-search-metrics", "base_search_metric": "time-in-queue-max", "cohesive_ad": { "sensitivity": 8 }, "cohesive_anomaly_detection_is_enabled": false, "datamodel": { "datamodel": "", "field": "", "object": "", "owner_field": "" }, "datamodel_filter": [], "datamodel_filter_clauses": null, "description": "", "enabled": true, "entity_filter_field": "host", "entity_split_field": "host", "entity_statop": "max", "entity_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "fill_gaps": "null_value", "gap_custom_alert_value": 0.0, "gap_severity": "unknown", "gap_severity_color": "#CCCCCC", "gap_severity_color_light": "#EEEEEE", "gap_severity_value": "-1", "is_filter_entities_to_service": true, "is_split_by_entity": true, "key": "da-itsi-cp-monitoring-splunk-1bd78e81ec1a7ed4124656a2", "kpi_base_search": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-1bd78e81ec1a7ed4124656a2 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ]", "kpi_template_kpi_id": "", "kpi_threshold_template_id": "", "metric_qualifier": "", "metric_search_spec": { "metric_index": "", "metric_name": "" }, "search": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-1bd78e81ec1a7ed4124656a2 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity(max, max_time_spent_in_queue, \"host\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(host, sec_grp)` | eval serviceid = \"da-itsi-cp-monitoring-splunk-splunk-indexer-queues\" | `aggregate_entity_into_service(max)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-1bd78e81ec1a7ed4124656a2, true, true, true)` | eval kpi=\"Time in Queue, Max\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-monitoring-splunk-splunk-indexer-queues\" | `assess_urgency` | `gettime`", "search_aggregate": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-1bd78e81ec1a7ed4124656a2 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_single_value(max, max, max_time_spent_in_queue, \"host\", 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-1bd78e81ec1a7ed4124656a2)`", "search_alert": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-1bd78e81ec1a7ed4124656a2 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity(max, max_time_spent_in_queue, \"host\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(host, sec_grp)` | eval serviceid = \"da-itsi-cp-monitoring-splunk-splunk-indexer-queues\" | `aggregate_entity_into_service(max)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-1bd78e81ec1a7ed4124656a2, true, true, true)` | eval kpi=\"Time in Queue, Max\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-monitoring-splunk-splunk-indexer-queues\" | `assess_urgency` | `gettime`", "search_alert_earliest": "5", "search_alert_entities": "", "search_buckets": "", "search_entities": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-1bd78e81ec1a7ed4124656a2 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_single_value(max, max, max_time_spent_in_queue, \"host\", 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-1bd78e81ec1a7ed4124656a2)`", "search_occurrences": 1.0, "search_time_compare": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-1bd78e81ec1a7ed4124656a2 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_and_compare(max, max, max_time_spent_in_queue, \"host\", 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-1bd78e81ec1a7ed4124656a2)`", "search_time_series": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-1bd78e81ec1a7ed4124656a2 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity_time_series(max, max_time_spent_in_queue, \"host\", 5)` | `aggregate_entity_into_service_time_series(max, 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-1bd78e81ec1a7ed4124656a2)`", "search_time_series_aggregate": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-1bd78e81ec1a7ed4124656a2 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity_time_series(max, max_time_spent_in_queue, \"host\", 5)` | `aggregate_entity_into_service_time_series(max, 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-1bd78e81ec1a7ed4124656a2)`", "search_time_series_entities": "index=_internal source=*/log/splunk/metrics.log sourcetype=splunkd (group=search_concurrency OR group=searchscheduler)\n| fields active_hist_searches, active_hist_searches, active_realtime_searches, active_realtime_searches _time, splunk_server, skipped, max_running, total_runtime, enqueue_seaches_count, avg_time_spent_in_queue, max_time_spent_in_queue, current_queue_size, largest_queue_size\n| rename splunk_server as host | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-1bd78e81ec1a7ed4124656a2 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_limited_entity_time_series(max, max_time_spent_in_queue, \"host\", 5)`", "search_type": "shared_base", "service_title": "Splunk Indexer Queues", "threshold_eval": "", "threshold_field": "max_time_spent_in_queue", "time_policies": { "policies": { "default_policy": { "aggregate_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "entity_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "policy_type": "static", "time_blocks": [], "title": "Default" } } }, "title": "Time in Queue, Max", "trending_ad": { "sensitivity": 8 }, "type": "kpis_primary", "tz_offset": null, "unit": "", "urgency": 5.0, "use_time_policies": false }, { "adaptive_thresholding_training_window": "-7d", "adaptive_thresholds_is_enabled": false, "aggregate_eval": "", "aggregate_statop": "avg", "aggregate_threshold_alert_enabled": false, "aggregate_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 110, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 110.0, "render_boundary_min": 0.0, "threshold_levels": [ { "dynamic_param": 0.0, "severity_color": "#FFE98C", "severity_color_light": "#FFF4C5", "severity_label": "low", "severity_label_localized": null, "severity_value": 3.0, "threshold_value": 75.0 }, { "dynamic_param": 0.0, "severity_color": "#FCB64E", "severity_color_light": "#FEE6C1", "severity_label": "medium", "severity_label_localized": null, "severity_value": 4.0, "threshold_value": 90.0 }, { "dynamic_param": 0.0, "severity_color": "#F26A35", "severity_color_light": "#FBCBB9", "severity_label": "high", "severity_label_localized": null, "severity_value": 5.0, "threshold_value": 95.0 }, { "dynamic_param": 0.0, "severity_color": "#B50101", "severity_color_light": "#E5A6A6", "severity_label": "critical", "severity_label_localized": null, "severity_value": 6.0, "threshold_value": 100.0 } ] }, "aggregate_thresholds_alert_enabled": false, "aggregate_thresholds_custom_alert_enabled": false, "aggregate_thresholds_custom_alert_rules": [], "alert_eval": "", "alert_lag": "5", "alert_on": "both", "alert_period": "5", "anomaly_detection_alerting_enabled": false, "anomaly_detection_is_enabled": false, "anomaly_detection_sensitivity": null, "anomaly_detection_training_window": null, "backfill_earliest_time": "-7d", "backfill_enabled": false, "base_search": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc)", "base_search_id": "da-itsi-cp-monitoring-splunk-splk-indexer-queues", "base_search_metric": "typing-queue-fill", "cohesive_ad": { "sensitivity": 8 }, "cohesive_anomaly_detection_is_enabled": false, "datamodel": { "datamodel": "", "field": "", "object": "", "owner_field": "" }, "datamodel_filter": [], "datamodel_filter_clauses": null, "description": "", "enabled": true, "entity_filter_field": "host", "entity_split_field": "host", "entity_statop": "avg", "entity_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 110, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 110.0, "render_boundary_min": 0.0, "threshold_levels": [ { "dynamic_param": 0.0, "severity_color": "#FFE98C", "severity_color_light": "#FFF4C5", "severity_label": "low", "severity_label_localized": null, "severity_value": 3.0, "threshold_value": 75.0 }, { "dynamic_param": 0.0, "severity_color": "#FCB64E", "severity_color_light": "#FEE6C1", "severity_label": "medium", "severity_label_localized": null, "severity_value": 4.0, "threshold_value": 90.0 }, { "dynamic_param": 0.0, "severity_color": "#F26A35", "severity_color_light": "#FBCBB9", "severity_label": "high", "severity_label_localized": null, "severity_value": 5.0, "threshold_value": 95.0 }, { "dynamic_param": 0.0, "severity_color": "#B50101", "severity_color_light": "#E5A6A6", "severity_label": "critical", "severity_label_localized": null, "severity_value": 6.0, "threshold_value": 100.0 } ] }, "fill_gaps": "null_value", "gap_custom_alert_value": 0.0, "gap_severity": "unknown", "gap_severity_color": "#CCCCCC", "gap_severity_color_light": "#EEEEEE", "gap_severity_value": "-1", "is_filter_entities_to_service": true, "is_split_by_entity": true, "key": "da-itsi-cp-monitoring-splunk-8cde6bdeaf6c77d32cdaaa96", "kpi_base_search": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc) | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-8cde6bdeaf6c77d32cdaaa96 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ]", "kpi_template_kpi_id": "", "kpi_threshold_template_id": "", "metric_qualifier": "", "metric_search_spec": { "metric_index": "", "metric_name": "" }, "search": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc) | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-8cde6bdeaf6c77d32cdaaa96 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity(avg, typing_fill_perc, \"host\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(host, sec_grp)` | eval serviceid = \"da-itsi-cp-monitoring-splunk-splunk-indexer-queues\" | `aggregate_entity_into_service(avg)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-8cde6bdeaf6c77d32cdaaa96, true, true, true)` | eval kpi=\"Typing Queue Fill\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-monitoring-splunk-splunk-indexer-queues\" | `assess_urgency` | `gettime`", "search_aggregate": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc) | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-8cde6bdeaf6c77d32cdaaa96 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_single_value(avg, avg, typing_fill_perc, \"host\", 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-8cde6bdeaf6c77d32cdaaa96)`", "search_alert": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc) | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-8cde6bdeaf6c77d32cdaaa96 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity(avg, typing_fill_perc, \"host\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(host, sec_grp)` | eval serviceid = \"da-itsi-cp-monitoring-splunk-splunk-indexer-queues\" | `aggregate_entity_into_service(avg)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-8cde6bdeaf6c77d32cdaaa96, true, true, true)` | eval kpi=\"Typing Queue Fill\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-monitoring-splunk-splunk-indexer-queues\" | `assess_urgency` | `gettime`", "search_alert_earliest": "5", "search_alert_entities": "", "search_buckets": "", "search_entities": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc) | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-8cde6bdeaf6c77d32cdaaa96 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_single_value(avg, avg, typing_fill_perc, \"host\", 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-8cde6bdeaf6c77d32cdaaa96)`", "search_occurrences": 1.0, "search_time_compare": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc) | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-8cde6bdeaf6c77d32cdaaa96 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_and_compare(avg, avg, typing_fill_perc, \"host\", 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-8cde6bdeaf6c77d32cdaaa96)`", "search_time_series": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc) | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-8cde6bdeaf6c77d32cdaaa96 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity_time_series(avg, typing_fill_perc, \"host\", 5)` | `aggregate_entity_into_service_time_series(avg, 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-8cde6bdeaf6c77d32cdaaa96)`", "search_time_series_aggregate": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc) | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-8cde6bdeaf6c77d32cdaaa96 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity_time_series(avg, typing_fill_perc, \"host\", 5)` | `aggregate_entity_into_service_time_series(avg, 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-indexer-queues, da-itsi-cp-monitoring-splunk-8cde6bdeaf6c77d32cdaaa96)`", "search_time_series_entities": "index=_internal source=*metrics.log group=queue | fields _time, current_size_kb, max_size_kb, name, host | eval fill_perc=round(current_size_kb / max_size_kb * 100, 2) | eval agg_fill_perc = case(match(name, \"aggqueue*\"), fill_perc), parsing_fill_perc = case(match(name, \"parsingqueue*\"), fill_perc), index_fill_perc = case(match(name, \"indexqueue*\"), fill_perc), typing_fill_perc = case(match(name, \"typingqueue*\"), fill_perc) | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-indexer-queues kpi_id=da-itsi-cp-monitoring-splunk-8cde6bdeaf6c77d32cdaaa96 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_limited_entity_time_series(avg, typing_fill_perc, \"host\", 5)`", "search_type": "shared_base", "service_title": "Splunk Indexer Queues", "threshold_eval": "", "threshold_field": "typing_fill_perc", "time_policies": { "policies": { "default_policy": { "aggregate_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "entity_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "policy_type": "static", "time_blocks": [], "title": "Default" } } }, "title": "Typing Queue Fill", "trending_ad": { "sensitivity": 8 }, "type": "kpis_primary", "tz_offset": null, "unit": "%", "urgency": 5.0, "use_time_policies": false } ], "service_tags": { "tags": [], "template_tags": [] }, "service_template_id": "", "services_depending_on_me": [ { "kpis_depending_on": [ "SHKPI-da-itsi-cp-monitoring-splunk-splunk-indexer-queues" ], "service_id": "da-itsi-cp-monitoring-splunk-splunk-indexer" } ], "services_depends_on": [], "team_id": "default_itsi_security_group", "title": "Splunk Indexer Queues", "version": "0.0.38" }