{ "algorithms": { "GradientBoostingRegressor": { "RMSE": 0, "modelId": "", "rSquared": 0, "recommended": false }, "LinearRegression": { "RMSE": 0, "modelId": "", "rSquared": 0, "recommended": false }, "LogisticRegression": { "accuracy": 0, "f1_score": 0, "modelId": "", "precision": 0, "recall": 0, "recommended": false }, "RandomForestRegressor": { "RMSE": 0, "modelId": "", "rSquared": 0, "recommended": false } }, "description": "", "enabled": true, "entity_rules": [ { "rule_condition": "AND", "rule_items": [ { "field": "splunk_role", "field_type": "info", "rule_type": "matches", "value": "search_head" } ] } ], "key": "da-itsi-cp-monitoring-splunk-splunk-search-head-users", "kpis": [ { "adaptive_thresholding_training_window": "-7d", "adaptive_thresholds_is_enabled": false, "aggregate_eval": "", "aggregate_statop": "avg", "aggregate_threshold_alert_enabled": false, "aggregate_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [ { "dynamic_param": null, "severity_color": "#B50101", "severity_color_light": "#E5A6A6", "severity_label": "critical", "severity_label_localized": null, "severity_value": 6.0, "threshold_value": 0.0 }, { "dynamic_param": null, "severity_color": "#F26A35", "severity_color_light": "#FBCBB9", "severity_label": "high", "severity_label_localized": null, "severity_value": 5.0, "threshold_value": 20.0 }, { "dynamic_param": null, "severity_color": "#FCB64E", "severity_color_light": "#FEE6C1", "severity_label": "medium", "severity_label_localized": null, "severity_value": 4.0, "threshold_value": 40.0 }, { "dynamic_param": null, "severity_color": "#FFE98C", "severity_color_light": "#FFF4C5", "severity_label": "low", "severity_label_localized": null, "severity_value": 3.0, "threshold_value": 60.0 }, { "dynamic_param": null, "severity_color": "#99D18B", "severity_color_light": "#DCEFD7", "severity_label": "normal", "severity_label_localized": null, "severity_value": 2.0, "threshold_value": 80.0 } ] }, "aggregate_thresholds_alert_enabled": false, "aggregate_thresholds_custom_alert_enabled": false, "aggregate_thresholds_custom_alert_rules": [], "alert_eval": "", "alert_lag": "30", "alert_on": "both", "alert_period": "1", "anomaly_detection_alerting_enabled": false, "anomaly_detection_is_enabled": false, "anomaly_detection_sensitivity": 0.999, "anomaly_detection_training_window": "-7d", "backfill_earliest_time": "-7d", "backfill_enabled": false, "base_search": "`get_full_itsi_summary_service_health_events(da-itsi-cp-monitoring-splunk-splunk-search-head-users)`", "base_search_id": null, "base_search_metric": null, "cohesive_ad": { "sensitivity": 8 }, "cohesive_anomaly_detection_is_enabled": false, "datamodel": { "datamodel": "", "field": "", "object": "", "owner_field": "" }, "datamodel_filter": [], "datamodel_filter_clauses": null, "description": "", "enabled": false, "entity_filter_field": "", "entity_split_field": "", "entity_statop": "avg", "entity_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [ { "dynamic_param": null, "severity_color": "#B50101", "severity_color_light": "#E5A6A6", "severity_label": "critical", "severity_label_localized": null, "severity_value": 6.0, "threshold_value": 0.0 }, { "dynamic_param": null, "severity_color": "#F26A35", "severity_color_light": "#FBCBB9", "severity_label": "high", "severity_label_localized": null, "severity_value": 5.0, "threshold_value": 20.0 }, { "dynamic_param": null, "severity_color": "#FCB64E", "severity_color_light": "#FEE6C1", "severity_label": "medium", "severity_label_localized": null, "severity_value": 4.0, "threshold_value": 40.0 }, { "dynamic_param": null, "severity_color": "#FFE98C", "severity_color_light": "#FFF4C5", "severity_label": "low", "severity_label_localized": null, "severity_value": 3.0, "threshold_value": 60.0 }, { "dynamic_param": null, "severity_color": "#99D18B", "severity_color_light": "#DCEFD7", "severity_label": "normal", "severity_label_localized": null, "severity_value": 2.0, "threshold_value": 80.0 } ] }, "fill_gaps": "null_value", "gap_custom_alert_value": 0.0, "gap_severity": "unknown", "gap_severity_color": "#CCCCCC", "gap_severity_color_light": "#EEEEEE", "gap_severity_value": "-1", "is_filter_entities_to_service": false, "is_split_by_entity": false, "key": "SHKPI-da-itsi-cp-monitoring-splunk-splunk-search-head-users", "kpi_base_search": "", "kpi_template_kpi_id": "", "kpi_threshold_template_id": "", "metric_qualifier": null, "metric_search_spec": { "metric_index": "", "metric_name": "" }, "search": "`get_full_itsi_summary_service_health_events(da-itsi-cp-monitoring-splunk-splunk-search-head-users)` | stats latest(health_score) AS aggregate", "search_aggregate": "`get_full_itsi_summary_service_health_events(da-itsi-cp-monitoring-splunk-splunk-search-head-users)` | stats latest(health_score) AS aggregate", "search_alert": "", "search_alert_earliest": "15", "search_alert_entities": "", "search_buckets": "", "search_entities": null, "search_occurrences": 1.0, "search_time_compare": "`get_full_itsi_summary_service_health_events(da-itsi-cp-monitoring-splunk-splunk-search-head-users)` [| stats count | addinfo | eval search= \"earliest=\" + tostring(info_min_time-(info_max_time-info_min_time))+ \" latest=\" + tostring(info_max_time) |fields search] | addinfo | eval bucket=if(_time0, \"increase\", if(window_delta < 0, \"decrease\", \"none\"))", "search_time_series": "`get_full_itsi_summary_service_health_events(da-itsi-cp-monitoring-splunk-splunk-search-head-users)` | timechart avg(health_score) AS aggregate", "search_time_series_aggregate": "`get_full_itsi_summary_service_health_events(da-itsi-cp-monitoring-splunk-splunk-search-head-users)` | timechart avg(health_score) AS aggregate", "search_time_series_entities": "", "search_type": "adhoc", "service_title": "Splunk Search Head Users", "threshold_eval": "", "threshold_field": "aggregate", "time_policies": { "policies": { "default_policy": { "aggregate_thresholds": { "base_severity_color": "#AED3E5", "base_severity_color_light": "#E3F0F6", "base_severity_label": "info", "base_severity_value": 1.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "entity_thresholds": { "base_severity_color": "#AED3E5", "base_severity_color_light": "#E3F0F6", "base_severity_label": "info", "base_severity_value": 1.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "policy_type": "static", "time_blocks": [], "title": "Default" } } }, "title": "ServiceHealthScore", "trending_ad": { "sensitivity": 8 }, "type": "service_health", "tz_offset": null, "unit": "", "urgency": 11.0, "use_time_policies": false }, { "adaptive_thresholding_training_window": "-7d", "adaptive_thresholds_is_enabled": false, "aggregate_eval": "", "aggregate_statop": "avg", "aggregate_threshold_alert_enabled": false, "aggregate_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "aggregate_thresholds_alert_enabled": false, "aggregate_thresholds_custom_alert_enabled": false, "aggregate_thresholds_custom_alert_rules": [], "alert_eval": "", "alert_lag": "5", "alert_on": "both", "alert_period": "5", "anomaly_detection_alerting_enabled": false, "anomaly_detection_is_enabled": false, "anomaly_detection_sensitivity": null, "anomaly_detection_training_window": null, "backfill_earliest_time": "-7d", "backfill_enabled": false, "base_search": "index=_audit action=search (id=* OR search_id=*) \n| eval search_id = if(isnull(search_id), id, search_id) \n| where (apiStartTime!=\"'ZERO_TIME'\" AND apiEndTime!=\"'ZERO_TIME'\") OR isnotnull(total_run_time)\n| replace '*' with * in search_id \n| rex \"search='(?.*?)', autojoin\" \n| search search!=scheduler_*\n| convert num(total_run_time) \n| rex \"\\,\\s+user=(?\\S+),\" \n| eval user = if(user=\"n/a\", null(), user) \n| eval total_run_time = if(isnull(total_run_time), round((strptime(apiEndTime,\"'%a %b %d %T %Y'\")-strptime(apiStartTime, \"'%a %b %d %T %Y'\"))/60, 1), total_run_time) \n| stats min(_time) as _time first(user) as user max(total_run_time) as total_run_time first(search) as search values(host) as host by search_id \n| search search=search* search!=*_internal* search!=*_audit*", "base_search_id": "da-itsi-cp-monitoring-splunk-splk-search-head-usage", "base_search_metric": "run-time-avg", "cohesive_ad": { "sensitivity": 8 }, "cohesive_anomaly_detection_is_enabled": false, "datamodel": { "datamodel": "", "field": "", "object": "", "owner_field": "" }, "datamodel_filter": [], "datamodel_filter_clauses": null, "description": "", "enabled": true, "entity_filter_field": "host", "entity_split_field": "user", "entity_statop": "avg", "entity_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "fill_gaps": "null_value", "gap_custom_alert_value": 0.0, "gap_severity": "unknown", "gap_severity_color": "#CCCCCC", "gap_severity_color_light": "#EEEEEE", "gap_severity_value": "-1", "is_filter_entities_to_service": true, "is_split_by_entity": true, "key": "da-itsi-cp-monitoring-splunk-ce603d672d398d18bbaa5a58", "kpi_base_search": "index=_audit action=search (id=* OR search_id=*) \n| eval search_id = if(isnull(search_id), id, search_id) \n| where (apiStartTime!=\"'ZERO_TIME'\" AND apiEndTime!=\"'ZERO_TIME'\") OR isnotnull(total_run_time)\n| replace '*' with * in search_id \n| rex \"search='(?.*?)', autojoin\" \n| search search!=scheduler_*\n| convert num(total_run_time) \n| rex \"\\,\\s+user=(?\\S+),\" \n| eval user = if(user=\"n/a\", null(), user) \n| eval total_run_time = if(isnull(total_run_time), round((strptime(apiEndTime,\"'%a %b %d %T %Y'\")-strptime(apiStartTime, \"'%a %b %d %T %Y'\"))/60, 1), total_run_time) \n| stats min(_time) as _time first(user) as user max(total_run_time) as total_run_time first(search) as search values(host) as host by search_id \n| search search=search* search!=*_internal* search!=*_audit* | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-search-head-users kpi_id=da-itsi-cp-monitoring-splunk-ce603d672d398d18bbaa5a58 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ]", "kpi_template_kpi_id": "", "kpi_threshold_template_id": "", "metric_qualifier": "", "metric_search_spec": { "metric_index": "", "metric_name": "" }, "search": "index=_audit action=search (id=* OR search_id=*) \n| eval search_id = if(isnull(search_id), id, search_id) \n| where (apiStartTime!=\"'ZERO_TIME'\" AND apiEndTime!=\"'ZERO_TIME'\") OR isnotnull(total_run_time)\n| replace '*' with * in search_id \n| rex \"search='(?.*?)', autojoin\" \n| search search!=scheduler_*\n| convert num(total_run_time) \n| rex \"\\,\\s+user=(?\\S+),\" \n| eval user = if(user=\"n/a\", null(), user) \n| eval total_run_time = if(isnull(total_run_time), round((strptime(apiEndTime,\"'%a %b %d %T %Y'\")-strptime(apiStartTime, \"'%a %b %d %T %Y'\"))/60, 1), total_run_time) \n| stats min(_time) as _time first(user) as user max(total_run_time) as total_run_time first(search) as search values(host) as host by search_id \n| search search=search* search!=*_internal* search!=*_audit* | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-search-head-users kpi_id=da-itsi-cp-monitoring-splunk-ce603d672d398d18bbaa5a58 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity(avg, total_run_time, \"user\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(user, sec_grp)` | eval serviceid = \"da-itsi-cp-monitoring-splunk-splunk-search-head-users\" | `aggregate_entity_into_service(avg)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-search-head-users, da-itsi-cp-monitoring-splunk-ce603d672d398d18bbaa5a58, true, true, true)` | eval kpi=\"Run Time, Avg\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-monitoring-splunk-splunk-search-head-users\" | `assess_urgency` | `gettime`", "search_aggregate": "index=_audit action=search (id=* OR search_id=*) \n| eval search_id = if(isnull(search_id), id, search_id) \n| where (apiStartTime!=\"'ZERO_TIME'\" AND apiEndTime!=\"'ZERO_TIME'\") OR isnotnull(total_run_time)\n| replace '*' with * in search_id \n| rex \"search='(?.*?)', autojoin\" \n| search search!=scheduler_*\n| convert num(total_run_time) \n| rex \"\\,\\s+user=(?\\S+),\" \n| eval user = if(user=\"n/a\", null(), user) \n| eval total_run_time = if(isnull(total_run_time), round((strptime(apiEndTime,\"'%a %b %d %T %Y'\")-strptime(apiStartTime, \"'%a %b %d %T %Y'\"))/60, 1), total_run_time) \n| stats min(_time) as _time first(user) as user max(total_run_time) as total_run_time first(search) as search values(host) as host by search_id \n| search search=search* search!=*_internal* search!=*_audit* | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-search-head-users kpi_id=da-itsi-cp-monitoring-splunk-ce603d672d398d18bbaa5a58 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_single_value(avg, avg, total_run_time, \"user\", 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-search-head-users, da-itsi-cp-monitoring-splunk-ce603d672d398d18bbaa5a58)`", "search_alert": "index=_audit action=search (id=* OR search_id=*) \n| eval search_id = if(isnull(search_id), id, search_id) \n| where (apiStartTime!=\"'ZERO_TIME'\" AND apiEndTime!=\"'ZERO_TIME'\") OR isnotnull(total_run_time)\n| replace '*' with * in search_id \n| rex \"search='(?.*?)', autojoin\" \n| search search!=scheduler_*\n| convert num(total_run_time) \n| rex \"\\,\\s+user=(?\\S+),\" \n| eval user = if(user=\"n/a\", null(), user) \n| eval total_run_time = if(isnull(total_run_time), round((strptime(apiEndTime,\"'%a %b %d %T %Y'\")-strptime(apiStartTime, \"'%a %b %d %T %Y'\"))/60, 1), total_run_time) \n| stats min(_time) as _time first(user) as user max(total_run_time) as total_run_time first(search) as search values(host) as host by search_id \n| search search=search* search!=*_internal* search!=*_audit* | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-search-head-users kpi_id=da-itsi-cp-monitoring-splunk-ce603d672d398d18bbaa5a58 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity(avg, total_run_time, \"user\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(user, sec_grp)` | eval serviceid = \"da-itsi-cp-monitoring-splunk-splunk-search-head-users\" | `aggregate_entity_into_service(avg)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-search-head-users, da-itsi-cp-monitoring-splunk-ce603d672d398d18bbaa5a58, true, true, true)` | eval kpi=\"Run Time, Avg\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-monitoring-splunk-splunk-search-head-users\" | `assess_urgency` | `gettime`", "search_alert_earliest": "5", "search_alert_entities": "", "search_buckets": "", "search_entities": "index=_audit action=search (id=* OR search_id=*) \n| eval search_id = if(isnull(search_id), id, search_id) \n| where (apiStartTime!=\"'ZERO_TIME'\" AND apiEndTime!=\"'ZERO_TIME'\") OR isnotnull(total_run_time)\n| replace '*' with * in search_id \n| rex \"search='(?.*?)', autojoin\" \n| search search!=scheduler_*\n| convert num(total_run_time) \n| rex \"\\,\\s+user=(?\\S+),\" \n| eval user = if(user=\"n/a\", null(), user) \n| eval total_run_time = if(isnull(total_run_time), round((strptime(apiEndTime,\"'%a %b %d %T %Y'\")-strptime(apiStartTime, \"'%a %b %d %T %Y'\"))/60, 1), total_run_time) \n| stats min(_time) as _time first(user) as user max(total_run_time) as total_run_time first(search) as search values(host) as host by search_id \n| search search=search* search!=*_internal* search!=*_audit* | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-search-head-users kpi_id=da-itsi-cp-monitoring-splunk-ce603d672d398d18bbaa5a58 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_single_value(avg, avg, total_run_time, \"user\", 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-search-head-users, da-itsi-cp-monitoring-splunk-ce603d672d398d18bbaa5a58)`", "search_occurrences": 1.0, "search_time_compare": "index=_audit action=search (id=* OR search_id=*) \n| eval search_id = if(isnull(search_id), id, search_id) \n| where (apiStartTime!=\"'ZERO_TIME'\" AND apiEndTime!=\"'ZERO_TIME'\") OR isnotnull(total_run_time)\n| replace '*' with * in search_id \n| rex \"search='(?.*?)', autojoin\" \n| search search!=scheduler_*\n| convert num(total_run_time) \n| rex \"\\,\\s+user=(?\\S+),\" \n| eval user = if(user=\"n/a\", null(), user) \n| eval total_run_time = if(isnull(total_run_time), round((strptime(apiEndTime,\"'%a %b %d %T %Y'\")-strptime(apiStartTime, \"'%a %b %d %T %Y'\"))/60, 1), total_run_time) \n| stats min(_time) as _time first(user) as user max(total_run_time) as total_run_time first(search) as search values(host) as host by search_id \n| search search=search* search!=*_internal* search!=*_audit* | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-search-head-users kpi_id=da-itsi-cp-monitoring-splunk-ce603d672d398d18bbaa5a58 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_and_compare(avg, avg, total_run_time, \"user\", 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-search-head-users, da-itsi-cp-monitoring-splunk-ce603d672d398d18bbaa5a58)`", "search_time_series": "index=_audit action=search (id=* OR search_id=*) \n| eval search_id = if(isnull(search_id), id, search_id) \n| where (apiStartTime!=\"'ZERO_TIME'\" AND apiEndTime!=\"'ZERO_TIME'\") OR isnotnull(total_run_time)\n| replace '*' with * in search_id \n| rex \"search='(?.*?)', autojoin\" \n| search search!=scheduler_*\n| convert num(total_run_time) \n| rex \"\\,\\s+user=(?\\S+),\" \n| eval user = if(user=\"n/a\", null(), user) \n| eval total_run_time = if(isnull(total_run_time), round((strptime(apiEndTime,\"'%a %b %d %T %Y'\")-strptime(apiStartTime, \"'%a %b %d %T %Y'\"))/60, 1), total_run_time) \n| stats min(_time) as _time first(user) as user max(total_run_time) as total_run_time first(search) as search values(host) as host by search_id \n| search search=search* search!=*_internal* search!=*_audit* | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-search-head-users kpi_id=da-itsi-cp-monitoring-splunk-ce603d672d398d18bbaa5a58 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity_time_series(avg, total_run_time, \"user\", 5)` | `aggregate_entity_into_service_time_series(avg, 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-search-head-users, da-itsi-cp-monitoring-splunk-ce603d672d398d18bbaa5a58)`", "search_time_series_aggregate": "index=_audit action=search (id=* OR search_id=*) \n| eval search_id = if(isnull(search_id), id, search_id) \n| where (apiStartTime!=\"'ZERO_TIME'\" AND apiEndTime!=\"'ZERO_TIME'\") OR isnotnull(total_run_time)\n| replace '*' with * in search_id \n| rex \"search='(?.*?)', autojoin\" \n| search search!=scheduler_*\n| convert num(total_run_time) \n| rex \"\\,\\s+user=(?\\S+),\" \n| eval user = if(user=\"n/a\", null(), user) \n| eval total_run_time = if(isnull(total_run_time), round((strptime(apiEndTime,\"'%a %b %d %T %Y'\")-strptime(apiStartTime, \"'%a %b %d %T %Y'\"))/60, 1), total_run_time) \n| stats min(_time) as _time first(user) as user max(total_run_time) as total_run_time first(search) as search values(host) as host by search_id \n| search search=search* search!=*_internal* search!=*_audit* | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-search-head-users kpi_id=da-itsi-cp-monitoring-splunk-ce603d672d398d18bbaa5a58 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity_time_series(avg, total_run_time, \"user\", 5)` | `aggregate_entity_into_service_time_series(avg, 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-search-head-users, da-itsi-cp-monitoring-splunk-ce603d672d398d18bbaa5a58)`", "search_time_series_entities": "index=_audit action=search (id=* OR search_id=*) \n| eval search_id = if(isnull(search_id), id, search_id) \n| where (apiStartTime!=\"'ZERO_TIME'\" AND apiEndTime!=\"'ZERO_TIME'\") OR isnotnull(total_run_time)\n| replace '*' with * in search_id \n| rex \"search='(?.*?)', autojoin\" \n| search search!=scheduler_*\n| convert num(total_run_time) \n| rex \"\\,\\s+user=(?\\S+),\" \n| eval user = if(user=\"n/a\", null(), user) \n| eval total_run_time = if(isnull(total_run_time), round((strptime(apiEndTime,\"'%a %b %d %T %Y'\")-strptime(apiStartTime, \"'%a %b %d %T %Y'\"))/60, 1), total_run_time) \n| stats min(_time) as _time first(user) as user max(total_run_time) as total_run_time first(search) as search values(host) as host by search_id \n| search search=search* search!=*_internal* search!=*_audit* | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-search-head-users kpi_id=da-itsi-cp-monitoring-splunk-ce603d672d398d18bbaa5a58 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_limited_entity_time_series(avg, total_run_time, \"user\", 5)`", "search_type": "shared_base", "service_title": "Splunk Search Head Users", "threshold_eval": "", "threshold_field": "total_run_time", "time_policies": { "policies": { "default_policy": { "aggregate_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "entity_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "policy_type": "static", "time_blocks": [], "title": "Default" } } }, "title": "Run Time, Avg", "trending_ad": { "sensitivity": 8 }, "type": "kpis_primary", "tz_offset": null, "unit": "", "urgency": 5.0, "use_time_policies": false }, { "adaptive_thresholding_training_window": "-7d", "adaptive_thresholds_is_enabled": false, "aggregate_eval": "", "aggregate_statop": "max", "aggregate_threshold_alert_enabled": false, "aggregate_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "aggregate_thresholds_alert_enabled": false, "aggregate_thresholds_custom_alert_enabled": false, "aggregate_thresholds_custom_alert_rules": [], "alert_eval": "", "alert_lag": "5", "alert_on": "both", "alert_period": "5", "anomaly_detection_alerting_enabled": false, "anomaly_detection_is_enabled": false, "anomaly_detection_sensitivity": null, "anomaly_detection_training_window": null, "backfill_earliest_time": "-7d", "backfill_enabled": false, "base_search": "index=_audit action=search (id=* OR search_id=*) \n| eval search_id = if(isnull(search_id), id, search_id) \n| where (apiStartTime!=\"'ZERO_TIME'\" AND apiEndTime!=\"'ZERO_TIME'\") OR isnotnull(total_run_time)\n| replace '*' with * in search_id \n| rex \"search='(?.*?)', autojoin\" \n| search search!=scheduler_*\n| convert num(total_run_time) \n| rex \"\\,\\s+user=(?\\S+),\" \n| eval user = if(user=\"n/a\", null(), user) \n| eval total_run_time = if(isnull(total_run_time), round((strptime(apiEndTime,\"'%a %b %d %T %Y'\")-strptime(apiStartTime, \"'%a %b %d %T %Y'\"))/60, 1), total_run_time) \n| stats min(_time) as _time first(user) as user max(total_run_time) as total_run_time first(search) as search values(host) as host by search_id \n| search search=search* search!=*_internal* search!=*_audit*", "base_search_id": "da-itsi-cp-monitoring-splunk-splk-search-head-usage", "base_search_metric": "run-time-max", "cohesive_ad": { "sensitivity": 8 }, "cohesive_anomaly_detection_is_enabled": false, "datamodel": { "datamodel": "", "field": "", "object": "", "owner_field": "" }, "datamodel_filter": [], "datamodel_filter_clauses": null, "description": "", "enabled": true, "entity_filter_field": "host", "entity_split_field": "user", "entity_statop": "max", "entity_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "fill_gaps": "null_value", "gap_custom_alert_value": 0.0, "gap_severity": "unknown", "gap_severity_color": "#CCCCCC", "gap_severity_color_light": "#EEEEEE", "gap_severity_value": "-1", "is_filter_entities_to_service": true, "is_split_by_entity": true, "key": "da-itsi-cp-monitoring-splunk-2fbe52efca0b64a221000379", "kpi_base_search": "index=_audit action=search (id=* OR search_id=*) \n| eval search_id = if(isnull(search_id), id, search_id) \n| where (apiStartTime!=\"'ZERO_TIME'\" AND apiEndTime!=\"'ZERO_TIME'\") OR isnotnull(total_run_time)\n| replace '*' with * in search_id \n| rex \"search='(?.*?)', autojoin\" \n| search search!=scheduler_*\n| convert num(total_run_time) \n| rex \"\\,\\s+user=(?\\S+),\" \n| eval user = if(user=\"n/a\", null(), user) \n| eval total_run_time = if(isnull(total_run_time), round((strptime(apiEndTime,\"'%a %b %d %T %Y'\")-strptime(apiStartTime, \"'%a %b %d %T %Y'\"))/60, 1), total_run_time) \n| stats min(_time) as _time first(user) as user max(total_run_time) as total_run_time first(search) as search values(host) as host by search_id \n| search search=search* search!=*_internal* search!=*_audit* | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-search-head-users kpi_id=da-itsi-cp-monitoring-splunk-2fbe52efca0b64a221000379 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ]", "kpi_template_kpi_id": "", "kpi_threshold_template_id": "", "metric_qualifier": "", "metric_search_spec": { "metric_index": "", "metric_name": "" }, "search": "index=_audit action=search (id=* OR search_id=*) \n| eval search_id = if(isnull(search_id), id, search_id) \n| where (apiStartTime!=\"'ZERO_TIME'\" AND apiEndTime!=\"'ZERO_TIME'\") OR isnotnull(total_run_time)\n| replace '*' with * in search_id \n| rex \"search='(?.*?)', autojoin\" \n| search search!=scheduler_*\n| convert num(total_run_time) \n| rex \"\\,\\s+user=(?\\S+),\" \n| eval user = if(user=\"n/a\", null(), user) \n| eval total_run_time = if(isnull(total_run_time), round((strptime(apiEndTime,\"'%a %b %d %T %Y'\")-strptime(apiStartTime, \"'%a %b %d %T %Y'\"))/60, 1), total_run_time) \n| stats min(_time) as _time first(user) as user max(total_run_time) as total_run_time first(search) as search values(host) as host by search_id \n| search search=search* search!=*_internal* search!=*_audit* | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-search-head-users kpi_id=da-itsi-cp-monitoring-splunk-2fbe52efca0b64a221000379 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity(max, total_run_time, \"user\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(user, sec_grp)` | eval serviceid = \"da-itsi-cp-monitoring-splunk-splunk-search-head-users\" | `aggregate_entity_into_service(max)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-search-head-users, da-itsi-cp-monitoring-splunk-2fbe52efca0b64a221000379, true, true, true)` | eval kpi=\"Run Time, Max\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-monitoring-splunk-splunk-search-head-users\" | `assess_urgency` | `gettime`", "search_aggregate": "index=_audit action=search (id=* OR search_id=*) \n| eval search_id = if(isnull(search_id), id, search_id) \n| where (apiStartTime!=\"'ZERO_TIME'\" AND apiEndTime!=\"'ZERO_TIME'\") OR isnotnull(total_run_time)\n| replace '*' with * in search_id \n| rex \"search='(?.*?)', autojoin\" \n| search search!=scheduler_*\n| convert num(total_run_time) \n| rex \"\\,\\s+user=(?\\S+),\" \n| eval user = if(user=\"n/a\", null(), user) \n| eval total_run_time = if(isnull(total_run_time), round((strptime(apiEndTime,\"'%a %b %d %T %Y'\")-strptime(apiStartTime, \"'%a %b %d %T %Y'\"))/60, 1), total_run_time) \n| stats min(_time) as _time first(user) as user max(total_run_time) as total_run_time first(search) as search values(host) as host by search_id \n| search search=search* search!=*_internal* search!=*_audit* | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-search-head-users kpi_id=da-itsi-cp-monitoring-splunk-2fbe52efca0b64a221000379 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_single_value(max, max, total_run_time, \"user\", 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-search-head-users, da-itsi-cp-monitoring-splunk-2fbe52efca0b64a221000379)`", "search_alert": "index=_audit action=search (id=* OR search_id=*) \n| eval search_id = if(isnull(search_id), id, search_id) \n| where (apiStartTime!=\"'ZERO_TIME'\" AND apiEndTime!=\"'ZERO_TIME'\") OR isnotnull(total_run_time)\n| replace '*' with * in search_id \n| rex \"search='(?.*?)', autojoin\" \n| search search!=scheduler_*\n| convert num(total_run_time) \n| rex \"\\,\\s+user=(?\\S+),\" \n| eval user = if(user=\"n/a\", null(), user) \n| eval total_run_time = if(isnull(total_run_time), round((strptime(apiEndTime,\"'%a %b %d %T %Y'\")-strptime(apiStartTime, \"'%a %b %d %T %Y'\"))/60, 1), total_run_time) \n| stats min(_time) as _time first(user) as user max(total_run_time) as total_run_time first(search) as search values(host) as host by search_id \n| search search=search* search!=*_internal* search!=*_audit* | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-search-head-users kpi_id=da-itsi-cp-monitoring-splunk-2fbe52efca0b64a221000379 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity(max, total_run_time, \"user\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(user, sec_grp)` | eval serviceid = \"da-itsi-cp-monitoring-splunk-splunk-search-head-users\" | `aggregate_entity_into_service(max)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-search-head-users, da-itsi-cp-monitoring-splunk-2fbe52efca0b64a221000379, true, true, true)` | eval kpi=\"Run Time, Max\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-monitoring-splunk-splunk-search-head-users\" | `assess_urgency` | `gettime`", "search_alert_earliest": "5", "search_alert_entities": "", "search_buckets": "", "search_entities": "index=_audit action=search (id=* OR search_id=*) \n| eval search_id = if(isnull(search_id), id, search_id) \n| where (apiStartTime!=\"'ZERO_TIME'\" AND apiEndTime!=\"'ZERO_TIME'\") OR isnotnull(total_run_time)\n| replace '*' with * in search_id \n| rex \"search='(?.*?)', autojoin\" \n| search search!=scheduler_*\n| convert num(total_run_time) \n| rex \"\\,\\s+user=(?\\S+),\" \n| eval user = if(user=\"n/a\", null(), user) \n| eval total_run_time = if(isnull(total_run_time), round((strptime(apiEndTime,\"'%a %b %d %T %Y'\")-strptime(apiStartTime, \"'%a %b %d %T %Y'\"))/60, 1), total_run_time) \n| stats min(_time) as _time first(user) as user max(total_run_time) as total_run_time first(search) as search values(host) as host by search_id \n| search search=search* search!=*_internal* search!=*_audit* | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-search-head-users kpi_id=da-itsi-cp-monitoring-splunk-2fbe52efca0b64a221000379 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_single_value(max, max, total_run_time, \"user\", 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-search-head-users, da-itsi-cp-monitoring-splunk-2fbe52efca0b64a221000379)`", "search_occurrences": 1.0, "search_time_compare": "index=_audit action=search (id=* OR search_id=*) \n| eval search_id = if(isnull(search_id), id, search_id) \n| where (apiStartTime!=\"'ZERO_TIME'\" AND apiEndTime!=\"'ZERO_TIME'\") OR isnotnull(total_run_time)\n| replace '*' with * in search_id \n| rex \"search='(?.*?)', autojoin\" \n| search search!=scheduler_*\n| convert num(total_run_time) \n| rex \"\\,\\s+user=(?\\S+),\" \n| eval user = if(user=\"n/a\", null(), user) \n| eval total_run_time = if(isnull(total_run_time), round((strptime(apiEndTime,\"'%a %b %d %T %Y'\")-strptime(apiStartTime, \"'%a %b %d %T %Y'\"))/60, 1), total_run_time) \n| stats min(_time) as _time first(user) as user max(total_run_time) as total_run_time first(search) as search values(host) as host by search_id \n| search search=search* search!=*_internal* search!=*_audit* | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-search-head-users kpi_id=da-itsi-cp-monitoring-splunk-2fbe52efca0b64a221000379 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_and_compare(max, max, total_run_time, \"user\", 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-search-head-users, da-itsi-cp-monitoring-splunk-2fbe52efca0b64a221000379)`", "search_time_series": "index=_audit action=search (id=* OR search_id=*) \n| eval search_id = if(isnull(search_id), id, search_id) \n| where (apiStartTime!=\"'ZERO_TIME'\" AND apiEndTime!=\"'ZERO_TIME'\") OR isnotnull(total_run_time)\n| replace '*' with * in search_id \n| rex \"search='(?.*?)', autojoin\" \n| search search!=scheduler_*\n| convert num(total_run_time) \n| rex \"\\,\\s+user=(?\\S+),\" \n| eval user = if(user=\"n/a\", null(), user) \n| eval total_run_time = if(isnull(total_run_time), round((strptime(apiEndTime,\"'%a %b %d %T %Y'\")-strptime(apiStartTime, \"'%a %b %d %T %Y'\"))/60, 1), total_run_time) \n| stats min(_time) as _time first(user) as user max(total_run_time) as total_run_time first(search) as search values(host) as host by search_id \n| search search=search* search!=*_internal* search!=*_audit* | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-search-head-users kpi_id=da-itsi-cp-monitoring-splunk-2fbe52efca0b64a221000379 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity_time_series(max, total_run_time, \"user\", 5)` | `aggregate_entity_into_service_time_series(max, 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-search-head-users, da-itsi-cp-monitoring-splunk-2fbe52efca0b64a221000379)`", "search_time_series_aggregate": "index=_audit action=search (id=* OR search_id=*) \n| eval search_id = if(isnull(search_id), id, search_id) \n| where (apiStartTime!=\"'ZERO_TIME'\" AND apiEndTime!=\"'ZERO_TIME'\") OR isnotnull(total_run_time)\n| replace '*' with * in search_id \n| rex \"search='(?.*?)', autojoin\" \n| search search!=scheduler_*\n| convert num(total_run_time) \n| rex \"\\,\\s+user=(?\\S+),\" \n| eval user = if(user=\"n/a\", null(), user) \n| eval total_run_time = if(isnull(total_run_time), round((strptime(apiEndTime,\"'%a %b %d %T %Y'\")-strptime(apiStartTime, \"'%a %b %d %T %Y'\"))/60, 1), total_run_time) \n| stats min(_time) as _time first(user) as user max(total_run_time) as total_run_time first(search) as search values(host) as host by search_id \n| search search=search* search!=*_internal* search!=*_audit* | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-search-head-users kpi_id=da-itsi-cp-monitoring-splunk-2fbe52efca0b64a221000379 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity_time_series(max, total_run_time, \"user\", 5)` | `aggregate_entity_into_service_time_series(max, 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-search-head-users, da-itsi-cp-monitoring-splunk-2fbe52efca0b64a221000379)`", "search_time_series_entities": "index=_audit action=search (id=* OR search_id=*) \n| eval search_id = if(isnull(search_id), id, search_id) \n| where (apiStartTime!=\"'ZERO_TIME'\" AND apiEndTime!=\"'ZERO_TIME'\") OR isnotnull(total_run_time)\n| replace '*' with * in search_id \n| rex \"search='(?.*?)', autojoin\" \n| search search!=scheduler_*\n| convert num(total_run_time) \n| rex \"\\,\\s+user=(?\\S+),\" \n| eval user = if(user=\"n/a\", null(), user) \n| eval total_run_time = if(isnull(total_run_time), round((strptime(apiEndTime,\"'%a %b %d %T %Y'\")-strptime(apiStartTime, \"'%a %b %d %T %Y'\"))/60, 1), total_run_time) \n| stats min(_time) as _time first(user) as user max(total_run_time) as total_run_time first(search) as search values(host) as host by search_id \n| search search=search* search!=*_internal* search!=*_audit* | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-search-head-users kpi_id=da-itsi-cp-monitoring-splunk-2fbe52efca0b64a221000379 entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_limited_entity_time_series(max, total_run_time, \"user\", 5)`", "search_type": "shared_base", "service_title": "Splunk Search Head Users", "threshold_eval": "", "threshold_field": "total_run_time", "time_policies": { "policies": { "default_policy": { "aggregate_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "entity_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "policy_type": "static", "time_blocks": [], "title": "Default" } } }, "title": "Run Time, Max", "trending_ad": { "sensitivity": 8 }, "type": "kpis_primary", "tz_offset": null, "unit": "", "urgency": 5.0, "use_time_policies": false }, { "adaptive_thresholding_training_window": "-7d", "adaptive_thresholds_is_enabled": false, "aggregate_eval": "", "aggregate_statop": "sum", "aggregate_threshold_alert_enabled": false, "aggregate_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "aggregate_thresholds_alert_enabled": false, "aggregate_thresholds_custom_alert_enabled": false, "aggregate_thresholds_custom_alert_rules": [], "alert_eval": "", "alert_lag": "5", "alert_on": "both", "alert_period": "5", "anomaly_detection_alerting_enabled": false, "anomaly_detection_is_enabled": false, "anomaly_detection_sensitivity": null, "anomaly_detection_training_window": null, "backfill_earliest_time": "-7d", "backfill_enabled": false, "base_search": "index=_audit action=search (id=* OR search_id=*) \n| eval search_id = if(isnull(search_id), id, search_id) \n| where (apiStartTime!=\"'ZERO_TIME'\" AND apiEndTime!=\"'ZERO_TIME'\") OR isnotnull(total_run_time)\n| replace '*' with * in search_id \n| rex \"search='(?.*?)', autojoin\" \n| search search!=scheduler_*\n| convert num(total_run_time) \n| rex \"\\,\\s+user=(?\\S+),\" \n| eval user = if(user=\"n/a\", null(), user) \n| eval total_run_time = if(isnull(total_run_time), round((strptime(apiEndTime,\"'%a %b %d %T %Y'\")-strptime(apiStartTime, \"'%a %b %d %T %Y'\"))/60, 1), total_run_time) \n| stats min(_time) as _time first(user) as user max(total_run_time) as total_run_time first(search) as search values(host) as host by search_id \n| search search=search* search!=*_internal* search!=*_audit*", "base_search_id": "da-itsi-cp-monitoring-splunk-splk-search-head-usage", "base_search_metric": "user-count", "cohesive_ad": { "sensitivity": 8 }, "cohesive_anomaly_detection_is_enabled": false, "datamodel": { "datamodel": "", "field": "", "object": "", "owner_field": "" }, "datamodel_filter": [], "datamodel_filter_clauses": null, "description": "", "enabled": true, "entity_filter_field": "host", "entity_split_field": "user", "entity_statop": "dc", "entity_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "fill_gaps": "null_value", "gap_custom_alert_value": 0.0, "gap_severity": "unknown", "gap_severity_color": "#CCCCCC", "gap_severity_color_light": "#EEEEEE", "gap_severity_value": "-1", "is_filter_entities_to_service": true, "is_split_by_entity": true, "key": "da-itsi-cp-monitoring-splunk-4c2969076be8390cf68bd6fd", "kpi_base_search": "index=_audit action=search (id=* OR search_id=*) \n| eval search_id = if(isnull(search_id), id, search_id) \n| where (apiStartTime!=\"'ZERO_TIME'\" AND apiEndTime!=\"'ZERO_TIME'\") OR isnotnull(total_run_time)\n| replace '*' with * in search_id \n| rex \"search='(?.*?)', autojoin\" \n| search search!=scheduler_*\n| convert num(total_run_time) \n| rex \"\\,\\s+user=(?\\S+),\" \n| eval user = if(user=\"n/a\", null(), user) \n| eval total_run_time = if(isnull(total_run_time), round((strptime(apiEndTime,\"'%a %b %d %T %Y'\")-strptime(apiStartTime, \"'%a %b %d %T %Y'\"))/60, 1), total_run_time) \n| stats min(_time) as _time first(user) as user max(total_run_time) as total_run_time first(search) as search values(host) as host by search_id \n| search search=search* search!=*_internal* search!=*_audit* | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-search-head-users kpi_id=da-itsi-cp-monitoring-splunk-4c2969076be8390cf68bd6fd entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ]", "kpi_template_kpi_id": "", "kpi_threshold_template_id": "", "metric_qualifier": "", "metric_search_spec": { "metric_index": "", "metric_name": "" }, "search": "index=_audit action=search (id=* OR search_id=*) \n| eval search_id = if(isnull(search_id), id, search_id) \n| where (apiStartTime!=\"'ZERO_TIME'\" AND apiEndTime!=\"'ZERO_TIME'\") OR isnotnull(total_run_time)\n| replace '*' with * in search_id \n| rex \"search='(?.*?)', autojoin\" \n| search search!=scheduler_*\n| convert num(total_run_time) \n| rex \"\\,\\s+user=(?\\S+),\" \n| eval user = if(user=\"n/a\", null(), user) \n| eval total_run_time = if(isnull(total_run_time), round((strptime(apiEndTime,\"'%a %b %d %T %Y'\")-strptime(apiStartTime, \"'%a %b %d %T %Y'\"))/60, 1), total_run_time) \n| stats min(_time) as _time first(user) as user max(total_run_time) as total_run_time first(search) as search values(host) as host by search_id \n| search search=search* search!=*_internal* search!=*_audit* | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-search-head-users kpi_id=da-itsi-cp-monitoring-splunk-4c2969076be8390cf68bd6fd entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity(dc, user, \"user\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(user, sec_grp)` | eval serviceid = \"da-itsi-cp-monitoring-splunk-splunk-search-head-users\" | `aggregate_entity_into_service(sum)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-search-head-users, da-itsi-cp-monitoring-splunk-4c2969076be8390cf68bd6fd, true, true, true)` | eval kpi=\"User Count\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-monitoring-splunk-splunk-search-head-users\" | `assess_urgency` | `gettime`", "search_aggregate": "index=_audit action=search (id=* OR search_id=*) \n| eval search_id = if(isnull(search_id), id, search_id) \n| where (apiStartTime!=\"'ZERO_TIME'\" AND apiEndTime!=\"'ZERO_TIME'\") OR isnotnull(total_run_time)\n| replace '*' with * in search_id \n| rex \"search='(?.*?)', autojoin\" \n| search search!=scheduler_*\n| convert num(total_run_time) \n| rex \"\\,\\s+user=(?\\S+),\" \n| eval user = if(user=\"n/a\", null(), user) \n| eval total_run_time = if(isnull(total_run_time), round((strptime(apiEndTime,\"'%a %b %d %T %Y'\")-strptime(apiStartTime, \"'%a %b %d %T %Y'\"))/60, 1), total_run_time) \n| stats min(_time) as _time first(user) as user max(total_run_time) as total_run_time first(search) as search values(host) as host by search_id \n| search search=search* search!=*_internal* search!=*_audit* | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-search-head-users kpi_id=da-itsi-cp-monitoring-splunk-4c2969076be8390cf68bd6fd entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_single_value(dc, sum, user, \"user\", 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-search-head-users, da-itsi-cp-monitoring-splunk-4c2969076be8390cf68bd6fd)`", "search_alert": "index=_audit action=search (id=* OR search_id=*) \n| eval search_id = if(isnull(search_id), id, search_id) \n| where (apiStartTime!=\"'ZERO_TIME'\" AND apiEndTime!=\"'ZERO_TIME'\") OR isnotnull(total_run_time)\n| replace '*' with * in search_id \n| rex \"search='(?.*?)', autojoin\" \n| search search!=scheduler_*\n| convert num(total_run_time) \n| rex \"\\,\\s+user=(?\\S+),\" \n| eval user = if(user=\"n/a\", null(), user) \n| eval total_run_time = if(isnull(total_run_time), round((strptime(apiEndTime,\"'%a %b %d %T %Y'\")-strptime(apiStartTime, \"'%a %b %d %T %Y'\"))/60, 1), total_run_time) \n| stats min(_time) as _time first(user) as user max(total_run_time) as total_run_time first(search) as search values(host) as host by search_id \n| search search=search* search!=*_internal* search!=*_audit* | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-search-head-users kpi_id=da-itsi-cp-monitoring-splunk-4c2969076be8390cf68bd6fd entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity(dc, user, \"user\")` | eval sec_grp = \"default_itsi_security_group\" | `match_entities(user, sec_grp)` | eval serviceid = \"da-itsi-cp-monitoring-splunk-splunk-search-head-users\" | `aggregate_entity_into_service(sum)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-search-head-users, da-itsi-cp-monitoring-splunk-4c2969076be8390cf68bd6fd, true, true, true)` | eval kpi=\"User Count\", urgency=\"5.0\", alert_period=\"5\", serviceid=\"da-itsi-cp-monitoring-splunk-splunk-search-head-users\" | `assess_urgency` | `gettime`", "search_alert_earliest": "5", "search_alert_entities": "", "search_buckets": "", "search_entities": "index=_audit action=search (id=* OR search_id=*) \n| eval search_id = if(isnull(search_id), id, search_id) \n| where (apiStartTime!=\"'ZERO_TIME'\" AND apiEndTime!=\"'ZERO_TIME'\") OR isnotnull(total_run_time)\n| replace '*' with * in search_id \n| rex \"search='(?.*?)', autojoin\" \n| search search!=scheduler_*\n| convert num(total_run_time) \n| rex \"\\,\\s+user=(?\\S+),\" \n| eval user = if(user=\"n/a\", null(), user) \n| eval total_run_time = if(isnull(total_run_time), round((strptime(apiEndTime,\"'%a %b %d %T %Y'\")-strptime(apiStartTime, \"'%a %b %d %T %Y'\"))/60, 1), total_run_time) \n| stats min(_time) as _time first(user) as user max(total_run_time) as total_run_time first(search) as search values(host) as host by search_id \n| search search=search* search!=*_internal* search!=*_audit* | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-search-head-users kpi_id=da-itsi-cp-monitoring-splunk-4c2969076be8390cf68bd6fd entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_single_value(dc, sum, user, \"user\", 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-search-head-users, da-itsi-cp-monitoring-splunk-4c2969076be8390cf68bd6fd)`", "search_occurrences": 1.0, "search_time_compare": "index=_audit action=search (id=* OR search_id=*) \n| eval search_id = if(isnull(search_id), id, search_id) \n| where (apiStartTime!=\"'ZERO_TIME'\" AND apiEndTime!=\"'ZERO_TIME'\") OR isnotnull(total_run_time)\n| replace '*' with * in search_id \n| rex \"search='(?.*?)', autojoin\" \n| search search!=scheduler_*\n| convert num(total_run_time) \n| rex \"\\,\\s+user=(?\\S+),\" \n| eval user = if(user=\"n/a\", null(), user) \n| eval total_run_time = if(isnull(total_run_time), round((strptime(apiEndTime,\"'%a %b %d %T %Y'\")-strptime(apiStartTime, \"'%a %b %d %T %Y'\"))/60, 1), total_run_time) \n| stats min(_time) as _time first(user) as user max(total_run_time) as total_run_time first(search) as search values(host) as host by search_id \n| search search=search* search!=*_internal* search!=*_audit* | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-search-head-users kpi_id=da-itsi-cp-monitoring-splunk-4c2969076be8390cf68bd6fd entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_and_compare(dc, sum, user, \"user\", 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-search-head-users, da-itsi-cp-monitoring-splunk-4c2969076be8390cf68bd6fd)`", "search_time_series": "index=_audit action=search (id=* OR search_id=*) \n| eval search_id = if(isnull(search_id), id, search_id) \n| where (apiStartTime!=\"'ZERO_TIME'\" AND apiEndTime!=\"'ZERO_TIME'\") OR isnotnull(total_run_time)\n| replace '*' with * in search_id \n| rex \"search='(?.*?)', autojoin\" \n| search search!=scheduler_*\n| convert num(total_run_time) \n| rex \"\\,\\s+user=(?\\S+),\" \n| eval user = if(user=\"n/a\", null(), user) \n| eval total_run_time = if(isnull(total_run_time), round((strptime(apiEndTime,\"'%a %b %d %T %Y'\")-strptime(apiStartTime, \"'%a %b %d %T %Y'\"))/60, 1), total_run_time) \n| stats min(_time) as _time first(user) as user max(total_run_time) as total_run_time first(search) as search values(host) as host by search_id \n| search search=search* search!=*_internal* search!=*_audit* | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-search-head-users kpi_id=da-itsi-cp-monitoring-splunk-4c2969076be8390cf68bd6fd entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity_time_series(dc, user, \"user\", 5)` | `aggregate_entity_into_service_time_series(sum, 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-search-head-users, da-itsi-cp-monitoring-splunk-4c2969076be8390cf68bd6fd)`", "search_time_series_aggregate": "index=_audit action=search (id=* OR search_id=*) \n| eval search_id = if(isnull(search_id), id, search_id) \n| where (apiStartTime!=\"'ZERO_TIME'\" AND apiEndTime!=\"'ZERO_TIME'\") OR isnotnull(total_run_time)\n| replace '*' with * in search_id \n| rex \"search='(?.*?)', autojoin\" \n| search search!=scheduler_*\n| convert num(total_run_time) \n| rex \"\\,\\s+user=(?\\S+),\" \n| eval user = if(user=\"n/a\", null(), user) \n| eval total_run_time = if(isnull(total_run_time), round((strptime(apiEndTime,\"'%a %b %d %T %Y'\")-strptime(apiStartTime, \"'%a %b %d %T %Y'\"))/60, 1), total_run_time) \n| stats min(_time) as _time first(user) as user max(total_run_time) as total_run_time first(search) as search values(host) as host by search_id \n| search search=search* search!=*_internal* search!=*_audit* | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-search-head-users kpi_id=da-itsi-cp-monitoring-splunk-4c2969076be8390cf68bd6fd entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_entity_time_series(dc, user, \"user\", 5)` | `aggregate_entity_into_service_time_series(sum, 5)` | `assess_severity(da-itsi-cp-monitoring-splunk-splunk-search-head-users, da-itsi-cp-monitoring-splunk-4c2969076be8390cf68bd6fd)`", "search_time_series_entities": "index=_audit action=search (id=* OR search_id=*) \n| eval search_id = if(isnull(search_id), id, search_id) \n| where (apiStartTime!=\"'ZERO_TIME'\" AND apiEndTime!=\"'ZERO_TIME'\") OR isnotnull(total_run_time)\n| replace '*' with * in search_id \n| rex \"search='(?.*?)', autojoin\" \n| search search!=scheduler_*\n| convert num(total_run_time) \n| rex \"\\,\\s+user=(?\\S+),\" \n| eval user = if(user=\"n/a\", null(), user) \n| eval total_run_time = if(isnull(total_run_time), round((strptime(apiEndTime,\"'%a %b %d %T %Y'\")-strptime(apiStartTime, \"'%a %b %d %T %Y'\"))/60, 1), total_run_time) \n| stats min(_time) as _time first(user) as user max(total_run_time) as total_run_time first(search) as search values(host) as host by search_id \n| search search=search* search!=*_internal* search!=*_audit* | search [ | rest splunk_server=local report_as=text service_id=da-itsi-cp-monitoring-splunk-splunk-search-head-users kpi_id=da-itsi-cp-monitoring-splunk-4c2969076be8390cf68bd6fd entity_id_fields=host entity_alias_filtering_fields=host search_type=adhoc \"/servicesNS/nobody/SA-ITOA/itoa_interface/generate_entity_filter\" | return $value ] | `aggregate_raw_into_limited_entity_time_series(dc, user, \"user\", 5)`", "search_type": "shared_base", "service_title": "Splunk Search Head Users", "threshold_eval": "", "threshold_field": "user", "time_policies": { "policies": { "default_policy": { "aggregate_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "entity_thresholds": { "base_severity_color": "#99D18B", "base_severity_color_light": "#DCEFD7", "base_severity_label": "normal", "base_severity_value": 2.0, "gauge_max": 100, "gauge_min": 0, "is_max_static": false, "is_min_static": true, "metric_field": "count", "render_boundary_max": 100.0, "render_boundary_min": 0.0, "threshold_levels": [] }, "policy_type": "static", "time_blocks": [], "title": "Default" } } }, "title": "User Count", "trending_ad": { "sensitivity": 8 }, "type": "kpis_primary", "tz_offset": null, "unit": "", "urgency": 5.0, "use_time_policies": false } ], "service_tags": { "tags": [], "template_tags": [] }, "service_template_id": "", "services_depending_on_me": [ { "kpis_depending_on": [ "SHKPI-da-itsi-cp-monitoring-splunk-splunk-search-head-users" ], "service_id": "da-itsi-cp-monitoring-splunk-splunk-search" } ], "services_depends_on": [], "team_id": "default_itsi_security_group", "title": "Splunk Search Head Users", "version": "0.0.38" }