[similar_episodes] default_fields = ["title"] [common_fields] number_of_fields = 50 [migration] kv_store_batch_size = 10000 cluster_manager_check_required = 1 itsi_grouped_alerts_index_lookback = 90 itsi_grouped_alerts_index_search_wait_time = 7200 [precheck] kv_store_collection_size_limit = 1050000 [tracked_alert] sort_notable_events = 0 [ingest_service] notable_events_batch_size = 10000 max_retries = 3 retry_interval = 1 [event_onboarding] preview_results_limit = 300 preview_results_search_wait_time = 10 [export_csv] max_batch_size = 5000 delete_period = 7 [telemetry] latency_query = | search `itsi_grouped_alerts_index` OR `itsi_tracked_alerts_index` earliest=-24h | rename _indextime as it\ | stats earliest(it) as it by index event_id | xyseries event_id index it\ | search itsi_grouped_alerts=* AND itsi_tracked_alerts=* | eval latency=itsi_grouped_alerts-itsi_tracked_alerts\ | fields itsi_tracked_alerts latency | bin itsi_tracked_alerts span=10m\ | stats avg(latency) as eventProcessingLatency queue_enabled_query = | rest splunk_server=local /services/configs/conf-app_common_flags/itsi-rulesengine-adhoc\ | rename disabled as reMode\ | eval reMode = if(reMode == "0", "Adhoc", "RealTime")\ | append [\ | rest splunk_server=local /services/configs/conf-app_common_flags/itsi-rulesengine-queue\ | rename disabled as reMode\ | eval reMode = if(reMode == "0", "Queue", "RealTime")\ ]\ | stats count(eval(reMode="Adhoc")) as AdhocCount, count(eval(reMode="RealTime")) as RealTimeCount, count(eval(reMode="Queue")) as QueueCount\ | rename QueueCount as queueEnabled\ | table queueEnabled cpu_mem_query = | search index="itsi_nats_metrics" sourcetype="varz" earliest=-24h\ | eval memInMB=round(mem/1024/1024,2)\ | stats avg(cpu) as cpuAverage avg(memInMB) as memAverage backfill_rate_query = | search index="itsi_nats_metrics" sourcetype="jsz" earliest=-24h\ | spath input=_raw path=account_details{}.stream_detail{}.consumer_detail{}.num_pending output=num_pending\ | stats sum(num_pending) as numPending\ | eval eventsBackfilledPerMinute = numPending/1440\ | table eventsBackfilledPerMinute events_processed_rate_query = | search index="_internal" source="*itsi_rules_engine*" earliest=-24h | search NOT reMode IN ("Preview") EventId Status\ | stats count(eval(searchmatch("Status=Received PolicyExecutor"))) as totalEventsProcessed\ | eval eventsProcessedPerMinute = totalEventsProcessed/1440\ | table eventsProcessedPerMinute messages_pushed_to_nats_rate_query = | search index="itsi_nats_metrics" sourcetype="jsz" earliest=-24h\ | sort _time\ | spath input=_raw path=account_details{}.stream_detail{}.state.last_seq output=nats_messages_in\ | streamstats current=f last(nats_messages_in) as last_nats_messages_in\ | eval messages_in_data = if(nats_messages_in > last_nats_messages_in, nats_messages_in - last_nats_messages_in, 0)\ | table _time,messages_in_data\ | stats sum(messages_in_data) as messageSum\ | eval eventsIngestedPerMinute = messageSum/1440\ | table eventsIngestedPerMinute rules_engine_start_stop_query = | search index="_internal" source="*itsi_rules_engine*" RulesEngineTask reMode earliest=-24h\ | stats count(eval(searchmatch("RulesEngineTask=RealTimeSearch OR RulesEngineTask=QueueProcessing Status=Started"))) AS "rulesEngineStarted"\ count(eval(searchmatch("RulesEngineTask=RealTimeSearch OR RulesEngineTask=QueueProcessing Status=Stopped"))) AS "rulesEngineStopped"\ | table rulesEngineStarted, rulesEngineStopped