# macros.conf

# auto span
[auto_span]
definition = [ | stats count | addinfo\
| eval earliest = if(info_min_time == "0.000", info_search_time, info_min_time), latest=if(info_max_time == "+Infinity", info_search_time, info_max_time)\
| eval searchStartTIme = strftime(earliest, "%a %d %B %Y %H:%M"), searchEndTime=strftime(latest, "%a %d %B %Y %H:%M")\
| eval Difference = (latest - earliest)\
| eval span = case(\
info_min_time == "0.000", "2m",\
Difference > (3000*24*60*60), "4d",\
Difference > (2000*24*60*60), "3d",\
Difference > (1000*24*60*60), "2d",\
Difference > (500*24*60*60), "1d",\
Difference > (333*24*60*60), "12h",\
Difference > (166*24*60*60), "8h",\
Difference > (83*24*60*60), "4h",\
Difference > (41*24*60*60), "2h",\
Difference > (916*60*60), "1h",\
Difference > (833*60*60), "55m",\
Difference > (750*60*60), "50m",\
Difference > (666*60*60), "45m",\
Difference > (583*60*60), "40m",\
Difference > (500*60*60), "35m",\
Difference > (416*60*60), "30m",\
Difference > (333*60*60), "25m",\
Difference > (250*60*60), "20m",\
Difference > (166*60*60), "15m",\
Difference > (83*60*60), "10m",\
Difference <= (83*60*60), "5m"\
)\
| return span ]
iseval = 0

########################################
# UI related utilities
########################################

[eval_state_icon_code]
definition = eval state_icon_code=case( \
object_state="green" AND ack_state="inactive", "001", \
object_state="green" AND ack_state="active", "002", \
object_state="green", "003", \
object_state="red" AND ack_state="inactive", "004", \
object_state="red" AND ack_state="active", "005", \
object_state="red", "006", \
object_state="orange" AND ack_state="inactive", "007", \
object_state="orange" AND ack_state="active", "008", \
object_state="orange", "009", \
object_state="blue" AND ack_state="inactive", "010", \
object_state="blue" AND ack_state="active", "011", \
object_state="blue", "012" \
)
iseval = false

########################################
# Virtual tenancy and user preferences #
########################################

#
# Shared DSM/DHM/MHM
#

[trackme_return_root_constraint(2)]
definition = inputlookup trackme_virtual_tenants where tenant_id="$tenant_id$" | table tenant_$target$_root_constraint | rename tenant_$target$_root_constraint as constraint | return $constraint
args = tenant_id,target
iseval = 0

#
# Virtual tenants UI
#

[scheduler_completness]
definition = (index=_internal sourcetype=scheduler app="trackme") \
| rex field=savedsearch_name "_tenant_(?<tenant_id>.*)$" \
| lookup trackme_virtual_tenants tenant_id OUTPUT tenant_id as found | where isnotnull(found) | fields - found \
| eval alert_actions=if((isnull(alert_actions) OR (alert_actions == "")),"none",alert_actions) \
| eval status=case(status=="success" OR status=="completed", "completed", status=="skipped", "skipped", status=="continued", "deferred") \
| search (status="completed" OR status="deferred" OR status="skipped") | stats count by status | sort - count | eventstats sum(count) AS total \
| eval percent=(round(((count / total) * 100),2)) | fields - total \
| fields status percent \
| eval status=upper(status) \
| eval color=case( \
status=="COMPLETED", "#45D4BA", \
status=="SKIPPED", "#FBC02D", \
status=="DEFERRED", "#e85b79" \
) \
| append [ | makeresults | head 1 | eval status="NO ACTIVITY YET!", percent="100.00", color="#3c444d" | fields - _time ] \
| streamstats count as result_count \
| eventstats count as total \
| where NOT (status=="NO ACTIVITY YET!" AND total>1) | fields - result_count, total
iseval = false

[scheduler_completness_per_tenant]
definition = (index=_internal sourcetype=scheduler app="trackme") \
| rex field=savedsearch_name "_tenant_(?<tenant_id>.*)$" \
| lookup trackme_virtual_tenants tenant_id OUTPUT tenant_id as found | where isnotnull(found) | fields - found \
| eval alert_actions=if((isnull(alert_actions) OR (alert_actions == "")),"none",alert_actions) \
| eval status=case(((status == "success") OR (status == "completed")),"completed",(status == "skipped"),"skipped",(status == "continued"),"deferred") \
| search (status="completed" OR status="deferred" OR status="skipped") \
| stats count(eval(status=="completed")) as count_completed, count(eval(status=="skipped")) as count_skipped, count by tenant_id, savedsearch_name \
| eval "% completed"=round(((count_completed / count) * 100),2) \
| eval "% completed"=if('% completed'==100, "✅ " . '% completed', "❌ " . '% completed') \
| sort 0 tenant_id, savedsearch_name
iseval = false

[scheduler_completness_overtime]
definition = (index=_internal sourcetype=scheduler app="trackme") \
| rex field=savedsearch_name "_tenant_(?<tenant_id>.*)$" \
| lookup trackme_virtual_tenants tenant_id OUTPUT tenant_id as found | where isnotnull(found) | fields - found \
| eval status=case(status=="success" OR status=="completed", "completed", status=="skipped", "skipped", status=="continued", "deferred") \
| eval alert_actions = if(isnull(alert_actions) OR alert_actions == "", "none", alert_actions) \
| search (status="completed" OR status="deferred" OR status="skipped") \
| timechart minspan=30m count by status
iseval = false

[per_tenant_ops_statusv2(1)]
args = tenant_id
definition = trackmetenantstatus tenant_id="$tenant_id$" \
| eval jsonData=_raw \
| eval status_num=if(last_status="success", 1, 0) \
| eventstats count as job_count, sum(status_num) as status_num, max(last_exec) as tenant_last_exec by tenant_id \
| eval last_status=if((last_status == "success"),"OPERATIONAL","DEGRADED") \
| eval per_job_status = "report=" . report . "|earliest=" . earliest . "|latest=" . latest . "|last_status=" . last_status . "|last_exec=" . strftime(last_exec, "%c") \
| table tenant_id, per_job_status, * \
| eval overall_ops_pct=if(job_count==status_num, 100, round(status_num/job_count*100, 2)) \
| stats first(overall_ops_pct) as overall_ops_pct, first(tenant_last_exec) as tenant_last_exec by tenant_id \
| fillnull value=0 overall_ops_pct \
| sort 0 tenant_id \
| eval status=if((overall_ops_pct == 100),"OPERATIONAL","DEGRADED"), last_exec=strftime(last_exec, "%c") \
| eval tenant_last_exec=strftime(tenant_last_exec, "%c") \
| append \
    [ | trackmetenantstatus tenant_id="$tenant_id$" \
    | eval job_component_register=_raw \
    | fields tenant_id job_component_register ] \
| stats values(tenant_last_exec) as tenant_last_exec, first(overall_ops_pct) as overall_ops_pct, first(status) as status, values(job_component_register) as job_component_register by tenant_id \
| eval job_component_register = "[" . mvjoin(job_component_register, ", ") . "]"
iseval = false

[per_tenant_ops_status_raw(1)]
args = tenant_id
definition = trackmetenantstatus tenant_id="$tenant_id$" \
| eval jsonData=_raw \
| eval status_num=if(last_status="success", 1, 0) \
| eventstats count as job_count, sum(status_num) as status_num, max(last_exec) as tenant_last_exec by tenant_id \
| eval last_status=if((last_status == "success"),"OPERATIONAL","DEGRADED") \
| eval per_job_status = "report=" . report . "|earliest=" . earliest . "|latest=" . latest . "|last_status=" . last_status . "|last_exec=" . strftime(last_exec, "%c") \
| table tenant_id, per_job_status, * \
| eval overall_ops_pct=if(job_count==status_num, 100, round(status_num/job_count*100, 2)) \
| stats first(overall_ops_pct) as overall_ops_pct, first(tenant_last_exec) as tenant_last_exec by tenant_id \
| fillnull value=0 overall_ops_pct \
| sort 0 tenant_id \
| eval status=if((overall_ops_pct == 100),"OPERATIONAL","DEGRADED"), last_exec=strftime(last_exec, "%c") \
| eval tenant_last_exec=strftime(tenant_last_exec, "%c") \
| append \
    [ | trackmetenantstatus tenant_id="$tenant_id$" \
    | eval job_component_register=_raw \
    | fields tenant_id job_component_register ] \
| stats values(tenant_last_exec) as tenant_last_exec, first(overall_ops_pct) as overall_ops_pct, first(status) as status, values(job_component_register) as job_component_register by tenant_id \
| eval job_component_register = "[" . mvjoin(job_component_register, ", ") . "]"
iseval = false

[per_tenant_ops_summary_activity]
definition = (`trackme_audit_idx` OR [ | trackmeload | search tenant_idx_settings!="global" | fields tenant_idx_settings | spath input=tenant_idx_settings | stats count by trackme_audit_idx | dedup trackme_audit_idx | rename trackme_audit_idx as index | fields index | format ] ) [ | inputlookup trackme_virtual_tenants | fields tenant_id | format | fields search ] sourcetype="trackme:health" last_status="failure" | timechart limit=40 minspan=5m count as degraded by tenant_id
iseval = false

[tenant_get_info(2)]
definition = trackmegetcoll tenant_id=$tenant_id$ component=$component$ mode=cachedstats \
``` format as expected by the Virtual Tenants UI ``` \
| eval count = count_total - count_total_disabled, count_red = count_red_critical_priority_enabled + count_red_high_priority_enabled + count_red_other_priority_enabled, count_others = count_blue_enabled + count_orange_enabled \
| rename count_green_enabled as count_green, count_low_enabled as count_low_priority, count_medium_enabled as count_medium_priority, count_high_enabled as count_high_priority, count_critical_enabled as count_critical_priority, count_red_low_priority_enabled as count_red_low_priority, count_red_medium_priority_enabled as count_red_medium_priority, count_red_high_priority_enabled as count_red_high_priority, count_red_critical_priority_enabled as count_red_critical_priority \
| table count, count_green, count_red, count_red_low_priority, count_red_medium_priority, count_red_high_priority, count_red_critical_priority, count_low_priority, count_medium_priority, count_high_priority, count_critical_priority, count_others \
| foreach count, count_green, count_red, count_red_low_priority, count_red_medium_priority, count_red_high_priority, count_red_critical_priority, count_low_priority, count_medium_priority, count_high_priority, count_critical_priority, count_others [ eval <<FIELD>> = if(isnum('<<FIELD>>'), '<<FIELD>>', 0) ]
iseval = false
args = tenant_id, component

[get_tenant_json_dict]
definition = trackmeload | table _raw
iseval = false

[get_tenant_entities_summary(4)]
definition = inputlookup trackme_$component$_tenant_$tenant_id$ \
| eval entity=coalesce(object,object,object,object), entity_state=coalesce(object_state,object_state,object_state,object_state) \
| eval entity_state_search=case(match(entity_state, "(green|red)"), entity_state, match(entity_state, ".*"), "other") \
| search entity_state_search="$status$" \
| search priority="$priority$" \
| sort - 0 latest_flip_time \
| head 100 \
| stats values(entity) as entities \
| eval summary="['" . mvjoin(entities, "', '") . "']" \
| fields summary | rename summary as _raw
args = tenant_id, component, status, priority
iseval = false

[get_tenant_entities_summary_wlk(4)]
definition = inputlookup trackme_$component$_tenant_$tenant_id$ \
| lookup local=t trackme_wlk_apps_enablement_tenant_$tenant_id$ app OUTPUT enabled as app_is_enabled | where NOT app_is_enabled="False" \
| eval entity=coalesce(object,object,object,object), entity_state=coalesce(object_state,object_state,object_state,object_state) \
| eval entity_state_search=case(match(entity_state, "(green|red)"), entity_state, match(entity_state, ".*"), "other") \
| search entity_state_search="$status$" \
| search priority="$priority$" \
| sort - 0 latest_flip_time \
| head 100 \
| stats values(entity) as entities \
| eval summary="['" . mvjoin(entities, "', '") . "']" \
| fields summary | rename summary as _raw
args = tenant_id, component, status, priority
iseval = false

#
# All these macros were replaced in TrackMe 2.0.86 with a pure Python logic for best flexibility, performance and to avoid consuming a new search slot
# To maintain compatibility, these macros now only report a simple stats count as report_entities_count
#

# register tenant component state summary
[register_tenant_component_summary(2)]
definition = fields report_entities_count, report_objects_list
args = tenant_id, component
iseval = false

# register tenant component state summary (version with mv fields to be expanded)
[register_tenant_component_summary(3)]
definition = fields report_entities_count, report_objects_list
args = tenant_id, component, mvfields
iseval = false

# register tenant component state summary (version for composants with no allowlists / blocklists concepts)
[register_tenant_component_summary_nofilter(2)]
definition = fields report_entities_count, report_objects_list
args = tenant_id, component
iseval = false

# register tenant component state summary (version for wlk)
[register_tenant_component_summary_wlk(2)]
definition = fields report_entities_count, report_objects_list
args = tenant_id, component
iseval = false

# register tenant component state summary for splk-fqm collect jobs
[register_tenant_component_summary_fqm_collect_job(2)]
definition = fields report_results_count
args = tenant_id, component
iseval = false

# get tenant knowledge objects
[get_tenants_reports(1)]
definition = rest "/servicesNS/-/trackme/saved/searches" count=0 splunk_server=local \
| where ('eai:acl.app' == "trackme") \
| rex field=title "_tenant_(?<tenant_id>[^\\_]+)" \
| rex field=title "tenant_id\\:(?<tenant_id>[^\\s]+)" \
| rex field=title "^trackme_(?<component>[^\\_]+)_\\w+" \
| eval component=if(match(title,"tenant_id\\:[^\\s]+"),"alert",component) \
| search tenant_id="$tenant_id$" \
| eval type=if(isnotnull('alert.suppress') AND 'alert.suppress'!="", "alerts", "savedsearches") \
| fields + tenant_id, component, type, title \
| append \
    [| rest "/servicesNS/-/-/admin/macros" count=0 splunk_server=local \
    | where ('eai:acl.app' == "trackme") \
    | rex field=title "_tenant_(?<tenant_id>[^\\_]+)" \
    | rex field=title "tenant_id\\:(?<tenant_id>[^\\s]+)" \
    | search tenant_id="$tenant_id$" \
    | rex field=title "^trackme_(?<component>[^\\_]+)_\\w+" \
    | rex field=definition mode=sed "s/\"/\\\"/g" \
    | eval type="macros" \
    | fields + tenant_id, component, type, title ] \
| append \
    [| rest /servicesNS/-/trackme/data/transforms/lookups count=0 splunk_server=local \
    | where ('eai:acl.app' == "trackme") \
    | rex field=title "_tenant_(?<tenant_id>[^\\_]+)" \
    | rex field=title "tenant_id\\:(?<tenant_id>[^\\s]+)" \
    | search tenant_id="$tenant_id$" \
    | rex field=title "^trackme_(?<component>[^\\_]+)_\\w+" \
    | eval type="lookup_definitions" \
    | fields + tenant_id, component, type, title ] \
| append \
    [| rest /servicesNS/nobody/trackme/storage/collections/config count=0 splunk_server=local \
    | where ('eai:acl.app' == "trackme") \
    | rex field=title "_tenant_(?<tenant_id>[^\\_]+)" \
    | rex field=title "tenant_id\\:(?<tenant_id>[^\\s]+)" \
    | search tenant_id="$tenant_id$" \
    | rex field=title "^kv_trackme_(?<component>[^\\_]+)_\\w+" \
    | eval type="kvstore_collections" \
    | fields + tenant_id, component, type, title ]
args = tenant_id
iseval = false

######################
# splk outliers engine 
######################

[set_splk_outliers_rules(2)]
args = tenant_id, component
definition = trackmesplkoutlierssetrules tenant_id="$tenant_id$" component="$component$"
iseval = false

[get_splk_outliers_entities(3)]
args = tenant_id, component, object
definition = trackmesplkoutliersgetrules tenant_id="$tenant_id$" component="$component$" object="$object$" \
``` anything pending is not ready yet ``` \
| where ml_model_render_search!="pending" AND isnotnull(model_id) AND is_disabled=0 \
``` table ``` \
| table object, model_id \
``` lookup the last exec ``` \
| lookup trackme_$component$_outliers_entity_data_tenant_$tenant_id$ object OUTPUT mtime as last_exec_monitor \
``` group models for a same context ``` \
| stats values(model_id) as model_id, min(last_exec_monitor) as last_exec_monitor by object \
| eval model_id=mvjoin(model_id, ",") \
| sort limit=0 - last_exec_monitor
iseval = false

[get_splk_outliers_status(2)]
args = tenant_id, component
definition = lookup trackme_$component$_outliers_entity_data_tenant_$tenant_id$ object_category, object OUTPUT isOutlier, isOutlierReason, models_in_anomaly \
| lookup trackme_$component$_outliers_entity_rules_tenant_$tenant_id$ object_category, object OUTPUT is_disabled AS OutliersDisabled \
| eval OutliersDisabled=if(isnum(OutliersDisabled), OutliersDisabled, 0), isOutlier = if(OutliersDisabled=1, 0, isOutlier)
iseval = False

[splk_outliers_extract_boundaries]
definition = rex field=BoundaryRanges "(-Infinity:(?<LowerBound>[\d|\.]*))|((?<UpperBound>[\d|\.]*):Infinity)"
iseval = False

[splk_outliers_get_model_files_for_tenant(2)]
definition = inputlookup trackme_$component$_outliers_entity_rules_tenant_$tenant_id$ | fields entities_outliers | rex field=entities_outliers "\"ml_model_filename\": \"(?<ml_model_filename>[^\"]*)\"" | where isnotnull(ml_model_filename) AND ml_model_filename!="pending" | fields ml_model_filename
args = tenant_id, component

#
# Deprecated macros get components table
#
# These macros were deprecated in TrackMe 2.0.86 and replaced by a pure Python logic.
# For compatibility purposes, these macros remain available but call the new logic instead.

##################
# splk-dsm table #
##################

# note: since TrackMe 2.0.86, only tenant_id/keyid are used in these macros
[get_splk_dsm_table(2)]
args = tenant_id, keyid
definition = trackmegetcoll tenant_id="$tenant_id$" component="dsm" | fields - _raw | table *
iseval = false

##################
# splk-dhm table #
##################

# note: since TrackMe 2.0.86, only tenant_id/keyid are used in these macros
[get_splk_dhm_table(4)]
args = tenant_id, mode, keyid, object
definition = trackmegetcoll tenant_id="$tenant_id$" component="dhm" | fields - _raw | table *
iseval = 0

##################
# splk-mhm table #
##################

# note: since TrackMe 2.0.86, only tenant_id/keyid are used in these macros
[get_splk_mhm_table(4)]
args = tenant_id, mode, keyid, object
definition = trackmegetcoll tenant_id="$tenant_id$" component="mhm" | fields - _raw | table *
iseval = 0

[get_splk_mhm_table_expanded(3)]
args = tenant_id, keyid, object
definition = inputlookup trackme_mhm_tenant_$tenant_id$ where _key="$keyid$"\
| eval keyid=_key \
| search object="$object$" \
| sort limit=0 object\
| trackmeexpandsplkmhm field_current="metric_details" \
\
| foreach metric_index, metric_category [ eval <<FIELD>> = split('<<FIELD>>', ",") ]\
| eval lag_summary=if(last_lag_seen>60, tostring(metric_last_lag_seen, "duration"), metric_last_lag_seen . " sec")
iseval = 0

##################
# splk-flx table #
##################

# note: since TrackMe 2.0.86, only tenant_id/keyid are used in these macros
[get_splk_flx_table(3)]
args = tenant_id, keyid, object
definition = trackmegetcoll tenant_id="$tenant_id$" component="flx" | fields - _raw | table *
iseval = false

##################
# splk-wlk table #
##################

# note: since TrackMe 2.0.86, only tenant_id/keyid are used in these macros
[get_splk_wlk_table(3)]
args = tenant_id, keyid, object
definition = trackmegetcoll tenant_id="$tenant_id$" component="wlk" | fields - _raw | table *
iseval = false

##########
# Others #
##########

#
# Outliers anomaly detection
#

[trackme_outliers_readiness_status]
definition = eval outliers_readiness = if(outliers_readiness=="True", outliers_readiness, "False")
iseval = 0

#
# get alerts
#

[get_tenant_alerts(1)]
args = tenant_id
definition = rest splunk_server=local /servicesNS/nobody/trackme/saved/searches \
| search eai:acl.app="trackme" alert.track=1 title="TrackMe alert tenant_id:$tenant_id$ - *" \
| fields title, cron_schedule, schedule_window, alert.suppress.fields, alert.suppress.period, disabled, next_scheduled_time, id, actions, alert.track \
| rename alert.suppress.fields as suppress_fields, alert.suppress.period as suppress_period \
| eval actions=trim(actions) \
| makemv actions delim="," \
| eval actions=trim(actions) \
| eval actions=case('alert.track'=1, mvappend(actions, "add_to_triggered"), 1=1, actions) \
| eval actions=mvjoin(mvsort(mvdedup(actions)), ",") \
| rex field=id "saved/searches/(?<id>.*)" \
| sort limit=0 title
iseval = 0

# trackMe global search filter, this filter is applies globally in general searches looking potentially at all TrackMe related indexes
[trackme_idx_search_filter]
definition = [ | trackmegetconf target="index_settings" | spath path=index_settings.trackme_idx_search_filter output=trackme_idx_search_filter | rename trackme_idx_search_filter as index | return index ]
iseval = 0

# trackMe summary index, customise its value to use your own index naming convention, default to summary
[trackme_idx]
definition = [ | trackmegetconf target="index_settings" | spath path=index_settings.trackme_summary_idx output=trackme_summary_idx | rename trackme_summary_idx as index | return index ]
iseval = 0

[trackme_idx(1)]
args = tenant_id
definition = [ | trackme mode=post url=/services/trackme/v2/vtenants/tenant_idx_settings body="{'tenant_id': '$tenant_id$', 'idx_stanza': 'trackme_summary_idx'}" | spath | rename trackme_summary_idx as index | return index ]
iseval = 0

# index target for the notable events
[trackme_notable_idx]
definition = [ | trackmegetconf target="index_settings" | spath path=index_settings.trackme_notable_idx output=trackme_notable_idx | rename trackme_notable_idx as index | return index ]
iseval = false

[trackme_notable_idx(1)]
args = tenant_id
definition = [ | trackme mode=post url=/services/trackme/v2/vtenants/tenant_idx_settings body="{'tenant_id': '$tenant_id$', 'idx_stanza': 'trackme_notable_idx'}" | spath | rename trackme_notable_idx as index | return index ]
iseval = false

# index target for the audit events
[trackme_audit_idx]
definition = [ | trackmegetconf target="index_settings" | spath path=index_settings.trackme_audit_idx output=trackme_audit_idx | rename trackme_audit_idx as index | return index ]
iseval = false

[trackme_audit_idx(1)]
args = tenant_id
definition = [ | trackme mode=post url=/services/trackme/v2/vtenants/tenant_idx_settings body="{'tenant_id': '$tenant_id$', 'idx_stanza': 'trackme_audit_idx'}" | spath | rename trackme_audit_idx as index | return index ]
iseval = false

# trackMe metric index, customise its value depending on your preference
# default to trackme_metrics which has to be created
[trackme_metrics_idx]
definition = [ | trackmegetconf target="index_settings" | spath path=index_settings.trackme_metric_idx output=trackme_metric_idx | rename trackme_metric_idx as index | return index ]
iseval = 0

[trackme_metrics_idx(1)]
args = tenant_id
definition = [ | trackme mode=post url=/services/trackme/v2/vtenants/tenant_idx_settings body="{'tenant_id': '$tenant_id$', 'idx_stanza': 'trackme_metric_idx'}" | spath | rename trackme_metric_idx as index | return index ]
iseval = 0

# Data source availability default monitored state
# customize this macro to change the way the monitored state is defined by default, such as conditional operations
# using the index or sourcetype naming convention

# used as the top of the populating searches for metric indexes
[trackme_mstats_main_filter]
definition = metric_name="*" metric_name!="trackme*"
iseval = 0

[trackme_default_monitored_state]
definition = eval monitored_state=if(isnull(monitored_state), "enabled", monitored_state)
iseval = 0

[trackme_default_host_monitored_state]
definition = eval monitored_state=if(isnull(monitored_state), "enabled", monitored_state)
iseval = 0

[trackme_default_splk_mhm_monitored_state]
definition = eval monitored_state=if(isnull(monitored_state), "enabled", monitored_state)
iseval = 0

[trackme_default_min_dcount_host]
definition = eval min_dcount_host=if(isnull(min_dcount_host), "any", min_dcount_host), min_dcount_field=if(isnull(min_dcount_field), "latest_dcount_host_5m", min_dcount_field) | trackmesplksetcurrentdcounthost
iseval = 0

[trackme_default_lag]
definition = eval default_data_max_lag_allowed=[ | trackmegetconf target="splk_general" | spath path=splk_general.splk_general_dsm_threshold_default output=splk_general_dsm_threshold_default | eval value = "\"" . splk_general_dsm_threshold_default . "\"" | return $value ], data_max_lag_allowed=if(isnum(data_max_lag_allowed), data_max_lag_allowed, default_data_max_lag_allowed) | fields - default_data_max_lag_allowed
iseval = 0

[trackme_default_delay]
definition = eval default_data_max_delay_allowed=[ | trackmegetconf target="splk_general" | spath path=splk_general.splk_general_dsm_delay_default output=splk_general_dsm_delay_default | eval value = "\"" . splk_general_dsm_delay_default . "\"" | return $value ], data_max_delay_allowed=if(isnum(data_max_delay_allowed), data_max_delay_allowed, default_data_max_delay_allowed) | fields - default_data_max_delay_allowed
iseval = 0

[trackme_default_allow_adaptive_delay]
definition = eval allow_adaptive_delay=if(isnull(allow_adaptive_delay), "true", allow_adaptive_delay)
iseval = 0

[trackme_default_host_lag]
definition = eval default_data_max_lag_allowed=[ | trackmegetconf target="splk_general" | spath path=splk_general.splk_general_dhm_threshold_default output=splk_general_dhm_threshold_default | eval value = "\"" . splk_general_dhm_threshold_default . "\"" | return $value ], data_max_lag_allowed=if(isnum(data_max_lag_allowed), data_max_lag_allowed, default_data_max_lag_allowed) | fields - default_data_max_lag_allowed
iseval = 0

[trackme_default_host_delay]
definition = eval default_data_max_delay_allowed=[ | trackmegetconf target="splk_general" | spath path=splk_general.splk_general_dhm_delay_default output=splk_general_dhm_delay_default | eval value = "\"" . splk_general_dhm_delay_default . "\"" | return $value ], data_max_delay_allowed=if(isnum(data_max_delay_allowed), data_max_delay_allowed, default_data_max_delay_allowed) | fields - default_data_max_delay_allowed
iseval = 0

[trackme_default_host_allow_adaptive_delay]
definition = eval allow_adaptive_delay=if(isnull(allow_adaptive_delay), "true", allow_adaptive_delay)
iseval = 0

[trackme_default_splk_mhm_lag]
definition = eval default_metric_max_lag_allowed=[ | trackmegetconf target="splk_general" | spath path=splk_general.splk_general_mhm_threshold_default output=splk_general_mhm_threshold_default | eval value = "\"" . splk_general_mhm_threshold_default . "\"" | return $value ], metric_max_lag_allowed=if(isnum(metric_max_lag_allowed), metric_max_lag_allowed, default_metric_max_lag_allowed) | fields - default_metric_max_lag_allowed
iseval = 0

# deprecated: trackme_default_alert_over_kpis macro has been removed - data_lag_alert_kpis is no longer used

# deprecated: this macro is now deprecated and will be removed in TrackMe 2.2.6
[trackme_default_monitored_wdays]
definition = eval data_monitoring_wdays=if(isnull(data_monitoring_wdays), "auto:all_days", data_monitoring_wdays)
iseval = 0

# deprecated: this macro is now deprecated and will be removed in TrackMe 2.2.6
[trackme_default_monitored_wdays(1)]
args = field
definition = eval $field$=if(isnull($field$), "auto:all_days", $field$)
iseval = 0

# deprecated: this macro is now deprecated and will be removed in TrackMe 2.2.6
[trackme_default_host_monitored_wdays]
definition = eval data_monitoring_wdays=if(isnull(data_monitoring_wdays), "auto:all_days", data_monitoring_wdays)
iseval = 0

# deprecated: this macro is now deprecated and will be removed in TrackMe 2.2.6
[trackme_default_monitored_hours_ranges]
definition = eval data_monitoring_hours_ranges=if(isnull(data_monitoring_hours_ranges), "auto:all_ranges", data_monitoring_hours_ranges)
iseval = 0

# deprecated: this macro is now deprecated and will be removed in TrackMe 2.2.6
[trackme_default_monitored_hours_ranges(1)]
args = field
definition = eval $field$=if(isnull($field$), "auto:all_ranges", $field$)
iseval = 0

# deprecated: this macro is now deprecated and will be removed in TrackMe 2.2.6
[trackme_default_host_monitored_hours_ranges]
definition = eval data_monitoring_hours_ranges=if(isnull(data_monitoring_hours_ranges), "auto:all_ranges", data_monitoring_hours_ranges)
iseval = 0

# deprecation note:, this macro is deprecated from TrackMe 2.0.95, it has been replaced by a Virtual Tenant account option and will be removed in a future release
[trackme_default_priority]
definition = eval priority=if(isnull(priority), "medium", priority)

[trackme_default_priority(1)]
args = tenant_id
definition = eval priority=if(isnull(priority) OR priority="", [ | trackme url="/services/trackme/v2/vtenants/vtenants_accounts" mode="post" body="{'tenant_id': '$tenant_id$'}" | spath | table default_priority | eval default_priority = "\"" . default_priority . "\"" | return $default_priority ], priority)
iseval = 0

# used by the metrics tracker report, can be customised at large scale to reduce computing costs
[trackme_mstats_span]
definition = span=1m
iseval = 0

# can be customized for filtering
[trackme_splk_dsm_filtering]
definition = search object=*
iseval = 0

# can be customized for date format
[trackme_date_format(1)]
args = input_field
definition = eval "$input_field$ (translated)"=strftime($input_field$, "%d %b %Y %H:%M")
iseval = 0

# can be customized for date format
[trackme_multi_date_format(1)]
args = input_fields
definition = foreach $input_fields$ [ eval "<<FIELD>> (translated)"=strftime('<<FIELD>>', "%d %b %Y %H:%M") ]
iseval = 0

# defined pattern filter for indexers
[trackme_idx_filter]
definition = [ | trackmegetconf target="splk_general" | spath path=splk_general.splk_general_idx_filter output=splk_general_idx_filter | return $splk_general_idx_filter ]
iseval = 0

# define the tolerance in negative seconds regarding the detection of data indexed in the future (default 30 seconds)
[trackme_future_indexing_tolerance]
definition = [ | trackmegetconf target="splk_general" | spath path=splk_general.splk_general_feeds_future_tolerance output=splk_general_feeds_future_tolerance | return $splk_general_feeds_future_tolerance ]
iseval = 0

# for alerts, provides macros that can be customised if necessary to order fields upon results

[trackme_alerts_order_splk_dsm]
definition = fields object, data_index, data_sourcetype, object_state, monitored_state, status_message, *
iseval = 0

[trackme_alerts_order_splk_dhm]
definition = fields object, data_index, sourcetype_summary, object_state, monitored_state, status_message, *
iseval = 0

[trackme_alerts_order_splk_mhm]
definition = fields object, metric_index, object_state, monitored_state, status_message, *
iseval = 0

# define a filtering rule for host names, sometimes complex ingestion method that requires host Meta overwrite can fail which results in pollution of wrong hosts
# A host should match a traditional naming convention
# By default: alphanumeric chars, literal dots and hyphens, less than 100 chars
[trackme_splk_dhm_rule_filter(1)]
args = key
definition = where match($key$, "[\w|\-|\.]") AND len($key$)<100
iseval = 0

# Evaluate the icon fields rendering
[trackme_eval_icons]
definition = fillnull value="NA" "data_last_time_seen (translated)", data_last_lag_seen, data_max_lag_allowed, data_max_delay_allowed, data_last_ingestion_lag_seen\
| eval isOutlier=if(isnum(isOutlier), isOutlier, 0)\
| eval state = "icon|" + case(\
object_state=="green" AND isOutlier=0, "ico_good ico_small|icon-check|Good: entity status is green, latest data available is " . 'data_last_time_seen (translated)' . " (" . data_last_lag_seen . " seconds from now), and monitoring conditions are met.",\
object_state=="red" AND isOutlier=0 AND isAnomaly=0 AND (isnum(min_dcount_host) AND min_dcount_threshold<min_dcount_host), "ico_error ico_small|icon-close|Alert: entity status is red, monitoring conditions are not met due to the number of distinct hosts in the entity being below the accepted threshold (min_dcount_host: " . min_dcount_host . " / dcount_host: " . min_dcount_threshold . ").",\
object_state=="red" AND isOutlier=0 AND isAnomaly=0, "ico_error ico_small|icon-close|Alert: entity status is red, monitoring conditions are not met due to lagging or interruption in the data flow, latest data available is " . 'data_last_time_seen (translated)' . " (" . data_last_lag_seen . " seconds from now, duration: " . tostring(round(data_last_lag_seen, 0), "duration") . ") and ingestion latency is approximately " . round(data_last_ingestion_lag_seen, 3) . " seconds (duration: " . tostring(round(data_last_ingestion_lag_seen, 0), "duration") . "), max ingest latency configured is " . data_max_lag_allowed . " seconds, max events delay configured is " . data_max_delay_allowed . " seconds.",\
object_state=="red" AND isOutlier=1 AND isAnomaly=0, "ico_error ico_small|icon-close|Alert: entity status is red, monitoring conditions are not met due to outlier detection, review the the data sampling screen to investigate. For this source, latest data available is " . 'data_last_time_seen (translated)' . " (" . data_last_lag_seen . " seconds from now, duration: " . tostring(round(data_last_lag_seen, 0), "duration") . ") and ingestion latency is approximately " . round(data_last_ingestion_lag_seen, 3) . " seconds (duration: " . tostring(round(data_last_ingestion_lag_seen, 0), "duration") . "), max ingest latency configured is " . data_max_lag_allowed . " seconds, max events delay configured is " . data_max_delay_allowed . " seconds.",\
object_state=="red" AND isAnomaly=1, "ico_error ico_small|icon-close|Alert: entity status is red, monitoring conditions are not met due to anomalies detected in the data sampling and format recognition, review the data sampling screen to investigate. This alert means that trackMe detected an issue in the format of the events compared to the format that was previously identified for this source.",\
object_state=="orange" AND isAnomaly=1, "ico_warn ico_small|icon-close|Warn: entity status is orange, monitoring conditions are not met due to anomalies detected in the data sampling and format recognition, review the data sampling screen to investigate. This alert means that trackMe detected an issue in the format of the events compared to the format that was previously identified for this source. Howener, week days and/or hours ranges rules conditions are not met.",\
object_state=="orange" AND isOutlier=1, "ico_warn ico_small|icon-close|Warn: entity status is orange, monitoring conditions are not met due to outlier detection, review the the data sampling screen to investigate. For this source, latest data available is " . 'data_last_time_seen (translated)' . " (" . data_last_lag_seen . " seconds from now, duration: " . tostring(round(data_last_lag_seen, 0), "duration") . ") and ingestion latency is approximately " . round(data_last_ingestion_lag_seen, 3) . " seconds (duration: " . tostring(round(data_last_ingestion_lag_seen, 0), "duration") . "), max ingest latency configured is " . data_max_lag_allowed . " seconds, max events delay configured is " . data_max_delay_allowed . " seconds.",\
object_state=="orange" AND isnull(object_group_name) AND data_last_lag_seen>=future_tolerance AND (dcount_host>=min_dcount_host OR min_dcount_host="any"), "ico_warn ico_small|icon-close|Warn: entity status is orange, lagging conditions are met, latest data available is " . '	data_last_time_seen (translated)' . " (" . data_last_lag_seen . " seconds from now), max lag configured in seconds is " . data_max_lag_allowed . ". However, week days and/or hours ranges rules conditions are not met.",\
object_state=="orange" AND (min_dcount_threshold<min_dcount_host AND min_dcount_host!="any"), "ico_warn ico_small|icon-close|Warn: entity status is orange, monitoring conditions are not met due to the number of distinct hosts in the entity being below the accepted threshold (min_dcount_host: " . min_dcount_host . " / dcount_host: " . min_dcount_threshold . "), however week days and/or hours ranges rules conditions are not met.",\
object_state=="blue" AND isnotnull(object_group_name) AND data_last_lag_seen>=future_tolerance, "ico_unknown ico_small|icon-close|Info: entity does not honour lagging or week days monitoring conditions however it is a member of a logical group named: " . object_group_name . " which is honouring monitoring rules , the group green status percentage is " . round(object_group_green_percent, 2) . " % which complies with a minimal " . object_group_min_green_percent . " % green members configured for that group." . "(members: " . 	object_group_members_count . "/ red status members count: " . 	object_group_members_red . ", latest data available for the group: " . object_group_last_lag_seen . " seconds from now)",\
object_state=="orange" AND data_last_lag_seen<future_tolerance, "ico_warn ico_small|icon-close|Warn: entity status is orange, detected data indexed in the future which is most likely due to timestamping misconfiguration, timezone or time synchronization issue, latest data available is " . '	data_last_time_seen (translated)' . " (" . data_last_lag_seen . " seconds from now), max lag configured in seconds is " . data_max_lag_allowed . "."),\
monitoring = "icon|" + if(monitored_state=="enabled", "ico_good ico_small|icon-check|Enabled: entity is being actively monitored", "ico_error ico_small|icon-close|Disabled: entity monitoring is disabled")\
| rex field=state "[^\|]*\|[^\|]*\|[^\|]*\|(?<status_message>.*)"
iseval = 0

#
# deprecated: this macro is now replaced by the decision maker and will be removed in a future release
#

[trackme_eval_anomaly_reason_splk_dsm]
definition = fillnull value="NA" "data_last_time_seen (translated)", data_last_lag_seen, data_max_lag_allowed, data_max_delay_allowed, data_last_ingestion_lag_seen\
| eval isOutlier=if(isnum(isOutlier), isOutlier, 0)\
| eval status_message = case(\
object_state=="green", "Good: entity status is green, latest data available is " . 'data_last_time_seen (translated)' . " (" . data_last_lag_seen . " seconds from now), and monitoring conditions are met.",\
object_state=="red" AND isOutlier!=1 AND isAnomaly!=1 AND (isnum(min_dcount_host) AND min_dcount_threshold<min_dcount_host), "Alert: entity status is red, monitoring conditions are not met due to the number of distinct hosts in the entity being below the accepted threshold (min_dcount_host: " . min_dcount_host . " / dcount_host: " . min_dcount_threshold . ").",\
object_state=="red" AND isOutlier!=1 AND isAnomaly!=1, "Alert: entity status is red, monitoring conditions are not met due to lagging or interruption in the data flow, latest data available is " . 'data_last_time_seen (translated)' . " (" . data_last_lag_seen . " seconds from now, duration: " . tostring(round(data_last_lag_seen, 0), "duration") . ") and ingestion latency is approximately " . round(data_last_ingestion_lag_seen, 3) . " seconds (duration: " . tostring(round(data_last_ingestion_lag_seen, 0), "duration") . "), max ingest latency configured is " . data_max_lag_allowed . " seconds, max events delay configured is " . data_max_delay_allowed . " seconds.",\
object_state=="red" AND isOutlier=1 AND isAnomaly=1, "Alert: entity status is red, monitoring conditions are not met due to outlier detection and anomalies detected in the data sampling and format recognition, review the Outlier detection and the data sampling screens to investigate. For this source, latest data available is " . 'data_last_time_seen (translated)' . " (" . data_last_lag_seen . " seconds from now, duration: " . tostring(round(data_last_lag_seen, 0), "duration") . ") and ingestion latency is approximately " . round(data_last_ingestion_lag_seen, 3) . " seconds (duration: " . tostring(round(data_last_ingestion_lag_seen, 0), "duration") . "), max ingest latency configured is " . data_max_lag_allowed . " seconds, max events delay configured is " . data_max_delay_allowed . " seconds.",\
object_state=="red" AND isOutlier=1, "Alert: entity status is red, monitoring conditions are not met due to outlier detection, review the the data sampling screen to investigate. For this source, latest data available is " . 'data_last_time_seen (translated)' . " (" . data_last_lag_seen . " seconds from now, duration: " . tostring(round(data_last_lag_seen, 0), "duration") . ") and ingestion latency is approximately " . round(data_last_ingestion_lag_seen, 3) . " seconds (duration: " . tostring(round(data_last_ingestion_lag_seen, 0), "duration") . "), max ingest latency configured is " . data_max_lag_allowed . " seconds, max events delay configured is " . data_max_delay_allowed . " seconds.",\
object_state=="red" AND isAnomaly=1, "Alert: entity status is red, monitoring conditions are not met due to anomalies detected in the data sampling and format recognition, review the data sampling screen to investigate. This alert means that trackMe detected an issue in the format of the events compared to the format that was previously identified for this source.",\
object_state=="orange" AND isAnomaly=1, "Warn: entity status is orange, monitoring conditions are not met due to anomalies detected in the data sampling and format recognition, review the data sampling screen to investigate. This alert means that trackMe detected an issue in the format of the events compared to the format that was previously identified for this source. Howener, week days and/or hours ranges rules conditions are not met.",\
object_state=="orange" AND isOutlier=1, "Warn: entity status is orange, monitoring conditions are not met due to outlier detection, review the the data sampling screen to investigate. For this source, latest data available is " . 'data_last_time_seen (translated)' . " (" . data_last_lag_seen . " seconds from now, duration: " . tostring(round(data_last_lag_seen, 0), "duration") . ") and ingestion latency is approximately " . round(data_last_ingestion_lag_seen, 3) . " seconds (duration: " . tostring(round(data_last_ingestion_lag_seen, 0), "duration") . "), max ingest latency configured is " . data_max_lag_allowed . " seconds, max events delay configured is " . data_max_delay_allowed . " seconds.",\
object_state=="orange" AND isnull(object_group_name) AND data_last_lag_seen>=future_tolerance AND (dcount_host>=min_dcount_host OR min_dcount_host="any"), "Warn: entity status is orange, lagging conditions are met, latest data available is " . '	data_last_time_seen (translated)' . " (" . data_last_lag_seen . " seconds from now), max lag configured in seconds is " . data_max_lag_allowed . ". However, week days and/or hours ranges rules conditions are not met.",\
object_state=="orange" AND (min_dcount_threshold<min_dcount_host AND min_dcount_host!="any"), "Warn: entity status is orange, monitoring conditions are not met due to the number of distinct hosts in the entity being below the accepted threshold (min_dcount_host: " . min_dcount_host . " / dcount_host: " . min_dcount_threshold . "), however week days and/or hours ranges rules conditions are not met.",\
object_state=="blue" AND isnotnull(object_group_name), "Info: entity does not meet monitoring conditions however it is a member of a logical group named: " . object_group_name . " which is honouring monitoring rules , the group green status percentage is " . round(object_group_green_percent, 2) . " % which complies with a minimal " . object_group_min_green_percent . " % green members configured for that group." . "(members: " . 	object_group_members_count . "/ red status members count: " . 	object_group_members_red . ", latest data available for the group: " . object_group_last_lag_seen . " seconds from now)",\
object_state=="orange" AND data_last_lag_seen<future_tolerance, "Warn: entity status is orange, detected data indexed in the future which is most likely due to timestamping misconfiguration, timezone or time synchronization issue, latest data available is " . '	data_last_time_seen (translated)' . " (" . data_last_lag_seen . " seconds from now), max lag configured in seconds is " . data_max_lag_allowed . ".")\
| eval anomaly_reason = case(\
object_state=="green", "none",\
object_state=="red" AND isOutlier!=1 AND isAnomaly!=1 AND (isnum(min_dcount_host) AND min_dcount_threshold<min_dcount_host), "min_hosts_dcount",\
object_state=="red" AND isOutlier!=1 AND isAnomaly!=1 AND data_last_ingestion_lag_seen>data_max_lag_allowed, "lag_threshold_breached",\
object_state=="red" AND isOutlier!=1 AND isAnomaly!=1 AND data_last_lag_seen>data_max_delay_allowed, "delay_threshold_breached",\
object_state=="red" AND isOutlier=1 AND isAnomaly=1, "ml_outliers_detection,data_sampling_anomaly",\
object_state=="red" AND isOutlier=1, "ml_outliers_detection",\
object_state=="red" AND isAnomaly=1, "data_sampling_anomaly",\
object_state=="orange" AND isOutlier=1 AND isAnomaly=1, "volume_outliers_anomaly_out_of_monitoring,data_sampling_anomaly_out_of_monitoring",\
object_state=="orange" AND isAnomaly=1, "data_sampling_anomaly_out_of_monitoring",\
object_state=="orange" AND isOutlier=1, "volume_outliers_anomaly_out_of_monitoring",\
object_state=="orange" AND isnull(object_group_name) AND data_last_lag_seen>=future_tolerance AND (dcount_host>=min_dcount_host OR min_dcount_host="any"), "lag_threshold_breached_out_of_monitoring",\
object_state=="orange" AND (min_dcount_threshold<min_dcount_host AND min_dcount_host!="any"), "min_hosts_dcount_out_of_monitoring",\
object_state=="blue" AND isnotnull(object_group_name), "in_logical_group",\
object_state=="orange" AND data_last_lag_seen<future_tolerance, "future_over_tolerance")
iseval = 0

[trackme_eval_icons_host]
definition = fillnull value="NA" "data_last_time_seen (translated)", data_last_lag_seen, data_max_lag_allowed, data_max_delay_allowed, data_last_ingestion_lag_seen\
| eval isOutlier=if(isnum(isOutlier), isOutlier, 0)\
| eval state = "icon|" + case(\
object_state=="green", "ico_good ico_small|icon-check|Good: entity status is green, latest data available is " . 'data_last_time_seen (translated)' . " (" . data_last_lag_seen . " seconds from now), and monitoring conditions are met.",\
object_state=="red" AND isnull(object_group_name) AND isOutlier=0 AND ( (splk_dhm_alerting_policy="global_policy" AND default_splk_dhm_alerting_policy="track_per_sourcetype") OR (splk_dhm_alerting_policy="track_per_sourcetype") ) AND match(splk_dhm_st_summary, "\'state\':\s\'red\'"), "ico_error ico_small|icon-close|Alert: entity status is red, monitoring conditions are not met due to lagging or interruption in the data flow for at least one monitored sourcetype for this host, latest data available is " . 'data_last_time_seen (translated)' . " (" . data_last_lag_seen . " seconds from now, duration: " . tostring(round(data_last_lag_seen, 0), "duration") . ") and ingestion latency is approximately " . round(data_last_ingestion_lag_seen, 3) . " seconds (duration: " . tostring(round(data_last_ingestion_lag_seen, 0), "duration") . "), max ingest latency configured is " . data_max_lag_allowed . " seconds, max events delay configured is " . data_max_delay_allowed . " seconds.",\
object_state=="red" AND isnull(object_group_name) AND isOutlier=0, "ico_error ico_small|icon-close|Alert: entity status is red, monitoring conditions are not met due to lagging or interruption in the data flow, latest data available is " . 'data_last_time_seen (translated)' . " (" . data_last_lag_seen . " seconds from now, duration: " . tostring(round(data_last_lag_seen, 0), "duration") . ") and ingestion latency is approximately " . round(data_last_ingestion_lag_seen, 3) . " seconds (duration: " . tostring(round(data_last_ingestion_lag_seen, 0), "duration") . "), max ingest latency configured is " . data_max_lag_allowed . " seconds, max events delay configured is " . data_max_delay_allowed . " seconds.",\
object_state=="red" AND isnull(object_group_name) AND isOutlier=1, "ico_error ico_small|icon-close|Alert: entity status is red, monitoring conditions are not met due to outlier detection, review the the data sampling screen to investigate. For this source, latest data available is " . 'data_last_time_seen (translated)' . " (" . data_last_lag_seen . " seconds from now, duration: " . tostring(round(data_last_lag_seen, 0), "duration") . ") and ingestion latency is approximately " . round(data_last_ingestion_lag_seen, 3) . " seconds (duration: " . tostring(round(data_last_ingestion_lag_seen, 0), "duration") . "), max ingest latency configured is " . data_max_lag_allowed . " seconds, max events delay configured is " . data_max_delay_allowed . " seconds.",\
object_state=="orange" AND isnull(object_group_name) AND isOutlier=1, "ico_warn ico_small|icon-close|Warn: entity status is orange, monitoring conditions are not met due to outlier detection, review the the data sampling screen to investigate. For this source, latest data available is " . 'data_last_time_seen (translated)' . " (" . data_last_lag_seen . " seconds from now, duration: " . tostring(round(data_last_lag_seen, 0), "duration") . ") and ingestion latency is approximately " . round(data_last_ingestion_lag_seen, 3) . " seconds (duration: " . tostring(round(data_last_ingestion_lag_seen, 0), "duration") . "), max ingest latency configured is " . data_max_lag_allowed . " seconds, max events delay configured is " . data_max_delay_allowed . " seconds.",\
object_state=="red" AND isnotnull(object_group_name), "ico_error ico_small|icon-close|Alert: entity does not honour lagging or week days monitoring conditions, in addition it is a member of a logical group named: " . object_group_name . " which is not honouring monitoring rules , the group green status percentage is " . round(object_group_green_percent, 2) . " % which does not comply with a minimal " . object_group_min_green_percent . " % green members configured for that group." . "(members: " . 	object_group_members_count . "/ red status members count: " . 	object_group_members_red . ", latest data available for the group: " . object_group_last_lag_seen . " seconds from now)",\
object_state=="orange" AND isnull(object_group_name) AND data_last_lag_seen>=future_tolerance, "ico_warn ico_small|icon-close|Warn: entity status is orange, lagging conditions are met, latest data available is " . '	data_last_time_seen (translated)' . " (" . data_last_lag_seen . " seconds from now), max lag configured in seconds is " . data_max_lag_allowed . " however week days and/or hours ranges rules conditions are not met.",\
object_state=="blue" AND isnotnull(object_group_name) AND data_last_lag_seen>=future_tolerance, "ico_unknown ico_small|icon-close|Info: entity does not honour lagging or week days monitoring conditions however it is a member of a logical group named: " . object_group_name . " which is honouring monitoring rules , the group green status percentage is " . round(object_group_green_percent, 2) . " % which complies with a minimal " . object_group_min_green_percent . " % green members configured for that group." . "(members: " . 	object_group_members_count . "/ red status members count: " . 	object_group_members_red . ", latest data available for the group: " . object_group_last_lag_seen . " seconds from now)",\
object_state=="orange" AND data_last_lag_seen<future_tolerance, "ico_warn ico_small|icon-close|Warn: entity status is orange, detected data indexed in the future which is most likely due to timestamping misconfiguration, timezone or time synchronization issue, latest data available is " . '	data_last_time_seen (translated)' . " (" . data_last_lag_seen . " seconds from now), max lag configured in seconds is " . data_max_lag_allowed . "."),\
monitoring = "icon|" + if(monitored_state=="enabled", "ico_good ico_small|icon-check|Enabled: entity is being actively monitored", "ico_error ico_small|icon-close|Disabled: entity monitoring is disabled")\
| rex field=state "[^\|]*\|[^\|]*\|[^\|]*\|(?<status_message>.*)"
iseval = 0

[trackme_eval_icons_splk_mhm]
definition = eval state = "icon|" + case(\
object_state=="green", "ico_good ico_small|icon-check|Good: entity status is green, latest data available is " . 'last time' . " (" . metric_last_lag_seen . " seconds from now)",\
object_state=="red" AND isnull(object_group_name), "ico_error ico_small|icon-close|Alert: entity status is red, lagging monitoring conditions are not met, latest data available is " . 'last time' . " (" . metric_last_lag_seen . " seconds from now)",\
object_state=="red" AND isnotnull(object_group_name), "ico_error ico_small|icon-close|Alert: entity does not honour lagging conditions, in addition it is a member of a logical group named: " . object_group_name . " which is not honouring monitoring rules , the group green status percentage is " . round(object_group_green_percent, 2) . " % which does not comply with a minimal " . object_group_min_green_percent . " % green members configured for that group." . "(members: " . 	object_group_members_count . "/ red status members count: " . 	object_group_members_red . ", latest data available for the group: " . object_group_last_lag_seen . " seconds from now)",\
object_state=="blue" AND isnotnull(object_group_name), "ico_unknown ico_small|icon-close|Info: entity does not honour lagging conditions however it is a member of a logical group named: " . object_group_name . " which is honouring monitoring rules , the group green status percentage is " . round(object_group_green_percent, 2) . " % which complies with a minimal " . object_group_min_green_percent . " % green members configured for that group." . "(members: " . 	object_group_members_count . "/ red status members count: " . 	object_group_members_red . ", latest data available for the group: " . object_group_last_lag_seen . " seconds from now)"),\
monitoring = "icon|" + if(monitored_state=="enabled", "ico_good ico_small|icon-check|Enabled: entity is being actively monitored", "ico_error ico_small|icon-close|Disabled: entity monitoring is disabled")\
| rex field=state "[^\|]*\|[^\|]*\|[^\|]*\|(?<status_message>.*)"
iseval = 0

[trackme_eval_icons_object_state_only]
definition = eval state = "icon|" + case(object_state=="green", "ico_good ico_small|icon-check|Up: metric category is available and marked as green due to monitoring rules being met (metric last time seen: " . 	metric_last_time . ", " . metric_current_lag_sec . " seconds from now, which complies with a lagging configured of " . metric_max_lag_allowed . "seconds.)",\
object_state=="red", "ico_error ico_small|icon-close|Down: metric category is not available and therefore marked as down due to rules monitoring (metric last time seen: " . 	metric_last_time . ", " . metric_current_lag_sec . " seconds from now, which does not comply with a lagging configured of " . metric_max_lag_allowed . " seconds.)")
iseval = 0

[trackme_eval_icons_flip]
definition = eval object_previous_state = "icon|" + case(\
object_previous_state=="green", "ico_good ico_small|icon-check|Up: object is available and marked as green due to monitoring rules being met",\
object_previous_state=="red", "ico_error ico_small|icon-close|Down: object is not available or marked as down due to rules monitoring",\
object_previous_state=="orange", "ico_warn ico_small|icon-close|Warn: object is not available but marked as warn due to monitoring rules",\
object_previous_state=="blue", "ico_unknown ico_small|icon-close|Info: object is not available however it is a member of a logical group which monitoring rules are met",\
object_previous_state=="discovered", "ico_unknown ico_small|icon-close|Info: object was discovered and added to the collections"),\
object_state = "icon|" + case(object_state=="green", "ico_good ico_small|icon-check|Up: object is available and marked as green due to monitoring rules being met",\
object_state=="red", "ico_error ico_small|icon-close|Down: object is not available or marked as down due to rules monitoring",\
object_state=="orange", "ico_warn ico_small|icon-close|Warn: object is not available but marked as warn due to monitoring rules",\
object_state=="blue", "ico_unknown ico_small|icon-close|Info: object is not available however it is a member of a logical group which monitoring rules are met")
iseval = 0

[trackme_eval_noicons_flip]
definition = eval object_previous_state = case(\
object_previous_state=="green", "Up: object is available and marked as green due to monitoring rules being met",\
object_previous_state=="red", "Down: object is not available or marked as down due to rules monitoring",\
object_previous_state=="orange", "Warn: object is not available but marked as warn due to monitoring rules",\
object_previous_state=="blue", "Info: object is not available however it is a member of a logical group which monitoring rules are met",\
object_previous_state=="discovered", "Info: object was discovered and added to the collections"),\
object_state = case(object_state=="green", "Up: object is available and marked as green due to monitoring rules being met",\
object_state=="red", "Down: object is not available or marked as down due to rules monitoring",\
object_state=="orange", "Warn: object is not available but marked as warn due to monitoring rules",\
object_state=="blue", "Info: object is not available however it is a member of a logical group which monitoring rules are met")
iseval = 0

[trackme_eval_icons_audit_changes]
definition = eval action_icon = "icon|" + case(\
action=="success", "ico_good ico_small|icon-check|Up: change action was successful",\
action!="success", "ico_error ico_small|icon-close|Down: change action was refused or its status is unknown")
iseval = 0

# generates summary events of latest statuses from the KVstore collections
[trackme_collect_state(3)]
definition = eval current_state=object_state, previous_state=object_previous_state, _time=now(), object=$object$ \
| addinfo | trackmecollect `trackme_idx($tenant_id$)` source="$source$"
args = source,object,tenant_id
iseval = 0

# for optimisations purposes
[trackme_inputlookup(3)]
args = lookup, filter_field, filter_value
definition = inputlookup append=t $lookup$ where ($filter_field$ = "$filter_value$")
iseval = 0

# splk-dsm persistent fields lookup call
[trackme_dsm_lookup_persistent_fields(1)]
args = tenant_id
definition = trackmehashobject input_field="object" output_field="keyid" | lookup local=t trackme_dsm_tenant_$tenant_id$ _key as keyid OUTPUT _key as key, ctime, priority, priority_updated, priority_external, priority_reason, alias, data_max_lag_allowed, data_max_delay_allowed, future_tolerance, monitored_state, monitoring_time_policy, monitoring_time_rules, data_override_lagging_class, min_dcount_host, min_dcount_field, min_dcount_threshold, impact_score_weights, tags_manual, latest_flip_time, latest_flip_state, object_state, tracker_runtime, anomaly_reason as previous_anomaly_reason, data_first_time_seen, outliers_readiness, allow_adaptive_delay, sla_class
iseval = 0

# splk-dsm - abstract version used for tracking
[trackme_dsm_tracker_abstract(2)]
args = tenant_id, search_mode
definition = ``` define the ingestion lag versus now, and a flag field defining an online status any results from live tstats ```\
| eval data_last_lag_seen=now()-data_last_time_seen, splk_dsm_is_online="true"\
\
``` TrackMe metrics: the following line would detect even small amounts of very late indexed events but may be too sensible in Prod context ``` \
```| eval data_last_ingestion_lag_seen=if(avg_latency_5m>data_last_ingestion_lag_seen, avg_latency_5m, data_last_ingestion_lag_seen)``` \
``` Use TrackMe avg_latency_5m metric here ``` \
| eval data_last_ingestion_lag_seen=if(isnum(avg_latency_5m), avg_latency_5m, data_last_ingestion_lag_seen) \
\
``` some rename to keep the same convention across other trackers ```\
| rename data_last_ingestion_lag_seen as live_data_last_ingestion_lag_seen, data_last_ingest as live_data_last_ingest, data_first_time_seen as live_data_first_time_seen, data_last_time_seen as live_data_last_time_seen, data_last_lag_seen as live_data_last_lag_seen, data_eventcount as live_data_eventcount\
\
``` Lookup the current collection, and retrieves the list of persistent fields ```\
\
| `trackme_dsm_lookup_persistent_fields($tenant_id$)`\
\
``` manage live information ```\
| eval data_last_ingest=if(isnotnull(live_data_last_ingest), live_data_last_ingest, data_last_ingest)\
| eval data_first_time_seen=if(isnotnull(data_first_time_seen), data_first_time_seen, live_data_first_time_seen)\
| eval data_last_time_seen=if(isnotnull(live_data_last_time_seen), live_data_last_time_seen, data_last_time_seen)\
| eval data_last_lag_seen=if(isnotnull(live_data_last_lag_seen), live_data_last_lag_seen, data_last_lag_seen)\
| eval data_last_ingestion_lag_seen=if(isnotnull(live_data_last_ingestion_lag_seen), live_data_last_ingestion_lag_seen, data_last_ingestion_lag_seen)\
\
``` note for data_eventcount: to avoid wrongly impacting the status with Outliers detection for cold data, keep the last eventcount ```\
| eval data_eventcount=if(isnotnull(live_data_eventcount), live_data_eventcount, data_eventcount)\
| fields - live_*\
\
``` manage previous values ```\
| rename object_state as object_previous_state, tracker_runtime as previous_tracker_runtime\
\
``` if the key is null, this is the first time we see this source and it will be added to the collection, create a key ```\
| eval key=if(isnull(key), sha256(object), key)\
\
``` set the object alias, if necessary, if remote keep everything but the account information ```\
| eval alias=if(isnull(alias) OR alias="", object, alias)\
| rex field=alias "^remote\|account:[^\|]*\|(?<alias>.*)$"\
\
``` apply default policies ```\
| `trackme_default_monitored_state`\
| `trackme_default_lag`\
| `trackme_default_delay`\
| `trackme_default_allow_adaptive_delay`\
| `trackme_default_priority($tenant_id$)`\
| `trackme_default_min_dcount_host`\
\
``` handle override lagging class ```\
| eval data_override_lagging_class=if(isnull(data_override_lagging_class) OR data_override_lagging_class="null", "false", data_override_lagging_class)\
\
``` lookup any defined rule for max lagging based on index or sourcetype ```\
\
``` match if object=all ```\
| eval lagging_class_object="all" | lookup trackme_common_lagging_classes_tenant_$tenant_id$ name as data_index, object as lagging_class_object OUTPUTNEW value_lag as data_custom_max_lag_allowed, value_delay as data_custom_max_delay_allowed, level as lagging_class_level | eval data_custom_max_lag_allowed=if(lagging_class_level!="index", null(), data_custom_max_lag_allowed), data_custom_max_delay_allowed=if(lagging_class_level!="index", null(), data_custom_max_delay_allowed) | fields - lagging_class_level, lagging_class_object\
``` match if object=splk-dsm ```\
| eval lagging_class_object="splk-dsm" | lookup trackme_common_lagging_classes_tenant_$tenant_id$ name as data_index, object as lagging_class_object OUTPUTNEW value_lag as data_custom_max_lag_allowed, value_delay as data_custom_max_delay_allowed, level as lagging_class_level | eval data_custom_max_lag_allowed=if(lagging_class_level!="index", null(), data_custom_max_lag_allowed), data_custom_max_delay_allowed=if(lagging_class_level!="index", null(), data_custom_max_delay_allowed) | fields - lagging_class_level, lagging_class_object\
\
``` match if object=all ```\
| eval lagging_class_object="all" | lookup trackme_common_lagging_classes_tenant_$tenant_id$ name as data_sourcetype, object as lagging_class_object OUTPUTNEW value_lag as data_custom_max_lag_allowed, value_delay as data_custom_max_delay_allowed, level as lagging_class_level | eval data_custom_max_lag_allowed=if(lagging_class_level!="sourcetype" AND isnotnull(data_custom_max_lag_allowed), null(), data_custom_max_lag_allowed), data_custom_max_delay_allowed=if(lagging_class_level!="sourcetype" AND isnotnull(data_custom_max_delay_allowed), null(), data_custom_max_delay_allowed) | fields - lagging_class_level, lagging_class_object\
``` match if object=splk-dsm ```\
| eval lagging_class_object="splk-dsm" | lookup trackme_common_lagging_classes_tenant_$tenant_id$ name as data_sourcetype, object as lagging_class_object OUTPUTNEW value_lag as data_custom_max_lag_allowed, value_delay as data_custom_max_delay_allowed, level as lagging_class_level | eval data_custom_max_lag_allowed=if(lagging_class_level!="sourcetype" AND isnotnull(data_custom_max_lag_allowed), null(), data_custom_max_lag_allowed), data_custom_max_delay_allowed=if(lagging_class_level!="sourcetype" AND isnotnull(data_custom_max_delay_allowed), null(), data_custom_max_delay_allowed) | fields - lagging_class_level, lagging_class_object\
\
``` match if object=all ```\
| eval lagging_class_object="all" | lookup trackme_common_lagging_classes_tenant_$tenant_id$ name as priority, object as lagging_class_object OUTPUTNEW value_lag as data_custom_max_lag_allowed, value_delay as data_custom_max_delay_allowed, level as lagging_class_level | eval data_custom_max_lag_allowed=if(lagging_class_level!="priority" AND isnotnull(data_custom_max_lag_allowed), null(), data_custom_max_lag_allowed), data_custom_max_delay_allowed=if(lagging_class_level!="priority" AND isnotnull(data_custom_max_delay_allowed), null(), data_custom_max_delay_allowed) | fields - lagging_class_level, lagging_class_object\
``` match if object=splk-dsm ```\
| eval lagging_class_object="splk-dsm" | lookup trackme_common_lagging_classes_tenant_$tenant_id$ name as priority, object as lagging_class_object OUTPUTNEW value_lag as data_custom_max_lag_allowed, value_delay as data_custom_max_delay_allowed, level as lagging_class_level | eval data_custom_max_lag_allowed=if(lagging_class_level!="priority" AND isnotnull(data_custom_max_lag_allowed), null(), data_custom_max_lag_allowed), data_custom_max_delay_allowed=if(lagging_class_level!="priority" AND isnotnull(data_custom_max_delay_allowed), null(), data_custom_max_delay_allowed) | fields - lagging_class_level, lagging_class_object\
\
``` conditionally handle data_max_lag_allowed ```\
| eval data_max_lag_allowed=if(isnum(data_custom_max_lag_allowed) AND data_override_lagging_class!="true", data_custom_max_lag_allowed, data_max_lag_allowed)\
| eval data_max_delay_allowed=if(isnum(data_custom_max_delay_allowed) AND data_override_lagging_class!="true", data_custom_max_delay_allowed, data_max_delay_allowed)\
| fields - data_custom_max_lag_allowed, data_custom_max_delay_allowed\
\
``` exclude any permanent deletion ```\
| `trackme_exclude_permanent_deletion($tenant_id$, "splk-dsm")`\
\
``` calculate last time seen and lag per index ```\
| eventstats max(data_last_time_seen) as data_last_time_seen_idx, min(data_last_lag_seen) as data_last_lag_seen_idx by data_index\
\
``` filter sources ```\
| `trackme_splk_dsm_filtering`\
\
``` define the object_category field which is used by further lookup operations ```\
| eval object_category="splk-dsm"\
\
``` Retrieve Data sampling detection status ```\
| lookup local=t trackme_dsm_data_sampling_tenant_$tenant_id$ object OUTPUT data_sample_feature, data_sample_status_colour, data_sample_mtime\
| eval data_sample_feature=if(isnull(data_sample_feature), "enabled", data_sample_feature)\
| eval data_sample_status_colour=if(isnull(data_sample_status_colour), "green", data_sample_status_colour)\
| eval isAnomaly=if(data_sample_status_colour=="red" AND data_sample_feature=="enabled", 1, 0)\
| eval data_sample_lastrun=if(isnum(data_sample_mtime), data_sample_mtime, 0)\
\
```ML management: handle feeds interruption ``` \
| foreach avg_eventcount_5m, latest_eventcount_5m, perc95_eventcount_5m, stdev_eventcount_5m, avg_dcount_host_5m, latest_dcount_host_5m, perc95_dcount_host_5m, stdev_dcount_host_5m [ eval <<FIELD>>=if(isnum(<<FIELD>>), <<FIELD>>, 0) ] \
\
```Retrieve ML anomaly detection status``` \
| `get_splk_outliers_status($tenant_id$, "dsm")` \
\
``` tracker_runtime is now ```\
| eval tracker_runtime=now()\
\
``` source group feature ```\
| `trackme_splk_dsm_group_lookup($tenant_id$)`\
\
``` Decision maker ``` \
| trackmedecisionmaker tenant_id="$tenant_id$" component="dsm" \
\
``` conditional verifications ```\
| eval object_previous_state=if(isnull(object_previous_state), "discovered", object_previous_state)\
| eval previous_tracker_runtime=if(isnull(previous_tracker_runtime), now(), previous_tracker_runtime)\
| eval latest_flip_state=if(isnull(latest_flip_state), object_previous_state, latest_flip_state)\
| eval latest_flip_time=if(isnull(latest_flip_time), previous_tracker_runtime, latest_flip_time)\
\
``` handle flip ```\
| eval latest_flip_state=if(object_state!=object_previous_state, object_state, latest_flip_state)\
| eval latest_flip_time=if(object_state!=object_previous_state, now(), latest_flip_time)\
\
| where isnotnull(data_last_time_seen)\
| eval data_last_lag_seen=if(splk_dsm_is_online="true", data_last_lag_seen, now()-data_last_time_seen)\
\
``` Apply tags policies ```\
| `trackme_tags_policies_apply($tenant_id$, object)`\
\
``` Set the tenant_id and search_mode ```\
| eval tenant_id="$tenant_id$", search_mode="$search_mode$"
iseval = 0

# splk-dsm - abstract version to be loaded from the collection rather than looking at the data
[trackme_dsm_tracker_abstract(1)]
args = tenant_id
definition = ``` define the ingestion lag versus now, and a flag field defining an online status any results from live tstats ```\
| eval data_last_lag_seen=now()-data_last_time_seen, splk_dsm_is_online="true"\
\
``` TrackMe metrics: the following line would detect even small amounts of very late indexed events but may be too sensible in Prod context ``` \
```| eval data_last_ingestion_lag_seen=if(avg_latency_5m>data_last_ingestion_lag_seen, avg_latency_5m, data_last_ingestion_lag_seen)``` \
``` Use TrackMe avg_latency_5m metric here ``` \
| eval data_last_ingestion_lag_seen=if(isnum(avg_latency_5m), avg_latency_5m, data_last_ingestion_lag_seen) \
\
``` some rename to keep the same convention across other trackers ```\
| rename data_last_ingestion_lag_seen as live_data_last_ingestion_lag_seen, data_last_ingest as live_data_last_ingest, data_first_time_seen as live_data_first_time_seen, data_last_time_seen as live_data_last_time_seen, data_last_lag_seen as live_data_last_lag_seen, data_eventcount as live_data_eventcount\
\
``` Lookup the current collection, and retrieves the list of persistent fields ```\
\
| `trackme_dsm_lookup_persistent_fields($tenant_id$)`\
\
``` manage live information ```\
| eval data_last_ingest=if(isnotnull(live_data_last_ingest), live_data_last_ingest, data_last_ingest)\
| eval data_first_time_seen=if(isnotnull(data_first_time_seen), data_first_time_seen, live_data_first_time_seen)\
| eval data_last_time_seen=if(isnotnull(live_data_last_time_seen), live_data_last_time_seen, data_last_time_seen)\
| eval data_last_lag_seen=if(isnotnull(live_data_last_lag_seen), live_data_last_lag_seen, data_last_lag_seen)\
| eval data_last_ingestion_lag_seen=if(isnotnull(live_data_last_ingestion_lag_seen), live_data_last_ingestion_lag_seen, data_last_ingestion_lag_seen)\
\
``` note for data_eventcount: to avoid wrongly impacting the status with Outliers detection for cold data, keep the last eventcount ```\
| eval data_eventcount=if(isnotnull(live_data_eventcount), live_data_eventcount, data_eventcount)\
| fields - live_*\
\
``` global_dcount_host was later introduced, set to 0 if not present ```\
| eval global_dcount_host=if(isnum(global_dcount_host), global_dcount_host, 0)\
\
``` manage previous values ```\
| rename object_state as object_previous_state, tracker_runtime as previous_tracker_runtime\
\
``` if the key is null, this is the first time we see this source and it will be added to the collection, create a key ```\
| eval key=if(isnull(key), sha256(object), key)\
\
``` set the object alias, if necessary ```\
| eval alias=if(isnull(alias) OR alias="", object, alias)\
\
``` apply default policies ```\
| `trackme_default_monitored_state`\
| `trackme_default_lag`\
| `trackme_default_delay`\
| `trackme_default_allow_adaptive_delay`\
| `trackme_default_priority($tenant_id$)`\
| `trackme_default_min_dcount_host`\
\
``` handle override lagging class ```\
| eval data_override_lagging_class=if(isnull(data_override_lagging_class) OR data_override_lagging_class="null", "false", data_override_lagging_class)\
\
``` lookup any defined rule for max lagging based on index or sourcetype ```\
\
``` match if object=all ```\
| eval lagging_class_object="all" | lookup trackme_common_lagging_classes_tenant_$tenant_id$ name as data_index, object as lagging_class_object OUTPUTNEW value_lag as data_custom_max_lag_allowed, value_delay as data_custom_max_delay_allowed, level as lagging_class_level | eval data_custom_max_lag_allowed=if(lagging_class_level!="index", null(), data_custom_max_lag_allowed), data_custom_max_delay_allowed=if(lagging_class_level!="index", null(), data_custom_max_delay_allowed) | fields - lagging_class_level, lagging_class_object\
``` match if object=splk-dsm ```\
| eval lagging_class_object="splk-dsm" | lookup trackme_common_lagging_classes_tenant_$tenant_id$ name as data_index, object as lagging_class_object OUTPUTNEW value_lag as data_custom_max_lag_allowed, value_delay as data_custom_max_delay_allowed, level as lagging_class_level | eval data_custom_max_lag_allowed=if(lagging_class_level!="index", null(), data_custom_max_lag_allowed), data_custom_max_delay_allowed=if(lagging_class_level!="index", null(), data_custom_max_delay_allowed) | fields - lagging_class_level, lagging_class_object\
\
``` match if object=all ```\
| eval lagging_class_object="all" | lookup trackme_common_lagging_classes_tenant_$tenant_id$ name as data_sourcetype, object as lagging_class_object OUTPUTNEW value_lag as data_custom_max_lag_allowed, value_delay as data_custom_max_delay_allowed, level as lagging_class_level | eval data_custom_max_lag_allowed=if(lagging_class_level!="sourcetype" AND isnotnull(data_custom_max_lag_allowed), null(), data_custom_max_lag_allowed), data_custom_max_delay_allowed=if(lagging_class_level!="sourcetype" AND isnotnull(data_custom_max_delay_allowed), null(), data_custom_max_delay_allowed) | fields - lagging_class_level, lagging_class_object\
``` match if object=splk-dsm ```\
| eval lagging_class_object="splk-dsm" | lookup trackme_common_lagging_classes_tenant_$tenant_id$ name as data_sourcetype, object as lagging_class_object OUTPUTNEW value_lag as data_custom_max_lag_allowed, value_delay as data_custom_max_delay_allowed, level as lagging_class_level | eval data_custom_max_lag_allowed=if(lagging_class_level!="sourcetype" AND isnotnull(data_custom_max_lag_allowed), null(), data_custom_max_lag_allowed), data_custom_max_delay_allowed=if(lagging_class_level!="sourcetype" AND isnotnull(data_custom_max_delay_allowed), null(), data_custom_max_delay_allowed) | fields - lagging_class_level, lagging_class_object\
\
``` match if object=all ```\
| eval lagging_class_object="all" | lookup trackme_common_lagging_classes_tenant_$tenant_id$ name as priority, object as lagging_class_object OUTPUTNEW value_lag as data_custom_max_lag_allowed, value_delay as data_custom_max_delay_allowed, level as lagging_class_level | eval data_custom_max_lag_allowed=if(lagging_class_level!="priority" AND isnotnull(data_custom_max_lag_allowed), null(), data_custom_max_lag_allowed), data_custom_max_delay_allowed=if(lagging_class_level!="priority" AND isnotnull(data_custom_max_delay_allowed), null(), data_custom_max_delay_allowed) | fields - lagging_class_level, lagging_class_object\
``` match if object=splk-dsm ```\
| eval lagging_class_object="splk-dsm" | lookup trackme_common_lagging_classes_tenant_$tenant_id$ name as priority, object as lagging_class_object OUTPUTNEW value_lag as data_custom_max_lag_allowed, value_delay as data_custom_max_delay_allowed, level as lagging_class_level | eval data_custom_max_lag_allowed=if(lagging_class_level!="priority" AND isnotnull(data_custom_max_lag_allowed), null(), data_custom_max_lag_allowed), data_custom_max_delay_allowed=if(lagging_class_level!="priority" AND isnotnull(data_custom_max_delay_allowed), null(), data_custom_max_delay_allowed) | fields - lagging_class_level, lagging_class_object\
\
``` conditionally handle data_max_lag_allowed ```\
| eval data_max_lag_allowed=if(isnum(data_custom_max_lag_allowed) AND data_override_lagging_class!="true", data_custom_max_lag_allowed, data_max_lag_allowed)\
| eval data_max_delay_allowed=if(isnum(data_custom_max_delay_allowed) AND data_override_lagging_class!="true", data_custom_max_delay_allowed, data_max_delay_allowed)\
| fields - data_custom_max_lag_allowed, data_custom_max_delay_allowed\
\
``` calculate last time seen and lag per index ```\
| eventstats max(data_last_time_seen) as data_last_time_seen_idx, min(data_last_lag_seen) as data_last_lag_seen_idx by data_index\
\
``` filter sources ```\
| `trackme_splk_dsm_filtering`\
\
``` define the object_category field which is used by further lookup operations ```\
| eval object_category="splk-dsm"\
\
``` Retrieve Data sampling detection status ```\
| lookup local=t trackme_dsm_data_sampling_tenant_$tenant_id$ object OUTPUT data_sample_feature, data_sample_status_colour, data_sample_mtime\
| eval data_sample_feature=if(isnull(data_sample_feature), "enabled", data_sample_feature)\
| eval data_sample_status_colour=if(isnull(data_sample_status_colour), "green", data_sample_status_colour)\
| eval isAnomaly=if(data_sample_status_colour=="red" AND data_sample_feature=="enabled", 1, 0)\
| eval data_sample_lastrun=if(isnum(data_sample_mtime), data_sample_mtime, 0)\
\
```ML management: handle feeds interruption ``` \
| foreach avg_eventcount_5m, latest_eventcount_5m, perc95_eventcount_5m, stdev_eventcount_5m, avg_dcount_host_5m, latest_dcount_host_5m, perc95_dcount_host_5m, stdev_dcount_host_5m [ eval <<FIELD>>=if(isnum(<<FIELD>>), <<FIELD>>, 0) ] \
\
```Retrieve ML anomaly detection status``` \
| `get_splk_outliers_status($tenant_id$, "dsm")` \
\
``` tracker_runtime is now ```\
| eval tracker_runtime=now()\
\
``` source group feature ```\
| `trackme_splk_dsm_group_lookup($tenant_id$)`\
\
``` Decision maker ``` \
| trackmedecisionmaker tenant_id="$tenant_id$" component="dsm" \
\
``` conditional verifications ```\
| eval object_previous_state=if(isnull(object_previous_state), "discovered", object_previous_state)\
| eval previous_tracker_runtime=if(isnull(previous_tracker_runtime), now(), previous_tracker_runtime)\
| eval latest_flip_state=if(isnull(latest_flip_state), object_previous_state, latest_flip_state)\
| eval latest_flip_time=if(isnull(latest_flip_time), previous_tracker_runtime, latest_flip_time)\
\
``` handle flip ```\
| eval latest_flip_state=if(object_state!=object_previous_state, object_state, latest_flip_state)\
| eval latest_flip_time=if(object_state!=object_previous_state, now(), latest_flip_time)\
\
| where isnotnull(data_last_time_seen)\
| eval data_last_lag_seen=if(splk_dsm_is_online="true", data_last_lag_seen, now()-data_last_time_seen)\
\
``` Apply tags policies ```\
| `trackme_tags_policies_apply($tenant_id$, object)`
iseval = 0

#
# SPLK-FLX
#

# splk-dsm persistent fields lookup call
[trackme_flx_lookup_persistent_fields(1)]
args = tenant_id
definition = trackmehashobject input_field="object" output_field="keyid" | lookup local=t trackme_flx_tenant_$tenant_id$ _key as keyid OUTPUT _key as key, ctime, priority as current_priority, priority_updated, priority_external, priority_reason, alias as current_alias, monitored_state, object_state as object_previous_state, anomaly_reason as previous_anomaly_reason, tracker_runtime as previous_tracker_runtime, latest_flip_time, latest_flip_state, outliers_readiness, sla_class, tags_manual \
| eval priority=coalesce(current_priority, priority)
iseval = 0

# data source macro abstract
[trackme_flx_tracker_abstract(1)]
args = tenant_id
definition = eval object_state = case( \
status=1, "green", \
status=2,"red", \
status=3,"orange", \
isnull(status) OR status=="", "red" \
) \
\
``` lookup persistent fields ``` \
| `trackme_flx_lookup_persistent_fields($tenant_id$)` \
\
``` Preserve tracker-keyed JSON fields before processing other fields ``` \
| eval metrics_preserved=metrics \
| eval status_preserved=status \
| eval status_description_preserved=status_description \
| eval status_description_short_preserved=status_description_short \
| eval object_description_preserved=object_description \
| eval tracker_name_preserved=tracker_name \
| eval tracker_runtime_preserved=tracker_runtime \
| eval disruption_min_time_sec_preserved=disruption_min_time_sec \
| eval max_sec_inactive_preserved=max_sec_inactive \
\
``` For all fields, keep the first value only ``` \
| foreach * [ eval <<FIELD>> = mvindex('<<FIELD>>', 0) ] \
\
``` Restore tracker-keyed JSON fields (use preserved value if available, otherwise keep current) ``` \
| eval metrics=coalesce(metrics_preserved, metrics) \
| eval status=coalesce(status_preserved, status) \
| eval status_description=coalesce(status_description_preserved, status_description) \
| eval status_description_short=coalesce(status_description_short_preserved, status_description_short) \
| eval object_description=coalesce(object_description_preserved, object_description) \
| eval tracker_name=coalesce(tracker_name_preserved, tracker_name) \
| eval tracker_runtime=coalesce(tracker_runtime_preserved, tracker_runtime) \
| eval disruption_min_time_sec=coalesce(disruption_min_time_sec_preserved, disruption_min_time_sec) \
| eval max_sec_inactive=coalesce(max_sec_inactive_preserved, max_sec_inactive) \
| fields - metrics_preserved, status_preserved, status_description_preserved, status_description_short_preserved, object_description_preserved, tracker_name_preserved, tracker_runtime_preserved, disruption_min_time_sec_preserved, max_sec_inactive_preserved \
\
``` if the key is null, this is the first time we see this source and it will be added to the collection, create a key ``` \
| eval key=if(isnull(key), sha256(object), key) \
\
``` for splk-flx, the alias is set by the parser, we use the persistent value if defined ``` \
| eval alias=coalesce(current_alias, alias) \
\
``` apply default policies ``` \
| `trackme_default_monitored_state` \
| `trackme_default_priority($tenant_id$)` \
\
``` exclude any permanent deletion ```\
| `trackme_exclude_permanent_deletion($tenant_id$, "splk-flx")` \
\
```Retrieve ML anomaly detection status``` \
| `get_splk_outliers_status($tenant_id$, "flx")` \
\
``` Logical group ``` \
| `trackme_splk_flx_group_lookup($tenant_id$)`\
\
``` Decision Maker ``` \
| trackmedecisionmaker tenant_id="$tenant_id$" component="flx" \
\
``` tracker_runtime is now ``` \
| eval tracker_runtime=now() \
\
``` conditional verifications ``` \
| eval object_previous_state=if(isnull(object_previous_state), "discovered", object_previous_state) \
| eval previous_tracker_runtime=if(isnull(previous_tracker_runtime), now(), previous_tracker_runtime) \
| eval latest_flip_state=if(isnull(latest_flip_state), object_previous_state, latest_flip_state) \
| eval latest_flip_time=if(isnull(latest_flip_time), previous_tracker_runtime, latest_flip_time) \
\
``` handle flip ``` \
| eval latest_flip_state=if(object_state!=object_previous_state, object_state, latest_flip_state) \
| eval latest_flip_time=if(object_state!=object_previous_state, now(), latest_flip_time) \
\
``` Set the tenant_id ``` \
| eval tenant_id="$tenant_id$"
iseval = 0

#
# SPLK-FQM
#

# splk-fqm persistent fields lookup call
[trackme_fqm_lookup_persistent_fields(1)]
args = tenant_id
definition = lookup local=t trackme_fqm_tenant_$tenant_id$ _key as object_id OUTPUT _key as key, ctime, priority as current_priority, priority_updated, priority_external, priority_reason, alias as current_alias, monitored_state, object_state as object_previous_state, anomaly_reason as previous_anomaly_reason, tracker_runtime as previous_tracker_runtime, latest_flip_time, latest_flip_state, outliers_readiness, sla_class, tags_manual \
| eval priority=coalesce(current_priority, priority)
iseval = 0

# Logical group for splk-fqm
[trackme_splk_fqm_group_lookup(1)]
args = tenant_id
definition = eval object_category="splk-fqm" | lookup trackme_common_logical_group_tenant_$tenant_id$ object_group_members as object OUTPUTNEW _key as object_group_key, object_group_name\
| foreach object_group_key, object_group_name [ eval <<FIELD>> = mvindex(<<FIELD>>, 0) ]
iseval = 0

[trackme_fqm_tracker_abstract(1)]
args = tenant_id
definition = `trackme_fqm_lookup_persistent_fields($tenant_id$)` \
\
``` For all fields, keep the first value only ``` \
| foreach * [ eval <<FIELD>> = mvindex('<<FIELD>>', 0) ] \
\
``` if the key is null, this is the first time we see this source and it will be added to the collection, create a key ``` \
| eval key=if(isnull(key), sha256(object), key) \
\
``` apply default policies ``` \
| `trackme_default_monitored_state` \
| `trackme_default_priority($tenant_id$)` \
\
``` exclude any permanent deletion ```\
| `trackme_exclude_permanent_deletion($tenant_id$, "splk-fqm")` \
\
```Retrieve ML anomaly detection status``` \
| `get_splk_outliers_status($tenant_id$, "fqm")` \
\
``` Logical group ``` \
| `trackme_splk_fqm_group_lookup($tenant_id$)`\
\
``` Decision Maker ``` \
| trackmedecisionmaker tenant_id="$tenant_id$" component="fqm" \
\
``` tracker_runtime is now ``` \
| eval tracker_runtime=now() \
\
``` conditional verifications ``` \
| eval object_previous_state=if(isnull(object_previous_state), "discovered", object_previous_state) \
| eval previous_tracker_runtime=if(isnull(previous_tracker_runtime), now(), previous_tracker_runtime) \
| eval latest_flip_state=if(isnull(latest_flip_state), object_previous_state, latest_flip_state) \
| eval latest_flip_time=if(isnull(latest_flip_time), previous_tracker_runtime, latest_flip_time) \
\
``` handle flip ``` \
| eval latest_flip_state=if(object_state!=object_previous_state, object_state, latest_flip_state) \
| eval latest_flip_time=if(object_state!=object_previous_state, now(), latest_flip_time) \
\
``` Set the tenant_id ``` \
| eval tenant_id="$tenant_id$"
iseval = 0

#
# SPLK-WLK
#

# splk-wlk persistent fields lookup call
[trackme_wlk_lookup_persistent_fields(1)]
args = tenant_id
definition = trackmehashobject input_field="object" output_field="keyid" | lookup local=t trackme_wlk_tenant_$tenant_id$ _key as keyid OUTPUT _key as key, ctime, priority, priority_updated, priority_external, priority_reason, alias, monitored_state, first_seen, sec_since_lastexec, tags_manual, object_state as object_previous_state, anomaly_reason as previous_anomaly_reason, tracker_runtime as previous_tracker_runtime, outliers_readiness, sla_class
iseval = 0

# manage metrics
[trackme_wlk_eval_metrics]
definition = eval skipped_pct=round((('count_skipped' / 'count_execution') * 100), 2) \
| eval skipped_pct=if(skipped_pct>0, skipped_pct, 0) \
| foreach count_completed, count_execution, count_skipped, count_errors, count_ess_notable [ eval <<FIELD>> = round('<<FIELD>>', 0) ] \
| foreach *pct_cpu, *pct_memory, *elapsed, *svc_usage [ eval <<FIELD>> = if('<<FIELD>>'>0, round('<<FIELD>>', 3), 0) ] \
| foreach *scan_count [ eval <<FIELD>> = if(isnum('<<FIELD>>'), round('<<FIELD>>', 0), 0) ]
iseval = 0

# manage metrics v2
[trackme_wlk_eval_metrics_v2]
definition = eval skipped_pct_last_24h=round((('count_skipped_last_24h' / 'count_execution_last_24h') * 100), 2), skipped_pct_last_4h=round((('count_skipped_last_4h' / 'count_execution_last_4h') * 100), 2), skipped_pct_last_60m=round((('count_skipped_last_60m' / 'count_execution_last_60m') * 100), 2) \
| foreach skipped_pct_* [ eval <<FIELD>> = if('<<FIELD>>'>0, '<<FIELD>>', 0) ] \
| foreach count_completed*, count_execution*, count_skipped*, count_errors*, count_ess_notable* [ eval <<FIELD>> = round('<<FIELD>>', 0) ] \
| foreach *pct_cpu*, *pct_memory*, *elapsed*, *svc_usage* [ eval <<FIELD>> = if('<<FIELD>>'>0, round('<<FIELD>>', 3), 0) ] \
| foreach *scan_count* [ eval <<FIELD>> = if(isnum('<<FIELD>>'), round('<<FIELD>>', 0), 0) ]
iseval = 0

# handle no metric utility
[trackme_wlk_null_metrics_json]
definition = foreach metrics_last_* [ eval <<FIELD>> = if(isnotnull('<<FIELD>>'), '<<FIELD>>', "{\"label\": \"<<FIELD>>\", \"metrics\": {}}") ]
iseval = 0

# lookup metadata
[trackme_wlk_lookup_metadata(1)]
args = tenant_id
definition = lookup local=t trackme_wlk_versioning_tenant_$tenant_id$ _key as keyid OUTPUT description as object_description | eval object_description=coalesce(object_description, "No description") \
| lookup local=t trackme_wlk_orphan_status_tenant_$tenant_id$ _key as keyid OUTPUT orphan
iseval = 0

# lookup metadata version_id
[trackme_wlk_lookup_metadata_version_id(1)]
args = tenant_id
definition = lookup local=t trackme_wlk_versioning_tenant_$tenant_id$ _key as object_id OUTPUT current_version_id as version_id | eval version_id=coalesce(version_id, "None")
iseval = 0

# Workload macro abstract
[trackme_wlk_tracker_abstract(1)]
args = tenant_id
definition = where sha256(object) == object_id \
\
| eval object_state = case( \
status=1, "green", \
status=2,"red", \
status=3,"orange", \
isnull(status) OR status=="", "red" \
) \
\
``` lookup collection ``` \
\
| `trackme_wlk_lookup_persistent_fields($tenant_id$)` \
\
``` For all fields, keep the first value only ``` \
| foreach * [ eval <<FIELD>> = mvindex('<<FIELD>>', 0) ] \
\
``` manage first_seen ``` \
| eval first_seen=if(isnum(first_seen), first_seen, now()) \
\
``` set the object alias, if necessary ``` \
| eval alias=if(isnull(alias) OR alias="", object, alias) \
\
``` apply default policies ``` \
| `trackme_default_monitored_state` \
| `trackme_default_priority($tenant_id$)` \
\
``` exclude any permanent deletion ```\
| `trackme_exclude_permanent_deletion($tenant_id$, "splk-wlk")` \
\
```Retrieve ML anomaly detection readiness status``` \
| `get_splk_outliers_status($tenant_id$, "wlk")` \
\
```Retrieve the cron sequence ``` \
| lookup local=t trackme_wlk_versioning_tenant_$tenant_id$ _key as object_id OUTPUT cron_exec_sequence_sec | eval cron_exec_sequence_sec=if(isnum(cron_exec_sequence_sec), cron_exec_sequence_sec, 0) \
\
``` Decision Maker ``` \
| trackmedecisionmaker tenant_id="$tenant_id$" component="wlk" \
\
``` tracker_runtime is now ``` \
| eval tracker_runtime=now() \
\
``` time in sec since last exec ``` \
| eval sec_since_lastexec = tracker_runtime-last_seen \
\
``` conditional verifications ``` \
| eval object_previous_state=if(isnull(object_previous_state), "discovered", object_previous_state) \
| eval previous_tracker_runtime=if(isnull(previous_tracker_runtime), now(), previous_tracker_runtime) \
| eval latest_flip_state=if(isnull(latest_flip_state), object_previous_state, latest_flip_state) \
| eval latest_flip_time=if(isnull(latest_flip_time), previous_tracker_runtime, latest_flip_time) \
\
``` handle flip ``` \
| eval latest_flip_state=if(object_state!=object_previous_state, object_state, latest_flip_state) \
| eval latest_flip_time=if(object_state!=object_previous_state, now(), latest_flip_time) \
\
``` Set the tenant_id ``` \
| eval tenant_id="$tenant_id$"
iseval = 0

# In use within the UI
[trackme_wlk_single_summary(2)]
args = tenant_id, object_id
definition = mstats sum(trackme.splk.wlk.count_execution) as count_execution, sum(trackme.splk.wlk.count_completed) as count_completed, sum(trackme.splk.wlk.count_errors) as count_errors, sum(trackme.splk.wlk.count_skipped) as count_skipped, avg(trackme.splk.wlk.elapsed) as elapsed where `trackme_metrics_idx($tenant_id$)` tenant_id="$tenant_id$" object_category="splk-wlk" object_id="$object_id$" \
| `trackme_parse_duration(elapsed, 60)` \
| foreach count_* [ eval <<FIELD>> = round('<<FIELD>>', 0) ] \
| eval pct_skipped=round(count_skipped/count_execution*100, 2) \
| eval pct_skipped=if(pct_skipped!=0, round(pct_skipped, 2), pct_skipped) \
| fields count_execution, count_errors, pct_skipped, elapsed
iseval = 0

#
# SPLK-DHM abstract
#

[trackme_dhm_tracker_abstract(2)]
args = tenant_id, search_mode
definition = eval host=lower(host), tenant_id="$tenant_id$"\
\
``` TrackMe metrics: the following line would detect even small amounts of very late indexed events but may be too sensible in Prod context ``` \
```| eval data_last_ingestion_lag_seen=if(avg_latency_5m>data_last_ingestion_lag_seen, avg_latency_5m, data_last_ingestion_lag_seen)``` \
``` Use TrackMe avg_latency_5m metric here ``` \
| eval data_last_ingestion_lag_seen=if(isnum(avg_latency_5m), avg_latency_5m, data_last_ingestion_lag_seen) \
\
``` build the st summary ```\
| foreach data_eventcount,data_last_ingest,avg_eventcount_5m,latest_eventcount_5m,perc95_eventcount_5m,avg_latency_5m,latest_latency_5m,perc95_latency_5m,stdev_latency_5m,stdev_eventcount_5m [ eval <<FIELD>> = if(isnum('<<FIELD>>'), '<<FIELD>>', 0) ] \
| eval current_summary = "'idx': '" . index . "', " . "'st': '" . sourcetype . "', " . "'last_eventcount': '" . data_eventcount . "', " . "'last_ingest': '" . data_last_ingest . "', " . "'avg_eventcount_5m': '" . avg_eventcount_5m . "', " . "'latest_eventcount_5m': '" . latest_eventcount_5m . "', " . "'perc95_eventcount_5m': '" . perc95_eventcount_5m . "', " . "'avg_latency_5m': '" . avg_latency_5m . "', " . "'latest_latency_5m': '" . latest_latency_5m . "', " . "'perc95_latency_5m': '" . perc95_latency_5m . "', " . "'stdev_latency_5m': '" . stdev_latency_5m . "', " . "'stdev_eventcount_5m': '" . stdev_eventcount_5m . "', " . "'first_time': '" . data_first_time_seen . "', " . "'last_time': '" . data_last_time_seen . "', " . "'last_ingest_lag': '" . round(data_last_ingestion_lag_seen, 2) . "', " . "'last_event_lag': '" . (now()-data_last_time_seen) . "', " . "'time_measure': '" . now() . "'" \
\
``` handle the summary ```\
\
``` form a unique hash for that host idx / st using sha256 ```\
| eval current_summary =  "'" . sha256(index . ":" . sourcetype) . "': {" . current_summary . "}"\
| stats values(current_summary) as current_summary, first(alias) as alias by host\
| eval current_summary=mvjoin(current_summary, ", ")\
| eval current_summary="{" . current_summary . "}"\
\
``` Retrieve the previous execution dictionnary per host ```\
| lookup trackme_dhm_tenant_$tenant_id$ object as host OUTPUT splk_dhm_st_summary as previous_summary\
| eval previous_summary=if(isnull(previous_summary), "", previous_summary)\
\
``` Call the merging backend ```\
| trackmemergesplkdhm field_host="host" field_current="current_summary" field_previous="previous_summary"\
\
``` spath ```\
| spath input=record\
\
``` lagging policies per index and sourcetype ```\
\
``` for lag ```\
``` match if object=all ```\
| eval lagging_class_object="all" | lookup trackme_common_lagging_classes_tenant_$tenant_id$ name as summary_idx, object as lagging_class_object OUTPUTNEW value_lag as data_custom_max_lag_allowed_per_index, level as data_custom_max_lag_allowed_per_index_level, object as data_custom_max_lag_allowed_per_index_object\
``` match if object=splk-dhm ```\
| eval lagging_class_object="splk-dhm" | lookup trackme_common_lagging_classes_tenant_$tenant_id$ name as summary_idx, object as lagging_class_object OUTPUTNEW value_lag as data_custom_max_lag_allowed_per_index, level as data_custom_max_lag_allowed_per_index_level, object as data_custom_max_lag_allowed_per_index_object\
\
``` match if object=all ```\
| eval lagging_class_object="all" | lookup trackme_common_lagging_classes_tenant_$tenant_id$ name as summary_st, object as lagging_class_object OUTPUTNEW value_lag as data_custom_max_lag_allowed_per_sourcetype, level as data_custom_max_lag_allowed_per_sourcetype_level, object as data_custom_max_lag_allowed_per_sourcetype_object\
``` match if object=splk-dhm ```\
| eval lagging_class_object="splk-dhm" | lookup trackme_common_lagging_classes_tenant_$tenant_id$ name as summary_st, object as lagging_class_object OUTPUTNEW value_lag as data_custom_max_lag_allowed_per_sourcetype, level as data_custom_max_lag_allowed_per_sourcetype_level, object as data_custom_max_lag_allowed_per_sourcetype_object\
\
``` for delay ```\
\
``` match if object=all ```\
| eval lagging_class_object="all" | lookup trackme_common_lagging_classes_tenant_$tenant_id$ name as summary_idx, object as lagging_class_object OUTPUTNEW value_delay as data_custom_max_delay_allowed_per_index, level as data_custom_max_delay_allowed_per_index_level, object as data_custom_max_delay_allowed_per_index_object\
``` match if object=splk-dhm ```\
| eval lagging_class_object="splk-dhm" | lookup trackme_common_lagging_classes_tenant_$tenant_id$ name as summary_idx, object as lagging_class_object OUTPUTNEW value_delay as data_custom_max_delay_allowed_per_index, level as data_custom_max_delay_allowed_per_index_level, object as data_custom_max_delay_allowed_per_index_object\
\
``` match if object=all ```\
| eval lagging_class_object="all" | lookup trackme_common_lagging_classes_tenant_$tenant_id$ name as summary_st, object as lagging_class_object OUTPUTNEW value_delay as data_custom_max_delay_allowed_per_sourcetype, level as data_custom_max_delay_allowed_per_sourcetype_level, object as data_custom_max_delay_allowed_per_sourcetype_object\
``` match if object=splk-dhm ```\
| eval lagging_class_object="splk-dhm" | lookup trackme_common_lagging_classes_tenant_$tenant_id$ name as summary_st, object as lagging_class_object OUTPUTNEW value_delay as data_custom_max_delay_allowed_per_sourcetype, level as data_custom_max_delay_allowed_per_sourcetype_level, object as data_custom_max_delay_allowed_per_sourcetype_object\
\
| eval data_custom_max_lag_allowed_per_index=if(data_custom_max_lag_allowed_per_index_level!="index", null(), data_custom_max_lag_allowed_per_index), data_custom_max_lag_allowed_per_sourcetype=if(data_custom_max_lag_allowed_per_sourcetype_level!="sourcetype", null(), data_custom_max_lag_allowed_per_sourcetype)\
| eval data_custom_max_lag_allowed_per_index=if(isnull(data_custom_max_lag_allowed_per_index_object) OR data_custom_max_lag_allowed_per_index_object="all" OR data_custom_max_lag_allowed_per_index_object="splk-dhm", data_custom_max_lag_allowed_per_index, null())\
| eval data_custom_max_lag_allowed_per_sourcetype=if(isnull(data_custom_max_lag_allowed_per_sourcetype_object) OR data_custom_max_lag_allowed_per_sourcetype_object="all" OR data_custom_max_lag_allowed_per_sourcetype_object="splk-dhm", data_custom_max_lag_allowed_per_sourcetype, null())\
| fields - data_custom_max_lag_allowed_per_index_level, data_custom_max_lag_allowed_per_sourcetype_level, data_custom_max_lag_allowed_per_index_object, data_custom_max_lag_allowed_per_sourcetype_object\
\
| eval data_custom_max_delay_allowed_per_index=if(data_custom_max_delay_allowed_per_index_level!="index", null(), data_custom_max_delay_allowed_per_index), data_custom_max_delay_allowed_per_sourcetype=if(data_custom_max_delay_allowed_per_sourcetype_level!="sourcetype", null(), data_custom_max_delay_allowed_per_sourcetype)\
| eval data_custom_max_delay_allowed_per_index=if(isnull(data_custom_max_delay_allowed_per_index_object) OR data_custom_max_delay_allowed_per_index_object="all" OR data_custom_max_delay_allowed_per_index_object="splk-dhm", data_custom_max_delay_allowed_per_index, null())\
| eval data_custom_max_delay_allowed_per_sourcetype=if(isnull(data_custom_max_delay_allowed_per_sourcetype_object) OR data_custom_max_delay_allowed_per_sourcetype_object="all" OR data_custom_max_delay_allowed_per_sourcetype_object="splk-dhm", data_custom_max_delay_allowed_per_sourcetype, null())\
| fields - data_custom_max_delay_allowed_per_index_level, data_custom_max_delay_allowed_per_sourcetype_level, data_custom_max_delay_allowed_per_index_object, data_custom_max_delay_allowed_per_sourcetype_object\
\
``` for any match within the lagging policy, the highest lagging value has the precedence ```\
| fillnull value=0 data_custom_max_lag_allowed_per_index, data_custom_max_lag_allowed_per_sourcetype\
| eval data_custom_max_lag_allowed=max(data_custom_max_lag_allowed_per_index, data_custom_max_lag_allowed_per_sourcetype)\
\
| fillnull value=0 data_custom_max_delay_allowed_per_index, data_custom_max_delay_allowed_per_sourcetype\
| eval data_custom_max_delay_allowed=max(data_custom_max_delay_allowed_per_index, data_custom_max_delay_allowed_per_sourcetype)\
\
``` Get default lag and latency thresholds ```\
| `trackme_default_host_lag`\
| `trackme_default_host_delay`\
| `trackme_default_host_allow_adaptive_delay`\
\
``` lookup existing entity values ```\
| trackmehashobject input_field="host" output_field="keyid" | lookup trackme_dhm_tenant_$tenant_id$ _key as keyid OUTPUT _key as key, alias as current_alias, monitored_state, data_override_lagging_class, data_max_lag_allowed as current_host_level_data_max_lag_allowed, data_max_delay_allowed as current_host_level_data_max_delay_allowed, splk_dhm_alerting_policy, monitoring_time_policy, monitoring_time_rules, tags_manual, priority, priority_updated, priority_external, priority_reason, future_tolerance, object_state as object_previous_state, anomaly_reason as previous_anomaly_reason, tracker_runtime as previous_tracker_runtime, latest_flip_state, latest_flip_time, outliers_readiness, host_idx_blocklists, host_st_blocklists, impact_score_weights\
| eval alias=if(isnotnull(current_alias) AND current_alias!="", current_alias, alias) | fields - current_alias\
\
| eval summary_max_lag_allowed=if(isnum(data_custom_max_lag_allowed) AND data_custom_max_lag_allowed>0, data_custom_max_lag_allowed, data_max_lag_allowed)\
| fields - data_custom_max_lag_allowed_per_index, data_custom_max_lag_allowed_per_sourcetype\
\
| eval summary_max_delay_allowed=if(isnum(data_custom_max_delay_allowed) AND data_custom_max_delay_allowed>0, data_custom_max_delay_allowed, data_max_delay_allowed)\
| fields - data_custom_max_delay_allowed_per_index, data_custom_max_delay_allowed_per_sourcetype\
\
``` Handle delay and latency for the global host level ```\
\
| eventstats max(summary_max_lag_allowed) as host_summary_max_lag_allowed, max(summary_max_delay_allowed) as host_summary_max_delay_allowed by host\
\
``` host level values always win ``` \
| eval data_max_lag_allowed=if(isnum(current_host_level_data_max_lag_allowed) AND current_host_level_data_max_lag_allowed!=host_summary_max_delay_allowed, current_host_level_data_max_lag_allowed, host_summary_max_lag_allowed) \
| eval data_max_delay_allowed=if(isnum(current_host_level_data_max_delay_allowed) AND current_host_level_data_max_delay_allowed!=host_summary_max_delay_allowed, current_host_level_data_max_delay_allowed, host_summary_max_delay_allowed) \
\
``` define the status per sourcetype for the source summary mvfield ```\
| eval summary_st_state=if(summary_last_event_lag>summary_max_delay_allowed OR summary_last_ingest_lag>summary_max_lag_allowed, "red", "green")\
| eval splk_dhm_st_summary = "'idx': '" . summary_idx . "', " . "'st': '" . summary_st . "', " . "'last_eventcount': '" . summary_last_eventcount . "', " . "'max_lag_allowed': '" . summary_max_lag_allowed . "', " . "'max_delay_allowed': '" . summary_max_delay_allowed . "', " . "'last_ingest': '" . summary_last_ingest . "', " . "'first_time': '" . summary_first_time . "', " . "'last_time': '" . summary_last_time . "', " . "'last_ingest_lag': '" . summary_last_ingest_lag . "', " . "'last_event_lag': '" . (now()-summary_last_time) . "', " . "'time_measure': '" . now() . "', " . "'state': '" . summary_st_state . "'"\
| eval splk_dhm_st_summary =  "'" . hash . "': {" . splk_dhm_st_summary . "}"\
\
``` perform intermediate calculation table ```\
| stats first(key) as key, first(alias) as alias, max(summary_last_ingest) as data_last_ingest, min(summary_first_time) as data_first_time_seen,\
max(summary_last_time) as data_last_time_seen, avg(summary_last_ingest_lag) as data_last_ingestion_lag_seen,\
values(summary_idx) as data_index, values(summary_st) as data_sourcetype,\
max(data_custom_max_lag_allowed) as data_custom_max_lag_allowed, max(data_max_lag_allowed) as data_max_lag_allowed, max(data_max_delay_allowed) as data_max_delay_allowed, max(summary_max_lag_allowed) as summary_max_lag_allowed, max(summary_max_delay_allowed) as summary_max_delay_allowed, values(splk_dhm_st_summary) as splk_dhm_st_summary, \
sum(summary_last_eventcount) as data_eventcount, first(splk_dhm_alerting_policy) as splk_dhm_alerting_policy, \
first(object_state) as object_state, first(data_last_lag_seen) as data_last_lag_seen, first(monitored_state) as monitored_state, \
first(data_override_lagging_class) as data_override_lagging_class, first(object_previous_state) as object_previous_state, \
first(monitoring_time_policy) as monitoring_time_policy, first(monitoring_time_rules) as monitoring_time_rules, \
first(previous_tracker_runtime) as previous_tracker_runtime, first(tracker_runtime) as tracker_runtime, first(isOutlier) as isOutlier, first(outliers_readiness) as outliers_readiness\
first(latest_flip_state) as latest_flip_state, first(latest_flip_time) as latest_flip_time, first(object_category) as object_category, first(priority) as priority, first(priority_updated) as priority_updated, first(priority_external) as priority_external, first(priority_reason) as priority_reason, first(future_tolerance) as future_tolerance,\
first(sla_class) as sla_class, first(tags_manual) as tags_manual,\
first(impact_score_weights) as impact_score_weights,\
first(host_idx_blocklists) as host_idx_blocklists, first(host_st_blocklists) as host_st_blocklists, \
sum(avg_eventcount_5m) as avg_eventcount_5m, sum(latest_eventcount_5m) as latest_eventcount_5m, \
sum(perc95_eventcount_5m) as perc95_eventcount_5m, sum(stdev_eventcount_5m) as stdev_eventcount_5m, \
avg(perc95_latency_5m) as perc95_latency_5m, avg(stdev_latency_5m) as stdev_latency_5m, \
avg(avg_latency_5m) as avg_latency_5m, avg(latest_latency_5m) as latest_latency_5m by host \
\
| eval data_last_ingestion_lag_seen=round(data_last_ingestion_lag_seen, 0)\
\
``` define the ingestion lag versus now, and a flag field defining an online status any results from live tstats ```\
| eval data_last_lag_seen=now()-data_last_time_seen, splk_dhm_is_online="true"\
\
``` rename host and lower case ```\
| rename host as object | eval object=lower(object)\
\
``` exclude any permanent deletion ```\
| `trackme_exclude_permanent_deletion($tenant_id$, "splk-dhm")`\
\
``` if the key is null, this is the first time we see this source and it will be added to the collection, create a key ```\
| eval key=if(isnull(key), sha256(object), key)\
\
``` set the object alias, if necessary ```\
| eval alias=if(isnull(alias) OR alias="", object, alias)\
\
``` apply various macro defining default values and policies ```\
| `trackme_default_host_monitored_state`\
| `trackme_default_host_monitored_wdays`\
| `trackme_default_host_monitored_hours_ranges`\
| `trackme_default_priority($tenant_id$)`\
\
``` support for priority based lagging classes ```\
\
``` for lag ``` \
\
``` match if object=all ```\
| eval lagging_class_object="all" | lookup trackme_common_lagging_classes_tenant_$tenant_id$ name as priority, object as lagging_class_object OUTPUT value_lag as data_custom_max_lag_allowed_per_priority, level as data_custom_max_lag_allowed_per_priority_level, object as data_custom_max_lag_allowed_per_priority_object\
``` match if object=splk-dhm ```\
| eval lagging_class_object="splk-dhm" | lookup trackme_common_lagging_classes_tenant_$tenant_id$ name as priority, object as lagging_class_object OUTPUTNEW value_lag as data_custom_max_lag_allowed_per_priority, level as data_custom_max_lag_allowed_per_priority_level, object as data_custom_max_lag_allowed_per_priority_object\
\
``` for delay ```\
\
``` match if object=all ```\
| eval lagging_class_object="all" | lookup trackme_common_lagging_classes_tenant_$tenant_id$ name as priority, object as lagging_class_object OUTPUT value_delay as data_custom_max_delay_allowed_per_priority, level as data_custom_max_delay_allowed_per_priority_level, object as data_custom_max_delay_allowed_per_priority_object\
``` match if object=splk-dhm ```\
| eval lagging_class_object="splk-dhm" | lookup trackme_common_lagging_classes_tenant_$tenant_id$ name as priority, object as lagging_class_object OUTPUTNEW value_delay as data_custom_max_delay_allowed_per_priority, level as data_custom_max_delay_allowed_per_priority_level, object as data_custom_max_delay_allowed_per_priority_object\
\
| eval data_max_lag_allowed=case(\
data_override_lagging_class="true", data_max_lag_allowed,\
data_override_lagging_class!="true", max(data_max_lag_allowed, data_custom_max_lag_allowed_per_priority),\
1=1, data_max_lag_allowed\
)\
| eval data_max_delay_allowed=case(\
data_override_lagging_class="true", data_max_delay_allowed,\
data_override_lagging_class!="true", max(data_max_delay_allowed, data_custom_max_delay_allowed_per_priority),\
1=1, data_max_delay_allowed\
)\
\
``` if data_override_lagging_class is defined ```\
| eval data_override_lagging_class=if(isnull(data_override_lagging_class) OR data_override_lagging_class="null", "false", data_override_lagging_class)\
\
``` create a comma separated list of known indexes and sourcetype for live/lookup merge in next phase ```\
| eval data_index=mvjoin(data_index, ","), data_sourcetype=mvjoin(data_sourcetype, ",")\
\
``` define the object_category field which is used by further lookup operations ```\
| eval object_category="splk-dhm"\
\
``` source group feature ```\
| `trackme_splk_dhm_group_lookup($tenant_id$)`\
\
```ML management: handle feeds interruption ``` \
| foreach avg_eventcount_5m, latest_eventcount_5m, perc95_eventcount_5m, stdev_eventcount_5m [ eval <<FIELD>>=if(isnum(<<FIELD>>), <<FIELD>>, 0) ] \
\
```Retrieve ML anomaly detection status``` \
| `get_splk_outliers_status($tenant_id$, "dhm")` \
\
``` Decision maker ```\
| trackmedecisionmaker tenant_id="$tenant_id$" component="dhm" \
\
``` define tracker_runtime ```\
| eval tracker_runtime=now()\
\
``` conditional verifications ```\
| eval object_previous_state=if(isnull(object_previous_state), "discovered", object_previous_state)\
| eval previous_tracker_runtime=if(isnull(previous_tracker_runtime), now(), previous_tracker_runtime)\
| eval latest_flip_state=if(isnull(latest_flip_state), object_previous_state, latest_flip_state)\
| eval latest_flip_time=if(isnull(latest_flip_time), previous_tracker_runtime, latest_flip_time)\
\
``` handle flip ```\
| eval latest_flip_state=if(object_state!=object_previous_state, object_state, latest_flip_state)\
| eval latest_flip_time=if(object_state!=object_previous_state, now(), latest_flip_time)\
\
| where isnotnull(data_last_time_seen)\
| eval data_last_lag_seen=if(splk_dhm_is_online="true", data_last_lag_seen, now()-data_last_time_seen)\
\
``` apply host filtering ```\
| `trackme_splk_dhm_rule_filter(object)`\
\
``` form the data host sourcetype summary ```\
| eval splk_dhm_st_summary=mvjoin(splk_dhm_st_summary, ", ")\
| eval splk_dhm_st_summary="{" . splk_dhm_st_summary . "}"\
\
``` set the tenant_id and search_mode ```\
| eval tenant_id="$tenant_id$", search_mode="$search_mode$"\
\
``` Call the extract summary tool and store all results for faster rendering purposes ``` \
| trackmeextractsplkdhm tenant_id="$tenant_id$" field_current="splk_dhm_st_summary" mode="all" gen_metrics="True"\
| spath
iseval = 0

# This version is designed to be called for untracked entities purposes in the health tracker
[trackme_dhm_tracker_abstract(1)]
args = tenant_id
definition = ``` offline version ``` \
| eval host=object, tenant_id="$tenant_id$"\
\
| rename splk_dhm_st_summary as current_summary\
| eval previous_summary=if(isnull(previous_summary), "", previous_summary)\
\
``` Call the merging backend ```\
| trackmemergesplkdhm field_host="host" field_current="current_summary" field_previous="previous_summary"\
\
``` spath ```\
| spath input=record\
\
``` lagging policies per index and sourcetype ```\
\
``` for lag ```\
``` match if object=all ```\
| eval lagging_class_object="all" | lookup trackme_common_lagging_classes_tenant_$tenant_id$ name as summary_idx, object as lagging_class_object OUTPUTNEW value_lag as data_custom_max_lag_allowed_per_index, level as data_custom_max_lag_allowed_per_index_level, object as data_custom_max_lag_allowed_per_index_object\
``` match if object=splk-dhm ```\
| eval lagging_class_object="splk-dhm" | lookup trackme_common_lagging_classes_tenant_$tenant_id$ name as summary_idx, object as lagging_class_object OUTPUTNEW value_lag as data_custom_max_lag_allowed_per_index, level as data_custom_max_lag_allowed_per_index_level, object as data_custom_max_lag_allowed_per_index_object\
\
``` match if object=all ```\
| eval lagging_class_object="all" | lookup trackme_common_lagging_classes_tenant_$tenant_id$ name as summary_st, object as lagging_class_object OUTPUTNEW value_lag as data_custom_max_lag_allowed_per_sourcetype, level as data_custom_max_lag_allowed_per_sourcetype_level, object as data_custom_max_lag_allowed_per_sourcetype_object\
``` match if object=splk-dhm ```\
| eval lagging_class_object="splk-dhm" | lookup trackme_common_lagging_classes_tenant_$tenant_id$ name as summary_st, object as lagging_class_object OUTPUTNEW value_lag as data_custom_max_lag_allowed_per_sourcetype, level as data_custom_max_lag_allowed_per_sourcetype_level, object as data_custom_max_lag_allowed_per_sourcetype_object\
\
``` for delay ```\
\
``` match if object=all ```\
| eval lagging_class_object="all" | lookup trackme_common_lagging_classes_tenant_$tenant_id$ name as summary_idx, object as lagging_class_object OUTPUTNEW value_delay as data_custom_max_delay_allowed_per_index, level as data_custom_max_delay_allowed_per_index_level, object as data_custom_max_delay_allowed_per_index_object\
``` match if object=splk-dhm ```\
| eval lagging_class_object="splk-dhm" | lookup trackme_common_lagging_classes_tenant_$tenant_id$ name as summary_idx, object as lagging_class_object OUTPUTNEW value_delay as data_custom_max_delay_allowed_per_index, level as data_custom_max_delay_allowed_per_index_level, object as data_custom_max_delay_allowed_per_index_object\
\
``` match if object=all ```\
| eval lagging_class_object="all" | lookup trackme_common_lagging_classes_tenant_$tenant_id$ name as summary_st, object as lagging_class_object OUTPUTNEW value_delay as data_custom_max_delay_allowed_per_sourcetype, level as data_custom_max_delay_allowed_per_sourcetype_level, object as data_custom_max_delay_allowed_per_sourcetype_object\
``` match if object=splk-dhm ```\
| eval lagging_class_object="splk-dhm" | lookup trackme_common_lagging_classes_tenant_$tenant_id$ name as summary_st, object as lagging_class_object OUTPUTNEW value_delay as data_custom_max_delay_allowed_per_sourcetype, level as data_custom_max_delay_allowed_per_sourcetype_level, object as data_custom_max_delay_allowed_per_sourcetype_object\
\
| eval data_custom_max_lag_allowed_per_index=if(data_custom_max_lag_allowed_per_index_level!="index", null(), data_custom_max_lag_allowed_per_index), data_custom_max_lag_allowed_per_sourcetype=if(data_custom_max_lag_allowed_per_sourcetype_level!="sourcetype", null(), data_custom_max_lag_allowed_per_sourcetype)\
| eval data_custom_max_lag_allowed_per_index=if(isnull(data_custom_max_lag_allowed_per_index_object) OR data_custom_max_lag_allowed_per_index_object="all" OR data_custom_max_lag_allowed_per_index_object="splk-dhm", data_custom_max_lag_allowed_per_index, null())\
| eval data_custom_max_lag_allowed_per_sourcetype=if(isnull(data_custom_max_lag_allowed_per_sourcetype_object) OR data_custom_max_lag_allowed_per_sourcetype_object="all" OR data_custom_max_lag_allowed_per_sourcetype_object="splk-dhm", data_custom_max_lag_allowed_per_sourcetype, null())\
| fields - data_custom_max_lag_allowed_per_index_level, data_custom_max_lag_allowed_per_sourcetype_level, data_custom_max_lag_allowed_per_index_object, data_custom_max_lag_allowed_per_sourcetype_object\
\
| eval data_custom_max_delay_allowed_per_index=if(data_custom_max_delay_allowed_per_index_level!="index", null(), data_custom_max_delay_allowed_per_index), data_custom_max_delay_allowed_per_sourcetype=if(data_custom_max_delay_allowed_per_sourcetype_level!="sourcetype", null(), data_custom_max_delay_allowed_per_sourcetype)\
| eval data_custom_max_delay_allowed_per_index=if(isnull(data_custom_max_delay_allowed_per_index_object) OR data_custom_max_delay_allowed_per_index_object="all" OR data_custom_max_delay_allowed_per_index_object="splk-dhm", data_custom_max_delay_allowed_per_index, null())\
| eval data_custom_max_delay_allowed_per_sourcetype=if(isnull(data_custom_max_delay_allowed_per_sourcetype_object) OR data_custom_max_delay_allowed_per_sourcetype_object="all" OR data_custom_max_delay_allowed_per_sourcetype_object="splk-dhm", data_custom_max_delay_allowed_per_sourcetype, null())\
| fields - data_custom_max_delay_allowed_per_index_level, data_custom_max_delay_allowed_per_sourcetype_level, data_custom_max_delay_allowed_per_index_object, data_custom_max_delay_allowed_per_sourcetype_object\
\
``` for any match within the lagging policy, the highest lagging value has the precedence ```\
| fillnull value=0 data_custom_max_lag_allowed_per_index, data_custom_max_lag_allowed_per_sourcetype\
| eval data_custom_max_lag_allowed=max(data_custom_max_lag_allowed_per_index, data_custom_max_lag_allowed_per_sourcetype)\
\
| fillnull value=0 data_custom_max_delay_allowed_per_index, data_custom_max_delay_allowed_per_sourcetype\
| eval data_custom_max_delay_allowed=max(data_custom_max_delay_allowed_per_index, data_custom_max_delay_allowed_per_sourcetype)\
\
``` Get default lag and latency thresholds ```\
| `trackme_default_host_lag`\
| `trackme_default_host_delay`\
| `trackme_default_host_allow_adaptive_delay`\
\
``` lookup existing entity values ```\
| trackmehashobject input_field="host" output_field="keyid" | lookup trackme_dhm_tenant_$tenant_id$ _key as keyid OUTPUT _key as key, alias as current_alias, monitored_state, data_override_lagging_class, data_max_lag_allowed as current_host_level_data_max_lag_allowed, data_max_delay_allowed as current_host_level_data_max_delay_allowed, splk_dhm_alerting_policy, monitoring_time_policy, monitoring_time_rules, tags_manual, priority, priority_updated, priority_external, priority_reason, future_tolerance, object_state as object_previous_state, anomaly_reason as previous_anomaly_reason, tracker_runtime as previous_tracker_runtime, latest_flip_state, latest_flip_time, outliers_readiness, host_idx_blocklists, host_st_blocklists, impact_score_weights\
| eval alias=if(isnotnull(current_alias) AND current_alias!="", current_alias, alias) | fields - current_alias\
\
| eval summary_max_lag_allowed=if(isnum(data_custom_max_lag_allowed) AND data_custom_max_lag_allowed>0, data_custom_max_lag_allowed, data_max_lag_allowed)\
| fields - data_custom_max_lag_allowed_per_index, data_custom_max_lag_allowed_per_sourcetype\
\
| eval summary_max_delay_allowed=if(isnum(data_custom_max_delay_allowed) AND data_custom_max_delay_allowed>0, data_custom_max_delay_allowed, data_max_delay_allowed)\
| fields - data_custom_max_delay_allowed_per_index, data_custom_max_delay_allowed_per_sourcetype\
\
``` Handle delay and latency for the global host level ```\
\
| eventstats max(summary_max_lag_allowed) as host_summary_max_lag_allowed, max(summary_max_delay_allowed) as host_summary_max_delay_allowed by host\
\
``` host level values always win ``` \
| eval data_max_lag_allowed=if(isnum(current_host_level_data_max_lag_allowed) AND current_host_level_data_max_lag_allowed!=host_summary_max_delay_allowed, current_host_level_data_max_lag_allowed, host_summary_max_lag_allowed) \
| eval data_max_delay_allowed=if(isnum(current_host_level_data_max_delay_allowed) AND current_host_level_data_max_delay_allowed!=host_summary_max_delay_allowed, current_host_level_data_max_delay_allowed, host_summary_max_delay_allowed) \
\
``` define the status per sourcetype for the source summary mvfield ```\
| eval summary_st_state=if(summary_last_event_lag>summary_max_delay_allowed OR summary_last_ingest_lag>summary_max_lag_allowed, "red", "green")\
| eval splk_dhm_st_summary = "'idx': '" . summary_idx . "', " . "'st': '" . summary_st . "', " . "'last_eventcount': '" . summary_last_eventcount . "', " . "'max_lag_allowed': '" . summary_max_lag_allowed . "', " . "'max_delay_allowed': '" . summary_max_delay_allowed . "', " . "'last_ingest': '" . summary_last_ingest . "', " . "'first_time': '" . summary_first_time . "', " . "'last_time': '" . summary_last_time . "', " . "'last_ingest_lag': '" . summary_last_ingest_lag . "', " . "'last_event_lag': '" . (now()-summary_last_time) . "', " . "'time_measure': '" . now() . "', " . "'state': '" . summary_st_state . "'"\
| eval splk_dhm_st_summary =  "'" . hash . "': {" . splk_dhm_st_summary . "}"\
\
``` perform intermediate calculation table ```\
| stats first(key) as key, first(alias) as alias, max(summary_last_ingest) as data_last_ingest, min(summary_first_time) as data_first_time_seen,\
max(summary_last_time) as data_last_time_seen, avg(summary_last_ingest_lag) as data_last_ingestion_lag_seen,\
values(summary_idx) as data_index, values(summary_st) as data_sourcetype,\
max(data_custom_max_lag_allowed) as data_custom_max_lag_allowed, max(data_max_lag_allowed) as data_max_lag_allowed, max(data_max_delay_allowed) as data_max_delay_allowed, max(summary_max_lag_allowed) as summary_max_lag_allowed, max(summary_max_delay_allowed) as summary_max_delay_allowed, values(splk_dhm_st_summary) as splk_dhm_st_summary, \
sum(summary_last_eventcount) as data_eventcount, first(splk_dhm_alerting_policy) as splk_dhm_alerting_policy, \
first(object_state) as object_state, first(data_last_lag_seen) as data_last_lag_seen, first(monitored_state) as monitored_state, \
first(data_override_lagging_class) as data_override_lagging_class, first(object_previous_state) as object_previous_state, \
first(previous_anomaly_reason) as previous_anomaly_reason, first(previous_tracker_runtime) as previous_tracker_runtime, first(tracker_runtime) as tracker_runtime, first(isOutlier) as isOutlier, first(outliers_readiness) as outliers_readiness,\
first(latest_flip_state) as latest_flip_state, first(latest_flip_time) as latest_flip_time, first(object_category) as object_category, first(priority) as priority, first(priority_updated) as priority_updated, first(priority_external) as priority_external, first(priority_reason) as priority_reason,\
first(sla_class) as sla_class, first(tags_manual) as tags_manual,\
first(impact_score_weights) as impact_score_weights,\
first(host_idx_blocklists) as host_idx_blocklists, first(host_st_blocklists) as host_st_blocklists, \
first(monitoring_time_policy) as monitoring_time_policy, first(monitoring_time_rules) as monitoring_time_rules, \
sum(avg_eventcount_5m) as avg_eventcount_5m, sum(latest_eventcount_5m) as latest_eventcount_5m, \
sum(perc95_eventcount_5m) as perc95_eventcount_5m, sum(stdev_eventcount_5m) as stdev_eventcount_5m, \
avg(perc95_latency_5m) as perc95_latency_5m, avg(stdev_latency_5m) as stdev_latency_5m, \
avg(avg_latency_5m) as avg_latency_5m, avg(latest_latency_5m) as latest_latency_5m by host \
\
| eval data_last_ingestion_lag_seen=round(data_last_ingestion_lag_seen, 0)\
\
``` define the ingestion lag versus now, and a flag field defining an online status any results from live tstats ```\
| eval data_last_lag_seen=now()-data_last_time_seen, splk_dhm_is_online="true"\
\
``` rename host and lower case ```\
| rename host as object | eval object=lower(object)\
\
``` if the key is null, this is the first time we see this source and it will be added to the collection, create a key ```\
| eval key=if(isnull(key), sha256(object), key)\
\
``` set the object alias, if necessary ```\
| eval alias=if(isnull(alias) OR alias="", object, alias)\
\
``` apply various macro defining default values and policies ```\
| `trackme_default_host_monitored_state`\
| `trackme_default_priority($tenant_id$)`\
\
``` support for priority based lagging classes ```\
\
``` for lag ``` \
\
``` match if object=all ```\
| eval lagging_class_object="all" | lookup trackme_common_lagging_classes_tenant_$tenant_id$ name as priority, object as lagging_class_object OUTPUT value_lag as data_custom_max_lag_allowed_per_priority, level as data_custom_max_lag_allowed_per_priority_level, object as data_custom_max_lag_allowed_per_priority_object\
``` match if object=splk-dhm ```\
| eval lagging_class_object="splk-dhm" | lookup trackme_common_lagging_classes_tenant_$tenant_id$ name as priority, object as lagging_class_object OUTPUTNEW value_lag as data_custom_max_lag_allowed_per_priority, level as data_custom_max_lag_allowed_per_priority_level, object as data_custom_max_lag_allowed_per_priority_object\
\
``` for delay ```\
\
``` match if object=all ```\
| eval lagging_class_object="all" | lookup trackme_common_lagging_classes_tenant_$tenant_id$ name as priority, object as lagging_class_object OUTPUT value_delay as data_custom_max_delay_allowed_per_priority, level as data_custom_max_delay_allowed_per_priority_level, object as data_custom_max_delay_allowed_per_priority_object\
``` match if object=splk-dhm ```\
| eval lagging_class_object="splk-dhm" | lookup trackme_common_lagging_classes_tenant_$tenant_id$ name as priority, object as lagging_class_object OUTPUTNEW value_delay as data_custom_max_delay_allowed_per_priority, level as data_custom_max_delay_allowed_per_priority_level, object as data_custom_max_delay_allowed_per_priority_object\
\
| eval data_max_lag_allowed=case(\
data_override_lagging_class="true", data_max_lag_allowed,\
data_override_lagging_class!="true", max(data_max_lag_allowed, data_custom_max_lag_allowed_per_priority),\
1=1, data_max_lag_allowed\
)\
| eval data_max_delay_allowed=case(\
data_override_lagging_class="true", data_max_delay_allowed,\
data_override_lagging_class!="true", max(data_max_delay_allowed, data_custom_max_delay_allowed_per_priority),\
1=1, data_max_delay_allowed\
)\
\
``` if data_override_lagging_class is defined ```\
| eval data_override_lagging_class=if(isnull(data_override_lagging_class) OR data_override_lagging_class="null", "false", data_override_lagging_class)\
\
``` create a comma separated list of known indexes and sourcetype for live/lookup merge in next phase ```\
| eval data_index=mvjoin(data_index, ","), data_sourcetype=mvjoin(data_sourcetype, ",")\
\
``` define the object_category field which is used by further lookup operations ```\
| eval object_category="splk-dhm"\
\
``` source group feature ```\
| `trackme_splk_dhm_group_lookup($tenant_id$)`\
\
```ML management: handle feeds interruption ``` \
| foreach avg_eventcount_5m, latest_eventcount_5m, perc95_eventcount_5m, stdev_eventcount_5m [ eval <<FIELD>>=if(isnum(<<FIELD>>), <<FIELD>>, 0) ] \
\
```Retrieve ML anomaly detection status``` \
| `get_splk_outliers_status($tenant_id$, "dhm")` \
\
``` Decision maker ```\
| trackmedecisionmaker tenant_id="$tenant_id$" component="dhm" \
\
``` define tracker_runtime ```\
| eval tracker_runtime=now()\
\
``` conditional verifications ```\
| eval object_previous_state=if(isnull(object_previous_state), "discovered", object_previous_state)\
| eval previous_tracker_runtime=if(isnull(previous_tracker_runtime), now(), previous_tracker_runtime)\
| eval latest_flip_state=if(isnull(latest_flip_state), object_previous_state, latest_flip_state)\
| eval latest_flip_time=if(isnull(latest_flip_time), previous_tracker_runtime, latest_flip_time)\
\
``` handle flip ```\
| eval latest_flip_state=if(object_state!=object_previous_state, object_state, latest_flip_state)\
| eval latest_flip_time=if(object_state!=object_previous_state, now(), latest_flip_time)\
\
| where isnotnull(data_last_time_seen)\
| eval data_last_lag_seen=if(splk_dhm_is_online="true", data_last_lag_seen, now()-data_last_time_seen)\
\
``` apply host filtering ```\
| `trackme_splk_dhm_rule_filter(object)`\
\
``` form the data host sourcetype summary ```\
| eval splk_dhm_st_summary=mvjoin(splk_dhm_st_summary, ", ")\
| eval splk_dhm_st_summary="{" . splk_dhm_st_summary . "}"\
\
``` set the tenant_id and search_mode ```\
| lookup trackme_dhm_tenant_$tenant_id$ object OUTPUT search_mode\
| eval tenant_id="$tenant_id$"\
\
``` Call the extract summary tool and store all results for faster rendering purposes ``` \
| trackmeextractsplkdhm field_current="splk_dhm_st_summary" mode="all" \
| spath
iseval = 0

#
# SPLK-MHM abstract
#
[trackme_mhm_tracker_abstract(3)]
args = tenant_id, break_by, object
definition = rename $break_by$ as object, index as metric_index \
\
``` extract the metric category based on the metric name naming convention, the metric_category field is the essential key of metric monitoring in trackMe concept ``` \
| rex field=metric_name "(?<metric_category>[^.]*).{0,1}" \
\
``` intermediate calculation ``` \
| stats max(_time) as _time, min(_time) as first_time, first(alias) as alias by metric_category, metric_index, object \
\
``` build the metric summary ``` \
| eval current_summary = "'metric_category': '" . metric_category . "', " . "'idx': '" . metric_index . "', " . "'first_time': '" . round(first_time, 0) . "', " . "'last_time': '" . _time . "', " . "'last_metric_lag': '" . (now() - _time) . "', " . "'time_measure': '" . now() . "'" \
\
``` handle the summary ``` \
\
``` form a unique hash for that host metric_category using sha256 ``` \
| eval current_summary =  "'" . sha256(metric_category) . "': {" . current_summary . "}" \
| stats values(current_summary) as current_summary, first(alias) as alias by object \
| eval current_summary=mvjoin(current_summary, ", ") \
| eval current_summary="{" . current_summary . "}" \
\
``` Retrieve the previous execution dictionnary per host ``` \
| lookup trackme_mhm_tenant_$tenant_id$ object OUTPUT metric_details as previous_summary \
| eval previous_summary=if(isnull(previous_summary), "", previous_summary) \
\
``` retrieve current collection and merge (ensure handling inactive endpoints ``` \
| inputlookup append=t trackme_mhm_tenant_$tenant_id$ \
| eval current_summary=coalesce(current_summary, metric_details) \
| trackmehashobject input_field="object" output_field="keyid" \
| stats first(current_summary) as current_summary, first(previous_summary) as previous_summary, first(alias) as alias, first(object) as object by keyid \
\
``` exclude any permanent deletion ```\
| `trackme_exclude_permanent_deletion($tenant_id$, "splk-mhm")`\
\
``` Call the merging backend ``` \
| trackmemergesplkmhm field_host="object" field_current="current_summary" field_previous="previous_summary" \
\
``` spath ``` \
| spath input=record \
\
``` refresh last_metric_lag ``` \
| eval last_metric_lag=now()-last_time \
\
``` lookup max lag allowed per metric ``` \
| lookup trackme_mhm_lagging_classes_tenant_$tenant_id$ metric_category OUTPUT metric_max_lag_allowed \
\
``` apply default policy ``` \
| `trackme_default_splk_mhm_lag` \
\
``` set metric category state ``` \
| `trackme_eval_metric_category_state` \
\
``` build the metric_category summary ``` \
| eval time=last_time \
| eval current_summary = "'metric_category': '" . metric_category . "', " . "'idx': '" . metric_index . "', " . "'first_time': '" . round(first_time, 0) . "', " . "'last_time': '" . round(time, 0) . "', " . "'last_metric_lag': '" . round((now() - _time), 2) . "', " . "'time_measure': '" . round(now(), 0) . "', " . "'state': '" . object_state . "', " . "'lag_allowed': '" . metric_max_lag_allowed . "'" \
\
``` form a unique hash for that host metric_category using sha256 ``` \
| eval current_summary =  "'" . sha256(metric_category) . "': {" . current_summary . "}" \
\
``` intermediate calculation ``` \
| stats max(last_time) as metric_last_time_seen, min(_time) as first_time, values(metric_index) as metric_index, values(metric_category) as metric_category, values(current_summary) as current_summary, first(alias) as alias by host \
| rename host as object \
\
``` form the dict ``` \
| eval current_summary=mvjoin(current_summary, ", ") \
| eval current_summary="{" . current_summary . "}" \
\
``` rename the dict as the target field in the collection ``` \
| rename current_summary as metric_details\
\
``` retrieve key, metric_details and other fields to be preserved from lookup ``` \
| lookup trackme_mhm_tenant_$tenant_id$ object OUTPUT _key as key, \
object_state as object_previous_state, anomaly_reason as previous_anomaly_reason, tracker_runtime as metric_previous_tracker_runtime, \
metric_first_time_seen, \
monitored_state, metric_override_lagging_class, \
latest_flip_state, latest_flip_time, priority, priority_updated, priority_external, priority_reason, tags_manual \
\
``` handle first time seen ``` \
| eval metric_first_time_seen=if(isnull(metric_first_time_seen), now(), metric_first_time_seen) \
\
``` handle new endpoints ``` \
| eval key=if(isnull(key), sha256(object), key) \
\
``` set the object alias, if necessary ```\
| eval alias=if(isnull(alias) OR alias="", object, alias)\
\
``` define priority ``` \
| `trackme_default_priority($tenant_id$)` \
\
``` handle logical group mapping ``` \
| `trackme_splk_mhm_group_lookup($tenant_id$)` \
\
``` Decision Maker ``` \
| trackmedecisionmaker tenant_id="$tenant_id$" component="mhm" \
\
``` some final steps, define the state if new, by safety requires a metric_category, define the monitored state ``` \
| eval object_previous_state=if(isnull(object_previous_state), "discovered", object_previous_state) \
| eval metric_previous_tracker_runtime=if(isnull(metric_previous_tracker_runtime), now(), metric_previous_tracker_runtime) \
| eval latest_flip_state=if(isnull(latest_flip_state), object_previous_state, latest_flip_state) \
| eval latest_flip_time=if(isnull(latest_flip_time), metric_previous_tracker_runtime, latest_flip_time) \
\
``` handle flip ```\
| eval latest_flip_state=if(object_state!=object_previous_state, object_state, latest_flip_state)\
| eval latest_flip_time=if(object_state!=object_previous_state, now(), latest_flip_time)\
\
| where isnotnull(metric_category) \
| `trackme_default_splk_mhm_monitored_state`\
\
``` store the metric_index, metric_category as csv fields ``` \
| eval metric_index=mvjoin(metric_index, ","), metric_category=mvjoin(metric_category, ",") \
\
``` expected by the collection ```\
| eval metric_last_lag_seen=last_lag_seen\
\
``` the tracker run time ```\
| eval tracker_runtime = now()\
\
``` set the tenant ``` \
| eval tenant_id="$tenant_id$" \
\
``` Call the extract summary tool and store both results for faster rendering purposes ``` \
| trackmeextractsplkmhm field_current="metric_details" mode="all" tenant_id="$tenant_id$" gen_metrics="True" | spath
iseval = 0

#
# deprecated: this macro is now replaced by the decision maker and will be removed in a future release
#

# Evaluate the entity status
[trackme_eval_splk_mhm_state]
definition = eval object_state=if(match(metric_details, "'state':\s'red'"), "red", "green")

# Evaluate the metric category status
[trackme_eval_metric_category_state]
definition = eval object_state=if(last_metric_lag>metric_max_lag_allowed, "red", "green")

#
# Docs note and link global deprecation:
# These macros have been deprecated in 2.0.87 and replaced with a more robust and valuable approach relying on TrackMe's configuration level
# They will be removed in a future release
#

# TrackMe data source identity card
[trackme_get_identity_card(2)]
args = tenant_id, key
definition = lookup trackme_dsm_knowledge_tenant_$tenant_id$ object as $key$ OUTPUT doc_link, doc_note, object as doc_key_match\
| eval doc_link=if(isnull(doc_link), "null", doc_link), doc_note=if(isnull(doc_note), "null", doc_note)\
| eval doc_link_global=`trackme_identity_card_default_url`, doc_note_global=`trackme_identity_card_default_note`, doc_identity_card_is_global=if(doc_link_global!="none" AND doc_note_global!="none" AND (doc_link=="null" OR doc_note=="null"), "true", "false")\
| eval doc_key_match=case(isnotnull(doc_key_match) AND match(doc_key_match, "\*"), "wildcard", isnotnull(doc_key_match), "strict")\
| eval doc_link=if(doc_link_global!="none" AND doc_note_global!="none" AND doc_link=="null", doc_link_global, doc_link), doc_note=if(doc_link_global!="none" AND doc_note_global!="none" AND doc_note=="null", doc_note_global, doc_note) | fields - doc_link_global, doc_note_global
iseval = 0

# default URL for identity card, can be customised to set a default URL for all
[trackme_identity_card_default_url]
definition = "none"
iseval = 0

# default note for identity card, can be customised to set a default note for all
[trackme_identity_card_default_note]
definition = "none"
iseval = 0

#
# End deprecation note
#

# Ack default duration in seconds
[trackme_ack_default_duration]
definition = [ | trackmegetconf target="trackme_general" | spath path=trackme_general.trackme_ack_duration_default output=trackme_ack_duration_default | return $trackme_ack_duration_default ]
iseval = 0

# Use to populate the list of durations
[trackme_ack_populate_dropdown]
definition = makeresults | head 1\
| eval choice1=`trackme_ack_default_duration` . "|" . "default" . "|" . 1, choice2=2*24*60*60 . "|" . "default" . "|" . 2, choice3=7*24*60*60 . "|" . "default" . "|" . 3, choice4=15*24*60*60 . "|" . "default" . "|" . 4, choice5=30*24*60*60 . "|" . 5, choice5=90*24*60*60 . "|" . "default" . "|" . 6\
| fields - _time\
| transpose | stats list("row 1") as choices\
| mvexpand choices\
| rex field=choices "(?<duration>[^\|]+)\|(?<label>[^\|]+)\|(?<priority>\d)"\
| fields duration, label, priority\
| eval label=tostring(duration, "duration")\
| sort priority | fields duration, label
iseval = 0

# Ack lookup
[trackme_ack_lookup_main(2)]
definition = lookup trackme_common_alerts_ack_tenant_$tenant_id$ object as $object$ object_category | eval ack_state=if(isnull(ack_state), "inactive", ack_state)
args = tenant_id, object
iseval = 0

# Logical group for splk-dsm
[trackme_splk_dsm_group_lookup(1)]
args = tenant_id
definition = eval object_category="splk-dsm" | lookup trackme_common_logical_group_tenant_$tenant_id$ object_group_members as object OUTPUTNEW _key as object_group_key, object_group_name\
| foreach object_group_key, object_group_name [ eval <<FIELD>> = mvindex(<<FIELD>>, 0) ]
iseval = 0

# Logical group for splk-dhm
[trackme_splk_dhm_group_lookup(1)]
args = tenant_id
definition = eval object_category="splk-dhm" | lookup trackme_common_logical_group_tenant_$tenant_id$ object_group_members as object OUTPUTNEW _key as object_group_key, object_group_name\
| foreach object_group_key, object_group_name [ eval <<FIELD>> = mvindex(<<FIELD>>, 0) ]
iseval = 0

# Logical group for splk-mhm
[trackme_splk_mhm_group_lookup(1)]
args = tenant_id
definition = eval object_category="splk-mhm" | lookup trackme_common_logical_group_tenant_$tenant_id$ object_group_members as object OUTPUTNEW _key as object_group_key, object_group_name\
| foreach object_group_key, object_group_name [ eval <<FIELD>> = mvindex(<<FIELD>>, 0) ]
iseval = 0

# Logical group for splk-flx
[trackme_splk_flx_group_lookup(1)]
args = tenant_id
definition = eval object_category="splk-flx" | lookup trackme_common_logical_group_tenant_$tenant_id$ object_group_members as object OUTPUTNEW _key as object_group_key, object_group_name\
| foreach object_group_key, object_group_name [ eval <<FIELD>> = mvindex(<<FIELD>>, 0) ]
iseval = 0

# Load audit flip collection for SLA compliance

[trackme_get_sla_pct_per_entity(2)]
definition = `trackme_idx_search_filter` sourcetype="trackme:state" object_category="$object_category$" object="$object$"\
| fields _time object_category, object, current_state\
| sort limit=0 _time\
| stats first(current_state) as current_state by _time, object_category, object\
| streamstats last(_time) as "prev_time", last(current_state) as prev_state current=f by object_category, object\
| where NOT [ | trackmereturnmaintenancedb tenant_id="$tenant_id$" | table search_str | return $search_str ]\
| eval range_duration=_time-prev_time, green_time=case(current_state="green" OR current_state="blue", range_duration), not_green_time=case(current_state!="green" AND current_state!="blue", range_duration)\
| stats sum(range_duration) as range_duration, sum(green_time) as green_time, sum(not_green_time) as not_green_time by object_category, object\
| eval green_time=if(isnum(green_time), green_time, 0)\
| eval percent_sla=round(green_time/range_duration*100, 2) | fields percent_sla
args = object_category, object
iseval = 0

[trackme_get_sla_pct_per_entity_key(3)]
definition = `trackme_idx("$tenant_id$")` sourcetype="trackme:state" object_category="$object_category$" tenant_id="$tenant_id$" key="$key$"\
| fields _time object_category, object, current_state\
| sort limit=0 _time\
| stats first(current_state) as current_state by _time, object_category, object\
| streamstats last(_time) as "prev_time", last(current_state) as prev_state current=f by object_category, object\
| where NOT [ | trackmereturnmaintenancedb tenant_id="$tenant_id$" | table search_str | return $search_str ]\
| eval range_duration=_time-prev_time, green_time=case(current_state="green" OR current_state="blue", range_duration), not_green_time=case(current_state!="green" AND current_state!="blue", range_duration)\
| stats sum(range_duration) as range_duration, sum(green_time) as green_time, sum(not_green_time) as not_green_time by object_category, object\
| eval green_time=if(isnum(green_time), green_time, 0)\
| eval percent_sla=round(green_time/range_duration*100, 2) | fields percent_sla
args = tenant_id, object_category, key
iseval = 0

#
# new in TrackMe v2.0.68: SLA is now in metrics, metric_name=trackme.sla.object_state
#

[trackme_sla_eval_current_state]
definition = eval current_state=case(\
    object_state = 1, "green",\
    object_state = 2, "red",\
    object_state = 3, "orange",\
    object_state = 4, "blue",\
    object_state = 5, "unknown"\
    )
iseval = 0

[trackme_get_sla_pct_metrics_per_entity_key(3)]
definition = mstats latest(trackme.sla.object_state) as object_state where `trackme_metrics_idx("$tenant_id$")` object_category="$object_category$" tenant_id="$tenant_id$" object_id IN ($key$) by object_category, object span=1m\
| `trackme_sla_eval_current_state`\
| sort limit=0 _time\
| stats first(current_state) as current_state by _time, object_category, object\
| streamstats last(_time) as "prev_time", last(current_state) as prev_state current=f by object_category, object\
| where NOT [ | trackmereturnmaintenancedb tenant_id="$tenant_id$" | table search_str | return $search_str ]\
| eval range_duration=_time-prev_time, green_time=case(current_state="green" OR current_state="blue", range_duration), not_green_time=case(current_state!="green" AND current_state!="blue", range_duration)\
| stats sum(range_duration) as range_duration, sum(green_time) as green_time, sum(not_green_time) as not_green_time by object_category, object\
| eval green_time=if(isnum(green_time), green_time, 0)\
| eval percent_sla=round(green_time/range_duration*100, 2) | fields percent_sla
args = tenant_id, object_category, key
iseval = 0

# for multiple entities at once
[trackme_get_sla_pct_metrics_per_entity_keys(3)]
definition = mstats latest(trackme.sla.object_state) as object_state where `trackme_metrics_idx("$tenant_id$")` object_category="$object_category$" tenant_id="$tenant_id$" object_id IN ($key$) by object_category, object span=1m\
| `trackme_sla_eval_current_state`\
| sort limit=0 _time\
| stats first(current_state) as current_state by _time, object_category, object\
| streamstats last(_time) as "prev_time", last(current_state) as prev_state current=f by object_category, object\
| where NOT [ | trackmereturnmaintenancedb tenant_id="$tenant_id$" | table search_str | return $search_str ]\
| eval range_duration=_time-prev_time, green_time=case(current_state="green" OR current_state="blue", range_duration), not_green_time=case(current_state!="green" AND current_state!="blue", range_duration)\
| stats sum(range_duration) as range_duration, sum(green_time) as green_time, sum(not_green_time) as not_green_time by object_category, object\
| eval green_time=if(isnum(green_time), green_time, 0)\
| eval percent_sla=round(green_time/range_duration*100, 2) | fields percent_sla | stats avg(percent_sla) as percent_sla
args = tenant_id, object_category, key
iseval = 0

# Get SLA compliance table
# notes: benchmarks on Production scale have shown the fastest solution was a lookup call at the latest stage post streamstats
[trackme_get_sla(5)]
definition = `trackme_idx("$tenant_id$")` sourcetype="trackme:state" tenant_id=$tenant_id$ object_category="$object_category$" object="$object_freetext$" object="$object$"\
``` to be actively monitored, the tracker_runtime must be higher or equal to the relative last 4 hours ```\
| where tracker_runtime>=relative_time(now(), "-4h") \
| fields _time tenant_id, object_category, object, current_state, priority, monitored_state\
| stats first(current_state) as current_state, first(priority) as priority, first(monitored_state) as monitored_state by _time, tenant_id, object_category, object\
| streamstats last(_time) as "prev_time", last(current_state) as prev_state current=f by tenant_id, object_category, object\
| where NOT [ | trackmereturnmaintenancedb tenant_id="$tenant_id$" | table search_str | return $search_str ]\
| eval range_duration=_time-prev_time, green_time=case(current_state="green" OR current_state="blue", range_duration), not_green_time=case(current_state!="green" AND current_state!="blue", range_duration)\
| stats sum(range_duration) as range_duration, sum(green_time) as green_time, sum(not_green_time) as not_green_time, latest(priority) as priority, latest(monitored_state) as monitored_state by tenant_id, object_category, object\
| eval green_time=if(isnum(green_time), green_time, 0), not_green_time=if(isnum(not_green_time), not_green_time, 0), percent_sla=round(green_time/range_duration*100, 2)\
| foreach range_duration green_time not_green_time [ eval <<FIELD>> = tostring('<<FIELD>>', "duration") ]\
| search priority="$priority$" AND monitored_state="enabled"\
| fields tenant_id, object, object_category, priority, monitored_state, range_duration, green_time, not_green_time, percent_sla\
| rename range_duration as "total duration ([D+]HH:MM:SS)", green_time as "duration spent in green state ([D+]HH:MM:SS)", not_green_time as "duration spent in non green state ([D+]HH:MM:SS)"
args = tenant_id,object_category,object,object_freetext,priority
iseval = 0

# Get SLA compliance results for a particular object
[trackme_get_sla(2)]
definition = `trackme_idx_search_filter` sourcetype="trackme:state" source="current_state_tracking:$object_category$" object_category="$object_category$" object="$object$"\
| fields _time object_category, object, current_state, priority, monitored_state\
| stats first(current_state) as current_state, first(priority) as priority, first(monitored_state) as monitored_state by _time, object_category, object\
| streamstats last(_time) as "prev_time", last(current_state) as prev_state current=f by object_category, object\
| where NOT [ | trackmereturnmaintenancedb tenant_id="$tenant_id$" | table search_str | return $search_str ]\
| eval range_duration=_time-prev_time, green_time=case(current_state="green" OR current_state="blue", range_duration), not_green_time=case(current_state!="green" AND current_state!="blue", range_duration)\
| stats sum(range_duration) as range_duration, sum(green_time) as green_time, sum(not_green_time) as not_green_time, latest(priority) as priority, latest(monitored_state) as monitored_state by object_category, object\
| eval green_time=if(isnum(green_time), green_time, 0), not_green_time=if(isnum(not_green_time), not_green_time, 0), percent_sla=round(green_time/range_duration*100, 2)\
| fields object, object_category, priority, monitored_state, percent_sla
args = object, object_category
iseval = 0

# Shows details for drilldown purposes
[trackme_get_sla_details(2)]
definition = `trackme_idx_search_filter` sourcetype="trackme:state" source="current_state_tracking:$object_category$" object_category="$object_category$" object="$object$"\
| fields _time, object_category, object, current_state\
| stats first(current_state) as current_state by _time, object_category, object\
| streamstats last(_time) as "prev_time", last(current_state) as prev_state current=f by object_category, object\
| where NOT [ | trackmereturnmaintenancedb tenant_id="$tenant_id$" | table search_str | return $search_str ]\
| eval range_duration=_time-prev_time, green_time=case(((current_state == "green") OR (current_state == "blue")),range_duration), not_green_time=case(((current_state != "green") AND (current_state != "blue")),range_duration)\
| stats sum(range_duration) as range_duration, sum(green_time) as green_time, sum(not_green_time) as not_green_time by object_category, object\
| foreach range_duration green_time not_green_time [ eval <<FIELD>> = tostring('<<FIELD>>', "duration") ]
args = object, object_category
iseval = 0

# Simple regex to extract source, sourcetype and host from component=DateParserVerbose
[trackme_rex_dateparserverbose]
definition = rex "Context:\ssource=(?<data_source>[^\|]*)\|host=(?<object>[^\|]*)\|(?<data_sourcetype>[^\|]*)"
iseval = 0

#
# Machine Learning Outliers anomalies detection
#

# Define the isOutlier status based on the data
[trackme_isOutlier_status]
definition = eval isOutlier=0
iseval = 0

#
# outputlookup macros from trackers
#

# to rely on TrackMe's custom command doing both operations at the same time
[trackme_outputlookup(2)]
definition = eval mtime=now() | eval $key$=if(isnull($key$) OR $key$="", sha256(object), $key$) | trackmepersistentfields collection="$collection$" key="$key$" update_collection="True"
args = collection, key
iseval = 0

[trackme_outputlookup(3)]
definition = eval tenant_id="$tenant_id$", mtime=now() | eval $key$=if(isnull($key$) OR $key$="", sha256(object), $key$) | trackmepersistentfields collection="$collection$" key="$key$" update_collection="True"
args = collection, key, tenant_id
iseval = 0

# specific for the trackmehealthtracker
[trackme_outputlookup_tracker_health(2)]
definition = eval mtime=now() | eval tracker_health_runtime=now() | eval $key$=if(isnull($key$) OR $key$="", sha256(object), $key$) | trackmepersistentfields collection="$collection$" key="$key$" update_collection="True"
args = collection, key
iseval = 0

# to rely on the Splunk outputlookup command
#[trackme_outputlookup(2)]
#definition = eval mtime=now() | trackmepersistentfields collection="$collection$" key="$key$" update_collection="False" | outputlookup $collection$ append=t key_field=$key$
#args = collection, key
#iseval = 0

#[trackme_outputlookup(3)]
#definition = eval tenant_id="$tenant_id$", mtime=now() | trackmepersistentfields collection="$collection$" key="$key$" update_collection="False" | outputlookup $collection$ append=t key_field=$key$
#args = collection, key, tenant_id
#iseval = 0

#
# collect to summary index
#

[trackme_sumarycollect(1)]
definition = eval _time=now() | collect `trackme_idx` source=$report$
args = report
iseval = 0

#
# mcollect macro
#

[trackme_mcollect(5)]
definition = eval _time=now(), object=$object$, object_category="$object_category$"\
| eval $metrics$\
| fields $dimensions$, metric_name:*\
| rename "metric_name:*" as "*"\
| mcollect split=t `trackme_metrics_idx($tenant_id$)` $dimensions$
args = object, object_category, metrics, dimensions, tenant_id
iseval = 0

# use to populate the Elastic type of search dropdown in the UI
[trackme_elastic_gen_type]
definition = makeresults count=10 \
| streamstats count | head 10 \
| eval label=case( \
count=1, "tstats (local)", \
count=2, "raw (local)", \
count=3, "from (local)", \
count=4, "mstats (local)", \
count=5, "mpreview (local)", \
count=6, "tstats (remote)", \
count=7, "raw (remote)", \
count=8, "from (remote)", \
count=9, "mstats (remote)", \
count=10, "mpreview (remote)" \
) \
| eval value=case( \
count=1, "tstats", \
count=2, "raw", \
count=3, "from", \
count=4, "mstats", \
count=5, "mpreview", \
count=6, "remote_tstats", \
count=7, "remote_raw", \
count=8, "remote_from", \
count=9, "remote_mstats", \
count=10, "remote_mpreview" \
) \
| rex field="label" "\((?<type>[^\)]+)" | fields - _time | sort count
iseval = 0

# used to generate SPL for simulation
[trackme_eval_spl]
definition = eval spl=case(\
search_mode="tstats", "| " . search_mode . " max(_indextime) as data_last_ingest, min(_time) as data_first_time_seen, max(_time) as data_last_time_seen, count as data_eventcount, dc(host) as dcount_host" . " where " . search_constraint . " | eval object=\"" . object . "\", data_index=\"" . elastic_index . "\", data_sourcetype=\"" . elastic_sourcetype . "\", data_last_ingestion_lag_seen=data_last_ingest-data_last_time_seen",\
search_mode="raw", search_constraint . " | stats max(_indextime) as data_last_ingest, min(_time) as data_first_time_seen, max(_time) as data_last_time_seen, count as data_eventcount, dc(host) as dcount_host" . " | eval object=\"" . object . "\", data_index=\"" . elastic_index . "\", data_sourcetype=\"" . elastic_sourcetype . "\", data_last_ingestion_lag_seen=data_last_ingest-data_last_time_seen",\
search_mode="from" AND match(from_part1, "(?i)datamodel\:"), "| " . search_mode . " " . from_part1 . " | " . from_part2 . " | stats max(_indextime) as data_last_ingest, min(_time) as data_first_time_seen, max(_time) as data_last_time_seen, count as data_eventcount, dc(host) as dcount_host" . " | eval object=\"" . object . "\", data_index=\"" . elastic_index . "\", data_sourcetype=\"" . elastic_sourcetype . "\", data_last_ingestion_lag_seen=data_last_ingest-data_last_time_seen",\
search_mode="from" AND match(from_part1, "(?i)lookup\:"), "| " . search_mode . " " . from_part1 . " | " . from_part2 . " | eventstats max(_time) as indextime | eval _indextime=if(isnum(_indextime), _indextime, indextime) | fields - indextime | eval host=if(isnull(host), \"none\", host) | stats max(_indextime) as data_last_ingest, min(_time) as data_first_time_seen, max(_time) as data_last_time_seen, count as data_eventcount, dc(host) as dcount_host" . " | eval object=\"" . object . "\", data_index=\"" . elastic_index . "\", data_sourcetype=\"" . elastic_sourcetype . "\", data_last_ingestion_lag_seen=data_last_ingest-data_last_time_seen",\
search_mode="mstats", "| " . search_mode . " latest(_value) as value" . " where " . search_constraint . " by host, metric_name span=1m | stats min(_time) as data_first_time_seen, max(_time) as data_last_time_seen, dc(metric_name) as data_eventcount, dc(host) as dcount_host | eval object=\"" . object . "\", data_index=\"" . elastic_index . "\", data_sourcetype=\"" . elastic_sourcetype . "\", data_last_ingest=data_last_time_seen, data_last_ingestion_lag_seen=now()-data_last_time_seen",\
search_mode="mpreview", "| " . search_mode . " " . search_constraint . " | stats max(_indextime) as data_last_ingest, min(_time) as data_first_time_seen, max(_time) as data_last_time_seen, count as data_eventcount, dc(host) as dcount_host" . " | eval object=\"" . object . "\", data_index=\"" . elastic_index . "\", data_sourcetype=\"" . elastic_sourcetype . "\", data_last_ingestion_lag_seen=data_last_ingest-data_last_time_seen",\
search_mode="remote_tstats", "| splunkremotesearch " . remote_target . " search=\"" . "| tstats" . " max(_indextime) as data_last_ingest, min(_time) as data_first_time_seen, max(_time) as data_last_time_seen, count as data_eventcount, dc(host) as dcount_host" . " where " . remote_constrainst . " | eval object=\\\"" . object . "\\\", data_index=\\\"" . elastic_index . "\\\", data_sourcetype=\\\"" . elastic_sourcetype . "\\\", data_last_ingestion_lag_seen=data_last_ingest-data_last_time_seen\"",\
search_mode="remote_raw", "| splunkremotesearch " . remote_target . " search=\"" . "search " . remote_constrainst . " | stats max(_indextime) as data_last_ingest, min(_time) as data_first_time_seen, max(_time) as data_last_time_seen, count as data_eventcount, dc(host) as dcount_host" . " | eval object=\\\"" . object . "\\\", data_index=\\\"" . elastic_index . "\\\", data_sourcetype=\\\"" . elastic_sourcetype . "\\\", data_last_ingestion_lag_seen=data_last_ingest-data_last_time_seen\"",\
search_mode="remote_from" AND match(remote_constrainst_part1, "(?i)datamodel\:"), "| splunkremotesearch " . remote_target . " search=\"" . "| from " . remote_constrainst_part1 . " | " . remote_constrainst_part2 . " | stats max(_indextime) as data_last_ingest, min(_time) as data_first_time_seen, max(_time) as data_last_time_seen, count as data_eventcount, dc(host) as dcount_host" . " | eval object=\\\"" . object . "\\\", data_index=\\\"" . elastic_index . "\\\", data_sourcetype=\\\"" . elastic_sourcetype . "\\\", data_last_ingestion_lag_seen=data_last_ingest-data_last_time_seen\"",\
search_mode="remote_from" AND match(remote_constrainst_part1, "(?i)lookup\:"), "| splunkremotesearch " . remote_target . " search=\"" . "| from " . remote_constrainst_part1 . " | " . remote_constrainst_part2 . " | eventstats max(_time) as indextime | eval _indextime=if(isnum(_indextime), _indextime, indextime) | fields - indextime | eval host=if(isnull(host), \\\"none\\\", host) | stats max(_indextime) as data_last_ingest, min(_time) as data_first_time_seen, max(_time) as data_last_time_seen, count as data_eventcount, dc(host) as dcount_host" . " | eval object=\\\"" . object . "\\\", data_index=\\\"" . elastic_index . "\\\", data_sourcetype=\\\"" . elastic_sourcetype . "\\\", data_last_ingestion_lag_seen=data_last_ingest-data_last_time_seen\"",\
search_mode="remote_mstats", "| splunkremotesearch " . remote_target . " search=\"" . "| mstats" . " latest(_value) as value" . " where " . remote_constrainst . " by host, metric_name span=1m | stats min(_time) as data_first_time_seen, max(_time) as data_last_time_seen, dc(metric_name) as data_eventcount, dc(host) as dcount_host | eval data_last_ingest=data_last_time_seen | eval object=\\\"" . object . "\\\", data_index=\\\"" . elastic_index . "\\\", data_sourcetype=\\\"" . elastic_sourcetype . "\\\", data_last_ingestion_lag_seen=data_last_ingest-data_last_time_seen\"",\
search_mode="remote_mpreview", "| splunkremotesearch " . remote_target . " search=\"" . "| mpreview" . " " . remote_constrainst . " | stats min(_time) as data_first_time_seen, max(_time) as data_last_time_seen, count as data_eventcount, dc(host) as dcount_host" . " | eval object=\\\"" . object . "\\\", data_index=\\\"" . elastic_index . "\\\", data_sourcetype=\\\"" . elastic_sourcetype . "\\\"\""\
)
iseval = 0

# used to simulate elastic sources

[trackme_elastic_sources_simulate_genspl(3)]
definition = rex field=search_constraint "^(?<from_part1>[^\|]*)\|{0,1}(?<from_part2>.*)" | eval from_part2=if(search_mode="from" AND (isnull(from_part2) OR from_part2=""), "search *", from_part2)\
``` remote search ```\
| rex field=search_constraint "(?<remote_target>(?:account)\=[^\|]*)\s{0,}\|\s{0,}(?<remote_constrainst>.*)"\
| rex field=search_constraint "(?<remote_target>(?:account)\=[^\|]*)\s{0,}\|\s{0,}(?<remote_constrainst_part1>[^\|]*\s{0,})\|{0,1}\s{0,}(?<remote_constrainst_part2>.*)"\
| rex field=remote_constrainst mode=sed "s/\"/\\\"/g"\
| rex field=remote_constrainst_part1 mode=sed "s/\"/\\\"/g"\
| rex field=remote_constrainst_part2 mode=sed "s/\"/\\\"/g"\
| eval remote_constrainst_part2=if(search_mode="remote_from" AND (isnull(remote_constrainst_part2) OR remote_constrainst_part2=""), "search *", remote_constrainst_part2)\
\
| `trackme_eval_spl`\
| eval spl=if(match(search_mode, "^remote_\w+"), spl . " earliest=\"$earliest_time$\" " . " latest=\"$latest_time$\" ", spl)\
\
| fields spl
args = object,earliest_time,latest_time
iseval = 0

[trackme_elastic_sources_simulate_get_results(2)]
definition =  eval data_index=if(isnull(data_index) OR data_index="none", object, data_index)\
| eval data_sourcetype=if(isnull(data_sourcetype) OR data_sourcetype="none", object, data_sourcetype)\
| eval simulation="true"\
`trackme_dsm_tracker_abstract($tenant_id$, elastic)`\
| append [ | makeresults | head 1 | eval object="$object$", simulation_results="No results found, please verify your search." | fields - _time ]\
| fillnull value="Success, you can now add this new source to the shared tracker or as a dedicated tracker." simulation_results\
| lookup trackme_dsm_elastic_shared_tenant_$tenant_id$ object OUTPUTNEW object as object_found\
| lookup trackme_dsm_elastic_dedicated_tenant_$tenant_id$ object OUTPUTNEW object as object_found\
| eval simulation_results=if(isnotnull(object_found), "ERROR: this data_source was found in the collection!", simulation_results)\
| eval " " = case(\
simulation_results="No results found, please verify your search.", "icon|ico_warn ico_small|icon-close|no results",\
simulation_results="ERROR: this data_source was found in the collection!", "icon|ico_error ico_small|icon-close|error",\
simulation_results="Success, you can now add this new source to the shared tracker or as a dedicated tracker.", "icon|ico_good ico_small|icon-check|success")\
| fields " ", simulation_results, object, data*, * | where object="$object$"\
| head 1
args = tenant_id, object
iseval = 0

# used within the UI to get search definition for elastic sources
[trackme_lookup_elastic_sources(1)]
args = tenant_id
definition = lookup trackme_dsm_elastic_shared_tenant_$tenant_id$ object OUTPUTNEW search_constraint as elastic_source_search_constraint, search_mode as elastic_source_search_mode, earliest as elastic_earliest, latest as elastic_latest\
| lookup trackme_dsm_elastic_dedicated_tenant_$tenant_id$ object OUTPUTNEW search_constraint as elastic_source_search_constraint, search_mode as elastic_source_search_mode, elastic_report\
| eval elastic_earliest=if(isnotnull(elastic_source_search_constraint) AND isnull(elastic_earliest), "-4h", elastic_earliest)\
| eval elastic_latest=if(isnotnull(elastic_source_search_constraint) AND isnull(elastic_latest), "+4h", elastic_latest)\
| rex field=elastic_source_search_constraint "^\s{0,}(?<elastic_source_from_part1>[^\|]*)\|{0,1}\s{0,}(?<elastic_source_from_part2>.*)?"\
| eval elastic_source_from_part2=case(\
elastic_source_search_mode="from" AND match(elastic_source_from_part1, "(?i)datamodel\:") AND (isnull(elastic_source_from_part2) OR elastic_source_from_part2=""), "search *",\
elastic_source_search_mode="from" AND match(elastic_source_from_part1, "(?i)lookup\:") AND (isnull(elastic_source_from_part2) OR elastic_source_from_part2=""), "eventstats max(_time) as indextime | eval _indextime=if(isnum(_indextime), _indextime, indextime) | fields - indextime | eval host=if(isnull(host), \"none\", host)",\
elastic_source_search_mode="from" AND match(elastic_source_from_part1, "(?i)lookup\:") AND (isnotnull(elastic_source_from_part2) AND elastic_source_from_part2!=""), elastic_source_from_part2 . "eventstats max(_time) as indextime | eval _indextime=if(isnum(_indextime), _indextime, indextime) | fields - indextime | eval host=if(isnull(host), \"none\", host)",\
(elastic_source_search_mode="remote_from") AND match(elastic_source_from_part2, "(?i)datamodel\:"), elastic_source_from_part2,\
(elastic_source_search_mode="remote_from") AND match(elastic_source_from_part2, "(?i)lookup\:"), elastic_source_from_part2 . " | eventstats max(_time) as indextime | eval _indextime=if(isnum(_indextime), _indextime, indextime) | fields - indextime | eval host=if(isnull(host), \"none\", host)",\
isnotnull(elastic_source_from_part2) AND elastic_source_from_part2!="", elastic_source_from_part2\
)\
| rex field=elastic_source_search_constraint "(?P<elastic_mstats_idx>index=[\w|\*]*)(?P<elastic_mstats_filters>.*)"\
| eval elastic_mstats_idx=if(elastic_source_search_mode="mstats" OR elastic_source_search_mode="remote_mstats", elastic_mstats_idx, "")\
| eval elastic_mstats_filters=if(elastic_source_search_mode="mstats" OR elastic_source_search_mode="remote_mstats", elastic_mstats_filters, "")
iseval = 0

# Same but excludes some lookup based from Elastic Sources
[trackme_lookup_elastic_sources_for_data_sampling(1)]
definition = lookup trackme_dsm_elastic_shared_tenant_$tenant_id$ object OUTPUTNEW search_constraint as elastic_source_search_constraint, search_mode as elastic_source_search_mode\
| lookup trackme_dsm_elastic_dedicated_tenant_$tenant_id$ object OUTPUTNEW search_constraint as elastic_source_search_constraint, search_mode as elastic_source_search_mode\
| rex field=elastic_source_search_constraint "^\s{0,}(?<elastic_source_from_part1>[^\|]*)\|{0,1}\s{0,}(?<elastic_source_from_part2>.*)?"\
\
| rex field=elastic_source_from_part1 mode=sed "s/\\\\//g"\
\
```| where NOT match(elastic_source_search_mode, "^(remote_|remoteraw_)") ```\
| eval elastic_source_from_part2=case(\
elastic_source_search_mode="from" AND match(elastic_source_from_part1, "(?i)datamodel\:") AND (isnull(elastic_source_from_part2) OR elastic_source_from_part2=""), "search *",\
elastic_source_search_mode="from" AND match(elastic_source_from_part1, "(?i)lookup\:") AND (isnull(elastic_source_from_part2) OR elastic_source_from_part2=""), "eventstats max(_time) as indextime | eval _indextime=if(isnum(_indextime), _indextime, indextime) | fields - indextime | eval host=if(isnull(host), \"none\", host)",\
elastic_source_search_mode="from" AND match(elastic_source_from_part1, "(?i)lookup\:") AND (isnotnull(elastic_source_from_part2) AND elastic_source_from_part2!=""), elastic_source_from_part2 . "eventstats max(_time) as indextime | eval _indextime=if(isnum(_indextime), _indextime, indextime) | fields - indextime | eval host=if(isnull(host), \"none\", host)",\
(elastic_source_search_mode="remote_from") AND match(elastic_source_from_part2, "(?i)datamodel\:"), elastic_source_from_part2,\
(elastic_source_search_mode="remote_from") AND match(elastic_source_from_part2, "(?i)lookup\:"), elastic_source_from_part2 . " | eventstats max(_time) as indextime | eval _indextime=if(isnum(_indextime), _indextime, indextime) | fields - indextime | eval host=if(isnull(host), \"none\", host)",\
isnotnull(elastic_source_from_part2) AND elastic_source_from_part2!="", elastic_source_from_part2\
)\
| rex field=elastic_source_search_constraint "(?P<elastic_mstats_idx>index=[\w|\*]*)(?P<elastic_mstats_filters>.*)"\
| eval elastic_mstats_idx=if(elastic_source_search_mode="mstats" OR elastic_source_search_mode="remote_mstats", elastic_mstats_idx, "")\
| eval elastic_mstats_filters=if(elastic_source_search_mode="mstats" OR elastic_source_search_mode="remote_mstats", elastic_mstats_filters, "")
args = tenant_id
iseval = 0

# used to complete the code of elastic dedicated trackers
[trackme_elastic_dedicated_tracker(1)]
definition = eval tenant_id="$tenant_id$"\
\
| where isnotnull(object) AND data_eventcount>0\
\
``` define data_last_ingestion_lag_seen ```\
| eval data_last_ingestion_lag_seen=data_last_ingest-data_last_time_seen\
\
``` Specific to elastic sources ```\
| eval data_index=if(isnull(data_index) OR data_index="none", object, data_index)\
| eval data_sourcetype=if(isnull(data_sourcetype) OR data_sourcetype="none", object, data_sourcetype)\
\
``` call the abstract macro ```\
`trackme_dsm_tracker_abstract($tenant_id$, elastic)`\
\
``` collects latest collection state into the summary index ```\
| `trackme_collect_state("current_state_tracking:splk-dsm:$tenant_id$", "object", "$tenant_id$")`\
\
``` output flipping change status if changes ```\
| trackmesplkgetflipping tenant_id="$tenant_id$" object_category="splk-dsm"\
\
```Generate splk outliers rules``` \
| `set_splk_outliers_rules($tenant_id$, "dsm")`\
\
``` run collect and updates the KVstore ```\
| `trackme_outputlookup(trackme_dsm_tenant_$tenant_id$, key)`\
| `trackme_mcollect(object, splk-dsm, "metric_name:trackme.splk.feeds.avg_eventcount_5m=avg_eventcount_5m, metric_name:trackme.splk.feeds.latest_eventcount_5m=latest_eventcount_5m, \
metric_name:trackme.splk.feeds.perc95_eventcount_5m=perc95_eventcount_5m, metric_name:trackme.splk.feeds.stdev_eventcount_5m=stdev_eventcount_5m, \
metric_name:trackme.splk.feeds.avg_latency_5m=avg_latency_5m, metric_name:trackme.splk.feeds.latest_latency_5m=latest_latency_5m, \
metric_name:trackme.splk.feeds.perc95_latency_5m=perc95_latency_5m, metric_name:trackme.splk.feeds.stdev_latency_5m=stdev_latency_5m, \
metric_name:trackme.splk.feeds.avg_dcount_host_5m=avg_dcount_host_5m, metric_name:trackme.splk.feeds.latest_dcount_host_5m=latest_dcount_host_5m, \
metric_name:trackme.splk.feeds.global_dcount_host=global_dcount_host, \
metric_name:trackme.splk.feeds.perc95_dcount_host_5m=perc95_dcount_host_5m, metric_name:trackme.splk.feeds.stdev_dcount_host_5m=stdev_dcount_host_5m, \
metric_name:trackme.splk.feeds.eventcount_4h=data_eventcount, metric_name:trackme.splk.feeds.hostcount_4h=dcount_host, metric_name:trackme.splk.feeds.lag_event_sec=data_last_lag_seen, \
metric_name:trackme.splk.feeds.lag_ingestion_sec=data_last_ingestion_lag_seen", "tenant_id, object_category, object", "$tenant_id$")`\
| stats c
args = tenant_id
iseval = 0

[trackme_elastic_dedicated_tracker(2)]
definition = eval tenant_id="$tenant_id$"\
\
| where isnotnull(object) AND data_eventcount>0\
\
``` define data_last_ingestion_lag_seen ```\
| eval data_last_ingestion_lag_seen=data_last_ingest-data_last_time_seen\
\
``` Specific to elastic sources ```\
| eval data_index=if(isnull(data_index) OR data_index="none", object, data_index)\
| eval data_sourcetype=if(isnull(data_sourcetype) OR data_sourcetype="none", object, data_sourcetype)\
\
``` call the abstract macro ```\
`trackme_dsm_tracker_abstract($tenant_id$, elastic)`\
\
| where object="$tracker_name$"\
\
``` collects latest collection state into the summary index ```\
| `trackme_collect_state("current_state_tracking:splk-dsm:$tenant_id$", "object", "$tenant_id$")`\
\
``` output flipping change status if changes ```\
| trackmesplkgetflipping tenant_id="$tenant_id$" object_category="splk-dsm"\
\
```Generate splk outliers rules``` \
| `set_splk_outliers_rules($tenant_id$, "dsm")`\
\
``` run collect and updates the KVstore ```\
| `trackme_outputlookup(trackme_dsm_tenant_$tenant_id$, key)`\
| `trackme_mcollect(object, splk-dsm, "metric_name:trackme.splk.feeds.avg_eventcount_5m=avg_eventcount_5m, metric_name:trackme.splk.feeds.latest_eventcount_5m=latest_eventcount_5m, \
metric_name:trackme.splk.feeds.perc95_eventcount_5m=perc95_eventcount_5m, metric_name:trackme.splk.feeds.stdev_eventcount_5m=stdev_eventcount_5m, \
metric_name:trackme.splk.feeds.avg_latency_5m=avg_latency_5m, metric_name:trackme.splk.feeds.latest_latency_5m=latest_latency_5m, \
metric_name:trackme.splk.feeds.perc95_latency_5m=perc95_latency_5m, metric_name:trackme.splk.feeds.stdev_latency_5m=stdev_latency_5m, \
metric_name:trackme.splk.feeds.avg_dcount_host_5m=avg_dcount_host_5m, metric_name:trackme.splk.feeds.latest_dcount_host_5m=latest_dcount_host_5m, \
metric_name:trackme.splk.feeds.global_dcount_host=global_dcount_host, \
metric_name:trackme.splk.feeds.perc95_dcount_host_5m=perc95_dcount_host_5m, metric_name:trackme.splk.feeds.stdev_dcount_host_5m=stdev_dcount_host_5m, \
metric_name:trackme.splk.feeds.eventcount_4h=data_eventcount, metric_name:trackme.splk.feeds.hostcount_4h=dcount_host, metric_name:trackme.splk.feeds.lag_event_sec=data_last_lag_seen, \
metric_name:trackme.splk.feeds.lag_ingestion_sec=data_last_ingestion_lag_seen", "tenant_id, object_category, object", "$tenant_id$")`
args = tenant_id, tracker_name
iseval = 0

# used to complete the hybrid tracker simulation results
[trackme_hybrid_tracker_simulation]
definition = eval simulation_result=if(dcount_entities>=1, "Success, one or more entities were found, you can now add this new hybrid tracker if you wish to do so.", "No results found, please verify your search.")\
| eval status = if(dcount_entities>=1, "success", "no_results")\
| fields status, simulation_result, dcount_entities, *
iseval = 0

# used to complete the replica tracker simulation results
[trackme_replica_tracker_simulation]
definition = stats dc(object) as dcount_entities, first(tenant_id) as parent_tenant_id\
| eval simulation_result=if(dcount_entities>=1, "Success, one or more entities were found, you can now add this new replica tracker if you wish to do so.", "No results found, please verify your search.")\
| eval status = if(dcount_entities>=1, "success", "no_results")\
| fields " ", simulation_result, dcount_entities, parent_tenant_id
iseval = 0

# Data Quality - rex for extractions
[trackme_data_quality_parse]
definition = rex "Context:\s*source=(?<data_source>[^\|]*)\|host=(?<object>[^\|]*)\|(?<data_sourcetype>[^\|]*)\|"
iseval = 0

# Simple macro to parse duration in seconds and show duration string if higher than x secs
[trackme_parse_duration(2)]
definition = eval $inputfield$=if($inputfield$>$maxsec$, tostring(round($inputfield$, 0), "duration"), round($inputfield$, 1) . " sec")
args = inputfield, maxsec
iseval = 0

# tags policies genspl
[trackme_tags_policies_genspl(2)]
definition = makeresults | head 1 | eval spl = "case(" | fields - _time\
| append [ | inputlookup trackme_common_tag_policies_tenant_$tenant_id$ | eval tags_policy_value=lower(mvjoin(tags_policy_value, ",")) | rex field=tags_policy_regex mode=sed "s/\"/\\\"/g"\
| eval spl = "match($target_field$, \"" . tags_policy_regex . "\"), \"" . tags_policy_value . "\""\
| streamstats count as record_id\
| eventstats max(record_id) as total_records\
| eval spl = if(record_id!=total_records, spl . ",", spl) ]\
| append [ | makeresults | head 1 | eval spl = ")" | fields - _time ]\
| stats list(spl) as spl | nomv spl | eval spl=if(spl == "case( )", "noop", spl)
iseval = 0
args = tenant_id, target_field

# tags policies merged tags
[trackme_tags_policies_apply(2)]
definition = lookup trackme_dsm_tags_tenant_$tenant_id$ _key as key OUTPUT tags_auto \
| eval tags_merged = case(\
(isnotnull(tags_manual) AND tags_manual!="") AND (isnotnull(tags_auto) AND tags_auto!=""), lower(tags_manual) . "," . lower(mvjoin(tags_auto, ",")),\
isnotnull(tags_manual) AND tags_manual!="", lower(tags_manual),\
isnotnull(tags_auto) AND tags_auto!="", lower(mvjoin(tags_auto, ","))\
)\
| eval tags=lower(mvjoin(mvsort(mvdedup(split(tags_merged, ","))), ",")) | fields - tags_merged
iseval = 0
args = tenant_id, target_field

# Host alerting policy
# Since TrackMe 1.2.27, data host monitoring can be managed in a track fashion, where users can decide if an host should be triggering red 
# if all sourcetypes are red (default) or if at least one sourcetype is red
# The default policy behaviour is to trigger red only when none of the known and accessible sourcetypes of a given host do not meet monitoring conditions
[trackme_default_splk_dhm_alert_policy]
definition = "track_per_host"
iseval = 0

# Default policy
[trackme_default_splk_dhm_alert_global_policy]
definition = eval splk_dhm_alerting_policy=if(isnull(splk_dhm_alerting_policy), "global_policy", splk_dhm_alerting_policy)
iseval = 0

# data host get summary for smart status
[trackme_smart_status_summary_dh(1)]
definition = tstats max(_indextime) as data_last_ingest, min(_time) as data_first_time_seen, max(_time) as data_last_time_seen, count as data_eventcount where [ | inputlookup trackme_host_monitoring where object="$object$" | fields data_index | makemv delim="," data_index | mvexpand data_index | rename data_index as index | table index ] host="$object$" by index, sourcetype, host\
| eval data_last_ingestion_lag_seen=data_last_ingest-data_last_time_seen\
| `trackme_splk_dhm_tracker_abstract`\
| where object="$object$"\
| fields splk_dhm_st_summary\
| makemv delim="," splk_dhm_st_summary\
| mvexpand splk_dhm_st_summary\
| rex field="splk_dhm_st_summary" "^idx=(?<summary_idx>[^\|]*)\|st=(?<summary_st>[^\|]*)\|max_lag_allowed=(?<summary_max_lag_allowed>[^\|]*)\|last_ingest=(?<summary_last_ingest>[^\|]*)\|first_time=(?<summary_first_time>[^\|]*)\|last_time=(?<summary_last_time>[^\|]*)\|last_ingest_lag=(?<summary_last_ingest_lag>[^\|]*)\|last_event_lag=(?<summary_last_event_lag>[^\|]*)\|time_measure=(?<time>[^\|]*)\|state=(?<summary_state>[^\|]*)"\
| where (summary_state!="green" AND summary_state!="info")\
| fields summary_idx summary_st summary_max_lag_allowed summary_last_ingest_lag summary_last_ingest summary_last_time summary_last_event_lag summary_state\
| rename summary_* as "*"\
| eval last_ingest=strftime(last_ingest, "%c"), last_time=strftime(last_time, "%c"), last_ingest_lag=if(last_ingest_lag>60, tostring(last_ingest_lag, "duration"), last_ingest_lag), last_event_lag=if(last_event_lag>60, tostring(last_event_lag, "duration"), last_event_lag)\
| eval summary = "[ " . "idx=" . idx . ", st=" . st . ", max_lag_allowed=" . max_lag_allowed . ", last_ingest=" . last_ingest . ", last_ingest_lag=" . last_ingest_lag . ", last_event=" . last_time . ", last_event_lag=" . last_event_lag . ", state=" . state . " ]"\
| fields summary\
| stats values(summary) as summary\
| eval summary = mvjoin(summary, ", ")
args = object
iseval = 0

# data host get summary for data in the future for smart status
[trackme_smart_status_summary_future_dh(1)]
definition = tstats max(_indextime) as data_last_ingest, min(_time) as data_first_time_seen, max(_time) as data_last_time_seen, count as data_eventcount where [ | inputlookup trackme_host_monitoring where object="$object$" | fields data_index | makemv delim="," data_index | mvexpand data_index | rename data_index as index | table index ] host="$object$" by index, sourcetype, host\
| eval data_last_ingestion_lag_seen=data_last_ingest-data_last_time_seen\
| `trackme_splk_dhm_tracker_abstract`\
| where object="$object$"\
| fields splk_dhm_st_summary\
| makemv delim="," splk_dhm_st_summary\
| mvexpand splk_dhm_st_summary\
| rex field="splk_dhm_st_summary" "^idx=(?<summary_idx>[^\|]*)\|st=(?<summary_st>[^\|]*)\|max_lag_allowed=(?<summary_max_lag_allowed>[^\|]*)\|last_ingest=(?<summary_last_ingest>[^\|]*)\|first_time=(?<summary_first_time>[^\|]*)\|last_time=(?<summary_last_time>[^\|]*)\|last_ingest_lag=(?<summary_last_ingest_lag>[^\|]*)\|last_event_lag=(?<summary_last_event_lag>[^\|]*)\|time_measure=(?<time>[^\|]*)\|state=(?<summary_state>[^\|]*)"\
| eval summary_state=if(summary_last_event_lag<-600, "orange", summary_state)\
| where (summary_state!="green" AND summary_state!="info")\
| fields summary_idx summary_st summary_max_lag_allowed summary_last_ingest_lag summary_last_ingest summary_last_time summary_last_event_lag summary_state\
| rename summary_* as "*"\
| eval last_ingest=strftime(last_ingest, "%c"), last_time=strftime(last_time, "%c"), last_ingest_lag=if(last_ingest_lag>60, tostring(last_ingest_lag, "duration"), last_ingest_lag), last_event_lag=if(last_event_lag>60, tostring(last_event_lag, "duration"), last_event_lag)\
| eval summary = "[ " . "idx=" . idx . ", st=" . st . ", max_lag_allowed=" . max_lag_allowed . ", last_ingest=" . last_ingest . ", last_ingest_lag=" . last_ingest_lag . ", last_event=" . last_time . ", last_event_lag=" . last_event_lag . ", state=" . state . " ]"\
| fields summary\
| stats values(summary) as summary\
| eval summary = mvjoin(summary, ", ")
args = object
iseval = 0

# metric host get summary for smart status
[trackme_smart_status_summary_mh(1)]
definition = savedsearch "trackMe - metric per host table report" host="$object$"\
| where object_state!="green"\
\
| append [ | mcatalog values(metric_name) as metric_name where index=* metric_name=* earliest="-5m" latest="now"\
| mvexpand metric_name\
| rex field=metric_name "(?<metric_category>[^\.]*)\.{0,1}"\
| stats dc(metric_name) as metrics by metric_category\
| eval isActive = "true (" + metrics + " distinct metrics active for any hosts during the past 5 min - these metrics are active for other hosts - the issue is likely limited to this specific host" ]\
| stats first(*) as "*" by metric_category\
| eval isActive = if(isnull(isActive), "false (no active metric activity for any hosts during the past 5 min - these metrics are apparently inactive - the issue is likely to be concerning all hosts and not limited to this specific host)", isActive)\
| eval summary="metric_category=" . metric_category . ", max_lag_allowed=" . metric_max_lag_allowed . ", current_lag=" . metric_current_lag_sec . ", lag_duration=". duration . ", last_time=" . metric_last_time . ", state=" . object_state . ", isActive=" . isActive\
\
| fields summary\
| stats values(summary) as summary\
| eval summary=mvjoin(summary, ", ")
args = object
iseval = 0

# convert wdays monitoring days to human
[trackme_smart_status_weekdays_to_human]
definition = rex field=data_monitoring_wdays mode=sed "s/0/Sunday/g"\
| rex field=data_monitoring_wdays mode=sed "s/1/Mondayy/g"\
| rex field=data_monitoring_wdays mode=sed "s/2/Tuesday/g"\
| rex field=data_monitoring_wdays mode=sed "s/3/Wednesday/g"\
| rex field=data_monitoring_wdays mode=sed "s/4/Thursday/g"\
| rex field=data_monitoring_wdays mode=sed "s/5/Friday/g"\
| rex field=data_monitoring_wdays mode=sed "s/6/Saturday/g"
iseval = 0

# time filter - used in alert tracking to provide time filtering in alerting
[AnyTime]
definition = eval time_filtering="false" | fields - time_filtering
iseval = 0

[Day_BusinessDays_8h-20h]
definition = eval local_time=strftime(_time, "%H:%M"), date_wday=lower(strftime(_time, "%A")) | search (local_time>="08:00" AND local_time<="20:00") AND (date_wday!="sunday" date_wday!="saturday") | fields - local_time, date_wday
iseval = 0

[Day_WeekEnd_8h-20h]
definition = eval local_time=strftime(_time, "%H:%M"), date_wday=lower(strftime(_time, "%A")) | search (local_time>="08:00" AND local_time<="20:00") AND (date_wday="sunday" OR date_wday="saturday") | fields - local_time, date_wday
iseval = 0

[Day_AllDays_8h-20h]
definition = eval local_time=strftime(_time, "%H:%M"), date_wday=lower(strftime(_time, "%A")) | search (local_time>="08:00" AND local_time<="20:00") | fields - local_time, date_wday
iseval = 0

[Night_BusinessDays_20h-8h]
definition = eval local_time=strftime(_time, "%H:%M"), date_wday=lower(strftime(_time, "%A")) | search (local_time>="20:00" AND local_time<="23:59") OR (local_time>="00:00" AND local_time<="08:00") AND (date_wday!="sunday" date_wday!="saturday") | fields - local_time, date_wday
iseval = 0

[Night_WeekEnd_20h-8h]
definition = eval local_time=strftime(_time, "%H:%M"), date_wday=lower(strftime(_time, "%A")) | search (local_time>="20:00" AND local_time<="23:59") OR (local_time>="00:00" AND local_time<="08:00") AND (date_wday="sunday" OR date_wday="saturday") | fields - local_time, date_wday
iseval = 0

[Night_AllDays_20h-8h]
definition = eval local_time=strftime(_time, "%H:%M"), date_wday=lower(strftime(_time, "%A")) | search (local_time>="20:00" AND local_time<="23:59") OR (local_time>="00:00" AND local_time<="08:00") | fields - local_time, date_wday
iseval = 0

# gen checkbox hours ranges
[genHoursRanges]
definition = makeresults | head 1 | eval label="00h-to-01h59", value="0,1"\
| append [ | makeresults | head 1 | eval label="02h-to-03h59", value="2,3" ]\
| append [ | makeresults | head 1 | eval label="04h-to-05h59", value="4,5" ]\
| append [ | makeresults | head 1 | eval label="06h-to-07h59", value="6,7" ]\
| append [ | makeresults | head 1 | eval label="08h-to-09h59", value="8,9" ]\
| append [ | makeresults | head 1 | eval label="10h-to-11h59", value="10,11" ]\
| append [ | makeresults | head 1 | eval label="12h-to-13h59", value="12,13" ]\
| append [ | makeresults | head 1 | eval label="14h-to-15h59", value="14,15" ]\
| append [ | makeresults | head 1 | eval label="16h-to-17h59", value="16,17" ]\
| append [ | makeresults | head 1 | eval label="18h-to-19h59", value="18,19" ]\
| append [ | makeresults | head 1 | eval label="20h-to-21h59", value="20,21" ]\
| append [ | makeresults | head 1 | eval label="22h-to-23h59", value="22,23" ]\
| fields - _time
iseval = 0

# exclude permanent entities deletetion when running the trackers, using lookup call for scalability
[trackme_exclude_permanent_deletion(2)]
args = tenant_id, object_category
definition = eval object_category=if(isnull(object_category), "$object_category$", object_category) | lookup trackme_common_permanently_deleted_objects_tenant_$tenant_id$ object, object_category OUTPUT _key as permanently_deleted_key | where isnull(permanently_deleted_key) | fields - permanently_deleted_key
iseval = 0

# decomissioned in TrackMe 2.2.4, will be removed in future versions
# exclude permanent entities deletetion when running the trackers
[trackme_exclude_permanent_deletion(3)]
args = tenant_id, object_category, object_target
definition = search NOT [ | inputlookup trackme_common_permanently_deleted_objects_tenant_$tenant_id$ where (object_category="$object_category$") | dedup object | rename object as $object_target$ | fields $object_target$ | format | return $search ]
iseval = 0

# decomissioned in TrackMe 2.2.4, will be removed in future versions
# variation to include the object itself
[trackme_exclude_permanent_deletion(4)]
args = tenant_id, object_category, object_target, object
definition = search NOT [ | inputlookup trackme_common_permanently_deleted_objects_tenant_$tenant_id$ where (object_category="$object_category$" AND object="$object$") | dedup object | rename object as $object_target$ | fields $object_target$ | format | return $search ]
iseval = 0

# mhm loading
[trackme_mhm_load_collection(1)]
args = tenant_id
definition = inputlookup trackme_mhm_tenant_$tenant_id$\
| eval keyid=_key\
| foreach metric_index, metric_category [ eval <<FIELD>> = split('<<FIELD>>', ",") ]\
| eval metric_index_raw=metric_index, metric_category_raw=metric_category, metric_details_raw=metric_details\
| foreach metric_details [ eval <<FIELD>> = split('<<FIELD>>', ",") ]\
| mvexpand metric_details\
| rex field=metric_details "metric_category=(?<detail_metric_category>[^\|]*)\|metric_last_time=(?<detail_metric_last_time>[^\|]*)\|metric_max_lag_allowed=(?<detail_metric_max_lag_allowed>[^\|]*)\|metric_current_lag_sec=(?<detail_metric_current_lag_sec>[^\|]*)\|object_state=(?<detail_object_state>[^\|]*)"\
| `trackme_date_format(detail_metric_last_time)`\
| eval metric_details_human  = "metric_category=" . detail_metric_category . "|" . "metric_last_time=" . 'detail_metric_last_time (translated)' . "|" . "metric_current_lag_sec=" . detail_metric_current_lag_sec . "|" . "object_state=" . detail_object_state\
| fields - detail_* | where NOT match(metric_details, "metric_category=trackme") | stats values(*) as "*" by keyid
iseval = 0

# mhm loading per host
[trackme_mhm_load_collection(2)]
args = tenant_id, keyid
definition = inputlookup trackme_mhm_tenant_$tenant_id$ where _key=$keyid$\\
| foreach metric_index, metric_category [ eval <<FIELD>> = split('<<FIELD>>', ",") ]\
| eval metric_index_raw=metric_index, metric_category_raw=metric_category, metric_details_raw=metric_details\
| foreach metric_details [ eval <<FIELD>> = split('<<FIELD>>', ",") ]\
| mvexpand metric_details\
| rex field=metric_details "metric_category=(?<detail_metric_category>[^\|]*)\|metric_last_time=(?<detail_metric_last_time>[^\|]*)\|metric_max_lag_allowed=(?<detail_metric_max_lag_allowed>[^\|]*)\|metric_current_lag_sec=(?<detail_metric_current_lag_sec>[^\|]*)\|object_state=(?<detail_object_state>[^\|]*)"\
| `trackme_date_format(detail_metric_last_time)`\
| eval metric_details_human  = "metric_category=" . detail_metric_category . "|" . "metric_last_time=" . 'detail_metric_last_time (translated)' . "|" . "metric_current_lag_sec=" . detail_metric_current_lag_sec . "|" . "object_state=" . detail_object_state\
| fields metric_details_human\
| mvexpand metric_details_human\
| rex field=metric_details_human "metric_category=(?<metric_category>[^\|]*)\|metric_last_time=(?<metric_last_time>[^\|]*)\|metric_current_lag_sec=(?<metric_current_lag_sec>[^\|]*)\|object_state=(?<object_state>\w*)"\
| lookup trackme_metric_lagging_classes_tenant_$tenant_id$ metric_category OUTPUT metric_max_lag_allowed\
| `trackme_default_splk_mhm_lag`\
| `trackme_eval_icons_object_state_only`\
| eval duration=tostring(metric_current_lag_sec, "duration")\
| fields metric_category metric_last_time metric_max_lag_allowed metric_current_lag_sec duration state object_state
iseval = 0

[trackme_mhm_perhost_update(2)]
args = tenant_id, host
definition = trackme_mhm_abstract_root_tenant_$tenant_id$($host$)\
``` collects latest collection state into the summary index ```\
| `trackme_collect_state("current_state_tracking:splk-mhm:$tenant_id$", "object", "$tenant_id$")`\
``` output flipping change status if changes ```\
| trackmesplkgetflipping tenant_id="$tenant_id$" object_category="splk-mhm"\
| eval metric_category=mvjoin(metric_category, ","), metric_index=mvjoin(metric_index, ","), metric_details=mvjoin(metric_details, ",")\
| search NOT [ | inputlookup trackme_common_audit_changes_tenant_$tenant_id$ | where action="success" AND change_type="delete permanent" AND object_category="splk-mhm" | eval _time=time/1000 | where _time>relative_time(now(), "-7d") | table object | dedup object | sort limit=0 object | rename object as object ]\
| `trackme_outputlookup(trackme_mhm_tenant_$tenant_id$, key, $tenant_id$)`\
| stats c
iseval = 0

# Decomission note: this macro is used by the legacy view, and will be removed in future versions
[trackme_splk_dsm_populate_overview(1)]
args = cat
definition = makeresults | head 1 | eval cat="trackme", label="eventcount_5m / latency_5m / delay", value="latest_eventcount_5m, avg_latency_5m, lag_event_sec"\
| append [ | makeresults | head 1 | eval cat="trackme", label="eventcount_5m / latency_5m", value="latest_eventcount_5m, avg_latency_5m" ]\
| append [ | makeresults | head 1 | eval cat="trackme", label="eventcount_5m / delay", value="latest_eventcount_5m, lag_event_sec" ]\
| append [ | makeresults | head 1 | eval cat="trackme", label="eventcount_5m / latest_dcount_host_5m", value="latest_eventcount_5m, latest_dcount_host_5m" ]\
| append [ | makeresults | head 1 | eval cat="splunk", label="events / ingest latency", value="events_count, avg_latency" ]\
| append [ | makeresults | head 1 | eval cat="splunk", label="events / host dcount", value="events_count, dcount_host" ]\
| fields - _time | search cat="$cat$"
iseval = 0

# small macro to list available accounts
[trackme_list_accounts]
definition = trackme mode=get url="/services/trackme/v2/configuration/list_accounts" | spath | rename accounts{} as account | table account | mvexpand account | streamstats count as order | eval order=if(account=="local", 0, order) | sort 0 order
iseval = 0

# small macro to exclude badly formed entities such as containing double quotes crap
[trackme_exclude_badentities]
definition = where NOT match(object, "(?<!\\\\)\\\"")

# small function to populate the email accounts
[trackme_populate_email_accounts]
definition = rest splunk_server=local /servicesNS/nobody/trackme/trackme_emails | dedup title | append [ | makeresults | eval title="localhost" | fields - _time ] | table title
iseval = 0

# simple function to apply the maintenance mode in TrackMe components alerts
[trackme_apply_maintenance_mode]
definition = appendcols [ | inputlookup trackme_maintenance_mode ]\
| filldown tenants_scope, maintenance_mode\
| eval tenant_in_scope = mvfind(tenants_scope, "^" . tenant_id . "$"), tenant_in_scope=if(isnotnull(tenant_in_scope) OR tenants_scope=="*", "true", "false")\
| where NOT (maintenance_mode="enabled" AND tenant_in_scope="true")\
| fields - tenants_scope, tenant_in_scope\
``` exclude entities with priority pending ```\
| where NOT (priority="pending")
iseval = 0

# enhanced function to apply the maintenance mode in TrackMe components alerts
[trackme_apply_maintenance_mode_v2]
definition = eval [ | inputlookup trackme_maintenance_mode | table maintenance | rename maintenance as maintenance_mode | return maintenance_mode ], [ | inputlookup trackme_maintenance_mode | table tenants_scope | rename tenants_scope as maintenance_tenants_scope | return maintenance_tenants_scope ]\
| eval maintenance_tenant_in_scope = mvfind(maintenance_tenants_scope, "^" . tenant_id . "$"), maintenance_tenant_in_scope=if(isnotnull(maintenance_tenant_in_scope) OR maintenance_tenants_scope=="*", 1, 0)\
| where NOT (maintenance_mode=1 AND maintenance_tenant_in_scope=1)\
| fields - maintenance_mode, maintenance_tenants_scope, maintenance_tenant_in_scope\
``` exclude entities with priority pending ```\
| where NOT (priority="pending") \
``` exclude entities with bank holidays active ```\
| `trackme_apply_bank_holidays`
iseval = 0

# enhanced function to apply bank holidays in TrackMe components alerts
[trackme_apply_bank_holidays]
definition = eval [ | inputlookup trackme_bank_holidays | eval bank_holidays_active=if(now()>=start_date AND now()<=end_date, 1, 0) | stats max(bank_holidays_active) as bank_holidays_active | append [ | makeresults | eval bank_holidays_active=0 | fields - _time ] | head 1 | return bank_holidays_active ]\
| eval bank_holidays_active=coalesce(bank_holidays_active, 0)\
| where NOT (bank_holidays_active=1)\
| fields - bank_holidays_active
iseval = 0

#
# splk-fqm
#

[trackme_splk_fqm_simulation_summary]
definition = lookup trackme_cim_recommended_fields field as fieldname OUTPUT is_recommended\
| eval recommended=json_extract(w,"comment.recommended"), recommended=if(is_recommended=="true" OR match(recommended, "(?i)true|1"), "true", "false")\
``` rename ```\
| rename fieldname as @fieldname, fieldstatus as @fieldstatus\
``` prevents failure if no value for metadata.nodename or not in the breakby definition ```\
| eval metadata.nodename=if(isnull('metadata.nodename'), 'metadata.datamodel', 'metadata.nodename')\
``` calculate ```\
| stats\
values(metadata.index) as metadata.index, values(metadata.sourcetype) as metadata.sourcetype,\
values(eval(if('@fieldstatus'=="success" AND recommended=="true", '@fieldname', null()))) as list_fields_recommended_passed, values(eval(if('@fieldstatus'=="failure" AND recommended=="true", '@fieldname', null()))) as list_fields_recommended_failed,\
values(eval(if('@fieldstatus'=="success", '@fieldname', null()))) as list_fields_passed, values(eval(if('@fieldstatus'=="failure", '@fieldname', null()))) as list_fields_failed,\
max(total_events) as total_events_parsed by metadata.datamodel, metadata.nodename\
``` all fields ```\
| eval all_fields = mvappend(list_fields_failed, list_fields_passed)\
| eval all_fields = mvdedup(all_fields)\
| eval final_state = mvmap( all_fields, if( mvfind(list_fields_failed, "^" . all_fields . "$") >= 0, all_fields . "|failed", all_fields . "|success" ) )\
| eval success_fields = mvfilter(match(final_state, "\|success$"))\
| eval failed_fields = mvfilter(match(final_state, "\|failed$"))\
| eval success_fields = mvmap(success_fields, mvindex(split(success_fields, "|"), 0))\
| eval failed_fields = mvmap(failed_fields, mvindex(split(failed_fields, "|"), 0))\
| fields - final_state\
| eval success_fields=if(isnull(success_fields), null(), success_fields), failed_fields=if(isnull(failed_fields), null(), failed_fields)\
| fields - list_fields_passed, list_fields_failed\
``` all recommended fields ```\
| eval all_recommended_fields = mvappend(list_fields_recommended_failed, list_fields_recommended_passed)\
| eval all_recommended_fields = mvdedup(all_recommended_fields)\
| eval final_recommended_state = mvmap( all_recommended_fields, if( mvfind(list_fields_recommended_failed, "^" . all_recommended_fields . "$") >= 0, all_recommended_fields . "|failed", all_recommended_fields . "|success" ) )\
| eval success_recommended_fields = mvfilter(match(final_recommended_state, "\|success$"))\
| eval failed_recommended_fields = mvfilter(match(final_recommended_state, "\|failed$"))\
| eval success_recommended_fields = mvmap(success_recommended_fields, mvindex(split(success_recommended_fields, "|"), 0))\
| eval failed_recommended_fields = mvmap(failed_recommended_fields, mvindex(split(failed_recommended_fields, "|"), 0))\
| fields - final_recommended_state\
| eval success_recommended_fields=if(isnull(success_recommended_fields), null(), success_recommended_fields), failed_recommended_fields=if(isnull(failed_recommended_fields), null(), failed_recommended_fields)\
| fields - list_recommended_fields_passed, list_recommended_fields_failed\
``` calculate ```\
| eventstats\
values(metadata.index) as metadata.index, values(metadata.sourcetype) as metadata.sourcetype,\
dc(all_fields) as total_fields_checked, dc(success_fields) as total_fields_passed, dc(failed_fields) as total_fields_failed,\
dc(all_recommended_fields) as total_fields_recommended_checked, dc(success_recommended_fields) as total_fields_recommended_passed, dc(failed_recommended_fields) as total_fields_recommended_failed\
by metadata.datamodel, metadata.nodename\
| eval percentage_passed=round(total_fields_passed/total_fields_checked*100, 2), percentage_failed=round(total_fields_failed/total_fields_checked*100, 2)\
| eval percentage_recommended_passed=round(total_fields_recommended_passed/total_fields_recommended_checked*100, 2), percentage_recommended_failed=round(total_fields_recommended_failed/total_fields_recommended_checked*100, 2)\
``` render ```\
| rename metadata.* as "*"
iseval = 0

# this macro is used the description_extended field and is called by the monitor jobs
[trackme_fqm_get_description_extended]
definition = eventstats\
count(eval(description=="Field exists and is valid.")) as description_fields_exists_and_is_valid,\
count(eval(description=="Field does not exist but is allowed to be missing.")) as description_fields_does_not_exist_but_allowed,\
count(eval(description=="Field is empty but is allowed to be empty.")) as description_fields_empty_but_allowed,\
count(eval(description=="Field does not exist.")) as description_fields_does_not_exist,\
count(eval(description=="Field exists but contains 'unknown'.")) as description_fields_contains_unknown,\
count(eval(description=="Field exists but one or more values in the list do not match the required pattern.")) as description_fields_list_regex_not_matched,\
count(eval(description=="Field is empty.")) as description_fields_is_empty,\
count(eval(description=="Field is 'unknown'.")) as description_fields_is_unknown,\
count(eval(description=="Field exists but value does not match the required pattern.")) as description_fields_regex_not_matched\
count as count_total by fieldname\
``` calculate pct ```\
| eval pct_fields_exists_and_valid = round(description_fields_exists_and_is_valid/count_total*100, 2)\
| eval pct_fields_does_not_exist_but_allowed = round(description_fields_does_not_exist_but_allowed/count_total*100, 2)\
| eval pct_fields_empty_but_allowed = round(description_fields_empty_but_allowed/count_total*100, 2)\
| eval pct_fields_does_not_exist = round(description_fields_does_not_exist/count_total*100, 2)\
| eval pct_fields_contains_unknown = round(description_fields_contains_unknown/count_total*100, 2)\
| eval pct_fields_list_regex_not_matched = round(description_fields_list_regex_not_matched/count_total*100, 2)\
| eval pct_fields_is_empty = round(description_fields_is_empty/count_total*100, 2)\
| eval pct_fields_is_unknown = round(description_fields_is_unknown/count_total*100, 2)\
| eval pct_fields_regex_not_matched = round(description_fields_regex_not_matched/count_total*100, 2)\
``` eval description ```\
| eval description_extended=if(description_fields_exists_and_is_valid>0, mvappend(description_extended, "category: Field exists and is valid, count: " . description_fields_exists_and_is_valid . ", pct: " . pct_fields_exists_and_valid), description_extended)\
| eval description_extended=if(description_fields_does_not_exist_but_allowed>0, mvappend(description_extended, "category: Field does not exist but is allowed to be missing, count: " . description_fields_does_not_exist_but_allowed . ", pct: " . pct_fields_does_not_exist_but_allowed), description_extended)\
| eval description_extended=if(description_fields_empty_but_allowed>0, mvappend(description_extended, "category: Field is empty but is allowed to be empty, count: " . description_fields_empty_but_allowed . ", pct: " . pct_fields_empty_but_allowed), description_extended)\
| eval description_extended=if(description_fields_does_not_exist>0, mvappend(description_extended, "category: Field does not exist, count: " . description_fields_does_not_exist . ", pct: " . pct_fields_does_not_exist), description_extended)\
| eval description_extended=if(description_fields_contains_unknown>0, mvappend(description_extended, "category: Field exists but contains 'unknown', count: " . description_fields_contains_unknown . ", pct: " . pct_fields_contains_unknown), description_extended)\
| eval description_extended=if(description_fields_list_regex_not_matched>0, mvappend(description_extended, "category: Field exists but one or more values in the list do not match the required pattern, count: " . description_fields_list_regex_not_matched . ", pct: " . pct_fields_list_regex_not_matched), description_extended)\
| eval description_extended=if(description_fields_is_empty>0, mvappend(description_extended, "category: Field is empty, count: " . description_fields_is_empty . ", pct: " . pct_fields_is_empty), description_extended)\
| eval description_extended=if(description_fields_is_unknown>0, mvappend(description_extended, "category: Field is 'unknown', count: " . description_fields_is_unknown . ", pct: " . pct_fields_is_unknown), description_extended)\
| eval description_extended=if(description_fields_regex_not_matched>0, mvappend(description_extended, "category: Field exists but value does not match the required pattern, count: " . description_fields_regex_not_matched . ", pct: " . pct_fields_regex_not_matched), description_extended)\
``` replace description by description_extended if available ```\
| eval description=if(isnotnull(description_extended) AND description_extended!="", description_extended, description)\
| fields - description_extended
iseval = 0