# General
[instrumentation.lastSent]
search = index=_telemetry source=telemetry sourcetype=splunk_telemetry_log status=success | fillnull value=anonymous visibility | eval anonymous_send_time = if(visibility LIKE "%anonymous%", _time, null) | eval license_send_time = if(visibility LIKE "%license%", _time, null) | eval support_send_time = if(visibility LIKE "%support%", _time, null) | stats latest(anonymous_send_time) as latest_anonymous_send_time latest(license_send_time) as latest_license_send_time latest(support_send_time) as latest_support_send_time

[instrumentation.reportingErrorCount]
search = index=_telemetry source=telemetry sourcetype=splunk_telemetry_log status=failed | fillnull value=anonymous visibility | stats count(eval(visibility LIKE "%anonymous%")) as anonymous_errors count(eval(visibility LIKE "%license%")) as license_errors count(eval(visibility LIKE "%support%")) as support_errors

# Anonymous
# For splunk core <= 7.0.x and splunk_instrumentation <= 3.0.x, anonymous usage data is indexed in _telemtry
# For later versions, data is indexed in _introspection
[instrumentation.anonymized.eventsByTime]
search = (index=_introspection OR index=_telemetry) sourcetype=splunk_telemetry source="http-stream" visibility=*anonymous* | append [| savedsearch instrumentation.licenseUsage]

# Support
# For splunk core <= 7.0.x and splunk_instrumentation <= 3.0.x, support usage data is indexed in _telemtry
# For later versions, data is indexed in _introspection
[instrumentation.support.eventsByTime]
search = (index=_introspection OR index=_telemetry) sourcetype=splunk_telemetry source="http-stream" visibility=*support* | append [| savedsearch instrumentation.licenseUsage]

# Deployment
[instrumentation.deployment.clustering.indexer]
search = | makeresults annotate=true | append [localop | rest /services/cluster/config] | sort -mode | head 1 | eval data=if(mode=="master","{\"host\":\""+splunk_server+"\",\"timezone\":\""+strftime(now(),"%z")+"\",\"multiSite\":"+multisite+",\"summaryReplication\":"+if(summary_replication=1,"true","false")+",\"enabled\":true,\"replicationFactor\":"+tostring(replication_factor)+",\"siteReplicationFactor\":"+coalesce(replace(replace(site_replication_factor, "origin", "\"origin\""), "total", "\"total\""), "null")+",\"siteSearchFactor\":"+coalesce(replace(replace(site_search_factor, "origin", "\"origin\""), "total", "\"total\""),"null")+",\"searchFactor\":"+tostring(search_factor)+"}","{\"host\":\""+splunk_server+"\",\"timezone\":\""+strftime(now(),"%z")+"\",\"enabled\":false}") | eval _time=now() | eval date=strftime(_time, "%Y-%m-%d") | fields _time date data

[instrumentation.deployment.forwarders]
search = index=_internal source=*metrics.log* TERM(group=tcpin_connections) (TERM(connectionType=cooked) OR TERM(connectionType=cookedSSL)) fwdType=* guid=* | rename sourceIp as forwarderHost | eval connectionType=case(fwdType=="uf" or fwdType=="lwf" or fwdType=="full", fwdType, 1==1,"Splunk fwder") | eval version=if(isnull(version),"pre 4.2",version) |  bin _time span=1d | stats sum(kb) as kb, latest(connectionType) as connectionType, latest(arch) as arch, latest(os) as os, latest(version) as version, latest(protocolLevel) as protocolLevel, latest(forwarderHost) as forwarderHost by guid _time| eval protocolLevel=if(isnull(protocolLevel), "no info", tostring(protocolLevel)) | stats estdc(forwarderHost) as numHosts estdc(guid) as numInstances `instrumentation_distribution_values(kb)` by connectionType arch os version _time protocolLevel| eval data="{\"hosts\":"+tostring(numHosts)+",\"instances\":"+tostring(numInstances)+",\"architecture\":\""+arch+"\",\"os\":\""+os+"\",\"splunkVersion\":\""+version+"\",\"type\":\""+connectionType+"\", \"protocolLevel\":\""+protocolLevel+"\", \"bytes\":{" + `instrumentation_distribution_strings("kb",1024,0)` + "}}" | eval date=strftime(_time, "%Y-%m-%d") | fields _time date data

[instrumentation.deployment.app]
search = | rest /services/apps/local | eval _time=now() | fields splunk_server title updated version disabled | eval data="{\"host\":\""+splunk_server+"\",\"name\":\""+title+"\",\"version\":\""+coalesce(version, "")+"\",\"enabled\":"+if(disabled=0, "true", "false")+"}" | eval date=strftime(_time, "%Y-%m-%d") | fields data _time date

[instrumentation.deployment.node]
search = index=_introspection sourcetype=splunk_disk_objects component::Partitions \
| bin _time span=1d \
| stats latest(data.free) as partitionFree, latest(data.capacity) as partitionCapacity by host data.fs_type data.mount_point _time \
| eval partitionUtilized=round(1-partitionFree/partitionCapacity,2) \
| eval partitions="{\"utilization\":"+`instrumentation_number_format(partitionUtilized,1,2)`+",\"capacity\":"+`instrumentation_number_format(partitionCapacity,1048576,0)`+",\"fileSystem\":\""+'data.fs_type' + "\"}" \
| stats delim="," values(partitions) as partitions by host _time \
| rename _time as date \
| mvcombine partitions \
| rename date as _time \
| join type=left host _time [ \
    search index=_introspection sourcetype=splunk_resource_usage component::Hostwide \
    | eval cpuUsage = 'data.cpu_system_pct' + 'data.cpu_user_pct' \
    | rename data.mem_used as memUsage \
    | bin _time span=1d \
    | stats latest(data.cpu_count) as coreCount, \
            latest(data.virtual_cpu_count) as virtualCoreCount, \
            latest(data.mem) as memAvailable, \
            latest(data.splunk_version) as splunkVersion, \
            latest(data.cpu_arch) as cpuArch, \
            latest(data.os_name) as osName, \
            latest(data.os_name_ext) as osNameExt, \
            latest(data.os_version) as osVersion, \
            `instrumentation_distribution_values(cpuUsage)`, \
            `instrumentation_distribution_values(memUsage)`, \
            latest(data.instance_guid) as guid \
        by host _time \
    ] \
| fillnull value="null" coreCount virtualCoreCount memAvailable \
| eval splunkVersion=coalesce("\""+splunkVersion+"\"", "null"), \
       cpuArch=coalesce("\""+cpuArch+"\"", "null"), \
       osName=coalesce("\""+osName + "\"", "null"), \
       osNameExt=coalesce("\""+osNameExt+"\"", "null"), \
       osVersion=coalesce("\""+osVersion+"\"", "null"), \
       guid=coalesce("\""+guid+"\"", "null") \
| eval data = "{\"guid\":"+guid+",\"host\":\""+replace(host,"\"", "\\\"")+"\",\"partitions\": " + coalesce("[" + partitions + "]", "null") + ",\"cpu\":{\"architecture\":"+cpuArch+",\"coreCount\":" + tostring(coreCount)+ ",\"virtualCoreCount\":"+tostring(virtualCoreCount)+",\"utilization\":{" + `instrumentation_distribution_strings("cpuUsage",.01,2)` + "}},\"memory\":"+"{\"capacity\":"+ `instrumentation_number_format(memAvailable,1048576,0)`+",\"utilization\":{" + `instrumentation_distribution_strings("memUsage",1/memAvailable,2)` + "}},\"os\":"+osName+",\"osExt\":"+osNameExt + ",\"osVersion\":"+osVersion+",\"splunkVersion\":"+splunkVersion+"}" \
| eval date=strftime(_time, "%Y-%m-%d") \
| fields _time date data

[instrumentation.deployment.index]
search = | rest /services/data/indexes | join type=outer splunk_server title [| rest /services/data/indexes-extended] \
| append [| rest /services/data/indexes datatype=metric | join type=outer splunk_server title [| rest /services/data/indexes-extended datatype=metric]] \
| eval warm_bucket_size = if(isnotnull('bucket_dirs.home.warm_bucket_size'), 'bucket_dirs.home.warm_bucket_size', 'bucket_dirs.home.size') \
| eval cold_bucket_size_gb = tostring(round(coalesce('bucket_dirs.cold.bucket_size', 'bucket_dirs.cold.size', 0) / 1024, 2)) \
| eval warm_bucket_size_gb = tostring(round(coalesce(warm_bucket_size,0) / 1024, 2)) \
| eval hot_bucket_size = tostring(round(coalesce(total_size / 1024 - cold_bucket_size_gb - warm_bucket_size_gb, 0),2)) \
| eval hot_bucket_size_gb = tostring(round(coalesce(hot_bucket_size,0) / 1024, 2)) \
| eval thawed_bucket_size_gb = tostring(round(coalesce('bucket_dirs.thawed.bucket_size', 'bucket_dirs.thawed.size',0) / 1024, 2)) \
| eval warm_bucket_count = tostring(coalesce('bucket_dirs.home.warm_bucket_count', 0)) \
| eval hot_bucket_count = tostring(coalesce('bucket_dirs.home.hot_bucket_count',0)) \
| eval cold_bucket_count = tostring(coalesce('bucket_dirs.cold.bucket_count',0)) \
| eval thawed_bucket_count = tostring(coalesce('bucket_dirs.thawed.bucket_count',0)) \
| eval home_event_count = tostring(coalesce('bucket_dirs.home.event_count',0)) \
| eval cold_event_count = tostring(coalesce('bucket_dirs.cold.event_count',0)) \
| eval thawed_event_count = tostring(coalesce('bucket_dirs.thawed.event_count',0)) \
| eval home_bucket_capacity_gb = coalesce(if('homePath.maxDataSizeMB' == 0, "\"unlimited\"", round('homePath.maxDataSizeMB' / 1024, 2)), "\"unlimited\"") \
| eval cold_bucket_capacity_gb = coalesce(if('coldPath.maxDataSizeMB' == 0, "\"unlimited\"", round('coldPath.maxDataSizeMB' / 1024, 2)), "\"unlimited\"") \
| eval currentDBSizeGB = tostring(round(coalesce(currentDBSizeMB,0) / 1024, 2)) \
| eval maxTotalDataSizeGB = tostring(if(maxTotalDataSizeMB = 0, "\"unlimited\"", coalesce(round(maxTotalDataSizeMB / 1024, 2), "null"))) \
| eval maxHotBuckets= if(maxHotBuckets="auto","\"auto\"", maxHotBuckets) \
| eval minTime = tostring(coalesce(strptime(minTime,"%Y-%m-%dT%H:%M:%S%z"),"null")) \
| eval maxTime = tostring(coalesce(strptime(maxTime,"%Y-%m-%dT%H:%M:%S%z"),"null")) \
| eval total_bucket_count = tostring(if(isnotnull(total_bucket_count), total_bucket_count, 0)) \
| eval totalEventCount = tostring(coalesce(totalEventCount, 0)) \
| eval total_raw_size_gb = tostring(coalesce(round(total_raw_size / 1024, 2), "null")) \
| eval timeResolution = IF('metric.timestampResolution'=="ms","millisec","sec") \
| eval index_type = coalesce(datatype ,"event") \
| rename eai:acl.app as App \
| eval _time=now() \
| fields splunk_server, title,index_type, timeResolution,\
currentDBSizeGB, totalEventCount, total_bucket_count, \
total_raw_size_gb, minTime, maxTime, home_bucket_capacity_gb, cold_bucket_capacity_gb, \
hot_bucket_size_gb, warm_bucket_size_gb, cold_bucket_size_gb, thawed_bucket_size_gb, \
hot_bucket_count, warm_bucket_count, cold_bucket_count, thawed_bucket_count, \
home_event_count, cold_event_count, thawed_event_count, \
maxTotalDataSizeGB, maxHotBuckets, maxWarmDBCount App _time \
| eval data="{\"host\":\""+splunk_server+"\",\"name\":\""+title+"\",\"type\":\""+index_type+"\",\"timeResolution\":\""+timeResolution+"\",\"app\":\""+App+"\",\"total\":{\"currentDBSizeGB\":"+currentDBSizeGB+",\"maxDataSizeGB\":"+maxTotalDataSizeGB+",\"events\":"+totalEventCount+",\"buckets\":"+total_bucket_count+",\"rawSizeGB\":"+total_raw_size_gb+",\"minTime\":"+minTime+",\"maxTime\":"+maxTime+"},\"buckets\":{\"homeCapacityGB\":"+home_bucket_capacity_gb+",\"homeEventCount\":"+home_event_count+",\"coldCapacityGB\":"+cold_bucket_capacity_gb+",\"hot\":{\"sizeGB\":"+hot_bucket_size_gb+",\"count\":"+hot_bucket_count+",\"max\":"+maxHotBuckets+"},\"warm\":{\"sizeGB\":"+warm_bucket_size_gb+",\"count\":"+warm_bucket_count+"},\"cold\":{\"sizeGB\":"+cold_bucket_size_gb+",\"count\":"+cold_bucket_count+",\"events\":"+cold_event_count+"},\"thawed\":{\"sizeGB\":"+thawed_bucket_size_gb+",\"count\":"+thawed_bucket_count+",\"events\":"+thawed_event_count+"}}}" \
| eval date=strftime(_time, "%Y-%m-%d") | fields data _time date

[instrumentation.deployment.dscluster]
search = index=_dsphonehome earliest=-24h@h | stats dc(data.clientId) as total_clients \
| appendcols [ | search index=_dsphonehome earliest=-24h@h | rex field=source ".*phonehomes_(?<ds_guid>.*).log" | stats  dc(data.clientId) as clients_connected by ds_guid | makejson ds_guid, clients_connected output=count_by_ds | fields count_by_ds | mvcombine delim="," count_by_ds | nomv count_by_ds | eval count_by_ds="["+count_by_ds+"]" ] \
| appendcols [ | search index=_dsphonehome earliest=-24h@h | join data.clientId max=1 [search index=_dsclient] | stats dc(data.clientId) as version_package_count by data.splunkVersion data.package | rename data.splunkVersion as version | rename data.package as package| makejson version package version_package_count output=count_by_version_and_package | fields count_by_version_and_package | mvcombine delim="," count_by_version_and_package | nomv count_by_version_and_package | eval count_by_version_and_package="["+count_by_version_and_package+"]" ] \
| fields total_clients count_by_ds count_by_version_and_package \
| eval data="{\"total_clients\":"+tostring(total_clients)+",\"count_by_ds\":"+count_by_ds+",\"count_by_version_and_package\":"+count_by_version_and_package+"}" | eval _time=now() | eval date=strftime(_time, "%Y-%m-%d") | fields data _time date

[instrumentation.deployment.dsclients]
search = index="_internal" uri="/services/broker/phonehome/connection*" sourcetype=splunkd earliest=-24h | table host | dedup host | rename host AS hostname | join inner hostname [search index="_internal" sourcetype="splunkd" fwdtype version | fields hostname, version, fwdType, os] | table hostname version fwdType os | rename fwdType as agentType | stats count AS agentsCount by version, agentType, os \
| makejson version, agentType, os, agentsCount output=countByAgentTypes | fields countByAgentTypes | mvcombine delim="," countByAgentTypes | nomv countByAgentTypes | eval countByAgentTypes="["+countByAgentTypes+"]" \
| eval data="{\"countByAgentTypes\":"+tostring(countByAgentTypes)+"}" | eval _time=now() | eval date=strftime(_time, "%Y-%m-%d") | fields data _time date

[instrumentation.deployment.dsservers]
search = index="_internal" uri="/services/broker/phonehome/connection*" sourcetype="splunkd_access" earliest=-24h | stats distinct_count(uri) as agentsConnected by host | join inner host [search index="_internal" source="*metrics.log" instance_guid | table host instance_guid instance_roles ] | rename instance_guid AS dsGuid instance_roles as instanceRoles \
| makejson dsGuid, instanceRoles, agentsConnected output=countByManager | fields countByManager | mvcombine delim="," countByManager | nomv countByManager | eval countByManager="["+countByManager+"]" \
| fields countByManager | eval data="{\"countByManager\":"+countByManager+"}" | eval _time=now() | eval date=strftime(_time, "%Y-%m-%d") | fields data _time date

[instrumentation.deployment.dseffectiveconfig]
search = index=_internal source="*splunkd_access.log" earliest=-24h \
| appendpipe [ \
 search method=POST "uri_path"="/services/agent-management/v1/agents/effective-configuration" status=201 \
 | stats distinct_count(uri_query) as count BY useragent \
 | rename useragent AS ta_info \
 | eval resultType="uploads" \
 | fillnull value="[]" ta_info count \
] \
| appendpipe [ \
 search status=200 method=GET \
 | rex field=uri_path "/services/agent-management/v1/agents/(?<ec_id>[A-Za-z0-9+/=]+)/effective-configuration/(?<confType>.+)" \
 | stats count BY confType \
 | eval resultType="requests" \
 | fillnull value="[]" confType count \
] \
| eval uploads_data=if(resultType=="uploads", "{\"ta_info\":\"".ta_info."\",\"count\":".count."}", null()) \
| eval requests_data=if(resultType=="requests", "{\"confType\":\"".confType."\",\"count\":".count."}", null()) \
| stats list(uploads_data) AS uploads, list(requests_data) AS requests \
| eval uploads=if(isnull(uploads),"[]","["+mvjoin(uploads, ",")+"]") \
| eval requests=if(isnull(requests),"[]","["+mvjoin(requests, ",")+"]") \
| eval data="{\"uploads\":"+uploads+",\"requests\":"+requests+"}" \
| fields data

[instrumentation.deployment.index.config]
search = | rest /services/configs/conf-indexes | eval _time=now() | fields splunk_server title tsidxWritingLevel | eval data="{\"host\":\""+splunk_server+"\",\"index\":\""+title+"\",\"tsidxWritingLevel\":"+tsidxWritingLevel+"}" | eval date=strftime(_time, "%Y-%m-%d") | fields data _time date

[instrumentation.deployment.remoteupgrade]
search = index=_internal TERM(group=deploy-server) TERM(name=upgrade_status) source=*metrics.log* earliest=-24h@h\
|stats sum(nRemoteUpgrade) AS total_nRemoteUpgrade, sum(nRemoteUpgradeSuccess) AS total_nRemoteUpgradeSuccess, sum(nRemoteUpgradeFailure) AS total_nRemoteUpgradeFailure, sum(nRemoteUpgradePkg*) AS total_nRemoteUpgradePkg* sum(nRemoteUpgradeOS*) AS total_nRemoteUpgradeOS* \
| fields total_nRemoteUpgrade, total_nRemoteUpgradeSuccess, total_nRemoteUpgradeFailure, total_nRemoteUpgradePkg*, total_nRemoteUpgradeOS* \
| eval totalnRemoteUpgradePkgStr="", total_nRemoteUpgradeOsStr="" \
| foreach total_nRemoteUpgradePkg* \
     [eval totalnRemoteUpgradePkgStr = totalnRemoteUpgradePkgStr.",\"total_remote_upgrade_pkg"."<<MATCHSTR>>"."\":".tostring('<<FIELD>>')] \
| foreach total_nRemoteUpgradeOS* \
     [eval total_nRemoteUpgradeOsStr = total_nRemoteUpgradeOsStr.",\"total_remote_upgrade_os"."<<MATCHSTR>>"."\":".tostring('<<FIELD>>')] \
| fields total_nRemoteUpgrade, total_nRemoteUpgradeSuccess, total_nRemoteUpgradeFailure, totalnRemoteUpgradePkgStr, total_nRemoteUpgradeOsStr \
| eval data="{\"total_remote_upgrade\":"+tostring(total_nRemoteUpgrade)+",\"total_remote_ugprade_success\":"+tostring(total_nRemoteUpgradeSuccess)+",\"total_remote_ugprade_failure\":"+tostring(total_nRemoteUpgradeFailure)+totalnRemoteUpgradePkgStr+total_nRemoteUpgradeOsStr+"}" \
| eval _time=now() \
| eval date=strftime(_time, "%Y-%m-%d") \
| fields data _time date

# Licensing
[instrumentation.licenseUsage]
# Why start with append? Otherwise, when running this saved search by itself, the results of the
# stats command are not reflected in the events. Instead, the events tab will only show the events
# as they existed in the pipeline before stats.
search = NOT() | append [search index=_telemetry type=RolloverSummary | eval date=strftime(_time-43200, "%Y-%m-%d") | eval licenseIDs=coalesce(replace(replace(replace(replace(licenseGuids,"\[","[\""),"\]","\"]"),",","\",\"")," ", ""),"null"), subgroup_id=coalesce(subgroupId, "Production"), group_id=coalesce("\""+licenseGroup+"\"", "null"), lmGuid=coalesce("\""+guid+"\"", "null"), productType=coalesce("\""+productType+"\"", "null"), type_id=if(substr(stack,1,16)="fixed-sourcetype", "fixed-sourcetype",stack) | stats max(_time) as lastTime latest(stacksz) as stack_quota, latest(poolsz) as pool_quota, sum(b) as consumption by pool stack host lmGuid licenseIDs type_id group_id subgroup_id productType date | rename stack as stack_id | eval pool="{\"quota\":" + pool_quota+",\"consumption\":"+consumption+"}" | stats delim="," values(pool) as pools, max(lastTime) as lastTime max(stack_quota) as stack_quota sum(consumption) as stack_consumption by stack_id group_id subgroup_id type_id lmGuid host licenseIDs productType date | mvcombine pools | eval _raw="{\"component\":\"licensing.stack\",\"data\":{\"host\":\""+host+"\",\"guid\":"+lmGuid+",\"name\":\""+replace(stack_id,"\"", "\\\"")+"\",\"type\":\"" + type_id + "\",\"subgroup\":\"" + subgroup_id + "\",\"product\":"+productType+",\"quota\":" + stack_quota+",\"consumption\":"+stack_consumption+",\"pools\":["+pools+"],\"licenseIDs\":"+licenseIDs+"}, \"date\":\""+date+"\",\"visibility\":\"anonymous,license\"}", _time=lastTime]

[instrumentation.licensing.stack]
search = index=_telemetry source=*license_usage_summary.log* sourcetype=splunkd TERM(type=RolloverSummary) | eval date=strftime(_time, "%m-%d-%Y"), licenseIDs=coalesce(replace(replace(replace(replace(licenseGuids,"\[","[\""),"\]","\"]"),",","\",\"")," ", ""),"null"), subgroup_id=coalesce(subgroupId, "Production"), group_id=coalesce("\""+licenseGroup+"\"", "null"), lmGuid=coalesce("\""+guid+"\"", "null"), productType=coalesce("\""+productType+"\"", "null"), type_id=if(substr(stack,1,16)="fixed-sourcetype", "fixed-sourcetype",stack) | stats latest(stacksz) as stack_quota, latest(poolsz) as pool_quota, sum(b) as consumption by pool stack host lmGuid licenseIDs type_id group_id subgroup_id productType date | rename stack as stack_id | eval pool="{\"quota\":" + pool_quota+",\"consumption\":"+consumption+"}" | stats delim="," values(pool) as pools, max(stack_quota) as stack_quota sum(consumption) as stack_consumption by stack_id group_id subgroup_id type_id lmGuid host licenseIDs productType date | mvcombine pools | eval data="{\"host\":\""+host+"\",\"guid\":"+lmGuid+",\"name\":\""+replace(stack_id,"\"", "\\\"")+"\",\"type\":\"" + type_id + "\",\"subgroup\":\"" + subgroup_id + "\",\"product\":"+productType+",\"quota\":" + stack_quota+",\"consumption\":"+stack_consumption+",\"pools\":["+pools+"],\"licenseIDs\":"+licenseIDs+"}" | eval _time=strptime(date, "%m-%d-%Y")-43200 | fields data _time

[instrumentation.licensing.lmredundancy.errors]
search = index=_internal sourcetype=splunkd component=LMStackMgr error=LMRedundancy | table host type text _time | eval date=strftime(_time, "%Y-%m-%d") | dedup text date | eval data="{\"host\":\""+host+"\",\"errorType\":\""+type+"\"}" | fields _time date data

# Performance
[instrumentation.performance.indexing]
search = index=_internal TERM(group=thruput) TERM(name=index_thruput) source=*metrics.log* | bin _time span=30s | stats sum(kb) as kb sum(instantaneous_kbps) as instantaneous_kbps by host _time | bin _time span=1d | stats sum(kb) as totalKB `instrumentation_distribution_values(instantaneous_kbps)` by host _time | eval data="{\"host\":\""+host+"\",\"thruput\":{\"total\":" + tostring(round(totalKB*1024)) + "," + `instrumentation_distribution_strings("instantaneous_kbps",1024,0)`+"}}" | eval date=strftime(_time, "%Y-%m-%d") | fields _time date data

[instrumentation.performance.search]
search = index=_audit sourcetype=audittrail TERM(action=search) TERM(info=completed) total_run_time=* | eval search_et=if(search_et="N/A", 0, search_et) | eval search_lt=if(search_lt="N/A", exec_time, min(exec_time,search_lt)) | eval timerange=search_lt-search_et | bin _time span=1d | stats latest(searched_buckets) as searched_buckets latest(total_slices) as total_slices latest(scan_count) as scan_count latest(timerange) as timerange latest(total_run_time) as runtime by search_id _time | stats `instrumentation_distribution_values(runtime)`, `instrumentation_distribution_values(searched_buckets)`, `instrumentation_distribution_values(total_slices)`, `instrumentation_distribution_values(scan_count)`, `instrumentation_distribution_values(timerange)` count as numSearches by _time | eval data="{\"searches\":"+tostring(numSearches)+",\"latency\":{"+`instrumentation_distribution_strings("runtime",1,2)`+"},\"buckets\":{"+`instrumentation_distribution_strings("searched_buckets",1,2)`+"},\"slices\":{"+`instrumentation_distribution_strings("total_slices",1,2)`+"},\"scanCount\":{"+`instrumentation_distribution_strings("scan_count",1,2)`+"},\"dayRange\":{"+`instrumentation_distribution_strings("timerange",1/86400,2)`+"}}" | eval date=strftime(_time, "%Y-%m-%d") | fields _time date data



# Templates
[instrumentation.anonymous.firstEvent]
search = (index=_introspection OR index=_telemetry) sourcetype=splunk_telemetry source="http-stream" visibility=*anonymous* | append [savedsearch instrumentation.licenseUsage] | where date >= "$beginDate$" AND date <= "$endDate$" | head 1

[instrumentation.support.firstEvent]
search = (index=_introspection OR index=_telemetry) sourcetype=splunk_telemetry source="http-stream" visibility=*support* | append [savedsearch instrumentation.licenseUsage] | where date >= "$beginDate$" AND date <= "$endDate$" | head 1

[instrumentation.license.firstEvent]
search = | savedsearch instrumentation.licenseUsage | where date >= "$beginDate$" AND date <= "$endDate$" | head 1

[instrumentation.reporting]
search = index=_telemetry source=telemetry sourcetype=splunk_telemetry_log | fields _raw | spath | eval time_formatted = strftime(_time, "%Y-%m-%d %H:%M:%S") | search (status=success OR status=failed)

[instrumentation.reporting.errors]
search = index=_telemetry source=telemetry sourcetype=splunk_telemetry_log status=failed visibility=*$visibility$*



# Usage
[instrumentation.usage.app.page]
search = index=_internal sourcetype=splunk_web_access uri_path="/*/app/*/*" NOT uri_path="/*/static/*" | eval uri_parts=split(uri_path, "/"),locale=mvindex(uri_parts,1), app=mvindex(uri_parts,3), page=mvindex(uri_parts,4) | bin _time span=1d | eventstats estdc(user) as appUsers count as appOccurrences by app _time | bin _time span=1d | stats latest(locale) as locale count as occurrences estdc(user) as users by app page appUsers appOccurrences _time | sort app -occurrences | streamstats count as pageRank by app _time | where pageRank<=10 | eval data="{\"app\":\""+app+"\",\"page\":\""+page+"\",\"locale\":\""+locale+"\",\"occurrences\":" + tostring(occurrences) + ",\"users\":" + tostring(users) + "}" | eval data=if(pageRank==1,data+";{\"app\":\""+app+"\",\"locale\":\""+locale+"\",\"occurrences\":" + tostring(appOccurrences) + ",\"users\":" + tostring(appUsers) + "}", data) | stats values(data) as data by app appOccurrences appUsers _time | sort _time -appOccurrences | streamstats count as appRank by _time | where appRank<=25 | mvexpand data | makemv delim=";" data | mvexpand data | eval date=strftime(_time, "%Y-%m-%d") | fields _time date data

[instrumentation.usage.indexing.sourcetype]
search = index=_internal source=*metrics.log* TERM(group=per_sourcetype_thruput) | bin _time span=1d | stats sum(ev) as events, sum(kb) as size, estdc(host) as hosts by series _time | eval data="{\"name\":\""+replace(series,"\"", "\\\"") + "\",\"events\":"+tostring(events)+",\"bytes\":"+tostring(round(size*1024))+",\"hosts\":"+tostring(hosts)+"}" | eval date=strftime(_time, "%Y-%m-%d") | fields _time date data

[instrumentation.usage.search.concurrent]
search = index=_introspection sourcetype=splunk_resource_usage component::PerProcess data.search_props.sid::* | bin _time span=10s | stats estdc(data.search_props.sid) AS concurrent_searches by _time host | bin _time span=1d | stats `instrumentation_distribution_values(concurrent_searches)` by host _time | eval data="{\"host\":\""+host+"\",\"searches\":{" + `instrumentation_distribution_strings("concurrent_searches",1,0)` +"}}" | eval date=strftime(_time, "%Y-%m-%d") | fields _time date data

[instrumentation.usage.search.type]
search = index=_introspection sourcetype=splunk_resource_usage component::PerProcess data.search_props.sid::* | rename data.search_props.type as searchType | bin _time span=1d | stats estdc(data.search_props.sid) AS search_count by searchType _time | eval data="\""+searchType+"\":"+tostring(search_count) | stats delim="," values(data) as data by _time | rename _time as date | mvcombine data | eval data="{"+data+"}" | rename date as _time | eval date=strftime(_time, "%Y-%m-%d") | fields _time date data

[instrumentation.usage.users.active]
search = index=_audit sourcetype=audittrail TERM(action=search) user!="splunk-system-user" user!="n/a" | bin _time span=1d | stats estdc(user) as active by _time | eval data="{\"active\":"+tostring(active)+"}" | eval date=strftime(_time, "%Y-%m-%d") | fields _time date data

[instrumentation.usage.kvstore]
search = | rest splunk_server=local /services/kvstore/info | appendcols [ | rest splunk_server=local /services/server/introspection/kvstore/collectionstats | mvexpand data | spath input=data | rex field=ns "(?<App>.*)\.(?<Collection>.*)" | eval dbsize=round(size/1024/1024, 2) | stats sum(dbsize) as data_size count(ns) as numOfCollections | eval collData = "\"usage.dataSizeMB\": \"" + data_size + "\", \"usage.numOfCollections\": \"" + numOfCollections + "\""] | eval data = "{" | foreach usage.* [eval data = data + "\"<<FIELD>>\":\"" + '<<FIELD>>' + "\", " ] | eval data = data + collData | eval data = rtrim(data, ", ") + "}", _time = now(), date=strftime(_time, "%Y-%m-%d") | fields data _time date

# kvstore backup/restore
[instrumentation.usage.kvstorebackuprestore]
search = index=_internal sourcetype=splunkd source=*splunkd.log* component=KVStoreBackupRestore (started AND archiveName AND method) | eval usageMethod = if(method==2, "pointInTime", "nonPointInTime") | stats count by usageMethod, action | makejson usageMethod, action, count output=data | eval _time=now(), date=strftime(_time, "%Y-%m-%d") | fields _time date data

#Topology
[instrumentation.topology.deployment.clustering.member]
search = | localop | rest /services/cluster/master/peers | eval data="{\"master\":\""+splunk_server+"\",\"member\":{\"host\":\""+label+"\",\"guid\":\""+title+"\",\"status\":\""+status+"\"},\"site\":\""+site+"\"}" | where isnotnull(data) | eval _time=now() | eval date=strftime(_time, "%Y-%m-%d") | fields _time date data

[instrumentation.topology.deployment.clustering.searchhead]
search = | localop | rest /services/cluster/master/searchheads | where splunk_server!=label | eval data="{\"master\":\""+splunk_server+"\",\"searchhead\":{\"host\":\""+label+"\",\"guid\":\""+title+"\",\"status\":\""+status+"\"},\"site\":\""+site+"\"}" | where isnotnull(data) | eval _time=now() | eval date=strftime(_time, "%Y-%m-%d") | fields _time date data

[instrumentation.topology.deployment.shclustering.member]
search = | localop | rest /services/shcluster/captain/members | eval data="{\"site\":\""+site+"\",\"captain\":\""+splunk_server+"\",\"member\":{\"host\":\""+label+"\",\"guid\":\""+title+"\",\"status\":\""+status+"\"}}" | where isnotnull(data) | eval _time=now() | eval date=strftime(_time, "%Y-%m-%d") | fields _time date data

[instrumentation.topology.deployment.distsearch.peer]
search = | localop | rest /services/search/distributed/peers | eval data="{\"host\":\""+splunk_server+"\",\"peer\":{\"host\":\""+peerName+"\",\"guid\":\""+guid+"\",\"status\":\""+status+"\"}}" | where isnotnull(data) | eval _time=now() | eval date=strftime(_time, "%Y-%m-%d") | fields _time date data

[instrumentation.topology.deployment.licensing.slave]
search = | localop | rest /services/licenser/slaves | eval data="{\"master\":\""+splunk_server+"\",\"slave\":{\"host\":\""+label+"\",\"guid\":\""+title+"\",\"pool\":\""+active_pool_ids+"\"}}" | where isnotnull(data) | eval _time=now() | eval date=strftime(_time, "%Y-%m-%d") | fields _time date data

[instrumentation.topology.deployment.licensing.manager]
search = | localop | rest /services/licenser/managers | eval data="{\"host\":\""+lm_manager_label+"\",\"guid\":\""+title+"\"}" | where isnotnull(data) | eval _time=now() | eval date=strftime(_time, "%Y-%m-%d") | fields _time date data

#Workload management
[instrumentation.usage.workloadManagement.enabled]
search = NOT() | append [rest splunk_server=local /services/workloads/status/workload-management-status | eval support='general.isSupported', enabled='general.enabled', os_name='general.os_name', os_version='general.os_version'| fields support, enabled, os_name, os_version]

[instrumentation.usage.workloadManagement.category]
search = NOT() | append [rest splunk_server=local /services/workloads/categories | eval data="\""+title+"\":{\"allocated cpu percent\":\""+cpu_allocated_percent+"\", \"allocated mem limit\":\""+mem_allocated_percent+"\"}" | stats list(data) AS categoryList | eval categoryCombined=mvjoin(categoryList, ", ") | fields categoryCombined]

[instrumentation.usage.workloadManagement.pools]
search = NOT() | append [rest splunk_server=local /services/workloads/pools | eval isDeafultPool=if(default_category_pool=1, "True", "False"), poolList="\""+title+"\":{\"allocated cpu percent\":\""+cpu_allocated_percent+"\", \"allocated mem limit\":\""+mem_allocated_percent+"\", \"default category pool\":\""+isDeafultPool+"\"}" | stats list(poolList) AS poolList, count BY category | eval poolList="\""+category+"\":{\"count\":"+count+","+mvjoin(poolList, ", ")+"}" | stats sum(count) AS poolTotal list(poolList) AS poolList| eval poolCombined=mvjoin(poolList, ", ") | fields poolCombined, poolTotal]

[instrumentation.usage.workloadManagement.rules]
search = NOT() | append [rest splunk_server=local /services/workloads/rules | eval data="\""+title+"\":{\"order\":\""+order+"\", \"predicate\":\""+predicate+"\", \"workload pool\":\""+workload_pool+"\"}" | stats list(data) AS ruleList, count AS ruleTotal by splunk_server | eval ruleCombined=mvjoin(ruleList, ", ") | fields ruleTotal, ruleCombined]

[instrumentation.usage.workloadManagement.rulesTriggered]
search = NOT() | append [search index=_internal source=*/var/log/splunk/metrics.log* group=workload_management name=workload-metrics | stats sum(admission_rules_triggered) as total_admission_rules_triggered sum(monitoring_rules_triggered) as total_monitoring_rules_triggered | fields total_admission_rules_triggered, total_monitoring_rules_triggered]

[instrumentation.usage.workloadManagement.report]
action.outputtelemetry = 1
action.outputtelemetry.param.anonymous     = 1
action.outputtelemetry.param.support       = 1
action.outputtelemetry.param.license       = 0
action.outputtelemetry.param.optinrequired = 3
action.outputtelemetry.param.component = usage.workloadManagement.report
action.outputtelemetry.param.input = data
action.outputtelemetry.param.type = aggregate
alert.suppress = 0
alert.track = 0
counttype = number of events
cron_schedule = 0 3 * * 1
dispatch.earliest_time = -1w
dispatch.latest_time = now
display.general.type = statistics
display.page.search.tab = statistics
enableSched = 1
quantity = 0
relation = greater than
search = |rest splunk_server=local /services/server/info | appendcols [|rest splunk_server=local /servicesNS/nobody/splunk_instrumentation/telemetry | fields telemetrySalt]| eval telemetrySalt=if(isnull(telemetrySalt), "", telemetrySalt), hashHost=sha1(telemetrySalt+splunk_server), roleCombine=mvjoin(server_roles, ", ") | fields guid, hashHost, roleCombine| appendcols [|savedsearch instrumentation.usage.workloadManagement.enabled] | appendcols [|savedsearch instrumentation.usage.workloadManagement.category]| appendcols [|savedsearch instrumentation.usage.workloadManagement.pools] | appendcols [|savedsearch instrumentation.usage.workloadManagement.rules] | appendcols [|savedsearch instrumentation.usage.workloadManagement.rulesTriggered] | fillnull value=0 | eval total_admission_rules_triggered=if(isnull(total_admission_rules_triggered),0, total_admission_rules_triggered), total_monitoring_rules_triggered=if(isnull(total_monitoring_rules_triggered),0, total_monitoring_rules_triggered), data="{\"host\": \""+hashHost+"\", \"guid\": \""+guid+"\", \"wlm supported\": \""+support+"\", \"os\": \""+os_name+"\", \"osVersion\": \""+os_version+"\", \"wlm enabled\": \""+enabled+"\", \"server roles\": \""+roleCombine+"\", \"totalAdmissionRulesTriggered\": \""+total_admission_rules_triggered+"\", \"totalMonitoringRulesTriggered\": \""+total_monitoring_rules_triggered+"\"", poolTotal=if(isnull(poolTotal),0, poolTotal), ruleTotal=if(isnull(ruleTotal),0, ruleTotal) | eval data=if(support==1, data+", \"categories\":{"+categoryCombined+"}, \"pools\":{\"total count\":\""+poolTotal+"\""+ if(poolTotal>0, ", "+poolCombined+"", "") + "}, \"rules\":{\"total count\":\""+ruleTotal+"\""+if(ruleTotal>0, ", "+ruleCombined, "")+"}}", data+"}"), _time=now(), date=strftime(_time, "%Y-%m-%d")| fields _time date data

#Admission rules
[instrumentation.usage.admissionRules.enabled]
search = NOT() | append [rest splunk_server=local /services/workloads/status/admission-control-status | fields enabled]

[instrumentation.usage.admissionRules.rules]
search = NOT() | append [rest splunk_server=local "/services/workloads/rules?workload_rule_type=search_filter" | `hash_admission_rule_predicate(index)` | `hash_admission_rule_predicate(app)` | `hash_admission_rule_predicate(user)` | `hash_admission_rule_predicate(role)` | eval data="\""+sha1(title)+"\":{\"predicate\":\""+predicate+"\"}" | stats list(data) AS ruleList, count AS ruleTotal | eval ruleCombined=mvjoin(ruleList, ", ") | fields ruleTotal, ruleCombined]

[instrumentation.usage.admissionRules.rulesTriggered]
search = index=_internal sourcetype=wlm_monitor prefilter_action=filter | stats count by prefilter_rule | fields prefilter_rule, count | eval data="{\"searchFilterRule\":\""+sha1(prefilter_rule)+"\", \"filteredSearchesCount\":\""+count+"\"}" | stats list(data) AS rulesTriggered, count AS rulesTriggeredTotal | eval rulesTriggeredCombined=mvjoin(rulesTriggered, ", ") | fields rulesTriggeredTotal, rulesTriggeredCombined

[instrumentation.usage.admissionRules.report]
action.outputtelemetry = 1
action.outputtelemetry.param.anonymous     = 1
action.outputtelemetry.param.support       = 1
action.outputtelemetry.param.license       = 0
action.outputtelemetry.param.optinrequired = 3
action.outputtelemetry.param.component = usage.admissionRules.report
action.outputtelemetry.param.input = data
action.outputtelemetry.param.type = aggregate
alert.suppress = 0
alert.track = 0
counttype = number of events
cron_schedule = 0 3 * * 1
dispatch.earliest_time = -1w
dispatch.latest_time = now
display.general.type = statistics
display.page.search.tab = statistics
enableSched = 1
quantity = 0
relation = greater than
search = |rest splunk_server=local /services/server/info | appendcols [|rest splunk_server=local /servicesNS/nobody/splunk_instrumentation/telemetry | fields telemetrySalt]| eval telemetrySalt=if(isnull(telemetrySalt), "", telemetrySalt), hashHost=sha1(telemetrySalt+splunk_server), roleCombine=mvjoin(server_roles, ", ") | fields guid, hashHost, roleCombine| appendcols [|savedsearch instrumentation.usage.admissionRules.enabled] | appendcols [|savedsearch instrumentation.usage.admissionRules.rules] | appendcols [|savedsearch instrumentation.usage.admissionRules.rulesTriggered] | fillnull value=0 | eval data="{\"host\": \""+hashHost+"\", \"guid\": \""+guid+"\", \"admissionRulesEnabled\": \""+enabled+"\", \"serverRoles\": \""+roleCombine+"\", \"rules\":{\"totalCount\":\""+ruleTotal+"\""+if(ruleTotal>0, ", "+ruleCombined, "")+"}, \"rulesTriggered\":["+if(rulesTriggeredTotal>0, rulesTriggeredCombined, "")+"]}", _time=now(), date=strftime(_time, "%Y-%m-%d")| fields _time date data

#Password policy management
[instrumentation.usage.passwordPolicy.config]
action.outputtelemetry = 1
action.outputtelemetry.param.anonymous     = 1
action.outputtelemetry.param.support       = 1
action.outputtelemetry.param.license       = 0
action.outputtelemetry.param.optinrequired = 3
action.outputtelemetry.param.component = usage.passwordPolicy.config
action.outputtelemetry.param.input = data
action.outputtelemetry.param.type = aggregate
alert.suppress = 0
alert.track = 0
counttype = number of events
cron_schedule = 0 3 * * 1
dispatch.earliest_time = -1w
dispatch.latest_time = now
display.general.type = statistics
display.page.search.tab = statistics
enableSched = 1
quantity = 0
relation = greater than
search = |rest splunk_server=local /services/admin/Splunk-auth/splunk_auth| join type=left splunk_server [|rest splunk_server=local /services/server/info | fields guid, splunk_server] | appendcols [|rest splunk_server=local /servicesNS/nobody/splunk_instrumentation/telemetry | fields telemetrySalt]| eval telemetrySalt=if(isnull(telemetrySalt), "", telemetrySalt), hashHost=sha1(telemetrySalt+splunk_server)| replace "1" with "true", "0" with "false" in enablePasswordHistory,expireUserAccounts, forceWeakPasswordChange, lockoutUsers, verboseLoginFailMsg | eval data="{\"host\": \""+hashHost+"\",\"guid\": \""+guid+"\", \"constant login time\":\""+constantLoginTime+"\", \"enable password history\":\""+enablePasswordHistory+"\", \"expiration alert in days\":\""+expireAlertDays+"\", \"days until password expires\":\""+expirePasswordDays+"\", \"enable password expiration\":\""+expireUserAccounts+"\", \"force existing users to change weak passwords\":\""+forceWeakPasswordChange+"\", \"failed login attempts\":\""+lockoutAttempts+"\", \"lockout duration in minutes\":\""+lockoutMins+"\", \"lockout threshold in minutes\":\""+lockoutThresholdMins+"\", \"enable lockout users\":\""+lockoutUsers+"\", \"minimum number of digits\":\""+minPasswordDigit+"\", \"minimum number of characters\":\""+minPasswordLength+"\", \"minimum number of lowercase letters\":\""+minPasswordLowercase+"\", \"minimum number of special characters\":\""+minPasswordSpecial+"\", \"minimum number of uppercase letters\":\""+minPasswordUppercase+"\", \"password history count\":\""+passwordHistoryCount+"\", \"enable verbose login fail message\":\""+verboseLoginFailMsg+"\"}",_time=now(), date=strftime(_time, "%Y-%m-%d") | fields data _time date

#Health monitoring
[instrumentation.usage.healthMonitor.report]
action.outputtelemetry = 1
action.outputtelemetry.param.anonymous     = 1
action.outputtelemetry.param.support       = 1
action.outputtelemetry.param.license       = 0
action.outputtelemetry.param.optinrequired = 3
action.outputtelemetry.param.component = usage.healthMonitor.report
action.outputtelemetry.param.input = data
action.outputtelemetry.param.type = aggregate
alert.suppress = 0
alert.track = 0
counttype = number of events
cron_schedule = 0 3 * * 1
dispatch.earliest_time = -1w
dispatch.latest_time = now
display.general.type = statistics
display.page.search.tab = statistics
enableSched = 1
quantity = 0
relation = greater than
search = |rest splunk_server=local /services/server/health-config | eval thresh="" | foreach indicator*red,indicator*yellow [eval thresh =if('<<FIELD>>'!="", thresh+"\"<<FIELD>>\":" + '<<FIELD>>' + ",", thresh)] | eval thresh=rtrim(thresh, ","), enabled=if(disabled=='' or disabled==0 or isnull(disabled), 1,0) | eval feature="\""+title+"\":{\"threshold\": {"+thresh+"}, \"enabled\": \""+enabled+"\"}", distinct=if(like(title, "feature%"), "feature", "alert") | eval disable=coalesce('alert.disabled', disabled), action=coalesce('alert.actions','action.to','action.url', 'action.integration_url_override') | eval action=if(action=="" or isnull(action), "empty", action) | eval alert="\""+title+"\": {\"disabled\": \""+disable+"\", \"action/ action.to/ action.url/ action.integration_url_override\": \""+action+"\"}" | stats list(alert) AS alertList, list(feature) AS feaList by distinct | eval alertCombined=mvjoin(alertList, ","), feaCombined=mvjoin(feaList, ",") | eval alertCombined="\"alert\":{"+alertCombined+"}" | eval feaCombined=if(distinct=="alert", null, feaCombined), alertCombined=if(distinct=="feature", null, alertCombined) | eval dataCombined=coalesce(alertCombined, feaCombined) | stats list(dataCombined) AS dataList| eval data=mvjoin(dataList, ",") | eval data="{"+data+"}",_time=now(), date=strftime(_time, "%Y-%m-%d") | fields data _time date

#Health Reporter Usage
[instrumentation.usage.healthMonitor.currentState]
search = | rest /services/apps/local splunk_server=local | search title=search | rename version as splunk_version | fields splunk_version \
 | appendcols [| rest services/server/health-config/distributed_health_reporter splunk_server=local | eval dist_hr_enabled=if(isnull(disabled) or disabled=='' or disabled==0, 1, 0) | fields dist_hr_enabled] \
 | appendcols [| search index=_internal earliest=-1d source=*splunkd_ui_access.log ("server/health/splunkd/details" OR "server/health/deployment/details") | stats count as click_count] \
 | appendcols [ \
    | rest services/server/health/splunkd/details splunk_server=local \
    | fields + features.* health | rename health as features.health \
    | fields - *.reasons.* *.messages.* \
    | foreach features.* [ eval newname="splunkd."+replace(lower("<<MATCHSTR>>"), " ", "_") | rex field=newname mode=sed "s/features\.|\.health//g" | eval {newname}='<<FIELD>>'] \
    | fields - features.*, newname \
    | transpose column_name="features" | rename "row 1" as current_color \
    | join type=outer features [ \
        | search index=_internal earliest=-1d source=*health.log component=PeriodicHealthReporter \
            | stats count as num,  values(color) as colors by node_path \
            | rename node_path as features \
            | eval colors = mvjoin(colors, ",") \
            | eval worst_color = if(match(colors, "red"), "red", (if(match(colors, "yellow"), "yellow", "green"))) \
            | fields features, num, colors, worst_color \
    ] \
    | sort by features \
    | eval nodes="", combin_column=1 \
    | foreach features* [eval nodes = "{\"nodePath\": \"" + '<<FIELD>>' + "\", \"color\": \"" + current_color + "\", \"worstColorInLast24Hours\": \"" + worst_color + "\"}"] \
    | stats count list(nodes) as node_list by combin_column \
    | eval node_status = mvjoin(node_list, ", ") \
 ] \
 | eval data = "{\"splunkVersion\": \"" + splunk_version + "\", \"distribuedHealthReporter\": {\"enabled\": " + dist_hr_enabled + "}, \"healthReportClicks\": " + click_count + ", \"nodeStatus\": [" + node_status + "]}", _time=now(), date=strftime(_time, "%Y-%m-%d") \
 | fields data _time date

#Authentication methods
[instrumentation.usage.authMethod.config]
action.outputtelemetry = 1
action.outputtelemetry.param.anonymous     = 1
action.outputtelemetry.param.support       = 1
action.outputtelemetry.param.license       = 0
action.outputtelemetry.param.optinrequired = 3
action.outputtelemetry.param.component = usage.authMethod.config
action.outputtelemetry.param.input = data
action.outputtelemetry.param.type = aggregate
alert.suppress = 0
alert.track = 0
counttype = number of events
cron_schedule = 0 3 * * 1
dispatch.earliest_time = -1w
dispatch.latest_time = now
display.general.type = statistics
display.page.search.tab = statistics
enableSched = 1
quantity = 0
relation = greater than
search = |rest splunk_server=local /services/admin/auth-services| join type=left splunk_server [|rest splunk_server=local /services/server/info | fields guid, splunk_server] | appendcols [|rest splunk_server=local /servicesNS/nobody/splunk_instrumentation/telemetry | fields telemetrySalt]| eval telemetrySalt=if(isnull(telemetrySalt), "", telemetrySalt), hashHost=sha1(telemetrySalt+splunk_server)| eval data="{\"host\": \""+hashHost+"\",\"guid\": \""+guid+"\", \"authentication method\": \""+active_authmodule+"\",\"mfa type\": " +"\"" + if(mfa_type=="", "none", mfa_type) +"\"}", _time=now(), date=strftime(_time, "%Y-%m-%d") | fields data _time date

#JsonWebToken metrics
[instrumentation.authentication.jwt]
search = | rest splunk_server=local /services/properties/authorize/tokens_auth/disabled | table value | rename value as disabled | appendcols [| search index=_audit action=create_token | stats count | rename count as created] | appendcols [| rest splunk_server=local /services/configs/conf-authentication |  eval test=if(isnull(scriptPath), 0, 1) | table test | stats sum | rename sum(test) as scriptedExtensionsEnabled] | appendcols [| search index = _internal source=*/splunkd.log jsonwebtoken validation failed | stats count | rename count as failures] | makejson output=data

#S2 configuration
[instrumentation.usage.smartStore.global]
search = |rest splunk_server=local /services/configs/conf-server | where title in ("cachemanager","diskUsage", "clustering") | eval data="\""+title+"\":",hotlist_recency_secs=if(isnull(hotlist_recency_secs), "none", hotlist_recency_secs), hotlist_bloom_filter_recency_hours=if(isnull(hotlist_bloom_filter_recency_hours), "none", hotlist_bloom_filter_recency_hours) | eval data=if(title="diskUsage", data+"{\"minFreeSpace\":\""+minFreeSpace+"\"}", data), data=if(title="cachemanager", data+"{\"eviction_padding\":\""+eviction_padding+"\",\"max_cache_size\":\""+max_cache_size+"\", \"hotlist_recency_secs\":\""+hotlist_recency_secs+"\", \"hotlist_bloom_filter_recency_hours\":\""+hotlist_bloom_filter_recency_hours+"\"}", data), data=if(title="clustering", data+"{\"mode\":\""+mode+"\""+if(mode="master", ",\"search_factor\":\""+search_factor+"\",\"multisite\":\""+multisite+"\",\"site_replication_factor\":\""+site_replication_factor+"\",\"site_search_factor\":\""+site_search_factor+"\"}", "}"), data) | stats list(data) AS dataList BY splunk_server | eval globalConfig="\"global config\":{" + mvjoin(dataList, ",") + "}" | fields globalConfig, splunk_server

[instrumentation.usage.smartStore.perIndex]
search = |rest splunk_server=local /services/configs/conf-indexes | appendcols [|rest splunk_server=local /servicesNS/nobody/splunk_instrumentation/telemetry | fields telemetrySalt]| eval title_dist=if(match(title, "^([^_].*?)\s*"),"external","internal"), s2Enabled=if(isnotnull(remotePath),"SmartStore enabled", "non-SmartStore enabled"),hotlist_recency_secs=if(isnull(hotlist_recency_secs), "none", hotlist_recency_secs), hotlist_bloom_filter_recency_hours=if(isnull(hotlist_bloom_filter_recency_hours), "none", hotlist_bloom_filter_recency_hours) | makejson frozenTimePeriodInSecs, hotlist_recency_secs, hotlist_bloom_filter_recency_hours, maxHotSpanSecs, maxGlobalDataSizeMB, output="indexConfig" | eval telemetrySalt=if(isnull(telemetrySalt), "", telemetrySalt), hashTitle=sha1(telemetrySalt+title), title_combine=title_dist+"_"+hashTitle, indexConfig="\""+title_combine+"\":" + indexConfig | stats list(hashTitle) AS titleList,list(indexConfig) AS indexList BY s2Enabled, splunk_server | eval indexConfig=mvjoin(indexList, ","), titleCombined="\""+s2Enabled+"\":\"" + mvjoin(titleList, ",") +"\"" | stats list(titleCombined) AS s2List, list(indexConfig) AS indexList BY splunk_server| eval s2Enabled="\"list of indexes\":{" + mvjoin(s2List, ",") + "}", indexConfig="\"per index config\":{" + mvjoin(indexList, ",") + "}"  | fields s2Enabled, indexConfig, splunk_server

[instrumentation.usage.smartStore.capacity]
search = |rest splunk_server=local /services/server/status/partitions-space | makejson available, capacity, free, fs_type, output="cap" | eval cap="\""+title+"\": "+cap+"" | stats list(cap) AS capList BY splunk_server | eval capCombined="\"total storage capacity\":{" + mvjoin(capList, ", ") + "}" | fields capCombined, splunk_server

[instrumentation.usage.smartStore.accountTypes]
search = |rest splunk_server=local /services/configs/conf-indexes | eval scheme=if(storageType == "remote", mvindex(split(path,":"), 0), "local") | eval scheme=case(scheme=="gs","gcp",scheme=="s3","aws",scheme!="local",scheme) | stats list(scheme) as accountTypes delim="," by splunk_server | nomv accountTypes | fillnull value="" accountTypes | eval accountTypes="\"accountTypes\":\"" + accountTypes + "\"" | fields accountTypes, splunk_server

[instrumentation.usage.smartStore.config]
action.outputtelemetry = 1
action.outputtelemetry.param.anonymous     = 1
action.outputtelemetry.param.support       = 1
action.outputtelemetry.param.license       = 0
action.outputtelemetry.param.optinrequired = 3
action.outputtelemetry.param.component = usage.smartStore.Config
action.outputtelemetry.param.input = data
action.outputtelemetry.param.type = aggregate
alert.suppress = 0
alert.track = 0
counttype = number of events
cron_schedule = 0 3 * * 1
dispatch.earliest_time = -1w
dispatch.latest_time = now
display.general.type = statistics
display.page.search.tab = statistics
enableSched = 1
quantity = 0
relation = greater than
search = |savedsearch instrumentation.usage.smartStore.global | join type=left splunk_server [|savedsearch instrumentation.usage.smartStore.perIndex] | join type=left splunk_server [|savedsearch instrumentation.usage.smartStore.capacity] | join type=left splunk_server [|savedsearch instrumentation.usage.smartStore.accountTypes] | eval data="{"+globalConfig+", "+capCombined+", "+indexConfig+", "+s2Enabled+", "+accountTypes+"}",_time=now(), date=strftime(_time, "%Y-%m-%d") | fields data _time date

#Metrics
[instrumentation.usage.search.report_acceleration]
search = | localop | rest /servicesNS/-/-/admin/summarization | stats count as existing_report_accelerations, sum(summary.access_count) as access_count_of_existing_report_accelerations | makejson access_count_of_existing_report_accelerations(int) existing_report_accelerations(int) output="data" | eval _time=now(), date=strftime(_time, "%Y-%m-%d") | fields _time date data

#searchtelemetry
[instrumentation.usage.search.searchTelemetry]
search = index=_introspection sourcetype=search_telemetry | rename search_commands{}.name as name, search_commands{}.duration as duration | stats perc50(duration), perc90(duration), perc95(duration), perc99(duration), sum(duration) as totalDuration, sum(bytes_read) as sumBytesRead, count(bytes_read) as countBytesRead max(bytes_read) as maxBytesRead by name, type | makejson output=searchTypeInformation | fields searchTypeInformation | mvcombine delim="," searchTypeInformation | nomv searchTypeInformation | eval _time=now(), date=strftime(_time, "%Y-%m-%d") | eval data="{ \"searchTypeInformation\" : [".searchTypeInformation."]}" | fields _time date data

#searchtelemetry sourcetypeusage
[instrumentation.usage.search.searchtelemetry.sourcetypeUsage]
search = index=_audit | stats count(sourcetype_count__*) as * | makejson output=sourcetypeUsage | fields sourcetypeUsage | mvcombine delim="," sourcetypeUsage | nomv sourcetypeUsage | eval _time=now(), date=strftime(_time, "%Y-%m-%d") | eval data="{ \"sourcetypeUsage\" : [".sourcetypeUsage."]}" | fields _time date data

#Lookup Definitions
[instrumentation.usage.lookups.lookupDefinitions]
search = |rest splunk_server=local /services/admin/transforms-lookup getsize=true | eval name = 'eai:acl.app' + "." + title | rename "eai:acl.sharing" AS sharing | eval is_temporal = if(isnull(time_field),0,1) | table name type is_temporal size sharing | join type=left name [rest splunk_server=local /services/admin/kvstore-collectionstats | table data | mvexpand data | spath input=data | table ns size | rename ns as name] | eval name=sha1(name) | makejson output=lookups | stats list(lookups) as lookups | eval data = "{ \"lookups\" : [" . mvjoin(lookups,",") . "]}", _time = now(), date=strftime(_time, "%Y-%m-%d") | fields data _time date

#Bundle Replication
[instrumentation.performance.bundleReplication]
search = index=_internal source=*/metrics.log TERM(group=bundles_uploads) | bin _time span=1d | stats count as bundles_uploads_count avg(peer_count) as avg_peer_count avg(average_baseline_bundle_bytes) as avg_baseline_bundle_bytes max(average_baseline_bundle_bytes) as max_baseline_bundle_bytes avg(average_delta_bundle_bytes) as avg_delta_bundle_bytes max(average_delta_bundle_bytes) as max_delta_bundle_bytes sum(total_count) as total_count sum(delta_count) as total_delta_count sum(success_count) as total_success_count sum(baseline_count) as total_baseline_count sum(already_present_count) as total_already_present_count sum(total_msec_spent) as total_msec_spent sum(delta_msec_spent) as total_delta_msec_spent sum(total_bytes) as total_bytes sum(delta_bytes) as total_delta_bytes by host _time | makejson output=data | eval date=strftime(_time, "%Y-%m-%d") | fields _time date data

#Bundle Replication Cycle
[instrumentation.performance.bundleReplicationCycle]
search = index=_internal source=*/metrics.log splunk_server=local TERM(group=bundle_replication) TERM(name=cycle_dispatch) \
| stats count(cycle_id) as cycleCount avg(peer_count) as avgPeerCount avg(peer_success_count) as avgPeerSuccessCount avg(replication_time_msec) as avgReplicationTimeMsec avg(bundle_bytes) as avgBundleBytes avg(delta_bundle_bytes) as avgDeltaBundleBytes \
| appendcols [| rest /services/search/distributed/bundle/replication/config splunk_server=local | fields replicationPolicy] \
| eval avgPeerCount=round(avgPeerCount,2) | eval avgPeerSuccessCount=round(avgPeerSuccessCount,2) \
| eval avgReplicationTimeMsec=round(avgReplicationTimeMsec,2) | eval avgBundleBytes=round(avgBundleBytes,2) | eval avgDeltaBundleBytes=round(avgDeltaBundleBytes,2) \
| makejson output=data | eval _time=now(), date=strftime(_time, "%Y-%m-%d") | fields _time date data

#Metrics Info
[instrumentation.usage.metrics]
search = | mcatalog values(_dims) values(sourcetype) values(metric_type) where index=* earliest=-15m by metric_name, index | stats count(values(_dims)) AS dimension_count list(values(sourcetype)) AS sourcetype list(values(metric_type)) AS metrictype by metric_name, index | eval metrictype = if(isnull(metrictype), "N/A", metrictype) | fields metric_name, index,  dimension_count, sourcetype, metrictype | eval data="{ \"metricName\" : \""+metric_name+"\", \"indexName\" : \""+index+"\", \"dimensionCount\" : \""+dimension_count+"\", \"sourcetype\" : \""+sourcetype+"\", \"metricType\" : \""+metrictype+"\"}", _time = now(), date=strftime(_time, "%Y-%m-%d") | fields _time date data

#Rollup
[instrumentation.usage.rollup]
search = | rest servicesNS/-/-/catalog/metricstore/rollup | eval summaryCount=0, target_index_list="", metricOverrideCount=0, name=title, hasDimensionList=if(isnull(dimensionList), "false", "true") | foreach summaries*rollupIndex [| eval summaryCount=if(isnull('<<FIELD>>'), summaryCount, summaryCount+1)] | foreach aggregation* [| eval metricOverrideCount=if(isnull('<<FIELD>>'), metricOverrideCount, metricOverrideCount+1)] | foreach summaries*rollupIndex [eval target_index_list=if(isnotnull('<<FIELD>>'), target_index_list.",".'<<FIELD>>', target_index_list)] | eval targetIndex=split('target_index_list',",")| mvexpand  targetIndex | search NOT targetIndex="" | join type=left targetIndex [| rest /services/data/indexes datatype=metric | eval targetIndexDBSizeGB_temp=tostring(round(coalesce(currentDBSizeMB,0) / 1024, 2)) | stats sum(targetIndexDBSizeGB_temp) as targetIndexDBSizeGB by title | rename title as targetIndex | fields targetIndex, targetIndexDBSizeGB] | fields name, defaultAggregation, summaryCount, hasDimensionList, metricOverrideCount, targetIndex, targetIndexDBSizeGB | eval targetIndexDBSizeGB=if(targetIndexDBSizeGB==0, "0 (Check Index to Verify)", targetIndexDBSizeGB) | makejson name defaultAggregation targetIndex targetIndexDBSizeGB hasDimensionList summaryCount(int) metricOverrideCount(int) output="data" | eval _time=now(), date=strftime(_time, "%Y-%m-%d") | fields _time date data

# Feature Flags
[instrumentation.deployment.featureFlags]
search = | rest /services/properties splunk_server=local | rename title as conf |  map maxsearches=200 search="| rest /services/properties/$conf$ fillContents=1 | rename title as featureFlag  | search featureFlag=_feature-flag*" | fields featureFlag, name, creationDate, disabled, description |  makejson featureFlag, name, description, disabled, creationDate output=data | outputtelemetry input=data anonymous=true support=true component="deployment.featureFlags" optinrequired=3 type=event

# Savedsearches Alert
[instrumentation.usage.savedSearches.alerts]
search = | rest servicesNS/-/-/admin/savedsearch | search NOT title=instrumentation.* | eval name=sha1(title), alertConditionType=alert_type, actionList=actions, triggerMode=if('alert.digest_mode'==1, "Once", "For each result"), alertSeverity='alert.severity', alertSuppress=if('alert.suppress'==1, "Yes", "No"), alertSuppressGroup=if('alert.suppress.group_name'=="", "N/A", sha1('alert.suppress.group_name')), alertTrackable=if('alert.track'==1, "Yes", "No"), cronSchedule=cron_schedule, dispatchAllowPartialResults=if('dispatch.allow_partial_results'==1, "Yes", "No") | fields name, alertConditionType, actionList, triggerMode, alertSeverity, alertTrackable, alertSuppress, alertSuppressGroup, cronSchedule, dispatchAllowPartialResults | makejson name alertConditionType actionList triggerMode alertSeverity alertTrackable alertSuppress cronSchedule alertSuppressGroup dispatchAllowPartialResults output="data" | eval _time=now(), date=strftime(_time, "%Y-%m-%d") | fields _time date data

# Streaming Metric Alert
[instrumentation.usage.streamingMetricAlerts]
search = | rest servicesNS/-/-/alerts/metric_alerts | eval name=sha1(title), hasFilter=if(filter=="", "No", "Yes"), hasGroupby=if(groupby=="", "No", "Yes"), triggerActionPerGroup=if('trigger.action_per_group'==1, "Yes", "No"), triggerEvaluationPerGroup=if('trigger.evaluation_per_group'==1, "Yes", "No"), triggerSuppress=if('trigger.suppress'==1, "Yes", "No"), triggerPrepare=if(isnotnull('trigger.prepare'), "Yes", "No"), alertTrackable=if('splunk_ui.track'==1, "Yes", "No"), triggerThreshold=if(isnotnull('trigger.threshold'), 'trigger.threshold', "N/A"), actionList="", hasDescription=if(description=="", "No", "Yes"), alertSeverity=if(isnull('splunk_ui.severity'), "N/A", 'splunk_ui.severity'), triggerExpires='trigger.expires', triggerMaxTracked='trigger.max_tracked' | eval actionList=if('action.email'==1, actionList.","."email", actionList), actionList=if('action.logevent'==1, actionList.","."logevent", actionList), actionList=if('action.rss'==1, actionList.","."rss", actionList), actionList=if('action.script'==1, actionList.","."script", actionList), actionList=if('action.webhook'==1, actionList.","."webhook", actionList) | eval actionList=if(actionList=="", actionList, substr(actionList,2)) | eval hasLabels="No", hasComplexCondition="No" | foreach label* [| eval hasLabels=if(isnull('<<FIELD>>'), hasLabels, "Yes")] | eval hasMultipleMetricIndexes=if(match(metric_indexes, ","), "Yes", "No"), hasComplexCondition=if(match(condition, "OR") OR match(condition, "AND"), "Yes", hasComplexCondition) | fields name, hasFilter, hasGroupby, triggerActionPerGroup, triggerEvaluationPerGroup, triggerThreshold, triggerSuppress, triggerPrepare, alertTrackable, actionList, hasDescription, alertSeverity, triggerExpires, triggerMaxTracked, hasLabels, hasMultipleMetricIndexes, hasComplexCondition | makejson name hasFilter hasGroupby triggerActionPerGroup triggerEvaluationPerGroup triggerThreshold triggerSuppress triggerPrepare alertTrackable actionList hasDescription alertSeverity triggerExpires triggerMaxTracked hasLabels hasMultipleMetricIndexes hasComplexCondition output="data" | eval _time=now(), date=strftime(_time, "%Y-%m-%d") | fields _time date data

# Durable Search
[instrumentation.usage.durableSearch]
search = | rest servicesNS/-/-/admin/savedsearch | search NOT title=instrumentation.* AND NOT durable.track_time_type="" | eval name=sha1(title), durableTrackTimeType='durable.track_time_type', durableLagTime='durable.lag_time', durableBackfillType='durable.backfill_type', durableMaxBackfillIntervals='durable.max_backfill_intervals', enableSummaryIndex=if('action.summary_index'==1, "Yes", "No")  | fields name, durableTrackTimeType, durableLagTime, durableBackfillType, durableMaxBackfillIntervals, enableSummaryIndex | makejson name durableTrackTimeType durableLagTime durableBackfillType durableMaxBackfillIntervals enableSummaryIndex output="data" | eval _time=now(), date=strftime(_time, "%Y-%m-%d") | fields _time date data

#
[instrumentation.usage.rest]
search = index=_internal useragent=splunk-sdk* | eval endpointuri = case(like(uri_path, "%/authorization/capabilities%"), "authorization/capabilities", like(uri_path, "%/authorization/roles%"), "authorization/roles", like(uri_path, "%/configs/conf-%s%"), "configs/conf-%s", like(uri_path, "%/properties/%"), "properties", like(uri_path, "%/saved/eventtypes%"), "saved/eventtypes", like(uri_path, "%/deployment/client%"), "deployment/client", like(uri_path, "%/deployment/tenants%"), "deployment/tenants", like(uri_path, "%/deployment/server%"), "deployment/server", like(uri_path, "%/deployment/serverclass%"), "deployment/serverclass", like(uri_path, "%/storage/passwords%"), "storage/passwords", like(uri_path, "%/services/receivers/stream%"), "/services/receivers/stream", like(uri_path, "%/services/receivers/simple%"), "/services/receivers/simple", like(uri_path, "%/authentication/users%"), "authentication/users", like(uri_path, "%/saved/searches%"), "saved/searches", like(uri_path, "%/data/modular_inputs%"), "data/modular_inputs", like(uri_path, "%/data/input%"), "data/input", like(uri_path, "%/data/indexes%"), "data/indexes", like(uri_path, "%/alerts/fired_alerts%"), "/alerts/fired_alerts", like(uri_path, "%messages%"), "messages", like(uri_path, "%/search/jobs%"), "search/jobs" ) | stats count by endpointuri, status, method, useragent | eval _time=now(), date=strftime(_time, "%Y-%m-%d") | eval data="{ \"endpointUri\" : \""+endpointuri+"\", \"status\" : \""+status+"\", \"method\" : \""+method+"\", \"useragent\" : \""+useragent+"\", \"count\" : \""+count+"\" }" | fields _time date data

# bucket merge
[instrumentation.usage.bucketmerge.standalone]
search=index=_internal source=*splunkd-utility.log* component=BucketMergerCmd command=merge OR command=dryrun OR command=list | table command, newBucketsCount, oldBucketsCount, durationSec | makejson command newBucketsCount oldBucketsCount durationSec output="data" | eval _time=now(), date=strftime(_time, "%Y-%m-%d") | fields _time date data

[instrumentation.usage.bucketmerge.clustered]
search = index=_internal source=*mergebuckets.log* ("all peers" action=merge OR action=dryrun) OR action=list | eval command=action | eval newBucketsCount=mergedBuckets | eval oldBucketsCount=totalBucketsToMerge | eval indexersCount=if(isnotnull(peers),peers,0) | eval bucketsFailedToMergeCount=if(isnotnull(bucketsUnableToMerge),bucketsUnableToMerge,0) |table command, newBucketsCount, oldBucketsCount, bucketsFailedToMergeCount, indexersCount | makejson command newBucketsCount oldBucketsCount bucketsFailedToMergeCount indexersCount output="data" | eval _time=now(), date=strftime(_time, "%Y-%m-%d") | fields _time date data

# local-proxy feature usage
[instrumentation.usage.localproxy.config]
search = | rest splunk_server=local /services/local_proxy/config | eval disabled='info.disabled', max_concurrent_requests='info.max_concurrent_requests', response_timeout_ms='info.response_timeout_ms' | makejson disabled, max_concurrent_requests, response_timeout_ms output=data | eval _time=now(), date=strftime(_time, "%Y-%m-%d") | fields _time date data

[instrumentation.usage.localproxy.info]
search = index=_internal sourcetype=splunkd component=LocalProxyRestHandler log_level=INFO pending_responses duration_msec status=200 | bin _time span=1m | stats avg(duration_msec) as avgLatency min(duration_msec) as minLatency max(duration_msec) as maxLatency avg(pending_responses) as avgRequests min(pending_responses) as minRequests max(pending_responses) as maxRequests count as requests by _time | makejson output=data | eval date=strftime(_time, "%Y-%m-%d") | fields _time date data

[instrumentation.usage.localproxy.client]
search = index=_internal sourcetype=splunkd_ui_access status=200 method!=GET | rex field=uri "(.*)/services(?<matching>/.*)" | join left=uiAccess right=lpAccess where uiAccess.matching = lpAccess.uri_path [| search index=_internal component=LocalProxyRestHandler sourcetype=splunkd log_level=INFO status=200 uri_path | stats count by uri_path | fields uri_path] | stats count by uiAccess.matching, uiAccess.useragent | fields uiAccess.matching, uiAccess.useragent, count | rename uiAccess.useragent as client, uiAccess.matching as uri | makejson uri,client, count output=data | eval _time=now(), date=strftime(_time, "%Y-%m-%d") | fields _time date data

#supervisor stability
[instrumentation.usage.supervisor.stability]
search = index=_internal sourcetype=splunkd component=SidecarThread log_level=ERROR Sidecar "failed to run" Status | stats count by Sidecar | eval _time=now() | eval date=strftime(_time, "%Y-%m-%d") | makejson Sidecar, count output=data | fields _time date data

# config tracker feature usage
[instrumentation.usage.configtracker.info]
search = | rest splunk_server=local /services/config_tracker/config | eval disabled='info.disabled', mode='info.mode', denylist='info.denylist', exclude_fields='info.exclude_fields', uses_inotify='info.uses_inotify' | makejson disabled, mode, denylist, exclude_fields, uses_inotify output=data | eval _time=now(), date=strftime(_time, "%Y-%m-%d") | fields _time date data

[instrumentation.usage.configtracker.introspection]
search = index=_audit action=search info=granted (search="*index=_c*" OR search="*index=*_configtracker*") NOT (search=*"index=*_audit"*) | stats dc(user) AS "user_count" count AS "total_search_count" | eval _time=now() | eval date=strftime(_time, "%Y-%m-%d") | makejson user_count total_search_count output=data | fields data _time

[instrumentation.usage.configtracker.searches]
search = index="_configtracker" | top limit=500 data.action data.path data.changes{}.stanza data.changes{}.properties{}.name | rex field=data.path ".*\/etc\/*(?<path>.*)" | eval path = "$SPLUNK_HOME/etc/" + 'path' | eval stanza = 'data.changes{}.stanza' | eval prop = 'data.changes{}.properties{}.name' | eval _time=now() | eval date=strftime(_time, "%Y-%m-%d") | makejson path stanza prop count output=data | fields data _time

[instrumentation.usage.cmredundancy]
search = index=_internal source=*metrics.log* splunk_server=local group=cm_redundancy* | stats sum(indexers_pinged_for_switchover) as indexersPingedForSwitchover, sum(indexers_reachable_by_this_cm) as indexersReachableByThisCm, sum(indexers_can_reach_active_cm) as indexersCanReachActiveCm, sum(hb_received) as hbReceived, sum(retry_sent) as retrySent, sum(switchover_attempts) as switchoverAttempts, sum(lb_healthcheck_count) as lbHealthcheckCount, median(standby_bundle_sync_rate_kBps) as standbyBundleSyncRateKBps, median(standby_generation_sync_avg_elapsed_sec) as standbyGenerationSyncAvgElapsedSec | makejson indexersPingedForSwitchover indexersReachableByThisCm indexersCanReachActiveCm hbReceived retrySent switchoverAttempts lbHealthcheckCount standbyBundleSyncRateKBps standbyGenerationSyncAvgElapsedSec output="data" | eval _time=now(), date=strftime(_time, "%Y-%m-%d") | fields _time date data

# s3 tsidx streaming compression
[instrumentation.usage.s3tsidxcompression]
search = index=_internal source=*/metrics.log  group=s3client name=get | stats sum(num_files) as downloadedFiles, sum(decompressed_kb) as downloadedDecompressedKb, sum(compressed_kb) as downloadedCompressedKb | appendcols [| search index=_internal source=*/metrics.log  group=s3client name=put | stats sum(num_files) as uploadedFiles, sum(decompressed_kb) as uploadedOriginalKb, sum(compressed_kb) as uploadedCompressedKb] | makejson downloadedFiles downloadedCompressedKb downloadedDecompressedKb uploadedFiles uploadedOriginalKb uploadedCompressedKb output=data | eval _time=now(), date=strftime(_time, "%Y-%m-%d") | fields _time date data

#KVStore upgrade
[instrumentation.upgrade.kvstore.standalone]
search = | rest splunk_server=local /services/server/info \
| fields server_roles host \
| join host [ search index=_internal sourcetype=splunkd "KV Store upgrade version tool completed in seconds=" OR "KV Store version upgrade failed" | stats latest(host) as host, latest(seconds) as duration, latest(_time) as end_time, latest(event_message) as event_message | eval kvstore_upgrade_status=case(match(event_message, "KV Store upgrade version tool completed in seconds="), "Success", match(event_message, "KV Store version upgrade failed"), "Failed") | fields host, kvstore_upgrade_status, duration, end_time] \
| makejson duration host kvstore_upgrade_status server_roles end_time output=data \
| eval _time=now(), date=strftime(_time, "%Y-%m-%d") \
| fields _time date data

[instrumentation.upgrade.kvstore.shc]
search = index=_internal component=SHCMasterKVStoreMigrationState "Completed KVStore update for search head cluster" \
| stats latest(start_time) as start_time, latest(end_time) as end_time, latest(version) as version, latest(type) as type, latest(host) as host \
| eval duration = end_time - start_time \
| eval version = version | eval type = type \
| makejson start_time end_time duration type version host output=data \
| eval _time=now(), date=strftime(_time, "%Y-%m-%d") \
| fields _time date data

[instrumentation.upgrade.kvstore.standaloneBackfill]
search = | rest splunk_server=local /services/server/info \
| fields server_roles host \
| join host [ search index=_internal earliest=12/09/2024:00:00:00 latest=now sourcetype=splunkd "KV Store upgrade version tool completed in seconds=" OR "KV Store version upgrade failed" | stats latest(host) as host, latest(seconds) as duration, latest(_time) as end_time, latest(event_message) as event_message | eval kvstore_upgrade_status=case(match(event_message, "KV Store upgrade version tool completed in seconds="), "Success", match(event_message, "KV Store version upgrade failed"), "Failed") | fields host, kvstore_upgrade_status, duration, end_time] \
| makejson duration host kvstore_upgrade_status server_roles end_time output=data \
| eval _time=now(), date=strftime(_time, "%Y-%m-%d") \
| fields _time date data

[instrumentation.upgrade.kvstore.shcBackfill]
search = index=_internal earliest=12/09/2024:00:00:00 latest=now component=SHCMasterKVStoreMigrationState "Completed KVStore update for search head cluster" \
| stats latest(start_time) as start_time, latest(end_time) as end_time, latest(version) as version, latest(type) as type, latest(host) as host \
| eval duration = end_time - start_time \
| eval version = version | eval type = type \
| makejson start_time end_time duration type version host output=data \
| eval _time=now(), date=strftime(_time, "%Y-%m-%d") \
| fields _time date data

# TLS feature usage
[instrumentation.usage.tlsBestPractices]
search = | rest /services/configs/conf-server/kvstore | eval sslVerifyServerCert=if(isnull(sslVerifyServerCert),"unset",sslVerifyServerCert), splunk_server=sha256(splunk_server),  kvstore_configuredSystem=if(app_dir="system","true", "false")  \
| stats values(sslVerifyServerCert) as kvstore_sslVerifyServerCert values(eai:acl.app) as kvstore_configuredApp by splunk_server | eval kvstore_configuredSystem=if(kvstore_configuredApp="system","true","false") | fields kvstore_sslVerifyServerCert, splunk_server, kvstore_configuredSystem \
| append [| rest /services/configs/conf-server/sslConfig | eval sslVerifyServerName=if(isnull(sslVerifyServerName),"unset",sslVerifyServerName), splunk_server=sha256(splunk_server) | stats values(sslVerifyServerName) as servername_sslVerifyServerName values(eai:acl.app) as servername_configuredApp by splunk_server | eval servername_configuredSystem=if(servername_configuredApp="system","true","false") | fields servername_sslVerifyServerCert, splunk_server, servername_configuredSystem] \
| append [| rest /services/configs/conf-server/sslConfig | eval sslVerifyServerCert=if(isnull(sslVerifyServerCert),"unset",sslVerifyServerCert), splunk_server=sha256(splunk_server) | stats values(eai:acl.app) as global_configuredApp values(sslVerifyServerCert) as global_sslVerifyServerCert by splunk_server | eval global_configuredSystem=if(global_configuredApp="system","true","false") | fields global_sslVerifyServerCert, splunk_server, global_configuredSystem] \
| append [| rest /services/configs/conf-pythonSslClientConfig | eval sslVerifyServerCert=if(isnull(sslVerifyServerCert),"unset",sslVerifyServerCert), splunk_server=sha256(splunk_server)| stats values(eai:acl.app) as python_configuredApp values(sslVerifyServerCert) as python_sslVerifyServerCert by splunk_server | eval python_configuredSystem=if(python_configuredApp="system","true","false") | fields python_sslVerifyServerCert, splunk_server, python_configuredSystem] \
| append [| rest /services/configs/conf-web/settings | eval mgmtHostPort=if(isnull(mgmtHostPort),"unset",mgmtHostPort), splunk_server=sha256(splunk_server)| stats values(eai:acl.app) as fwdrMgmtHostPort_configuredApp values(mgmtHostPort) as fwdr_mgmtHostPort by splunk_server | eval fwdrMgmtHostPort_configuredSystem=if(fwdrMgmtHostPort_configuredApp="system","true","false") | fields fwdrMgmtHostPort_sslVerifyServerCert, splunk_server, fwdrMgmtHostPort_configuredSystem] \
| append [| rest /services/configs/conf-server/sslConfig | eval cliVerifyServerName=if(isnull(cliVerifyServerName),"feature",cliVerifyServerName), splunk_server=sha256(splunk_server) | stats values(cliVerifyServerName) as servername_cliVerifyServerName values(eai:acl.app) as servername_configuredApp by splunk_server | eval cli_configuredSystem=if(cli_configuredApp="system","true","false") | fields cli_sslVerifyServerCert, splunk_server, cli_configuredSystem] | stats values(*) as * by splunk_server | makejson output=data \
| outputtelemetry input=data anonymous=true support=true component="usage.tlsBestPractices" optinrequired=3 type=event

# Federated Search Usage
[instrumentation.usage.federatedsearch.fsh]
search = index=_audit [| rest /services/authentication/users splunk_server=local | dedup splunk_server | rename splunk_server AS host | table host] sourcetype=audittrail TERM(action=search) user!="splunk-system-user" user!="n/a" is_federated_search=1 | eval search_type="fsh_search" | table _time user app search_type info search_id has_error_warn fully_completed_search total_run_time exec_time | eval user=sha256(user) | makejson output=data | fields + data _time\
| outputtelemetry input=data anonymous=true support=true component="usage.federatedsearch.fsh" optinrequired=3 type=event

[instrumentation.usage.federatedsearch.rsh]
search = index=_audit \
    [| rest /services/server/status splunk_server=local \
    | dedup splunk_server \
    | rename splunk_server AS host \
    | table host] \
    sourcetype=audittrail TERM(action=search) user!="splunk-system-user" user!="n/a" federated_user!="" \
| eval search_type="rsh_search" \
| table _time user app search_type info search_id has_error_warn fully_completed_search total_run_time exec_time federated_sid federated_user federated_version federated_guid use_fsh_ko fsh_streaming_phase_only\
| eval user=sha256(user), federated_user=sha256(federated_user) \
| makejson output=data \
| fields + data _time\
| outputtelemetry input=data anonymous=true support=true component="usage.federatedsearch.rsh" optinrequired=3 type=event


# Monitoring Console
[instrumentation.app.splunk_monitoring_console.info]
search = | rest splunk_server=local /services/server/roles | fields role_list \
  | appendcols [ | rest splunk_server=local /servicesNS/-/splunk_monitoring_console/configs/conf-splunk_monitoring_console_assets/settings \
     | eval mode = case(configuredPeers="", "standalone", true(), "distributed") \
     | fields disabled, mode, mc_auto_config ] |  makejson output=data \
     | eval _time=now(), date=strftime(_time, "%Y-%m-%d") | fields _time date data \

#Ingest actions usages
[instrumentation.usage.ingestactions.rulesets]
search = | rest splunk_server=local /services/data/ingest/rulesets serialize_rules=true | spath input=rules | rename {}.* as * \
| fields dest | mvexpand dest| eval dest=split(dest,",") | mvexpand dest | eval dest = replace(dest,"rfs:", "") | dedup dest \
| join type=left [|rest splunk_server=local services/data/ingest/rfsdestinations \
| rex field=path "(?<destinationType>.*?):" | rename title as dest | fields dest destinationType] \
| stats count by destinationType | eval data="\""+destinationType+"\":"+count| fields data \
| append [| rest splunk_server=local /services/server/info | eval data="\"instanceRole\":\""+mvjoin(server_roles, ", ") + "\"" | fields data] \
| append [| rest splunk_server=local /services/data/ingest/rulesets serialize_rules=true | spath input=rules | rename {}.* as * \
| stats count by action | eval data="\""+action+"\":"+count  | fields data \
| append [| rest splunk_server=local /services/data/ingest/rulesets serialize_rules=true | spath input=rules | rename {}.* as * \
| stats count by clone | where clone = "true" | eval data="\"clone\":"+count | fields data]] \
| append [| rest splunk_server=local /services/data/ingest/rulesets serialize_rules=true | spath input=rules | rename {}.* as * \
| stats count(match) as maskRegexCount count(cond.filter.match) as filterRegexCount count(cond.filter.expr) as filterEvalExprCount count(cond.route.match) as routeRegexCount count(cond.route.expr) as routeEvalExprCount dc(expr) as uniqueIndexCount \
| makejson output=data| fields data | eval data = replace(data, "{", "")| eval data = replace(data, "}", "")] \
| mvcombine delim="," data | nomv data | eval data = "{" + data +"} " |  eval _time=now(), date=strftime(_time, "%Y-%m-%d") | fields _time date data \

[instrumentation.usage.ingestactions.destinations]
search = | rest splunk_server=local services/data/ingest/rfsdestinations | rex field=path "(?<destinationType>.*?):" | rename remote.s3.* as * \
| eval authMethodAccesskey=if(destinationType="s3"AND isnotnull(access_key) AND isnotnull(secret_key) , "true", "false") \
| eval authMethodIAM=if(destinationType="s3"AND isnull(access_key) AND isnull(secret_key) , "true", "false") \
| rename signature_version as signatureVersion supports_versioning as supportsVersioning url_version as urlVersion \
| fields authMethodAccesskey authMethodIAM batchSizeThresholdKB batchTimeout compression destinationType dropEventsOnUploadError encryption signatureVersion supportsVersioning urlVersion \
| makejson output=data| fields data | mvcombine delim="," data | nomv data | eval data="{ \"destinations\" : ["+data+"]}"|  eval _time=now(), date=strftime(_time, "%Y-%m-%d") | fields _time date data \

[instrumentation.usage.ingestactions.deletions]
search = index=_internal sourcetype=splunkd source=*splunkd.log* component=IngestActionsRulesHandler OR component=IngestActionsRfsDestinationHandler ("Deleted ruleset" OR "Deleted rfs destination") \
| eval actionType=if(match(_raw, "Deleted ruleset"), "rulesetDeletion", "destinationDeletion") \
| stats count by actionType | eval data="\""+actionType+"\":"+count | fields data | mvcombine delim="," data | nomv data \
| eval data = "{"+data+"}" | eval _time=now(), date=strftime(_time, "%Y-%m-%d") | fields _time date data \

#Field filters telemetry
[instrumentation.usage.search.fieldFilters]
search = | rest /servicesNS/-/-/authorization/fieldfilters \
| eval filterName = sha1(title), isFieldFilteringEnabled = if('disabled'==1, "false", "true"), isTargetIndexSpecified = if(isnull('index'), "false", "true"), limitType = 'limit.key', fieldType = if('action.field'=="_raw", "raw", "indexed"), replacementMethod = case('action.operator'=="sha256()","SHA256", 'action.operator'=="sha512()","SHA512",'action.operator'=="null()","NULL",true(),"STRING"), areExemptionRolesSpecified =    if(isnull('roleExemptions'), "false", "true") \
| fields filterName, isFieldFilteringEnabled, isTargetIndexSpecified, limitType, fieldType, replacementMethod, areExemptionRolesSpecified \
| makejson filterName isFieldFilteringEnabled isTargetIndexSpecified limitType fieldType replacementMethod areExemptionRolesSpecified output="  data" | eval _time=now(), date=strftime(_time, "%Y-%m-%d") | fields _time date data

# Version Control telemetry metrics
[instrumentation.usage.versionControl.report]
action.outputtelemetry = 1
action.outputtelemetry.param.anonymous     = 1
action.outputtelemetry.param.support       = 1
action.outputtelemetry.param.license       = 0
action.outputtelemetry.param.optinrequired = 3
action.outputtelemetry.param.component = usage.versionControl.report
action.outputtelemetry.param.input = data
action.outputtelemetry.param.type = aggregate
alert.suppress = 0
alert.track = 0
counttype = number of events
cron_schedule = 0 3 * * *
dispatch.earliest_time = -1d
dispatch.latest_time = now
display.general.type = statistics
display.page.search.tab = statistics
enableSched = 1
quantity = 0
relation = greater than
search = index=_internal ( \
        (source=*/var/log/splunk/metrics.log* group=version_control name=stats) OR \
        (sourcetype=splunkd_access uri="/services/configs/conf-server/_reload")) \
                [| rest /services/server/status splunk_server=local | return 1 host=splunk_server] \
                | eval status=coalesce(status, "X") \
                | stats latest(bytes) as bytes, \
                        latest(tracked_file_count) as tracked_file_count, \
                        latest(total_commits) as total_commits, \
                        count(eval(sourcetype="splunkd_access")) as count \
                        by status \
                | eval repo = if(isnull(total_commits), null(), json_object("bytes", bytes, "tracked_file_count", tracked_file_count, "total_commits", total_commits)), \
                        endpoint = if(isnotnull(total_commits), null(), json_object(status, count)) \
                | fields repo, endpoint \
                | mvcombine endpoint \
                | eval endpoint=if(mvcount(endpoint)=0, null(), mv_to_json_array(endpoint, true())) \
                | stats latest(repo) AS repository_snapshot, latest(endpoint) AS reload_rest_endpoint \
        | appendcols \
                [| rest splunk_server=local /services/server/info \
                | fields versionControlEnabled \
                | rename versionControlEnabled as enabled] \
        | makejson output=data
#Splunk remote upgrader telemetry
[instrumentation.usage.splunkUpgrader.status]
search = index=_internal source="*/var/log/splunk/splunk-upgrade-install.log" TELEMETRY | rex ".*is running on (?<platform>\w*)\."   | rename version AS upgrader_version | eval date=strftime(_time, "%Y-%m-%d") | dedup host date | stats count by date upgrader_version kernel_version platform | makejson count upgrader_version kernel_version platform output=data | fields date data

[instrumentation.usage.splunkUpgrader.ufUpgradeStatus]
search = index=_internal source="*/var/log/splunk/splunk-upgrade.log" TELEMETRY | transaction host maxspan=180s  | eval date=strftime(_time, "%Y-%m-%d") | eval status=if(like(_raw, "%Splunk upgrade has been completed successfully%"), "success", if(like(_raw, "%Splunk upgrade failed%"), "fail", null())) | eval product="UF" | rename current_splunk_version AS from target_splunk_version AS to package_type AS package | stats count by date product from to status package platform | makejson count product from to status package platform  output=data | fields date data

[instrumentation.usage.splunkUpgrader.upgraderUpgradeStatus]
search = index=_internal source="*/var/log/splunk/splunk-upgrade.log" TELEMETRY | transaction host maxspan=180s  | eval date=strftime(_time, "%Y-%m-%d") | eval status=if(like(_raw, "%self-upgrade failed%"), "failed", if(like(_raw, "%Splunk Upgrader has been upgraded%"), "successed", null())) | eval product="RU" | rename old_upgrader_version AS from new_upgrader_version AS to package_type AS package | stats count by date product from to status package platform | makejson count product from to status package platform  output=data | fields date data

[instrumentation.usage.splunkUpgrader.upgradeAttempts]
search = index=_internal source="*/var/log/splunk/splunk-upgrade.log" TELEMETRY | transaction host maxspan=180s  | eval date=strftime(_time, "%Y-%m-%d") | eval upgrade_started=if(like(_raw, "%Found upgrade flag file%"), 1,null()) | stats count(upgrade_started) as total_upgrade_attempts by date | makejson total_upgrade_attempts output=data | fields date data

[instrumentation.usage.edgeprocessor.datavolume]
search = | mstats fillnull_value="null" sum(processor_bytes_in_total) as processor_bytes_in_total, sum(processor_bytes_out_total) as processor_bytes_out_total, sum(processor_events_in_total) as processor_events_in_total, sum(processor_events_out_total) as processor_events_out_total WHERE index=_metrics by processor_id, pipeline_id, evt_source_type span=1d \
| makejson processor_id pipeline_id processor_bytes_in_total processor_bytes_out_total processor_events_in_total processor_events_out_total  evt_source_type output=data \
| search processor_bytes_in_total > 1.00 OR processor_bytes_out_total > 1.00 \
| eval date=strftime(_time, "%Y-%m-%d") | fields date data

[instrumentation.usage.edgeprocessor.pipelines]
search = index=_internal source="*/var/log/splunk/sup-pkg-edge-processor-config*" PIPELINE_STATS | rex max_match=0 field=_raw "(?<json>[^\n]+)" | mvexpand json | eval _raw = json \
| spath metric_id | search metric_id=acies_active_pipelines*| spath | stats max(value) as "count" by metric_id | transpose header_field=metric_id | fields acies_active_pipelines, acies_active_pipelines_jobs \
| appendcols [| search index=_internal source="*/var/log/splunk/sup-pkg-edge-processor-config*" PIPELINE_STATS | rex max_match=0 field=_raw "(?<json>[^\n]+)" | mvexpand json | eval _raw = json \
| spath | search metric_id=acies_functions_in_pipelines | stats max(value) as "count" by function_name | transpose 0 header_field=function_name | fields- column | makejson output=functions_in_pipelines_with_function_name] \
| appendcols [|search index=_internal source="*/var/log/splunk/sup-pkg-edge-processor-config*" PIPELINE_STATS | rex max_match=0 field=_raw "(?<json>[^\n]+)" | mvexpand json | eval _raw = json \
| spath | search metric_id=acies_pipeline_destinations | stats max(value) as "count" by destination_type | transpose 0 header_field=destination_type | fields- column | makejson output=pipeline_destinations] \
| rename acies_active_pipelines as active_pipelines, acies_active_pipelines_jobs as active_pipelines_jobs \
| fields active_pipelines, active_pipelines_jobs, pipeline_destinations, functions_in_pipelines_with_function_name | tojson auto(*) output_field=data | eval _time=now() | eval date=strftime(_time, "%Y-%m-%d") | fields date data

[instrumentation.usage.edgeprocessor.instances]
search = index=_internal source="*/var/log/splunk/sup-pkg-edge-processor-config*" PIPELINE_STATS | rex max_match=0 field=_raw "(?<json>[^\n]+)" | mvexpand json | eval _raw = json \
| spath | stats max(value) as "count" by status | transpose header_field=status | fields- column | makejson output=instances_by_status \
| appendcols [| search index=_internal source="*/var/log/splunk/sup-pkg-edge-processor-config*" PIPELINE_STATS | rex max_match=0 field=_raw "(?<json>[^\n]+)" | mvexpand json | fields - _raw  | eval _raw = json \
| spath | stats max(value) as "count" by edge_processor_version | eval instances_per_version = "\"" . edge_processor_version . "\": " . count \
| stats list(instances_per_version) as instances_per_version | eval instances_per_version = "{" . mvjoin(instances_per_version, ", ") . "}"] \
| appendcols [| search index=_internal source="*/var/log/splunk/sup-pkg-edge-processor-config*" PIPELINE_STATS | rex max_match=0 field=_raw "(?<json>[^\n]+)" | mvexpand json | fields - _raw  | eval _raw = json \
| spath | stats max(value) as "count" by source_type | transpose 0 header_field=source_type | fields- column | makejson output=instances_with_sources] \
| fields instances_by_status, instances_per_version, instances_with_sources \
| tojson auto(*) output_field=data | eval _time=now() | eval date=strftime(_time, "%Y-%m-%d") | fields date data

[instrumentation.usage.datamanagement.agents]
search = index=_internal source="*/var/log/splunk/sup-pkg-opamp-svc*" | spath data{} output=data | dedup telemetry_name | stats list(data) by telemetry_name | transpose header_field=telemetry_name | fields- column | tojson auto(*) output_field=data | eval _time=now() | eval date=strftime(_time, "%Y-%m-%d") | fields date, data

[instrumentation.usage.datamanagement.syncedSourceTypes]
search = index=_internal service=orchestrator operationId="syncSourcetypeConfigurationFederatedConnection" message="Ssv2 post selected sourcetype configurations from federated connection with name" | stats count as sourcetype_sync_requests \
| appendcols [ search index=_internal service=orchestrator operationId="syncSourcetypeConfigurationFederatedConnection" message="Source type sync details" | stats avg(processedSourcetypeCount) as average_processed_sourcetypes_per_request, avg(synchronizedSourcetypeCount) as average_synced_sourcetypes_per_request | eval average_processed_sourcetypes_per_request=ceil(average_processed_sourcetypes_per_request), average_synced_sourcetypes_per_request=ceil(average_synced_sourcetypes_per_request) ] \
| appendcols [ search index=_internal service=orchestrator message="POST /orchestrator/v1/runtimes/edge-processor/refresh/sourcetypes/sync" | stats min(durationMs) as minimum, max(durationMs) as maximum, median(durationMs) as median, avg(durationMs) as average | eval average=ceil(average) | tojson output_field=sourcetype_sync_request_duration_ms_stats num(minimum), num(maximum), num(median), num(average) ] \
| appendcols [ search index=_internal service=orchestrator message="POST /orchestrator/v1/runtimes/edge-processor/refresh/sourcetypes/sync" | stats count by statusCode | sort statusCode | eval dummy_x="row" | xyseries dummy_x statusCode count | fields - dummy_x | tojson output_field=sourcetype_sync_response_http_status_codes | fields + sourcetype_sync_response_http_status_codes ] \
| appendcols [ search index=_internal service=orchestrator message="Sourcetype contains unsupported key and will not be synced" | stats count by key | eval dummy_x="row" | xyseries dummy_x key count | fields - dummy_x | tojson output_field=incompatible_sourcetype_setting_stats | fields + incompatible_sourcetype_setting_stats ] \
| tojson output_field=data num(sourcetype_sync_requests) num(average_processed_sourcetypes_per_request) num(average_synced_sourcetypes_per_request) json(sourcetype_sync_request_duration_ms_stats) json(sourcetype_sync_response_http_status_codes) json(incompatible_sourcetype_setting_stats) \
| eval _time=now() | eval date=strftime(_time, "%Y-%m-%d") | fields date, data

[instrumentation.usage.bulkdatamove.bucketsplitcmd]
search = index=_internal sourcetype=splunkd source=*splunkd-utility.log* component=BucketSplitCmd split_stats | table dryrun, split_status, buckets_split, source_buckets_created, destination_buckets_created, time_taken | makejson dryrun, split_status, buckets_split, source_buckets_created, destination_buckets_created, time_taken output=data | eval _time=now(), date=strftime(_time, "%Y-%m-%d") | fields _time date data


[instrumentation.usage.fips.provider]
search = index=_internal "FIPS provider enabled" AND NOT host=*.splunk*.* \
| rex "provider:\s*(?<fips_provider>\S+)," \
| stats latest(_time) as _time by host, fips_provider \
| stats count as count by fips_provider \
| eval kv = "\"" . fips_provider . "\":" . count \
| stats values(kv) as kv_list \
| eval data = "{" . mvjoin(kv_list, ",") . "}" \
| eval _time = now() \
| fields data, _time