You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
927 lines
38 KiB
927 lines
38 KiB
##############
|
|
#
|
|
# Customise these macros to ensure the SplunkAdmins / Alerts for Splunk Admins
|
|
# application works as expected
|
|
#
|
|
##############
|
|
[indexerhosts]
|
|
definition = host=*
|
|
iseval = 0
|
|
|
|
[heavyforwarderhosts]
|
|
definition = host=*
|
|
iseval = 0
|
|
|
|
[searchheadhosts]
|
|
definition = host=*
|
|
iseval = 0
|
|
|
|
#Designed for searches where returning data from other search heads
|
|
#would not provide valid results...
|
|
[localsearchheadhosts]
|
|
definition = host=*
|
|
iseval = 0
|
|
|
|
[splunkenterprisehosts]
|
|
definition = host=*
|
|
iseval = 0
|
|
|
|
[deploymentserverhosts]
|
|
definition = host=*
|
|
iseval = 0
|
|
|
|
[licensemasterhost]
|
|
definition = host=*
|
|
iseval = 0
|
|
|
|
[cluster_masters]
|
|
definition = host=*
|
|
iseval = 0
|
|
|
|
[sysloghosts]
|
|
definition = host=*
|
|
iseval = 0
|
|
|
|
[searchheadsplunkservers]
|
|
definition = splunk_server=*
|
|
iseval = 0
|
|
|
|
[splunkindexerhostsvalue]
|
|
definition = splunk_server=*
|
|
iseval = 0
|
|
|
|
[splunkadmins_splunkd_source]
|
|
definition = source=*splunkd.log*
|
|
iseval = 0
|
|
|
|
[splunkadmins_splunkuf_source]
|
|
definition = source=*splunkd.log*
|
|
iseval = 0
|
|
|
|
[splunkadmins_mongo_source]
|
|
definition = source=*mongod.log*
|
|
iseval = 0
|
|
|
|
[splunkadmins_license_usage_source]
|
|
definition = source=*license_usage.log*
|
|
iseval = 0
|
|
|
|
[splunkadmins_clustermaster_oshost]
|
|
definition = host=changeme
|
|
iseval = 0
|
|
|
|
#Only used in a few searches, customise this if you have the cluster master as a search
|
|
#peer, if not you may wish to leave this to local and run the ClusterMasterLevel searches on
|
|
#the cluster master server...
|
|
[splunkadmins_clustermaster_host]
|
|
definition = splunk_server=local
|
|
|
|
##############
|
|
#
|
|
# Utility functions
|
|
#
|
|
##############
|
|
[comment(1)]
|
|
args = text
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
#
|
|
#Dynamically generate a Splunk SPL statement to filter out a list of hosts / time periods where the particular hosts
|
|
#were restarting, for example if search heads were restarting we probably don't care about delayed scheduled searches at this point in time
|
|
#Allowing a macro name to be passed in allows this function to be used for search heads or indexers or anything else
|
|
#Furthermore allowing contingency time allows some time for the server to recover from the restart if required...
|
|
#This macro returns in the form of ((host=X _time>start _time<end) OR (host=Y _time>start _time<end)
|
|
#
|
|
#The query is checking various shutdown messages as different types of server have different messages signalling the start of the shutdown process
|
|
#simplifying this in the past has resulted in missing at least 1 type of shutdown or the start of the shutdown process...
|
|
[splunkadmins_shutdown_list(3)]
|
|
args = macroName, minTimeContingency, maxTimeContingency
|
|
definition = search ```Send an exclusion list in terms of a search result for when this particular Splunk server was shutdown, plus any contingency time as requested```\
|
|
index=_internal (`$macroName$`) sourcetype=splunkd `splunkadmins_splunkd_source` (CASE("Shutting down")) OR "Shutdown complete in" OR "Received shutdown signal." OR "master has instructed peer to restart" OR "Performing early shutdown tasks"\
|
|
| eval message=coalesce(message,event_message)\
|
|
| stats min(_time) AS logTime by message, host\
|
|
| stats min(logTime) AS minTime, max(logTime) AS maxTime by host\
|
|
| eval minTime=minTime - $minTimeContingency$, maxTime=maxTime + $maxTimeContingency$\
|
|
| eval search="host=" . host . " _time>" . minTime . " _time<" .maxTime\
|
|
| fields search\
|
|
| format\
|
|
| rex mode=sed field=search "s/\"//g"
|
|
iseval = 0
|
|
|
|
#
|
|
#Dynamically generate a Splunk SPL statement to filter out a list of keywords (hostnames without the host=) / time periods where the particular hosts
|
|
#were restarting, for example if search heads were restarting we probably don't care about delayed scheduled searches at this point in time
|
|
#Allowing a macro name to be passed in allows this function to be used for search heads or indexers or anything else
|
|
#Furthermore allowing contingency time allows some time for the server to recover from the restart if required...
|
|
#This macro returns in the form of ((X _time>start _time<end) OR (Y _time>start _time<end)
|
|
#
|
|
#The query is checking various shutdown messages as different types of server have different messages signalling the start of the shutdown process
|
|
#simplifying this in the past has resulted in missing at least 1 type of shutdown or the start of the shutdown process...
|
|
[splunkadmins_shutdown_keyword(3)]
|
|
args = macroName, minTimeContingency, maxTimeContingency
|
|
definition = search ```Send an exclusion list in terms of a search result for when this particular Splunk server was shutdown, plus any contingency time as requested```\
|
|
index=_internal (`$macroName$`) sourcetype=splunkd `splunkadmins_splunkd_source` (CASE("Shutting down")) OR "Shutdown complete in" OR "Received shutdown signal." OR "master has instructed peer to restart" OR "Performing early shutdown tasks"\
|
|
| eval message=coalesce(message,event_message)\
|
|
| stats min(_time) AS logTime by message, host\
|
|
| stats min(logTime) AS minTime, max(logTime) AS maxTime by host\
|
|
| eval minTime=minTime - $minTimeContingency$, maxTime=maxTime + $maxTimeContingency$\
|
|
| eval search=host . " _time>" . minTime . " _time<" .maxTime\
|
|
| fields search\
|
|
| format\
|
|
| rex mode=sed field=search "s/\"//g"
|
|
iseval = 0
|
|
|
|
#
|
|
#Dynamically generate a Splunk SPL statement to filter out a list of hosts / time periods where the particular hosts
|
|
#were restarting, for example if search heads were restarting we probably don't care about delayed scheduled searches at this point in time
|
|
#Allowing a macro name to be passed in allows this function to be used for search heads or indexers or anything else
|
|
#Furthermore allowing contingency time allows some time for the server to recover from the restart if required...
|
|
#This macro returns in the form of (_time>start _time<end) this allows entire indexer cluster restarts to be filtered out.
|
|
#
|
|
#The query is checking various shutdown messages as different types of server have different messages signalling the start of the shutdown process
|
|
#simplifying this in the past has resulted in missing at least 1 type of shutdown or the start of the shutdown process...
|
|
[splunkadmins_shutdown_time(3)]
|
|
args = macroName, minTimeContingency, maxTimeContingency
|
|
definition = search ```Send an exclusion list in terms of a search result for the time when any indexer was shutdown```\
|
|
index=_internal (`$macroName$`) sourcetype=splunkd `splunkadmins_splunkd_source` (CASE("Shutting down")) OR "Shutdown complete in" OR "Received shutdown signal." OR "master has instructed peer to restart" OR "Performing early shutdown tasks"\
|
|
| eval message=coalesce(message,event_message)\
|
|
| stats min(_time) AS logTime by message, host\
|
|
| stats min(logTime) AS minTime, max(logTime) AS maxTime\
|
|
| eval minTime=minTime - $minTimeContingency$, maxTime=maxTime + $maxTimeContingency$\
|
|
| eval search=" _time>" . minTime . " _time<" .maxTime\
|
|
| fields search\
|
|
| format\
|
|
| rex mode=sed field=search "s/\"//g"
|
|
iseval = 0
|
|
|
|
# variation of the above to utilise smaller blocks of time during the search
|
|
[splunkadmins_shutdown_time_by_period(4)]
|
|
args = macroName, minTimeContingency, maxTimeContingency, period
|
|
definition = search ```Send an exclusion list in terms of a search result for the time when any indexer was shutdown```\
|
|
index=_internal (`$macroName$`) sourcetype=splunkd `splunkadmins_splunkd_source` (CASE("Shutting down")) OR "Shutdown complete in" OR "Received shutdown signal." OR "master has instructed peer to restart" OR "Performing early shutdown tasks"\
|
|
| eval message=coalesce(message,event_message)\
|
|
| bin _time span=$period$\
|
|
| stats min(_time) AS logTime by message, host, _time\
|
|
| stats min(logTime) AS minTime, max(logTime) AS maxTime by _time\
|
|
| eval minTime=minTime - $minTimeContingency$, maxTime=maxTime + $maxTimeContingency$\
|
|
| eval search=" _time>" . minTime . " _time<" .maxTime\
|
|
| fields search\
|
|
| format\
|
|
| rex mode=sed field=search "s/\"//g"
|
|
iseval = 0
|
|
|
|
|
|
##############
|
|
#
|
|
# Per-alert macros that can be customised for
|
|
# filtering unncessary data from alerts where required
|
|
#
|
|
##############
|
|
|
|
[splunkadmins_acc_datamodels]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_runscript]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_timeskew]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_changedprops]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_changedprops_count]
|
|
definition = 3
|
|
iseval = 0
|
|
|
|
[splunkadmins_btoolvalidation_ds]
|
|
definition = ```Splunk for stream doesn't include a config file which causes errors, however it appears to work without it...``` NOT "/opt/splunk/etc/deployment-apps/Splunk_TA_stream*"
|
|
iseval = 0
|
|
|
|
[splunkadmins_bandwidth]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_toosmall_checkcrc]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_forwarderdown]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_heavylogging]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_exceeding_filedescriptor]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_sending_data]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_sending_data_nonhf_count]
|
|
definition = 0
|
|
iseval = 0
|
|
|
|
[splunkadmins_sending_data_hf_count]
|
|
definition = 5
|
|
iseval = 0
|
|
|
|
[splunkadmins_unusual_duplication]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_unusual_duplication_count]
|
|
definition = 10
|
|
iseval = 0
|
|
|
|
[splunkadmins_crcsalt_initcrc]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_uf_timeshifting]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_future_dated]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_failuretoparse_timestamp]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_failuretoparse_timestamp_count]
|
|
definition = 0
|
|
iseval = 0
|
|
|
|
[splunkadmins_failuretoparse_timestamp_binperiod]
|
|
definition = 1m
|
|
iseval = 0
|
|
|
|
[splunkadmins_failuretoparse_timestamp2]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_indexconfig_warn]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_indexerqueue_fillperc_nonindexqueue]
|
|
definition = 50
|
|
iseval = 0
|
|
|
|
[splunkadmins_indexerqueue_fillperc_indexqueue]
|
|
definition = 90
|
|
iseval = 0
|
|
|
|
[splunkadmins_indexer_replication_queue_count]
|
|
definition = 15
|
|
iseval = 0
|
|
|
|
[splunkadmins_uneven_indexed_perc]
|
|
definition = 25
|
|
iseval = 0
|
|
|
|
[splunkadmins_weekly_brokenevents]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_weekly_truncated]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_weekly_truncated_count]
|
|
definition = 0
|
|
iseval = 0
|
|
|
|
|
|
[splunkadmins_valid_timestamp_invalidparsed]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_longrunning_searches]
|
|
definition = ```Exclude various standard/expected searches``` savedsearch_name!="Generate Meta Woot! every 15 mins" savedsearch_name!="Generate NMON*"
|
|
iseval = 0
|
|
|
|
[splunkadmins_realtime_scheduledsearches]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_scheduledsearches_cannot_run]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_scheduledsearches_without_earliestlatest]
|
|
definition = NOT (eai:acl.app=splunk_app_aws author=nobody)
|
|
iseval = 0
|
|
|
|
#Ignore Splunk apps which will trigger this
|
|
[splunkadmins_scheduledsearches_without_index]
|
|
definition = eai:acl.app!="splunk_archiver" eai:acl.app!="splunk_app_windows_infrastructure" eai:acl.app!="splunk_app_aws" eai:acl.app!="nmon"
|
|
iseval = 0
|
|
|
|
[splunkadmins_scriptfailures]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_users_violating_searchquota]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_users_exceeding_diskquota]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_execprocessor]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_timeformat_change]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_loginattempts]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_insufficient_permissions]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_tcpoutput_paused]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_streamerrors]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_unable_distribute_to_peer]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_dashboards_allindexes]
|
|
definition = NOT (eai:appName=simple_xml_examples eai:acl.sharing=app) NOT (eai:appName=nmon eai:acl.sharing=app)
|
|
iseval = 0
|
|
|
|
[splunkadmins_scheduled_incorrectsharing]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_realtime_dashboard]
|
|
definition = NOT (eai:appName=simple_xml_examples eai:acl.sharing=app) NOT (eai:appName=nmon eai:acl.sharing=app)
|
|
iseval = 0
|
|
|
|
[splunkadmins_olddata]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_olddata_lookback]
|
|
definition = -7d
|
|
|
|
[splunkadmins_olddata_earliest]
|
|
definition = -2600d
|
|
|
|
[splunkadmins_olddata_latest]
|
|
definition = -60d
|
|
|
|
[splunkadmins_forwarders_nottalking_ds]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
#Ignore enterprise security related sendalert errors, they are often false alarms here, also filter the data a bit further...
|
|
[splunkadmins_sendmodalert_errors]
|
|
definition = ```We look for the sendalert commands to provide context around the errors where possible. Since notable/risks fail more often they are removed from this particular alert``` NOT action=notable NOT action=risk NOT (" - INFO]" OR "Results Link" OR "Alert Name:")
|
|
iseval = 0
|
|
|
|
[splunkadmins_bucketrolling_count]
|
|
definition = 20
|
|
iseval = 0
|
|
|
|
[splunkadmins_readop_expectingack]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_repfailures]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_lowdisk]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_lowdisk_perc]
|
|
definition = 10
|
|
iseval = 0
|
|
|
|
[splunkadmins_lowdisk_mb]
|
|
definition = 90000
|
|
iseval = 0
|
|
|
|
[splunkadmins_kvstore_terminated]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_fileintegritycheck]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_multiline_linemerge]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_warninifile]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_toomany_sametimestamp]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_colddata_percused]
|
|
definition = 80
|
|
iseval = 0
|
|
|
|
#Ignore internal indexes introspection & main
|
|
[splunkadmins_colddata]
|
|
definition = ```Some internal indexes roll based on size by default such as introspection``` index!=_introspection index!=defaultdb
|
|
iseval = 0
|
|
|
|
#Ignore internal indexes introspection & main
|
|
[splunkadmins_bucketfrozen]
|
|
definition = ```Some internal indexes roll based on size by default such as introspection``` bkt!="*_introspection*" bkt!="*defaultdb*"
|
|
iseval = 0
|
|
|
|
[splunkadmins_permissions]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
#Ignore internal indexes introspection & main
|
|
[splunkadmins_warmdbcount]
|
|
definition = ```We probably don't care about the warm limits for the internal indexes...``` index!=_introspection index!=defaultdb
|
|
iseval = 0
|
|
|
|
[splunkadmins_warmdbcount_perc]
|
|
definition = 80
|
|
iseval = 0
|
|
|
|
[splunkadmins_clustermaster_failurecount]
|
|
definition = 1
|
|
iseval = 0
|
|
|
|
#My environment appears to have random SSL interconnectivity issues with mongo which are harmless/never cause an issue
|
|
[splunkadmins_mongodb_errors]
|
|
definition = NOT "SSL: error"
|
|
iseval = 0
|
|
|
|
[splunkadmins_mongodb_errors2]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
#Many of these applications contain macros which have embedded macros, attempting to expand them proved to be ... complicated so ignoring them!
|
|
[splunkadmins_scheduledsearches_without_index_macro]
|
|
definition = NOT ((eai:acl.app="splunk_app_windows_infrastructure" OR eai:acl.app="splunk_app_aws" OR eai:acl.app="splunk_app_for_nix" OR eai:acl.app="app-docker" OR eai:acl.app="nmon") AND (eai:acl.sharing=app OR eai:acl.sharing=global))
|
|
iseval = 0
|
|
|
|
[splunkadmins_privilegedowners]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
#Not sure why but this "Success" message appears in my instance...
|
|
[splunkadmins_searchfailures]
|
|
definition = message!="Success"
|
|
iseval = 0
|
|
|
|
[splunkadmins_captain_switchover]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_resource_starvation]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_s2sfilereceiver]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
#
|
|
#Dynamically generate a Splunk SPL statement to filter out a list of hosts / time periods where the particular hosts
|
|
#were having a transfer of captain
|
|
#Allowing a macro name to be passed in allows this function to be used for different search head clusters
|
|
#This macro returns in the form of (_time>start _time<end) this allows entire indexer cluster restarts to be filtered out.
|
|
[splunkadmins_transfer_captain_times(3)]
|
|
args = macroName, minTimeContingency, maxTimeContingency
|
|
definition = search ```Send an exclusion list in terms of a search result for the time when a search head captain transfer occurred``` index=_internal (`$macroName$`) sourcetype=splunkd `splunkadmins_splunkd_source` "Got Transfer captaincy" | eval message=coalesce(message,event_message) | stats min(_time) AS logTime by message, host | stats min(logTime) AS minTime, max(logTime) AS maxTime | eval minTime=minTime - $minTimeContingency$, maxTime=maxTime + $maxTimeContingency$ | eval search=" _time>" . minTime . " _time<" .maxTime | fields search | format | rex mode=sed field=search "s/\"//g"
|
|
iseval = 0
|
|
|
|
[splunkadmins_replicationfactor]
|
|
definition = 2
|
|
iseval = 0
|
|
|
|
[whataccessdoihave]
|
|
definition = rest /services/authentication/users splunk_server=local\
|
|
| search ```REST query is limited to the current search head this is running on so we see the index access from this instances point of view```\
|
|
[| rest /services/authentication/current-context/context splunk_server=local\
|
|
| head 1 \
|
|
| fields username \
|
|
| rename username AS title] \
|
|
| table title roles | rename title as user | mvexpand roles\
|
|
| join type=left roles \
|
|
[rest /services/authorization/roles splunk_server=local\
|
|
| table title srchIndexesAllowed srchIndexesDefault srchIndexesDisallowed imported_srchIndexesAllowed imported_srchIndexesDefault imported_srchIndexesDisallowed | rename title as roles]\
|
|
| fillnull value="" srchIndexesAllowed, srchIndexesDefault, srchIndexesDisallowed, imported_srchIndexesAllowed, imported_srchIndexesDefault imported_srchIndexesDisallowed\
|
|
| eval srchIndexesAllowed = srchIndexesAllowed . " " . imported_srchIndexesAllowed, srchIndexesDefault = srchIndexesDefault . " " . imported_srchIndexesDefault, srchIndexesDisallowed = srchIndexesDisallowed . " " . imported_srchIndexesDisallowed \
|
|
| makemv srchIndexesAllowed tokenizer=(\S+) | makemv srchIndexesDefault tokenizer=(\S+) | makemv srchIndexesDisallowed tokenizer=(\S+) \
|
|
| eval indexes= [ | eventcount summarize=false index=* index=_* | stats values(index) AS indexes | eval theindexes="\"" . mvjoin(indexes, " ") . "\"" | return $theindexes ]\
|
|
| makemv indexes\
|
|
| stats values(roles) AS roles, values(indexes) AS indexes, values(srchIndexesAllowed) AS srchIndexesAllowed, values(srchIndexesDefault) AS srchIndexesDefault, values(srchIndexesDisallowed) AS srchIndexesDisallowed by user
|
|
|
|
[diskusage]
|
|
definition = rest /services/authentication/current-context/context splunk_server=local \
|
|
| head 1 \
|
|
| fields username \
|
|
| map \
|
|
[| rest /services/search/jobs splunk_server=local search="eai:acl.owner=$username$" ] \
|
|
| eval run_time=tostring(round(runDuration),"duration"), time_to_live_remaining=tostring(ttl,"duration"), disk_usage=round(diskUsage/1024/1024) \
|
|
| eventstats sum(disk_usage) AS total_disk_usage \
|
|
| eval disk_usage=disk_usage . "MB", total_disk_usage=total_disk_usage . "MB" \
|
|
| stats list(disk_usage) AS disk_usage, list(eai:acl.app) AS apps, list(provenance) AS provenance, list(resultCount) AS result_count, list(run_time) AS run_time, list(time_to_live_remaining) AS time_to_live_remaining, list(updated) AS updated, list(title) AS title, values(total_disk_usage) AS total_disk_usage by dispatchState \
|
|
| table total_disk_usage, disk_usage, apps, provenance, time_to_live_remaining, run_time, dispatchState, result_count, updated, title \
|
|
| eval total_disk_usage=if(dispatchState!="DONE",null(),total_disk_usage)
|
|
iseval = 0
|
|
|
|
[splunkadmins_restmacro]
|
|
definition = splunk_server=local
|
|
iseval = 0
|
|
|
|
#Number of metric logs printed per minute, defaults to 30 seconds but can be changed by the user...
|
|
#if you log metrics every minute change this to 1
|
|
[splunkadmins_metrics_permin]
|
|
definition = 2
|
|
iseval = 0
|
|
|
|
#Substitute `<macro name>` within the audit.log files with the audit definition based on a lookup file
|
|
#note this version only substitutes the first macro seen...the Splunk 8 version can handle multiple macros at once
|
|
[splunkadmins_audit_logs_macro_sub]
|
|
definition = ```Set all values to null() in case this macro is called again within the same search. Subsitute a macro used inside a search with the definition found in the lookup file```\
|
|
| eval definition=null(), commas=null(), commas2=null(), argCount2=null(), argCount=null(), match=null()\
|
|
| rex field=search max_match=1 "\`(?!\")(?!')(?P<macro>[^\`]+)\`" \
|
|
```You can have multiple macro definitions with either 0 or more arguments so we have to count them...``` \
|
|
| rex max_match=10 field=macro "([^\"]+\")|([^']+')\s*(?P<commas>,)" \
|
|
| rex max_match=10 field=macro "(?P<commas2>,)" \
|
|
| rex max_match=1 field=macro "(?P<match>[^\(]+\()" \
|
|
```Two count methods are used as if we have macro(arg1) that has no commas, but macro(arg1,arg2) will work as expected...``` \
|
|
| eval argCount2=if(match(macro,"([^\"]+\")|([^']+')") AND isnull(commas),-1,if(isnotnull(commas2),mvcount(commas2),null())) \
|
|
| eval argCount=if(isnull(argCount2),0,argCount2+1) \
|
|
| eval argCount=if(argCount==0,if(isnotnull(match),1,0),argCount) \
|
|
| rex field=macro "(?P<macro>^[^\( ]+)" \
|
|
| eval macroName=if(argCount==0,macro,macro . "(" . argCount . ")") \
|
|
| lookup splunkadmins_macros title AS macroName, app AS app_name, splunk_server \
|
|
| eval app_name2="global"\
|
|
| lookup splunkadmins_macros title AS macroName, app AS app_name2, splunk_server OUTPUTNEW definition\
|
|
| lookup splunkadmins_macros title AS macroName, splunk_server OUTPUTNEW definition\
|
|
| eval macroReplace=if((argCount == 0),(("`" . macro) . "`"),(("`" . macro) . "\\(.*?\\)`")), search=if(isnotnull(definition),replace(search,macroReplace,mvindex(definition,0)),search)
|
|
iseval = 0
|
|
|
|
#Substitute `<macro name>` within the audit.log files with the audit definition based on a lookup file
|
|
#note this version only works on Splunk 8 due to the use of mvmap
|
|
[splunkadmins_audit_logs_macro_sub_v8]
|
|
definition = ```Set all values to null() in case this macro is called again within the same search. Subsitute a macro used inside a search with the definition found in the lookup file``` \
|
|
eval definition=null(), definition2=null(), definition3=null(), commas=null(), commas2=null(), argCount2=null(), argCount=null(), match=null() \
|
|
| rex field=search "\\`(?!\")(?!')(?P<macro>[^\\`]+)\\`" max_match=20 \
|
|
```remove any commas inside double quotes or single quotes inside a macro, they are probably not arguments to the macro itself``` \
|
|
| eval remove_commas_inside_macros=mvmap(macro,replace(macro,"(\"[^\"]+\"|'[^']+')","")) \
|
|
```Originally a regex, the replace+len works in mvmap and determines number of commas so we can find a macro name``` \
|
|
| eval commas2=mvmap(remove_commas_inside_macros,if(match(remove_commas_inside_macros,"^[^\(]+$"),"-1",len(replace(remove_commas_inside_macros,"[^,]+",""))+1)) \
|
|
| rex field=macro "(?P<macro_name>^[^\( ]+)" max_match=20 \
|
|
| eval macro_commas=mvzip(macro_name,commas2,"!!!!!!!") \
|
|
```A macro with zero arguments is -1 from the previous mvmap, if it has non-zero arguments the definition changes to macro(number)...``` \
|
|
| eval macroName=mvmap(macro_commas,if(mvindex(split(macro_commas,"!!!!!!!"),1)=="-1",mvindex(split(macro_commas,"!!!!!!!"),0),mvindex(split(macro_commas,"!!!!!!!"),0) . "(" . mvindex(split(macro_commas,"!!!!!!!"),1) . ")")) \
|
|
| lookup splunkadmins_macros title AS macroName, app AS app_name, splunk_server \
|
|
| eval app_name2="global" \
|
|
```The original version just did an OUTPUTNEW definition, however this has the limitation that if 1 of the 5 macros found resolves, output stops. And this can result in missing macros. So this version over-matches but that appears to be the tradeoff...without making this even more complicated``` \
|
|
| lookup splunkadmins_macros title AS macroName, app AS app_name2, splunk_server OUTPUT definition AS definition2 \
|
|
| lookup splunkadmins_macros title AS macroName, splunk_server OUTPUT definition AS definition3 \
|
|
| eval definition=mvdedup(mvappend(definition,definition2,definition3)) \
|
|
| fillnull definition value="macronotfound" \
|
|
| nomv definition \
|
|
| eval definition=" " . definition . " " \
|
|
```While an mvmap could replace per-macro that results in a multivalue output. Also replace doesn't handle a multivalued replacement argument so just replace the first macro if it exists with the definitions of all the macros, close enough for what we want``` \
|
|
| eval search=if(isnotnull(macro_name),replace(search,mvindex(macro_name,0),definition),search)
|
|
iseval = 0
|
|
|
|
#Substitute `<macro name>` within the any file
|
|
[splunkadmins_macro_sub(1)]
|
|
args = fieldname
|
|
definition = ```Set all values to null() in case this macro is called again within the same search. Subsitute a macro used inside a search with the definition found in the lookup file``` \
|
|
eval definition=null(), definition2=null(), definition3=null(), commas=null(), commas2=null(), argCount2=null(), argCount=null(), match=null() \
|
|
| rex field=$fieldname$ "\\`(?!\")(?!')(?P<macro>[^\\`]+)\\`" max_match=20 \
|
|
```remove any commas inside double quotes or single quotes inside a macro, they are probably not arguments to the macro itself``` \
|
|
| eval remove_commas_inside_macros=mvmap(macro,replace(macro,"(\"[^\"]+\"|'[^']+')","")) \
|
|
```Originally a regex, the replace+len works in mvmap and determines number of commas so we can find a macro name``` \
|
|
| eval commas2=mvmap(remove_commas_inside_macros,if(match(remove_commas_inside_macros,"^[^\(]+$"),"-1",len(replace(remove_commas_inside_macros,"[^,]+",""))+1)) \
|
|
| rex field=macro "(?P<macro_name>^[^\( ]+)" max_match=20 \
|
|
| eval macro_commas=mvzip(macro_name,commas2,"!!!!!!!") \
|
|
```A macro with zero arguments is -1 from the previous mvmap, if it has non-zero arguments the definition changes to macro(number)...``` \
|
|
| eval macroName=mvmap(macro_commas,if(mvindex(split(macro_commas,"!!!!!!!"),1)=="-1",mvindex(split(macro_commas,"!!!!!!!"),0),mvindex(split(macro_commas,"!!!!!!!"),0) . "(" . mvindex(split(macro_commas,"!!!!!!!"),1) . ")")) \
|
|
| lookup splunkadmins_macros title AS macroName, app AS app_name, splunk_server \
|
|
| eval app_name2="global" \
|
|
```The original version just did an OUTPUTNEW definition, however this has the limitation that if 1 of the 5 macros found resolves, output stops. And this can result in missing macros. So this version over-matches but that appears to be the tradeoff...without making this even more complicated``` \
|
|
| lookup splunkadmins_macros title AS macroName, app AS app_name2, splunk_server OUTPUT definition AS definition2 \
|
|
| lookup splunkadmins_macros title AS macroName, splunk_server OUTPUT definition AS definition3 \
|
|
| eval definition=mvdedup(mvappend(definition,definition2,definition3)) \
|
|
| fillnull definition value="macronotfound" \
|
|
| nomv definition \
|
|
| eval definition=" " . definition . " " \
|
|
```While an mvmap could replace per-macro that results in a multivalue output. Also replace doesn't handle a multivalued replacement argument so just replace the first macro if it exists with the definitions of all the macros, close enough for what we want``` \
|
|
| eval search=if(isnotnull(macro_name),replace($fieldname$,mvindex(macro_name,0),definition),$fieldname$)
|
|
iseval = 0
|
|
|
|
#Note this macro requires TA-webtools
|
|
#Alternatively the "Mothership app" on SplunkBase can be used for this purpose...
|
|
[splunkadmins_remote_macros(3)]
|
|
args = url,user,pass
|
|
definition = | curl method=get uri="$url$/servicesNS/-/-/configs/conf-macros?count=-1&output_mode=json" user=$user$ pass=$pass$\
|
|
| spath input=curl_message path="entry{}.name" output=title\
|
|
| spath input=curl_message path="entry{}.acl.app" output=app\
|
|
| spath input=curl_message path="entry{}.content.definition" output=definition\
|
|
| spath input=curl_message path="entry{}.acl.sharing" output=sharing\
|
|
| fields - curl_* \
|
|
| fields title, app, definition, sharing \
|
|
| eval data=mvzip(mvzip(mvzip(title, 'app', "%%%%"),definition,"%%%%"),sharing,"%%%%")\
|
|
| fields data \
|
|
| mvexpand data \
|
|
| makemv data delim="%%%%" \
|
|
| eval title=mvindex(data,0),app=mvindex(data,1), definition=mvindex(data,2), sharing=mvindex(data,3)\
|
|
| search sharing!=user\
|
|
| fields - data
|
|
iseval = 0
|
|
|
|
#Not currently in use by searches but attempts to pull the roles from a remote Splunk server
|
|
#Alternatively the "Mothership app" on SplunkBase can be used for this purpose...
|
|
[splunkadmins_remote_roles(3)]
|
|
args = url,user,pass
|
|
definition = | curl method=get uri="$url$/services/authentication/users?output_mode=json&count=0&f=roles" user="$user$" pass="$pass$"\
|
|
| rex field=curl_message max_match=10000 "{\"name\":\"(?P<user>[^\"]+)\".*?\"roles\":\[(?P<roles>[^\]]+)" \
|
|
| fields - curl_* \
|
|
| eval data=mvzip(user,roles,"%%%%") \
|
|
| mvexpand data \
|
|
| table data \
|
|
| makemv data delim="%%%%" \
|
|
| eval user=mvindex(data,0), roles=mvindex(data,1)\
|
|
| fields - data\
|
|
| eval roles=replace(roles,"\"","")\
|
|
| makemv roles delim=","
|
|
iseval = 0
|
|
|
|
#Macro to determine search head cluster name, potentially using a case statement or similar
|
|
[search_head_cluster]
|
|
definition = "default"
|
|
iseval = 0
|
|
|
|
#Macro to determine which indexer cluster name, potentially using a case statement or similar
|
|
[indexer_cluster_name(1)]
|
|
args = indexer
|
|
definition = "default"
|
|
iseval = 0
|
|
|
|
#Macro to define indexer cluster name
|
|
[indexer_cluster_name]
|
|
definition = "default"
|
|
iseval = 0
|
|
|
|
[forwarder_name(1)]
|
|
args = hostname
|
|
definition = "default"
|
|
iseval = 0
|
|
|
|
[search_type_from_sid(1)]
|
|
args = search_id
|
|
definition = eval from=null(), username=null(), searchname2=null(), searchname=null()\
|
|
| rex field=$search_id$ "'?(_rt)?(_?subsearch)*_?(?P<from>[^_]+)((_(?P<base64username>[^_]+))|(__(?P<username>[^_]+)))((__(?P<app>[^_]+)__(?P<searchname2>[^_]+))|(_(?P<base64appname>[^_]+)__(?P<searchname>[^_]+)))"\
|
|
| rex field=$search_id$ "^_?(?P<from>SummaryDirector)"\
|
|
```Pattern appears to vary but remote_<hostname>_ is consistent along with the optional _subsearch, the _from can be <username>__ownername__appname__RMD for dashboards as one pattern, it can also be unixepoch (ad-hoc), or scheduler__username__appname (scheduled search), or username__owner__(something)__dashboardview, among others. RMD values can be translated via audit.log, scheduler.log or remote_searches.log (if savedsearch_name is there)!```\
|
|
| fillnull from value="adhoc"\
|
|
| eval searchname=coalesce(searchname,searchname2)\
|
|
| eval type=case(from=="scheduler","scheduled",from=="SummaryDirector","acceleration",match(search_id,"^'?alertsmanager_"),"scheduled",isnotnull(searchname),"dashboard",1=1,"ad-hoc")
|
|
iseval = 0
|
|
|
|
[base64decode(1)]
|
|
args = afield
|
|
definition = eval $afield$=null() ```As per https://docs.splunk.com/Documentation/Splunk/latest/Report/Createandeditreports usernames/apps can be base64 encrypted, remove the eval when ready to use this...decrypt2 (splunkbase) can be used to decrypt with (remove the backslashes): eval $afield$=$afield$ . "===" | decrypt field=$afield$ atob emit('$afield$')```
|
|
iseval = 0
|
|
|
|
[dashboard_depends_filter1]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[dashboard_depends_filter2]
|
|
definition = ```potentially a where clause to only filter when a certain number of tokens exist...``` ""
|
|
iseval = 0
|
|
|
|
[dashboard_depends_filter3]
|
|
definition = ```potentially a where clause to only filter when a certain number of tokens were matched or similar...``` ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_wineventlog_index]
|
|
definition = wineventlog
|
|
iseval = 0
|
|
|
|
[splunkadmins_unexpected_term_count]
|
|
definition = 5
|
|
iseval = 0
|
|
|
|
#Note getsize=true appears to be added in 7.3.3+ and above so this will only work on newer versions and only for lookup definitions
|
|
#the /admin/file-explorer/ will work for all CSV files but is admin only so using this option as a macro...
|
|
[mylookups]
|
|
definition = rest splunk_server=local /servicesNS/-/-/admin/transforms-lookup getsize=true \
|
|
| search [| rest /services/authentication/current-context/context splunk_server=local | head 1 | fields username | rename username AS eai:acl.owner] \
|
|
| eval name = 'eai:acl.app' + "." + title \
|
|
| rename "eai:acl.sharing" AS sharing \
|
|
| table name type size sharing \
|
|
| sort - size
|
|
|
|
[splunkadmins_tailreader_ignorepath]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_splunk_server_name]
|
|
definition = "default"
|
|
|
|
[splunkadmins_audit_alltime]
|
|
definition = ""
|
|
|
|
[splunkadmins_dashboards_alltime]
|
|
definition = ""
|
|
|
|
#Just a nicer way to format the returned data from the conf-props or conf-similar (borrowed from slack)
|
|
[conf_rest_endpoint(1)]
|
|
args = endpoint
|
|
definition = rest /services/configs/conf-$endpoint$ splunk_server=local \
|
|
| eval _raw="", acl="" \
|
|
| foreach "*" \
|
|
[| eval field=if(match("<<FIELD>>","^(title|eai:|splunk_server|author|id|updated|published)"),"","<<FIELD>> = ".'<<FIELD>>') \
|
|
| eval acl_field=if(match("<<FIELD>>","^(eai:|author|updated|published)"),"<<FIELD>> = ".'<<FIELD>>',"") \
|
|
| eval _raw=mvappend(_raw,field) \
|
|
| eval acl=mvappend(acl,acl_field)] \
|
|
| fields splunk_server title _raw acl \
|
|
| eval _raw=mvappend("[".title."]",_raw)
|
|
iseval = 0
|
|
|
|
[splunkadmins_excessive_rest_api_httplib]
|
|
definition = "Python-httplib2/0.13.1 (gzip)"
|
|
|
|
[splunkadmins_excessive_rest_api_threshold]
|
|
definition = 100
|
|
|
|
#Convert a time string into epoch time
|
|
[splunkadmins_epoch(1)]
|
|
args = time
|
|
definition = strptime("$time$","%Y-%m-%d %T")
|
|
iseval = 1
|
|
|
|
[splunkadmins_audit_logs_datamodel_sub]
|
|
definition = eval definition=null(), datamodel3=null(), datamodel1=null(), datamodel2=null()\
|
|
| rex field=search "^\s*\|\s*((from\s+datamodel\s*:?\s*\"?(?P<datamodel1>[^\"\.\s]+))|(datamodel\s+\"?(?P<datamodel2>[^\s\"\.]+)\"?\s+[^\|]*search))" \
|
|
| rex field=search "datamodel\s*=\s*\"?(?P<datamodel3>[^\s\"\.]+)" \
|
|
| eval datamodel_res=case(isnotnull(datamodel3) AND match(search,"\s*\|\s*(tstats)"),datamodel3,isnotnull(datamodel1),datamodel1,isnotnull(datamodel2),datamodel2,true(),null()) \
|
|
| lookup splunkadmins_datamodels datamodel AS datamodel_res, app AS app_name, splunk_server OUTPUT definition\
|
|
| eval app_name2="global"\
|
|
| lookup splunkadmins_datamodels datamodel AS datamodel_res, app AS app_name2, splunk_server OUTPUTNEW definition\
|
|
| lookup splunkadmins_datamodels datamodel AS datamodel_res, splunk_server OUTPUTNEW definition\
|
|
| nomv definition \
|
|
| eval definition=" " . definition . " "\
|
|
```While an mvmap could replace per-datamodel that results in a multivalue output. Also replace doesn't handle a multivalued replacement argument so just replace the first macro if it exists with the definitions of all the datamodels``` \
|
|
| eval search=if(isnotnull(datamodel_res),replace(search,mvindex(datamodel_res,0),definition),search)
|
|
iseval = 0
|
|
|
|
[splunkadmins_audit_logs_tags_sub]
|
|
definition = eval pretag=null(), tag=null(), definition=null(), definition2=null(), definition3=null() \
|
|
| rex field=search max_match=50 "(?P<pre_tag>tag\s*=\s*)(?P<tag>[^\s\)\"]+)" \
|
|
| lookup splunkadmins_tags tag, app AS app_name, splunk_server OUTPUT definition \
|
|
| eval app_name2="global" \
|
|
| lookup splunkadmins_tags tag, app AS app_name2, splunk_server OUTPUT definition AS definition2 \
|
|
| lookup splunkadmins_tags tag, splunk_server OUTPUT definition AS definition3 \
|
|
| eval definition=mvdedup(mvappend(definition, definition2, definition3)) \
|
|
| nomv definition \
|
|
| eval search=if(isnotnull(definition),replace(search,pre_tag . tag," " . definition . " "),search)
|
|
iseval = 0
|
|
|
|
[splunkadmins_audit_logs_eventtypes_sub]
|
|
definition = eval pre_eventtype=null(), eventtype=null(), eventtype2=null(), definition=null(), definition2=null(), definition3=null() \
|
|
| rex field=search max_match=20 "(?P<pre_eventtype>eventtype\s*=\s*)((\"(?P<eventtype>[^\"]+))|((?P<eventtype2>[^\s\)]+)))" \
|
|
| eval eventtype=coalesce(eventtype,eventtype2) \
|
|
| lookup splunkadmins_eventtypes eventtype, app AS app_name, splunk_server OUTPUT definition \
|
|
| eval app_name2="global" \
|
|
| lookup splunkadmins_eventtypes eventtype, app AS app_name2, splunk_server OUTPUT definition AS definition2 \
|
|
| lookup splunkadmins_eventtypes eventtype, splunk_server OUTPUT definition AS definition3 \
|
|
| eval definition=mvdedup(mvappend(definition, definition2, definition3)) \
|
|
| nomv definition \
|
|
| eval search=if(isnotnull(definition),replace(search,pre_eventtype . "\"?" . eventtype," " . definition . " "),search)
|
|
iseval = 0
|
|
|
|
[splunkadmins_slowpeer_time]
|
|
definition = 60
|
|
iseval = 0
|
|
|
|
[splunkadmins_slowpeer_threshold]
|
|
definition = 10
|
|
iseval = 0
|
|
|
|
[splunkadmins_searchmessages_user_1]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_searchmessages_user_2]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_searchmessages_admin_1]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_searchmessages_admin_2]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_splunkd_log_messages]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_alertactions_max_action_results]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_authorize_conf_prevent_users]
|
|
definition = role!="can_delete"
|
|
iseval = 0
|
|
|
|
[splunkadmins_indexer_remotesearches_alltime]
|
|
definition = host=localhost
|
|
iseval = 0
|
|
|
|
[splunkadmins_dataparsing_error]
|
|
definition = ""
|
|
iseval = 0
|
|
|
|
[splunkadmins_shutdown_time_by_shc(3)]
|
|
args = macroName, minTimeContingency, maxTimeContingency
|
|
definition = search ```Send an exclusion list in terms of a search result for the time when any SH was shutdown```\
|
|
index=_internal (`$macroName$`) sourcetype=splunkd `splunkadmins_splunkd_source` (CASE("Shutting down")) OR "Shutdown complete in" OR "Received shutdown signal." OR "Shutdown signal received" OR "master has instructed peer to restart" OR "Performing early shutdown tasks"\
|
|
| eval message=coalesce(message,event_message)\
|
|
| stats min(_time) AS logTime by message, host\
|
|
| eval search_head=host\
|
|
| eval search_head_cluster=`search_head_cluster`\
|
|
| stats min(logTime) AS minTime, max(logTime) AS maxTime by search_head_cluster\
|
|
| eval minTime=minTime - $minTimeContingency$, maxTime=maxTime + $maxTimeContingency$\
|
|
| eval search=" _time>" . minTime . " _time<" .maxTime . " search_head_cluster=" . search_head_cluster\
|
|
| fields search\
|
|
| format\
|
|
| rex mode=sed field=search "s/\"//g"
|
|
iseval = 0
|
|
|
|
[splunkadmins_indexerqueue_count]
|
|
definition = 1
|
|
iseval = 0
|
|
|
|
[splunkadmins_deploymentserver_splunkserver]
|
|
definition = splunk_server=local
|
|
iseval = 0
|
|
|
|
[splunkadmins_sh_knowledgebundle_metrics_filter]
|
|
definition = where replication_time_msec>200000
|
|
iseval = 0
|
|
|
|
[splunkadmins_sh_knowledgebundle_metrics_timespan]
|
|
definition = 60m
|
|
iseval = 0
|
|
|
|
[splunkadmins_bundlepush_span]
|
|
definition = 10m
|
|
iseval = 0
|
|
|
|
[splunkadmins_metrics_source]
|
|
definition = source=*metrics.log*
|
|
iseval = 0
|
|
|
|
[splunkadmins_hec_metrics_source]
|
|
definition = source=*http_event_collector_metrics.log*
|
|
iseval = 0
|
|
|
|
[splunkadmins_summaryindex_durablesearch]
|
|
definition = NOT title IN ("SearchHeadLevel - summary indexing searches not using durable search") next_scheduled_time!=""
|
|
iseval = 0
|
|
|
|
[splunkadmins_events_per_second]
|
|
definition = desc.savedsearch_name IN ("Example")
|
|
iseval = 0
|