#Splunk does not index the search.log files from the dispatch directory by default #so create a stanza to take only the parts we care about... #Example lines to look for include #05-24-2018 08:31:03.881 ERROR SearchResultTransaction - Got status 502 from https://x.x.x.x:8089/services/streams/search?sh_sid=1527150641.164891_315974D3-2FA6-4A16-839A-A95A0376BA14 #05-24-2018 08:31:03.881 ERROR SearchResultTransaction - HTTP error status message from https://x.x.x.x:8089/services/streams/search?sh_sid=1527150641.164891_315974D3-2FA6-4A16-839A-A95A0376BA14: Error connecting: Connect Timeout #05-24-2018 08:31:03.881 ERROR DispatchThread - sid:1527150641.164891_315974D3-2FA6-4A16-839A-A95A0376BA14 Unknown error for peer indexername. Search Results might be incomplete. If this occurs frequently, please check on the peer. #05-28-2018 00:52:17.245 INFO DispatchThread - sid:1527468707.34320_315974D3-DFFC-48EC-86C8-33BD6744EE4F Search auto-finalized after time limit (30 seconds) reached. #however a better alternative may be [search] #log_search_messages = true #In the limits.conf file and then use the search_messages.log file... [splunk:searchlog] TIME_PREFIX = ^ TIME_FORMAT = %m-%d-%Y %H:%M:%S.%3N SHOULD_LINEMERGE = false TRANSFORMS-set = setNull,setError,setAutoFinalize #Example inputs.conf if you want to use the above in Linux #[monitor:///opt/splunk/var/run/splunk/dispatch/*/search.log] #sourcetype = splunk:searchlog #index = _internal #Splunk records failures from search heads to indexer for corrupt buckets in the info.csv log only on the search head level #the search.log on the indexer peers *will* record this so if your ingesting the search.log from the peers you probably don't need this one... #The info.csv does show you what the end user will see in terms of errors such as this... #Examples include: #,,,,,,,,,,,,,,,,,ERROR,"[hostname] Failed to read size=1 event(s) from rawdata in bucket='_internal~43~E21ADB4E-02B7-4877-8A42-A15CE7F422BD' path='.../db_1515304396_1515080916_.... Rawdata may be corrupt, see search.log. Results may be incomplete!","{}",,,,,,, #Note that a better alternative may be [search] #log_search_messages = true #In the limits.conf file and then use the search_messages.log file... [splunk:search:info] SHOULD_LINEMERGE = false DATETIME_CONFIG = NONE TRANSFORMS-set = setNull,setWARNorERROR,setAutoFinalize #Example inputs.conf if you want to use the above in Linux #[monitor:///opt/splunk/var/run/splunk/dispatch/*/info.csv] #sourcetype = splunk:search:info #index = _internal #crcSalt =