#!/usr/bin/env python # coding=utf-8 __author__ = "TrackMe Limited" __copyright__ = "Copyright 2022-2026, TrackMe Limited, U.K." __credits__ = "TrackMe Limited, U.K." __license__ = "TrackMe Limited, all rights reserved" __version__ = "0.1.0" __maintainer__ = "TrackMe Limited, U.K." __email__ = "support@trackme-solutions.com" __status__ = "PRODUCTION" # Standard library imports import json import logging import os import re import sys import time # Third-party library imports import urllib3 from logging.handlers import RotatingFileHandler # Disable insecure request warnings for urllib3 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) # set splunkhome splunkhome = os.environ["SPLUNK_HOME"] # set logging filehandler = RotatingFileHandler( "%s/var/log/splunk/trackme_splk_flx_converging.log" % splunkhome, mode="a", maxBytes=10000000, backupCount=1, ) formatter = logging.Formatter( "%(asctime)s %(levelname)s %(filename)s %(funcName)s %(lineno)d %(message)s" ) logging.Formatter.converter = time.gmtime filehandler.setFormatter(formatter) log = logging.getLogger() # root logger - Good to get it only once. for hdlr in log.handlers[:]: # remove the existing file handlers if isinstance(hdlr, logging.FileHandler): log.removeHandler(hdlr) log.addHandler(filehandler) # set the new handler # set the log level to INFO, DEBUG as the default is ERROR log.setLevel(logging.INFO) # append current directory sys.path.append(os.path.dirname(os.path.abspath(__file__))) # import libs import import_declare_test # Import Splunk libs from splunklib.searchcommands import ( dispatch, GeneratingCommand, Configuration, Option, validators, ) # Import trackme libs from trackme_libs import trackme_reqinfo, run_splunk_search @Configuration(distributed=False) class TrackMeFlxConverging(GeneratingCommand): tenants_scope = Option( doc=""" **Syntax:** **tenants_scope=**** **Description:** Comma seperated list of tenants id where to source entities from.""", require=True, default=None, ) group = Option( doc=""" **Syntax:** **group=**** **Description:** The group this entity belongs to.""", require=True, default=None, ) object = Option( doc=""" **Syntax:** **object=**** **Description:** The entity object name.""", require=True, default=None, ) object_description = Option( doc=""" **Syntax:** **object_description=**** **Description:** The entity object description.""", require=True, default=None, ) root_constraint = Option( doc=""" **Syntax:** **root_constraint=**** **Description:** The Flex Object root constraint""", require=True, validate=validators.Match("root_constraint", r"^.*$"), ) consider_orange_as_up = Option( doc=""" **Syntax:** **consider_orange_as_up=**** **Description:** Consider orange as up""", require=False, default=True, validate=validators.Boolean(), ) remove_extra_attributes = Option( doc=""" **Syntax:** **remove_extra_attributes=**** **Description:** Remove the extra_attributes field from the results.""", require=False, default=False, validate=validators.Boolean(), ) min_pct_for_green = Option( doc=""" **Syntax:** **min_pct_for_green=**** **Description:** Minimum percentage of availability required for the status to be green (1). Default is 100.""", require=False, default=100, validate=validators.Integer(0, 100), ) def generate(self, **kwargs): # Start performance counter start = time.time() # Get request info and set logging level reqinfo = trackme_reqinfo( self._metadata.searchinfo.session_key, self._metadata.searchinfo.splunkd_uri ) log.setLevel(reqinfo["logging_level"]) # Get tenants_scope and turn into a list tenants_scope = self.tenants_scope.split(",") # for troubleshooting purposes searches_list = [] # initialise search_results search_results = [] results_dict = {} results_summary_dict = {} results_light_list_up = [] results_light_list_down = [] # counters count_entities = 0 count_entities_list = [] count_entities_up = 0 count_entities_up_list = [] count_entities_down = 0 count_entities_down_list = [] # percentage of availability, as the percentage of entities up against the total number of entities pct_availability = 0 # iterate for tenant_item in tenants_scope: # we accept : as a key pair in the list # if not, we set the component to flx if ":" in tenant_item: tenant_id, component = tenant_item.split(":") else: tenant_id = tenant_item component = "flx" # flx_converging_search flx_converging_search = f""" | trackmegetcoll tenant_id={tenant_id} component={component} ``` exclude converging flx_type ``` | where NOT flx_type="converging" ``` exclude disabled entities ``` | where NOT monitored_state="disabled" ``` root constraints ``` | search ({self.root_constraint}) """ searches_list.append(flx_converging_search) try: reader = run_splunk_search( self.service, flx_converging_search, { "earliest_time": "-5m", "latest_time": "now", "count": 0, "output_mode": "json", }, 24, 5, ) for item in reader: if isinstance(item, dict): logging.debug(f'search_results="{item}"') # append to the list of searches search_results.append(item) # add to the dict by _key results_dict[item["_key"]] = item except Exception as e: logging.error(f'An exception was encountered, exception="{str(e)}"') yield_record = { "_time": time.time(), "action": "failure", "searches": searches_list, "response": "The search failed to be executed", "_raw": { "action": "failure", "response": "The search failed to be executed", "exception": str(e), }, } yield yield_record # # main processing # # for entity in results_dict, get the status (object_state), if green or orange, count as up, if red, count as down and add to the associated lists for entity in results_dict: entity_record = results_dict[entity] # get the alias entity_alias = entity_record.get("alias") # set the summary dict entity_summary_dict = {} # for fields: object, alias, priority, object_state, status_message_json, status_description, status_description_short, tracker_name # if available in the entity record, add to the entity_summary_dict for field in [ "tenant_id", "object", "keyid", "priority", "object_state", "status_message_json", "status_description", "status_description_short", "tracker_name", ]: if field in entity_record: entity_summary_dict[field] = entity_record[field] # add to our summary results_summary_dict[results_dict[entity]["object"]] = entity_summary_dict count_entities += 1 count_entities_list.append(entity_summary_dict) if self.consider_orange_as_up: if results_dict[entity]["object_state"] in ["green", "blue", "orange"]: count_entities_up += 1 count_entities_up_list.append(entity_summary_dict) results_light_list_up.append(entity_alias) entity_summary_dict["converging_status"] = "up" elif results_dict[entity]["object_state"] == "red": count_entities_down += 1 count_entities_down_list.append(entity_summary_dict) results_light_list_down.append(entity_alias) entity_summary_dict["converging_status"] = "down" else: if results_dict[entity]["object_state"] in ["green", "blue"]: count_entities_up += 1 count_entities_up_list.append(entity_summary_dict) results_light_list_up.append(entity_alias) entity_summary_dict["converging_status"] = "up" elif results_dict[entity]["object_state"] in ["orange", "red"]: count_entities_down += 1 count_entities_down_list.append(entity_summary_dict) results_light_list_down.append(entity_alias) entity_summary_dict["converging_status"] = "down" # Sort the lists by tenant_id and object def sort_key(entity): return (entity.get("tenant_id", ""), entity.get("object", "")) count_entities_down_list.sort(key=sort_key) count_entities_list.sort(key=sort_key) # Sort the results_summary_dict by tenant_id and object sorted_results = {} for key in sorted( results_summary_dict.keys(), key=lambda k: ( results_summary_dict[k].get("tenant_id", ""), results_summary_dict[k].get("object", ""), ), ): sorted_results[key] = results_summary_dict[key] # Reorganize results_summary_dict with down_entities and all_entities sections new_results_summary_dict = {"all_entities": sorted_results} if count_entities_down > 0: new_results_summary_dict = { "down_entities": count_entities_down_list, **new_results_summary_dict, } results_summary_dict = new_results_summary_dict # calculate the percentage of availability if count_entities > 0: pct_availability = round((count_entities_up / count_entities) * 100, 2) # status (1 / 2 / 3) if pct_availability >= self.min_pct_for_green: status = 1 elif pct_availability >= 0: status = 2 else: status = 3 # set the value of metrics metrics = { "pct_availability": pct_availability, "count_entities": count_entities, "count_entities_up": count_entities_up, "count_entities_down": count_entities_down, } # set status_description_short status_description_short = f"Availability={pct_availability}%, up={count_entities_up}, down={count_entities_down}" # set status_description if pct_availability == 100: status_description = f"The availability percentage is {pct_availability}, all {count_entities} entities are up" elif pct_availability > 0: status_description = f"The availability percentage is {pct_availability}, {count_entities_up} entities are up and {count_entities_down} entities are down: {results_light_list_down}" else: status_description = f"The availability percentage is {pct_availability}, all {count_entities} entities are down: {results_light_list_down}" # final results records # if count_entities > 0: if count_entities > 0: final_results_records = { "group": self.group, "object:": self.object, "object_description": self.object_description, "status": status, "status_description": status_description, "status_description_short": status_description_short, "extra_attributes": results_summary_dict, "metrics": metrics, } else: # no entities found final_results_records = { "group": self.group, "object:": self.object, "object_description": self.object_description, "status": 3, "status_description": "No entities found", "status_description_short": "No entities found", "extra_attributes": {}, "metrics": { "pct_availability": 0, "count_entities": 0, "count_entities_up": 0, "count_entities_down": 0, }, "searches": searches_list, } # remove extra_attributes if requested if self.remove_extra_attributes: results_summary_dict = {} final_results_records["extra_attributes"] = {} yield_record = { "_time": time.time(), "action": "success", "search": flx_converging_search, "group": self.group, "object": self.object, "object_description": self.object_description, "status": status, "status_description": status_description, "status_description_short": status_description_short, "extra_attributes": results_summary_dict, "metrics": metrics, "_raw": final_results_records, } # yield yield yield_record # Log the run time logging.info( f'trackmesplkflxconverging has terminated, run_time={round(time.time() - start, 3)}, search="{flx_converging_search}"' ) dispatch(TrackMeFlxConverging, sys.argv, sys.stdin, sys.stdout, __name__)