Splunk_Deploiement/apps/trackme/bin/trackmesplkflxconverging.py

#!/usr/bin/env python
# coding=utf-8

__author__ = "TrackMe Limited"
__copyright__ = "Copyright 2022-2026, TrackMe Limited, U.K."
__credits__ = "TrackMe Limited, U.K."
__license__ = "TrackMe Limited, all rights reserved"
__version__ = "0.1.0"
__maintainer__ = "TrackMe Limited, U.K."
__email__ = "support@trackme-solutions.com"
__status__ = "PRODUCTION"

# Standard library imports
import json
import logging
import os
import re
import sys
import time

# Third-party library imports
import urllib3
from logging.handlers import RotatingFileHandler

# Disable insecure request warnings for urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

# set splunkhome
splunkhome = os.environ["SPLUNK_HOME"]

# set logging
filehandler = RotatingFileHandler(
    "%s/var/log/splunk/trackme_splk_flx_converging.log" % splunkhome,
    mode="a",
    maxBytes=10000000,
    backupCount=1,
)
formatter = logging.Formatter(
    "%(asctime)s %(levelname)s %(filename)s %(funcName)s %(lineno)d %(message)s"
)
logging.Formatter.converter = time.gmtime
filehandler.setFormatter(formatter)
log = logging.getLogger()  # root logger - Good to get it only once.
for hdlr in log.handlers[:]:  # remove the existing file handlers
    if isinstance(hdlr, logging.FileHandler):
        log.removeHandler(hdlr)
log.addHandler(filehandler)  # set the new handler
# set the log level to INFO, DEBUG as the default is ERROR
log.setLevel(logging.INFO)

# append current directory
sys.path.append(os.path.dirname(os.path.abspath(__file__)))

# import libs
import import_declare_test

# Import Splunk libs
from splunklib.searchcommands import (
    dispatch,
    GeneratingCommand,
    Configuration,
    Option,
    validators,
)

# Import trackme libs
from trackme_libs import trackme_reqinfo, run_splunk_search


@Configuration(distributed=False)
class TrackMeFlxConverging(GeneratingCommand):

    tenants_scope = Option(
        doc="""
        **Syntax:** **tenants_scope=****
        **Description:** Comma seperated list of tenants id where to source entities from.""",
        require=True,
        default=None,
    )

    group = Option(
        doc="""
        **Syntax:** **group=****
        **Description:** The group this entity belongs to.""",
        require=True,
        default=None,
    )

    object = Option(
        doc="""
        **Syntax:** **object=****
        **Description:** The entity object name.""",
        require=True,
        default=None,
    )

    object_description = Option(
        doc="""
        **Syntax:** **object_description=****
        **Description:** The entity object description.""",
        require=True,
        default=None,
    )

    root_constraint = Option(
        doc="""
        **Syntax:** **root_constraint=****
        **Description:** The Flex Object root constraint""",
        require=True,
        validate=validators.Match("root_constraint", r"^.*$"),
    )

    consider_orange_as_up = Option(
        doc="""
        **Syntax:** **consider_orange_as_up=****
        **Description:** Consider orange as up""",
        require=False,
        default=True,
        validate=validators.Boolean(),
    )

    remove_extra_attributes = Option(
        doc="""
        **Syntax:** **remove_extra_attributes=****
        **Description:** Remove the extra_attributes field from the results.""",
        require=False,
        default=False,
        validate=validators.Boolean(),
    )

    min_pct_for_green = Option(
        doc="""
        **Syntax:** **min_pct_for_green=****
        **Description:** Minimum percentage of availability required for the status to be green (1). Default is 100.""",
        require=False,
        default=100,
        validate=validators.Integer(0, 100),
    )

    def generate(self, **kwargs):
        # Start performance counter
        start = time.time()

        # Get request info and set logging level
        reqinfo = trackme_reqinfo(
            self._metadata.searchinfo.session_key, self._metadata.searchinfo.splunkd_uri
        )
        log.setLevel(reqinfo["logging_level"])

        # Get tenants_scope and turn into a list
        tenants_scope = self.tenants_scope.split(",")

        # for troubleshooting purposes
        searches_list = []

        # initialise search_results
        search_results = []
        results_dict = {}
        results_summary_dict = {}
        results_light_list_up = []
        results_light_list_down = []

        # counters
        count_entities = 0
        count_entities_list = []
        count_entities_up = 0
        count_entities_up_list = []
        count_entities_down = 0
        count_entities_down_list = []

        # percentage of availability, as the percentage of entities up against the total number of entities
        pct_availability = 0

        # iterate

        for tenant_item in tenants_scope:

            # we accept <tenant_id>:<component>  as a key pair in the list
            # if not, we set the component to flx
            if ":" in tenant_item:
                tenant_id, component = tenant_item.split(":")
            else:
                tenant_id = tenant_item
                component = "flx"

            # flx_converging_search
            flx_converging_search = f"""        
                | trackmegetcoll tenant_id={tenant_id} component={component}
                ``` exclude converging flx_type ```
                | where NOT flx_type="converging"
                ``` exclude disabled entities ```
                | where NOT monitored_state="disabled"
                ``` root constraints ```
                | search ({self.root_constraint})
            """
            searches_list.append(flx_converging_search)

            try:
                reader = run_splunk_search(
                    self.service,
                    flx_converging_search,
                    {
                        "earliest_time": "-5m",
                        "latest_time": "now",
                        "count": 0,
                        "output_mode": "json",
                    },
                    24,
                    5,
                )

                for item in reader:
                    if isinstance(item, dict):
                        logging.debug(f'search_results="{item}"')
                        # append to the list of searches
                        search_results.append(item)

                        # add to the dict by _key
                        results_dict[item["_key"]] = item

            except Exception as e:
                logging.error(f'An exception was encountered, exception="{str(e)}"')
                yield_record = {
                    "_time": time.time(),
                    "action": "failure",
                    "searches": searches_list,
                    "response": "The search failed to be executed",
                    "_raw": {
                        "action": "failure",
                        "response": "The search failed to be executed",
                        "exception": str(e),
                    },
                }
                yield yield_record

        #
        # main processing
        #

        # for entity in results_dict, get the status (object_state), if green or orange, count as up, if red, count as down and add to the associated lists
        for entity in results_dict:

            entity_record = results_dict[entity]

            # get the alias
            entity_alias = entity_record.get("alias")

            # set the summary dict
            entity_summary_dict = {}

            # for fields: object, alias, priority, object_state, status_message_json, status_description, status_description_short, tracker_name
            # if available in the entity record, add to the entity_summary_dict
            for field in [
                "tenant_id",
                "object",
                "keyid",
                "priority",
                "object_state",
                "status_message_json",
                "status_description",
                "status_description_short",
                "tracker_name",
            ]:
                if field in entity_record:
                    entity_summary_dict[field] = entity_record[field]

            # add to our summary
            results_summary_dict[results_dict[entity]["object"]] = entity_summary_dict

            count_entities += 1
            count_entities_list.append(entity_summary_dict)

            if self.consider_orange_as_up:
                if results_dict[entity]["object_state"] in ["green", "blue", "orange"]:
                    count_entities_up += 1
                    count_entities_up_list.append(entity_summary_dict)
                    results_light_list_up.append(entity_alias)
                    entity_summary_dict["converging_status"] = "up"
                elif results_dict[entity]["object_state"] == "red":
                    count_entities_down += 1
                    count_entities_down_list.append(entity_summary_dict)
                    results_light_list_down.append(entity_alias)
                    entity_summary_dict["converging_status"] = "down"
            else:
                if results_dict[entity]["object_state"] in ["green", "blue"]:
                    count_entities_up += 1
                    count_entities_up_list.append(entity_summary_dict)
                    results_light_list_up.append(entity_alias)
                    entity_summary_dict["converging_status"] = "up"
                elif results_dict[entity]["object_state"] in ["orange", "red"]:
                    count_entities_down += 1
                    count_entities_down_list.append(entity_summary_dict)
                    results_light_list_down.append(entity_alias)
                    entity_summary_dict["converging_status"] = "down"

        # Sort the lists by tenant_id and object
        def sort_key(entity):
            return (entity.get("tenant_id", ""), entity.get("object", ""))

        count_entities_down_list.sort(key=sort_key)
        count_entities_list.sort(key=sort_key)

        # Sort the results_summary_dict by tenant_id and object
        sorted_results = {}
        for key in sorted(
            results_summary_dict.keys(),
            key=lambda k: (
                results_summary_dict[k].get("tenant_id", ""),
                results_summary_dict[k].get("object", ""),
            ),
        ):
            sorted_results[key] = results_summary_dict[key]

        # Reorganize results_summary_dict with down_entities and all_entities sections
        new_results_summary_dict = {"all_entities": sorted_results}
        if count_entities_down > 0:
            new_results_summary_dict = {
                "down_entities": count_entities_down_list,
                **new_results_summary_dict,
            }
        results_summary_dict = new_results_summary_dict

        # calculate the percentage of availability
        if count_entities > 0:
            pct_availability = round((count_entities_up / count_entities) * 100, 2)

        # status (1 / 2 / 3)
        if pct_availability >= self.min_pct_for_green:
            status = 1
        elif pct_availability >= 0:
            status = 2
        else:
            status = 3

        # set the value of metrics
        metrics = {
            "pct_availability": pct_availability,
            "count_entities": count_entities,
            "count_entities_up": count_entities_up,
            "count_entities_down": count_entities_down,
        }

        # set status_description_short
        status_description_short = f"Availability={pct_availability}%, up={count_entities_up}, down={count_entities_down}"

        # set status_description
        if pct_availability == 100:
            status_description = f"The availability percentage is {pct_availability}, all {count_entities} entities are up"
        elif pct_availability > 0:
            status_description = f"The availability percentage is {pct_availability}, {count_entities_up} entities are up and {count_entities_down} entities are down: {results_light_list_down}"
        else:
            status_description = f"The availability percentage is {pct_availability}, all {count_entities} entities are down: {results_light_list_down}"

        # final results records
        # if count_entities > 0:
        if count_entities > 0:
            final_results_records = {
                "group": self.group,
                "object:": self.object,
                "object_description": self.object_description,
                "status": status,
                "status_description": status_description,
                "status_description_short": status_description_short,
                "extra_attributes": results_summary_dict,
                "metrics": metrics,
            }
        else:  # no entities found
            final_results_records = {
                "group": self.group,
                "object:": self.object,
                "object_description": self.object_description,
                "status": 3,
                "status_description": "No entities found",
                "status_description_short": "No entities found",
                "extra_attributes": {},
                "metrics": {
                    "pct_availability": 0,
                    "count_entities": 0,
                    "count_entities_up": 0,
                    "count_entities_down": 0,
                },
                "searches": searches_list,
            }

        # remove extra_attributes if requested
        if self.remove_extra_attributes:
            results_summary_dict = {}
            final_results_records["extra_attributes"] = {}

        yield_record = {
            "_time": time.time(),
            "action": "success",
            "search": flx_converging_search,
            "group": self.group,
            "object": self.object,
            "object_description": self.object_description,
            "status": status,
            "status_description": status_description,
            "status_description_short": status_description_short,
            "extra_attributes": results_summary_dict,
            "metrics": metrics,
            "_raw": final_results_records,
        }

        # yield
        yield yield_record

        # Log the run time
        logging.info(
            f'trackmesplkflxconverging has terminated, run_time={round(time.time() - start, 3)}, search="{flx_converging_search}"'
        )


dispatch(TrackMeFlxConverging, sys.argv, sys.stdin, sys.stdout, __name__)