Splunk_Deploiement/apps/trackme/bin/trackmeexpandsplkmhm.py

#!/usr/bin/env python
# coding=utf-8

__author__ = "TrackMe Limited"
__copyright__ = "Copyright 2022-2026, TrackMe Limited, U.K."
__credits__ = "TrackMe Limited, U.K."
__license__ = "TrackMe Limited, all rights reserved"
__version__ = "0.1.0"
__maintainer__ = "TrackMe Limited, U.K."
__email__ = "support@trackme-solutions.com"
__status__ = "PRODUCTION"

# Standard library imports
import os
import sys
import time
import json
import ast

# Logging imports
import logging
from logging.handlers import RotatingFileHandler

# Networking imports
import urllib3

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

# splunk home
splunkhome = os.environ["SPLUNK_HOME"]

# set logging
filehandler = RotatingFileHandler(
    "%s/var/log/splunk/trackme_expand_splk_mhm.log" % splunkhome,
    mode="a",
    maxBytes=10000000,
    backupCount=1,
)
formatter = logging.Formatter(
    "%(asctime)s %(levelname)s %(filename)s %(funcName)s %(lineno)d %(message)s"
)
logging.Formatter.converter = time.gmtime
filehandler.setFormatter(formatter)
log = logging.getLogger()  # root logger - Good to get it only once.
for hdlr in log.handlers[:]:  # remove the existing file handlers
    if isinstance(hdlr, logging.FileHandler):
        log.removeHandler(hdlr)
log.addHandler(filehandler)  # set the new handler
# set the log level to INFO, DEBUG as the default is ERROR
log.setLevel(logging.INFO)

# append current directory
sys.path.append(os.path.dirname(os.path.abspath(__file__)))

# import libs
import import_declare_test

# import Splunk libs
from splunklib.searchcommands import (
    dispatch,
    StreamingCommand,
    Configuration,
    Option,
    validators,
)

# Import trackme libs
from trackme_libs import trackme_reqinfo


@Configuration(distributed=False)
class TrackMeMergeSplkDhm(StreamingCommand):
    chunk_size = Option(
        doc="""
        **Syntax:** **chunk_size=****
        **Description:** Specify the number of records to be processed in a single chunk, defaults to 10K records.""",
        require=False,
        default=10000,
        validate=validators.Match("chunk_size", r"^\d*$"),
    )

    field_current = Option(
        doc="""
        **Syntax:** **field_current=****
        **Description:** field name containing the current object dictionnary.""",
        require=True,
    )

    # status will be statically defined as imported

    def stream(self, records):
        # Get request info and set logging level
        reqinfo = trackme_reqinfo(
            self._metadata.searchinfo.session_key, self._metadata.searchinfo.splunkd_uri
        )
        log.setLevel(reqinfo["logging_level"])

        # empty array to store our processed records
        records_list = []

        # Loop in the results
        records_count = 0
        for splrecord in records:
            # increment
            records_count += 1

            # get fields and values
            current_dict = str(splrecord[self.field_current])

            # handle the raw records
            rawdict = splrecord

            # remove fields not needed any longer
            del rawdict[self.field_current]

            # Create the record to be added to our array of processed events
            record = {
                "object": str(splrecord["object"]),
                "current_dict": str(current_dict),
                "rawdict": str(rawdict),
            }

            # logging debug
            logging.info(f'host="{splrecord["object"]}"')
            logging.debug(f'downstream record="{json.dumps(record, indent=1)}"')

            # Append to the records array
            records_list.append(record)

        ################################
        # Export to the table by chunk #
        ################################

        # total number of messages to be processed
        results_count = len(records_list)
        logging.info(
            f'number of records to be processed, record_count="{results_count}"'
        )

        # process by chunk
        chunks = [
            records_list[i : i + int(self.chunk_size)]
            for i in range(0, len(records_list), int(self.chunk_size))
        ]
        for chunk in chunks:
            for subrecord in chunk:
                # attempt to get the current_dict
                try:
                    current_dict = ast.literal_eval(subrecord.get("current_dict"))
                except Exception as e:
                    current_dict = None

                rawdict = subrecord.get("rawdict")

                # if we have both
                if current_dict:
                    # loop through the records
                    for p_id, p_info in current_dict.items():
                        yield {
                            "_time": time.time(),
                            "_raw": json.dumps(subrecord, indent=1),
                            "object": subrecord["object"],
                            "metric_index": p_info["idx"],
                            "metric_category": p_info["metric_category"],
                            "metric_last_time_seen": time.strftime(
                                "%d %b %Y %H:%M:%S",
                                time.localtime(int(p_info["last_time"])),
                            ),
                            "metric_last_lag_seen": p_info["last_metric_lag"],
                            "time_measure": time.strftime(
                                "%d %b %Y %H:%M:%S",
                                time.localtime(int(p_info["time_measure"])),
                            ),
                            "metric_max_lag_allowed": p_info["lag_allowed"],
                            "state": p_info["state"],
                        }


dispatch(TrackMeMergeSplkDhm, sys.argv, sys.stdin, sys.stdout, __name__)