Splunk_Deploiement/apps/trackme/bin/trackmesplkflxinactiveinspector.py

#!/usr/bin/env python
# coding=utf-8

__author__ = "TrackMe Limited"
__copyright__ = "Copyright 2022-2026, TrackMe Limited, U.K."
__credits__ = ["Guilhem Marchand"]
__license__ = "TrackMe Limited, all rights reserved"
__version__ = "0.1.0"
__maintainer__ = "TrackMe Limited, U.K."
__email__ = "support@trackme-solutions.com"
__status__ = "PRODUCTION"

# Standard library
import os
import sys
import time
import json
import uuid
import hashlib
import threading
from logging.handlers import RotatingFileHandler

# External libraries
import urllib3

# Networking imports
import requests
import urllib3

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

# Configure logging
import logging
from logging.handlers import RotatingFileHandler

# set splunkhome
splunkhome = os.environ["SPLUNK_HOME"]

# set logging
filehandler = RotatingFileHandler(
    f"{splunkhome}/var/log/splunk/trackme_splkflx_inactive_inspector.log",
    mode="a",
    maxBytes=10000000,
    backupCount=1,
)
formatter = logging.Formatter(
    "%(asctime)s %(levelname)s %(filename)s %(funcName)s %(lineno)d %(message)s"
)
logging.Formatter.converter = time.gmtime
filehandler.setFormatter(formatter)
log = logging.getLogger()  # root logger - Good to get it only once.
for hdlr in log.handlers[:]:  # remove the existing file handlers
    if isinstance(hdlr, logging.FileHandler):
        log.removeHandler(hdlr)
log.addHandler(filehandler)  # set the new handler
# set the log level to INFO, DEBUG as the default is ERROR
log.setLevel(logging.INFO)

# append current directory
sys.path.append(os.path.dirname(os.path.abspath(__file__)))

# import libs
import import_declare_test

# import Splunk libs (after lib appended)
from splunklib.searchcommands import (
    dispatch,
    GeneratingCommand,
    Configuration,
    Option,
    validators,
)

# import trackme libs (after lib appended)
from trackme_libs import (
    trackme_reqinfo,
    trackme_audit_event,
    trackme_audit_flip,
    trackme_register_tenant_object_summary,
    trackme_register_tenant_component_summary,
    trackme_gen_state,
    trackme_handler_events,
)

# import trackme libs sla
from trackme_libs_sla import trackme_sla_gen_metrics

# Import TrackMe splk-flx libs
from trackme_libs_splk_flx import trackme_flx_gen_metrics


@Configuration(distributed=False)
class SplkFlxInactiveEntitiesInspector(GeneratingCommand):
    tenant_id = Option(
        doc="""
        **Syntax:** **tenant_id=****
        **Description:** The tenant identifier.""",
        require=True,
        default=None,
    )

    register_component = Option(
        doc="""
        **Syntax:** **register_component=****
        **Description:** If the search is invoked by a tracker, register_component can be called to capture and register any execution exception.""",
        require=False,
        default=True,
    )

    report = Option(
        doc="""
        **Syntax:** **report=****
        **Description:** If register_component is set, a value for report is required.""",
        require=False,
        default=None,
        validate=validators.Match("report", r"^.*$"),
    )

    max_days_since_inactivity_before_purge = Option(
        doc="""
        **Syntax:** **max_days_since_inactivity_before_purge=****
        **Description:** value for max_days_since_inactivity_before_purge is required. (0 disabled this feature)""",
        require=False,
        default="30",
        validate=validators.Match(
            "max_days_since_inactivity_before_purge", r"^(\d|\.)*$"
        ),
    )

    """
    convert_seconds_to_duration
    behaviour: converts seconds to duration, duration is a string from as [D+]HH:MM:SS
    The first segment represents the number of days, the second the number of hours, third the number of minutes, and the fourth the number of seconds.
    """

    def convert_seconds_to_duration(self, seconds):

        try:
            original_seconds = int(seconds)
        except ValueError:
            return 0

        # Check if the original seconds were negative
        is_negative = original_seconds < 0
        seconds = abs(original_seconds)

        # Calculate days, hours, minutes, and seconds
        days = seconds // (24 * 3600)
        seconds = seconds % (24 * 3600)
        hours = seconds // 3600
        seconds %= 3600
        minutes = seconds // 60
        seconds %= 60

        # Format the duration string
        if days > 0:
            duration = f"{days}+{hours:02d}:{minutes:02d}:{seconds:02d}"
        else:
            duration = f"{hours:02d}:{minutes:02d}:{seconds:02d}"

        # Add "-" if the original seconds were negative
        if is_negative:
            duration = "-" + duration

        return duration

    """
    Function to return a unique uuid which is used to trace performance run_time of each subtask.
    """

    def get_uuid(self):
        return str(uuid.uuid4())

    """
    Queries and processes records from a collection based on specific criteria.

    :param collection: The collection object to query.
    :return: Tuple containing collection records and a dictionary of records.
    """

    def get_collection_records(self, component):

        # data_records
        collection_records = []
        collection_records_objects = set()
        collection_records_keys = set()
        collection_records_dict = {}

        params = {
            "tenant_id": self.tenant_id,
            "component": component,
            "page": 1,
            "size": 0,
        }

        # Define an header for requests authenticated communications with splunkd
        header = {
            "Authorization": f"Splunk {self._metadata.searchinfo.session_key}",
            "Content-Type": "application/json",
        }

        # Set url
        url = f"{self._metadata.searchinfo.splunkd_uri}/services/trackme/v2/component/load_component_data"

        try:
            response = requests.get(
                url,
                headers=header,
                params=params,
                verify=False,
                timeout=600,
            )

            if response.status_code not in (200, 201, 204):
                msg = f'get component has failed, response.status_code="{response.status_code}", response.text="{response.text}"'
                raise Exception(msg)

            else:
                response_json = response.json()
                data = response_json.get("data", [])

                # add the data to the data_records
                for record in data:
                    collection_records.append(record)
                    collection_records_objects.add(record.get("object"))
                    collection_records_dict[record.get("_key")] = {
                        "object": record.get("object"),
                        "object_state": record.get("object_state"),
                        "status_message": record.get("status_message", []),
                        "anomaly_reason": record.get("anomaly_reason", []),
                    }
                    collection_records_keys.add(record.get("_key"))

            return (
                collection_records,
                collection_records_objects,
                collection_records_keys,
                collection_records_dict,
            )

        except Exception as e:
            msg = f'get component has failed, exception="{str(e)}"'
            logging.error(msg)
            raise Exception(msg)

    """
    retrieve the tenant metric index
    """

    def get_tenant_metric_idx(self):
        # Define an header for requests authenticated communications with splunkd
        header = {
            "Authorization": "Splunk %s" % self._metadata.searchinfo.session_key,
            "Content-Type": "application/json",
        }

        # get the index conf for this tenant
        url = "%s/services/trackme/v2/vtenants/tenant_idx_settings" % (
            self._metadata.searchinfo.splunkd_uri
        )
        data = {"tenant_id": self.tenant_id, "idx_stanza": "trackme_metric_idx"}

        # Retrieve and set the tenant idx, if any failure, logs and use the global index
        try:
            response = requests.post(
                url,
                headers=header,
                data=json.dumps(data, indent=1),
                verify=False,
                timeout=600,
            )
            if response.status_code not in (200, 201, 204):
                error_msg = f'failed to retrieve the tenant metric index, response.status_code="{response.status_code}", response.text="{response.text}"'
                logging.error(error_msg)
                raise Exception(error_msg)
            else:
                response_data = json.loads(json.dumps(response.json(), indent=1))
                tenant_trackme_metric_idx = response_data["trackme_metric_idx"]
        except Exception as e:
            error_msg = (
                f'failed to retrieve the tenant metric index, exception="{str(e)}"'
            )
            logging.error(error_msg)
            raise Exception(error_msg)

        return tenant_trackme_metric_idx

    """
    Purge entities using the trackme API.

    :param keys_list: List of keys to be purged.
    :param system_deletion_period: The system wide auto-deletion period.
    :param instance_id: The instance identifier.
    :param task_name: The task name.
    :param task_instance_id: The task instance identifier.
    :return: None
    """

    def purge_entities(
        self,
        keys_list,
        system_deletion_period,
        instance_id,
        task_name,
        task_instance_id,
    ):

        # turn entities_to_be_deleted_csv list into CSV
        entities_to_be_deleted_csv = ",".join(keys_list)

        # endpoint target
        target_url = f"{self._metadata.searchinfo.splunkd_uri}/services/trackme/v2/splk_flx/write/flx_delete"

        # header
        header = {
            "Authorization": f"Splunk {self._metadata.searchinfo.session_key}",
            "Content-Type": "application/json",
        }

        try:
            response = requests.post(
                target_url,
                headers=header,
                data=json.dumps(
                    {
                        "tenant_id": self.tenant_id,
                        "keys_list": entities_to_be_deleted_csv,
                        "deletion_type": "temporary",
                        "update_comment": f"auto-deleted by the system, last seen data is beyond the system wide auto-deletion period of {system_deletion_period} days.",
                    }
                ),
                verify=False,
                timeout=600,
            )

            if response.status_code not in (200, 201, 204):
                msg = f'tenant_id="{self.tenant_id}", instance_id={instance_id}, task="{task_name}", task_instance_id={task_instance_id}, query has failed, response.status_code="{response.status_code}", response.text="{response.text}"'
                logging.error(msg)
                raise Exception(msg)
            else:
                try:
                    success_count = response.json().get("success_count")
                except Exception as e:
                    success_count = 0
                msg = f'tenant_id="{self.tenant_id}", instance_id={instance_id}, task="{task_name}", task_instance_id={task_instance_id}, request was successful, success_count="{success_count}"'
                logging.info(msg)
                return True

        except Exception as e:
            msg = f'tenant_id="{self.tenant_id}", instance_id={instance_id}, ctask="{task_name}", task_instance_id={task_instance_id}, request failed with exception="{str(e)}"'
            logging.error(msg)

    """
    Register the component summary.
    """

    def register_component_summary_async(
        self, session_key, splunkd_uri, tenant_id, component
    ):
        try:
            summary_register_response = trackme_register_tenant_component_summary(
                session_key,
                splunkd_uri,
                tenant_id,
                component,
            )
            logging.debug(
                f'function="trackme_register_tenant_component_summary", response="{json.dumps(summary_register_response, indent=2)}"'
            )
        except Exception as e:
            logging.error(
                f'failed to register the component summary with exception="{str(e)}"'
            )

    # main
    def generate(self, **kwargs):

        # start perf duration counter
        start = time.time()

        # Get request info and set logging level
        reqinfo = trackme_reqinfo(
            self._metadata.searchinfo.session_key,
            self._metadata.searchinfo.splunkd_uri,
        )
        log.setLevel(reqinfo["logging_level"])

        # set instance_id
        instance_id = self.get_uuid()

        # end of main
        logging.info(
            f'tenant_id="{self.tenant_id}", instance_id={instance_id}, trackmesplkflxinactiveinspector is starting'
        )

        # get metric index
        metric_index = self.get_tenant_metric_idx()

        # Get configuration and define metadata
        trackme_summary_idx = reqinfo["trackme_conf"]["index_settings"][
            "trackme_summary_idx"
        ]

        # Data collection
        data_collection_name = f"kv_trackme_flx_tenant_{self.tenant_id}"
        data_collection = self.service.kvstore[data_collection_name]

        # convert the max day in sec (purge)
        max_sec_record_age_before_purge = self.max_days_since_inactivity_before_purge
        if (
            max_sec_record_age_before_purge == "0"
            or max_sec_record_age_before_purge == 0
        ):
            max_sec_record_age_before_purge = 0

        else:
            max_sec_record_age_before_purge = (
                float(self.max_days_since_inactivity_before_purge) * 86400
            )

        # end of get configuration

        #
        # loop through the KVstore records
        #

        # call the function get collection records
        collection_records_get_start = time.time()
        (
            collection_records,
            collection_records_objects,
            collection_records_keys,
            collection_records_dict,
        ) = self.get_collection_records("flx")

        logging.info(
            f'tenant_id="{self.tenant_id}", instance_id={instance_id}, component="splk-flx", successfully retrieved {len(collection_records_keys)} records in the KVstore collection={data_collection_name}, run_time={round(time.time()-collection_records_get_start, 3)}'
        )

        # records to be processed counter
        count = 0

        # capture exceptions
        errors_count = 0
        errors_list = []

        # entities_to_be_deleted
        entities_to_be_deleted = []
        entities_to_be_deleted_dict = {}

        # counter for the number of actions engaged
        count_actions_engaged = 0

        task_start = time.time()
        task_instance_id = self.get_uuid()
        task_name = "manage_inactive_entities"

        # for the handler events
        report_objects_list = []

        logging.info(
            f'tenant_id="{self.tenant_id}", instance_id={instance_id}, task="{task_name}", task_instance_id={task_instance_id}, starting task.'
        )

        # Loop in the results
        for record in collection_records:

            # object
            object_name = record.get("object")
            object_id = record.get("_key")

            # entity_is_inactive record boolean
            entity_is_inactive = False

            # get the age in seconds since the latest execution
            sec_since_last_execution = time.time() - float(
                record.get("tracker_runtime")
            )

            # get status (handle tracker-keyed JSON format from concurrent trackers)
            status_value = record.get("status")
            try:
                if isinstance(status_value, str):
                    try:
                        status_by_tracker = json.loads(status_value)
                        if isinstance(status_by_tracker, dict):
                            # Tracker-keyed format: apply worst-status logic (2 > 3 > 1)
                            status_values = list(status_by_tracker.values())
                            if status_values:
                                if 2 in status_values:
                                    record_status = 2  # Red
                                elif 3 in status_values:
                                    record_status = 3  # Orange
                                else:
                                    record_status = 1  # Green
                            else:
                                record_status = 1  # Default to green if empty
                        else:
                            # Not a dict, treat as old format integer string
                            record_status = int(status_value)
                    except (json.JSONDecodeError, TypeError):
                        # If parsing fails, treat as old format integer string
                        record_status = int(status_value)
                elif isinstance(status_value, dict):
                    # Already a dict (tracker-keyed format)
                    status_values = list(status_value.values())
                    if status_values:
                        if 2 in status_values:
                            record_status = 2  # Red
                        elif 3 in status_values:
                            record_status = 3  # Orange
                        else:
                            record_status = 1  # Green
                    else:
                        record_status = 1  # Default to green if empty
                else:
                    # Simple integer value
                    record_status = int(status_value)
            except (ValueError, TypeError) as e:
                logging.error(
                    f'instance_id="{instance_id}", failed to parse status for object="{object_name}", '
                    f'value="{status_value}", exception="{str(e)}"'
                )
                record_status = 1  # Default to green on error

            # get current_state
            record_current_state = record.get("current_state")

            # attempt to retrieve max_sec_inactive on a per record basic
            max_sec_inactive = record.get("max_sec_inactive", None)
            if not max_sec_inactive:
                max_sec_inactive = 0
                # If max_sec_inactive is 0, skip updating this record
            else:
                # Handle tracker-keyed JSON format (from concurrent trackers)
                try:
                    if isinstance(max_sec_inactive, str):
                        try:
                            parsed = json.loads(max_sec_inactive)
                            if isinstance(parsed, dict):
                                # Tracker-keyed format: extract minimum value (excluding 0)
                                non_zero_values = [float(v) for v in parsed.values() if float(v) > 0]
                                if non_zero_values:
                                    max_sec_inactive = min(non_zero_values)
                                else:
                                    # All values are 0, use 0 (disabled)
                                    max_sec_inactive = 0
                            else:
                                # Not a dict, treat as simple numeric string
                                max_sec_inactive = float(max_sec_inactive)
                        except (json.JSONDecodeError, TypeError, ValueError):
                            # Not JSON, treat as simple numeric string
                            max_sec_inactive = float(max_sec_inactive)
                    elif isinstance(max_sec_inactive, dict):
                        # Already a dict (tracker-keyed format)
                        non_zero_values = [float(v) for v in max_sec_inactive.values() if float(v) > 0]
                        if non_zero_values:
                            max_sec_inactive = min(non_zero_values)
                        else:
                            max_sec_inactive = 0
                    else:
                        # Simple numeric value
                        max_sec_inactive = float(max_sec_inactive)
                except (ValueError, TypeError) as e:
                    logging.error(
                        f'instance_id="{instance_id}", failed to parse max_sec_inactive for object="{object_name}", '
                        f'value="{record.get("max_sec_inactive")}", exception="{str(e)}"'
                    )
                    max_sec_inactive = 0

            #
            # update inactive entities status
            #

            # A record will be processed only if max_sec_inactive is more than 0
            if max_sec_inactive > 0:
                entity_is_inactive = True

            if entity_is_inactive:

                # if the entity is inactive and already red, generate the status metrics
                if (
                    float(sec_since_last_execution) > max_sec_inactive
                    and record_status == 2
                ):

                    try:
                        trackme_flx_gen_metrics(
                            time.time(),
                            self.tenant_id,
                            object_name,
                            object_id,
                            metric_index,
                            json.dumps({"status": 2}),
                        )
                    except Exception as e:
                        error_msg = f'Failed to call trackme_flx_gen_metrics with exception="{str(e)}"'
                        logging.error(error_msg)

                    # add to the report_objects_list if not already present
                    if object_name not in report_objects_list:
                        report_objects_list.append(object_name)

                # if the entity is inactive but does not yet have a status red
                elif (
                    float(sec_since_last_execution) > max_sec_inactive
                    and record_status != 2
                ):
                    count += 1

                    # add to the report_objects_list if not already present
                    if object_name not in report_objects_list:
                        report_objects_list.append(object_name)

                    duration_since_last_execution = self.convert_seconds_to_duration(
                        sec_since_last_execution
                    )

                    # append object_name to the report_objects_list if not already present
                    if object_name not in report_objects_list:
                        report_objects_list.append(object_name)

                    # Update the record
                    record["object_state"] = "red"
                    record["current_state"] = "red"
                    record["status"] = 2
                    record["status_message"] = (
                        f"This entity has been inactive for more than {duration_since_last_execution} (D+HH:MM:SS) and was not actively managed by any tracker, its status was updated automatically by the inactive entities tracker"
                    )
                    record["status_description_short"] = (
                        "entity is red due to inactivity"
                    )
                    record["status_description"] = (
                        f"The entity status is red due to inactivity, it was not actively managed by any tracker for more than {duration_since_last_execution} (D+HH:MM:SS)"
                    )
                    record["anomaly_reason"] = "inactive"
                    flip_previous_state = record.get("latest_flip_state")
                    record["latest_flip_state"] = "red"
                    record["latest_flip_time"] = time.time()

                    try:
                        data_collection.data.update(
                            str(record.get("_key")), json.dumps(record)
                        )
                        logging.info(
                            f'successfully updated record="{json.dumps(record, indent=2)}"'
                        )
                        count_actions_engaged += 1

                        # flip
                        new_latest_flip_state = "red"
                        new_latest_flip_time = time.time()
                        new_flip_time_human = time.strftime(
                            "%d/%m/%Y %H:%M:%S",
                            time.localtime(new_latest_flip_time),
                        )
                        flip_result = f'{new_flip_time_human}, object="{object_name}" has flipped from previous_state="{flip_previous_state}" to state="{new_latest_flip_state}" with anomaly_reason="inactive"'

                        # Generate the flipping state event
                        try:
                            trackme_audit_flip(
                                self._metadata.searchinfo.session_key,
                                reqinfo["server_rest_uri"],
                                tenant_id=self.tenant_id,
                                keyid=object_id,
                                alias=record.get("alias"),
                                object=object_name,
                                object_category="splk-flx",
                                priority=record.get("priority"),
                                object_state="red",
                                object_previous_state=flip_previous_state,
                                latest_flip_time=new_latest_flip_time,
                                latest_flip_state=new_latest_flip_state,
                                anomaly_reason="inactive",
                                result=flip_result,
                            )
                        except Exception as e:
                            logging.error(
                                f'tenant_id="{self.tenant_id}", object="{object_name}", failed to generate a flipping state event with exception="{str(e)}"'
                            )

                        # Generate the audit event
                        trackme_audit_event(
                            self._metadata.searchinfo.session_key,
                            reqinfo["server_rest_uri"],
                            self.tenant_id,
                            "system",
                            "success",
                            "update status inactive entity",
                            object_name,
                            "splk-flx",
                            str(json.dumps(record, indent=1)),
                            "Entity was updated successfully",
                            "automated management of inactive entities",
                        )

                        # Generate the summary event
                        record["event_id"] = hashlib.sha256(
                            json.dumps(record).encode()
                        ).hexdigest()
                        try:
                            trackme_gen_state(
                                index=trackme_summary_idx,
                                sourcetype="trackme:state",
                                source=f"current_state_tracking:splk-flx:{self.tenant_id}",
                                event=record,
                            )
                            logging.debug(
                                f'TrackMe summary event created successfully, record="{json.dumps(record, indent=1)}"'
                            )
                        except Exception as e:
                            logging.error(
                                f'TrackMe summary event creation failure, record="{json.dumps(record, indent=1)}", exception="{str(e)}"'
                            )

                        # Generate SLA metrics
                        sla_record = [
                            {
                                "tenant_id": self.tenant_id,
                                "object_id": object_id,
                                "object": object_name,
                                "alias": record.get("alias"),
                                "object_category": "splk-flx",
                                "monitored_state": record.get("current_state"),
                                "priority": record.get("priority"),
                                "metrics_event": {"object_state": 2},
                            }
                        ]

                        try:
                            sla_metrics = trackme_sla_gen_metrics(
                                self.tenant_id,
                                metric_index,
                                sla_record,
                            )
                            logging.debug(
                                f'context="sla_gen_metrics", tenant_id="{self.tenant_id}", object="{object_name}, function trackme_sla_gen_metrics was successful'
                            )
                        except Exception as e:
                            logging.error(
                                f'context="sla_gen_metrics", tenant_id="{self.tenant_id}", object="{object_name}, function trackme_sla_gen_metrics failed with exception {str(e)}'
                            )

                        # generate the status metric
                        try:
                            trackme_flx_gen_metrics(
                                time.time(),
                                self.tenant_id,
                                object_name,
                                object_id,
                                metric_index,
                                json.dumps({"status": 2}),
                            )
                        except Exception as e:
                            error_msg = f'Failed to call trackme_flx_gen_metrics with exception="{str(e)}"'
                            logging.error(error_msg)

                        # yield record
                        yield_record = {
                            "_time": time.time(),
                            "_raw": {
                                "response": f"processing with record update, _key={record.get('_key')}, object={object_name}",
                                "action": "success",
                                "record": record,
                            },
                        }

                        yield yield_record

                    except Exception as e:
                        error_msg = f'An exception was encountered while attempting to update the KVstore record, exception="{str(e)}", record="{json.dumps(record, indent=2)}"'
                        errors_count += 1
                        errors_list.append(error_msg)
                        yield_record = {
                            "_time": time.time(),
                            "_raw": {
                                "response": f"processing with record deletion, _key={record.get('_key')}, object={object_name}",
                                "action": "failure",
                                "exception": str(e),
                                "record": record,
                            },
                        }

                        yield yield_record

                # if the entity is already red but inactive, gen a summary event
                elif float(sec_since_last_execution) > max_sec_inactive:
                    if record_current_state != "red":
                        try:
                            record["current_state"] = "red"
                            data_collection.data.update(
                                str(record.get("_key")), json.dumps(record)
                            )
                            logging.info(
                                f'successfully updated record="{json.dumps(record, indent=2)}"'
                            )
                            count_actions_engaged += 1
                        except Exception as e:
                            error_msg = f'An exception was encountered while attempting to update the KVstore record, exception="{str(e)}", record="{json.dumps(record, indent=2)}"'
                            errors_count += 1
                            errors_list.append(error_msg)
                            yield_record = {
                                "_time": time.time(),
                                "_raw": {
                                    "response": f"processing with record update, _key={record.get('_key')}, object={object_name}",
                                    "action": "failure",
                                    "exception": str(e),
                                    "record": record,
                                },
                            }

                            yield yield_record

                    # sunnary state
                    record["event_id"] = hashlib.sha256(
                        json.dumps(record).encode()
                    ).hexdigest()
                    try:
                        trackme_gen_state(
                            index=trackme_summary_idx,
                            sourcetype="trackme:state",
                            source=f"current_state_tracking:splk-flx:{self.tenant_id}",
                            event=record,
                        )
                        logging.debug(
                            f'TrackMe summary event created successfully, record="{json.dumps(record, indent=1)}"'
                        )
                    except Exception as e:
                        logging.error(
                            f'TrackMe summary event creation failure, record="{json.dumps(record, indent=1)}", exception="{str(e)}"'
                        )

            #
            # purge inactive entities
            #

            # if the requested conditions are met
            if max_sec_record_age_before_purge > 0:

                if float(sec_since_last_execution) > max_sec_record_age_before_purge:
                    count += 1

                    # append the key to the entities_to_be_deleted list
                    logging.info(
                        f'tenant_id="{self.tenant_id}", instance_id={instance_id}, task="{task_name}", task_instance_id={task_instance_id}, object="{record.get("object")}", this entity has been inactive for too long and will be purged automatically. (last seen update {round(sec_since_last_execution, 3)} is beyond the system wide auto-deletion period of {max_sec_record_age_before_purge} seconds)'
                    )
                    entities_to_be_deleted.append(record.get("_key"))
                    # add to a dict for yield and logging purposes
                    entities_to_be_deleted_dict[record.get("_key")] = {
                        "object": record.get("object"),
                        "object_state": record.get("object_state"),
                        "status_message": record.get("status_message", []),
                    }

                else:
                    logging.debug(
                        f'tenant_id="{self.tenant_id}", instance_id={instance_id}, task="{task_name}", task_instance_id={task_instance_id}, object="{record.get("object")}", this entity is active and will not be purged. (last seen update: {round(sec_since_last_execution, 3)} is not beyond the system wide auto-deletion period of {max_sec_record_age_before_purge} seconds)'
                    )

        # end task
        logging.info(
            f'tenant_id="{self.tenant_id}", instance_id={instance_id}, task="{task_name}", task_instance_id={task_instance_id}, run_time="{round(time.time()-task_start, 3)}", task has terminated.'
        )

        #
        # purge inactive entities
        #

        task_start = time.time()
        task_instance_id = self.get_uuid()
        task_name = "purge_inactive_entities"

        logging.info(
            f'tenant_id="{self.tenant_id}", instance_id={instance_id}, task="{task_name}", task_instance_id={task_instance_id}, starting task.'
        )

        # set a global success flag for the mass deletion
        mass_deletion_success = False

        if len(entities_to_be_deleted) > 0:

            try:
                entities_purge_response = self.purge_entities(
                    entities_to_be_deleted,
                    max_sec_record_age_before_purge,
                    instance_id,
                    task_name,
                    task_instance_id,
                )
                mass_deletion_success = True
                count_actions_engaged += len(entities_to_be_deleted)

            except Exception as e:
                mass_deletion_success = False
                error_msg = f'An exception was encountered while attempting to purge the entities, exception="{str(e)}"'
                errors_count += 1
                errors_list.append(error_msg)
                logging.error(error_msg)

        # yield the entities to be deleted
        for key, entity in entities_to_be_deleted_dict.items():

            # append object_name to the report_objects_list if not already present
            if entity.get("object") not in report_objects_list:
                report_objects_list.append(entity.get("object"))

            yield_record = {
                "_time": time.time(),
                "_raw": {
                    "response": f'processed with record deletion attempt, _key={key}, object={entity.get("object")}',
                    "record": entity,
                    "success": mass_deletion_success,
                },
            }

            yield yield_record

        logging.info(
            f'tenant_id="{self.tenant_id}", instance_id={instance_id}, task="{task_name}", task_instance_id={task_instance_id}, run_time="{round(time.time()-task_start, 3)}", task has terminated.'
        )

        # handler event
        if report_objects_list:

            # if report_objects_list is a string (a single object was reported), convert it to a list
            if isinstance(report_objects_list, str):
                report_objects_list = [report_objects_list]

            handler_events_records = []
            for object_name in report_objects_list:
                # Find the object_id (key) for this object from collection_records_dict
                object_id = None
                for key, data in collection_records_dict.items():
                    if data.get("object") == object_name:
                        object_id = key
                        break

                handler_events_records.append(
                    {
                        "object": object_name,
                        "object_id": object_id,
                        "object_category": "splk-flx",
                        "handler": self.report,
                        "handler_message": "Entity was inspected by the Flex inactive entities inspector, which tracks entities that are not actively managed by any tracker.",
                        "handler_troubleshoot_search": f"index=_internal sourcetype=trackme:custom_commands:trackmetrackerexecutor tenant_id={self.tenant_id} report={self.report}",
                        "handler_time": time.time(),
                    }
                )

            # notification event
            try:
                trackme_handler_events(
                    session_key=self._metadata.searchinfo.session_key,
                    splunkd_uri=self._metadata.searchinfo.splunkd_uri,
                    tenant_id=self.tenant_id,
                    sourcetype="trackme:handler",
                    source=f"trackme:handler:{self.tenant_id}",
                    handler_events=handler_events_records,
                )
            except Exception as e:
                logging.error(
                    f'tenant_id="{self.tenant_id}", component="splk-flx", could not send notification event, exception="{e}"'
                )

        #
        # check job status
        #

        task_start = time.time()
        task_instance_id = self.get_uuid()
        task_name = "check_job_status"

        logging.info(
            f'tenant_id="{self.tenant_id}", instance_id={instance_id}, task="{task_name}", task_instance_id={task_instance_id}, starting task.'
        )

        # capture the job status
        if errors_count > 0:
            if self.register_component and self.tenant_id and self.report:
                try:
                    trackme_register_tenant_object_summary(
                        self._metadata.searchinfo.session_key,
                        self._metadata.searchinfo.splunkd_uri,
                        self.tenant_id,
                        "splk-flx",
                        self.report,
                        "failure",
                        time.time(),
                        round(time.time() - start, 3),
                        errors_list,
                        "-5m",
                        "now",
                    )
                except Exception as e:
                    logging.error(
                        f'tenant_id="{self.tenant_id}", instance_id={instance_id}, component="splk-flx", Failed to call trackme_register_tenant_object_summary with exception="{str(e)}"'
                    )
        else:
            if self.register_component and self.tenant_id and self.report:
                try:
                    trackme_register_tenant_object_summary(
                        self._metadata.searchinfo.session_key,
                        self._metadata.searchinfo.splunkd_uri,
                        self.tenant_id,
                        "splk-flx",
                        self.report,
                        "success",
                        time.time(),
                        round(time.time() - start, 3),
                        "the job was executed successfully",
                        "-5m",
                        "now",
                    )
                except Exception as e:
                    logging.error(
                        f'tenant_id="{self.tenant_id}", instance_id={instance_id}, component="splk-flx", Failed to call trackme_register_tenant_object_summary with exception="{str(e)}"'
                    )

        logging.info(
            f'tenant_id="{self.tenant_id}", instance_id={instance_id}, task="{task_name}", task_instance_id={task_instance_id}, run_time="{round(time.time()-task_start, 3)}", task has terminated.'
        )

        #
        # Call the trackme_register_tenant_component_summary
        #

        if count_actions_engaged > 0:

            # Use threading to do an async call to the register summary without waiting for it to complete
            thread = threading.Thread(
                target=self.register_component_summary_async,
                args=(
                    self._metadata.searchinfo.session_key,
                    self._metadata.searchinfo.splunkd_uri,
                    self.tenant_id,
                    "splk-flx",
                ),
            )
            thread.start()

        #
        # process
        #

        if not count > 0:
            yield_record = {
                "_time": time.time(),
                "_raw": {
                    "response": "There are not records to be processed at the moment, nothing to do.",
                    "action": "success",
                },
            }

            yield yield_record

        # end of main
        logging.info(
            f'tenant_id="{self.tenant_id}", instance_id={instance_id}, trackmesplkflxinactiveinspector has terminated, run_time="{round(time.time() - start, 3)}"'
        )


dispatch(SplkFlxInactiveEntitiesInspector, sys.argv, sys.stdin, sys.stdout, __name__)