Splunk_Deploiement/apps/trackme/bin/trackmesplkfeedsdelayedinspector.py

#!/usr/bin/env python
# coding=utf-8

__author__ = "TrackMe Limited"
__copyright__ = "Copyright 2022-2026, TrackMe Limited, U.K."
__credits__ = "TrackMe Limited, U.K."
__license__ = "TrackMe Limited, all rights reserved"
__version__ = "0.1.0"
__maintainer__ = "TrackMe Limited, U.K."
__email__ = "support@trackme-solutions.com"
__status__ = "PRODUCTION"

# Standard library imports
import json
import logging
import os
import sys
import time
import requests

# Third-party library imports
import urllib3
from logging.handlers import RotatingFileHandler

# Disable insecure request warnings for urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

# set splunkhome
splunkhome = os.environ["SPLUNK_HOME"]

# set logging
filehandler = RotatingFileHandler(
    "%s/var/log/splunk/trackme_splk_feeds_delayed_inspector.log" % splunkhome,
    mode="a",
    maxBytes=10000000,
    backupCount=1,
)
formatter = logging.Formatter(
    "%(asctime)s %(levelname)s %(filename)s %(funcName)s %(lineno)d %(message)s"
)
logging.Formatter.converter = time.gmtime
filehandler.setFormatter(formatter)
log = logging.getLogger()  # root logger - Good to get it only once.
for hdlr in log.handlers[:]:  # remove the existing file handlers
    if isinstance(hdlr, logging.FileHandler):
        log.removeHandler(hdlr)
log.addHandler(filehandler)  # set the new handler
# set the log level to INFO, DEBUG as the default is ERROR
log.setLevel(logging.INFO)

# append current directory
sys.path.append(os.path.dirname(os.path.abspath(__file__)))

# import libs
import import_declare_test

# Import Splunk libs
from splunklib.searchcommands import (
    dispatch,
    GeneratingCommand,
    Configuration,
    Option,
    validators,
)

# Import trackme libs
from trackme_libs import (
    trackme_reqinfo,
    run_splunk_search,
    trackme_vtenant_component_info,
    trackme_register_tenant_object_summary,
    trackme_idx_for_tenant,
    trackme_handler_events,
)

# import splk-feeds
from trackme_libs_splk_feeds import (
    generate_dsm_report_search,
    generate_dhm_report_search,
)

# import trackme libs croniter
from trackme_libs_croniter import cron_to_seconds


@Configuration(distributed=False)
class TrackMeFeedsDelayedInspector(GeneratingCommand):

    tenant_id = Option(
        doc="""
        **Syntax:** **tenant_id=****
        **Description:** The value for tenant_id.""",
        require=True,
        validate=validators.Match("tenant_id", r"^.*$"),
    )

    component = Option(
        doc="""
        **Syntax:** **component=****
        **Description:** The component category.""",
        require=True,
        default=None,
        validate=validators.Match("component", r"^(?:dsm|dhm|flx|wlk)$"),
    )

    max_runtime = Option(
        doc="""
        **Syntax:** **max_runtime=****
        **Description:** Optional, The max value in seconds for the total runtime of the job, defaults to 900 (15 min) which is substracted by 120 sec of margin. Once the job reaches this, it gets terminated""",
        require=False,
        default="900",
        validate=validators.Match("max_runtime", r"^\d*$"),
    )

    max_errors_count_per_entity_search = Option(
        doc="""
        **Syntax:** **max_errors_count_per_entity_search=****
        **Description:** Optional, The maximum number of errors allowed per entity search, defaults to 3.
        """,
        require=False,
        default="3",
        validate=validators.Match("max_errors_count_per_entity_search", r"^\d*$"),
    )

    object_name = Option(
        doc="""
        **Syntax:** **object_name=****
        **Description:** Optional, The object name.""",
        require=False,
        default=None,
    )

    """
    Function to check if we have a record in the delayed inspector KV collection based
    on the _key field, and return the record if found, otherwise return empty dict
    """

    def get_delayed_inspector_record(
        self, delayed_inspectodelayed_inspector_collection, _key
    ):

        # check if we have a KVrecord already for this object
        query_string = {
            "$and": [
                {
                    "_key": _key,
                }
            ]
        }

        # record from the component
        try:
            kvrecord = delayed_inspectodelayed_inspector_collection.data.query(
                query=(json.dumps(query_string))
            )[0]
        except Exception as e:
            kvrecord = {}

        return kvrecord

    """
    Function to return the range category appartenance based on the data_last_lag_seen value and the delayed inspector configuration

    data_last_lag_seen: int
    splk_feeds_auto_disablement_period: str

    # behavior:
    - if the data_last_lag_seen is less than 24 hours, we return "24h"
    - if the data_last_lag_seen is between 24 hours and 7 days, we return "7d"
    - if the data_last_lag_seen is between 7 days and the auto disablement period, we return "until_disabled"
    - if the data_last_lag_seen is greater than the auto disablement period, we return "do_not_proceed"

    # returns:
    - str: The range category
    - str: The entity search earliest time
    - str: The span value, the longer the period of the search, the less granular the search
    """

    def get_range_category(
        self,
        data_last_lag_seen,
        splk_feeds_auto_disablement_period,
    ):
        # extract the number of days from splk_feeds_auto_disablement_period (format ex: 30d)
        splk_feeds_auto_disablement_period_days = int(
            splk_feeds_auto_disablement_period.split("d")[0]
        )

        if data_last_lag_seen < 3600 * 24:
            return "24h", "-24h", "1m"
        elif data_last_lag_seen >= 3600 * 24 and data_last_lag_seen < 3600 * 24 * 7:
            return "7d", "-7d", "5m"
        elif (
            data_last_lag_seen >= 3600 * 24 * splk_feeds_auto_disablement_period_days
            and data_last_lag_seen < 3600 * 24 * splk_feeds_auto_disablement_period_days
        ) and splk_feeds_auto_disablement_period != "0d":
            return (
                "until_disabled",
                f"-{splk_feeds_auto_disablement_period_days}d",
                "1d",
            )
        else:
            return "do_not_proceed", None, None

    """
    Function to define the proceed entity boolean depending on the range category and the last inspection time

    range_category: str
    last_inspection_time: int
    splk_feeds_delayed_inspector_24hours_range_min_sec: int
    splk_feeds_delayed_inspector_7days_range_min_sec: int
    splk_feeds_delayed_inspector_until_disabled_range_min_sec: int
    splk_feeds_auto_disablement_period: str

    # behavior:
    - if the range category is "24h" and the last inspection time is greater than the 24h range min sec, we proceed
    - if the range category is "7d" and the last inspection time is greater than the 7d range min sec, we proceed
    - if the range category is "until_disabled" and the last inspection time is greater than the until_disabled range min sec, we proceed
    - if the range category is "do_not_proceed", we do not proceed

    # returns:
    - bool: True if the entity should be proceeded, False otherwise
    - reaons: A string message explaining the reason for the proceed_entity_bool value
    """

    def define_proceed_entity_bool(
        self,
        range_category,
        last_inspection_time,
        splk_feeds_delayed_inspector_24hours_range_min_sec,
        splk_feeds_delayed_inspector_7days_range_min_sec,
        splk_feeds_delayed_inspector_until_disabled_range_min_sec,
    ):
        reason = ""
        proceed_entity_bool = False

        # If any of the range minimum seconds values are 0, do not proceed
        if (
            range_category == "24h"
            and splk_feeds_delayed_inspector_24hours_range_min_sec == 0
        ):
            reason = "The delayed inspector is disabled for the 24h range category (splk_feeds_delayed_inspector_24hours_range_min_sec is set to 0)"
            return proceed_entity_bool, reason
        elif (
            range_category == "7d"
            and splk_feeds_delayed_inspector_7days_range_min_sec == 0
        ):
            reason = "The delayed inspector is disabled for the 7d range category (splk_feeds_delayed_inspector_7days_range_min_sec is set to 0)"
            return proceed_entity_bool, reason
        elif (
            range_category == "until_disabled"
            and splk_feeds_delayed_inspector_until_disabled_range_min_sec == 0
        ):
            reason = "The delayed inspector is disabled for the until_disabled range category (splk_feeds_delayed_inspector_until_disabled_range_min_sec is set to 0)"
            return proceed_entity_bool, reason

        if range_category == "24h":

            if last_inspection_time == 0:
                proceed_entity_bool = True
                reason = f"The entity is within the 24h range category and the last inspection time {last_inspection_time} is 0"
            elif (
                last_inspection_time
                > splk_feeds_delayed_inspector_24hours_range_min_sec
            ):
                proceed_entity_bool = True
                reason = f"The entity is within the 24h range category and the last inspection time {last_inspection_time} is greater than the 24h range min sec {splk_feeds_delayed_inspector_24hours_range_min_sec}"
            else:
                reason = f"The entity is within the 24h range category but the last inspection time {last_inspection_time} is less than the 24h range min sec {splk_feeds_delayed_inspector_24hours_range_min_sec}"

        elif range_category == "7d":

            if last_inspection_time == 0:
                proceed_entity_bool = True
                reason = f"The entity is within the 7d range category and the last inspection time {last_inspection_time} is 0"
            elif (
                last_inspection_time > splk_feeds_delayed_inspector_7days_range_min_sec
            ):
                proceed_entity_bool = True
                reason = f"The entity is within the 7d range category and the last inspection time {last_inspection_time} is greater than the 7d range min sec {splk_feeds_delayed_inspector_7days_range_min_sec}"
            else:
                reason = f"The entity is within the 7d range category but the last inspection time {last_inspection_time} is less than the 7d range min sec {splk_feeds_delayed_inspector_7days_range_min_sec}"

        elif range_category == "until_disabled":

            if last_inspection_time == 0:
                proceed_entity_bool = True
                reason = f"The entity is within the until_disabled range category and the last inspection time {last_inspection_time} is 0"
            elif (
                last_inspection_time
                > splk_feeds_delayed_inspector_until_disabled_range_min_sec
            ):
                proceed_entity_bool = True
                reason = f"The entity is within the until_disabled range category and the last inspection time {last_inspection_time} is greater than the until_disabled range min sec {splk_feeds_delayed_inspector_until_disabled_range_min_sec}"
            else:
                reason = f"The entity is within the until_disabled range category but the last inspection time {last_inspection_time} is less than the until_disabled range min sec {splk_feeds_delayed_inspector_until_disabled_range_min_sec}"

        else:
            reason = f"The entity is not within any range category, the range category is {range_category}"

        return proceed_entity_bool, reason

    def generate(self, **kwargs):
        # Start performance counter
        start = time.time()

        # Get request info and set logging level
        reqinfo = trackme_reqinfo(
            self._metadata.searchinfo.session_key, self._metadata.searchinfo.splunkd_uri
        )
        log.setLevel(reqinfo["logging_level"])

        # get vtenant component info
        vtenant_component_info = trackme_vtenant_component_info(
            self._metadata.searchinfo.session_key,
            self._metadata.searchinfo.splunkd_uri,
            self.tenant_id,
        )
        logging.debug(
            f'tenant_id="{self.tenant_id}", component="splk-{self.component}", vtenant_component_info="{json.dumps(vtenant_component_info, indent=2)}"'
        )

        # get configuration values for the delayed inspector
        splk_feeds_delayed_inspector_24hours_range_min_sec = int(
            vtenant_component_info["splk_feeds_delayed_inspector_24hours_range_min_sec"]
        )
        splk_feeds_delayed_inspector_7days_range_min_sec = int(
            vtenant_component_info["splk_feeds_delayed_inspector_7days_range_min_sec"]
        )
        splk_feeds_delayed_inspector_until_disabled_range_min_sec = int(
            vtenant_component_info[
                "splk_feeds_delayed_inspector_until_disabled_range_min_sec"
            ]
        )
        splk_feeds_auto_disablement_period = str(
            vtenant_component_info["splk_feeds_auto_disablement_period"]
        )

        # check schema version migration state
        try:
            schema_version = int(vtenant_component_info["schema_version"])
            schema_version_upgrade_in_progress = bool(
                int(vtenant_component_info["schema_version_upgrade_in_progress"])
            )
            logging.debug(
                f'schema_version_upgrade_in_progress="{schema_version_upgrade_in_progress}"'
            )
        except Exception as e:
            schema_version = 0
            schema_version_upgrade_in_progress = False
            logging.error(
                f'failed to retrieve schema_version_upgrade_in_progress=, exception="{str(e)}"'
            )

        # Do not proceed if the schema version upgrade is in progress
        if schema_version_upgrade_in_progress:
            yield_json = {
                "_time": time.time(),
                "tenant_id": self.tenant_id,
                "component": self.component,
                "response": f'tenant_id="{self.tenant_id}", schema upgrade is currently in progress, we will wait until the process is completed before proceeding, the schema upgrade is handled by the health_tracker of the tenant and is completed once the schema_version field of the Virtual Tenants KVstore (trackme_virtual_tenants) matches TrackMe\'s version, schema_version="{schema_version}", schema_version_upgrade_in_progress="{schema_version_upgrade_in_progress}"',
                "schema_version": schema_version,
                "schema_version_upgrade_in_progress": schema_version_upgrade_in_progress,
            }
            logging.info(json.dumps(yield_json, indent=2))
            yield {
                "_time": yield_json["_time"],
                "_raw": yield_json,
            }

        # log start
        logging.info(
            f'tenant_id="{self.tenant_id}", component="splk-{self.component}", starting delayed entities inspector'
        )

        # get the target index
        tenant_indexes = trackme_idx_for_tenant(
            self._metadata.searchinfo.session_key,
            self._metadata.searchinfo.splunkd_uri,
            self.tenant_id,
        )

        # initialise search_results
        search_results = []

        # initialise results_dict
        results_dict = {}

        # initialise yield_record
        yield_record = {}

        # range category dict (by entity key)
        range_category_dict = {}
        # counters
        count_entities_processed = 0
        count_entities_failed = 0

        # delayed_inspector_search
        delayed_inspector_search = f"""        
            | trackmegetcoll tenant_id={self.tenant_id} component={self.component}
            ``` root constraints: filter on enabled entities, and entities that have been managed by the health tracker within at least the 15 minutes ```
            | where monitored_state=="enabled" AND tracker_health_runtime>=(now()-900)
            ``` filter on positive data_last_lag_seen ```
            | where data_last_lag_seen > 0
            ``` lookup against the delayed inspector KV collection ```
            | lookup trackme_{self.component}_delayed_entities_inspector_tenant_{self.tenant_id} _key as _key OUTPUT mtime as inspector_mtime, inspector_error_counters
            | eval inspector_mtime=if(isnull(inspector_mtime), 0, inspector_mtime), inspector_error_counters=if(isnull(inspector_error_counters), 0, inspector_error_counters)                       
            ``` calculate the time spent since the last inspection ```
            | eval time_since_last_inspection=if(inspector_mtime == 0, 0, now()-inspector_mtime)
            ``` round time_since_last_inspection ```
            | eval time_since_last_inspection=round(time_since_last_inspection, 0)
            ``` do not proceed if the inspector_error_counters is greater than {self.max_errors_count_per_entity_search}, this means we allow up to {self.max_errors_count_per_entity_search} attempts to run the search ```
            | where inspector_error_counters <= {self.max_errors_count_per_entity_search}
            ``` round data_last_lag_seen ```
            | eval data_last_lag_seen=round(data_last_lag_seen, 0)
            ``` table fields needed for the delayed inspector ```
            | table _key, object, alias, inspector_mtime, inspector_error_counters, time_since_last_inspection, data_last_lag_seen
            ``` sort by the older inspector_mtime ```
            | sort - inspector_mtime
        """

        # if object_name is set, add a constraint to the search
        if self.object_name:
            delayed_inspector_search += f"""
                | search object="{self.object_name}"
            """

        # delayed inspector KV collection
        delayed_inspector_collection_name = f"kv_trackme_{self.component}_delayed_entities_inspector_tenant_{self.tenant_id}"

        # connect to the delayed inspector KV collection
        delayed_inspectodelayed_inspector_collection = self.service.kvstore[
            delayed_inspector_collection_name
        ]

        # report name for logging purposes
        report_name = f"trackme_{self.component}_delayed_entities_inspector_tracker_tenant_{self.tenant_id}"

        # max runtime
        max_runtime = int(self.max_runtime)

        # Initialize sum of execution times and count of iterations
        total_execution_time = 0
        iteration_count = 0

        # Retrieve the search cron schedule
        savedsearch = self.service.saved_searches[report_name]
        savedsearch_cron_schedule = savedsearch.content["cron_schedule"]

        # get the cron_exec_sequence_sec
        try:
            cron_exec_sequence_sec = int(cron_to_seconds(savedsearch_cron_schedule))
        except Exception as e:
            logging.error(
                f'tenant_id="{self.tenant_id}", component="splk-{self.component}", failed to convert the cron schedule to seconds, error="{str(e)}"'
            )
            cron_exec_sequence_sec = max_runtime

        # the max_runtime cannot be bigger than the cron_exec_sequence_sec
        if max_runtime > cron_exec_sequence_sec:
            max_runtime = cron_exec_sequence_sec

        logging.info(
            f'tenant_id="{self.tenant_id}", component="splk-{self.component}", max_runtime="{max_runtime}",  savedsearch_name="{report_name}", savedsearch_cron_schedule="{savedsearch_cron_schedule}", cron_exec_sequence_sec="{cron_exec_sequence_sec}"'
        )

        #
        # main processing
        #

        try:
            reader = run_splunk_search(
                self.service,
                delayed_inspector_search,
                {
                    "earliest_time": "-5m",
                    "latest_time": "now",
                    "count": 0,
                    "output_mode": "json",
                },
                24,
                5,
            )

            for item in reader:
                if isinstance(item, dict):
                    logging.debug(
                        f'tenant_id="{self.tenant_id}", component="splk-{self.component}", search_results="{item}"'
                    )
                    # append to the list of searches
                    search_results.append(item)

                    # get entity_key
                    entity_key = item["_key"]

                    # add to the dict by _key
                    results_dict[entity_key] = item

                    # get range category
                    range_category, entity_search_earliest_time, span_value = (
                        self.get_range_category(
                            int(item["data_last_lag_seen"]),
                            str(splk_feeds_auto_disablement_period),
                        )
                    )

                    # add to the range category dict
                    range_category_dict[entity_key] = {
                        "range_category": range_category,
                        "entity_search_earliest_time": entity_search_earliest_time,
                        "span_value": span_value,
                    }

        except Exception as e:
            logging.error(
                f'tenant_id="{self.tenant_id}", component="splk-{self.component}", An exception was encountered, exception="{str(e)}"'
            )
            yield_record = {
                "_time": time.time(),
                "action": "failure",
                "search": delayed_inspector_search,
                "response": f'The entity providing delayed inspector search failed to be executed, exception="{str(e)}"',
                "_raw": {
                    "tenant_id": self.tenant_id,
                    "component": self.component,
                    "action": "failure",
                    "response": "The entity providing delayed inspector search failed to be executed",
                    "exception": str(e),
                },
            }

            trackme_register_tenant_object_summary(
                self._metadata.searchinfo.session_key,
                self._metadata.searchinfo.splunkd_uri,
                self.tenant_id,
                f"splk-{self.component}",
                report_name,
                "failure",
                time.time(),
                str(time.time() - start),
                f'The entity providing delayed inspector search failed to be executed, exception="{str(e)}"',
                "-5m",
                "now",
            )

            # yield the record
            yield yield_record

            # raise an exception
            raise Exception(
                f'The entity providing delayed inspector search failed to be executed, exception="{str(e)}"'
            )

        #
        # entities processing
        #

        # logic:
        # - Retrieve the list of entities for dsm/dhm which are reported in delayed anomaly
        # - For each entity, check if there is a delayed inspector record (potential fields: _key, mtime, object, inspector_exec_counters, inspector_error_counters, inspector_last_error, inspector_last_status)
        # - depending on our logic, we will attempt to verify if the entity is receiving data and process to updates as needed to avoid false positives due to tracker time range restrictions

        # proceed entity bool
        proceed_entity_bool = False

        # iterate over results_dict
        for key, value in results_dict.items():

            # break if reaching the max run time less 30 seconds of margin
            if (time.time() - int(start)) - 30 >= max_runtime:
                logging.info(
                    f'tenant_id="{self.tenant_id}", component="splk-{self.component}", max_runtime="{max_runtime}" was reached with current_runtime="{start}", job will be terminated now'
                )
                break

            # iteration start
            iteration_start_time = time.time()

            # get object value
            object_value = results_dict[key]["object"]

            # get the range category for this particular entity
            range_category = range_category_dict[key]["range_category"]
            entity_search_earliest_time = range_category_dict[key][
                "entity_search_earliest_time"
            ]
            span_value = range_category_dict[key]["span_value"]

            # set the boolean depending on the range category and the last inspection time
            proceed_entity_bool, proceed_entity_reason = (
                self.define_proceed_entity_bool(
                    range_category,
                    int(results_dict[key]["time_since_last_inspection"]),
                    splk_feeds_delayed_inspector_24hours_range_min_sec,
                    splk_feeds_delayed_inspector_7days_range_min_sec,
                    splk_feeds_delayed_inspector_until_disabled_range_min_sec,
                )
            )

            # if the entity is not to be proceeded, yield a record
            if not proceed_entity_bool:

                logging.info(
                    f'tenant_id="{self.tenant_id}", component="splk-{self.component}", object="{object_value}", key="{key}", proceed_entity_bool="{proceed_entity_bool}", proceed_entity_reason="{proceed_entity_reason}", range_category="{range_category}", last_inspection_time="{results_dict[key]["time_since_last_inspection"]}", data_last_lag_seen="{results_dict[key]["data_last_lag_seen"]}", the conditions for this entity to be proceeded were not met'
                )
                continue

            else:

                # get the delayed inspector record, if any
                logging.info(
                    f'tenant_id="{self.tenant_id}", component="splk-{self.component}", key="{key}"'
                )
                delayed_inspector_record = self.get_delayed_inspector_record(
                    delayed_inspectodelayed_inspector_collection, key
                )

                if not delayed_inspector_record:
                    logging.info(
                        f'tenant_id="{self.tenant_id}", component="splk-{self.component}", object="{object_value}", key="{key}", delayed_inspector_record="None"'
                    )

                # else, get exec and error counters
                exec_counters = int(
                    delayed_inspector_record.get("inspector_exec_counters", 0)
                )
                error_counters = int(
                    delayed_inspector_record.get("inspector_error_counters", 0)
                )

                # Initialize search variable
                delayed_entity_search = None

                # Get the entity info
                try:
                    json_data = {
                        "tenant_id": self.tenant_id,
                        "object": object_value,
                    }

                    component_url = {
                        "dsm": "/services/trackme/v2/splk_dsm/ds_entity_info",
                        "dhm": "/services/trackme/v2/splk_dhm/dh_entity_info",
                    }
                    target_url = f"{self._metadata.searchinfo.splunkd_uri}{component_url[self.component]}"
                    response = requests.post(
                        target_url,
                        headers={
                            "Authorization": f"Splunk {self._metadata.searchinfo.session_key}",
                            "Content-Type": "application/json",
                        },
                        verify=False,
                        timeout=600,
                        data=json.dumps(json_data),
                    )

                    entity_info = json.loads(response.text)
                    logging.debug(
                        f'tenant_id="{self.tenant_id}", component="splk-{self.component}", object="{object_value}", key="{key}", entity_info="{entity_info}"'
                    )

                except Exception as e:
                    logging.error(
                        f'tenant_id="{self.tenant_id}", component="splk-{self.component}", object="{object_value}", key="{key}", could not retrieve entity info for entity delayed tracking search, exception="{str(e)}"'
                    )

                if self.component == "dsm":

                    # specific to dsm, check is_elastic (0 or 1) from entity_info, elastic sources are excluded fron the delayed inspector
                    if entity_info["is_elastic"] == 1:
                        logging.info(
                            f'tenant_id="{self.tenant_id}", component="splk-{self.component}", object="{object_value}", key="{key}", is_elastic="{entity_info["is_elastic"]}", entity is anelastic source, skipping delayed inspector'
                        )
                        continue

                    delayed_entity_search = generate_dsm_report_search(
                        entity_info=entity_info,
                        search_mode=entity_info["search_mode"],
                        tenant_id=self.tenant_id,
                        root_constraint=entity_info["search_constraint"],
                        index_earliest_time=entity_search_earliest_time,
                        index_latest_time="now",
                        dsm_tstats_root_time_span=span_value,
                        breakby_field=entity_info.get("breakby_key", "none"),
                        account=entity_info["account"],
                        earliest_time=entity_search_earliest_time,
                        latest_time="now",
                        dsm_tstats_root_breakby_include_splunk_server=False,
                        dsm_tstats_root_breakby_include_host=False,
                    )

                elif self.component == "dhm":
                    delayed_entity_search = generate_dhm_report_search(
                        entity_info=entity_info,
                        search_mode=entity_info["search_mode"],
                        tenant_id=self.tenant_id,
                        root_constraint=entity_info["search_constraint"],
                        index_earliest_time=entity_search_earliest_time,
                        index_latest_time="now",
                        dhm_tstats_root_time_span=span_value,
                        breakby_field=entity_info.get("breakby_key", "none"),
                        account=entity_info["account"],
                        earliest_time=entity_search_earliest_time,
                        latest_time="now",
                        dhm_tstats_root_breakby_include_splunk_server=False,
                    )

                logging.info(
                    f'tenant_id="{self.tenant_id}", component="splk-{self.component}", object="{object_value}", key="{key}", delayed_entity_search="{delayed_entity_search}"'
                )

                # Run the main report, every result is a Splunk search to be executed on its own thread
                entity_search_start = time.time()
                if not delayed_entity_search:
                    logging.error(
                        f'tenant_id="{self.tenant_id}", component="splk-{self.component}", object="{object_value}", key="{key}", Could not retrieve entity info entity delayed tracking search, this entity was not found'
                    )
                    continue

                # Proceed
                logging.info(
                    f'tenant_id="{self.tenant_id}", component="splk-{self.component}", object="{object_value}", key="{key}", Executing entity delayed tracking resulting search="{delayed_entity_search}"'
                )

                # run the delayed entity search
                entity_search_results = []
                entity_search_failed = False
                entity_search_failed_reason = None
                entity_search_summary_record = {}

                try:
                    reader = run_splunk_search(
                        self.service,
                        delayed_entity_search,
                        {
                            "earliest_time": "-7d",
                            "latest_time": "now",
                            "count": 0,
                            "output_mode": "json",
                        },
                        24,
                        5,
                    )

                    for item in reader:
                        logging.debug(f'delayed_entity_search_results="{item}"')
                        entity_search_results.append(item)
                    logging.info(
                        f'tenant_id="{self.tenant_id}", component="splk-{self.component}", object="{object_value}", key="{key}", successfully executed in {round(time.time() - entity_search_start, 3)} seconds, delayed_entity_search_results="{entity_search_results}"'
                    )
                    # increment the exec counters
                    exec_counters += 1
                    count_entities_processed += 1

                    # create a summary record for the KV store taking into account our fields
                    entity_search_summary_record = {
                        "_key": key,
                        "mtime": time.time(),
                        "object": object_value,
                        "inspector_exec_counters": exec_counters,
                        "inspector_error_counters": error_counters,
                        "inspector_last_error": None,
                        "inspector_last_status": "success",
                    }

                    # notification event
                    try:
                        trackme_handler_events(
                            session_key=self._metadata.searchinfo.session_key,
                            splunkd_uri=self._metadata.searchinfo.splunkd_uri,
                            tenant_id=self.tenant_id,
                            sourcetype="trackme:handler",
                            source=f"trackme:handler:{self.tenant_id}",
                            handler_events=[
                                {
                                    "object": object_value,
                                    "object_id": key,
                                    "object_category": f"splk-{self.component}",
                                    "handler": "delayed_inspector",
                                    "key": key,
                                    "handler_message": "Entity was inspected by the delayed inspector, it is out of the scope of any hybrid tracker due to high delay and/or latency. The delay inspector performs regular backward searches to refresh the entity status and up to date knowledge.",
                                    "handler_troubleshoot_search": f"index=_internal sourcetype=trackme:custom_commands:trackme:custom_commands:trackmesplkfeedsdelayedinspector tenant_id={self.tenant_id} component={self.component} object={object_value}",
                                    "handler_time": time.time(),
                                }
                            ],
                        )
                    except Exception as e:
                        logging.error(
                            f'tenant_id="{self.tenant_id}", component="splk-{self.component}", object="{object_value}", key="{key}", could not send notification event, exception="{e}"'
                        )

                except Exception as e:
                    logging.error(
                        f'tenant_id="{self.tenant_id}", component="splk-{self.component}", object="{object_value}", key="{key}", could not execute delayed entity search, exception="{e}"'
                    )
                    # increment the error counters
                    error_counters += 1
                    entity_search_failed = True
                    entity_search_failed_reason = str(e)

                    # increment main counter
                    count_entities_processed += 1

                    # increment the error counter
                    count_entities_failed += 1

                    # create a summary record for the KV store taking into account our fields
                    # inspector_exec_counters, inspector_error_counters, inspector_last_error, inspector_last_status
                    entity_search_summary_record = {
                        "_key": key,
                        "mtime": time.time(),
                        "object": object_value,
                        "inspector_exec_counters": exec_counters,
                        "inspector_error_counters": error_counters,
                        "inspector_last_error": str(e),
                        "inspector_last_status": "failed",
                        "proceed_entity_bool": proceed_entity_bool,
                        "range_category": range_category,
                        "last_inspection_time": int(
                            results_dict[key]["time_since_last_inspection"]
                        ),
                        "delayed_inspector_message": f'The entity search failed to be executed, exception="{str(e)}"',
                    }

                # KVstore insert/update
                if not delayed_inspector_record:
                    # insert
                    try:
                        delayed_inspectodelayed_inspector_collection.data.insert(
                            json.dumps(entity_search_summary_record),
                        )
                    except Exception as e:
                        logging.error(
                            f'tenant_id="{self.tenant_id}", component="splk-{self.component}", object="{object_value}", key="{key}", could not insert delayed inspector record, exception="{e}"'
                        )
                else:
                    # update
                    try:
                        delayed_inspectodelayed_inspector_collection.data.update(
                            key,
                            json.dumps(entity_search_summary_record),
                        )
                    except Exception as e:
                        logging.error(
                            f'tenant_id="{self.tenant_id}", component="splk-{self.component}", object="{object_value}", key="{key}", could not update delayed inspector record, exception="{e}"'
                        )

                # add to yield_record
                yield {
                    "_time": time.time(),
                    "_raw": results_dict[key],
                    "entity_has_delayed_record": bool(delayed_inspector_record),
                    "keyid": key,
                    "object": results_dict[key]["object"],
                    "alias": results_dict[key]["alias"],
                    "delayed_entity_search": delayed_entity_search,
                    "entity_search_results": entity_search_results,
                    "entity_search_failed": entity_search_failed,
                    "entity_search_failed_reason": entity_search_failed_reason,
                    "proceed_entity_bool": proceed_entity_bool,
                    "range_category": range_category,
                    "last_inspection_time": int(
                        results_dict[key]["time_since_last_inspection"]
                    ),
                    "delayed_inspector_message": f"The entity is to be proceeded based on its current conditions and the delayed inspector configuration.",
                }

                # Calculate the execution time for this iteration
                iteration_end_time = time.time()
                execution_time = iteration_end_time - iteration_start_time

                # Update total execution time and iteration count
                total_execution_time += execution_time
                iteration_count += 1

                # Calculate average execution time
                if iteration_count > 0:
                    average_execution_time = total_execution_time / iteration_count
                else:
                    average_execution_time = 0

                # Check if there is enough time left to continue
                current_time = time.time()
                elapsed_time = current_time - start
                if elapsed_time + average_execution_time + 120 >= max_runtime:
                    logging.info(
                        f'tenant_id="{self.tenant_id}", component="splk-{self.component}", max_runtime="{max_runtime}" is about to be reached, current_runtime="{elapsed_time}", job will be terminated now'
                    )
                    break

        # if no entities were processed, yield a record
        if count_entities_processed == 0:
            yield {
                "_time": time.time(),
                "_raw": {
                    "tenant_id": self.tenant_id,
                    "component": self.component,
                    "search": delayed_inspector_search,
                    "result": "there were no entities to process at this time.",
                    "count_entities_processed": count_entities_processed,
                    "count_entities_failed": count_entities_failed,
                },
            }
            logging.info(
                f'tenant_id="{self.tenant_id}", component="splk-{self.component}", there were no entities to process at this time, count_entities_processed="{count_entities_processed}", count_entities_failed="{count_entities_failed}"'
            )

        trackme_register_tenant_object_summary(
            self._metadata.searchinfo.session_key,
            self._metadata.searchinfo.splunkd_uri,
            self.tenant_id,
            f"splk-{self.component}",
            report_name,
            "success",
            time.time(),
            str(time.time() - start),
            "The report was executed successfully",
            "-5m",
            "now",
        )

        #
        # End processing
        #

        # Log the run time
        logging.info(
            f'tenant_id="{self.tenant_id}", component="splk-{self.component}", trackmesplkfeedsdelayedinspector has terminated, run_time={round(time.time() - start, 3)}, search="{delayed_inspector_search}", count_entities_processed="{count_entities_processed}", count_entities_failed="{count_entities_failed}"'
        )


dispatch(TrackMeFeedsDelayedInspector, sys.argv, sys.stdin, sys.stdout, __name__)