SH-Deployer/apps/SA-ITSI-DriftDetection/bin/batchdetectdrift.py

import json
import os
import sys
import time

import exec_anaconda

exec_anaconda.exec_anaconda()

sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "lib"))

# Add the directory where this script resides to the Python path
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))

from splunklib.searchcommands import dispatch, GeneratingCommand, Configuration

from constants import (ITSI_DRIFT_DETECTION_KPIS_URI,
                       ITSI_DRIFT_DETECTION_TEMPLATE_URI,
                       LOOKBACK_PERIOD,
                       AGGREGATION_SPAN,
                       AGGREGATION_FUNCTION,
                       TOLERANCE_IN_PERCENT,
                       THRESHOLD_DIRECTION,
                       DRIFT_DETECTION_TEMPLATE_ID,
                       KVSTORE_KEY, ITSI_APP_NAME, ITSI_APP_OWNER)

from logger import get_logger
from util.telemetry_logger import log_telemetry

logger = get_logger()


@Configuration()
class BatchDriftDetectionCommand(GeneratingCommand):
    template_cache = {}

    @staticmethod
    def safe_network_call(func_, *args, **kwargs):
        try:
            response = func_(*args, **kwargs)
            if response.status != 200:
                logger.error(f"Network call failed with status {response.status}")
                return None
            return json.loads(response.body.read())
        except Exception as e:
            logger.error(f"Exception occurred during network call: {str(e)}")
            return None

    def get_kpis_configured_for_drift_detection(self):
        return self.safe_network_call(self.service.get,
                                      ITSI_DRIFT_DETECTION_KPIS_URI,
                                      owner=ITSI_APP_OWNER,
                                      app=ITSI_APP_NAME) or []

    def get_template_info(self, template_id):
        if template_id in self.template_cache:
            return self.template_cache[template_id]

        # Proceed with the network call if template_id is not in cache
        endpoint_uri = f"{ITSI_DRIFT_DETECTION_TEMPLATE_URI}/{template_id}"
        template_info = self.safe_network_call(self.service.get, endpoint_uri, owner=ITSI_APP_OWNER, app=ITSI_APP_NAME)

        # Cache the template_info if the network call was successful
        if template_info:
            self.template_cache[template_id] = template_info

        return template_info

    @staticmethod
    def construct_spl_query(template_info, kpi_id):
        spl_template = """| mstats latest(alert_value) AS alert_value latest(alert_level) AS alert_level WHERE index=itsi_summary_metrics earliest={lookback_period} latest=now() AND itsi_kpi_id="{kpi_id}" AND is_filled_gap_event!=1 AND is_null_alert_value=0 by itsi_kpi_id, itsi_service_id span=1m
        | where alert_level!=-2
        | bin _time span={aggregation_span}
        | stats {aggregation_function}(alert_value) as alert_value by _time, itsi_kpi_id, itsi_service_id
        | table _time, alert_value, itsi_kpi_id, itsi_service_id
        | detectdrift threshold_direction="{threshold_direction}", threshold={tolerance_in_percent}"""

        return spl_template.format(
            lookback_period=template_info[LOOKBACK_PERIOD],
            aggregation_span=template_info[AGGREGATION_SPAN],
            aggregation_function=template_info[AGGREGATION_FUNCTION],
            tolerance_in_percent=template_info[TOLERANCE_IN_PERCENT],
            threshold_direction=template_info[THRESHOLD_DIRECTION],
            kpi_id=kpi_id
        )

    def execute_spl_query(self, spl_query):
        """
        Executes an SPL query in blocking mode using the Splunk service.

        This method creates a search job, waits for it to complete, and logs the job's SID (Search ID)
        along with its final status. It returns the SID of the completed job or None if the execution fails.

        Parameters:
        - spl_query (str): The SPL query to be executed.

        Returns:
        - sid (str): The SID of the successfully executed job or None upon failure.

        Note:
        The job's success is evaluated based on the 'isFailed' flag and the 'dispatchState'.
        A job is considered successful if 'isFailed' is 0 (false) and 'dispatchState' is "DONE".
        Any other states indicate a failure or an incomplete job.

        For more details on parameters to `jobs.create` and job management,
        visit: https://dev.splunk.com/enterprise/docs/devtools/python/sdk-python/howtousesplunkpython/howtorunsearchespython/
        """
        try:
            # Replace newline characters with spaces
            single_line_query = spl_query.replace('\n', ' ')
            logger.info(f"Executing SPL query: {single_line_query}")

            job = self.service.jobs.create(spl_query, exec_mode="blocking")
            sid = job.sid

            logger.info(f"Search job initiated: SID={sid}")

            while not job.is_done():
                time.sleep(1)

            # Retrieve the final job status details
            is_failed = job["isFailed"]  # 0: success, 1: failure
            dispatch_state = job["dispatchState"]  # "DONE": success, "FAILED": failure

            # Evaluate job success
            if is_failed == "0" and dispatch_state == "DONE":
                job_status = "Success"
            else:
                job_status = "Failure"

            logger.info(f"Search job finished: SID={sid}, Status={job_status}, Dispatch State={dispatch_state}")

            return sid
        except Exception as e:
            logger.error(f"Failed to execute SPL query: {str(e)}")
            return None

    @staticmethod
    def get_kpi_id(kpi):
        return kpi.get(KVSTORE_KEY, "unknown")

    @staticmethod
    def get_template_id(kpi):
        return kpi.get(DRIFT_DETECTION_TEMPLATE_ID)

    def generate(self):
        time_0 = time.time()
        log_telemetry(
            event_type = 'batchdetectdrift_start',
        )

        kpis = self.get_kpis_configured_for_drift_detection()

        cnt_kpi = len(kpis)
        log_telemetry(
            event_type = 'get_drift_configured_kpi',
            count_kpi = cnt_kpi
        )

        if cnt_kpi == 0:
            logger.info("No KPIs configured for drift detection.")
            return

        for kpi in kpis:
            try:
                kpi_id = self.get_kpi_id(kpi)
                template_info = self.get_template_info(self.get_template_id(kpi))

                if not template_info:
                    log_telemetry(
                        event_type = 'no_template_found',
                        kpi_id = kpi_id
                    )
                    logger.error(f"No template info found for KPI ID: {kpi_id}. Skipping drift detection.")
                    continue

                logger.info(f"Initiating drift detection job for KPI ID: {kpi_id}")
                time_1 = time.time()

                # Construct the SPL query using the template information and KPI ID
                spl_query = self.construct_spl_query(template_info=template_info, kpi_id=kpi_id)

                # Execute the SPL query and proceed only if an SID was successfully returned
                sid = self.execute_spl_query(spl_query)

                log_telemetry(
                    event_type = 'inner_csc_complete',
                    kpi_id = kpi_id,
                    sid = sid,
                    total_time = f'{time.time() - time_1:.3f}s',
                )

                if not sid:
                    logger.error(f"Execution failed for KPI ID: {kpi_id}")
                    continue

                logger.info(f"Completed drift detection job: KPI ID={kpi_id}, SID={sid}")
                yield {'SID': sid}

            except Exception as e:
                logger.error(f"An error occurred during drift detection for KPI ID: {self.get_kpi_id(kpi)}: {e}")

        log_telemetry(
            event_type = 'batchdetectdrift_complete',
            total_time = f'{time.time() - time_0:.3f}s',
        )

dispatch(BatchDriftDetectionCommand, sys.argv, sys.stdin, sys.stdout, __name__)