# Copyright (C) 2005-2024 Splunk Inc. All Rights Reserved. import json import sys import time from splunk import ResourceNotFound from splunk.clilib.bundle_paths import make_splunkhome_path sys.path.append(make_splunkhome_path(['etc', 'apps', 'SA-ITOA', 'lib'])) sys.path.append(make_splunkhome_path(['etc', 'apps', 'SA-ITOA', 'lib', 'SA_ITOA_app_common'])) import itsi_path from itsi_py3 import _ import itsi_py3 from ITOA.setup_logging import logger as itsi_logger from ITOA.itoa_common import is_feature_enabled from ITOA.saved_search_utility import SavedSearch from ITOA.event_management.notable_event_aggregation_policy import NotableEventAggregationPolicy from ITOA.event_management.notable_event_utils import get_collection_name_for_event_management_objects, \ NotableEventConfiguration sys.path.append(make_splunkhome_path(['etc', 'apps', 'SA-UserAccess', 'lib'])) from user_access_utils import UserAccess class NotableEventValidator(object): """ Notable event validator """ def __init__(self, session_key, logger, required_keys=None): self.session_key = session_key self.owner_key = 'owner' self.status_key = 'status' self.severity_key = 'severity' if required_keys is None: self.required_keys = ['_time', 'mod_time', 'title', self.owner_key, self.status_key, self.severity_key] else: self.required_keys = required_keys if logger: self.logger = logger else: raise ValueError(_('`logger` is not provided.')) self.notable_configuration_object = NotableEventConfiguration(session_key, logger) self.valid_owners = self.notable_configuration_object.get_owners() self.valid_statuses = self.notable_configuration_object.get_statuses() self.valid_severities = self.notable_configuration_object.get_severities() def validate_schema(self, data): """ Validate schema before user CURD operation on notable event @type data: dict @param data: data which hold notable schema to create @rtype: bool @return: True - if data contains all required fields, False - otherwise or throw exception """ # Check for status, owner and severity is defined, otherwise set to default. if data.get(self.owner_key) is None or data.get(self.owner_key) == '': self.logger.info('No owner is being set for event title=%s, hence setting to default owner=%s.', data.get('title'), self.notable_configuration_object.get_default_owner()) data[self.owner_key] = self.notable_configuration_object.get_default_owner() if data.get(self.status_key) is None or data.get(self.status_key) == '': self.logger.info('No status is being set for event title=%s, hence setting to default status=%s.', data.get('title'), self.notable_configuration_object.get_default_status()) data[self.status_key] = self.notable_configuration_object.get_default_status() if data.get(self.severity_key) is None or data.get(self.severity_key) == '': self.logger.info('No severity is being set for event title=%s, hence setting to default severity=%s.', data.get('title'), self.notable_configuration_object.get_default_severity()) data[self.severity_key] = self.notable_configuration_object.get_default_severity() for key in self.required_keys: if key not in data: message = _("%s key does not exist in the data=%s.") % (key, str(data)) self.logger.error(message) raise ValueError(message) # Make sure status, severity value is str data[self.owner_key] = str(data.get(self.owner_key, '')) data[self.status_key] = str(data.get(self.status_key, '')) data[self.severity_key] = str(data.get(self.severity_key, '')) # Lets have more logging and proper error if validation failed. is_validate_owner = self.check_owner(data.get(self.owner_key)) if not is_validate_owner: warning_message = _('Invalid owner={0} for event title={1}. ' 'Unable to find owner in valid Splunk user list, ' 'hence setting to default owner={2}.').format( data.get(self.owner_key), data.get('title'), self.notable_configuration_object.get_default_owner()) self.logger.warning(warning_message) data[self.owner_key] = self.notable_configuration_object.get_default_owner() is_validate_status = self.check_status(data.get(self.status_key)) if not is_validate_status: warning_message = _('Invalid status={0} for event title={1}. ' 'Unable to find status in itsi_notable_event_status.conf, ' 'hence setting to default status={2}.').format( data.get(self.status_key), data.get('title'), self.notable_configuration_object.get_default_status()) self.logger.warning(warning_message) data[self.status_key] = self.notable_configuration_object.get_default_status() is_validate_severity = self.check_severity(data.get(self.severity_key)) if not is_validate_severity: warning_message = _('Invalid severity={0} for event title={1}. ' 'Unable to find severity in itsi_notable_event_severity.conf, ' 'hence setting to default severity={2}.').format( data.get(self.severity_key), data.get('title'), self.notable_configuration_object.get_default_severity()) self.logger.warning(warning_message) data[self.severity_key] = self.notable_configuration_object.get_default_severity() return is_validate_owner or is_validate_status or is_validate_severity def check_severity(self, severity): """ Check severity @type severity: basestring @param severity: severity @rtype: bool @return: True if valid severity otherwise False """ return severity in self.valid_severities def check_status(self, status): """ Check status @type status: basestring @param status: Status @rtype: bool @return: True if valid status otherwise False """ return status in self.valid_statuses def check_owner(self, owner): """ Check owner is valid or not @type owner: basestring @param owner: owner @rtype: bool @return: True if valid owner otherwise False """ return owner in self.valid_owners class NotableEventDefaultPoliciesLoader(object): """ This class is being used to load default aggregation policy """ DEFAULT_POLICY = """ { "_key": "itsi_default_policy", "group_title": "%title%", "group_description": "%description%", "group_status": "%status%", "group_assignee": "%owner%", "disabled": 0, "is_default": 1, "object_type": "notable_aggregation_policy", "title": "Default Policy", "description": "Applies to events that do not meet the criteria of any other active policy.", "split_by_field": "source", "priority": 5, "group_severity": "%severity%", "filter_criteria": { "condition": "OR", "items": [] }, "breaking_criteria": { "condition": "OR", "items": [ { "type": "pause", "config": { "limit": "7200" } } ] }, "rules": [] } """ DEFAULT_SNMP_POLICY = """ { "_key": "itsi_default_snmp_policy", "group_title": "%title%", "group_description": "%description%", "group_status": "%status%", "group_assignee": "%owner%", "disabled": 1, "is_default": 0, "object_type": "notable_aggregation_policy", "title": "Default SNMP Policy", "description": "Aggregation policy for SNMP traps", "split_by_field": "node,description", "priority": "", "group_severity": "%severity%", "filter_criteria": { "items": [ { "type": "clause", "config": { "items": [ { "type": "notable_event_field", "config": { "operator": "=", "field": "node", "value": "*" } }, { "type": "notable_event_field", "config": { "operator": "=", "field": "description", "value": "*" } } ], "condition": "AND" } } ], "condition": "OR" }, "breaking_criteria": { "items": [ { "type": "clause", "config": { "items": [ { "type": "notable_event_field", "config": { "operator": "=", "field": "severity", "value": "2" } } ], "condition": "AND" } } ], "condition": "OR" }, "rules": [ ] } """ NORMALIZED_AGGREGATION_POLICY = """ { "_key": "normalized_aggregation_policy", "group_title": "Normalized Alert for %itsiInstance% (%itsiSubInstance%) : %itsiAlert%", "group_description": "%last_description%", "group_status": "%status%", "group_assignee": "%owner%", "disabled": 0, "is_default": 0, "object_type": "notable_aggregation_policy", "title": "Normalized Policy (Splunk App for Infrastructure)", "description": "Applies to events that contain ITSI normalized fields.", "split_by_field": "itsiAlert,itsiInstance", "priority": "", "group_severity": "%last_severity%", "group_instruction": "%last_instruction%", "filter_criteria": { "items": [ { "config": { "items": [ { "config": { "operator": "=", "field": "itsiAlert", "value": "*" }, "type": "notable_event_field" }, { "config": { "operator": "=", "field": "itsiInstance", "value": "*" }, "type": "notable_event_field" }, { "config": { "operator": "=", "field": "itsiSubInstance", "value": "*" }, "type": "notable_event_field" }, { "config": { "operator": "=", "field": "itsiSeverity", "value": "*" }, "type": "notable_event_field" } ], "condition": "AND" }, "type": "clause" } ], "condition": "OR" }, "breaking_criteria": { "items": [ { "config": { "limit": "3600" }, "type": "pause" }, { "config": { "items": [ { "config": { "operator": "=", "field": "severity", "value": "2" }, "type": "notable_event_field" } ], "condition": "AND" }, "type": "clause" } ], "condition": "OR" }, "rules": [ { "_key": "normalized_aggregation_policy_action_1", "priority": 5, "activation_criteria": { "items": [ { "type": "breaking_criteria" }, { "config": { "operator": ">=", "limit": "2" }, "type": "notable_event_count" } ], "condition": "AND" }, "actions": [ { "items": [ { "execution_criteria": { "execute_on": "GROUP" }, "type": "notable_event_change", "config": { "field": "status", "value": "4" } } ], "condition": "AND" } ], "title": "", "description": "" }, { "_key": "normalized_aggregation_policy_action_2", "priority": 5, "activation_criteria": { "items": [ { "type": "breaking_criteria" }, { "config": { "operator": "==", "limit": "1" }, "type": "notable_event_count" } ], "condition": "AND" }, "actions": [ { "items": [ { "execution_criteria": { "execute_on": "GROUP" }, "type": "notable_event_change", "config": { "field": "status", "value": "5" } } ], "condition": "AND" } ], "title": "", "description": "" } ] } """ ENTITY_TYPE_ALERT_AGGREGATION_POLICY = """ { "_key": "entity_type_ootb_aggregation_policy", "group_title": "Entity Type Alerts: %entity_type%, Severity: %itsiSeverity%", "group_description": "Alerts for %entity_type% with Severity level %itsiSeverity%", "group_status": "%status%", "group_assignee": "%owner%", "disabled": 0, "is_default": 0, "object_type": "notable_aggregation_policy", "title": "Entity Type Alerts", "description": "Applies to events for entities that match a custom or out-of-box Entity Type", "split_by_field": "entity_type,itsiSeverity", "priority": "", "group_severity": "%last_severity%", "group_instruction": "%last_instruction%", "filter_criteria": { "items": [ { "config": { "items": [ { "config": { "operator": "=", "field": "entity_type", "value": "*" }, "type": "notable_event_field" }, { "config": { "operator": "=", "field": "itsiAlert", "value": "*" }, "type": "notable_event_field" }, { "config": { "operator": "=", "field": "itsiInstance", "value": "*" }, "type": "notable_event_field" }, { "config": { "operator": "=", "field": "itsiSeverity", "value": "*" }, "type": "notable_event_field" }, { "config": { "operator": "=", "field": "alert_source", "value": "entity_type" }, "type": "notable_event_field" } ], "condition": "AND" }, "type": "clause" } ], "condition": "OR" }, "breaking_criteria": { "items": [ { "config": { "limit": "3600" }, "type": "pause" }, { "config": { "items": [ { "config": { "operator": "=", "field": "severity", "value": "2" }, "type": "notable_event_field" } ], "condition": "AND" }, "type": "clause" } ], "condition": "OR" }, "rules": [ { "_key": "entity_type_ootb_aggregation_policy_action_1", "priority": 5, "activation_criteria": { "items": [ { "type": "breaking_criteria" }, { "config": { "operator": ">=", "limit": "2" }, "type": "notable_event_count" } ], "condition": "AND" }, "actions": [ { "items": [ { "execution_criteria": { "execute_on": "GROUP" }, "type": "notable_event_change", "config": { "field": "status", "value": "4" } } ], "condition": "AND" } ], "title": "Resolve episodes that are broken and have greater than or equal to 2 events", "description": "When the episode is broken, and the number of events in the episode is >= 2, \ change the episode status to Resolved." }, { "_key": "entity_type_ootb_aggregation_policy_action_2", "priority": 5, "activation_criteria": { "items": [ { "type": "breaking_criteria" }, { "config": { "operator": "==", "limit": "1" }, "type": "notable_event_count" } ], "condition": "AND" }, "actions": [ { "items": [ { "execution_criteria": { "execute_on": "GROUP" }, "type": "notable_event_change", "config": { "field": "status", "value": "5" } } ], "condition": "AND" } ], "title": "Close episodes that are broken which have only 1 event.", "description": "If the episode is broken, and the episode has only 1 event, \ then change the episode status to Closed." } ] } """ KPI_ALERTING_POLICY = """ { "_key": "kpi_alerting_policy", "_user": "nobody", "_owner": "nobody", "title": "KPI Alerting Policy", "group_title": "KPI Alerts from Service: %service_title%", "sub_group_limit": "", "group_assignee": "%last_owner%", "group_status": "%last_status%", "breaking_criteria": { "condition": "OR", "items": [ { "type": "pause", "config": { "limit": "3600" } } ] }, "disabled": 0, "is_default": 0, "priority": "", "identifying_name": "kpi alerting policy", "split_by_field": "service_ids", "rules": [ ], "object_type": "notable_aggregation_policy", "description": "Aggregation policy for KPI state-change alerts", "source_itsi_da": "itsi", "group_description": "Grouped alerts from service %service_title%; most recent alert from KPI %kpi_title%", "group_severity": "%last_severity%", "group_instruction": "%last_instruction%", "filter_criteria": { "condition": "OR", "items": [ { "type": "clause", "config": { "condition": "AND", "items": [ { "type": "notable_event_field", "config": { "value": "*", "operator": "=", "field": "kpiid" } }, { "type": "notable_event_field", "config": { "field": "alert_type", "operator": "=", "value": "KPI alert" } } ] } } ] } } """ def __init__(self, session_key, logger=None): if not isinstance(session_key, itsi_py3.string_type): raise TypeError(_('Invalid session key.')) self.session_key = session_key self.logger = logger if logger is not None else itsi_logger self.notable_event_aggregator = NotableEventAggregationPolicy(session_key, is_validate=False) def upload_default_policies(self): """ @rtype: bool @return: True/False - if default policies are loaded successfully or not """ default_success = self.upload_policy("itsi_default_policy", self.DEFAULT_POLICY) normalized_success = self.upload_policy("normalized_aggregation_policy", self.NORMALIZED_AGGREGATION_POLICY) snmp_success = self.upload_policy("itsi_default_snmp_policy", self.DEFAULT_SNMP_POLICY) kpi_alert_success = self.upload_policy("kpi_alerting_policy", self.KPI_ALERTING_POLICY) entity_type_success = self.upload_policy("entity_type_ootb_aggregation_policy", self.ENTITY_TYPE_ALERT_AGGREGATION_POLICY) return default_success and normalized_success and snmp_success and kpi_alert_success and entity_type_success def upload_policy(self, _id, policy): """ Upload policy @type _id: basestring @param _id: Aggregation policy id @type policy: basestring @param policy: Aggregation policy JSON string @rtype: bool @return: True/False - if policy is loaded successfully or not """ if not self.notable_event_aggregator.storage_interface.wait_for_storage_init(self.session_key): raise Exception(_("KV store failed to initialize in time")) try: result = self.notable_event_aggregator.get(_id) update_policy = False if result: if _id == "normalized_aggregation_policy" or _id == "entity_type_ootb_aggregation_policy": for actions in result['rules']: if '_key' not in actions or actions.get('_key') == "": update_policy = True if update_policy: ret = self.notable_event_aggregator.update(_id, json.loads(policy)) return True if ret else False else: self.logger.info('Action rule key Found for %s' % _id) return True else: self.logger.info('Found %s' % _id) return True else: self.logger.info('Could not find %s' % _id) raise ResourceNotFound('%s does not exist, this is expected when ITSI is first installed.' % _id) except ResourceNotFound as e: # load now self.logger.exception(e) self.logger.info('creating policy because we did not find %s', _id) ret = self.notable_event_aggregator.create(json.loads(policy)) # note this is not object type o_type = 'notable_event_aggregation_policy' success, rval = UserAccess.bulk_update_perms( object_ids=[_id], acl={'read': ['*'], 'write': ['*'], 'delete': []}, object_app='itsi', object_type=o_type, object_storename=get_collection_name_for_event_management_objects(o_type), session_key=self.session_key, logger=self.logger ) if not success: self.logger.error('Unable to save acl for %s. Response: `%s`', _id, rval) else: self.logger.info('Successfully saved acl for %s. Response:`%s`', _id, rval) return True if ret else False class CorrelationSearchDefaultAclLoader(object): """ This class sets the ACL permissions for the default correlation searches 'Monitor Critical Services Based on Health Score', 'Splunk App for Infrastructure Alerts', and 'Normalized Correlation Search' """ DEFAULT_ACL = {'read': ['*'], 'write': ['*'], 'delete': ['*']} DEFAULT_CS_ACL = {'read': ['*'], 'write': ['*'], 'delete': []} def __init__(self, session_key, logger=None): if not isinstance(session_key, itsi_py3.string_type): raise TypeError(_('Invalid session key.')) self.session_key = session_key self.logger = logger if logger is not None else itsi_logger self.retrys = 120 def default_acl_loader(self): """ Set perms for default correlation searches @return: """ correlation_search_ids = ['Monitor Critical Services Based on Health Score', 'Splunk App for Infrastructure Alerts', 'Normalized Correlation Search', 'SNMP Traps', 'BMC Remedy Bidirectional Ticketing', 'Bidirectional Ticketing', 'Jira Bidirectional Ticketing', 'High Scale EA Backfill'] for id in correlation_search_ids: _id = id o_type = 'correlation_search' rval = 'Already created' perms = UserAccess.get_perms( object_id=_id, object_app='itsi', object_type=o_type, object_storename=get_collection_name_for_event_management_objects(o_type), session_key=self.session_key, logger=self.logger, object_owner='nobody' ) if _id == 'Bidirectional Ticketing' and not is_feature_enabled('itsi-bidirectional-ticketing', self.session_key): if perms: disabled_search = SavedSearch.update_search(self.session_key, _id, disabled=1) remove_perms = UserAccess.delete_perms( object_id=_id, object_app='itsi', object_type=o_type, object_storename=get_collection_name_for_event_management_objects(o_type), session_key=self.session_key, logger=self.logger, object_owner='nobody' ) if disabled_search and remove_perms: self.logger.info('Successfully removed acl and disabled %s because itsi-bidirectional-ticketing' + ' feature flag has been disabled', _id) else: self.logger.error('Failed to remove acl and disable %s', _id) continue if _id == 'BMC Remedy Bidirectional Ticketing' and not is_feature_enabled( 'itsi-remedy-bidirectional-ticketing', self.session_key): if perms: disabled_search = SavedSearch.update_search(self.session_key, _id, disabled=1) remove_perms = UserAccess.delete_perms( object_id=_id, object_app='itsi', object_type=o_type, object_storename=get_collection_name_for_event_management_objects(o_type), session_key=self.session_key, logger=self.logger, object_owner='nobody' ) if disabled_search and remove_perms: self.logger.info('Successfully removed acl and disabled %s because' 'itsi-remedy-bidirectional-ticketing feature flag has been disabled', _id) else: self.logger.error('Failed to remove acl and disable %s', _id) continue if _id == 'Jira Bidirectional Ticketing' and not is_feature_enabled( 'itsi-jira-bidirectional-ticketing', self.session_key): if perms: disabled_search = SavedSearch.update_search(self.session_key, _id, disabled=1) remove_perms = UserAccess.delete_perms( object_id=_id, object_app='itsi', object_type=o_type, object_storename=get_collection_name_for_event_management_objects(o_type), session_key=self.session_key, logger=self.logger, object_owner='nobody' ) if disabled_search and remove_perms: self.logger.info('Successfully removed acl and disabled %s because' 'itsi-jira-bidirectional-ticketing feature flag has been disabled', _id) else: self.logger.error('Failed to remove acl and disable %s', _id) continue if _id == 'High Scale EA Backfill' and not is_feature_enabled( 'itsi-high-scale-ea', self.session_key): if perms: disabled_search = SavedSearch.update_search(self.session_key, _id, disabled=1) remove_perms = UserAccess.delete_perms( object_id=_id, object_app='itsi', object_type=o_type, object_storename=get_collection_name_for_event_management_objects(o_type), session_key=self.session_key, logger=self.logger, object_owner='nobody' ) if disabled_search and remove_perms: self.logger.info('Successfully removed acl and disabled %s because' 'itsi-high-scale-ea feature flag has been disabled', _id) else: self.logger.error('Failed to remove acl and disable %s', _id) continue while not perms and self.retrys > 0: self.logger.info('Trying to save acl for %s.', _id) success, rval = UserAccess.update_perms( object_id=_id, acl=self.DEFAULT_CS_ACL, object_app='itsi', object_type=o_type, object_storename=(get_collection_name_for_event_management_objects(o_type)), session_key=self.session_key, logger=self.logger ) perms = UserAccess.get_perms( object_id=_id, object_app='itsi', object_type=o_type, object_storename=get_collection_name_for_event_management_objects(o_type), session_key=self.session_key, logger=self.logger, object_owner='nobody' ) self.retrys -= 1 time.sleep(0.5) if not perms: self.logger.error( 'Unable to save acl for %s. Response: `%s`', _id, rval) else: self.logger.info( 'Successfully saved acl for %s. Response:`%s`', _id, rval)