You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

920 lines
38 KiB

# Copyright (C) 2005-2024 Splunk Inc. All Rights Reserved.
import json
import sys
import time
from splunk import ResourceNotFound
from splunk.clilib.bundle_paths import make_splunkhome_path
sys.path.append(make_splunkhome_path(['etc', 'apps', 'SA-ITOA', 'lib']))
sys.path.append(make_splunkhome_path(['etc', 'apps', 'SA-ITOA', 'lib', 'SA_ITOA_app_common']))
import itsi_path
from itsi_py3 import _
import itsi_py3
from ITOA.setup_logging import logger as itsi_logger
from ITOA.itoa_common import is_feature_enabled
from ITOA.saved_search_utility import SavedSearch
from ITOA.event_management.notable_event_aggregation_policy import NotableEventAggregationPolicy
from ITOA.event_management.notable_event_utils import get_collection_name_for_event_management_objects, \
NotableEventConfiguration
sys.path.append(make_splunkhome_path(['etc', 'apps', 'SA-UserAccess', 'lib']))
from user_access_utils import UserAccess
class NotableEventValidator(object):
"""
Notable event validator
"""
def __init__(self, session_key, logger, required_keys=None):
self.session_key = session_key
self.owner_key = 'owner'
self.status_key = 'status'
self.severity_key = 'severity'
if required_keys is None:
self.required_keys = ['_time', 'mod_time', 'title', self.owner_key, self.status_key, self.severity_key]
else:
self.required_keys = required_keys
if logger:
self.logger = logger
else:
raise ValueError(_('`logger` is not provided.'))
self.notable_configuration_object = NotableEventConfiguration(session_key, logger)
self.valid_owners = self.notable_configuration_object.get_owners()
self.valid_statuses = self.notable_configuration_object.get_statuses()
self.valid_severities = self.notable_configuration_object.get_severities()
def validate_schema(self, data):
"""
Validate schema before user CURD operation on notable event
@type data: dict
@param data: data which hold notable schema to create
@rtype: bool
@return: True - if data contains all required fields, False - otherwise or throw exception
"""
# Check for status, owner and severity is defined, otherwise set to default.
if data.get(self.owner_key) is None or data.get(self.owner_key) == '':
self.logger.info('No owner is being set for event title=%s, hence setting to default owner=%s.',
data.get('title'),
self.notable_configuration_object.get_default_owner())
data[self.owner_key] = self.notable_configuration_object.get_default_owner()
if data.get(self.status_key) is None or data.get(self.status_key) == '':
self.logger.info('No status is being set for event title=%s, hence setting to default status=%s.',
data.get('title'),
self.notable_configuration_object.get_default_status())
data[self.status_key] = self.notable_configuration_object.get_default_status()
if data.get(self.severity_key) is None or data.get(self.severity_key) == '':
self.logger.info('No severity is being set for event title=%s, hence setting to default severity=%s.',
data.get('title'),
self.notable_configuration_object.get_default_severity())
data[self.severity_key] = self.notable_configuration_object.get_default_severity()
for key in self.required_keys:
if key not in data:
message = _("%s key does not exist in the data=%s.") % (key, str(data))
self.logger.error(message)
raise ValueError(message)
# Make sure status, severity value is str
data[self.owner_key] = str(data.get(self.owner_key, ''))
data[self.status_key] = str(data.get(self.status_key, ''))
data[self.severity_key] = str(data.get(self.severity_key, ''))
# Lets have more logging and proper error if validation failed.
is_validate_owner = self.check_owner(data.get(self.owner_key))
if not is_validate_owner:
warning_message = _('Invalid owner={0} for event title={1}. '
'Unable to find owner in valid Splunk user list, '
'hence setting to default owner={2}.').format(
data.get(self.owner_key),
data.get('title'),
self.notable_configuration_object.get_default_owner())
self.logger.warning(warning_message)
data[self.owner_key] = self.notable_configuration_object.get_default_owner()
is_validate_status = self.check_status(data.get(self.status_key))
if not is_validate_status:
warning_message = _('Invalid status={0} for event title={1}. '
'Unable to find status in itsi_notable_event_status.conf, '
'hence setting to default status={2}.').format(
data.get(self.status_key),
data.get('title'),
self.notable_configuration_object.get_default_status())
self.logger.warning(warning_message)
data[self.status_key] = self.notable_configuration_object.get_default_status()
is_validate_severity = self.check_severity(data.get(self.severity_key))
if not is_validate_severity:
warning_message = _('Invalid severity={0} for event title={1}. '
'Unable to find severity in itsi_notable_event_severity.conf, '
'hence setting to default severity={2}.').format(
data.get(self.severity_key),
data.get('title'),
self.notable_configuration_object.get_default_severity())
self.logger.warning(warning_message)
data[self.severity_key] = self.notable_configuration_object.get_default_severity()
return is_validate_owner or is_validate_status or is_validate_severity
def check_severity(self, severity):
"""
Check severity
@type severity: basestring
@param severity: severity
@rtype: bool
@return: True if valid severity otherwise False
"""
return severity in self.valid_severities
def check_status(self, status):
"""
Check status
@type status: basestring
@param status: Status
@rtype: bool
@return: True if valid status otherwise False
"""
return status in self.valid_statuses
def check_owner(self, owner):
"""
Check owner is valid or not
@type owner: basestring
@param owner: owner
@rtype: bool
@return: True if valid owner otherwise False
"""
return owner in self.valid_owners
class NotableEventDefaultPoliciesLoader(object):
"""
This class is being used to load default aggregation policy
"""
DEFAULT_POLICY = """
{
"_key": "itsi_default_policy",
"group_title": "%title%",
"group_description": "%description%",
"group_status": "%status%",
"group_assignee": "%owner%",
"disabled": 0,
"is_default": 1,
"object_type": "notable_aggregation_policy",
"title": "Default Policy",
"description": "Applies to events that do not meet the criteria of any other active policy.",
"split_by_field": "source",
"priority": 5,
"group_severity": "%severity%",
"filter_criteria": {
"condition": "OR",
"items": []
},
"breaking_criteria": {
"condition": "OR",
"items": [
{
"type": "pause",
"config": {
"limit": "7200"
}
}
]
},
"rules": []
}
"""
DEFAULT_SNMP_POLICY = """
{
"_key": "itsi_default_snmp_policy",
"group_title": "%title%",
"group_description": "%description%",
"group_status": "%status%",
"group_assignee": "%owner%",
"disabled": 1,
"is_default": 0,
"object_type": "notable_aggregation_policy",
"title": "Default SNMP Policy",
"description": "Aggregation policy for SNMP traps",
"split_by_field": "node,description",
"priority": "",
"group_severity": "%severity%",
"filter_criteria": {
"items": [
{
"type": "clause",
"config": {
"items": [
{
"type": "notable_event_field",
"config": {
"operator": "=",
"field": "node",
"value": "*"
}
},
{
"type": "notable_event_field",
"config": {
"operator": "=",
"field": "description",
"value": "*"
}
}
],
"condition": "AND"
}
}
],
"condition": "OR"
},
"breaking_criteria": {
"items": [
{
"type": "clause",
"config": {
"items": [
{
"type": "notable_event_field",
"config": {
"operator": "=",
"field": "severity",
"value": "2"
}
}
],
"condition": "AND"
}
}
],
"condition": "OR"
},
"rules": [
]
}
"""
NORMALIZED_AGGREGATION_POLICY = """
{
"_key": "normalized_aggregation_policy",
"group_title": "Normalized Alert for %itsiInstance% (%itsiSubInstance%) : %itsiAlert%",
"group_description": "%last_description%",
"group_status": "%status%",
"group_assignee": "%owner%",
"disabled": 0,
"is_default": 0,
"object_type": "notable_aggregation_policy",
"title": "Normalized Policy (Splunk App for Infrastructure)",
"description": "Applies to events that contain ITSI normalized fields.",
"split_by_field": "itsiAlert,itsiInstance",
"priority": "",
"group_severity": "%last_severity%",
"group_instruction": "%last_instruction%",
"filter_criteria": {
"items": [
{
"config": {
"items": [
{
"config": {
"operator": "=",
"field": "itsiAlert",
"value": "*"
},
"type": "notable_event_field"
},
{
"config": {
"operator": "=",
"field": "itsiInstance",
"value": "*"
},
"type": "notable_event_field"
},
{
"config": {
"operator": "=",
"field": "itsiSubInstance",
"value": "*"
},
"type": "notable_event_field"
},
{
"config": {
"operator": "=",
"field": "itsiSeverity",
"value": "*"
},
"type": "notable_event_field"
}
],
"condition": "AND"
},
"type": "clause"
}
],
"condition": "OR"
},
"breaking_criteria": {
"items": [
{
"config": {
"limit": "3600"
},
"type": "pause"
},
{
"config": {
"items": [
{
"config": {
"operator": "=",
"field": "severity",
"value": "2"
},
"type": "notable_event_field"
}
],
"condition": "AND"
},
"type": "clause"
}
],
"condition": "OR"
},
"rules": [
{
"_key": "normalized_aggregation_policy_action_1",
"priority": 5,
"activation_criteria": {
"items": [
{
"type": "breaking_criteria"
},
{
"config": {
"operator": ">=",
"limit": "2"
},
"type": "notable_event_count"
}
],
"condition": "AND"
},
"actions": [
{
"items": [
{
"execution_criteria": {
"execute_on": "GROUP"
},
"type": "notable_event_change",
"config": {
"field": "status",
"value": "4"
}
}
],
"condition": "AND"
}
],
"title": "",
"description": ""
},
{
"_key": "normalized_aggregation_policy_action_2",
"priority": 5,
"activation_criteria": {
"items": [
{
"type": "breaking_criteria"
},
{
"config": {
"operator": "==",
"limit": "1"
},
"type": "notable_event_count"
}
],
"condition": "AND"
},
"actions": [
{
"items": [
{
"execution_criteria": {
"execute_on": "GROUP"
},
"type": "notable_event_change",
"config": {
"field": "status",
"value": "5"
}
}
],
"condition": "AND"
}
],
"title": "",
"description": ""
}
]
}
"""
ENTITY_TYPE_ALERT_AGGREGATION_POLICY = """
{
"_key": "entity_type_ootb_aggregation_policy",
"group_title": "Entity Type Alerts: %entity_type%, Severity: %itsiSeverity%",
"group_description": "Alerts for %entity_type% with Severity level %itsiSeverity%",
"group_status": "%status%",
"group_assignee": "%owner%",
"disabled": 0,
"is_default": 0,
"object_type": "notable_aggregation_policy",
"title": "Entity Type Alerts",
"description": "Applies to events for entities that match a custom or out-of-box Entity Type",
"split_by_field": "entity_type,itsiSeverity",
"priority": "",
"group_severity": "%last_severity%",
"group_instruction": "%last_instruction%",
"filter_criteria": {
"items": [
{
"config": {
"items": [
{
"config": {
"operator": "=",
"field": "entity_type",
"value": "*"
},
"type": "notable_event_field"
},
{
"config": {
"operator": "=",
"field": "itsiAlert",
"value": "*"
},
"type": "notable_event_field"
},
{
"config": {
"operator": "=",
"field": "itsiInstance",
"value": "*"
},
"type": "notable_event_field"
},
{
"config": {
"operator": "=",
"field": "itsiSeverity",
"value": "*"
},
"type": "notable_event_field"
},
{
"config": {
"operator": "=",
"field": "alert_source",
"value": "entity_type"
},
"type": "notable_event_field"
}
],
"condition": "AND"
},
"type": "clause"
}
],
"condition": "OR"
},
"breaking_criteria": {
"items": [
{
"config": {
"limit": "3600"
},
"type": "pause"
},
{
"config": {
"items": [
{
"config": {
"operator": "=",
"field": "severity",
"value": "2"
},
"type": "notable_event_field"
}
],
"condition": "AND"
},
"type": "clause"
}
],
"condition": "OR"
},
"rules": [
{
"_key": "entity_type_ootb_aggregation_policy_action_1",
"priority": 5,
"activation_criteria": {
"items": [
{
"type": "breaking_criteria"
},
{
"config": {
"operator": ">=",
"limit": "2"
},
"type": "notable_event_count"
}
],
"condition": "AND"
},
"actions": [
{
"items": [
{
"execution_criteria": {
"execute_on": "GROUP"
},
"type": "notable_event_change",
"config": {
"field": "status",
"value": "4"
}
}
],
"condition": "AND"
}
],
"title": "Resolve episodes that are broken and have greater than or equal to 2 events",
"description": "When the episode is broken, and the number of events in the episode is >= 2, \
change the episode status to Resolved."
},
{
"_key": "entity_type_ootb_aggregation_policy_action_2",
"priority": 5,
"activation_criteria": {
"items": [
{
"type": "breaking_criteria"
},
{
"config": {
"operator": "==",
"limit": "1"
},
"type": "notable_event_count"
}
],
"condition": "AND"
},
"actions": [
{
"items": [
{
"execution_criteria": {
"execute_on": "GROUP"
},
"type": "notable_event_change",
"config": {
"field": "status",
"value": "5"
}
}
],
"condition": "AND"
}
],
"title": "Close episodes that are broken which have only 1 event.",
"description": "If the episode is broken, and the episode has only 1 event, \
then change the episode status to Closed."
}
]
}
"""
KPI_ALERTING_POLICY = """
{
"_key": "kpi_alerting_policy",
"_user": "nobody",
"_owner": "nobody",
"title": "KPI Alerting Policy",
"group_title": "KPI Alerts from Service: %service_title%",
"sub_group_limit": "",
"group_assignee": "%last_owner%",
"group_status": "%last_status%",
"breaking_criteria": {
"condition": "OR",
"items": [
{
"type": "pause",
"config": {
"limit": "3600"
}
}
]
},
"disabled": 0,
"is_default": 0,
"priority": "",
"identifying_name": "kpi alerting policy",
"split_by_field": "service_ids",
"rules": [ ],
"object_type": "notable_aggregation_policy",
"description": "Aggregation policy for KPI state-change alerts",
"source_itsi_da": "itsi",
"group_description": "Grouped alerts from service %service_title%; most recent alert from KPI %kpi_title%",
"group_severity": "%last_severity%",
"group_instruction": "%last_instruction%",
"filter_criteria": {
"condition": "OR",
"items": [
{
"type": "clause",
"config": {
"condition": "AND",
"items": [
{
"type": "notable_event_field",
"config": {
"value": "*",
"operator": "=",
"field": "kpiid"
}
},
{
"type": "notable_event_field",
"config": {
"field": "alert_type",
"operator": "=",
"value": "KPI alert"
}
}
]
}
}
]
}
}
"""
def __init__(self, session_key, logger=None):
if not isinstance(session_key, itsi_py3.string_type):
raise TypeError(_('Invalid session key.'))
self.session_key = session_key
self.logger = logger if logger is not None else itsi_logger
self.notable_event_aggregator = NotableEventAggregationPolicy(session_key, is_validate=False)
def upload_default_policies(self):
"""
@rtype: bool
@return: True/False - if default policies are loaded successfully or not
"""
default_success = self.upload_policy("itsi_default_policy", self.DEFAULT_POLICY)
normalized_success = self.upload_policy("normalized_aggregation_policy", self.NORMALIZED_AGGREGATION_POLICY)
snmp_success = self.upload_policy("itsi_default_snmp_policy", self.DEFAULT_SNMP_POLICY)
kpi_alert_success = self.upload_policy("kpi_alerting_policy", self.KPI_ALERTING_POLICY)
entity_type_success = self.upload_policy("entity_type_ootb_aggregation_policy",
self.ENTITY_TYPE_ALERT_AGGREGATION_POLICY)
return default_success and normalized_success and snmp_success and kpi_alert_success and entity_type_success
def upload_policy(self, _id, policy):
"""
Upload policy
@type _id: basestring
@param _id: Aggregation policy id
@type policy: basestring
@param policy: Aggregation policy JSON string
@rtype: bool
@return: True/False - if policy is loaded successfully or not
"""
if not self.notable_event_aggregator.storage_interface.wait_for_storage_init(self.session_key):
raise Exception(_("KV store failed to initialize in time"))
try:
result = self.notable_event_aggregator.get(_id)
update_policy = False
if result:
if _id == "normalized_aggregation_policy" or _id == "entity_type_ootb_aggregation_policy":
for actions in result['rules']:
if '_key' not in actions or actions.get('_key') == "":
update_policy = True
if update_policy:
ret = self.notable_event_aggregator.update(_id, json.loads(policy))
return True if ret else False
else:
self.logger.info('Action rule key Found for %s' % _id)
return True
else:
self.logger.info('Found %s' % _id)
return True
else:
self.logger.info('Could not find %s' % _id)
raise ResourceNotFound('%s does not exist, this is expected when ITSI is first installed.' % _id)
except ResourceNotFound as e:
# load now
self.logger.exception(e)
self.logger.info('creating policy because we did not find %s', _id)
ret = self.notable_event_aggregator.create(json.loads(policy))
# note this is not object type
o_type = 'notable_event_aggregation_policy'
success, rval = UserAccess.bulk_update_perms(
object_ids=[_id],
acl={'read': ['*'], 'write': ['*'], 'delete': []},
object_app='itsi',
object_type=o_type,
object_storename=get_collection_name_for_event_management_objects(o_type),
session_key=self.session_key,
logger=self.logger
)
if not success:
self.logger.error('Unable to save acl for %s. Response: `%s`', _id, rval)
else:
self.logger.info('Successfully saved acl for %s. Response:`%s`', _id, rval)
return True if ret else False
class CorrelationSearchDefaultAclLoader(object):
"""
This class sets the ACL permissions for the default correlation searches
'Monitor Critical Services Based on Health Score',
'Splunk App for Infrastructure Alerts', and
'Normalized Correlation Search'
"""
DEFAULT_ACL = {'read': ['*'], 'write': ['*'], 'delete': ['*']}
DEFAULT_CS_ACL = {'read': ['*'], 'write': ['*'], 'delete': []}
def __init__(self, session_key, logger=None):
if not isinstance(session_key, itsi_py3.string_type):
raise TypeError(_('Invalid session key.'))
self.session_key = session_key
self.logger = logger if logger is not None else itsi_logger
self.retrys = 120
def default_acl_loader(self):
"""
Set perms for default correlation searches
@return:
"""
correlation_search_ids = ['Monitor Critical Services Based on Health Score',
'Splunk App for Infrastructure Alerts',
'Normalized Correlation Search',
'SNMP Traps',
'BMC Remedy Bidirectional Ticketing',
'Bidirectional Ticketing',
'Jira Bidirectional Ticketing',
'High Scale EA Backfill']
for id in correlation_search_ids:
_id = id
o_type = 'correlation_search'
rval = 'Already created'
perms = UserAccess.get_perms(
object_id=_id,
object_app='itsi',
object_type=o_type,
object_storename=get_collection_name_for_event_management_objects(o_type),
session_key=self.session_key,
logger=self.logger,
object_owner='nobody'
)
if _id == 'Bidirectional Ticketing' and not is_feature_enabled('itsi-bidirectional-ticketing',
self.session_key):
if perms:
disabled_search = SavedSearch.update_search(self.session_key, _id, disabled=1)
remove_perms = UserAccess.delete_perms(
object_id=_id,
object_app='itsi',
object_type=o_type,
object_storename=get_collection_name_for_event_management_objects(o_type),
session_key=self.session_key,
logger=self.logger,
object_owner='nobody'
)
if disabled_search and remove_perms:
self.logger.info('Successfully removed acl and disabled %s because itsi-bidirectional-ticketing'
+ ' feature flag has been disabled', _id)
else:
self.logger.error('Failed to remove acl and disable %s', _id)
continue
if _id == 'BMC Remedy Bidirectional Ticketing' and not is_feature_enabled(
'itsi-remedy-bidirectional-ticketing', self.session_key):
if perms:
disabled_search = SavedSearch.update_search(self.session_key, _id, disabled=1)
remove_perms = UserAccess.delete_perms(
object_id=_id,
object_app='itsi',
object_type=o_type,
object_storename=get_collection_name_for_event_management_objects(o_type),
session_key=self.session_key,
logger=self.logger,
object_owner='nobody'
)
if disabled_search and remove_perms:
self.logger.info('Successfully removed acl and disabled %s because'
'itsi-remedy-bidirectional-ticketing feature flag has been disabled', _id)
else:
self.logger.error('Failed to remove acl and disable %s', _id)
continue
if _id == 'Jira Bidirectional Ticketing' and not is_feature_enabled(
'itsi-jira-bidirectional-ticketing', self.session_key):
if perms:
disabled_search = SavedSearch.update_search(self.session_key, _id, disabled=1)
remove_perms = UserAccess.delete_perms(
object_id=_id,
object_app='itsi',
object_type=o_type,
object_storename=get_collection_name_for_event_management_objects(o_type),
session_key=self.session_key,
logger=self.logger,
object_owner='nobody'
)
if disabled_search and remove_perms:
self.logger.info('Successfully removed acl and disabled %s because'
'itsi-jira-bidirectional-ticketing feature flag has been disabled', _id)
else:
self.logger.error('Failed to remove acl and disable %s', _id)
continue
if _id == 'High Scale EA Backfill' and not is_feature_enabled(
'itsi-high-scale-ea', self.session_key):
if perms:
disabled_search = SavedSearch.update_search(self.session_key, _id, disabled=1)
remove_perms = UserAccess.delete_perms(
object_id=_id,
object_app='itsi',
object_type=o_type,
object_storename=get_collection_name_for_event_management_objects(o_type),
session_key=self.session_key,
logger=self.logger,
object_owner='nobody'
)
if disabled_search and remove_perms:
self.logger.info('Successfully removed acl and disabled %s because'
'itsi-high-scale-ea feature flag has been disabled', _id)
else:
self.logger.error('Failed to remove acl and disable %s', _id)
continue
while not perms and self.retrys > 0:
self.logger.info('Trying to save acl for %s.', _id)
success, rval = UserAccess.update_perms(
object_id=_id,
acl=self.DEFAULT_CS_ACL,
object_app='itsi',
object_type=o_type,
object_storename=(get_collection_name_for_event_management_objects(o_type)),
session_key=self.session_key,
logger=self.logger
)
perms = UserAccess.get_perms(
object_id=_id,
object_app='itsi',
object_type=o_type,
object_storename=get_collection_name_for_event_management_objects(o_type),
session_key=self.session_key,
logger=self.logger,
object_owner='nobody'
)
self.retrys -= 1
time.sleep(0.5)
if not perms:
self.logger.error(
'Unable to save acl for %s. Response: `%s`', _id, rval)
else:
self.logger.info(
'Successfully saved acl for %s. Response:`%s`', _id, rval)