You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
235 lines
13 KiB
235 lines
13 KiB
# Copyright (C) 2005-2025 Splunk Inc. All Rights Reserved.
|
|
|
|
|
|
from . import itoa_change_handler
|
|
from .backfill_cleanup_utils import cancel_or_delete_backfill_records, get_backfill_records
|
|
from ITOA.itoa_common import post_splunk_user_message
|
|
from itsi.event_management.itsi_correlation_search import ItsiCorrelationSearch
|
|
from itsi.objects.itsi_custom_threshold_windows import ItsiCustomThresholdWindows
|
|
from itsi.objects.itsi_entity_filter import ItsiEntityFilterRule
|
|
from itsi.objects.itsi_kpi_state_cache import ItsiKPIStateCache
|
|
from itsi.objects.itsi_service import ItsiService, ItsiKpi
|
|
from itsi.mad.itsi_mad_utils import (delete_mad_trending_instances,
|
|
delete_mad_cohesive_instances,
|
|
get_mad_trending_instance_kpi_mapping,
|
|
get_mad_cohesive_instance_kpi_mapping)
|
|
|
|
|
|
class KpiDeleteHandler(itoa_change_handler.ItoaChangeHandler):
|
|
"""
|
|
When one or more Kpis are deleted we need to refresh correlation searches
|
|
Disable correlation searches when not all KPIs on a correlation search are deleted
|
|
Delete correlation searches when all KPIs on a correlation search are deleted
|
|
"""
|
|
|
|
def _get_correlation_search_object(self):
|
|
"""
|
|
Return correlation search instance
|
|
@return:
|
|
@rtype: ItsiCorrelationSearch
|
|
"""
|
|
return ItsiCorrelationSearch(
|
|
self.session_key,
|
|
user='nobody',
|
|
app='itsi',
|
|
logger=self.logger
|
|
)
|
|
|
|
def deferred(self, change, transaction_id=None):
|
|
"""
|
|
Will delete/disable correlation searches based on if correlation search still has valid KPIs
|
|
@param change: The original passed to assess_impacted_objects
|
|
@param impacted_objects: The dict returned from assess_impacted_objects
|
|
@param change:
|
|
change.changed_object_type: must equal `kpi`
|
|
change.change_type: must equal `service_kpi_deletion`
|
|
change.changed_object_key: list of KPI IDs being deleted
|
|
change.change_detail: dict with the following fields:
|
|
- service_kpi_mapping: dict of deleted KPI ids keyed by associated service id
|
|
@type transaction_id: basestring
|
|
@param transaction_id: transaction id for instrumentation purposes.
|
|
@return True if all operations are success, False otherwise
|
|
@rtype: boolean
|
|
"""
|
|
if change.get('changed_object_type') != 'kpi':
|
|
raise Exception('Expected changed_object_type to be "kpi"')
|
|
|
|
if change.get('change_type') != 'service_kpi_deletion':
|
|
raise Exception('Expected change_type to be "service_kpi_deletion"')
|
|
|
|
service_interface = ItsiService(self.session_key, 'nobody')
|
|
kpi_state_cache = ItsiKPIStateCache(self.session_key, 'nobody')
|
|
correlation_interface = self._get_correlation_search_object()
|
|
correlation_searches = correlation_interface.\
|
|
get_associated_search_with_service_or_kpi(kpi_ids=change.get('changed_object_key'))
|
|
|
|
# Update dependencies
|
|
change_detail = change.get("change_detail", {})
|
|
service_kpi_mapping = change_detail.get("service_kpi_mapping", {})
|
|
all_kpis = change.get('changed_object_key', [])
|
|
backfills_to_cancel = get_backfill_records(self.session_key, all_kpis)
|
|
updated_services = {}
|
|
if len(service_kpi_mapping) > 0:
|
|
updated_services = self._get_service_dependency_updates(
|
|
service_interface, service_kpi_mapping, transaction_id=transaction_id)
|
|
|
|
# Get saved search name for kpi
|
|
saved_searches_to_delete = []
|
|
|
|
# Mad trending instances
|
|
mad_trending_instance_to_delete = []
|
|
trending_kpi_mapping = get_mad_trending_instance_kpi_mapping(self.session_key)
|
|
|
|
# Mad cohesive instances
|
|
mad_cohesive_instance_to_delete = []
|
|
cohesive_kpi_mapping = get_mad_cohesive_instance_kpi_mapping(self.session_key)
|
|
|
|
entity_filter_rule_delete_filter = {'$or': []}
|
|
|
|
for kpi_id in change.get('changed_object_key'):
|
|
saved_search_name = ItsiKpi.get_kpi_saved_search_name(kpi_id)
|
|
saved_searches_to_delete.append(saved_search_name)
|
|
if kpi_id in trending_kpi_mapping:
|
|
for trending_instance in trending_kpi_mapping.get(kpi_id):
|
|
mad_trending_instance_to_delete.append(trending_instance)
|
|
if kpi_id in cohesive_kpi_mapping:
|
|
for cohesive_instance in cohesive_kpi_mapping.get(kpi_id):
|
|
mad_cohesive_instance_to_delete.append(cohesive_instance)
|
|
entity_filter_rule_delete_filter['$or'].append({'kpi_id': kpi_id})
|
|
cache_update = kpi_state_cache.delete('nobody', kpi_id, transaction_id=transaction_id)
|
|
if cache_update:
|
|
self.logger.debug('Successfully deleted cache entry for kpi with id ({})'.format(kpi_id))
|
|
|
|
impacted_objects = {"correlation_search": correlation_searches,
|
|
"saved_searches_to_delete": saved_searches_to_delete,
|
|
"backfills_to_cancel": backfills_to_cancel,
|
|
"updated_services": updated_services,
|
|
"mad_trending_instance_to_delete": mad_trending_instance_to_delete,
|
|
"mad_cohesive_instance_to_delete": mad_cohesive_instance_to_delete}
|
|
if (len(impacted_objects.get('correlation_search', [])) == 0
|
|
and len(impacted_objects.get('saved_searches_to_delete', [])) == 0
|
|
and len(impacted_objects.get('updated_services', {})) == 0
|
|
and len(impacted_objects.get('backfills_to_cancel', [])) == 0
|
|
and len(impacted_objects.get('mad_trending_instance_to_delete', [])) == 0
|
|
and len(impacted_objects.get('mad_cohesive_instance_to_delete', [])) == 0):
|
|
return True # Noop
|
|
|
|
entity_filter_rule_object = ItsiEntityFilterRule(self.session_key, 'nobody')
|
|
entity_filter_rule_object.delete_bulk('nobody', filter_data=entity_filter_rule_delete_filter)
|
|
self.logger.info(
|
|
"Deleted {} entity filter rules associated with KPIs being deleted.".format(
|
|
len(change.get('changed_object_key'))))
|
|
|
|
service_interface = ItsiService(self.session_key, 'nobody')
|
|
status_ok = True
|
|
# update service dependencies first and attempt updating saved searches and others as best effort
|
|
updated_services = impacted_objects.get('updated_services', {})
|
|
if len(updated_services) > 0:
|
|
status_ok = service_interface.batch_save_backend('nobody',
|
|
list(updated_services.values()),
|
|
transaction_id=transaction_id) and status_ok
|
|
|
|
# Disassociate the CTW Objects for deleted KPIS
|
|
self.logger.info("Calling Disassociation for the KPI from CTW Objects")
|
|
for kpi_id in change.get('changed_object_key'):
|
|
itsi_ctw_interface = ItsiCustomThresholdWindows(self.session_key, 'nobody')
|
|
itsi_ctw_interface.disconnect_deleted_kpis_from_all_ctws('nobody', kpi_id, transaction_id=transaction_id)
|
|
|
|
correlation_interface = self._get_correlation_search_object()
|
|
correlation_searches = impacted_objects.get('correlation_search', [])
|
|
try:
|
|
correlation_interface. update_service_or_kpi_in_correlation_search(
|
|
'kpiid', ids=change.get('changed_object_key'), searches=correlation_searches)
|
|
except Exception:
|
|
message = (
|
|
'Cannot disable/delete all impacted correlation searches. '
|
|
'We recommend that you update the impacted correlation searches by '
|
|
'the UI. Correlation search names are: {0}').format(
|
|
[search.get('name', '') for search in correlation_searches])
|
|
self.logger.exception(message)
|
|
post_splunk_user_message(message=message, session_key=self.session_key)
|
|
status_ok = False
|
|
|
|
cancel_or_delete_backfill_records(impacted_objects.get('backfills_to_cancel', []), self.logger)
|
|
|
|
# Delete MAD instances
|
|
mad_trending_instances_list = impacted_objects.get('mad_trending_instance_to_delete', [])
|
|
delete_mad_trending_instances(self.session_key, mad_trending_instances_list)
|
|
|
|
mad_cohesive_instances_list = impacted_objects.get('mad_cohesive_instance_to_delete', [])
|
|
delete_mad_cohesive_instances(self.session_key, mad_cohesive_instances_list)
|
|
|
|
# Delete saved searches for kpis as a best effort
|
|
if not service_interface.delete_kpi_saved_searches(impacted_objects.get("saved_searches_to_delete", [])):
|
|
message = ('Cannot delete all KPI saved searches. We recommend that you manually delete them. '
|
|
'Saves search names are: {0}').format(
|
|
impacted_objects.get("saved_searches_to_delete", [])
|
|
)
|
|
self.logger.error(message)
|
|
post_splunk_user_message(message=message, session_key=self.session_key)
|
|
status_ok = False
|
|
|
|
return status_ok
|
|
|
|
def _get_service_dependency_updates(self, service_interface, service_kpi_mapping, transaction_id=None):
|
|
"""
|
|
Find which services need to be updated based on kpi deletion
|
|
@param service_interface: The itoa_object instance to fetch services
|
|
@param service_kpi_mapping: dict of service key to list of kpis that have been deleted
|
|
@return: dict of service key to services that need to be updated
|
|
"""
|
|
updated_services = {}
|
|
for service_key, deleted_kpi_list in service_kpi_mapping.items():
|
|
# get service from updated_services if it was already updated in an earlier iteration
|
|
if service_key in updated_services:
|
|
service = updated_services.get(service_key)
|
|
# otherwise fetch from kvstore
|
|
else:
|
|
service = service_interface.get('nobody', service_key, transaction_id=transaction_id)
|
|
# if service had no dependent services then we can skip it
|
|
depending_on_me = service.get('services_depending_on_me')
|
|
if depending_on_me is None or len(depending_on_me) == 0:
|
|
continue
|
|
# loop through all dependent services, check for intersection of
|
|
for dependency in depending_on_me:
|
|
depending_kpis = dependency.get('kpis_depending_on')
|
|
# this will return only items in depending_kpis that are not in deleted_kpi_list
|
|
changed_depending_kpis = list(set(depending_kpis) - set(deleted_kpi_list))
|
|
# Nothing in depending_kpis is in deleted_kpi_list, we can skip this dependency
|
|
if len(depending_kpis) == len(changed_depending_kpis):
|
|
continue
|
|
# update this service with the deleted_kpi_list kpis removed
|
|
dependency['kpis_depending_on'] = changed_depending_kpis
|
|
updated_services[service_key] = service
|
|
# need to update target service as well
|
|
target_service_key = dependency.get('serviceid')
|
|
if target_service_key in updated_services:
|
|
target_service = updated_services.get(target_service_key)
|
|
else:
|
|
target_service = service_interface.get('nobody', target_service_key, transaction_id=transaction_id)
|
|
target_service_depends_on = target_service.get('services_depends_on')
|
|
matched_dependencies = [d for d in target_service_depends_on if d.get('serviceid') == service_key]
|
|
if len(matched_dependencies) > 0:
|
|
# There can be only one! - The Highlander
|
|
if len(matched_dependencies) > 1:
|
|
self.logger.error('Service "%s" referenced more than once in services_depends_on', service_key)
|
|
target_service_kpis_depending_on = matched_dependencies[0].get('kpis_depending_on')
|
|
# remove any kpis referenced in deleted_kpi_List
|
|
target_service_kpis_depending_on = list(
|
|
set(target_service_kpis_depending_on) - set(deleted_kpi_list))
|
|
matched_dependencies[0]['kpis_depending_on'] = target_service_kpis_depending_on
|
|
updated_services[target_service_key] = target_service
|
|
else:
|
|
self.logger.error('Could not find service %s to update', service_key)
|
|
|
|
# remove service dependencies if kpis_depending_on me is empty after above changes
|
|
for updated_service in list(updated_services.values()):
|
|
depends_on = updated_service.get('services_depends_on', [])
|
|
depending_on_me = updated_service.get('services_depending_on_me', [])
|
|
depends_on = [d for d in depends_on if len(d.get('kpis_depending_on')) > 0]
|
|
updated_service['services_depends_on'] = depends_on
|
|
depending_on_me = [d for d in depending_on_me if len(d.get('kpis_depending_on')) > 0]
|
|
updated_service['services_depending_on_me'] = depending_on_me
|
|
|
|
return updated_services
|