# Copyright (C) 2005-2024 Splunk Inc. All Rights Reserved. from itsi.content_packs.itoa import ( get_itoa_identifier_fields, get_itoa_object_class, get_itoa_object_title ) from itsi.content_packs.journal import EntryType from itsi.content_packs.constants import ContentPackInstallOptions, ContentType def get_conflict_resolution_strategy(resolution_type): """ Returns the conflict resolution strategy class based on the given resolution type. :param resolution_type: the conflict resolution type :type resolution_type: str :return: the resolution strategy class :rtype: class """ if resolution_type == ContentPackInstallOptions.OVERWRITE_ALL: return OverwriteAll return IncludeNewOnly class ConflictResolver(object): """ Employs strategies to resolve conflicts for content objects that may already exist in ITSI. """ def __init__(self, logger, session_key, resolution_type=ContentPackInstallOptions.SKIP): """ :param logger: a logger instance :type logger: Logger :param session_key: the session key :type session_key: str :param resolution_type: the conflict resolution type :type resolution_type: str """ self.logger = logger self.session_key = session_key self.resolution_type = resolution_type self.strategy = get_conflict_resolution_strategy(resolution_type)(logger) def process_objects(self, content_objects, journal): """ Employs the conflict resolution strategy on the given content objects. :param content_objects: the content objects data :type content_objects: dict :param journal: a journal instance :type journal: TransactionJournal :return: the content objects data :rtype: dict """ processed_objects = {} for content_type, objects in content_objects.items(): processed_objects[content_type] = self.process_content_type(content_type, objects, journal=journal) return processed_objects def process_content_type(self, content_type, objects, journal): """ Employs the conflict resolution strategy on the given content type objects. :param content_type: the content type :type content_type: str :param objects: the list of content object data :type objects: list :param journal: a journal instance :type journal: TransactionJournal :return: the content type objects data :rtype: dict """ existing_objects = self.fetch_existing_objects(content_type, objects) if content_type == ContentType.GLASS_TABLE_IMAGE and self.resolution_type == ContentPackInstallOptions.OVERWRITE_ALL: # Images are a special use case since save_batch does not have ability to update existing objects. self.clean_up_images(content_type, objects) return self.strategy.process_objects(content_type, objects, existing_objects, journal=journal) def fetch_existing_objects(self, content_type, objects): """ Returns the potential existing objects in ITSI for the given content type and list of objects. :param content_type: the content type :type content_type: str :param objects: the content objects data :type objects: list :return: a list of object ids :rtype: list """ id_fields = get_itoa_identifier_fields(content_type) id_filter = self.construct_id_filter(objects, id_fields) filter_data = {'$or': id_filter } if id_filter else {} itoa_object_class = get_itoa_object_class(content_type) handler = itoa_object_class(self.session_key, 'nobody') return handler.get_bulk( 'nobody', fields=['_key', 'title'], filter_data=filter_data ) def construct_id_filter(self, objects, id_fields): """ Returns the filter data containing the ids for the given list of objects. :param objects: a list of objects data :type objects: list :param id_fields: a list of field names used to identify an object :type id_fields: list :return: a list of filter objects :rtype: list """ id_filter = [] for obj in objects: for field in id_fields: value = obj.get(field) if value: id_filter.append({field: value}) else: self.logger.info('Unable to find "%s" field for object="%s"', field, obj) return id_filter def clean_up_images(self, content_type, objects): """ Images are a special case for the overwrite resolution strategy. To overwrite, delete the existing object on the environment first then reinstall it. :param content_type: the content type (going to be IMAGES) :type content_type: str :param objects: a list of content type objects :type objects: list :return: the filtered list of content objects data :rtype: list """ itoa_object_class = get_itoa_object_class(content_type) image_handler = itoa_object_class(self.session_key, 'nobody') try: for image in objects: image_handler.delete(image.get('_key', '')) except Exception as e: self.logger.error('Failed to delete content pack image objects from environment.') self.logger.exception(e) class IncludeNewOnly(object): """Conflict resolution strategy that filters out any existing objects.""" def __init__(self, logger): """ :param logger: a logger instance :type logger: Logger """ self.logger = logger def process_objects(self, content_type, objects, existing_objects, journal): """ Filters out any existing objects found in the given list of objects. :param content_type: the content type :type content_type: str :param objects: a list of content type objects :type objects: list :param existing_objects: a list of existing objects :type existing_objects: list :param journal: a journal instance :type journal: TransactionJournal :return: the filtered list of content objects data :rtype: list """ id_fields = get_itoa_identifier_fields(content_type) id_field = id_fields[0] processed = [] existing_ids = {} local_ids = set() for existing in existing_objects: for field in id_fields: value = existing.get(field) if value: existing_id_field = existing_ids.setdefault(field, set()) existing_id_field.add(value) for obj in objects: if self.object_exists(obj, existing_ids, id_fields, content_type, journal=journal): continue object_id = obj.get(id_field) if object_id in local_ids: error_message = f'Excluding duplicate object content_type="{content_type}" with "{id_field}"="{object_id}"' self.logger.error(error_message) journal.failure({ 'error_message': error_message, 'type': EntryType.DUPLICATE_OBJECT, 'content_type': content_type, 'id': object_id, 'title': get_itoa_object_title(content_type, obj) }) continue processed.append(obj) local_ids.add(object_id) return processed def object_exists(self, obj, existing_ids, id_fields, content_type, journal): """ Determines whether the given object already exists. :param obj: the object data :type obj: dict :param existing_ids: a mapping of id fields to existing object id values :type existing_ids: dict :param id_fields: a list of field names used to identify the object :type id_fields: list :param content_type: the content type :type content_type: str :param journal: a journal instance :type journal: TransactionJournal :return: True if the object exists, False otherwise :rtype: bool """ for field in id_fields: object_id = obj.get(field) for existing_id_values in existing_ids.get(field, set()): if object_id in existing_id_values: error_message = f'Excluding object content_type="{content_type}" with "{field}"="{object_id}" since it already exists' if content_type != ContentType.GLASS_TABLE_ICON: # for icons, we use common prefix 'da-itsi-cp-icon-' in # itsi-cli so finding pre-existing icon is ok especially if they are # reused. the references to such icons will continue to work # because the _key will be the same too. This is not a # failure. # so we will just log this but not include as a failure self.logger.error(error_message) journal.failure({ 'error_message': error_message, 'type': EntryType.OBJECT_ALREADY_EXISTS, 'content_type': content_type, 'id': object_id, 'title': get_itoa_object_title(content_type, obj) }) else: self.logger.info(error_message) return True return False class OverwriteAll(object): """Conflict resolution strategy that overwrites all existing objects with contents of content pack.""" def __init__(self, logger): """ :param logger: a logger instance :type logger: Logger """ self.logger = logger def process_objects(self, content_type, objects, existing_objects, journal): """ Processes all objects from the content pack. Don't check for existing objects and just return all content pack objects. :param content_type: the content type :type content_type: str :param objects: a list of content type objects :type objects: list :param existing_objects: a list of existing objects :type existing_objects: list :param journal: a journal instance :type journal: TransactionJournal :return: the filtered list of content objects data :rtype: list """ id_fields = get_itoa_identifier_fields(content_type) id_field = id_fields[0] processed = [] local_ids = set() for obj in objects: if ( content_type != ContentType.CORRELATION_SEARCH and content_type != ContentType.GLASS_TABLE_IMAGE and self.object_name_exists( obj, existing_objects, content_type, journal=journal ) ): continue object_id = obj.get(id_field) if object_id in local_ids: error_message = f'Excluding duplicate object content_type="{content_type}" with "{id_field}"="{object_id}"' self.logger.error(error_message) journal.failure({ 'error_message': error_message, 'type': EntryType.DUPLICATE_OBJECT, 'content_type': content_type, 'object_id': object_id, 'object_title': get_itoa_object_title(content_type, obj) }) continue processed.append(obj) local_ids.add(object_id) return processed def object_name_exists(self, obj, existing_objects, content_type, journal): """ Determines whether the given object's name already exists for other objects. :param obj: the object data :type obj: dict :param existing_objects: a list of existing objects :type existing_objects: list :param content_type: the content type :type content_type: str :param journal: a journal instance :type journal: TransactionJournal :return: True if the object exists, False otherwise :rtype: bool """ object_name = obj.get('title') object_id = obj.get('_key') for existing_obj in existing_objects: if object_name == existing_obj.get('title') and object_id != existing_obj.get('_key'): error_message = f'Excluding object content_type="{content_type}" with name="{object_name}" since it already exists' self.logger.error(error_message) journal.failure({ 'error_message': error_message, 'type': EntryType.OBJECT_NAME_ALREADY_EXISTS, 'content_type': content_type, 'id': object_id, 'title': get_itoa_object_title(content_type, obj) }) return True return False