You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
386 lines
13 KiB
386 lines
13 KiB
# Copyright (C) 2005-2024 Splunk Inc. All Rights Reserved.
|
|
|
|
from itsi.content_packs.itoa import (
|
|
get_itoa_identifier_fields,
|
|
get_itoa_object_class,
|
|
get_itoa_object_title
|
|
)
|
|
from itsi.content_packs.journal import EntryType
|
|
from itsi.content_packs.constants import ContentPackInstallOptions, ContentType
|
|
|
|
|
|
def get_conflict_resolution_strategy(resolution_type):
|
|
"""
|
|
Returns the conflict resolution strategy class based on the given resolution type.
|
|
|
|
:param resolution_type: the conflict resolution type
|
|
:type resolution_type: str
|
|
|
|
:return: the resolution strategy class
|
|
:rtype: class
|
|
"""
|
|
if resolution_type == ContentPackInstallOptions.OVERWRITE_ALL:
|
|
return OverwriteAll
|
|
return IncludeNewOnly
|
|
|
|
|
|
class ConflictResolver(object):
|
|
"""
|
|
Employs strategies to resolve conflicts for content objects that may already exist in ITSI.
|
|
"""
|
|
|
|
def __init__(self, logger, session_key, resolution_type=ContentPackInstallOptions.SKIP):
|
|
"""
|
|
:param logger: a logger instance
|
|
:type logger: Logger
|
|
|
|
:param session_key: the session key
|
|
:type session_key: str
|
|
|
|
:param resolution_type: the conflict resolution type
|
|
:type resolution_type: str
|
|
"""
|
|
self.logger = logger
|
|
self.session_key = session_key
|
|
self.resolution_type = resolution_type
|
|
self.strategy = get_conflict_resolution_strategy(resolution_type)(logger)
|
|
|
|
def process_objects(self, content_objects, journal):
|
|
"""
|
|
Employs the conflict resolution strategy on the given content objects.
|
|
|
|
:param content_objects: the content objects data
|
|
:type content_objects: dict
|
|
|
|
:param journal: a journal instance
|
|
:type journal: TransactionJournal
|
|
|
|
:return: the content objects data
|
|
:rtype: dict
|
|
"""
|
|
processed_objects = {}
|
|
|
|
for content_type, objects in content_objects.items():
|
|
processed_objects[content_type] = self.process_content_type(content_type, objects, journal=journal)
|
|
|
|
return processed_objects
|
|
|
|
def process_content_type(self, content_type, objects, journal):
|
|
"""
|
|
Employs the conflict resolution strategy on the given content type objects.
|
|
|
|
:param content_type: the content type
|
|
:type content_type: str
|
|
|
|
:param objects: the list of content object data
|
|
:type objects: list
|
|
|
|
:param journal: a journal instance
|
|
:type journal: TransactionJournal
|
|
|
|
:return: the content type objects data
|
|
:rtype: dict
|
|
"""
|
|
existing_objects = self.fetch_existing_objects(content_type, objects)
|
|
|
|
if content_type == ContentType.GLASS_TABLE_IMAGE and self.resolution_type == ContentPackInstallOptions.OVERWRITE_ALL:
|
|
# Images are a special use case since save_batch does not have ability to update existing objects.
|
|
self.clean_up_images(content_type, objects)
|
|
|
|
return self.strategy.process_objects(content_type, objects, existing_objects, journal=journal)
|
|
|
|
def fetch_existing_objects(self, content_type, objects):
|
|
"""
|
|
Returns the potential existing objects in ITSI for the given content type and list of objects.
|
|
|
|
:param content_type: the content type
|
|
:type content_type: str
|
|
|
|
:param objects: the content objects data
|
|
:type objects: list
|
|
|
|
:return: a list of object ids
|
|
:rtype: list
|
|
"""
|
|
id_fields = get_itoa_identifier_fields(content_type)
|
|
id_filter = self.construct_id_filter(objects, id_fields)
|
|
filter_data = {'$or': id_filter } if id_filter else {}
|
|
|
|
itoa_object_class = get_itoa_object_class(content_type)
|
|
|
|
handler = itoa_object_class(self.session_key, 'nobody')
|
|
|
|
return handler.get_bulk(
|
|
'nobody',
|
|
fields=['_key', 'title'],
|
|
filter_data=filter_data
|
|
)
|
|
|
|
def construct_id_filter(self, objects, id_fields):
|
|
"""
|
|
Returns the filter data containing the ids for the given list of objects.
|
|
|
|
:param objects: a list of objects data
|
|
:type objects: list
|
|
|
|
:param id_fields: a list of field names used to identify an object
|
|
:type id_fields: list
|
|
|
|
:return: a list of filter objects
|
|
:rtype: list
|
|
"""
|
|
id_filter = []
|
|
|
|
for obj in objects:
|
|
for field in id_fields:
|
|
value = obj.get(field)
|
|
if value:
|
|
id_filter.append({field: value})
|
|
else:
|
|
self.logger.info('Unable to find "%s" field for object="%s"', field, obj)
|
|
|
|
return id_filter
|
|
|
|
def clean_up_images(self, content_type, objects):
|
|
"""
|
|
Images are a special case for the overwrite resolution strategy.
|
|
|
|
To overwrite, delete the existing object on the environment first then reinstall it.
|
|
|
|
:param content_type: the content type (going to be IMAGES)
|
|
:type content_type: str
|
|
|
|
:param objects: a list of content type objects
|
|
:type objects: list
|
|
|
|
:return: the filtered list of content objects data
|
|
:rtype: list
|
|
"""
|
|
itoa_object_class = get_itoa_object_class(content_type)
|
|
image_handler = itoa_object_class(self.session_key, 'nobody')
|
|
|
|
try:
|
|
for image in objects:
|
|
image_handler.delete(image.get('_key', ''))
|
|
except Exception as e:
|
|
self.logger.error('Failed to delete content pack image objects from environment.')
|
|
self.logger.exception(e)
|
|
|
|
|
|
class IncludeNewOnly(object):
|
|
"""Conflict resolution strategy that filters out any existing objects."""
|
|
|
|
def __init__(self, logger):
|
|
"""
|
|
:param logger: a logger instance
|
|
:type logger: Logger
|
|
"""
|
|
self.logger = logger
|
|
|
|
def process_objects(self, content_type, objects, existing_objects, journal):
|
|
"""
|
|
Filters out any existing objects found in the given list of objects.
|
|
|
|
:param content_type: the content type
|
|
:type content_type: str
|
|
|
|
:param objects: a list of content type objects
|
|
:type objects: list
|
|
|
|
:param existing_objects: a list of existing objects
|
|
:type existing_objects: list
|
|
|
|
:param journal: a journal instance
|
|
:type journal: TransactionJournal
|
|
|
|
:return: the filtered list of content objects data
|
|
:rtype: list
|
|
"""
|
|
id_fields = get_itoa_identifier_fields(content_type)
|
|
id_field = id_fields[0]
|
|
|
|
processed = []
|
|
existing_ids = {}
|
|
local_ids = set()
|
|
|
|
for existing in existing_objects:
|
|
for field in id_fields:
|
|
value = existing.get(field)
|
|
if value:
|
|
existing_id_field = existing_ids.setdefault(field, set())
|
|
existing_id_field.add(value)
|
|
|
|
for obj in objects:
|
|
if self.object_exists(obj, existing_ids, id_fields, content_type, journal=journal):
|
|
continue
|
|
|
|
object_id = obj.get(id_field)
|
|
if object_id in local_ids:
|
|
error_message = f'Excluding duplicate object content_type="{content_type}" with "{id_field}"="{object_id}"'
|
|
self.logger.error(error_message)
|
|
journal.failure({
|
|
'error_message': error_message,
|
|
'type': EntryType.DUPLICATE_OBJECT,
|
|
'content_type': content_type,
|
|
'id': object_id,
|
|
'title': get_itoa_object_title(content_type, obj)
|
|
})
|
|
continue
|
|
|
|
processed.append(obj)
|
|
local_ids.add(object_id)
|
|
|
|
return processed
|
|
|
|
def object_exists(self, obj, existing_ids, id_fields, content_type, journal):
|
|
"""
|
|
Determines whether the given object already exists.
|
|
|
|
:param obj: the object data
|
|
:type obj: dict
|
|
|
|
:param existing_ids: a mapping of id fields to existing object id values
|
|
:type existing_ids: dict
|
|
|
|
:param id_fields: a list of field names used to identify the object
|
|
:type id_fields: list
|
|
|
|
:param content_type: the content type
|
|
:type content_type: str
|
|
|
|
:param journal: a journal instance
|
|
:type journal: TransactionJournal
|
|
|
|
:return: True if the object exists, False otherwise
|
|
:rtype: bool
|
|
"""
|
|
for field in id_fields:
|
|
object_id = obj.get(field)
|
|
|
|
for existing_id_values in existing_ids.get(field, set()):
|
|
if object_id in existing_id_values:
|
|
error_message = f'Excluding object content_type="{content_type}" with "{field}"="{object_id}" since it already exists'
|
|
if content_type != ContentType.GLASS_TABLE_ICON:
|
|
# for icons, we use common prefix 'da-itsi-cp-icon-' in
|
|
# itsi-cli so finding pre-existing icon is ok especially if they are
|
|
# reused. the references to such icons will continue to work
|
|
# because the _key will be the same too. This is not a
|
|
# failure.
|
|
# so we will just log this but not include as a failure
|
|
self.logger.error(error_message)
|
|
journal.failure({
|
|
'error_message': error_message,
|
|
'type': EntryType.OBJECT_ALREADY_EXISTS,
|
|
'content_type': content_type,
|
|
'id': object_id,
|
|
'title': get_itoa_object_title(content_type, obj)
|
|
})
|
|
else:
|
|
self.logger.info(error_message)
|
|
return True
|
|
|
|
return False
|
|
|
|
|
|
class OverwriteAll(object):
|
|
"""Conflict resolution strategy that overwrites all existing objects with contents of content pack."""
|
|
|
|
def __init__(self, logger):
|
|
"""
|
|
:param logger: a logger instance
|
|
:type logger: Logger
|
|
"""
|
|
self.logger = logger
|
|
|
|
def process_objects(self, content_type, objects, existing_objects, journal):
|
|
"""
|
|
Processes all objects from the content pack. Don't check for existing objects and just return all content
|
|
pack objects.
|
|
|
|
:param content_type: the content type
|
|
:type content_type: str
|
|
|
|
:param objects: a list of content type objects
|
|
:type objects: list
|
|
|
|
:param existing_objects: a list of existing objects
|
|
:type existing_objects: list
|
|
|
|
:param journal: a journal instance
|
|
:type journal: TransactionJournal
|
|
|
|
:return: the filtered list of content objects data
|
|
:rtype: list
|
|
"""
|
|
id_fields = get_itoa_identifier_fields(content_type)
|
|
id_field = id_fields[0]
|
|
|
|
processed = []
|
|
local_ids = set()
|
|
|
|
for obj in objects:
|
|
if (
|
|
content_type != ContentType.CORRELATION_SEARCH
|
|
and content_type != ContentType.GLASS_TABLE_IMAGE
|
|
and self.object_name_exists(
|
|
obj, existing_objects, content_type, journal=journal
|
|
)
|
|
):
|
|
continue
|
|
|
|
object_id = obj.get(id_field)
|
|
if object_id in local_ids:
|
|
error_message = f'Excluding duplicate object content_type="{content_type}" with "{id_field}"="{object_id}"'
|
|
self.logger.error(error_message)
|
|
journal.failure({
|
|
'error_message': error_message,
|
|
'type': EntryType.DUPLICATE_OBJECT,
|
|
'content_type': content_type,
|
|
'object_id': object_id,
|
|
'object_title': get_itoa_object_title(content_type, obj)
|
|
})
|
|
continue
|
|
|
|
processed.append(obj)
|
|
local_ids.add(object_id)
|
|
|
|
return processed
|
|
|
|
def object_name_exists(self, obj, existing_objects, content_type, journal):
|
|
"""
|
|
Determines whether the given object's name already exists for other objects.
|
|
|
|
:param obj: the object data
|
|
:type obj: dict
|
|
|
|
:param existing_objects: a list of existing objects
|
|
:type existing_objects: list
|
|
|
|
:param content_type: the content type
|
|
:type content_type: str
|
|
|
|
:param journal: a journal instance
|
|
:type journal: TransactionJournal
|
|
|
|
:return: True if the object exists, False otherwise
|
|
:rtype: bool
|
|
"""
|
|
object_name = obj.get('title')
|
|
object_id = obj.get('_key')
|
|
|
|
for existing_obj in existing_objects:
|
|
|
|
if object_name == existing_obj.get('title') and object_id != existing_obj.get('_key'):
|
|
error_message = f'Excluding object content_type="{content_type}" with name="{object_name}" since it already exists'
|
|
self.logger.error(error_message)
|
|
journal.failure({
|
|
'error_message': error_message,
|
|
'type': EntryType.OBJECT_NAME_ALREADY_EXISTS,
|
|
'content_type': content_type,
|
|
'id': object_id,
|
|
'title': get_itoa_object_title(content_type, obj)
|
|
})
|
|
return True
|
|
|
|
return False
|