# Copyright (C) 2005-2024 Splunk Inc. All Rights Reserved. import sys import re import http.client import json import urllib.parse from itsi_py3 import _ from splunk.clilib import bundle_paths from splunk.clilib.bundle_paths import make_splunkhome_path sys.path.append(make_splunkhome_path(['etc', 'apps', 'SA-ITOA', 'lib'])) from itsi.objects.object_manifest import object_manifest as itoa_objects from ITOA.event_management.event_management_object_manifest import object_manifest as event_management_objects from ITOA.controller_utils import ITOAError from splunk import ResourceNotFound import splunk.rest as rest import os import shutil itoa_objects.update(event_management_objects) class MissingCPConfigFile(Exception): # When an expected config file is not found pass class InvalidCPVersion(Exception): # When the content pack version is not valid pass class InvalidSplunkObjectFormat(Exception): # When an invalid format splunk object is encountered # expecting: # {'name': 'xxxx', 'app':'yyyy'} pass class InvalidTitle(Exception): # When the overview is too large to fit into kv store pass class InvalidDescription(Exception): # When the description is too large to fit into kv store pass class InvalidOverview(Exception): # When the overview is too large to fit into kv store pass class AuthorshipUtils(object): def __init__(self, logger, session_key, authored_content_key='', content_pack_id='', cp_root_path=''): self.logger = logger self.session_key = session_key self.authored_content_key = authored_content_key self.content_pack_id = content_pack_id self.cp_root_path = cp_root_path def validate_itsi_objects_intact(self, itsi_objects={}, user_selected_objects={}): """ Validate if ITSI objects within a content authorship object is intact Authorship has a list of references to different types of ITSI objects Validates if the referencing ITSI objects still exist in the KVStore by retrieving object titles @type: dict @param itsi_objects: ITSI object references eg: { 'service': [ '9a7b10a1-52d8-4216-af15-711fd917a46e', 'dfsfa1-52d8-4216-af1dsfakljio-2394lk' ], 'kpi_base_search': ['DA-ITSI-APPSERVER_Performance_Runtime'], 'kpi_threshold_template': ['kpi_threshold_template_7'], 'base_service_template': [], 'glass_table': [], 'deep_dive': [], 'entity_type': ['k8s_node'], 'home_view': [], 'notable_aggregation_policy': [], 'correlation_search': ['correlation_search1'], 'event_management_state': [], 'team': [] } @param user_selected_objects: Objects that the user clicked on specifically e.g user_selected_objects : { 'itsi_objects': { 'base_service_template': [], 'correlation_search': [], 'deep_dive': [], 'entity_type': [], 'event_management_state': [], 'glass_table': [], 'home_view': [], 'kpi_base_search': [], 'kpi_threshold_template': [], 'notable_aggregation_policy': [], 'service': [], 'team': [] }, 'splunk_objects': { 'dashboards': [], 'lookups': [], 'macros': [], 'props': [], 'savedsearches': [], 'transforms': [] } } @rtype: object {bool, dict, dict/None, dict} @return: {is_intact, itsi_objects_with_title, all_missing_objects, final_user_selected_objects_itsi} """ itsi_objects_with_title = { 'service': [], 'kpi_base_search': [], 'kpi_threshold_template': [], 'base_service_template': [], 'glass_table': [], 'deep_dive': [], 'entity_type': [], 'home_view': [], 'notable_aggregation_policy': [], 'correlation_search': [], 'event_management_state': [], 'team': [] } all_missing_objects = {} for itsi_object_type, key_list in itsi_objects.items(): ''' Ignore kpi_threshold_template with id as "custom". This is due to bad data which is reported by some customers over slack. Refer files "kpi_thresholds_template_update_handler.py" and "itsi_migration.py" which mentions about relation between "custom" and KPI Threshold Templates ''' if itsi_object_type == 'kpi_threshold_template': key_list = [key for key in key_list if key != "custom"] # if no values for this object type, do nothing if not key_list: continue try: if itsi_object_type == 'notable_aggregation_policy': itoa_object_class = itoa_objects.get('notable_event_aggregation_policy') obj = itoa_object_class(self.session_key, 'nobody') results = obj.get_bulk( key_list, fields=['_key', 'title'] ) elif itsi_object_type == 'correlation_search': itoa_object_class = itoa_objects.get(itsi_object_type) obj = itoa_object_class(self.session_key, 'nobody') results = obj.get_bulk( key_list, ) else: itoa_object_class = itoa_objects.get(itsi_object_type) obj = itoa_object_class(self.session_key, 'nobody') modified_key_list = [ {'_key': key} for key in key_list ] filter_data = {'$or': modified_key_list} results = obj.get_bulk( 'nobody', fields=['_key', 'title'], filter_data=filter_data ) if len(results) != len(key_list): missing_objects = self._get_missing_objects(itsi_object_type, key_list, results) all_missing_objects.update({itsi_object_type: missing_objects}) for item in results: if itsi_object_type == 'correlation_search': itsi_objects_with_title[itsi_object_type].append({ item.get('name'): item.get('name') }) else: itsi_objects_with_title[itsi_object_type].append({ item.get('_key'): item.get('title') }) except Exception as e: self.logger.exception(f'{self.authored_content_key} Failed to retrieve objects for object ' f'type {itsi_object_type}: exception {e}') raise ITOAError(status=http.client.BAD_REQUEST, message=_(f"Failed to retrieve content pack object from KVStore. " f"Check the logs for details. type {itsi_object_type}: exception {e}")) final_user_selected_objects = user_selected_objects if all_missing_objects and len(all_missing_objects) > 0: try: if user_selected_objects: temp_missing_object = all_missing_objects for key in temp_missing_object.keys(): for item in temp_missing_object[key]: if item not in itsi_objects_with_title[key] and item in final_user_selected_objects['itsi_objects'][key]: final_user_selected_objects['itsi_objects'][key].remove(item) return {'is_intact': False, 'itsi_objects_with_title' : itsi_objects_with_title, 'all_missing_objects': all_missing_objects, 'final_user_selected_objects_itsi': final_user_selected_objects } except Exception as e: self.logger.exception(f'{self.authored_content_key} Failed to process missing objects ' f'for user_selected_objects. exception {e}') raise ITOAError(status=http.client.BAD_REQUEST, message=_(f"Failure during user_selected_objects missing object processing " f"Check the logs for details. exception {e}")) else: return {'is_intact': True, 'itsi_objects_with_title' : itsi_objects_with_title, 'all_missing_objects': {}, 'final_user_selected_objects_itsi': final_user_selected_objects } @staticmethod def _get_missing_objects(itsi_object_type, expected_keys, actual_objects): """ Cross checking the list of expected objects listed in the content pack authorship meta data against the list of objects actually existing on the system and return the missing object list :param itsi_object_type: type of itsi objects (i.e. key in object_manifest, base_service_template, service, etc) :param expected_keys: list of keys/ids/names listed in content pack authorship meta for the given object_type :param actual_objects: list of objects actually returned from querying the expected_keys :return: missing_keys :rtype: list of _key or name (for correlation search) """ if itsi_object_type == 'correlation_search': actual_keys = [item.get('name') for item in actual_objects] else: actual_keys = [item.get('_key') for item in actual_objects] missing_keys = [key for key in expected_keys if key not in actual_keys] return missing_keys @staticmethod def is_valid_cp_version(cp_version): """ Verify if the given content pack version string is valid :param str cp_version: a content pack version string to be validated :return: whether the version string is valid or not :rtype: boolean """ pattern = re.compile(r'^([1-9]\d*|0)(\.\d+){2}$') if pattern.fullmatch(cp_version): return True else: return False def _make_request(self, method, relative_url, args={}): """ Perform a request to a Splunk endpoint pool. @type: string @param method: GET or POST @type: string @param relative_url: @param: dict @param args: args of the request @returns: content of the request @rtype: dict @raises: exception """ try: if method == 'GET': path = rest.makeSplunkdUri() + relative_url args['output_mode'] = 'json' response, content = rest.simpleRequest( path, method='GET', getargs=args, sessionKey=self.session_key, raiseAllErrors=False ) self.logger.info('%s GET for %s response %s, content %s', self.authored_content_key, relative_url, response, content) content = json.loads(content) return content elif method == 'POST': path = rest.makeSplunkdUri() + relative_url args['output_mode'] = 'json' response, content = rest.simpleRequest( path, method='POST', postargs=args, sessionKey=self.session_key, raiseAllErrors=False ) self.logger.info('%s POST for %s response %s, content %s, args %s', self.authored_content_key, relative_url, response, content, args) content = json.loads(content) return content except Exception as e: self.logger.exception( f'{self.authored_content_key} Encountered exception for method:{method} path:{path} exception:{e}') raise e def get_missing_splunk_objects(self, object_type, object_list_to_check): """find missing splunk objects in the content authorship metadata references Args: object_type (str): type of object (i.e: dashboards, lookups, props, etc) object_list_to_check (_type_): the list of dashboards, lookups etc to check Returns: dict: dict of object_type to list of missing objects """ missing_object_list = [] if object_type == 'dashboards': get_object_partial_path = 'servicesNS/nobody/{}/data/ui/views/{}' # if the object_type is 'savedsearch', get only disabled property for that stanza # since we are checking only if the object exists or not; not interested in all the # data content elif object_type == 'savedsearches': get_object_partial_path = 'servicesNS/nobody/{}/properties/{}/{}/disabled' else: get_object_partial_path = 'servicesNS/nobody/{}/configs/conf-{}/{}' for splunk_object in object_list_to_check: object_name = splunk_object.get('name') object_app = splunk_object.get('app') if object_type == 'dashboards': conf_object_full_path = get_object_partial_path.format(object_app, urllib.parse.quote(object_name)) else: conf_object_full_path = get_object_partial_path.format(object_app, object_type, urllib.parse.quote(object_name)) try: self._make_request('GET', conf_object_full_path) except ResourceNotFound: missing_object_list.append({ 'name': object_name, 'app': object_app }) except Exception as e: self.logger.exception( f'{self.authored_content_key} Retrieving splunk object for object type:{object_type} ' f'in app:{object_app} with name:{object_name} ' f'encountered exception {e}' ) return missing_object_list def validate_splunk_objects_intact(self, splunk_objects={}, user_selected_objects={}): """ Validate if the splunk objects within a content authorship object are present on the instance @type: dict @param splunk_objects: splunk object references eg: "splunk_objects": { "dashboards": [ { "name": "test_dashboard", "app": "itsi" } ], "lookups": [ { "name": "Entity.csv", "app": "DA-ITSI-CP-vmware-dashboards" }, { "name": "well_known_ports.csv", "app": "DA-ITSI-CP-aws-dashboards" } ] } @param user_selected_objects: Objects that the user clicked on specifically e.g user_selected_objects : { 'itsi_objects': { 'base_service_template': [], 'correlation_search': [], 'deep_dive': [], 'entity_type': [], 'event_management_state': [], 'glass_table': [], 'home_view': [], 'kpi_base_search': [], 'kpi_threshold_template': [], 'notable_aggregation_policy': [], 'service': [], 'team': [] }, 'splunk_objects': { 'dashboards': [], 'lookups': [], 'macros': [], 'props': [], 'savedsearches': [], 'transforms': [] } } @rtype: object of bool, dict/None @return: {is_intact, all_missing_objects} """ all_missing_objects = {} final_user_selected_objects = user_selected_objects temp_splunk_objects = splunk_objects try: for object_type, object_info_list in splunk_objects.items(): if not object_info_list: continue missing_objects = self.get_missing_splunk_objects(object_type, object_info_list) if missing_objects and len(missing_objects) > 0: all_missing_objects[object_type] = missing_objects if all_missing_objects and len(all_missing_objects) > 0: for keys in all_missing_objects.keys(): for item in all_missing_objects[keys]: if item in splunk_objects[keys]: temp_splunk_objects[keys].remove(item) if final_user_selected_objects: final_user_selected_objects['splunk_objects'][keys].remove(item) return {'is_intact': False, 'all_missing_objects': all_missing_objects, 'splunk_objects': temp_splunk_objects, 'final_user_selected_objects_splunk': final_user_selected_objects, } else: return {'is_intact': True, 'all_missing_objects': {}, 'splunk_objects': temp_splunk_objects, 'final_user_selected_objects_splunk': final_user_selected_objects, } except Exception as e: self.logger.exception(f'{self.authored_content_key}: Failed to retrieve splunk object. {e}') raise ITOAError(status=http.client.BAD_REQUEST, message=_(f'Failed to retrieve content pack splunk object. ' f'Check the logs for details. exception={e}')) def is_custom_image_in_kvstore(self, source): """ Determine if the icon/image/choropleth is a custom icon/image/choropleth uploaded on kvstore If the UUID of the source begins with "icon-" then it's default. If it's plain UUID, then it's custom. @type source: str @param source: source points to the URL or kvstore UUID of the visualization @rtype: bool @return: whether the source points to a custom image uploaded on kvstore """ if not source: return False if 'kvstore://' in source: res = source.split("kvstore://", 1) file_id = res[1] if not file_id: return False # default icon ID will have UUID preceding a descriptive name like icon-active-directory__ or # icon-datatacenter__, whereas the ones uploaded by the customer will just be a uuid if file_id.startswith('icon') or '__' in file_id: self.logger.info("Content pack: %s. Following is not a custom image: %s", self.authored_content_key, source) return False else: self.logger.info("Content pack: %s. Following is a custom image: %s", self.authored_content_key, source) return True return False def is_image_on_disk(self, source): """ Determine if the icon/image/choropleth is stored locally on the disk If the URL of the source starts with "/static" then we know that it's stored locally on the disk. @type source: str @param source: source points to the URL of the visualization @rtype: bool @returns: whether the source points to an icon/image/choropleth's static location locally on the disk """ if not source: return False # TODO: check if this works for Windows if source.startswith('/static') or source.startswith('static') or source.startswith(r'\\static'): return True return False def get_image_path(self, image_url): """ Returns the actual image file path based on the image URL provided If the image_url is "/static/app/itsi/dashboard_images/background_image.jpeg", the image path will be "$SPLUNK_HOME/etc/apps/itsi/appserver/static/dashboard_images/background_image.jpeg" @type image_url: string @param image_url: URL of the image location. Example of image_url: "/static/app/itsi/dashboard_images/background_image.jpeg" @rtype: string @returns: the actual file path relative to $SPLUNK_HOME for the given image_url """ if not image_url: return "" pattern_match = '' sub_image_url = '' first_slash_idx = '' if os.path.sep == '/': # Linux-type system pattern_match = re.search("static/app/", image_url) sub_image_url = image_url[pattern_match.end():] first_slash_idx = sub_image_url.find(os.path.sep) else: # Windows system # TODO: check if this is working for Windows pattern_match = re.search(r'static\\app\\', image_url) sub_image_url = image_url[pattern_match.end():] first_slash_idx = sub_image_url.find(os.path.sep) app_name = sub_image_url[:first_slash_idx] # Fetch the file path after the app name. # For example, for "/static/app/itsi/dashboard_images/background_image.jpeg" the file path will be # "dashboard_images/background_image.jpeg" # In Windows, "\\" will be regarded as one character. Hence, below code ([first_slash_idx + 1:]) # should provide the desired index on both Linux-type and Windows environment. file_path = sub_image_url[first_slash_idx + 1:] full_image_path = bundle_paths.make_splunkhome_path(['etc', 'apps', app_name, 'appserver', 'static', file_path]) self.logger.info('Content pack: %s. Fetched full path for the given image url. ' 'Image URL: %s. full image path: %s', self.authored_content_key, image_url, full_image_path) return full_image_path @staticmethod def _extract_file_name_from_path(filepath): """ Takes a file path and returns the file name from that path. Assumption is that the value after the last slash is the file name. @type filepath: str @param filepath: file path. Example: "/opt/etc/apps/itsi/appserver/static/images/image.jpeg" @rtype: str @returns: file name in the given file path. For the above example, it will return 'image.jpeg' """ if not filepath: return "" # remove the trailing slashes and then split the filepath tokens = filepath.strip(os.path.sep).split(os.path.sep) if len(tokens) > 1: return tokens[len(tokens) - 1] return "" def extract_image(self, source, type): """ It determines what data should be embedded in the dashboard for the visualization's source. If the source is custom KV Store UUID, then the source to be embedded in the dashboard would be base64 encoding stored in the KV Store for the given UUID. If the source is static file location, then the source to be embedded in the dashboard would be the static file location inside the Content Pack's app. File location inside the content pack app would be: "/static/app/{content_pack_id}/dashboard_images/{file_name}". For all other remaining sources, they would remain as is. @type source: str @param source: Source of the icon/image/choropleth. The source could point to one of the following: kvstore UUID, URL of local static location, base64 encoding or an HTTP URL. @rtype: str @returns: source to be embedded in the dashboard. The source to be embedded could be one of the following: default kvstore UUID, URL of local static location, base64 encoding or an HTTP URL. """ if not source: return '' if self.is_custom_image_in_kvstore(source): self.logger.info("Content pack: %s. This is a custom image is in kvstore. Image source: %s", self.authored_content_key, source) # get the image, get base64 res = source.split('kvstore://', 1) file_id = res[1] file_url = "" if type and 'icon' in type: # icons are stored in splunk-dashboard-icons collection file_url = '/servicesNS/nobody/splunk-dashboard-studio/storage/collections/data/' \ 'splunk-dashboard-icons/' + file_id else: # background images and images viz are stored in splunk-dashboard-images collection file_url = '/servicesNS/nobody/splunk-dashboard-studio/storage/collections/data/' \ 'splunk-dashboard-images/' + file_id self.logger.info('Content pack: %s. Getting custom imgage from kv store. KV Store image url: %s', self.authored_content_key, file_url) response = self._make_request('GET', file_url) self.logger.debug("Content pack: %s. Base64 encoded image from kvstore: %s", self.authored_content_key, response) base64_img = response['dataURI'] return base64_img elif self.is_image_on_disk(source): self.logger.info("Content pack: %s. Image is on disk. Image source: %s", self.authored_content_key, source) # Copy the image to the Content Pack App's appserver/static/dashboard_images directory. Return the static # location of the image relative to the Content Pack App's static directory. if not source: self.logger.warn('Content pack: %s. No URL provided for the image on disk', self.authored_content_key) return '' full_image_path = self.get_image_path(source) cp_static_dashboard_images_path = os.path.join(self.cp_root_path, 'appserver', 'static', 'dashboard_images') os.makedirs(cp_static_dashboard_images_path, exist_ok=True) file_name = self._extract_file_name_from_path(full_image_path) cp_static_file_path = os.path.join(cp_static_dashboard_images_path, file_name) shutil.copy(full_image_path, cp_static_file_path) self.logger.info('Content pack: %s. Local image copied over from %s to %s ', self.authored_content_key, full_image_path, cp_static_file_path) # path that will be embedded in the dashboard is "/static/app/{content_pack_id}/dashboard_image/{file_name}" # static_path_under_cp = '/static/app/{}/dashboard_images/{}'.format(self.content_pack_id, file_name) static_path_under_cp = os.path.join(os.path.sep, 'static', 'app', self.content_pack_id, 'dashboard_images', file_name) self.logger.info('Content pack: %s. Local image static path to be embedded in dashboard: %s.', self.authored_content_key, static_path_under_cp) return static_path_under_cp return source def get_image_source(self, visualization): """ It replaces visualization's source with source that would work out of the box when the Content Pack App is installed on a different instance. @type visualization: dict @param visualization: a visualization from the list of visualizations used in the dashboard Visualization could be one of the following types: viz.img(or splunk.image), icon, choropleth, or layout with backgroundImage @rtype: dict @returns: visualization with modified source for image/icon/choropleth. """ if not visualization: return {} source = '' if 'type' in visualization: type = visualization['type'] # icon svg (type is splunk.singlevalueicon) if 'icon' in type: source = None if 'options' in visualization and 'icon' in visualization['options']: source = visualization['options']['icon'] self.logger.info('Content pack: %s. Parsing dashboard. icon source: %s', self.authored_content_key, source) if source: visualization['options']['icon'] = self.extract_image(source, type) # image (type is viz.img or splunk.image) elif ('viz.img' in type) or ('splunk.image' in type): source = None if 'options' in visualization and 'src' in visualization['options']: source = visualization['options']['src'] self.logger.info('Content pack: %s. Parsing dashboard. viz.img (or splunk.image) source: %s', self.authored_content_key, source) if source: visualization['options']['src'] = self.extract_image(source, type) # choropleth svg (type is splunk.choropleth.svg) elif 'choropleth' in type: source = None if 'options' in visualization and 'svg' in visualization['options']: source = visualization['options']['svg'] self.logger.info('Content pack: %s. Parsing dashboard. choropleth source: %s', self.authored_content_key, source) if source: visualization['options']['svg'] = self.extract_image(source, type) # background image elif 'options' in visualization and 'backgroundImage' in visualization['options']: source = None if ('options' in visualization) and ('backgroundImage' in visualization['options']) \ and ('src' in visualization['options']['backgroundImage']): source = visualization['options']['backgroundImage']['src'] self.logger.info('Content pack: %s. Parsing dashboard. backgroundImg source: %s', self.authored_content_key, source) if source: visualization['options']['backgroundImage']['src'] = self.extract_image(source, 'viz.img') self.logger.info("Returning visualization: %s", visualization) return visualization @staticmethod def _is_savedsearch_in_list(savedsearches_list, savedsearch): ''' Check if the savedsearch being used in the Dashboard is present in the savedsearches_list provided @type savedsearches_list: list of dict's @param savedsearches_list: list of savedsearches which are specified to be part of the Content Pack Example of the list: [{'name': '10 Most Popular Executables Last Hour (UNIX - CPU)', 'app': 'DA-ITSI-CP-unix-dashboards'}, {'name': 'foo_warn', 'app': 'itsi'}, {'name': 'bar_error', 'app': 'itsi'}] @type savedsearch: dict @param savedsearch: savedsearch retrieved from dashboard. In the dashboard the savedsearch name is called 'ref', but before sending it for checking here, 'ref' will be changed to 'name'. {'name': 'baz_search', 'app': 'itsi'} @returns: whether the savedsearch is present in savedsearches_list @rtype: bool ''' if not savedsearches_list or len(savedsearches_list) == 0: return False name = savedsearch.get('name', None) # if no app is specified then search is the default app app = savedsearch.get('app', 'search') for search in savedsearches_list: s_name = search.get('name', None) s_app = search.get('app', 'search') if name == s_name and app == s_app: return True return False