# Copyright (C) 2005-2025 Splunk Inc. All Rights Reserved. import datetime import re import time import json import itsi_py3 import ITOA.itoa_common as utils from ITOA.itoa_exceptions import ItoaError, ItoaValidationError from ITOA.itoa_object import CRUDMethodTypes from ITOA.service_tree import generate_subgraphs_json from ITOA.setup_logging import InstrumentCall, logger from itsi.objects.changehandlers.sandbox_base_service_template_delete_handler import SandboxServiceTemplateDeleteHandler from itsi.objects.itsi_sandbox import ItsiSandbox, Status from itsi.objects.itsi_sandbox_service import ItsiSandboxService, SandboxServiceStatus, SANDBOX_SERVICES_LIMIT from itsi.objects.itsi_sandbox_sync_log import ( ItsiSandboxSyncLog, TYPE_VALIDATE, TYPE_PUBLISH, STATUS_NOT_STARTED, STATUS_IN_PROGRESS, STATUS_SUCCESS, STATUS_FAILED, ACTION_TYPE_SYNCHRONIZE ) from itsi.objects.itsi_security_group import ItsiSecGrp from itsi.objects.itsi_service import ItsiService from itsi_py3 import string_type from itsi.health_services.health_services_provider import Provider from copy import deepcopy from itsi.itsi_utils import SplunkMessageHandler def sandbox_lock_and_update_count_factory(intermediate_status, update_count=True, multiple_sandbox_publish_validate_status_check=False): """ Create a decorator to handle sandbox validation, locking, and unlocking automatically. Usage: ``` @sandbox_lock_and_update_count_factory(desired_status_for_duration_of_foo) def foo(): ``` :param intermediate_status: Status that should be used while the decorated function runs :type: int (see Status in itsi_sandbox.py) :param update_count: boolean flag that would update the count of services and uniqueu service template in service sandbox :type: bool :return: Decorator to handle sandbox validation, locking, and unlocking automatically :type: function """ def lock_and_update_count_sandbox(func): """ This function is the decorator to handle sandbox validation, locking, and unlocking automatically. :param func: Function to use :type: function :return: Wrapping function to do the actual work :type: function """ def lock_and_update_count_sandbox_function(obj, owner, sandbox_id, transaction_id, *args, **kwargs): """ This is the actual wrapping function that will: 1. Validate that the sandbox is in a ready state 2. Lock the sandbox 3. Perform some operation 4. Unlock the sandbox for all cases except publish 5. Set the status to partial publish, successful publish or edit based on the publish action result 6. Throw any errors encountered Wrapped functions should begin with the same parameters as this function. Object should have a `sandbox_utils` variable. :param obj: SandboxServiceUtils (or similar) object to use :type: SandboxServiceUtils (or similar) :param owner: Owner of the objects :type: string :param sandbox_id: ID of the sandbox :type: string :param transaction_id: ID of transaction for tracebacks :type: string :return: Sandbox sync log creation response :rtype: dict """ rv = None allowed_initial_statuses = Status.get_allowed_statuses(intermediate_status) # This is outside the try-loop to avoid locking and unlocking a non-existent sandbox, a sandbox that the # user shouldn't have permissions to, or a sandbox under lock by someone else. sandbox_status_before_action = obj.sandbox_utils.validate_and_return_current_sandbox_status( owner, sandbox_id, transaction_id=transaction_id, multiple_sandbox_publish_validate_status_check=multiple_sandbox_publish_validate_status_check, allowed_initial_statuses=allowed_initial_statuses) try: obj.sandbox_utils.sandbox_interface.update(owner, sandbox_id, data={'status': intermediate_status}, is_partial_data=True, transaction_id=transaction_id) rv = func(obj, owner, sandbox_id, transaction_id, *args, **kwargs) except Exception as e: raise e finally: # Edit is the default fall back status in case of any unknown case if intermediate_status == Status.STATUS_SYNC: data = { 'status' : sandbox_status_before_action} else: data = { 'status' : Status.STATUS_EDIT} if intermediate_status == Status.STATUS_PUBLISH: sandbox = obj.sandbox_interface.get(owner, sandbox_id) log_link = obj.sandbox_utils.get_search_log_url(transaction_id) sandbox_name = sandbox['identifying_name'] notification_type = SplunkMessageHandler.INFO notification_message = "Services from %s published. %s for more details." % (sandbox_name, log_link) sandbox_sync_filter = {'transaction_id': transaction_id} results = obj.sandbox_sync_log_interface.get_bulk( owner, filter_data=sandbox_sync_filter, fields=[ '_key', 'details', 'errors'], sort_key='timestamp', sort_dir='desc', transaction_id=transaction_id, ) # Parse the most recently updated record sandbox_sync_log_record = results[0] error_count = 0 if 'details' in sandbox_sync_log_record and 'errors' in sandbox_sync_log_record: errors = sandbox_sync_log_record.get('errors', []) data['last_published_record'] = { 'details': sandbox_sync_log_record['details'], 'errors': errors, 'publish_time': time.time()} service_tree = sandbox_sync_log_record['details'].get( 'service_tree', {}) total_publish_service_count_attempted = 0 service_count_by_tree_id = {} if service_tree and 'graphs' in service_tree and 'totalCount' in service_tree: for error in errors: if error['object_type'] == 'sandbox_service_tree': if len(service_count_by_tree_id) == 0: for tree in service_tree['graphs']: service_count_by_tree_id[tree['id']] = len( tree['vertices']) error_count += service_count_by_tree_id[error['_key']] elif error['object_type'] == obj.sandbox_utils.sandbox_interface.object_type: # This signifies that all the service in the sandbox failed to publish # as part of bulk import error_count = service_tree['totalCount'] break else: # Assuming the error object type to be obj.sandbox_utils.service_interface.object_type error_count += 1 total_publish_service_count_attempted = service_tree['totalCount'] if error_count == 0: # All services are published successfully # The status of the sandbox will be set back to edit when # a reset publish or revert publish call is made data['status'] = Status.STATUS_PUBLISH_SUCCESS notification_type = SplunkMessageHandler.INFO notification_message = "Services from %s published. %s for more details." % (sandbox_name, log_link) elif error_count < total_publish_service_count_attempted: # Partial publish where some service were published others were not # The status of the sandbox will be set back to edit when # a reset publish or revert publish call is made data['status'] = Status.STATUS_PARTIAL_PUBLISH notification_type = SplunkMessageHandler.WARNING notification_message = "Some services in %s failed to publish. %s for more details." % (sandbox_name, log_link) else: # Do not save the publish record in case of failed publish data.pop('last_published_record') notification_type = SplunkMessageHandler.ERROR notification_message = "Services in %s failed to published. %s for more details." % (sandbox_name, log_link) obj.sandbox_utils.message_handler.post_or_update_message( 'service_sandbox_%s' % ( transaction_id), notification_type, notification_message ) if update_count: count_of_services, count_of_unique_service_templates = obj.sandbox_utils.determine_counts(owner, sandbox_id, transaction_id=transaction_id) data['count_of_unique_service_templates'] = count_of_unique_service_templates data['count_of_services'] = count_of_services obj.sandbox_utils.sandbox_interface.update(owner, sandbox_id, data=data, is_partial_data=True, transaction_id=transaction_id) return rv return lock_and_update_count_sandbox_function return lock_and_update_count_sandbox class SandboxUtils(object): def __init__(self, session_key, current_user_name): """ @type: string @param: session_key @type: string @param current_user_name: user invoking this call """ self._session_key = session_key self.current_user_name = current_user_name self.base_service_templates = None self.sandbox_interface = ItsiSandbox(self._session_key, self.current_user_name) self.sandbox_service_interface = ItsiSandboxService(self._session_key, self.current_user_name) self.message_handler = SplunkMessageHandler(session_key) def validate_and_return_current_sandbox_status(self, owner, sandbox_id, transaction_id=None, multiple_sandbox_publish_validate_status_check=False, allowed_initial_statuses=[Status.STATUS_EDIT]): """ Validates if sandbox exists and if the status is Edit @type string @param sandbox_id: key of the sandbox @rtype None """ current_status = self.sandbox_interface.validate_and_return_current_sandbox_status(owner, sandbox_id, transaction_id=transaction_id, allowed_initial_statuses=allowed_initial_statuses) if multiple_sandbox_publish_validate_status_check: if not self.allow_publish_validate(owner): self.sandbox_interface.raise_error_bad_validation(logger, 'Concurrent publish/validate operation across different service sandboxes are not allowed', 409) return current_status def get_search_log_url(self, transaction_id): """ Builds the search log url to be provided as part of the notification message @type string @param transaction_id: transaction id of the action on which the logs are to be retrieved """ # sid (search id) is dynamically generated as part of search created. build_url = 'app/itsi/search?q=search index="_internal" transaction_id=%s OR tid=%s&display.page.search.mode=smart&dispatch.sample_ratio=1&workload_pool=standard_perf&earliest=0&latest=now' % ( transaction_id, transaction_id) log_url = '[[%s|View logs]]' % (build_url) return log_url def allow_publish_validate(self, owner, transaction_id=None): """ Checks if all the sandboxes are not in publish or validates state. @type string @param owner: owner value @type string @param transaction_id: Transaction Id @rtype bool @return boolean value for allow publish and validate """ results = self.sandbox_interface.get_bulk( 'nobody', fields=['title', 'status'], filter_data={'$or': [ {'status': Status.STATUS_PUBLISH}, {'status': Status.STATUS_VALIDATE}, ]} ) if len(results) > 0: for entry in results: status_pretty = Status.pretty_string(entry['status']) logger.error('Publish/Validate operations are blocked as sandbox service="%s" has current status="%s"' % (entry['title'], status_pretty)) return False return True def determine_counts(self, owner, sandbox_id, transaction_id=None): """ determines the number of services and unique set of service templates used in service sandbox @type string @param sandbox_id: key of the sandbox @rtype None """ sandbox_id_filter = {'sandbox_id': sandbox_id} results = self.sandbox_service_interface.get_bulk_skip_enforce_security( owner, filter_data=sandbox_id_filter, fields=['_key', 'base_service_template_id'], transaction_id=transaction_id, ) count_of_unique_service_templates = 0 count_of_services = 0 if results: count_of_services = len(results) service_templates = set() [service_templates.add( result['base_service_template_id']) for result in results if result.get('base_service_template_id', False)] count_of_unique_service_templates = len(service_templates) return count_of_services, count_of_unique_service_templates class SandboxServiceUtils(object): def __init__(self, session_key, current_user_name): """ Constructor @type: string @param: session_key @type: string @param owner: "current_user_name" user invoking this call @rtype: None @return: None """ self._session_key = session_key self.current_user_name = current_user_name self.base_service_templates = None self.sandbox_service_interface = ItsiSandboxService(self._session_key, self.current_user_name) self.sandbox_interface = ItsiSandbox(self._session_key, self.current_user_name) self.sandbox_sync_log_interface = ItsiSandboxSyncLog(self._session_key, self.current_user_name) self.service_interface = ItsiService(self._session_key, self.current_user_name) self.sandbox_utils = SandboxUtils(self._session_key, self.current_user_name) self._instrumentation = InstrumentCall(logger) self.health_provider = Provider(session_key, logger) @sandbox_lock_and_update_count_factory(Status.STATUS_ADD_SERVICE) @InstrumentCall(logger) def create_sandbox_service(self, owner, sandbox_id, transaction_id, data): """ Method to create a sandbox service object @type: string @param owner: Owner of the object @type: string @param sandbox_id: ID of the sandbox to associate the service with @type: string @param transaction_id: Unique identifier for the transaction @type: dict @param data: Details of the object to be created @rtype: dict @return: response containing key of object created """ logger.info('Sandbox logs before adding service="%s", transaction_id="%s"', data, transaction_id) sandbox_id_filter = {'sandbox_id': sandbox_id} results = self.sandbox_service_interface.get_bulk_skip_enforce_security( owner, filter_data=sandbox_id_filter, fields=['_key'], transaction_id=transaction_id, ) # Create adds one new Service. If the current count + 1 exceeds the limit then error if len(results) + 1 > SANDBOX_SERVICES_LIMIT: raise ItoaError( "You can only have a total of 250 services in the sandbox. To continue creating services,\ delete one or more of the existing services or reduce the number of services.", logger, status_code=400) # setting sync status as synced for the initial creation data['sync_status'] = '1' result = self.sandbox_service_interface.create(owner, data, transaction_id=transaction_id) service_key = result['_key'] # Used to generate telemetry base_service_template_id = data.get('base_service_template_id') if base_service_template_id: template_log_str = ', base_service_template_id="%s"' % base_service_template_id else: template_log_str = '' logger.info('Successfully added service="%s" to Sandbox, transaction_id="%s"%s' % (service_key, transaction_id, template_log_str)) if not isinstance(data.get('sec_grp'), string_type): data['sec_grp'] = self.sandbox_service_interface._get_security_enforcer(). \ get_default_itsi_security_group_key() log_object = { 'title': 'Service Added to Sandbox %s (%s)' % (self.sandbox_service_interface.object_type, transaction_id), 'action_type': 'Add Service', 'details': data, 'sec_grp': data['sec_grp'], } self.sandbox_sync_log_interface.create_record( owner, transaction_id, self.sandbox_interface.object_type, sandbox_id, log_object, ) return result def perform_sandbox_service_delete(self, owner, to_be_deleted_sandbox_id, transaction_id): delete_sandbox_filter = {'$or': [{'sandbox_id': to_be_deleted_sandbox_id}]} self.sandbox_service_interface.delete_bulk(owner, filter_data=delete_sandbox_filter, transaction_id=transaction_id) def update_sandbox_services_status( self, transaction_id, updated_services_list, status ): """ Performs bulk CRUD operation of sandbox services to update the status field. @type string @param owner @type string @param transaction_id: Transaction ID to associate logs with @type list @param updated_services_list: List of services to be updated """ for entry in updated_services_list: entry['sync_status'] = status self.sandbox_service_interface.save_batch( 'nobody', updated_services_list, method=CRUDMethodTypes.METHOD_UPDATE, validate_names=True, is_partial_data=True, transaction_id=transaction_id) @sandbox_lock_and_update_count_factory(Status.STATUS_SAVE) @InstrumentCall(logger) def save_sandbox_services( self, owner, sandbox_id, transaction_id, updated_services_list, deleted_services_list ): """ Performs bulk CRUD operation of sandbox services. Below is the execution order Put Sandbox status in Save mode update sandbox services delete the sandbox services update the sandbox sync log with the details of the changes and the affected sandbox object Put Sandbox status in Edit mode @type string @param owner @type string @param sandbox_id @type string @param transaction_id: Transaction ID to associate logs with @type list @param updated_services_list: List of services to be updated @type list @param deleted_services_list: List of services to be deleted @type boolean @param Enable or disable sandbox """ data = {} self.sandbox_service_interface.base_service_templates = self.base_service_templates if len(updated_services_list) == 0 and len(deleted_services_list) == 0: raise ItoaValidationError( 'The sandbox has no data. Check that the inputted parameters are valid and try again.', logger) sandbox_sync_data = { 'title': 'Sync Log for %s (%s)' % (self.sandbox_interface.object_type, transaction_id), } try: if updated_services_list: updated_keys = set([service_val['_key'] for service_val in updated_services_list]) update_sandbox_service_filter = {'$or': [{'_key': sandbox_service_id} for sandbox_service_id in updated_keys]} found_services = self.sandbox_service_interface.get_bulk( owner, sort_key=None, sort_dir=None, filter_data=update_sandbox_service_filter, fields=['_key'], skip=None, limit=None, req_source='unknown', transaction_id=transaction_id) found_keys = set([service_val['_key'] for service_val in found_services]) updated_keys_difference = updated_keys.difference(found_keys) if updated_keys_difference: error_msg = '{} key(s) could not be found'.format( ', '.join(updated_keys_difference)) raise ItoaError(error_msg, logger, status_code=404) self.sandbox_service_interface.base_service_templates = self.base_service_templates data['updated_sandbox_service_ids'] = self.sandbox_service_interface.save_batch( owner, updated_services_list, method=CRUDMethodTypes.METHOD_UPDATE, validate_names=True, is_partial_data=True, transaction_id=transaction_id) if deleted_services_list: delete_sandbox_service_filter = {'$or': []} for sandbox_service_id in deleted_services_list: delete_sandbox_service_filter['$or'].append({'_key': sandbox_service_id}) self.sandbox_service_interface.delete_bulk(owner, filter_data=delete_sandbox_service_filter) remnant_sandbox_services = self.sandbox_service_interface.get_bulk( owner, sort_key=None, sort_dir=None, filter_data=delete_sandbox_service_filter, fields=['title', '_key'], skip=None, limit=None, req_source='unknown', transaction_id=transaction_id) remnant_sandbox_services_titles = [] for rem in remnant_sandbox_services: remnant_sandbox_services_titles.append(rem['title']) deleted_services_list.remove(rem['_key']) data['deleted_sandbox_service_ids'] = deleted_services_list if remnant_sandbox_services: error_msg = 'Failed to delete sandbox services. {}'.format( ', '.join(remnant_sandbox_services_titles)) raise ItoaError(error_msg, logger, status_code=500) sandbox_sync_data['details'] = data sandbox_sync_data['sec_grp'] = self.set_first_assigned_security_group(owner, transaction_id, data) self.sandbox_sync_log_interface.create_record( owner, transaction_id, self.sandbox_interface.object_type, sandbox_id, sandbox_sync_data, ) except ItoaError as e: if data.get('updated_sandbox_service_ids') is None and data.get('deleted_sandbox_service_ids') is None: raise e data['error'] = str(e) sandbox_sync_data['details'] = data sandbox_sync_data['sec_grp'] = self.set_first_assigned_security_group(owner, transaction_id, data) self.sandbox_sync_log_interface.create_record( owner, transaction_id, self.sandbox_interface.object_type, sandbox_id, sandbox_sync_data, ) except Exception as e: if data.get('updated_sandbox_service_ids') is None and data.get('deleted_sandbox_service_ids') is None: raise e data['error'] = str(e) sandbox_sync_data['details'] = data sandbox_sync_data['sec_grp'] = self.set_first_assigned_security_group(owner, transaction_id, data) self.sandbox_sync_log_interface.create_record( owner, transaction_id, self.sandbox_interface.object_type, sandbox_id, sandbox_sync_data, ) return data def set_first_assigned_security_group(self, owner, transaction_id, data): """ Fetches the security groups associated to the user and then will select the first one and use it to create sync log. We cant use default sec grp as itoteamadmin does not have permission on the grp. @type: string @param owner: Object type for which sync log is generated @type: string @param transaction_id: Transaction ID for tracebacks @rtype: string @return: security grp key """ if data.get('sec_grp') is None: sec_grp_interface = ItsiSecGrp(self._session_key, self.current_user_name) teams = sec_grp_interface.get_bulk(owner, transaction_id=transaction_id) for sec_grp in teams: if sec_grp.get('_key') is not None: # setting the first security grp/team. data['sec_grp'] = sec_grp.get('_key') break return data.get('sec_grp') def get_sandbox_service_keys_from_titles(self, owner, titles, sandbox_id, transaction_id=None): """ Fetches the sandbox service keys from titles for a specific sandbox @type: string @param owner: Object type for which sync log is generated @type: list @param titles: titles of the services for which the keys are retrieved @type: string @param sandbox_id: ID of the sandbox for which the sandbox service is associated @type: string @param transaction_id: Transaction ID for tracebacks @rtype: list @return: list of dicts carrying the _key """ sandbox_id_filter = {'sandbox_id': sandbox_id} sandbox_service_filter = {'$and': [sandbox_id_filter]} if len(titles) > 0: sandbox_service_title_filter = { '$or': [{'title': title} for title in titles] } sandbox_service_filter['$and'].append(sandbox_service_title_filter) else: sandbox_service_filter = sandbox_id_filter return self.sandbox_service_interface.get_bulk(owner, filter_data=sandbox_service_filter, fields=['_key'], transaction_id=transaction_id) def get_all_services(self, owner, sandbox_id, transaction_id=None): """ Fetches all the services that are part of a given sandbox @type: string @param owner: Object type for which sync log is generated @type: string @param sandbox_id: ID of the sandbox for which the sandbox services are associated @type: string @param transaction_id: Transaction ID for tracebacks @rtype: list of dictionary @return: sandbox services associated with the given sandbox """ sandbox_id_filter = {'sandbox_id': sandbox_id} return self.sandbox_service_interface.get_bulk_skip_enforce_security( owner, filter_data=sandbox_id_filter, fields=['_key'], transaction_id=transaction_id, ) def get_sandbox_service_trees(self, owner, sandbox_id='default_itsi_sandbox', service_id_filter=[], services=[], transaction_id=None): """ Generates and returns graph of the sandbox service trees :param owner: Owner of the sandbox services :type: string :param sandbox_id: ID of sandbox to fetch services from :type: string :param service_id_filter: List of service IDs to filter to :type: list :param services: List of services to merge with before calculation :type list: :param transaction_id: ID of transaction for tracebacks :type: string :return: Graph of the sandbox services trees :type: dict """ # perform a partial fetch of services fields = ['title', '_key', 'services_depends_on'] data = self.sandbox_service_interface.get_bulk(owner, filter_data={'sandbox_id': sandbox_id}, fields=fields, transaction_id=transaction_id) # Replace any existing service data with any specifically provided data data = utils.combine_object_lists(data, services) return generate_subgraphs_json(data, service_id_filter=service_id_filter, partial_backup=False, max_depth_filter=-2, severity_filter_ids=[]) def process_health(self, services_by_id): """ Process the services and generates the health result of the services :param services_by_id: Services for which the service health must be processes :type: list :return: The health scores of individual services provided :type: dict """ health_result = {} def process_node(service_id, services_by_id): service = services_by_id.get(service_id) severity_urgency_list = [] service_dependencies = service.get('dependencies') for dep_service_key in service_dependencies: dep_service = services_by_id[dep_service_key] if not dep_service.get('visited'): # process the child node if not visited process_node(dep_service_key, services_by_id) for kpi_key in service_dependencies.get(dep_service_key): if kpi_key.startswith('SHKPI-'): # Get the computed service health score severity value as the service health score of the # depended service could change based on updates split_sh_service_id = kpi_key.split('SHKPI-', 1) dep_service_id = split_sh_service_id[1] if health_result.get(dep_service_id) is None: # Process the service health if the service health is not computed already process_node(dep_service_id, services_by_id) sev_name_value = health_result[dep_service_id]['score']['data']['health_severity_name'] else: sev_name_value = services_by_id[dep_service_key]['kpis'][kpi_key]['severity_name'] val = {'urgency': service_dependencies[dep_service_key][kpi_key]['urgency'], 'severity_name': sev_name_value} severity_urgency_list.append(val) # If only SHKPI is associate with the service with no other kpi or dependend kpis, # then use value of the SHKPI to compute the health only_shkpi_present = len(service['kpis']) == 1 and len(service.get('dependencies')) == 0 for kpi in service['kpis']: # Remove existing Service health score KPI for the calculation of the new Service health score if only_shkpi_present or "SHKPI-" + service_id != kpi: severity_urgency_list.append(service['kpis'][kpi]) score = deepcopy(self.health_provider.calculate_score(severity_urgency_list)) health_result[service_id] = { 'score' : score } for service_key in services_by_id: process_node(service_key, services_by_id) return health_result @InstrumentCall(logger) def process_services_health( self, services_by_id, nodes_to_process=[]): """ Generates and returns graph of the sandbox service trees :param services_by_id: The list of services and its dependency with the importance and severity levels :type: dict :param nodes_to_process: service nodes for which the service health must be simulated Assumes all services to be processed if empty list is provided :type: list :return: Services and with severity level and urgency by id :type: dict """ if len(nodes_to_process) == 0: # Assume all nodes to process if nodes_to_process is empty for service in services_by_id: services_by_id[service]['visited'] = False else: for node in nodes_to_process: service = services_by_id[node] parents = service['parent'] for parent in parents: if services_by_id.get(parent): services_by_id[parent]['visited'] = False health_result = self.process_health(services_by_id) return health_result def get_sandbox_service_with_severity_urgency(self, owner, sandbox_id, transaction_id=None): """ Generates and returns graph of the sandbox service trees :param owner: Owner of the sandbox services :type: string :param sandbox_id: ID of sandbox to fetch services from :type: string :param transaction_id: ID of transaction for tracebacks :type: string :return: Services and with severity level and urgency by id :type: dict """ # perform a partial fetch of services fields = ['title', '_key', 'kpis.title', 'kpis.urgency', 'kpis._key', 'kpis.severity_name', 'services_depends_on', 'services_depending_on_me'] services = self.sandbox_service_interface.get_bulk(owner, filter_data={'sandbox_id': sandbox_id}, fields=fields, transaction_id=transaction_id) service_urgency_severity_by_ids = {} for service in services: _key = service.get('_key') details = {} service_urgency_severity_by_ids[_key] = details parents = service.get('services_depending_on_me', []) details['parent'] = [] for parent in parents: details['parent'].append(parent['serviceid']) kpis = {} details['kpis'] = kpis for kpi in service.get('kpis'): severity_name = kpi.get('severity_name', 'normal') kpis[kpi['_key']] = { 'urgency': kpi['urgency'], 'severity_name': severity_name} dependencies = {} details['dependencies'] = dependencies for dependency in service.get('services_depends_on'): dep_kpis = {} for kpi in dependency.get('kpis_depending_on'): # Default importance is set to 5 for all kpis except SHKPI urgency = 5 if kpi.startswith('SHKPI-'): urgency = 11 severity_name = 'normal' if dependency.get('overloaded_urgencies'): urgency = dependency.get('overloaded_urgencies').get(kpi, urgency) if dependency.get('overloaded_severities'): severity_name = dependency.get('overloaded_severities').get(kpi, 'normal') dep_kpis[kpi] = { 'urgency': urgency, 'severity_name': severity_name} dependencies[dependency['serviceid']] = dep_kpis details['visited'] = True return service_urgency_severity_by_ids def filter_services_by_tree(self, tree, services): """ Generate a list of service keys belonging to a service tree :param tree: Representation of a service tree generated by `get_sandbox_service_trees()` :type: dict :param services: List of services :type: list of sandbox service objects :return: """ vertex_keys = set() for vertex in tree['vertices']: vertex_keys.add(vertex['id']) def helper(service): return service['_key'] in vertex_keys return list(filter(helper, services)) def get_sandbox_services_for_template( self, owner, template_id, filter_string, count, offset, sort_key, sort_dir, transaction_id=None, ): """ Method to return the Sandbox services for template :param owner: Owner of the sandbox services :type: string :param template_id: ID of template to fetch linked sandbox services :type: string :param count: Count of sandbox services to get :type: string :param offset: start of list of records to get :type: string :param transaction_id: ID of transaction for tracebacks :type: string :return: List of sandbox services associated to template with sandbox details. :type: List """ # perform fetch fields = ["title", "_key", "base_service_template_id", "sync_status", "sandbox_id", "identifying_name" ] filter_data_value = {'$and': [{'base_service_template_id': template_id}]} if isinstance(filter_string, itsi_py3.string_type) and len(filter_string) > 0: filter_data = json.loads(filter_string) if filter_data.get('filter_string') is not None and isinstance(filter_data['filter_string'], dict): filter_data_value.update(filter_data["filter_string"]) if filter_data.get('sync_status') is not None: sync_status = {'sync_status': filter_data["sync_status"]} filter_data_value.update(sync_status) sandbox_service_data = self.sandbox_service_interface.get_bulk( owner, filter_data=filter_data_value, fields=fields, sort_key=sort_key, sort_dir=sort_dir, transaction_id=transaction_id, ) sandbox_details_map = self.get_all_sandbox_details_in_map(transaction_id=transaction_id) for sandbox_service in sandbox_service_data: if sandbox_service: sandbox_object = sandbox_details_map.get(sandbox_service.get('sandbox_id'), None) if sandbox_object: sandbox_service['sandbox_title'] = sandbox_object.get('title') else: sandbox_service_data.remove(sandbox_service) if (count is not None) and (offset is not None): count = int(count) offset = int(offset) end_offset_requested = offset + count count_of_results = len(sandbox_service_data) if (count_of_results < offset) or (end_offset_requested < offset): raise Exception( ( "Invalid range requested. offset: {0}, count: {1}, result set count: {2}" ).format(offset, count, count_of_results) ) range_end = end_offset_requested if count_of_results < end_offset_requested: range_end = count_of_results return sandbox_service_data[offset: range_end] else: return sandbox_service_data def get_all_sandbox_details_in_map(self, transaction_id=None): """ Create the mapping of sandbox_id and sandbox_object. :param transaction_id: ID of transaction for tracebacks :type: string :return: dict of sandbox id and sandbox object :type: dict """ sandbox_details_map = {} fields = ["title", "_key"] all_sandbox_objects = self.sandbox_interface.get_bulk('nobody', fields=fields, transaction_id=transaction_id) for entry in all_sandbox_objects: key = entry['_key'] sandbox_details_map[key] = entry return sandbox_details_map def convert_sandbox_services_to_services(self, sandbox_services, transaction_id, backfill_enabled=False): """ Convert sandbox service objects to service objects in format :param sandbox_services: List of sandbox services to convert :type: list of sandbox service objects :param transaction_id: Transaction ID of operation :type: string :return: tuple of List of errors encountered and service-to-base service template map :rtype: tuple of list of dicts """ errors = [] service_base_service_template_map = {} current_time = time.time() for sandbox_service in sandbox_services: try: # Clean up sandbox service sandbox_service['object_type'] = self.service_interface.object_type sandbox_service['is_created_from_sandbox'] = True publish_history = sandbox_service.get('publish_history', []) publish_history.append({ 'sandbox_id': sandbox_service['sandbox_id'], 'publish_time': current_time, 'transaction_id': transaction_id, }) sandbox_service['publish_history'] = publish_history for field in self.sandbox_service_interface.SANDBOX_SERVICE_SPECIFIC_FIELDS: try: # Note update this to more generic code if there is a requirement to delete a nested object other than kpis if field.startswith('kpis.'): values = field.split(".") if len(values) == 2: kpis = sandbox_service.get(values[0]) for kpi in kpis: kpi.pop(values[1], None) else: del sandbox_service[field] except KeyError: continue # enable kpi backfill during publish kpis = sandbox_service.get('kpis', []) if backfill_enabled: for kpi in kpis: if not kpi.get('_key', "").startswith('SHKPI-'): kpi['backfill_enabled'] = True self.service_interface.ensure_required_fields([sandbox_service]) service_base_service_template_map[sandbox_service['_key']] = \ sandbox_service.get('base_service_template_id', '') except Exception as e: errors.append({ '_key': sandbox_service['_key'], 'object_type': self.sandbox_service_interface.object_type, 'description': str(e), }) return errors, service_base_service_template_map @sandbox_lock_and_update_count_factory(Status.STATUS_VALIDATE, update_count=False, multiple_sandbox_publish_validate_status_check=True) @InstrumentCall(logger) def validate_sandbox(self, owner, sandbox_id, transaction_id): """ Perform a best-effort validation that the sandbox has no issues and the sandbox services can be readily converted into real services. Note: This would be a good candidate for conversion to an asynchronous action when (if?) the refresh queue is able to handle priority. :param owner: Owner of the objects :type: string :param sandbox_id: ID of the sandbox :type: string :param transaction_id: ID of transaction for tracebacks :type: string :return: Sandbox sync log creation response :type: dict """ log_update_timestamp = time.time() record = { 'action_type': TYPE_VALIDATE, 'action_status': STATUS_IN_PROGRESS, 'errors': [], } self.sandbox_sync_log_interface.create_record( owner, transaction_id, self.sandbox_interface.object_type, sandbox_id, record, ) service_trees = {} sandbox_services = self.sandbox_service_interface.get_bulk( owner, filter_data={'sandbox_id': sandbox_id}, transaction_id=transaction_id, ) if not sandbox_services: record['errors'].append({ '_key': sandbox_id, 'object_type': self.sandbox_interface.object_type, 'description': 'No sandbox services to validate.', }) else: # Bulk service checks if not record['errors']: try: error_key = sandbox_id error_object_type = self.sandbox_interface.object_type title_conflict_dict = {} try: # Bulk check (moved from do_object_validation()) self.service_interface.validate_identifying_name( owner, sandbox_services, transaction_id=transaction_id, ignore_same_key=True, ) except ItoaValidationError as e: # Look for more detailed information about duplicate title conflicts if "Duplicate object name(s) found" in str(e): # Check for title conflicts in bulk case_insensitive_filter = { '$or': [ { 'title': { '$regex': re.escape(service['title']), # Case-insensitive regex flag '$options': 'i', }, } for service in sandbox_services ], } title_conflicts = self.service_interface.get_bulk( owner, filter_data=case_insensitive_filter, fields=['_key', 'title'], ) for service in title_conflicts: title_conflict_dict[service['title'].lower()] = service # Throw miscellaneous errors else: raise e services_simplified = [] for sandbox_service in sandbox_services: # Clean up sandbox service # Per service checks similar to convert_sandbox_services_to_services() but streamlined error_key = sandbox_service['_key'] error_object_type = self.sandbox_service_interface.object_type sandbox_service['object_type'] = self.service_interface.object_type for field in self.sandbox_service_interface.SANDBOX_SERVICE_SPECIFIC_FIELDS: try: del sandbox_service[field] except KeyError: continue self.service_interface.ensure_required_fields([sandbox_service]) services_simplified.append({ '_key': sandbox_service['_key'], 'title': sandbox_service['title'], 'services_depends_on': sandbox_service['services_depends_on'], }) # Conflicting service checks # We check over all sandbox services instead of over all conflicts to catch multiple sandbox # services that have the same conflict conflicting_service = title_conflict_dict.get(sandbox_service['title'].lower()) if conflicting_service and (conflicting_service['sec_grp'] == sandbox_service['sec_grp']): record['errors'].append({ '_key': sandbox_service['_key'], 'object_type': self.sandbox_service_interface.object_type, 'description': 'A conflicting title (%s) was found between sandbox service %s and service %s.' % ( conflicting_service['title'], sandbox_service['_key'], conflicting_service['_key'], ) }) service_trees = generate_subgraphs_json(services_simplified) # Check each graph individually self.service_interface.set_persisted_services_caching(True) record['details'] = { 'services_processed': 0, 'trees_processed': 0, } for tree in service_trees['graphs']: tree_services = self.filter_services_by_tree(tree, sandbox_services) error_key = tree['id'] # This is not a real object error_object_type = 'sandbox_service_tree' # Ensure no errors are present in the object setup self.service_interface.do_object_validation( owner, tree_services, validate_name=False, transaction_id=transaction_id, ignore_same_key=True, ) self.service_interface.do_additional_setup( owner, tree_services, method=CRUDMethodTypes.METHOD_CREATE, transaction_id=transaction_id, ) # Ensure no errors are present in the dependency generation self.service_interface.identify_dependencies( owner, tree_services, CRUDMethodTypes.METHOD_CREATE, transaction_id=transaction_id, dry_run=True, ) # Update the service cache ids = [{'_key': service['_key'] for service in tree_services}] self.service_interface.post_save_setup( owner, ids, tree_services, req_source="validate_sandbox", method=CRUDMethodTypes.METHOD_CREATE, transaction_id=transaction_id, dry_run=True, ) record['details']['services_processed'] += len(tree_services) record['details']['trees_processed'] += 1 # Update every 5 seconds (in cadence with front-end pull) current_time = time.time() if current_time - log_update_timestamp > 5: self.sandbox_sync_log_interface.update_record(owner, transaction_id, record) log_update_timestamp = current_time except Exception as e: record['errors'].append({ '_key': error_key, 'object_type': error_object_type, 'description': str(e), }) finally: self.service_interface.set_persisted_services_caching(False) if record['errors']: record['action_status'] = STATUS_FAILED else: record['action_status'] = STATUS_SUCCESS record['details'] = { 'service_tree': service_trees, } return self.sandbox_sync_log_interface.update_record(owner, transaction_id, record) @sandbox_lock_and_update_count_factory(Status.STATUS_PUBLISH, update_count=True, multiple_sandbox_publish_validate_status_check=True) @InstrumentCall(logger) def publish_sandbox(self, owner, sandbox_id, transaction_id, skip_validate_identifying_name=False): """ Publish the sandbox as services. Note: This would be a good candidate for conversion to an asynchronous action when (if?) the refresh queue is able to handle priority. :param owner: Owner of the objects :type: string :param sandbox_id: ID of the sandbox :type: string :param transaction_id: ID of transaction for tracebacks :type: string :param skip_validate_identifying_name: skip performing validate_identifying_name :type: bool :return: Sandbox sync log creation response :type: dict """ self.service_interface.set_persisted_services_caching(False) log_update_timestamp = time.time() record = { 'action_type': TYPE_PUBLISH, 'action_status': STATUS_IN_PROGRESS, 'errors': [], } self.sandbox_sync_log_interface.create_record( owner, transaction_id, self.sandbox_interface.object_type, sandbox_id, record, ) sandbox_detail = self.sandbox_interface.get(owner, sandbox_id) # Variable initialization to make error-handling cleaner service_trees = {} published_service_ids = [] failed_service_ids = [] # Actual publishing sandbox_services = self.sandbox_service_interface.get_bulk( owner, filter_data={'sandbox_id': sandbox_id}, transaction_id=transaction_id, ) if not sandbox_services: record['errors'].append({ '_key': sandbox_id, 'object_type': self.sandbox_interface.object_type, 'description': 'No sandbox services to publish.', }) else: try: services_simplified = [{ '_key': service['_key'], 'title': service['title'], 'services_depends_on': service['services_depends_on'], } for service in sandbox_services] service_trees = generate_subgraphs_json(services_simplified) except Exception as e: record['errors'].append({ '_key': sandbox_id, 'object_type': self.sandbox_interface.object_type, 'description': 'Error encountered in sandbox service tree generation: %s' % str(e), }) # Format conversion backfill_enabled = sandbox_detail.get('backfill_enabled', False) errors, service_base_service_template_map = \ self.convert_sandbox_services_to_services( sandbox_services, transaction_id, backfill_enabled) record['errors'].extend(errors) if not skip_validate_identifying_name: try: # Bulk check (moved from do_object_validation()) self.service_interface.validate_identifying_name( owner, sandbox_services, transaction_id=transaction_id, ignore_same_key=True, ) except Exception as e: record['errors'].append({ '_key': sandbox_id, 'object_type': self.sandbox_interface.object_type, 'description': 'Error encountered in sandbox services bulk object validation: %s' % str(e), }) if not record['errors']: record['details'] = { 'services_processed': 0, 'trees_processed': 0, } service_ids = None for tree in service_trees.get('graphs', []): try: tree_services = self.filter_services_by_tree(tree, sandbox_services) service_ids = self.service_interface.save_batch_via_publish( owner, tree_services, False, transaction_id=transaction_id, ) record['details']['services_processed'] += len(tree_services) record['details']['trees_processed'] += 1 # Update every 10 seconds (in cadence with front-end pull) # This is longer than in the Validate stage because the user is expected to wait longer here current_time = time.time() if current_time - log_update_timestamp > 10: self.sandbox_sync_log_interface.update_record(owner, transaction_id, record) log_update_timestamp = current_time except Exception as e: record['errors'].append({ '_key': tree['id'], # This is not a real object 'object_type': 'sandbox_service_tree', 'description': 'Publish failed: %s' % str(e), }) if service_ids: failed_service_ids.extend(service_ids) else: published_service_ids.extend(service_ids) service_ids = [] # Post-publish cleanup # Remove failed sandbox services from the production environment try: if failed_service_ids: filter_data = {'$or': [{'_key': service_id} for service_id in failed_service_ids]} self.service_interface.delete_bulk(owner, filter_data=filter_data, transaction_id=transaction_id) except Exception as e: for service_id in failed_service_ids: # This is technically inaccurate, but this exists to provide clearer information to the user record['errors'].append({ '_key': service_id, 'object_type': self.service_interface.object_type, 'description': 'Error encountered in failed services clean-up: %s' % str(e), }) if published_service_ids: # Map the service template with the list of service ids being published service_templates_services_map = {} for service_id in published_service_ids: service_template_id = service_base_service_template_map.get(service_id) if service_template_id not in service_templates_services_map.keys(): service_templates_services_map[service_template_id] = [] service_templates_services_map[service_template_id].append(service_id) service_template_get_filter = {'$or': []} for key in service_templates_services_map.keys(): service_template_get_filter['$or'].append({'_key': key}) # Fetch the persisted base service template and update the linked_services value with the published service # Necessary evil from itsi.objects.itsi_service_template import ItsiBaseServiceTemplate service_template_interface = ItsiBaseServiceTemplate(self._session_key, self.current_user_name) base_service_templates = service_template_interface.get_bulk(owner, filter_data=service_template_get_filter, transaction_id=transaction_id) for base_service_template in base_service_templates: if 'linked_services' not in base_service_template: base_service_template['linked_services'] = [] linked_services = base_service_template['linked_services'] for service_key in service_templates_services_map.get(base_service_template.get('_key')): if service_key in linked_services: continue linked_services.append(service_key) # update the base service template with the updated linked service count base_service_template['total_linked_services'] = len(base_service_template['linked_services']) try: service_template_interface.batch_save_backend( owner, base_service_templates, transaction_id=transaction_id) except Exception as error: logger.exception( f'Exception while saving service template interface {error}') if record['errors']: record['action_status'] = STATUS_FAILED else: record['action_status'] = STATUS_SUCCESS record['details'] = { 'service_tree': service_trees, } return self.sandbox_sync_log_interface.update_record(owner, transaction_id, record) def create_sync_records_for_synchronize(self, transaction_id, service_template, affected_sandbox_services, change): """ Create the Sync log record for the Synchronize process that is used to sync the template update with the Sandbox service. :param transaction_id: ID of transaction for tracebacks :type: string :param service_template: service_template object :type: dict :param affected_sandbox_services: list of affected services :type: list :param change: dict object of the change with details :type: dict :return: Sandbox sync log creation response :type: dict """ change_detail = change.get('change_detail') # Updating the Sandbox services status to pending self.update_sandbox_services_status(transaction_id, affected_sandbox_services, SandboxServiceStatus.STATUS_SERVICE_SYNC_PENDING) mapped_sandbox_with_services = self.get_sandbox_services_mapping(affected_sandbox_services) for sandbox_id, sandbox_services in mapped_sandbox_with_services.items(): change_detail[service_template.get('_key')]['affected_sandbox_services'] = sandbox_services change_detail['object_type'] = service_template.get('object_type') record = {'details': change, 'action_type': ACTION_TYPE_SYNCHRONIZE, 'action_status': STATUS_NOT_STARTED} self.sandbox_sync_log_interface.create_record( "nobody", transaction_id, 'sandbox', sandbox_id, record ) def get_sandbox_services_mapping(self, affected_sandbox_services): """ Create the mapping of sandbox and services. :param affected_sandbox_services: list of affected services :type: list :return: dict of sandbox services and sandbox :type: dict """ mapped_sandbox_with_services = {} for entry in affected_sandbox_services: sandbox_id = entry['sandbox_id'] key = entry['_key'] if sandbox_id not in mapped_sandbox_with_services: mapped_sandbox_with_services[sandbox_id] = [] mapped_sandbox_with_services[sandbox_id].append(key) return mapped_sandbox_with_services def create_sync_records_for_delete_synchronize(self, transaction_id, service_template, affected_sandbox_services, change): """ Create the Sync log record for the Synchronize process that is used to sync the template deletion with the Sandbox service. :param transaction_id: ID of transaction for tracebacks :type: string :param service_template: service_template object :type: dict :param affected_sandbox_services: list of affected services :type: list :param change: dict object of the change with details :type: dict :return: Sandbox sync log creation response :type: dict """ mapped_sandbox_with_services = self.get_sandbox_services_mapping(affected_sandbox_services) for sandbox_id, sandbox_services in mapped_sandbox_with_services.items(): record = {'details': change, 'action_type': ACTION_TYPE_SYNCHRONIZE, 'action_status': STATUS_NOT_STARTED} self.sandbox_sync_log_interface.create_record( 'nobody', transaction_id, 'sandbox', sandbox_id, record ) def get_all_sandbox_that_needs_sync(self, owner): """ Method to return all the sandbox objects that need sync. :param owner: owner of objects :type: string :return: sandbox details that need sync :type: list """ filter_data = {"$and": [{"action_type": ACTION_TYPE_SYNCHRONIZE}, {"$or": [{"action_status": STATUS_NOT_STARTED}, {"action_status": STATUS_FAILED}]}]} sync_data = self.sandbox_sync_log_interface.get_bulk(owner, fields=["_key", "title"], filter_data=filter_data) return sync_data def needs_sandbox_synchronize(self, owner, sandbox_id, last_mod_timestamp): """ Checks to ensure that sandbox needs to be synched or not. :param owner: Owner of the objects :type: string :param sandbox_id: ID of the sandbox :type: string :param last_mod_timestamp: Last modified time of the sandbox that is loaded in the UI :type: float (UTC time) :return: if the sandbox needs to perform a sync and/or browser needs a refresh or none :type: dict """ # validate the current status of sandbox. If already synching then return synching status sandbox_detail = self.sandbox_interface.get(owner, sandbox_id) needs_browser_refresh = False if not sandbox_detail: logger.error( "Unable to retrieve the sandbox details sandbox_id=%s", sandbox_id) self.raise_error_bad_validation( logger, 'Unable to retrieve the sandbox details. Check input.', 404) sandbox_status = sandbox_detail.get('status', Status.STATUS_EDIT) backfill_enabled, backfill_enabled_services_unused = self.get_backfill_enabled_value(owner, sandbox_id, sandbox_detail) # Compare the last modified time with the modified time of sandbox. # if Sandbox modified time value is greater than the last modified time # then set needs_browser_refresh to true if last_mod_timestamp is None: last_mod_timestamp = utils.get_current_timestamp_utc() # '+' is converted to ' ' through the web interface last_mod_timestamp = last_mod_timestamp.replace(' ', '+') sandbox_mod_time_epoch = int(datetime.datetime.fromisoformat( sandbox_detail.get('mod_timestamp')).timestamp()) loaded_sandbox_mod_time_epoch = int( datetime.datetime.fromisoformat(last_mod_timestamp).timestamp()) if sandbox_mod_time_epoch > loaded_sandbox_mod_time_epoch: needs_browser_refresh = True allow_publish_validate = self.sandbox_utils.allow_publish_validate(owner) sync_response = { "sandbox_status": sandbox_status, "needs_browser_refresh": needs_browser_refresh, "allow_publish_validate": allow_publish_validate, "backfill_enabled": backfill_enabled } return sync_response def get_backfill_enabled_value(self, owner, sandbox_id, sandbox_detail=None): """ Determines if the services published from sandbox are backfill_enabled or not :param owner: Owner of the objects :type: string :param sandbox_id: Kay of the sandbox :type: string :param sandbox_detail: sandbox object :type: object :return boolean value to determine if backfill is enabled and list of service with backfill enabled kpis :type bool, list of objects """ backfill_enabled_services = [] if sandbox_detail is None: sandbox_detail = self.sandbox_interface.get(owner, sandbox_id) backfill_enabled = sandbox_detail.get('backfill_enabled', False) sandbox_status = sandbox_detail.get('status', Status.STATUS_EDIT) if not backfill_enabled and (sandbox_status == Status.STATUS_PUBLISH_SUCCESS or sandbox_status == Status.STATUS_PARTIAL_PUBLISH): last_published_record = sandbox_detail.get( 'last_published_record', {}) services_to_be_reverted, failed_to_publish_services = self.find_services_by_publish_status( last_published_record) if services_to_be_reverted: filter_data = {"$or": [{"_key": service_id} for service_id in services_to_be_reverted]} backfill_filter = {"kpis.backfill_enabled": True} and_filter = {"$and": [filter_data, backfill_filter]} backfill_enabled_services = self.service_interface.get_bulk_skip_enforce_security( owner, fields=['title'], filter_data=and_filter) if len(backfill_enabled_services) > 0: backfill_enabled = True return backfill_enabled, backfill_enabled_services @sandbox_lock_and_update_count_factory(Status.STATUS_SYNC) @InstrumentCall(logger) def synchronize_sandbox(self, owner, sandbox_id, change, transaction_id): """ Synchronize the updates from Service Template or teams with Sandbox. We will be locking the sandbox and the status will be set to Sync. :param owner: Owner of the objects :type: string :param sandbox_id: ID of the sandbox :type: string :return: boolean of the operation commpletion :type: boolean """ sandbox_sync_completed = False # Get the list of changes from the sandbox sync log # Check the sandbox sync log for the presence of action_type as synchronize . from itsi.objects.changehandlers.sandbox_base_service_template_update_handler import SandboxBaseServiceTemplateUpdateHandler try: if (change.get("changed_object_type") == "base_service_template") and ( change.get("change_type") == "sandbox_base_service_template_update"): # Call the sandbox update handler to perform the sync operation with the affected services sandbox_sync_completed = SandboxBaseServiceTemplateUpdateHandler(logger, self._session_key).perform_template_sync( change=change, scheduled_for_later=False) elif (change.get("changed_object_type") == "base_service_template") and ( change.get("change_type") == "sandbox_delete_base_service_template"): sandbox_sync_completed = SandboxServiceTemplateDeleteHandler(logger, self._session_key).deferred( change=change, transaction_id=transaction_id) except BaseException as e: logger.error('Synchronizing not Completed for Sandbox sandboxid=%s', sandbox_id) raise e logger.info('Synchronizing Completed for Sandbox sandboxid=%s', sandbox_id) return sandbox_sync_completed @InstrumentCall(logger) def validated_publish_sandbox(self, owner, sandbox_id, transaction_id): """ Wrapper to ensure that publish operations can only follow validate operations for a sandbox. :param owner: Owner of the objects :type: string :param sandbox_id: ID of the sandbox :type: string :param transaction_id: ID of transaction for tracebacks :type: string :return: Sandbox sync log creation response :type: dict """ modified_since_last_validate = [] sandbox_services = self.sandbox_service_interface.get_bulk( owner, filter_data={'sandbox_id': sandbox_id}, fields=['_key', 'mod_timestamp'], transaction_id=transaction_id, ) log_filter = { '$and': [ {'mod_object_type': self.sandbox_interface.object_type}, {'mod_object_id': sandbox_id} ] } last_log = self.sandbox_sync_log_interface.get_bulk( owner, filter_data=log_filter, sort_key='timestamp', sort_dir='desc', fields=['_key', 'timestamp', 'mod_timestamp'], transaction_id=transaction_id, ) if last_log: for service in sandbox_services: service_timestamp = utils.parse_mod_timestamp(service['mod_timestamp']) if service_timestamp >= last_log[0]['timestamp']: # Re-using this to store warnings modified_since_last_validate.append({ '_key': service['_key'], 'object_type': self.sandbox_service_interface.object_type, 'description': 'Sandbox service %s was not validated when publish was attempted.' % service['_key'], }) else: modified_since_last_validate.append({ '_key': sandbox_id, 'object_type': self.sandbox_interface.object_type, 'description': 'Sandbox %s has no records.' % sandbox_id, }) if modified_since_last_validate: validation_response = self.validate_sandbox(owner, sandbox_id, 'pre_%s' % transaction_id) validation_results = self.sandbox_sync_log_interface.get(owner, validation_response['_key']) if validation_results['errors']: validation_results['warnings'].extend(modified_since_last_validate) return validation_results return self.publish_sandbox(owner, sandbox_id, transaction_id) def find_services_by_publish_status(self, last_published_record): """ Identifies the services that are succesfuly published and failed to publish from the sandbox's last_published_record :param last_published_record: Last published record detail from the sandbox :type: dict :return: List of successfully published service keys and failed to publish service keys :rtype: (list, list) """ errors = last_published_record.get('errors', []) successfuly_published_services = [] failed_service_trees = [] failed_to_publish_services = [] for error in errors: if error['object_type'] == 'sandbox_service_tree': failed_service_trees.append(error['_key']) elif error['object_type'] == self.service_interface.object_type: failed_to_publish_services.append(error['_key']) details = last_published_record.get('details', {}) if 'service_tree' in details and 'graphs' in details['service_tree']: service_tree_graphs = details['service_tree']['graphs'] for graph in service_tree_graphs: if graph['id'] not in failed_service_trees: for vertice in graph['vertices']: successfuly_published_services.append(vertice['id']) if graph['id'] in failed_service_trees: for vertice in graph['vertices']: failed_to_publish_services.append(vertice['id']) return successfuly_published_services, failed_to_publish_services @InstrumentCall(logger) def publish_revert_sandbox(self, owner, sandbox_id, transaction_id): """ Checks if revert of sandbox can be performed and then invokes the revert :param owner: Owner of the objects :type: string :param sandbox_id: ID of the sandbox :type: string :param transaction_id: ID of transaction for tracebacks :type: string :return: results from the revert operation :type: dict """ backfill_enabled, backfill_enabled_services = self.get_backfill_enabled_value(owner, sandbox_id) if backfill_enabled: if len(backfill_enabled_services) > 0: service_names = [ service['title'] for service in backfill_enabled_services] logger.error('Revert operations are blocked as services="%s" has the kpi backfill enabled' % (service_names)) raise ItoaError(logger=logger, status_code=409, message="Reverting of sandbox with service containing backfill enabled kpi is not allowed ") return self.process_publish_revert_sandbox(owner, sandbox_id, transaction_id) @sandbox_lock_and_update_count_factory(Status.STATUS_PUBLISH_REVERT, update_count=True, multiple_sandbox_publish_validate_status_check=False) @InstrumentCall(logger) def process_publish_revert_sandbox(self, owner, sandbox_id, transaction_id): """ Revert services back to published sandbox and clear the production services :param owner: Owner of the objects :type: string :param sandbox_id: ID of the sandbox :type: string :param transaction_id: ID of transaction for tracebacks :type: string :return: results from the revert operation :type: dict """ sandbox = self.sandbox_interface.get(owner, sandbox_id) log_link = self.sandbox_utils.get_search_log_url(transaction_id) sandbox_name = sandbox['identifying_name'] notification_type = SplunkMessageHandler.INFO notification_message = "%s reverted to the last published version. %s for more details." % (sandbox_name, log_link) last_published_record = sandbox.get('last_published_record', {}) services_to_be_reverted, failed_to_publish_services = self.find_services_by_publish_status(last_published_record) data = { 'services_to_be_reverted': services_to_be_reverted, 'services_skipped_from_revert': failed_to_publish_services, 'errors' : []} log_object = { 'title': 'Revert published services %s (%s)' % (self.service_interface.object_type, transaction_id), 'action_type': 'Revert publish', 'details': data, 'sec_grp': self.set_first_assigned_security_group(owner, transaction_id, data), } try: if services_to_be_reverted: filter_data = {'$or': [{'_key': service_id} for service_id in services_to_be_reverted]} self.service_interface.delete_bulk(owner, filter_data=filter_data, transaction_id=transaction_id) # Empty the last published record information from sandbox after revert self.sandbox_interface.update(owner, sandbox_id, data={'last_published_record': {}}, is_partial_data=True, transaction_id=transaction_id) except Exception as e: for service_id in services_to_be_reverted: # This is technically inaccurate, but this exists to provide clearer information to the user data['errors'].append({ '_key': service_id, 'object_type': self.service_interface.object_type, 'description': 'Error encountered in failed services clean-up: %s' % str(e), }) notification_type = SplunkMessageHandler.ERROR notification_message = "%s failed to revert. %s for more details." % (sandbox_name, log_link) self.sandbox_sync_log_interface.create_record( owner, transaction_id, self.sandbox_interface.object_type, sandbox_id, log_object) self.sandbox_utils.message_handler.post_or_update_message( 'service_sandbox_%s' % ( transaction_id), notification_type, notification_message ) return data def clear_publish_references_from_sandbox(self, owner, sandbox_id, transaction_id): """ Empties out the last_published_record information from the sandbox :param owner: Owner of the objects :type: string :param sandbox_id: ID of the sandbox :type: string :param transaction_id: ID of transaction for tracebacks :type: string """ sandbox = self.sandbox_interface.get(owner, sandbox_id) last_published_record = sandbox.get('last_published_record', {}) services_to_be_deleted, failed_to_publish_services_unused = self.find_services_by_publish_status(last_published_record) if services_to_be_deleted: filter_data = {'$or': [{'_key': service_id} for service_id in services_to_be_deleted]} self.clear_publish_history(owner, filter_data, transaction_id) def clear_publish_history(self, owner, filter_data, transaction_id): """ Clears the publish history from the published services :param owner: Owner of the objects :type: string :param sandbox_id: ID of the sandbox :type: string :param transaction_id: ID of transaction for tracebacks :type: string """ services_list = self.service_interface.get_bulk(owner, filter_data=filter_data, fields=[ '_key'], sort_key='timestamp', sort_dir='desc', transaction_id=transaction_id, ) for service in services_list: service['publish_history'] = [] # Empty the publish_history information from service after reset of sandbox if len(services_list) > 0: self.service_interface.save_batch( owner, services_list, method=CRUDMethodTypes.METHOD_UPDATE, validate_names=True, is_partial_data=True, transaction_id=transaction_id) @sandbox_lock_and_update_count_factory(Status.STATUS_PUBLISH_REVERT, update_count=True, multiple_sandbox_publish_validate_status_check=False) @InstrumentCall(logger) def publish_reset_sandbox(self, owner, sandbox_id, transaction_id): """ Resets the sandbox back to editable state The failed to publsih services will remain in the sandbox after publish. Successfully published services would be cleared off from the sandbox. :param owner: Owner of the objects :type: string :param sandbox_id: ID of the sandbox :type: string :param transaction_id: ID of transaction for tracebacks :type: string :return: results from the reset operation :type: dict """ sandbox = self.sandbox_interface.get(owner, sandbox_id) log_link = self.sandbox_utils.get_search_log_url(transaction_id) sandbox_name = sandbox['identifying_name'] notification_type = SplunkMessageHandler.INFO notification_message = "%s reset completed. %s for more details." % (sandbox_name, log_link) last_published_record = sandbox.get('last_published_record', {}) services_to_be_resetted, failed_to_publish_services = self.find_services_by_publish_status(last_published_record) data = { 'services_to_be_resetted': services_to_be_resetted, 'services_skipped_from_reset': failed_to_publish_services, 'errors' : []} log_object = { 'title': 'Reset published services %s (%s)' % (self.service_interface.object_type, transaction_id), 'action_type': 'Reset publish', 'details': data, 'sec_grp': self.set_first_assigned_security_group(owner, transaction_id, data), } try: if services_to_be_resetted: filter_data = {'$or': [{'_key': service_id} for service_id in services_to_be_resetted]} self.sandbox_service_interface.delete_bulk(owner, filter_data=filter_data, transaction_id=transaction_id) self.clear_publish_history(owner, filter_data, transaction_id) # Empty the last published record information from sandbox after revert # set the backfill enabled flag to false by default self.sandbox_interface.update(owner, sandbox_id, data={'last_published_record': {}, 'backfill_enabled': False}, is_partial_data=True, transaction_id=transaction_id) except Exception as e: for service_id in services_to_be_resetted: # This is technically inaccurate, but this exists to provide clearer information to the user data['errors'].append({ '_key': service_id, 'object_type': self.sandbox_service_interface.object_type, 'description': 'Error encountered in failed services clean-up: %s' % str(e), }) notification_type = SplunkMessageHandler.ERROR notification_message = "%s failed to reset. %s for more details." % (sandbox_name, log_link) self.sandbox_sync_log_interface.create_record( owner, transaction_id, self.sandbox_interface.object_type, sandbox_id, log_object) self.sandbox_utils.message_handler.post_or_update_message( 'service_sandbox_%s' % ( transaction_id), notification_type, notification_message ) return data