# Copyright (C) 2005-2024 Splunk Inc. All Rights Reserved. import sys import logging from copy import deepcopy import json import splunk.Intersplunk from splunk.clilib.bundle_paths import make_splunkhome_path from splunk import rest sys.path.append(make_splunkhome_path(['etc', 'apps', 'SA-ITOA', 'lib'])) import itsi_path from itsi_py3 import _ import itsi_py3 from ITOA.itoa_common import get_current_timestamp_utc from ITOA.setup_logging import getLogger4SearchCmd from itsi.itsi_utils import ITOAInterfaceUtils from itsi.objects.itsi_service import ItsiService from itsi.objects.itsi_entity import ItsiEntity from itsi.objects.itsi_security_group import ItsiSecGrp logger, settings, records = getLogger4SearchCmd(level=logging.ERROR, is_console_header=True, return_all=True) def parseArgs(): ''' Parse out the arguments passed into the scripted input entityfields is a multivalued field, where the values are separated by semicolons @retval: a dict of the valid arguments @rettype: dict ''' i = 1 required_parameters = ['title_field'] optional_parameters = ['identifier_fields','informational_fields','description_fields','services', 'preview','insertion_mode','service_fields', 'sec_grp'] valid_parameters = required_parameters + optional_parameters parsed_params = {} #We could probably list/dict comprehension the hell out of this #But we wont for arg in sys.argv[1:]: #We dont want the zeroth element found = False for param in valid_parameters: param_string = param+"=" if arg.find(param_string) != -1: if param in parsed_params: splunk.Intersplunk.parseError(_("Duplicate parameter found: %s.") % param) parsed_params[param] = arg[arg.find(param_string)+len(param_string):] found = True if not found: #The arg was not found in the list of valid params - invalid key splunk.Intersplunk.parseError(_("Invalid key specified: %s.") % arg) #Make sure that all of the valid parameters are there #And no invalid parameters are there for key in list(parsed_params.keys()): if key in required_parameters: required_parameters.remove(key) if len(required_parameters) != 0: splunk.Intersplunk.parseError(_("Missing required key(s): %s.") % ",".join(required_parameters)) #Special processing for anything that could be represented as a list special_params = ['identifier_fields','informational_fields','description_fields','services','service_fields'] for special in special_params: if special in parsed_params: parsed_params[special] = parsed_params[special].split(';') return parsed_params def getModStamp(sessionKey): #username = getCurrentUser() resp,content = rest.simpleRequest('/authentication/current-context',getargs={"output_mode":"json"},sessionKey=sessionKey,raiseAllErrors=False) content = json.loads(content) username = content['entry'][0]["content"]["username"] modified_time = get_current_timestamp_utc() source = 'splunk_search' return {"mod_by":username,"mod_time":modified_time,"mod_source":source} def appendEntity(sessionKey, owner, entity_object, entityRecord, preview): ''' Appends a new entity to the collection, used by the other modes such as upsert and replace ''' if preview: entityRecord['_key'] = 'New key' else: #Go throw it into the statestore mods = getModStamp(sessionKey) entityRecord["create_by"] = mods["mod_by"] entityRecord["create_time"] = mods["mod_time"] entityRecord["create_source"] = mods["mod_source"] entity_key = entity_object.create(owner, entityRecord) entityRecord['_key'] = entity_key['_key'] return [entityRecord] def replaceEntity(sessionKey, owner, entity_object, entityRecord, preview): ''' Attempts to find the existing entity and replaces it if there is a match ''' existing_entities = entity_object.get_bulk(owner, filter_data={"title":entityRecord['title']}) #Replace ALL entities that match on the existing one if len(existing_entities) == 0: return appendEntity(sessionKey, owner, entity_object, entityRecord, preview) new_entities = [] for e in existing_entities: #We want to make a deep copy of the entity here because we don't want #Information leakage between entities when creating them new_entity = deepcopy(entityRecord) #Transfer over all of the existing information that might be relevent for x in ['_key','create_time','create_by','create_source', 'services', '_version']: if x in e: new_entity[x] = e[x] new_entities.append(new_entity) if not preview: entity_object.save_batch(owner, new_entities, False, req_source = 'storeentities') return new_entities def upsertEntity(sessionKey, owner, entity_object, entityRecord, preview): ''' Attempts to insert an entity if there is no match, if there is, updates the existing record ''' array_fields = ['services'] existing_entities = entity_object.get_bulk(owner, filter_data={"title":entityRecord['title']}) #Upsert on ALL entities that match on the existing one if len(existing_entities) == 0: return appendEntity(sessionKey, owner, entity_object, entityRecord, preview) new_entities = [] for e in existing_entities: new_entity = deepcopy(entityRecord) for x in ['_key','create_time','create_by','create_source']: if x in e: new_entity[x] = e[x] for f in array_fields: if type(e[f]) is list: new_entity[f] += e[f] else: new_entity[f] += e[f].split(',') for n in ["identifier","informational"]: if len(e.get(n,{})) > 0: identifiers = e[n] fields = identifiers.get("fields",None) values = identifiers.get("values",None) if fields == None or values==None: continue for f in fields: values = e[f] if isinstance(values, itsi_py3.ext_string_type): values = values.split(',') #The new entity record should always be a list at this point new_entity[f] = list(set(e[f] + new_entity.get(f,[]))) new_entities.append(new_entity) if not preview: entity_object.save_batch(owner, new_entities, False, req_source = 'storeentities') return new_entities def storeEntities(sessionKey,owner,params,records): ''' Takes the records specified, and from them extracts the fields required. The ''' LOG_PREFIX = "[storeentities] " logger.debug(LOG_PREFIX + "begin execution") entity_object = ItsiEntity(sessionKey, 'nobody') new_records = [] #Load all the instructions for parsing insertion_mode = params.get('insertion_mode','append') informational_fields = params.get("informational_fields",None) #If this is None, then we should grab all of the fields identifier_fields = params.get("identifier_fields",None) #If this is None, then we should grab all of the fields service_fields = params.get("service_fields",None) #If this is None, then we should grab all of the fields description_fields = params.get("description_fields",None) #If this is None, then we should grab all of the fields title_field = params.get("title_field",None) sec_grp = params.get('sec_grp', ItsiSecGrp.get_default_itsi_security_group_key()) if identifier_fields == None: identifier_fields = [] if len(records) > 0: first_record = records[0] identifier_fields = list(first_record.keys()) identifier_fields = [field for field in identifier_fields if not field.startswith('__mv')] else: splunk.Intersplunk.parseError(_("No fields to automatically extract from results set.")) elif isinstance(identifier_fields, itsi_py3.ext_string_type): identifier_fields = identifier_fields.split(";") if informational_fields == None: informational_fields = [] elif isinstance(informational_fields, itsi_py3.ext_string_type): informational_fields = informational_fields.split(";") if service_fields == None: service_fields = [] elif isinstance(service_fields, itsi_py3.ext_string_type): service_fields = service_fields.split(";") if description_fields == None: description_fields = [] elif isinstance(description_fields, itsi_py3.ext_string_type): description_fields = description_fields.split(";") #Fields now should be an array of associated input fields. These are inputs #That are associated with each other logger.debug(LOG_PREFIX + "begin result analysis") try: mods = getModStamp(sessionKey) for result in records: new_entity = {} informational_values = set() informational_fields_to_remove = [] for field in informational_fields: entity = result.get(field,None) if entity is None: #This field doesn't exist in the informational fields informational_fields_to_remove.append(field) continue if isinstance(entity, itsi_py3.ext_string_type): ne = entity.split(",") else: ne = entity #Add each one as an array value new_entity[field] = ne informational_values.update(ne) #remove the informational fields that had null values for x in informational_fields_to_remove: informational_fields.remove(x) identifier_values = set() for field in identifier_fields: entity = result.get(field,None) if entity is None: splunk.Intersplunk.parseError(_("Storeentities error - identifying field {0} must be non-Null for all entities").format(field)) continue if isinstance(entity, itsi_py3.ext_string_type): ne = entity.split(",") else: ne = entity new_entity[field] = ne #Add each one as an array value identifier_values.update(ne) title = result.get(title_field,None) #NOTE: This should already be split correctly, per parseArgs above services = params.get('services',[]) services_for_entity = [] service_object = ItsiService(sessionKey, 'nobody') for field in service_fields: new_service_title = result.get(field,None) if new_service_title is None: continue existing_services = service_object.get_bulk(owner, filter_data={'$and':[ {"title":new_service_title}, {'sec_grp': sec_grp}]}) services_to_create = [] #If the name doesn't exist, add it if len(existing_services) == 0: serviceRecord = {"title":new_service_title} mods = getModStamp(sessionKey) serviceRecord["create_by"] = mods["mod_by"] serviceRecord["create_time"] = mods["mod_time"] serviceRecord["create_source"] = mods["mod_source"] serviceRecord["_key"] = serviceRecord.get('_key', ITOAInterfaceUtils.generate_backend_key()) serviceRecord['sec_grp'] = sec_grp services_to_create.extend([serviceRecord]) services_for_entity.append({'_key': serviceRecord['_key'], 'title': serviceRecord['title']}) else: services_for_entity.append({ '_key': existing_services[0]['_key'], 'title': existing_services[0]['title'] }) service_object.save_batch( owner, services_to_create, False, req_source = 'storeentities' ) if new_service_title not in services: services.append(new_service_title) new_entity['description'] = '' for field in description_fields: entity = result.get(field,None) if entity is None: continue new_entity['description'] = new_entity['description'] + ' ' + entity #So now we should have a set of entities and a list of fieldnames #These will be translated into the key-value pairs new_entity["identifier"] = {"fields":identifier_fields,"values":list(identifier_values)} new_entity["informational"] = {"fields":informational_fields,"values":list(informational_values)} new_entity["title"] = title new_entity["services"] = services_for_entity new_entity['sec_grp'] = sec_grp preview = params.get('preview',False) if preview == False or preview == '0' or preview.lower() == 'false' or preview.lower() == 'f': preview = False else: preview = True #Now that we're ready to update the db, update the creation fields new_entity['mod_time'] = mods['mod_time'] new_entity['mod_by'] = mods['mod_by'] new_entity['mod_source'] = mods['mod_source'] #Okay, now to actually do the deed if insertion_mode.lower() == 'append': new_entities = appendEntity(sessionKey, owner, entity_object, new_entity, preview) elif insertion_mode.lower() == 'upsert': new_entities = upsertEntity(sessionKey, owner, entity_object, new_entity, preview) elif insertion_mode.lower() == 'replace': new_entities = replaceEntity(sessionKey, owner, entity_object, new_entity, preview) for ne in new_entities: new_result = deepcopy(result) if preview: #Show them what would get put into the statestore at this point new_result['preview_identifier_fields'] = ne['identifier']['fields'] new_result['preview_identifier_values'] = ne['identifier']['values'] new_result['preview_informational_fields'] = ne['informational']['fields'] new_result['preview_informational_values'] = ne['informational']['values'] for f in ne['informational']['fields']: new_result['preview_' + f] = ne[f] for f in ne['identifier']['fields']: new_result['preview_' + f] = ne[f] new_result['preview_title'] = ne['title'] new_result['preview_services'] = ",".join([s.get('title') for s in ne.get('services',[])]) new_result['preview_mod_time'] = ne['mod_time'] new_result['preview_mod_by'] = ne['mod_by'] new_result['preview_mod_source'] = ne['mod_source'] new_result['preview_insertion_mode'] = insertion_mode new_result['preview_sec_grp'] = ne['sec_grp'] new_result['key'] = ne.get('_key','ERROR RETRIEVING KEY') new_records.append(new_result) except Exception as e: logger.exception(e) splunk.Intersplunk.parseError(_("Storeentities exception: %s") % str(e)) logger.debug(LOG_PREFIX + "end") return new_records def main(): params = parseArgs() # 'settings' and 'records' has been preloaded by getLogger4SearchCmd. sessionKey = settings['sessionKey'] owner = "nobody" # entities can only exist at app-level results = storeEntities(sessionKey,owner,params,records) splunk.Intersplunk.outputResults(results) main()