# Copyright (C) 2005-2024 Splunk Inc. All Rights Reserved. """ Low level interface for interacting with the kvstore """ import json from urllib.parse import quote_plus import re import sys import time import splunk.rest as rest from itsi_py3 import _ from splunk import ResourceNotFound, RESTException LOGGER = "itoa.storage.statestore" from ITOA.setup_logging import logger from ITOA.itoa_common import get_conf_stanza_single_entry, get_object_batch_size SUPPORTED_ITSI_OBJECT_TYPES = [ "backup_restore", "base_service_template", "custom_threshold_windows", "deep_dive", "entity", "entity_filter_rule", "entity_management_policies", "entity_management_rules", "entity_relationship", "entity_relationship_rule", "entity_type", "event_management_state", "glass_table", "home_view", "kpi", "kpi_at_info", "kpi_base_search", "kpi_entity_threshold", "kpi_template", "kpi_threshold_template", "saved_page", "service", "team", "temporary_kpi", "upgrade_readiness_prechecks", "sandbox", "sandbox_service", "sandbox_sync_log", "data_integration" ] # For special handling of objects that aren't first-class objects in the KVstore NONKVSTORE_OBJECT_TYPES = set([ "kpi", ]) class StateStoreError(Exception): def __init__(self, message, status_code=None): self.message = message self.status_code = status_code def __str__(self): s = repr(self.message) if self.status_code is not None: s = "status={} ".format(self.status_code) + s return s class StateStore(object): appname = "SA-ITOA" # Per http://docs.splunk.com/Documentation/Splunk/6.5.0/RESTREF/RESTkvstore#Limits, 16 MB is max limit for document # size in KV store and non-configurable _max_document_size_limit_bytes = 16777216 # 16 * 1024 * 1024 = 16 MB # Per http://docs.splunk.com/Documentation/Splunk/6.5.0/RESTREF/RESTkvstore#Limits, the max size per batch save in # MB is set in the kvstore stanza in the limits.conf file with the name max_size_per_batch_save_mb _max_size_per_batch_save = None # Per http://docs.splunk.com/Documentation/Splunk/6.5.0/RESTREF/RESTkvstore#Limits, the max count of documents per # batch save _max_documents_per_batch_save = None # Configurable field to set the timeout for each rest call this value is configurable in itsi_settings.conf # under the Rest section. The default value is 300 _rest_timeout = None def __init__(self, **kwargs): # Defaults here self.owner = "nobody" self.app = kwargs.get("namespace") or self.appname self.collectionname = kwargs.get("collection", "itsi_services") def _set_batch_save_size_limit(self, session_key, host_base_uri=''): """ Fetches the max size per batch save if not already fetched @param session_key: splunkd session key @type session_key: string @param host_base_uri: The base uri ://: of the target host. '' targets local host. @type host_base_uri: basestring """ # Sets static variable from limits conf file if not already set if StateStore._max_size_per_batch_save is None: resp = get_conf_stanza_single_entry(session_key, 'limits', 'kvstore', 'max_size_per_batch_save_mb', host_base_uri=host_base_uri) max_size_per_batch_save_mb = int(resp.get('content', 50)) # Used 0.99 of max_size_per_batch_save_mb in calculation _max_size_per_batch_save, # As during the payload size calculation, extra comma and space will be added. StateStore._max_size_per_batch_save = (0.99 * max_size_per_batch_save_mb) * 1024 * 1024 if StateStore._max_documents_per_batch_save is None: resp = get_conf_stanza_single_entry(session_key, 'limits', 'kvstore', 'max_documents_per_batch_save', host_base_uri=host_base_uri) StateStore._max_documents_per_batch_save = int(resp.get('content', 1000)) def _set_rest_configurations(self, session_key, host_base_uri=''): """ Fetches the rest configuration if not already fetched @param session_key: splunkd session key @type session_key: string @param host_base_uri: The base uri ://: of the target host. '' targets local host. @type host_base_uri: basestring """ # Sets static variable from itsi_settings conf file if not already set if StateStore._rest_timeout is None: try: resp = get_conf_stanza_single_entry(session_key, 'itsi_settings', 'rest', 'rest_timeout', host_base_uri=host_base_uri) StateStore._rest_timeout = int(resp.get('content', 300)) except Exception: StateStore._rest_timeout = 300 def check_payload_size(self, data_list, session_key, throw_on_violation=True, host_base_uri=''): """ Method to verify KV store payload size isnt larger than 16MB limit of per document size @type: list @param data_list: JSON list payload to verify @type: boolean @param throw_on_violation: True if violation should trigger exception, else returns bool indicating if violation detected @type host_base_uri: basestring @param host_base_uri: The base uri ://: of the target host. '' targets local host. @rtype: tuple (boolean, [int] or int) @return: (True, array of starting and ending index of each batch) if no violation detected, (False, size causing violation in bytes) if violation detected """ if not isinstance(data_list, list): raise StateStoreError(_('JSON payload is invalid.')) self._set_batch_save_size_limit(session_key, host_base_uri=host_base_uri) save_ranges = [] cur_size = 0 first_index = 0 for idx, data in enumerate(data_list): size_of_payload = sys.getsizeof(str(data)) if size_of_payload > self._max_document_size_limit_bytes: if throw_on_violation: raise StateStoreError( _('Object you are trying to save is too large (%s bytes). KV store only supports ' 'documents within 16MB sizes.') % size_of_payload ) else: # Return False indicating violation even if one object violates limits return False, size_of_payload cur_size += size_of_payload # Check that you have not reached the document limit if cur_size >= StateStore._max_size_per_batch_save or ( idx - first_index) == StateStore._max_documents_per_batch_save: save_ranges.append((first_index, idx)) first_index = idx cur_size = size_of_payload save_ranges.append((first_index, len(data_list))) return True, save_ranges def lazy_init(self, session_key, host_base_uri=''): ''' Method to initialize all the ITSI collections if it is done so. Query the kvstore config uri with the collection name, initialize the collection if 404 exception is returned, otherwise pass. @param session_key: The splunkd session key @type session_key: string @param host_base_uri: The base uri ://: of the target host. '' targets local host. @type host_base_uri: basestring ''' LOG_PREFIX = "[lazy_init] " entries = [] uri = host_base_uri + "/servicesNS/" + self.owner + "/" + self.app + "/storage/collections/config/" + \ self.collectionname try: response, content = rest.simpleRequest( uri, getargs={"output_mode": "json"}, sessionKey=session_key, raiseAllErrors=False ) parsed_content = json.loads(content) entries = parsed_content.get('entry', []) except ResourceNotFound: logger.debug('%s does not exist, it could be a new collection, will try to create it.', self.collectionname) except Exception as e: logger.exception(str(e)) if len(entries) == 0: # If it doesnt have the collection, we need to create it postargs = {"name": self.collectionname} postargs["output_mode"] = "json" response, content = rest.simpleRequest( uri, method="POST", postargs=postargs, sessionKey=session_key, raiseAllErrors=False ) if response.status != 200 and response.status != 201: logger.error("%s Unable to create collection=`%s`. URL=`%s`. Response=`%s`. Content=`%s`", LOG_PREFIX, self.collectionname, uri, response, content) else: logger.debug("%s Created collection successfully.", LOG_PREFIX) def is_available(self, session_key, host_base_uri=''): """ Tries to check the kvstore readiness via server info REST endpoint Parse kvStoreStatus status from the Json response. @param session_key: The splunkd session key @type session_key: string @param host_base_uri: The base uri ://: of the target host. '' targets local host. @type host_base_uri: basestring @returns true if response contains ready @rtype bool """ kvStoreStatus = '' uri = host_base_uri + '/services/server/info' getargs = {'output_mode': 'json'} response, content = rest.simpleRequest(uri, getargs=getargs, sessionKey=session_key, rawResult=True) try: parsed_content = json.loads(content) if isinstance(parsed_content, dict): entry = parsed_content.get('entry', []) if entry[0]: content = entry[0].get('content', {}) kvStoreStatus = content.get('kvStoreStatus', '') except Exception: pass return 'ready' in kvStoreStatus def create(self, session_key, owner, objecttype, data, host_base_uri=''): """ Create accepts different entity specifiers here, but can be reporposed for other collection tasks @param session_key: splunkd session key @type session_key: string @param objecttype: The type of object we are attempting to create @type objecttype: string @param data: The dict data (suitable for json-ification) @type data: dict @param host_base_uri: The host to run the REST request. '' defaults to localhost @type host_base_uri: string """ LOG_PREFIX = "[create_statestore_" + objecttype + "]" # Build the request uri = host_base_uri + "/servicesNS/" + owner + "/" + self.app + "/storage/collections/data/" + \ self.collectionname data['object_type'] = objecttype response, content = rest.simpleRequest( uri, method="POST", jsonargs=json.dumps(data), sessionKey=session_key, raiseAllErrors=False ) if response.status != 200 and response.status != 201: # Something failed in our request, raise an error message = _("Unable to save {0}, request failed. ").format(objecttype) logger.error('%s %s. response=`%s` content=`%s`', LOG_PREFIX, message, response, content) raise StateStoreError(content, response.status) try: # What we'll get back here is of the form {"_key":"somelongnumber"} (note the quotes) parsed_content = json.loads(content) return parsed_content except TypeError: message = _("Unable to parse response from statestore for {0} {1}.").format(objecttype, data) logger.exception(LOG_PREFIX + message) raise StateStoreError(message) def get(self, session_key, owner, objecttype, identifier, sort_key=None, sort_dir=None, filter_data={}, fields=None, skip=None, limit=None, host_base_uri=''): """ Retrieves the object specified by the identifier, which can be either internal or external @param session_key: The splunkd session key @type session_key: str @param objecttype: The type of object (currently service or entity) @type objecttype: str @param identifier: The object's primary identifier, if None retrieves all objects of the selected type @type identifier: str or None @param sort_key: the field on which to ask the server to sort the results @type sort_key: str @param fields: An array of fields to be returned. This is an array that is used to limit the field set returned @type fields: list @param host_base_uri: The host to to run the REST request. '' defaults to localhost @type host_base_uri: string """ LOG_PREFIX = "[get_statestore_" + objecttype + "]" self._set_rest_configurations(session_key, host_base_uri=host_base_uri) uri = host_base_uri + "/servicesNS/" + owner + "/" + self.app + "/storage/collections/data/" + \ self.collectionname get_args = None if identifier is None: filter_data = {} if filter_data is None else filter_data endpoint = uri # Here we plan on getting all elements # Pass in the filter_data and use the sort_key,sort_dir if defined if sort_key is not None and sort_dir is None: logger.error(LOG_PREFIX + "sort_key defined as {0} with no sort direction".format(sort_key)) elif sort_key is None and sort_dir is not None: logger.error(LOG_PREFIX + "sort_dir defined as {0} with no sort key".format(sort_dir)) elif sort_key is not None and sort_dir is not None: if sort_dir == "desc": sort_dir = -1 else: sort_dir = 1 # Default to ascending get_args = {"sort": sort_key + ":" + str(sort_dir)} filter_data["object_type"] = objecttype # ITOA-2913 if "" in filter_data: message = _("Empty field received - Rejecting filter.") logger.error(LOG_PREFIX + message) raise StateStoreError(message) if "filter_string" in filter_data: logger.debug(LOG_PREFIX + "filter_string=%s", filter_data["filter_string"]) filter_data.update(filter_data["filter_string"]) del filter_data["filter_string"] if "shared" in filter_data: get_args = {} if get_args is None else get_args get_args["shared"] = filter_data["shared"] del filter_data["shared"] if fields is not None: get_args = {} if get_args is None else get_args if "object_type" not in fields: fields.append("object_type") exclude = [field for field in fields if ':0' in field] # Mongodb does not allow field inclusion and exclusion in a single query. # The assumption is that if there is more than one field exclusion, # the system will ignore the field inclusion. if len(exclude) > 0: final_fields = exclude else: final_fields = fields get_args['fields'] = ','.join(final_fields) if skip is not None: get_args = {} if get_args is None else get_args get_args['skip'] = skip if limit is not None: get_args = {} if get_args is None else get_args get_args['limit'] = limit # At this point, 'filter_data' should have only the data for the 'query' param # Other params should be stored in 'get_args' and deleted from 'filter_data' if len(filter_data) > 0: get_args = {} if get_args is None else get_args try: get_args['query'] = json.dumps(filter_data) logger.debug(LOG_PREFIX + "json.dumps successful, get_args=%s", get_args) except ValueError: logger.error(LOG_PREFIX + "error parsing json of query - query=%s", filter_data) else: endpoint = uri + "/" + quote_plus(identifier) if objecttype in NONKVSTORE_OBJECT_TYPES: # Only KPIs are supported for now if identifier is None: # pwu: Impossible to hit? Needs further investigation. logger.info("KPI fetched with None identifier") get_args["object_type"] = "service" else: endpoint = uri get_args = { "query": json.dumps({ "object_type": "service", "kpis._key": identifier, }), } content = "[]" for retry in range(3): try: response, content = rest.simpleRequest( endpoint, method="GET", sessionKey=session_key, raiseAllErrors=True, getargs=get_args, timeout=self._rest_timeout ) if 300 > response.status > 199: break else: raise RESTException(response.status) except ResourceNotFound: logger.error("%s 404 Not Found on GET to %s", LOG_PREFIX, endpoint) # Return None when something is not found return None except RESTException as e: if e.statusCode == 503 and retry != 2: logger.warn( "%s status 503 on endpoint %s, assuming KV Store starting up and retrying request, retries=%s", LOG_PREFIX, endpoint, retry) time.sleep(2) else: logger.error("%s status %s on endpoint %s, raising exception", LOG_PREFIX, e.statusCode, endpoint) raise try: parsed_content = json.loads(content) if len(parsed_content) == 0: parsed_content = {} return parsed_content except TypeError: message = _("Unable to parse response from statestore for {0} {1}.").format(objecttype, identifier) logger.exception(LOG_PREFIX + message) raise StateStoreError(message) except ValueError: message = _("Unable to decode response from statestore for {0} {1}.").format(objecttype, identifier) logger.exception(LOG_PREFIX + message) raise StateStoreError(message) def get_via_post_bulk(self, session_key, owner, objecttype, sort_key=None, sort_dir=None, filter_data={}, fields=None, skip=None, limit=None, host_base_uri=""): """ Retrieves all objects that match the set of filters (the query) @param session_key: The splunkd session key @type session_key: str @param owner: owner making the request @type owner: str @param objecttype: The type of object (currently service or entity) @type objecttype: str @param sort_key: the field on which to ask the server to sort the results @type sort_key: str @param sort_dir: Sorting direction (Ascending or Descending) @type sort_dir: str @param filter_data: Filters based on which objects get retrieved @type filter_data: dict @param fields: An array of fields to be returned. This is an array that is used to limit the field set returned @type fields: list @param skip: number of items to skip from the start @type skip: str @param limit: Limit (value) for how many objects should be retrieved. @type limit: str @param host_base_uri: The host to run the REST request. '' defaults to localhost @type host_base_uri: string @return: List of records that are specified by the filters @rtype: list """ LOG_PREFIX = "[get_statestore_" + objecttype + "]" self._set_rest_configurations(session_key, host_base_uri=host_base_uri) endpoint = host_base_uri + "/servicesNS/" + owner + "/" + self.app + \ "/storage/collections/data/" + self.collectionname + "/batch_find?responseFormat=map" post_args = {} filter_data = {} if filter_data is None else filter_data # Here we plan on getting all elements # Pass in the filter_data and use the sort_key,sort_dir if defined if sort_key is not None and sort_dir is None: logger.error(LOG_PREFIX + "sort_key defined as {0} with no sort direction".format(sort_key)) elif sort_key is None and sort_dir is not None: logger.error(LOG_PREFIX + "sort_dir defined as {0} with no sort key".format(sort_dir)) elif sort_key is not None and sort_dir is not None: if sort_dir == "desc": sort_dir = -1 else: sort_dir = 1 # Default to ascending post_args = {"sort": [{sort_key: sort_dir}]} filter_data["object_type"] = objecttype if "" in filter_data: message = _("Empty field received - Rejecting filter.") logger.error(LOG_PREFIX + message) raise StateStoreError(message) if "filter_string" in filter_data: logger.debug(LOG_PREFIX + "filter_string=%s", filter_data["filter_string"]) filter_data.update(filter_data["filter_string"]) del filter_data["filter_string"] if "shared" in filter_data: if filter_data["shared"]: post_args["shared"] = True else: post_args["shared"] = False del filter_data["shared"] if fields is not None: if "object_type" not in fields: fields.append("object_type") exclude = [field for field in fields if ':0' in field] # Mongodb does not allow field inclusion and exclusion in a single query. # The assumption is that if there is more than one field exclusion, # the system will ignore the field inclusion. if len(exclude) > 0: final_fields = exclude # Fields which need to be excluded are provided as ['exclude1:0', 'exclude2:0], # So, striping the last two characters (':0') from the field string post_args["fields"] = {field[:len(field) - 2]: 0 for field in final_fields} else: final_fields = fields post_args["fields"] = {field: 1 for field in final_fields} # At this point, 'filter_data' should have only the data for the 'query' param # Other params should be stored in 'post_args' and deleted from 'filter_data' if len(filter_data) > 0: try: post_args['query'] = filter_data logger.debug(LOG_PREFIX + "json.dumps successful, post_args=%s", post_args) except ValueError: logger.error(LOG_PREFIX + "error parsing json of query - query=%s", filter_data) if skip is not None: skip_index = int(skip) else: skip_index = 0 post_args['skip'] = skip_index if limit is not None: limit_val = int(limit) else: limit_val = 0 post_args['limit'] = limit_val results_remaining = True rv = [] while results_remaining: for retry in range(3): try: response, content = rest.simpleRequest( endpoint, method="POST", sessionKey=session_key, raiseAllErrors=True, # converting post_args variable to list as it is necessary for "batch_find" keyword jsonargs=json.dumps([post_args]), timeout=self._rest_timeout ) if 300 > response.status > 199: break else: raise RESTException(response.status) except ResourceNotFound: logger.error("%s 404 Not Found on GET to %s", LOG_PREFIX, endpoint) # Return None when something is not found return None except RESTException as e: if e.statusCode == 503 and retry != 2: logger.warn( "%s status 503 on endpoint %s, assuming KV Store starting up and retrying request," "retries=%s", LOG_PREFIX, endpoint, retry, ) time.sleep(2) else: logger.error("%s status %s on endpoint %s, raising exception", LOG_PREFIX, e.statusCode, endpoint) raise try: parsed_content = json.loads(content) if len(parsed_content) == 0: results_remaining = False else: parsed_content = parsed_content[0] # Old-style /batch_find endpoint returns a list if type(parsed_content) is list: rv = parsed_content # On older versions of Splunk, we have no visibility results_remaining = False # New-style /batch_find endpoint returns a dict else: rv.extend(parsed_content['records']) # To prevent an infinite loop if a single object can't be fetched records_fetched = len(parsed_content['records']) records_truncated = int(parsed_content.get('recordsTruncated', '0')) if records_truncated and records_fetched == 0: message = _( 'Unable to fetch full results from statestore for query {0}, fetched {1}/{2}' ).format(post_args, len(rv), len(rv) + records_truncated) logger.error(LOG_PREFIX + message) # 507: Insufficient Storage raise RESTException(507) elif records_truncated: post_args['skip'] += records_fetched else: results_remaining = False except TypeError: message = _("Unable to parse response from statestore for {0}.").format( objecttype) logger.exception(LOG_PREFIX + message) raise StateStoreError(message) except ValueError: message = _("Unable to decode response from statestore for {0}.").format( objecttype) logger.exception(LOG_PREFIX + message) raise StateStoreError(message) return rv def edit(self, session_key, owner, objecttype, identifier, data, host_base_uri=''): """ Per the contract that we're defining, edit will only cover a single thing, and that thing will be found by its id which in this case is the statestore id. If no records are found, we will throw an error @param session_key: splunkd session key @type session_key: string @param objecttype: The type of object we are attempting to create - can currently be entity or service @type objecttype: string @param identifier: The id of the object @type identifier: string @param data: The dict data (suitable for json-ification) @type data: dict @param host_base_uri: The host to run the REST request. '' defaults to localhost @type host_base_uri: string """ LOG_PREFIX = "[edit_statestore_" + objecttype + "]" curr_owner = owner uri = host_base_uri + "/servicesNS/" + curr_owner + "/" + self.app + "/storage/collections/data/" + \ self.collectionname + "/" + quote_plus(identifier) # These are the two fields that we require if '_key' not in data: data['_key'] = identifier data['object_type'] = objecttype response0 = self.get(session_key, curr_owner, objecttype, identifier, host_base_uri=host_base_uri) if response0 is None or len(response0) == 0: # object doesn't exist under input namespace if curr_owner != "nobody": message = _("Cannot change permissions from 'app' to 'user' without cloning.") logger.error(LOG_PREFIX + message) raise StateStoreError(message) # find object (if possible) in other namespace if curr_owner == "nobody": curr_owner = data.get('acl').get('owner') if data.get('acl') else "nobody" else: curr_owner = "nobody" response1 = self.get(session_key, curr_owner, objecttype, identifier, host_base_uri=host_base_uri) if response1 is None or len(response1) == 0: # object doesn't exist in either namespace, error message = _("Object with ID: %s does not exist in statestore.") % (identifier) logger.error(LOG_PREFIX + message) raise StateStoreError(message) else: # DELETE object from original namespace self.delete(session_key, curr_owner, objecttype, identifier, host_base_uri=host_base_uri) # CREATE object in input name space return self.create(session_key, owner, objecttype, data, host_base_uri=host_base_uri) else: # found object under current namespace response, content = rest.simpleRequest( uri, method="POST", jsonargs=json.dumps(data), sessionKey=session_key, raiseAllErrors=True ) # Here. we're being generous about what we choose to accept if response.status not in (200, 201, 202): message = _("Unable to edit {0} {1}.").format(objecttype, identifier) logger.error('%s %s. Response=`%s` Content=`%s`', LOG_PREFIX, message, response, content) raise StateStoreError(message + ' ' + str(content), response.status) return {"_key": identifier} def delete(self, session_key, owner, objecttype, identifier, host_base_uri=''): """ Deletes only the record at the specified endpoint, so it will look for that specific record @param session_key: splunkd session key @type session_key: string @param objecttype: The type of object we are attempting to create - can currently be entity or service @type objecttype: string @param identifier: The id of the object @type identifier: string @param host_base_uri: The host to run the REST request. '' defaults to localhost @type host_base_uri: string """ LOG_PREFIX = "[delete_statestore_" + objecttype + "]" # First, we need to get the identifiers of the objects we plan on deleting uri = host_base_uri + "/servicesNS/" + owner + "/" + self.app + "/storage/collections/data/" + \ self.collectionname + "/" + quote_plus(identifier) try: response, content = rest.simpleRequest( uri, method="DELETE", sessionKey=session_key, raiseAllErrors=False ) except ResourceNotFound: # We tried to delete something that doesnt exist, just continue return # Here we're being generous about what we choose to accept if response.status not in (200, 201, 202): message = _("Unable to delete {0} {1}.").format(objecttype, identifier) logger.error('%s %s. Response=`%s`. Content=`%s`', LOG_PREFIX, message, response, content) raise StateStoreError(message + ' ' + str(content), response.status) def delete_all(self, session_key, owner, objecttype, filter_data, host_base_uri=''): """ Implements a bulk delete based on the object type and the other filter data that comes in. While you can put in things without filterdata, in this case I am requiring it to not be empty. Because that's how you delete your entire environment, and that isn't good """ LOG_PREFIX = "[delete_all_statestore_" + objecttype + "]" if filter_data is None or len(filter_data) == 0: logger.error(LOG_PREFIX + "filter data required for batch delete") return # ITOA-2913 if "" in filter_data: message = _("Empty field received - Rejecting filter.") logger.error(LOG_PREFIX + message) raise StateStoreError(message) uri = host_base_uri + "/servicesNS/" + owner + "/" + self.app + "/storage/collections/data/" + \ self.collectionname filter_data['object_type'] = objecttype get_args = {} try: get_args['query'] = json.dumps(filter_data) except ValueError: logger.exception("error parsing json of query - aborting request - query=%s", filter_data) return try: # get_args are used here because those are appended to the URI - so they work in this case for deletes response, content = rest.simpleRequest(uri, method="DELETE", sessionKey=session_key, getargs=get_args, raiseAllErrors=False) del content except ResourceNotFound: # We tried to delete something that doesnt exist, just continue logger.exception("error parsing json of query - aborting request - query=%s", filter_data) return # Here we're being generous about what we choose to accept if response.status != 200: logger.error( LOG_PREFIX + "could not batch delete status={0} query={1}".format(response.status, filter_data)) return def make_get_call_with_appropriate_method(self, session_key, owner, objecttype, sort_key=None, sort_dir=None, filter_data={}, fields=None, skip=None, limit=None, host_base_uri=''): """ Makes Get call to fetch all of data associated with the object type with appropriate method @param session_key: splunkd session key @type session_key: string @param owner: owner making the request @type owner: str @param objecttype: The type of object we are attempting to create - can currently be entity or service @type objecttype: string @param sort_key: the field on which to ask the server to sort the results @type sort_key: str @param sort_dir: Sorting direction (Ascending or Descending) @type sort_dir: str @param filter_data: Currently unused filter data @type filter_data: A dict of kv pairs that can be passed along to the backend for filtering. @param fields: An array of fields to be returned. This is an array that is used to limit the field set returned @type fields: list @param skip: number of items to skip from the start @type skip: str @param limit: Limit (value) for how many objects should be retrieved. @type limit: str @param host_base_uri: The host to run the REST request. '' defaults to localhost @type host_base_uri: string @return: List of records that are specified by the filters @rtype: list """ if objecttype in SUPPORTED_ITSI_OBJECT_TYPES: response = self.get_via_post_bulk( session_key, owner, objecttype, sort_key=sort_key, sort_dir=sort_dir, filter_data=filter_data, fields=fields, skip=skip, limit=limit, host_base_uri=host_base_uri ) else: response = self.get( session_key, owner, objecttype, None, sort_key=sort_key, sort_dir=sort_dir, filter_data=filter_data, fields=fields, skip=skip, limit=limit, host_base_uri=host_base_uri ) if response is None or len(response) == 0: return [] return response def get_all(self, session_key, owner, objecttype, sort_key=None, sort_dir=None, filter_data={}, fields=None, skip=None, limit=None, host_base_uri=''): """ Gets all of the data associated with the object type; like get all entities or services This function will subsequently call paged_get_all to fetch data in paginated format to remove limit=0 calls to collections @param session_key: splunkd session key @type session_key: string @param objecttype: The type of object we are attempting to create - can currently be entity or service @type objecttype: string @param filter_data: Currently unused filter data @type filter_data: A dict of kv pairs that can be passed along to the backend for filtering. """ if objecttype in NONKVSTORE_OBJECT_TYPES: # Only KPIs are supported for now objecttype = "service" if fields is not None: unique_fields = set(fields) fields = list(unique_fields) # Calling paged get call to fetch all objects via pagination results = self.paged_get_all( session_key, owner, objecttype, sort_key=sort_key, sort_dir=sort_dir, filter_data=filter_data, fields=fields, limit=limit, skip=skip, host_base_uri=host_base_uri, ) return results @staticmethod def _extract_content_error(content): """ Method takes in a content string from KV store API returned when API reports error and returns extracted first error message from it @type content: basestring @param content: XML containing content with error message in ...... Will return the first error message from the first element of type @rtype: basestring @return: extracted error message or a default if unavailable """ default_error = 'Unknown, check logs for details.' match = re.search(r'.*.*.*', content, flags=re.S) if match is not None and match.group(0) is not None: error_message = match.group(0) return error_message[error_message.find('>', error_message.find('')] return default_error def batch_save(self, session_key, owner, data, host_base_uri=''): """ WARNING: object_type must be set in everything or the data will be irretrievable with standard library Create accepts different entity specifiers here, but can be repurposed for other collection tasks @param session_key: splunkd session key @type session_key: string @param data: The list of dicts (suitable for json-ification) @type data: list @param host_base_uri: The host to run action. '' defaults to localhost @type host_base_uri: string """ condition, ranges = self.check_payload_size(data, session_key, host_base_uri=host_base_uri) LOG_PREFIX = "[batch_save]" if not isinstance(data, list): message = _("Batch save requires input be a list, actual data input={0}").format(data) logger.error(LOG_PREFIX + message) raise StateStoreError(message) if len(data) == 0: logger.warning("%s empty array passed to batch save, no op required", LOG_PREFIX) return [] parsed_content = [] for range_var in ranges: parsed_content.extend(self._execute_save_request(session_key, owner, data[range_var[0]:range_var[1]], host_base_uri=host_base_uri)) return parsed_content def _execute_save_request(self, session_key, owner, data, host_base_uri=''): """ Executes a save request for the specified data @param session_key: splunkd session key @type session_key: string @param owner: splunkd session key @type session_key: string @param data: The dict data (suitable for json-ification) @type data: dict @rtype: dict @return: return value from the save request @param host_base_uri: The host to run action. '' defaults to localhost @type host_base_uri: string """ LOG_PREFIX = "[batch_save]" uri = host_base_uri + "/servicesNS/" + owner + "/" + self.app + "/storage/collections/data/" + \ self.collectionname + "/batch_save" response, content = rest.simpleRequest( uri, method="POST", jsonargs=json.dumps(data), sessionKey=session_key, raiseAllErrors=False ) if response.status not in (200, 201): details = StateStore._extract_content_error(str(content)) message = _("Batch save to KV store failed with code {0}. Error details: {1}").format(response.status, details) logger.error('%s %s. Response=`%s`. Content=`%s`', LOG_PREFIX, message, response, content) raise StateStoreError(message) try: # What we'll get back here is of the form {"_key":"somelongnumber"} (note the quotes) parsed_content = json.loads(content) return parsed_content except TypeError: message = _("Unable to parse batch response from statestore for batch_edit") logger.exception(LOG_PREFIX + message) raise StateStoreError(message) def paged_get_all(self, session_key, owner, object_type, sort_key=None, sort_dir=None, filter_data={}, fields=None, skip=None, limit=None, host_base_uri=''): """ Paged implementation of get_all method to avoid limit=0 dependency @param session_key: The splunkd session key @type session_key: str @param owner: owner making the request @type owner: str @param object_type: The type of object (currently service or entity) @type object_type: str @param sort_key: the field on which to ask the server to sort the results @type sort_key: str @param sort_dir: Sorting direction (Ascending or Descending) @type sort_dir: str @param filter_data: Filters based on which objects get retrieved @type filter_data: dict @param fields: An array of fields to be returned. This is an array that is used to limit the field set returned @type fields: list @param skip: number of items to skip from the start @type skip: str @param limit: Limit (value) for how many objects should be retrieved. @type limit: str @param host_base_uri: The host to run the REST request. '' defaults to localhost @type host_base_uri: string @return: List of records that are specified by the filters @rtype: list """ LOG_PREFIX = "[paged_get_all_statestore_" + object_type + "]" batch_size = get_object_batch_size(session_key, object_type=object_type) requested_page = [] try: skip = 0 if skip is None else int(skip) except ValueError: logger.error(LOG_PREFIX + "skip value should be valid integer instead passed {0}".format(skip)) try: limit = 0 if limit is None else int(limit) except ValueError: logger.error(LOG_PREFIX + "limit value should be valid integer instead passed {0}".format(limit)) if 0 < limit <= batch_size: requested_page = self.make_get_call_with_appropriate_method(session_key, owner, objecttype=object_type, sort_key=sort_key, sort_dir=sort_dir, filter_data=filter_data, fields=fields, limit=limit, skip=skip, host_base_uri=host_base_uri) elif limit == 0: while True: results = self.make_get_call_with_appropriate_method(session_key, owner, objecttype=object_type, sort_key=sort_key, sort_dir=sort_dir, filter_data=filter_data, fields=fields, limit=batch_size, skip=skip, host_base_uri=host_base_uri) if not results: break skip += batch_size requested_page.extend(results) elif limit > batch_size: while limit > 0: results = self.make_get_call_with_appropriate_method(session_key, owner, objecttype=object_type, sort_key=sort_key, sort_dir=sort_dir, filter_data=filter_data, fields=fields, limit=min(limit, batch_size), skip=skip, host_base_uri=host_base_uri) if not results: break limit -= batch_size skip += batch_size requested_page.extend(results) return requested_page