You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
943 lines
32 KiB
943 lines
32 KiB
# Copyright (C) 2005-2025 Splunk Inc. All Rights Reserved.
|
|
from builtins import str
|
|
from builtins import zip
|
|
from builtins import object
|
|
import json
|
|
import splunk
|
|
import splunk.rest as rest
|
|
import logging
|
|
import time
|
|
|
|
from .api_documenter import api, api_operation, api_response, api_path_param,\
|
|
api_body_param, api_get_spec, api_query_param
|
|
|
|
from .packages.solnlib import log
|
|
from .packages.splunklib import binding
|
|
from .packages.solnlib import splunk_rest_client as rest_client
|
|
|
|
|
|
log.Logs.set_context(log_format='%(asctime)s %(levelname)s %(message)s')
|
|
logger = log.Logs().get_logger('gt_icon_collection')
|
|
logger.setLevel(logging.INFO)
|
|
|
|
import sys
|
|
if sys.version_info >= (3, 0):
|
|
string_type = (str, bytes)
|
|
else:
|
|
import __builtin__
|
|
string_type = __builtin__.basestring
|
|
|
|
|
|
"""
|
|
IconCollectionRestHandler
|
|
"""
|
|
class IconCollectionRestHandler(rest.BaseRestHandler):
|
|
|
|
'''
|
|
Class for handling icon objects.
|
|
'''
|
|
@api()
|
|
def __init__(self, *args, **kwargs):
|
|
'''
|
|
Initialize IconCollectionRestHandler and rest.BaseRestHandler
|
|
Initialize context instance.
|
|
Set default response headers.
|
|
:param args: from BaseRestHandlers
|
|
:param kwargs: from BaseRestHandlers
|
|
'''
|
|
rest.BaseRestHandler.__init__(self, *args, **kwargs)
|
|
self.context = ContextUtil.get_context(request=self.request,
|
|
sessionKey=self.sessionKey,
|
|
pathParts=self.pathParts)
|
|
|
|
self.response.setHeader('Content-Type', 'application/json')
|
|
|
|
@api_operation('get', 'Returns all/matched icons.', 'get_all')
|
|
@api_query_param(['sort_key', 'sort_dir', 'limit', 'offset', 'fields', 'filter', 'shared'])
|
|
@api_response(200, 'IconCollection', True)
|
|
@api_operation('get', 'Returns icon object by id')
|
|
@api_path_param()
|
|
@api_response(200, 'IconCollection')
|
|
def handle_GET(self):
|
|
'''
|
|
If id present, it returns the specific icon by id, else it returns multiple icons list.
|
|
:param id: (optional) returns icon by id.
|
|
:type: ``basestring``
|
|
:param id: (optional) list_categories.
|
|
:type: ``bool`` if set, returns a list of distinct categories
|
|
:params query: (optional) dict for icons search (i.e. sort, limit etc..)
|
|
:return: writes to response object.
|
|
'''
|
|
|
|
spec = self.context['query'].get('spec')
|
|
list_categories = self.context['query'].get('list_categories')
|
|
category = self.context['query'].get('category')
|
|
|
|
if spec: # TODO: figure out how this works
|
|
response = str(api_get_spec(self.context, ['GET', 'PUT', 'POST', 'DELETE']))
|
|
|
|
elif list_categories:
|
|
response = self._get_svc().get_categories()
|
|
|
|
elif category:
|
|
response = self._get_svc().get_category_icons(self.context['query'])
|
|
|
|
else:
|
|
if not self.context['id']:
|
|
response = self._get_svc().get_all(self.context['query'])
|
|
else:
|
|
response = self._get_svc().get(self.context['id'])
|
|
|
|
self.response.write(str(response))
|
|
|
|
@api_operation('put', 'Creates a new icon', 'create')
|
|
@api_body_param(True, 'IconCollection')
|
|
@api_response(200, 'IconCollection')
|
|
def handle_PUT(self):
|
|
'''
|
|
Creates the Icon record in KV.
|
|
:param: dict payload icon object.
|
|
:type: ``dict``
|
|
:return: id of created icon
|
|
:rtype: ``basestring``
|
|
'''
|
|
response = self._get_svc().create(self.context['payload'])
|
|
self.response.write(str(json.dumps(response)))
|
|
|
|
@api_operation('post', 'Updates icons by id or in bulk', 'update')
|
|
@api_body_param(True, 'IconCollection')
|
|
@api_path_param()
|
|
@api_response(200, 'IconCollection')
|
|
def handle_POST(self):
|
|
'''
|
|
Updates icon object by id.
|
|
:param: id of icon from path.
|
|
:param: payload of updated icon object.
|
|
:param: category selects icons by category
|
|
:param: new_category sets new category for those icons
|
|
:return: id of saved icon
|
|
:rtype: ``basestring``
|
|
'''
|
|
response = ''
|
|
if self.context['id']:
|
|
response = self._get_svc().update(self.context['id'], self.context['payload'])
|
|
elif self.context['payload']:
|
|
payload = self.context['payload']
|
|
category = payload.get('category')
|
|
new_category = payload.get('new_category')
|
|
if category and new_category:
|
|
response = self._get_svc().bulk_update_category(category, new_category)
|
|
|
|
self.response.write(str(response))
|
|
|
|
@api_operation('delete', 'Deletes the icon object by id')
|
|
@api_path_param()
|
|
@api_response(200)
|
|
def handle_DELETE(self):
|
|
'''
|
|
Deletes the icon by id or in bulk
|
|
:param category: if specified, deletes all icons under this category
|
|
:return: true if deleted
|
|
:rtype: ``basestring``
|
|
'''
|
|
category = self.context['query'].get('category')
|
|
filter = self.context['query'].get('filter')
|
|
if category:
|
|
category = category.strip()
|
|
self._get_svc().bulk_delete_category(category)
|
|
elif filter:
|
|
self._get_svc().bulk_delete(filter)
|
|
else:
|
|
self._get_svc().delete(self.context['id'])
|
|
res = {"Deleted": "True"}
|
|
self.response.write(str(json.dumps(res)))
|
|
|
|
def _get_svc(self):
|
|
'''
|
|
Creates the IconService object
|
|
:return: IconService()
|
|
'''
|
|
return IconService(self.context['app'], self.context['session'],
|
|
self.context['user'], self.context['collection'])
|
|
|
|
|
|
class ContextUtil(object):
|
|
|
|
def __init__(self):
|
|
pass
|
|
|
|
@staticmethod
|
|
def get_context(**kwargs):
|
|
|
|
'''
|
|
Creates and returns the dict for all the params needed for class
|
|
:param kwargs: dict of request, sessionKey and pathParts from baseRestHandler.
|
|
:type: ``dict``
|
|
:return: Context dict with all the params needed.
|
|
:rtype: ``dict``
|
|
'''
|
|
|
|
request = kwargs.get('request', None)
|
|
session = kwargs.get('sessionKey', None)
|
|
path = kwargs.get('pathParts', None)
|
|
|
|
if not request:
|
|
raise ArgValidationException(400, "Request is empty")
|
|
|
|
"""
|
|
# API --> services/app/version/api/id/action
|
|
# Required --> services, version, app, api
|
|
# Optional --> id, action
|
|
"""
|
|
path_keys = ['services', 'app', 'version', 'api', 'id', 'action']
|
|
path_params = dict(list(zip(path_keys, path)))
|
|
|
|
context = dict()
|
|
context['request'] = request
|
|
context['user'] = request['userName']
|
|
context['session'] = session
|
|
context['app'] = path_params.get('app')
|
|
context['api'] = path_params.get('api')
|
|
context['collection'] = context['app'] + '_' + context['api']
|
|
if request['payload']:
|
|
context['payload'] = json.loads(request['payload'])
|
|
else:
|
|
context['payload'] = None
|
|
context['id'] = path_params.get('id')
|
|
context['action'] = path_params.get('action')
|
|
context['version'] = path_params.get('version')
|
|
context['query'] = request['query']
|
|
context['headers'] = request['headers']
|
|
|
|
return context
|
|
|
|
|
|
class BaseService(object):
|
|
def __init__(self, app_name, session_id, user_name, collection_name):
|
|
self.session_id = session_id
|
|
self.app_name = app_name
|
|
self.user_name = user_name
|
|
self.collection_name = collection_name
|
|
self.options = {'collection': collection_name, 'app': app_name, 'owner': 'nobody'}
|
|
|
|
|
|
class IconService(BaseService):
|
|
|
|
"""
|
|
init params
|
|
"""
|
|
def __init__(self, *args, **kwargs):
|
|
|
|
'''
|
|
Creates kv store instance.
|
|
:param args:
|
|
:param kwargs:
|
|
'''
|
|
BaseService.__init__(self, *args, **kwargs)
|
|
self.kv_client = KvStoreHandler(self.collection_name, self.session_id, self.app_name)
|
|
|
|
def get_count(self, query_params=dict):
|
|
'''
|
|
Returns total number of matched icons.
|
|
:param query_params: params for sorting, pagination and filtering
|
|
:type: ``dict``
|
|
:return: number of matched icons
|
|
:rtype: ``int``
|
|
'''
|
|
filter = query_params.get('filter', None)
|
|
|
|
args = {'fields': '_key'}
|
|
|
|
if filter:
|
|
fg = FilterGenerator(filter)
|
|
filter_perm = fg.generate_kvstore_filter()
|
|
args['query'] = filter_perm
|
|
|
|
keys = self.kv_client.adv_query(args)
|
|
if keys: return len(keys)
|
|
|
|
return 0
|
|
|
|
def get_all(self, query_params=dict):
|
|
'''
|
|
Returns matched icons.
|
|
:param query_params: params for sorting, pagination and filtering
|
|
:type: ``dict``
|
|
:return: matched Icon objects
|
|
:rtype: ``basestring``
|
|
'''
|
|
|
|
count = self.get_count(query_params)
|
|
|
|
sort_key = query_params.get('sort_key', 'title')
|
|
sort_dir = query_params.get('sort_dir', 'asc')
|
|
limit = query_params.get('limit', 0)
|
|
|
|
offset = query_params.get('offset', 0)
|
|
fields = query_params.get('fields', None)
|
|
filter = query_params.get('filter', None)
|
|
shared = query_params.get('shared', None)
|
|
|
|
if sort_dir == "asc":
|
|
sort_dir = 1
|
|
else:
|
|
sort_dir = -1 # Default to descending
|
|
|
|
args = dict()
|
|
args['sort'] = sort_key + ":" + str(sort_dir)
|
|
args['limit'] = limit
|
|
args['skip'] = int(offset) * int(limit)
|
|
|
|
if fields:
|
|
args['fields'] = fields
|
|
|
|
if shared:
|
|
args["shared"] = shared
|
|
if filter:
|
|
fg = FilterGenerator(filter)
|
|
filter_perm = fg.generate_kvstore_filter()
|
|
|
|
args['query'] = filter_perm
|
|
|
|
content = self.kv_client.adv_query(args)
|
|
return '{"total": ' + str(count) + ', "result": ' + json.dumps(content) + '}'
|
|
|
|
"""
|
|
Get a single icon by id
|
|
"""
|
|
def get(self, id):
|
|
'''
|
|
Returns icon object by id (It will also migrate the old icon objects to current object)
|
|
:param id: id of the icon
|
|
:type: ``basestring``
|
|
:return: icon object from KV store
|
|
:rtype: ``basestring``
|
|
'''
|
|
IconService._validate_id(id)
|
|
content = self.kv_client.get(id)
|
|
|
|
response = json.loads(content)
|
|
|
|
return content
|
|
|
|
def get_categories(self):
|
|
'''
|
|
Fetch all entries with category set, then pick distinct values
|
|
'''
|
|
response = self.get_all({'fields': 'category,immutable'})
|
|
icons = json.loads(response)['result']
|
|
categories = []
|
|
seen = set()
|
|
for icon in icons:
|
|
category = icon.get('category')
|
|
if category is not None and not category in seen:
|
|
categories.append({
|
|
"name": category,
|
|
"immutable": 1 if 'immutable' in icon and icon['immutable'] == 1 else 0
|
|
})
|
|
seen.add(category)
|
|
return json.dumps(sorted(categories, key=lambda k: k['name']))
|
|
|
|
|
|
def get_category_icons(self, query_params):
|
|
'''
|
|
Fetch all entries within provided category
|
|
'''
|
|
if not 'category' in query_params:
|
|
return None
|
|
|
|
category = query_params['category'].strip()
|
|
filter = [{
|
|
"rule_condition": "AND",
|
|
"rule_items": [
|
|
{"value": category, "rule_type": "matches", "field": "category", "field_type": "title"}
|
|
]
|
|
}]
|
|
query_params.pop('category')
|
|
query_params['filter'] = filter
|
|
return self.get_all(query_params)
|
|
|
|
def validate_category_name(self, category):
|
|
'''
|
|
Returns false if category name is unacceptable
|
|
'''
|
|
invalid_chars = "~`!#$%\^&*+=\[\]\';,/{}|\":<>\?]#"
|
|
return not any(char in invalid_chars for char in category)
|
|
|
|
"""
|
|
Create a new icon
|
|
"""
|
|
def create(self, data):
|
|
'''
|
|
Validates the Icon against latest icon model object and creates a new icon
|
|
:param data: icon object
|
|
:type: ``dict``
|
|
:return: Saved icon id
|
|
:rtype: ``basestring``
|
|
'''
|
|
|
|
try:
|
|
content = ''
|
|
if isinstance(data, list):
|
|
self._validate_same_name_icon(data)
|
|
|
|
# bulk remove icons marked for removal
|
|
removal_list = []
|
|
save_list = []
|
|
skipped_categories = set()
|
|
|
|
if len(data) == 0:
|
|
return
|
|
|
|
for record in data:
|
|
if '__to_remove' in record:
|
|
removal_list.append(record)
|
|
else:
|
|
record['_owner'] = self.user_name
|
|
category = record.get('category')
|
|
if not category or len(category)==0:
|
|
continue
|
|
if not self.validate_category_name(category):
|
|
skipped_categories.add(category)
|
|
continue
|
|
save_list.append(record)
|
|
|
|
if len(skipped_categories) > 0:
|
|
logger.error('Skipped categories with names containing special characters: %s.' % ', '.join(skipped_categories))
|
|
raise IconException(400, "Special characters are not allowed in category names.")
|
|
|
|
if len(removal_list) > 0:
|
|
q = {"$or":[{"_key":record['_key']} for record in removal_list]}
|
|
q = json.dumps(q)
|
|
self.kv_client.bulk_delete(q)
|
|
|
|
logger.info('user:"%s" app:"%s" action:"%s" icons:"%s".' %
|
|
(
|
|
self.user_name,
|
|
self.app_name,
|
|
'removed',
|
|
','.join(["%s/%s"%(record['category'],record['title']) for record in removal_list])
|
|
)
|
|
)
|
|
|
|
if len(save_list) > 0:
|
|
content = self.kv_client.batch_create(save_list)
|
|
|
|
logger.info('user:"%s" app:"%s" action:"%s" icons:"%s".' %
|
|
(
|
|
self.user_name,
|
|
self.app_name,
|
|
'added',
|
|
','.join(["%s/%s"%(record['category'],record['title']) for record in save_list])
|
|
)
|
|
)
|
|
|
|
else:
|
|
raise IconException(400, "Payload is expected to be a list.")
|
|
return content
|
|
|
|
except IconException as ge:
|
|
logger.error("IconException: {}".format(ge))
|
|
raise ge
|
|
except Exception as e:
|
|
logger.error("Exception: {}".format(e))
|
|
if e.status == 403:
|
|
raise InsufficientPermissionsException(403, "Insufficient permissions to update icon collection.")
|
|
|
|
raise e
|
|
|
|
"""
|
|
Update existing icon by id
|
|
"""
|
|
def update(self, id, data):
|
|
'''
|
|
Validates the Icon against latest icon model object and updates a icon by id
|
|
:param id: icon id
|
|
:type: ``basestring``
|
|
:param data: icon object
|
|
:type: ``dict``
|
|
:return: Saved icon id
|
|
:rtype: ``basestring``
|
|
'''
|
|
IconService._validate_id(id)
|
|
|
|
get_response = self.get(id)
|
|
if not get_response:
|
|
raise IconException(404, "Icon not found.")
|
|
|
|
res_data = json.loads(get_response)
|
|
|
|
for k, v in data.items():
|
|
if v is not None:
|
|
res_data[k] = v
|
|
|
|
response = self.kv_client.single_update(id, res_data, True)
|
|
logger.info('user:"%s" app:"%s" action:"%s" id:"%s".' %
|
|
(
|
|
self.user_name,
|
|
self.app_name,
|
|
'updated',
|
|
id
|
|
)
|
|
)
|
|
return response
|
|
|
|
|
|
"""
|
|
Issue a simple KV store record deletion by category name
|
|
"""
|
|
def bulk_update_category(self, category, new_category):
|
|
if not self.validate_category_name(new_category):
|
|
raise IconException(400, "Category name cannot contain special characters.")
|
|
|
|
filter = [{
|
|
"rule_condition": "AND",
|
|
"rule_items": [
|
|
{"value": category, "rule_type": "matches", "field": "title", "field_type": "title"}
|
|
]
|
|
}]
|
|
|
|
fg = FilterGenerator(filter)
|
|
results = self.kv_client.adv_query({
|
|
'fields': ['title', '_key'],
|
|
'query': fg.generate_kvstore_filter()
|
|
})
|
|
|
|
updated_result = []
|
|
|
|
for i, res in enumerate(results):
|
|
results[i]['title'] = new_category
|
|
|
|
response = self.kv_client.batch_update(*results)
|
|
logger.info('user:"%s" app:"%s" action:"%s" category_old:"%s" category_new:"%s".' %
|
|
(
|
|
self.user_name,
|
|
self.app_name,
|
|
'renamed',
|
|
category,
|
|
new_category
|
|
)
|
|
)
|
|
return response
|
|
|
|
|
|
"""
|
|
Delete existing icon by id
|
|
"""
|
|
def delete(self, id):
|
|
'''
|
|
Deletes the icon object by id in KV
|
|
:param id: icon id
|
|
:type: ``basestring``
|
|
'''
|
|
IconService._validate_id(id)
|
|
response = self.kv_client.delete(id)
|
|
logger.info('user:"%s" app:"%s" action:"%s" id:"%s".' %
|
|
(
|
|
self.user_name,
|
|
self.app_name,
|
|
'deleted',
|
|
id
|
|
)
|
|
)
|
|
return response
|
|
|
|
"""
|
|
Delete multiple icons by query
|
|
"""
|
|
def bulk_delete(self, filter):
|
|
'''
|
|
Deletes icon objects by query in KV
|
|
:param filter: filtering query
|
|
:type: ``basestring``
|
|
'''
|
|
q = json.dumps(filter)
|
|
response = self.kv_client.bulk_delete(filter)
|
|
logger.info('user:"%s" app:"%s" action:"%s" filter:"%s".' %
|
|
(
|
|
self.user_name,
|
|
self.app_name,
|
|
'bulk_deleted',
|
|
str(filter)
|
|
)
|
|
)
|
|
return response
|
|
|
|
"""
|
|
Delete multiple icons by category
|
|
"""
|
|
def bulk_delete_category(self, category):
|
|
'''
|
|
Deletes icon objects by query in KV
|
|
:param filter: filtering query
|
|
:type: ``basestring``
|
|
'''
|
|
filter = [{
|
|
"rule_condition": "AND",
|
|
"rule_items": [
|
|
{"value": category, "rule_type": "matches", "field": "category", "field_type": "title"}
|
|
]
|
|
}]
|
|
fg = FilterGenerator(filter)
|
|
q = fg.generate_kvstore_filter()
|
|
q = json.dumps(q)
|
|
|
|
response = self.kv_client.bulk_delete(q)
|
|
logger.info('user:"%s" app:"%s" action:"%s" category:"%s".' %
|
|
(
|
|
self.user_name,
|
|
self.app_name,
|
|
'bulk_deleted_category',
|
|
str(category)
|
|
)
|
|
)
|
|
return response
|
|
|
|
|
|
@staticmethod
|
|
def _validate_id(_id):
|
|
'''
|
|
Validates for valid id string.
|
|
:param _id: icon id
|
|
:type: ``basestring``
|
|
:raises: ArgValidationException if missing or not a basestring
|
|
'''
|
|
if any([not isinstance(_id, string_type), (isinstance(_id, string_type) and not _id.strip())]):
|
|
raise ArgValidationException(400, 'Id is missing.')
|
|
|
|
def _validate_same_name_icon(self, data):
|
|
args = dict()
|
|
|
|
# First check for duplicates in incoming data:
|
|
seen = []
|
|
dups = set()
|
|
category = None
|
|
title = None
|
|
filter = [{
|
|
"rule_condition": "AND",
|
|
"rule_items": [
|
|
{"value": category, "rule_type": "matches", "field": "category", "field_type": "title"},
|
|
{"value": title, "rule_type": "matches", "field": "title", "field_type": "title"}
|
|
]
|
|
}]
|
|
for icon in data:
|
|
category = icon.get('category')
|
|
title = icon.get('title')
|
|
if not category or not title:
|
|
continue
|
|
filter[0]["rule_items"][0]["value"] = category
|
|
filter[0]["rule_items"][1]["value"] = title
|
|
fg = FilterGenerator(filter)
|
|
filter_perm = fg.generate_kvstore_filter()
|
|
args['query'] = filter_perm
|
|
content = self.kv_client.adv_query(args)
|
|
if content:
|
|
dups.add(title)
|
|
|
|
if '__to_remove' in icon: # don't count icons marked for deletion
|
|
continue
|
|
t = icon['title']
|
|
if t in seen:
|
|
dups.add(t)
|
|
else:
|
|
seen.append(t)
|
|
|
|
if len(dups) > 0:
|
|
dups = ','.join(dups)
|
|
raise AlreadyExistsException(400, "Trying to add icons with the same name: {}".format(dups))
|
|
|
|
|
|
|
|
class KvStoreHandler(object):
|
|
def __init__(self, collection_name, session_key, app, owner='nobody', **context):
|
|
self._collection_data = self._get_collection_data(collection_name,
|
|
session_key, app, owner, **context)
|
|
|
|
def _get_collection_data(self, collection_name, session_key, app, owner, **context):
|
|
'''
|
|
Returns collection instance
|
|
:param collection_name: collection name
|
|
:param session_key: session key
|
|
:param app: app name
|
|
:param owner: owner name
|
|
:param context: extra params
|
|
:return: collection instance
|
|
'''
|
|
kvstore = rest_client.SplunkRestClient(session_key,
|
|
app,
|
|
owner=owner,
|
|
**context).kvstore
|
|
|
|
try:
|
|
kvstore.get(name=collection_name)
|
|
except binding.HTTPError as e:
|
|
raise KVNotExists(404, 'Collection not exists')
|
|
|
|
collections = kvstore.list(search=collection_name)
|
|
for collection in collections:
|
|
if collection.name == collection_name:
|
|
return collection.data
|
|
else:
|
|
raise KVNotExists(404, 'Collection not exists')
|
|
|
|
def create(self, record, record_id, include_ts=True):
|
|
'''
|
|
Creates the object in KV
|
|
:param record: object
|
|
:param record_id: kv id
|
|
:param include_ts: boolean to add _time with record
|
|
:return: saved object id
|
|
'''
|
|
if record_id:
|
|
record['_key'] = record_id
|
|
|
|
if include_ts:
|
|
record['_time'] = time.time()
|
|
|
|
ret = self._collection_data.insert(json.dumps(record))
|
|
return json.dumps(ret)
|
|
|
|
def get(self, key):
|
|
'''Issue a simple KV store query by key. If key is empty, all records
|
|
will be returned.'''
|
|
|
|
if key is None:
|
|
key = ''
|
|
record = self._collection_data.query_by_id(key)
|
|
return json.dumps(record)
|
|
|
|
def delete(self, key):
|
|
'''Issue a simple KV store record deletion by key,
|
|
<tt>if key is not None and len(key) > 0</tt>.'''
|
|
|
|
if key and isinstance(key, string_type):
|
|
self._collection_data.delete_by_id(key)
|
|
|
|
return
|
|
|
|
def bulk_delete(self, q):
|
|
'''
|
|
Issue a simple KV store record deletion by category name
|
|
'''
|
|
return self._collection_data.delete(q)
|
|
|
|
def query(self, q):
|
|
# q = urllib2.quote(json.dumps(json_query))
|
|
return self._collection_data.query(**q)
|
|
|
|
def adv_query(self, getargs):
|
|
'''Issue a MORE complex KV store query. The query string is constructed
|
|
from a valid JSON object. Additional parameters such as "limit" can be
|
|
included in the query_options dictionary.
|
|
|
|
The allowable_params are: 'fields', 'limit', 'skip', 'sort', 'query'
|
|
'''
|
|
|
|
options = {}
|
|
|
|
for k, v in getargs.items():
|
|
if k == 'query':
|
|
options['query'] = json.dumps(v)
|
|
elif k == 'fields':
|
|
if isinstance(v, string_type):
|
|
options['fields'] = v
|
|
elif isinstance(v, list):
|
|
options['fields'] = ','.join(v)
|
|
else:
|
|
raise ArgValidationException(400, 'Invalid value for fields parameter in KV store query.')
|
|
elif k in ['limit', 'skip']:
|
|
# May raise ValueError
|
|
options[k] = str(int(v))
|
|
elif k == 'sort':
|
|
# Since sort order can be a bit complex, we just expect the
|
|
# consumer to construct their own sort string here.
|
|
if isinstance(v, string_type):
|
|
options['sort'] = v
|
|
else:
|
|
raise ArgValidationException(400, 'Invalid value for sort parameter in KV store query.')
|
|
else:
|
|
# Invalid parameter is ignored.
|
|
pass
|
|
|
|
# params = urllib.urlencode(options)
|
|
# logger.debug("params:: {}".format(params))
|
|
return self.query(options)
|
|
|
|
def single_update(self, id, record, include_ts=False):
|
|
# Caller is responsible for ensuring that the input IS NOT an array.
|
|
if include_ts:
|
|
record['_time'] = time.time()
|
|
|
|
ret = self._collection_data.update(id, json.dumps(record))
|
|
return json.dumps(ret)
|
|
|
|
def batch_update(self, records, include_ts=True):
|
|
for record in records:
|
|
if include_ts:
|
|
record['_time'] = time.time()
|
|
|
|
return self._collection_data.batch_save(*records)
|
|
|
|
def batch_create(self, records, include_ts=True):
|
|
for record in records:
|
|
if include_ts:
|
|
record['_time'] = time.time()
|
|
|
|
return self._collection_data.batch_save(*records)
|
|
|
|
|
|
class FilterGenerator(object):
|
|
"""
|
|
The filter does a couple separate things. First, it takes in a json filter specified
|
|
by the UI, which can be a combination of AND's OR's and NOT operations along
|
|
with items that may or may not be wildcarded according to the splunk wildcard specifications
|
|
(e.g. *str, str*, *str* s*tr).
|
|
"""
|
|
def __init__(self, source_json=None):
|
|
"""
|
|
Construct an itsi filter object
|
|
@param source_json: A parsed json object, dict or list
|
|
@type source_json: iterable (list,dict)
|
|
"""
|
|
self.kvstore_filter = None
|
|
if isinstance(source_json, string_type):
|
|
# We will need to extract the json, if they didn't read the documentation above
|
|
self.source = json.loads(source_json)
|
|
elif isinstance(source_json, dict) or isinstance(source_json, list):
|
|
# We're probably dealing with the right parameters here
|
|
self.source = source_json
|
|
elif source_json is None:
|
|
self.source = []
|
|
else:
|
|
raise ArgValidationException(400, "Source data could not be recognized as a string or parsed json. Data passed in: " + str(source_json))
|
|
|
|
def _generate_filter_expression(self, source):
|
|
"""
|
|
Generate the root filter expression given a source expression
|
|
There are three parameters
|
|
@param source: The source expression in a json format - defined in ITOA-2287
|
|
@type source: A dict
|
|
"""
|
|
log_prefix = "[generate_filter_expression] "
|
|
illegal_characters = ['=', '$', '^']
|
|
if not isinstance(source, dict):
|
|
message = "Expected a dictionary for the filter expression, got something else."
|
|
logger.error(log_prefix + message)
|
|
raise ArgValidationException(400, message)
|
|
rule_type = source.get('rule_type', '').lower()
|
|
field = source.get('field', None)
|
|
|
|
field_type = source.get('field_type')
|
|
|
|
if not any(field_type == allowed_type for allowed_type in ['alias', 'info', 'title']):
|
|
message = "Unexpected value='{0}' specified for field type, with type='{1}'.".format(field_type,
|
|
type(field_type))
|
|
logger.error(log_prefix + message)
|
|
raise ArgValidationException(400, message)
|
|
|
|
# Generate filter to identify presence of field in the respective field type
|
|
field_type_filter = {} # do not filter fields by default
|
|
if field_type == 'alias':
|
|
field_type_filter = {'identifier.fields': field}
|
|
elif field_type == 'info':
|
|
field_type_filter = {'informational.fields': field}
|
|
|
|
value = source.get('value', None)
|
|
if value is None or '':
|
|
message = "Expected value definition in the JSON {}".format(source)
|
|
logger.error(log_prefix + message)
|
|
raise ArgValidationException(400, message)
|
|
|
|
# For each value specified, construct the required filter
|
|
split_values = value.split(',')
|
|
values_to_regex = []
|
|
for split_value in split_values:
|
|
split_value = split_value.replace("\\", "\\\\");
|
|
for i in illegal_characters:
|
|
if i in split_value:
|
|
message = "Illegal character %s in value %s." % (i, split_value)
|
|
logger.error(log_prefix + message)
|
|
raise ArgValidationException(400, message)
|
|
|
|
# All done with validation, now build the filter
|
|
if split_value.find('*') != -1: # regex value identified
|
|
split_value = split_value.replace('*', '.*?')
|
|
if rule_type == 'not':
|
|
# Adjust regex to be an exclusion
|
|
split_value = '(?!' + split_value + ').*'
|
|
kv_filter = {field: {'$regex': '^' + split_value + '$', '$options': 'i'}}
|
|
elif rule_type == 'not':
|
|
# Construct exclusion filter
|
|
# Since the only way to perform case insensitive string compare is using regex,
|
|
# construct a regex for the single value lookup
|
|
# Regex cannot be used for empty value exclusion, so special handle it
|
|
if len(split_value) == 0:
|
|
kv_filter = {field: {"$ne": split_value}}
|
|
else:
|
|
# Since the only way to perform case insensitive string compare is using regex,
|
|
# construct a regex for the single value lookup
|
|
kv_filter = {field: {'$regex': '^(?!' + split_value + ').*$', '$options': 'i'}}
|
|
else:
|
|
kv_filter = {field: {'$regex': '^' + split_value + '$', '$options': 'i'}}
|
|
values_to_regex.append(kv_filter)
|
|
|
|
return {'$and': [field_type_filter, {'$or': values_to_regex}]}
|
|
|
|
def generate_kvstore_filter(self, regenerate=False):
|
|
"""
|
|
Generates the kvstore_filter from the source json
|
|
@param regenerate: Force a regeneration of the kvstore_filter, used more in testing
|
|
@type regenerate: Boolean
|
|
"""
|
|
if self.kvstore_filter is not None and regenerate is False:
|
|
return self.kvstore_filter
|
|
"""
|
|
We plan to currently support only one level of nesting as follows:
|
|
> All rule items are ORed at the top level.
|
|
> Only one level of Nesting is supported and all rule items in the nested level will be ANDed
|
|
> Sample:
|
|
key1=value1,value1.1 AND key=value2
|
|
OR
|
|
key3=value3 AND key4=value4
|
|
We will need to change the json formatting if we're expecting more nesting or combination of AND and OR
|
|
"""
|
|
if isinstance(self.source,list):
|
|
or_expressions = []
|
|
# Process the top level OR terms
|
|
for rule_group in self.source:
|
|
or_term = rule_group.get('rule_items')
|
|
and_expressions = []
|
|
# Process the first level nested AND terms
|
|
if isinstance(or_term,list):
|
|
for and_term in or_term:
|
|
leaf = self._generate_filter_expression(and_term)
|
|
and_expressions.append(leaf)
|
|
or_expressions.append({"$and":and_expressions})
|
|
generated_filter = {"$or": or_expressions }
|
|
else:
|
|
raise ArgValidationException(400, "source filter must be a list")
|
|
|
|
self.kvstore_filter = generated_filter
|
|
return self.kvstore_filter
|
|
|
|
|
|
class IconException(splunk.RESTException):
|
|
def __init__(self, status_code, msg):
|
|
splunk.RESTException.__init__(self, status_code, msg)
|
|
|
|
|
|
class ArgValidationException(IconException):
|
|
pass
|
|
|
|
|
|
class InsufficientPermissionsException(IconException):
|
|
pass
|
|
|
|
|
|
class AlreadyExistsException(IconException):
|
|
pass
|
|
|
|
|
|
class KVNotExists(IconException):
|
|
pass
|