You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
413 lines
20 KiB
413 lines
20 KiB
# Copyright (C) 2005-2024 Splunk Inc. All Rights Reserved.
|
|
|
|
|
|
import json
|
|
from ITOA.setup_logging import logger as itsi_logger
|
|
from . import itsi_module_common as utils
|
|
from .itsi_module_kpi_base_search import ItsiModuleKpiBaseSearch
|
|
from functools import reduce
|
|
|
|
|
|
class ItsiModuleKpiGroup(object):
|
|
"""
|
|
Class to create, update, get, getCount, delete KpiGroup
|
|
"""
|
|
|
|
"""
|
|
Class variables
|
|
"""
|
|
_base_url = '/servicesNS/nobody/%s/configs/conf-itsi_kpi_template'
|
|
_base_args = '?output_mode=json&count=-1'
|
|
ACCEPTED_KEYS = ['title', 'description', '_owner', 'source_itsi_da', 'kpis']
|
|
CONF_NAME = 'itsi_kpi_template'
|
|
|
|
_validation_error_messages = {
|
|
'id_prefix_mismatch': 'KPI template ID needs to be prefixed with module ID.',
|
|
'title_required': 'KPI template requires a title.',
|
|
'description_required': 'KPI template requires a description.',
|
|
'atleast_1_kpi': 'KPI template requires at least one KPI.',
|
|
'kpi_id_prefix_mismatch': 'ID for KPI {} needs to be prefixed with module ID.',
|
|
'kpi_base_search_not_exported': 'KPI {} refers to base search ID {} that is not part of an exported KPI base search.',
|
|
'kpi_base_search_metric_not_exported': 'KPI {} refers to base search ID {} and base search metric {} that is not part of an exported KPI base search.'
|
|
}
|
|
|
|
def __init__(self, session_key, app='SA-ITOA', owner='nobody', logger=None):
|
|
"""
|
|
Initializes the ItsiModuleKpiGroup object
|
|
|
|
@type session_key: string
|
|
@param session_key: the session key
|
|
|
|
@type app: string
|
|
@param app: the app context, defaults to SA-ITOA
|
|
|
|
@type owner: string
|
|
@param owner: the owner context, defaults to nobody
|
|
|
|
@type logger: string
|
|
@param logger: the logger
|
|
"""
|
|
self.session_key = session_key
|
|
self.owner = owner
|
|
self.app = app
|
|
self.object_type = 'kpi_group'
|
|
self.logger = logger if logger else itsi_logger
|
|
|
|
def create(self, itsi_module, data, **kwargs):
|
|
"""
|
|
Create itsi_kpi_template conf file
|
|
|
|
If itsi_object_id is given, then this endpoint would also require a list
|
|
of kpi titles to be grouped.
|
|
If itsi_object_id is not given, then emit the payload directly to conf file
|
|
|
|
@type itsi_module: string
|
|
@param itsi_module: ITSI module name
|
|
|
|
@type data: dict
|
|
@param data: key/value pairs to be written to conf file
|
|
|
|
@rtype: string
|
|
@return: newly created id (conf stanza name) or raise an exception
|
|
"""
|
|
kpi_group_id = data.get('itsi_object_id')
|
|
# If kpi_group_id is not given then construct the id
|
|
if not kpi_group_id:
|
|
kpi_template_title = data.get('title')
|
|
# Title should not be empty
|
|
if kpi_template_title:
|
|
kwargs = {'suffix': 'KPIs'}
|
|
kpi_group_id = utils.make_stanza_name(itsi_module, kpi_template_title, **kwargs)
|
|
else:
|
|
self.logger.error('KPI group title missing from the payload')
|
|
raise utils.ItsiModuleError(status=400, message='KPI group title is missing from the payload.')
|
|
|
|
# If include_kpi_base_search is set to true, then export itsi_kpi_base_search.conf based on base_search_id_list
|
|
if data.get('include_kpi_base_search') is True:
|
|
# Get base_search_id useds in all kpis within this kpi group and store in a list
|
|
# No duplicated base_search_id allowed
|
|
base_search_id_set = set([kpi.get('base_search_id') for kpi in data.get('kpis') if kpi.get('search_type') == 'shared_base'])
|
|
base_search_id_list = list(base_search_id_set)
|
|
if not base_search_id_list:
|
|
self.logger.info('No KPI base search id found in current KPI group')
|
|
else:
|
|
kpi_base_search_id_mapping = {}
|
|
kpi_base_search_id_metrics_mapping = {}
|
|
self.logger.debug('Creating templatized base searches for KPI group %s: %s', kpi_group_id, base_search_id_list)
|
|
base_search_instance = ItsiModuleKpiBaseSearch(self.session_key)
|
|
# Create a conf stanza for each base search id
|
|
for base_search_id in base_search_id_list:
|
|
base_search_new_id_with_metrics_mapping = base_search_instance.create(itsi_module, {'itsi_object_id': base_search_id})
|
|
# Set mapping between old base search id and newly returned base search id
|
|
kpi_base_search_id_mapping[base_search_id] = list(base_search_new_id_with_metrics_mapping.keys())[0]
|
|
# Set mapping between newly returned base search id, its metrics and old key and new key mapping for each metric
|
|
kpi_base_search_id_metrics_mapping.update(base_search_new_id_with_metrics_mapping)
|
|
|
|
# Update kpis with the new base_search id ad new metric key
|
|
self.logger.debug('Updating KPIs with the following base search id mappings: %s', json.dumps(kpi_base_search_id_mapping))
|
|
self.logger.debug('Updating KPIs with the following base search metric mappings: %s', json.dumps(kpi_base_search_id_metrics_mapping))
|
|
data['kpis'] = self._update_kpis(itsi_module, data.get('kpis'), kpi_base_search_id_mapping, kpi_base_search_id_metrics_mapping)
|
|
|
|
# Set kpi_template_kpi_id
|
|
data['kpis'] = self._set_kpi_template_kpi_id(itsi_module, data.get('kpis'))
|
|
# Fetch a list of kpi_template_kpi_id
|
|
kpi_template_kpi_id_list = [kpi.get('kpi_template_kpi_id') for kpi in data.get('kpis')]
|
|
|
|
# Filter out non-accepted keys
|
|
data_to_post = utils.filter_keys_reformat_certain_values_from_payload(itsi_module, self.ACCEPTED_KEYS, data)
|
|
|
|
# Set _owner to nobody and source_itsi_da to itsi_module if they are not given
|
|
if not data.get('_owner'):
|
|
data_to_post['_owner'] = 'nobody'
|
|
if not data.get('source_itsi_da'):
|
|
data_to_post['source_itsi_da'] = itsi_module
|
|
|
|
# Set name and write into conf
|
|
data_to_post['name'] = kpi_group_id
|
|
response, content = utils.create_conf_stanza(self.session_key, self.CONF_NAME, data_to_post, itsi_module)
|
|
if response.status == 200 or response.status == 201:
|
|
response = {kpi_group_id: kpi_template_kpi_id_list}
|
|
self.logger.debug(
|
|
'Created KPI template %s in module %s with following mapping: %s',
|
|
kpi_group_id,
|
|
itsi_module,
|
|
json.dumps(response))
|
|
return response
|
|
else:
|
|
self.logger.error('Error writing KPI template %s in module %s conf', kpi_group_id, itsi_module)
|
|
raise utils.ItsiModuleError(status=400, message='Error writing into conf file %s.' % content)
|
|
|
|
def get(self, itsi_module, object_id, **kwargs):
|
|
"""
|
|
Returns the kpi_group in a given module given an id, or all kpi groups in a module
|
|
|
|
If itsi_module is specified as "-", returns counts of kpi groups for all modules
|
|
|
|
@type itsi_module: string
|
|
@param itsi_module: ITSI module requested
|
|
|
|
@type object_id: string
|
|
@param object_id: ID of KPI group being requested
|
|
"""
|
|
|
|
url = self._base_url
|
|
service_id = kwargs.get('service_id')
|
|
|
|
# Get the endpoint for KPI groups
|
|
kpi_group_endpoint = utils.get_object_endpoint(url, self._base_args, itsi_module, object_id)
|
|
self.logger.debug('Attempting get from KPI group endpoint: %s', kpi_group_endpoint)
|
|
|
|
# Construct the response object based on the request for kpi_template
|
|
response = utils.construct_get_response(kpi_group_endpoint, self.object_type, object_id, self.session_key, ['kpis'])
|
|
|
|
# Add the necessary fields to each KPI in order to ensure proper operation of KPI Thresholding Template
|
|
parsed_response = self._add_aggregate_and_entity_fields_for_thresholding(response)
|
|
|
|
self.logger.debug('Get response for KPI group %s in module %s: %s', object_id, itsi_module, json.dumps(parsed_response))
|
|
if service_id is not None and service_id != '':
|
|
from .itsi_module_service_template import ItsiModuleServiceTemplate
|
|
service_template_instance = ItsiModuleServiceTemplate(self.session_key, self.app, self.owner, self.logger)
|
|
services = service_template_instance.get(itsi_module, None)
|
|
return utils.filter_kpis(parsed_response, services, service_id)
|
|
return parsed_response
|
|
|
|
def get_count(self, itsi_module, **kwargs):
|
|
"""
|
|
Returns the count of kpi_group in a given module
|
|
|
|
If itsi_module is specified as "-", returns counts of kpi groups for all modules
|
|
|
|
@type itsi_module: string
|
|
@param itsi_module: ITSI module requested
|
|
"""
|
|
# Set up the endpoint and make the request
|
|
kpi_group_endpoint = utils.get_object_endpoint(self._base_url, self._base_args, itsi_module, None)
|
|
self.logger.debug('Attempting get_count from KPI group endpoint: %s', kpi_group_endpoint)
|
|
|
|
# Construct the response object based on the request for kpi_template
|
|
response = utils.construct_count_response(kpi_group_endpoint, itsi_module, self.object_type, self.session_key)
|
|
self.logger.debug('Get_count response for KPI group %s in module %s: %s', kpi_group_endpoint, itsi_module, json.dumps(response))
|
|
return response
|
|
|
|
def update(self, itsi_module, object_id, data, **kwargs):
|
|
"""
|
|
Update itsi_kpi_template conf file stanza
|
|
|
|
@type itsi_module: string
|
|
@param itsi_module: ITSI module name
|
|
|
|
@type object_id: string
|
|
@param object_id: object id (stanza name)
|
|
|
|
@type data: dict
|
|
@param data: key/value pairs to be written to conf file
|
|
|
|
@rtype: string
|
|
@return: newly created id (conf stanza name) or raise an exception
|
|
"""
|
|
data_to_post = utils.filter_keys_reformat_certain_values_from_payload(itsi_module, self.ACCEPTED_KEYS, data)
|
|
response, content = utils.update_conf_stanza(self.session_key, self.CONF_NAME, object_id, data_to_post, itsi_module)
|
|
if response.status == 200:
|
|
self.logger.debug('Successfully updated KPI group %s in module %s', itsi_module, object_id)
|
|
return object_id
|
|
else:
|
|
self.logger.error('Failed to update KPI group object id %s', object_id)
|
|
raise utils.ItsiModuleError(status=400, message='Failed updating object id %s: %s.' % (object_id, content))
|
|
|
|
def delete(self, itsi_module, object_id, **kwargs):
|
|
"""
|
|
Delete a specific itsi_kpi_template conf file stanza
|
|
|
|
@type itsi_module: string
|
|
@param itsi_module: ITSI module name
|
|
|
|
@type object_id: string
|
|
@param object_id: object id (stanza name)
|
|
|
|
@rtype: string
|
|
@return: newly created id (conf stanza name) or raise an exception
|
|
"""
|
|
response, content = utils.delete_conf_stanza(self.session_key, self.CONF_NAME, object_id, itsi_module)
|
|
if response.status == 200:
|
|
self.logger.debug('Successfully deleted KPI group %s in module %s', itsi_module, object_id)
|
|
return
|
|
else:
|
|
self.logger.error('Failed deleting KPI group object id: %s', object_id)
|
|
raise utils.ItsiModuleError(status=400, message='Failed deleting object id %s: %s.' % (object_id, content))
|
|
|
|
def validate(self, itsi_module, object_id):
|
|
"""
|
|
Validates the KPI group objects
|
|
|
|
@type itsi_module: string
|
|
@param itsi_module: ITSI module name
|
|
|
|
@type object_id: string
|
|
@param object_id: stanza name (object id)
|
|
|
|
@rtype: dictionary
|
|
@return: dictionary of validation result type to actual contents
|
|
"""
|
|
validation_errors = []
|
|
kpi_groups = self.get(itsi_module, object_id)
|
|
kpi_base_search_object = ItsiModuleKpiBaseSearch(self.session_key)
|
|
kpi_base_searches = kpi_base_search_object.get(itsi_module, None)
|
|
|
|
def get_metric_ids_from_base_search(kpi_base_search):
|
|
return [metric.get('_key') for metric in kpi_base_search.get('content', {}).get('metrics')]
|
|
|
|
def make_kpi_base_search_dictionary(kpi_list, kpi_base_search):
|
|
kpi_list[kpi_base_search.get('id')] = get_metric_ids_from_base_search(kpi_base_search)
|
|
return kpi_list
|
|
|
|
kpi_base_search_id_to_metric = reduce(make_kpi_base_search_dictionary, kpi_base_searches, {})
|
|
|
|
for kpi_group in kpi_groups:
|
|
if not kpi_group.get('id', '').startswith(kpi_group.get('source_itsi_module')):
|
|
validation_errors.append(
|
|
utils.generate_validation_error_line(kpi_group, self._validation_error_messages['id_prefix_mismatch']))
|
|
|
|
if not kpi_group.get('content', {}).get('title'):
|
|
validation_errors.append(
|
|
utils.generate_validation_error_line(kpi_group, self._validation_error_messages['title_required']))
|
|
|
|
if not kpi_group.get('content', {}).get('description'):
|
|
validation_errors.append(
|
|
utils.generate_validation_error_line(kpi_group, self._validation_error_messages['description_required']))
|
|
|
|
if len(kpi_group.get('content', {}).get('kpis')) == 0:
|
|
validation_errors.append(
|
|
utils.generate_validation_error_line(kpi_group, self._validation_error_messages['atleast_1_kpi']))
|
|
|
|
for kpi in kpi_group.get('content', {}).get('kpis'):
|
|
if not kpi.get('kpi_template_kpi_id', '').startswith(kpi_group.get('source_itsi_module')):
|
|
validation_errors.append(
|
|
utils.generate_validation_error_line(
|
|
kpi_group,
|
|
self._validation_error_messages['kpi_id_prefix_mismatch'].format(kpi.get('title'))))
|
|
|
|
if kpi.get('search_type') == 'shared_base':
|
|
if kpi.get('base_search_id') not in kpi_base_search_id_to_metric:
|
|
validation_errors.append(
|
|
utils.generate_validation_error_line(
|
|
kpi_group,
|
|
self._validation_error_messages['kpi_base_search_not_exported']
|
|
.format(kpi.get('title'), kpi.get('base_search_id'))))
|
|
elif kpi.get('base_search_metric') not in kpi_base_search_id_to_metric[kpi.get('base_search_id')]:
|
|
validation_errors.append(
|
|
utils.generate_validation_error_line(
|
|
kpi_group,
|
|
self._validation_error_messages['kpi_base_search_metric_not_exported']
|
|
.format(kpi.get('title'), kpi.get('base_search_id'), kpi.get('base_search_metric'))))
|
|
|
|
self.logger.debug('Found following validation errors for KPI group %s in module %s: %s', object_id, itsi_module, validation_errors)
|
|
return utils.construct_validation_result(errors=validation_errors)
|
|
|
|
def build_module_kpi_mapping(self, itsi_module, **kwargs):
|
|
"""
|
|
Build the mapping between modules, KPI IDs, and KPI objects
|
|
Sample object:
|
|
{
|
|
'DA-ITSI-TEST': {
|
|
'KPI-ID-1': {
|
|
...
|
|
},
|
|
'KPI-ID-2': {
|
|
...
|
|
}
|
|
}
|
|
}
|
|
|
|
@type itsi_module: string
|
|
@param itsi_module: ITSI module that is being requested for KPI groups
|
|
"""
|
|
# Initialize the module-KPI mapping, and then request all modules' KPI groups
|
|
request_module = itsi_module if itsi_module else '-'
|
|
module_kpi_mapping = {}
|
|
response = self.get(itsi_module=request_module, object_id=None)
|
|
|
|
# Build the mapping, and add fields linking back to the parent KPI template ID and title
|
|
for kpi_group in response:
|
|
itsi_module = kpi_group['content']['source_itsi_da']
|
|
if itsi_module not in module_kpi_mapping:
|
|
module_kpi_mapping[itsi_module] = {}
|
|
for kpi in kpi_group['content']['kpis']:
|
|
kpi['parent_kpi_template_id'] = kpi_group['id']
|
|
kpi['parent_kpi_template_title'] = kpi_group['content']['title']
|
|
module_kpi_mapping[itsi_module][kpi['kpi_template_kpi_id']] = kpi
|
|
return module_kpi_mapping
|
|
|
|
def _add_aggregate_and_entity_fields_for_thresholding(self, response):
|
|
"""
|
|
Adds the fields 'search_time_series_aggregate' and 'search_time_series_entities' to the
|
|
response object within each KPI so that the fields get properly picked up by the thresholding
|
|
template in the module detail page
|
|
|
|
@type response: dict
|
|
@param response: Response object for KPI groups formatted to itsi_module_interface standards
|
|
"""
|
|
parsed_response = response
|
|
for kpi_group in parsed_response:
|
|
for kpi in kpi_group['content']['kpis']:
|
|
has_aggregate_search = True if 'aggregate_thresholds' in kpi \
|
|
and 'search' in kpi['aggregate_thresholds'] else False
|
|
has_entities_search = True if 'entity_thresholds' in kpi \
|
|
and 'search' in kpi['entity_thresholds'] else False
|
|
kpi['search_time_series_aggregate'] = kpi['aggregate_thresholds']['search'] if \
|
|
has_aggregate_search else ''
|
|
kpi['search_time_series_entities'] = kpi['entity_thresholds']['search'] if \
|
|
has_entities_search else ''
|
|
|
|
return parsed_response
|
|
|
|
def _set_kpi_template_kpi_id(self, itsi_module, kpi_obj_list):
|
|
"""
|
|
Set kpi_template_kpi_id for each kpi with constructed id value
|
|
|
|
@type itsi_module: string
|
|
@param itsi_module: ITSI module name
|
|
|
|
@type kpi_obj_list: list
|
|
@param kpi_obj_list: a list of kpi objects
|
|
|
|
@rtype: list
|
|
@return: a list with updated kpi objects
|
|
"""
|
|
for kpi in kpi_obj_list:
|
|
kpi['kpi_template_kpi_id'] = utils.make_stanza_name(itsi_module, kpi.get('title'))
|
|
return kpi_obj_list
|
|
|
|
def _update_kpis(self, itsi_module, kpis, kpi_base_search_id_mapping, kpi_base_search_id_metrics_mapping):
|
|
"""
|
|
Update kpis with new base_search_id
|
|
|
|
@type kpis: list
|
|
@param kpis: list of kpi object
|
|
|
|
@type kpi_base_search_id_mapping: dict
|
|
@param kpi_base_search_id_mapping: mapping between old base search id and new base search id
|
|
|
|
@type kpi_base_search_id_metrics_mapping: dict
|
|
@param kpi_base_search_id_metrics_mapping: mapping between new base search id, its metric and key mapping for each metric
|
|
|
|
@rtype: list
|
|
@return: list of updated kpi objects
|
|
"""
|
|
updated_kpis = []
|
|
for kpi in kpis:
|
|
if kpi.get('search_type') == 'shared_base':
|
|
old_base_search_id = kpi.get('base_search_id')
|
|
old_base_search_metric = kpi.get('base_search_metric')
|
|
new_base_search_id = kpi_base_search_id_mapping.get(old_base_search_id)
|
|
new_metrics_key = kpi_base_search_id_metrics_mapping.get(new_base_search_id).get('metrics').get(old_base_search_metric)
|
|
if old_base_search_id and old_base_search_id in kpi_base_search_id_mapping and new_metrics_key and new_base_search_id in kpi_base_search_id_metrics_mapping:
|
|
kpi['base_search_id'] = new_base_search_id
|
|
kpi['base_search_metric'] = new_metrics_key
|
|
|
|
updated_kpis.append(kpi)
|
|
self.logger.debug('Update %s KPI in in module %s: base_search_id from %s to %s, base_search_metric from %s to %s', kpi.get('title'), itsi_module,
|
|
old_base_search_id, new_base_search_id, old_base_search_metric, new_metrics_key)
|
|
else:
|
|
updated_kpis.append(kpi)
|
|
return updated_kpis
|