You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
401 lines
17 KiB
401 lines
17 KiB
# -*- coding: utf-8 -*-
|
|
# Core Python Imports
|
|
import sys
|
|
import os
|
|
import logging
|
|
import logging.handlers
|
|
import unicodedata
|
|
import bz2
|
|
import json
|
|
import contextlib
|
|
|
|
|
|
# CherryPy Web Controller Imports
|
|
import cherrypy
|
|
import splunk.appserver.mrsparkle.controllers as controllers
|
|
from splunk.appserver.mrsparkle.lib.decorators import expose_page
|
|
from splunk.appserver.mrsparkle.lib.routes import route
|
|
from splunk.clilib.bundle_paths import make_splunkhome_path
|
|
|
|
# Splunkd imports
|
|
import splunk
|
|
sys.path.append(make_splunkhome_path(['etc', 'apps', 'DA-ITSI-CP-vmware-dashboards', 'lib']))
|
|
from vmware_utils import six
|
|
|
|
sys.path.append(make_splunkhome_path(['etc', 'apps', 'DA-ITSI-CP-vmware-dashboards', 'local', 'data']))
|
|
|
|
# CONSTANTS
|
|
REST_ROOT_PATH = '/services'
|
|
|
|
|
|
def setupLogger(logger=None, log_format='%(asctime)s %(levelname)s [ReadStructuresService] %(message)s',
|
|
level=logging.DEBUG, log_name="read_structures_service.log", logger_name="read_structures_service"):
|
|
"""
|
|
Setup a logger suitable for splunkd consumption
|
|
"""
|
|
if logger is None:
|
|
logger = logging.getLogger(logger_name)
|
|
|
|
logger.propagate = False # Prevent the log messages from being duplicated in the python.log file
|
|
logger.setLevel(level)
|
|
|
|
file_handler = logging.handlers.RotatingFileHandler(make_splunkhome_path(['var', 'log', 'splunk', log_name]),
|
|
maxBytes=2500000, backupCount=5)
|
|
formatter = logging.Formatter(log_format)
|
|
file_handler.setFormatter(formatter)
|
|
|
|
logger.handlers = []
|
|
logger.addHandler(file_handler)
|
|
|
|
logger.debug("init read structures service logger")
|
|
|
|
return logger
|
|
|
|
|
|
logger = setupLogger()
|
|
splunk.setDefault()
|
|
local_host_path = splunk.mergeHostPath()
|
|
|
|
|
|
class SOLNSelectorError(cherrypy.HTTPError):
|
|
"""
|
|
This error class will be used to set the status and msg on the error
|
|
responses.
|
|
"""
|
|
|
|
def get_error_page(self, *args, **kwargs):
|
|
kwargs['noexname'] = 'true'
|
|
return super(SOLNSelectorError, self).get_error_page(*args, **kwargs)
|
|
|
|
|
|
class read_structures_service(controllers.BaseController):
|
|
'''Read Structures Service Controller'''
|
|
|
|
# Dictionary for single entity views
|
|
datastructDict = {}
|
|
|
|
# Dictionaries for Host/VM view
|
|
hostDataDict = {}
|
|
vmDataDict = {}
|
|
|
|
def _intersect(*d):
|
|
sets = iter(map(set, d))
|
|
result = next(sets)
|
|
for s in sets:
|
|
result = result.intersection(s)
|
|
return result
|
|
|
|
def __init__(self):
|
|
# logger.debuug('Read service initialization called')
|
|
try:
|
|
kwargs = {}
|
|
self._readStructuresAtLoad()
|
|
except AttributeError:
|
|
self.sessionKey = None
|
|
pass
|
|
except Exception as e:
|
|
logger.error(e)
|
|
super(read_structures_service, self).__init__()
|
|
|
|
def _readStructuresAtLoad(self):
|
|
folderName = 'hostsystem'
|
|
logger.debug('Reading Hostsystem data in _readStructuresAtLoad')
|
|
hread = self._idFieldsHash(folderName, 'host')
|
|
folderName = 'vm'
|
|
logger.debug('Reading VM data in _readStructuresAtLoad')
|
|
vread = self._readFilesFromFolder(folderName, 'vm')
|
|
folderName = 'cluster'
|
|
logger.debug('Reading Cluster data in _readStructuresAtLoad')
|
|
cread = self._readFilesFromFolder(folderName, 'cluster')
|
|
|
|
return
|
|
|
|
def _readFilesFromFolder(self, folderName, datatype):
|
|
""""Read data structures from folder.
|
|
Type field is used for host vm perf view to read host and vm data
|
|
|
|
"""
|
|
try:
|
|
folderPath = make_splunkhome_path(['etc', 'apps', 'DA-ITSI-CP-vmware-dashboards', 'local', 'data', folderName])
|
|
structDict = {}
|
|
logger.debug('There are files in this folder {0}', folderPath)
|
|
for file in os.listdir(folderPath):
|
|
logger.debug('Reading this file %s...', file)
|
|
with contextlib.closing(bz2.BZ2File(folderPath + "/" + file, 'rb')) as f:
|
|
logger.debug('Loading file {0} with json', file)
|
|
content = f.read()
|
|
dataDict = json.loads(content, encoding="utf-8")
|
|
|
|
logger.debug('Done loading dataDict...')
|
|
for key, val in six.iteritems(dataDict):
|
|
structDict[key] = json.loads(val, encoding="utf-8")
|
|
logger.debug('Key in %s', key)
|
|
if (key == 'idFieldsHash'):
|
|
idFieldsHash = json.loads(val, encoding="utf-8")
|
|
logger.debug('idFieldsHash %s', idFieldsHash)
|
|
structDict['hostHash'] = idFieldsHash['hostHash']
|
|
structDict['moidHash'] = idFieldsHash['moidHash']
|
|
#####Tau
|
|
logger.debug('Struct Dictionary %s', structDict['moidHash'])
|
|
if datatype == 'host':
|
|
logger.debug('Data Type is Host...')
|
|
self.hostDataDict = structDict
|
|
elif datatype == 'vm':
|
|
logger.debug('Data Type is VM...')
|
|
self.vmDataDict = structDict
|
|
else:
|
|
logger.debug('Data Type is Others...')
|
|
self.datastructDict = structDict
|
|
return True
|
|
except Exception as e:
|
|
logger.debug(
|
|
'Could not read files from folder={0}, for datatype={1} due to {2}'.format(folderPath, datatype, e))
|
|
msg = "[SOLNSelector_read_strcutures] Couldn't read files at " + folderPath
|
|
raise SOLNSelectorError(status="500", message=msg)
|
|
|
|
@route('/:app/:action=read_structures')
|
|
@expose_page(must_login=False, methods=['GET'])
|
|
def read_structures(self, app, action, **kwargs):
|
|
folderName = 'hostsystem'
|
|
logger.debug('Reading Hostsystem data in read_structures action')
|
|
hread = self._readFilesFromFolder(folderName, 'host')
|
|
folderName = 'vm'
|
|
logger.debug('Reading VM data in read_structures action')
|
|
vread = self._readFilesFromFolder(folderName, 'vm')
|
|
folderName = 'cluster'
|
|
logger.debug('Reading Cluster data in read_structures action')
|
|
cread = self._readFilesFromFolder(folderName, 'cluster')
|
|
|
|
return
|
|
|
|
@route('/:app/:action=find_matches')
|
|
@expose_page(must_login=False, methods=['GET'])
|
|
def find_matches(self, app, action, **kwargs):
|
|
"""Find the Entity level matches in the IITs"""
|
|
try:
|
|
searchString = kwargs.get("searchString")
|
|
if not searchString:
|
|
return {}
|
|
searchStringLower = searchString.lower()
|
|
|
|
hostVm = kwargs.get('hostVm', "")
|
|
datatype = ""
|
|
strInIIT = False
|
|
if not hostVm:
|
|
datatype = kwargs.get('datatype')
|
|
logger.debug("Search String %s", searchString)
|
|
suggestionsLimit = kwargs.get("suggestionsLimit", 10)
|
|
logger.debug("Suggestion Limit %s", suggestionsLimit)
|
|
matches = []
|
|
keyArr = []
|
|
if hostVm == 'host' or datatype == 'HostSystem':
|
|
iit = self.hostDataDict['iit']
|
|
keys = self.hostDataDict['keys']
|
|
keyIIT = self.hostDataDict['keyIIT']
|
|
allPrefixKeys = self.hostDataDict['allPrefixKeys']
|
|
elif hostVm == 'vm' or datatype == 'VirtualMachine':
|
|
iit = self.vmDataDict['iit']
|
|
keys = self.vmDataDict['keys']
|
|
keyIIT = self.vmDataDict['keyIIT']
|
|
allPrefixKeys = self.vmDataDict['allPrefixKeys']
|
|
else:
|
|
logger.debug("Inside find matches")
|
|
logger.debug("Data dictionary %s", self.datastructDict)
|
|
iit = self.datastructDict['iit']
|
|
logger.debug("IIT %s", iit)
|
|
keys = self.datastructDict['keys']
|
|
keyIIT = self.datastructDict['keyIIT']
|
|
allPrefixKeys = self.datastructDict['allPrefixKeys']
|
|
for key, value in six.iteritems(iit):
|
|
if key and key.lower() == searchStringLower:
|
|
strInIIT = True
|
|
matchKey = iit[key]
|
|
if isinstance(matchKey, list):
|
|
keyArr.extend(matchKey)
|
|
logger.debug("Key Arr in IIT: %s", keyArr)
|
|
if strInIIT:
|
|
logger.debug("search string exists in iit")
|
|
totalCount = len(keyArr)
|
|
for i in range(0, totalCount):
|
|
if i < suggestionsLimit:
|
|
matches.append(keys[keyArr[i]])
|
|
continue
|
|
else:
|
|
break
|
|
logger.debug(matches)
|
|
if ((not strInIIT) or (len(matches) < suggestionsLimit)):
|
|
logger.debug('Search string does not exist in iit or length of matches is less than suggestionsLimit')
|
|
keysArr = []
|
|
for key, value in six.iteritems(keyIIT):
|
|
if key and key.lower() == searchStringLower:
|
|
matchKey = keyIIT[key]
|
|
if isinstance(matchKey, list):
|
|
keysArr.extend(matchKey)
|
|
logger.debug('Key Arr in KeyIIT %s', keysArr)
|
|
for i in range(0, len(keysArr)):
|
|
if len(matches) > suggestionsLimit:
|
|
break
|
|
else:
|
|
if six.PY3:
|
|
suffixString = unicodedata.normalize('NFKD', allPrefixKeys[keysArr[i]])
|
|
else:
|
|
suffixString = unicodedata.normalize('NFKD', allPrefixKeys[keysArr[i]]).encode('utf-8',
|
|
'ignore')
|
|
logger.debug('suffix string = %s', suffixString)
|
|
matchArr = iit[str(suffixString)]
|
|
logger.debug('Matching Arr %s', matchArr)
|
|
for j in range(0, len(matchArr)):
|
|
if (len(matches) > suggestionsLimit):
|
|
break
|
|
else:
|
|
matches.append(keys[matchArr[j]])
|
|
logger.debug("Matches %s", matches)
|
|
return json.dumps(matches, ensure_ascii=False)
|
|
except Exception as e:
|
|
logger.error("Error while finding matches", e)
|
|
|
|
@route('/:app/:action=find_possible_matches')
|
|
@expose_page(must_login=False, methods=['GET'])
|
|
def find_possible_matches(self, app, action, **kwargs):
|
|
""""
|
|
/**
|
|
* Performs Word level Autocompletion. Splits the input
|
|
* string by "/" and look for individual entities in
|
|
* entityIIT to gather matches Input(searchString): String
|
|
* entered in the Input text box Output(matches): Array of
|
|
* matches
|
|
*/
|
|
"""
|
|
try:
|
|
searchString = kwargs.get("searchString")
|
|
if not searchString:
|
|
return {}
|
|
searchStringLower = searchString.lower()
|
|
hostVm = kwargs.get('hostVm', '')
|
|
datatype = ""
|
|
if not hostVm:
|
|
datatype = kwargs.get('datatype')
|
|
searchEntities = searchStringLower.split("/")
|
|
matchArrs = []
|
|
matches = []
|
|
resultsHash = {}
|
|
|
|
if hostVm == 'host' or datatype == 'HostSystem':
|
|
entityiit = self.hostDataDict['entityiit']
|
|
fullPathNames = self.hostDataDict['fullPathNames']
|
|
elif hostVm == 'vm' or datatype == 'VirtualMachine':
|
|
entityiit = self.vmDataDict['entityiit']
|
|
fullPathNames = self.vmDataDict['fullPathNames']
|
|
else:
|
|
entityiit = self.datastructDict['entityiit']
|
|
fullPathNames = self.datastructDict['fullPathNames']
|
|
entityiitLower = dict((key.lower(), value) for key, value in six.iteritems(entityiit))
|
|
for searchEntity in searchEntities:
|
|
logger.debug("Searching Entity %s", searchEntity)
|
|
if not (searchEntity == "" or searchEntity == "*"):
|
|
logger.debug("Finding matches for %s in find_possible_matches", searchEntity)
|
|
match = entityiitLower[searchEntity]
|
|
logger.debug("Matches found in find_possible_matches: %s",match)
|
|
if not match:
|
|
return json.dumps(matches, ensure_ascii=False)
|
|
matchArrs.append(match)
|
|
|
|
if len(matchArrs) > 0:
|
|
results = set(matchArrs[0]).intersection(*matchArrs)
|
|
resultsList = list(results)
|
|
for result in resultsList:
|
|
resultsHash[result] = 1
|
|
|
|
results = list(resultsHash.keys())
|
|
logger.debug("Results in find_possible_matches: %s",results)
|
|
for ind in results:
|
|
matches.append(str(fullPathNames[ind]))
|
|
return json.dumps(matches, ensure_ascii=False)
|
|
except Exception as e:
|
|
logger.error("Error while finding fullPath suggestions", e)
|
|
|
|
@route('/:app/:action=get_selected_key')
|
|
@expose_page(must_login=False, methods=['GET'])
|
|
def get_selected_key(self, app, action, **kwargs):
|
|
"""
|
|
Returns the full path for the selected key
|
|
stored in the 'storedKey'
|
|
"""
|
|
try:
|
|
hostVm = kwargs.get('hostVm', '')
|
|
datatype = ""
|
|
if (hostVm == ""):
|
|
datatype = kwargs.get('datatype')
|
|
if hostVm == 'host' or datatype == 'HostSystem':
|
|
selectedTypeHash = self.hostDataDict['selectedTypeHash']
|
|
elif hostVm == 'vm' or datatype == 'VirtualMachine':
|
|
selectedTypeHash = self.vmDataDict['selectedTypeHash']
|
|
else:
|
|
selectedTypeHash = self.datastructDict['selectedTypeHash']
|
|
|
|
storedKey = kwargs.get("storedKey")
|
|
logger.debug('Selected Type Hash %s', selectedTypeHash)
|
|
return json.dumps(selectedTypeHash[str(storedKey)], ensure_ascii=False)
|
|
except Exception as e:
|
|
logger.error("Error while fetching the selected key", e)
|
|
|
|
@route('/:app/:action=get_id_keys')
|
|
@expose_page(must_login=False, methods=['GET'])
|
|
def get_id_keys(self, app, action, **kwargs):
|
|
"""
|
|
Gets the id field values of the key
|
|
"""
|
|
try:
|
|
hostVm = kwargs.get('hostVm', '')
|
|
key = kwargs.get("key")
|
|
datatype = ""
|
|
if (hostVm == ""):
|
|
datatype = kwargs.get('datatype')
|
|
if hostVm == 'host' or datatype == 'HostSystem':
|
|
hostHash = self.hostDataDict['hostHash']
|
|
moidHash = self.hostDataDict['moidHash']
|
|
elif hostVm == 'vm' or datatype == 'VirtualMachine':
|
|
hostHash = self.vmDataDict['hostHash']
|
|
moidHash = self.vmDataDict['moidHash']
|
|
else:
|
|
hostHash = self.datastructDict['hostHash']
|
|
moidHash = self.datastructDict['moidHash']
|
|
|
|
idKeysHash = {}
|
|
idKeysHash['host'] = hostHash[key]
|
|
idKeysHash['moid'] = moidHash[key]
|
|
|
|
return json.dumps(idKeysHash, ensure_ascii=False)
|
|
except Exception as e:
|
|
logger.error("Error while fetching the id fields for the key", e)
|
|
|
|
@route('/:app/:action=validate_input')
|
|
@expose_page(must_login=False, methods=['GET'])
|
|
def validate_input(self, app, action, **kwargs):
|
|
"""
|
|
Validates that the full Path is present in host and moid hashes
|
|
"""
|
|
try:
|
|
hostVm = kwargs.get('hostVm', '')
|
|
datatype = ""
|
|
if (hostVm == ""):
|
|
datatype = kwargs.get('datatype')
|
|
inputStr = kwargs.get("inputStr")
|
|
logger.debug("Validating String %s", inputStr)
|
|
if hostVm == 'host' or datatype == 'HostSystem':
|
|
hostHash = self.hostDataDict['hostHash']
|
|
moidHash = self.hostDataDict['moidHash']
|
|
elif hostVm == 'vm' or datatype == 'VirtualMachine':
|
|
hostHash = self.vmDataDict['hostHash']
|
|
moidHash = self.vmDataDict['moidHash']
|
|
else:
|
|
hostHash = self.datastructDict['hostHash']
|
|
moidHash = self.datastructDict['moidHash']
|
|
logger.debug("Host Hash %s", hostHash)
|
|
logger.debug("Moid Hash %s", moidHash)
|
|
|
|
if (inputStr in hostHash) and (inputStr in moidHash):
|
|
return json.dumps(1, ensure_ascii=False);
|
|
return json.dumps(0, ensure_ascii=False);
|
|
except Exception as e:
|
|
logger.error("Input validation failed", e) |