""" This script will be used as a mod input to enable or disable NATS server """ import socket import subprocess import sys import time import json import splunk.rest as rest from splunk.rest import simpleRequest import signal from splunk.clilib.bundle_paths import make_splunkhome_path import platform import tarfile sys.path.append(make_splunkhome_path(['etc', 'apps', 'SA-ITOA', 'lib'])) sys.path.append(make_splunkhome_path(['etc', 'apps', 'SA-ITOA', 'lib', 'SA_ITOA_app_common'])) from ITOA.controller_utils import ITOAError from ITOA.mod_input_utils import skip_run_during_migration from ITOA.setup_logging import getLogger4ModInput import os from SA_ITOA_app_common.solnlib.modular_input import ModularInput from SA_ITOA_app_common.solnlib.conf_manager import ConfManager from ITOA.itoa_common import get_nats_credentials from itsi.itsi_utils import ITOAInterfaceUtils SPLUNK_HOME = os.environ.get("SPLUNK_HOME") class ITSINats(ModularInput): title = 'IT Service Intelligence NATS Modular Input' description = 'Modular Input to start and stop NATS server for Event Analytics' handlers = None app = 'SA-ITOA' name = 'itsi_nats_mod_input' use_single_instance = False use_kvstore_checkpointer = False use_hec_event_writer = False owner = 'nobody' nats_config_path = make_splunkhome_path(['etc', 'apps', 'SA-ITOA', 'bin', 'nats', 'nats-js.conf']) nats_command = make_splunkhome_path(['etc', 'apps', 'SA-ITOA', 'bin', 'nats', 'nats-server']) # https://docs.splunk.com/Documentation/Splunk/9.2.0/Installation/Systemrequirements SUPPORTED_OS = ['windows', 'linux', 'darwin'] # Arch types are possible outputs of uname -m or platform.machine() # https://en.wikipedia.org/wiki/Uname SUPPORTED_ARCH = { 'arm64': 'arm64', 'x86_64': 'amd64', 'amd64': 'amd64', 'i686': 'amd64', 'i386': 'amd64' } NATS_VERSION = 'v2.10.11' enable_rules_engine_in_queue_mode = "| itsichangerulesengineprocess is_disable_all=false is_use_queue_mode=true " \ "is_use_adhoc_search=false is_use_rt_search=false" disable_rules_engine_in_queue_mode = "| itsichangerulesengineprocess is_disable_all=false is_use_queue_mode=false " \ "is_use_adhoc_search=false is_use_rt_search=true" def __init__(self): super() self.logger = None signal.signal(signal.SIGINT, self.shutdown_nats) signal.signal(signal.SIGTERM, self.shutdown_nats) def extra_arguments(self): return [{ 'name': "log_level", 'title': "Logging Level", 'description': "This is the level at which the modular input will log data."}] def get_binary_name(self): """ Finds the right binary name based on OS and arch and returns the binary string name @return: name of the binary file """ os = platform.system().lower() os_architecture_raw = platform.machine() if os not in self.SUPPORTED_OS: raise ITOAError(f'Unsupported OS: {os}') if os_architecture_raw not in self.SUPPORTED_ARCH: raise ITOAError(f'Unsupported architecture: {os_architecture_raw}') binary_name = f'nats-server-{self.NATS_VERSION}-{os}-{self.SUPPORTED_ARCH[os_architecture_raw]}.tar.gz' self.logger.info(f'Nats binary name: {binary_name}') return binary_name def unzip_nats(self): """ Unzips the correct nats binary .tar.gz and moves the nats-server executable to SA-ITOA/bin/nats-server """ # First check if nats is already unzipped nats_binary_zip = self.get_binary_name() nats_binary_zip_path = make_splunkhome_path(['etc', 'apps', 'SA-ITOA', 'lib', 'nats', nats_binary_zip]) if os.path.isfile(self.nats_command): self.logger.info('Nats is already unzipped! Skipping unzipping step') return if not os.path.isfile(nats_binary_zip_path): raise ITOAError(f'Nats binary does not exist: {nats_binary_zip}') # Unzip nats binary tar = tarfile.open(nats_binary_zip_path, 'r:gz') tar.extractall(make_splunkhome_path(['etc', 'apps', 'SA-ITOA', 'bin', 'nats'])) tar.close() # Move nats executable to SA-ITOA/bin folder cur_binary_path = make_splunkhome_path( ['etc', 'apps', 'SA-ITOA', 'bin', 'nats', nats_binary_zip[:-7], 'nats-server']) os.rename(cur_binary_path, self.nats_command) def create_nats_conf_file(self): """ Creates a NATS configuration file for the host OS and arch. Writes nats-js.conf to SA-ITOA/bin/nats-js.conf """ if os.path.isfile(self.nats_config_path): self.logger.info('nats-js.conf is already created! Skipping the step') return cfm = ConfManager(self.session_key, 'SA-ITOA') conf = cfm.get_conf('itsi_nats') settings = conf.get('nats_settings') max_memory_store = int(settings['max_memory_store']) max_file_store = int(settings['max_file_store']) auth_enabled = int(settings['require_auth']) host_name = socket.gethostname() conf = { 'server_name': f'{host_name}-itsi-ea-cluster', 'listen': 4222, 'http_port': 8222, 'debug': False, 'trace': False, 'logfile_size_limit': 5242880, 'log_file': f'{SPLUNK_HOME}/var/log/splunk/itsi-nats-server.log' } jetstream = { 'store_dir': 'nats/data', 'max_memory_store': max_memory_store, 'max_file_store': max_file_store } # Add a user in default SYSTEM account in NATS accounts = { "$SYS": { 'users': [ { 'user': 'sys', 'pass': '' } ] } } conf['jetstream'] = jetstream # for now, disabling accounts support conf['accounts'] = accounts # get nats credentials from storage/passwords passwords_uri = "/services/storage/passwords/nats-admin?output_mode=json" credentials = get_nats_credentials(self.session_key, passwords_uri, auth_enabled) if auth_enabled == 1 and credentials: username = credentials['clear_password'].split(':')[0] password = credentials['clear_password'].split(':')[1] hash = credentials['clear_password'].split(':')[2] authorization = { 'ADMIN': { 'publish': '>', 'subscribe': '>' }, 'users': [ { 'user': username, 'password': hash, 'permissions': str('$ADMIN') } ] } conf['authorization'] = authorization peers = self.get_peers(host_name) if peers: if auth_enabled == 1: cluster = { 'name': 'itsi-ea-cluster', 'listen': f'{host_name}:4248', 'routes': [f'nats://{username}:{password}{peer}:4248' for peer in peers] } else: cluster = { 'name': 'itsi-ea-cluster', 'listen': f'{host_name}:4248', 'routes': [f'nats://{peer}:4248' for peer in peers] } conf['cluster'] = cluster f = open(self.nats_config_path, 'w') f.write(json.dumps(conf)) f.close def get_peers(self, cur_host_name): """ Gets the SHC peer host names @param cur_host_name: current machine's host name @return: list of peers host names """ response, content = simpleRequest('/services/shcluster/status', sessionKey=self.session_key, getargs={'output_mode': 'json'}, raiseAllErrors=False ) if response and response.status != 200: # no peers or shc is not supported return [] content = json.loads(content) peers = [] peers_list = content['entry'][0]['content']['peers'] for _, peer_info in peers_list.items(): peer_host_name = peer_info['label'] if peer_host_name != cur_host_name: peers.append(peer_host_name) return peers @staticmethod def wait_for_job(search_job, maxtime=10): """ Wait up to maxtime seconds for search_job to finish. If maxtime is negative, waits forever. Returns true, if job finished. """ pause = 0.2 lapsed = 0.0 while not search_job.is_done(): time.sleep(pause) lapsed += pause if maxtime >= 0 and lapsed > maxtime: break return search_job.is_done() def perform_search(self, search_query): try: enable_re_queue_mode_search_job = ITOAInterfaceUtils.run_search(self.session_key, self.logger, search_query) if not self.wait_for_job(enable_re_queue_mode_search_job, 500): raise Exception("Search for enabling/disabling Modular Input timed out.") except Exception as e: self.logger.error('Error occurred while enabling/disabling Modular Input: %s', e) @skip_run_during_migration def do_run(self, input_config): # Shutdown existing nats-server instances to make sure output is being captured logger = getLogger4ModInput(input_config) self.logger = logger logger.info('Starting ITSI NATS Modular Input') self.unzip_nats() self.shutdown_nats(None, None) self.create_nats_conf_file() bash_command = [self.nats_command, '-c', self.nats_config_path] # start Rules Engine in Queue Mode self.perform_search(self.enable_rules_engine_in_queue_mode) logger.info('Rules Engine has started in queue mode') try: with subprocess.Popen( bash_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=make_splunkhome_path(['etc', 'apps', 'SA-ITOA', 'bin']), universal_newlines=True ) as process: while process.poll() is None: for output in iter(process.stderr.readline, ''): logger.info(output) time.sleep(1) if self.should_enable_rt(self.session_key): # stop Rules Engine in Queue Mode self.perform_search(self.disable_rules_engine_in_queue_mode) logger.info('Rules Engine has started in rt search mode') except Exception as e: logger.error(str(e)) sys.exit(0) def shutdown_nats(self, signum, frame): shutdown_cmd = [self.nats_command, '--signal', 'quit'] subprocess.run(shutdown_cmd) def should_enable_rt(self, session_key): try: response, content = rest.simpleRequest( "/servicesNS/nobody/SA-ITOA/data/inputs/itsi_nats_mod_input?output_mode=json", sessionKey=session_key, method="GET", raiseAllErrors=True, ) parsed_content = json.loads(content) if parsed_content["entry"][0]["content"]["disabled"]: return True else: return False except Exception as e: self.logger.error('Error while validating the queue mode process : %s', str(e)) if __name__ == '__main__': worker = ITSINats() worker.execute() sys.exit(0)