import requests import logging import os import json import sys import logging.handlers from splunk.persistconn.application import PersistentServerConnectionApplication import signal import subprocess import tempfile import tarfile import re import splunk.clilib.cli_common import csv import re import boto3 import gzip sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "lib")) ''' # !!!!! DEBUG !!!! sys.path.append(os.path.join(os.environ['SPLUNK_HOME'],'etc','apps','SA-VSCode','bin')) import splunk_debug as dbg dbg.enable_debugging(timeout=25) ################# ''' try: #python3 from urllib.request import urlopen except: #python2 from urllib2 import urlopen splunk_home = os.environ['SPLUNK_HOME'] LOG_LEVEL = logging.INFO LOG_FILE_NAME = "acms.log" def setup_logger(): # setup logging global SPLUNK_HOME, LOG_LEVEL, LOG_FILE_NAME if 'SPLUNK_HOME' in os.environ: SPLUNK_HOME = os.environ['SPLUNK_HOME'] log_format = "%(asctime)s %(levelname)-s\t%(module)s[%(process)d]:%(lineno)d - %(message)s" logger = logging.getLogger('v') logger.setLevel(LOG_LEVEL) l = logging.handlers.RotatingFileHandler(os.path.join(SPLUNK_HOME, 'var', 'log', 'splunk', LOG_FILE_NAME), mode='a', maxBytes=1000000, backupCount=2) l.setFormatter(logging.Formatter(log_format)) logger.addHandler(l) # ..and (optionally) output to console logH = logging.StreamHandler() logH.setFormatter(logging.Formatter(fmt=log_format)) # logger.addHandler(logH) logger.propagate = False return logger logger = setup_logger() class Update_SbApps(PersistentServerConnectionApplication): def __init__(self, _command_line, _command_arg): super(PersistentServerConnectionApplication, self).__init__() # Handle a syncronous from splunkd. def handle(self, in_string): """ Called for a simple synchronous request. @param in_string: request data passed in @rtype: string or dict @return: String to return in response. If a dict was passed in, it will automatically be JSON encoded before being returned. """ #dbg.set_breakpoint() try : bucket_name = "splunkbaseassets" file_name = "splunkbase_apps.csv" s3_path = "splunkbase_apps/" + file_name s3 = boto3.client("s3", aws_access_key_id='AKIAWFRE2V5CBQOFF632', aws_secret_access_key='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', region_name='us-east-1') obj = s3.get_object(Bucket=bucket_name, Key=s3_path) with gzip.open(os.path.join(splunk_home,"etc","apps","appcontentmanager",'lookups','acms_splunkbase_apps.csv.gz'), 'wb') as gz_out_csv: gz_out_csv.write(obj['Body'].read()) gz_out_csv.close() return {'payload': {'status':'success'}, 'status': 200} except: return {'payload': {'status':'Error'}, 'status': 400} def handleStream(self, handle, in_string): """ For future use """ raise NotImplementedError( "PersistentServerConnectionApplication.handleStream") def done(self): """ Virtual method which can be optionally overridden to receive a callback after the request completes. """ pass def convert_to_dict(self, query): """ Create a dictionary containing the parameters. """ parameters = {} for key, val in query: # If the key is already in the list, but the existing entry isn't a list then make the # existing entry a list and add thi one if key in parameters and not isinstance(parameters[key], list): parameters[key] = [parameters[key], val] # If the entry is already included as a list, then just add the entry elif key in parameters: parameters[key].append(val) # Otherwise, just add the entry else: parameters[key] = val return parameters def parse_in_string(self, in_string): """ Parse the in_string """ params = json.loads(in_string) params['method'] = params['method'].lower() params['form_parameters'] = self.convert_to_dict(params.get('form', [])) params['query_parameters'] = self.convert_to_dict(params.get('query', [])) return params