Deploiement_Server/deployment-apps/appcontentmanager/bin/stack_helper.py

import requests
import logging
import os
import json
import sys
import logging.handlers
import time
from splunk.persistconn.application import PersistentServerConnectionApplication
import signal
import subprocess
import tempfile
import tarfile
import re
import splunk.clilib.cli_common
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "lib"))

'''
# !!!!! DEBUG !!!!
sys.path.append(os.path.join(os.environ['SPLUNK_HOME'],'etc','apps','SA-VSCode','bin'))
import splunk_debug as dbg
dbg.enable_debugging(timeout=25)
#################
'''

splunk_home = os.environ['SPLUNK_HOME']
LOG_LEVEL = logging.INFO
LOG_FILE_NAME = "acms.log"

def result_errors(msg):
    return (msg["level"] == "ERROR" or  msg["level"] == "CRITICAL")

def convertResponse_to_json(response,stackname):
   
    resp = {}
    resp["stack"] = stackname
    resp["user"] = user
    resp["response"] = {}
    resp["response"]["status_code"] = response.status_code
    resp["response"]["text"] = response.text

    resp["request"] = {}
    resp["request"]["headers"] = response.request.headers.__dict__['_store']
    resp["request"]["headers"]["authorization"] = list(["Authorization","xxxxx xxxxxxx"])
    if response.request.body != None :
        resp["request"]["body"] = response.request.body 
    else :
        resp["request"]["body"] = ""
    resp["request"]["method"] = response.request.method
    resp["request"]["url"] = response.request.url

    return resp

def setup_logger():  # setup logging
    global SPLUNK_HOME, LOG_LEVEL, LOG_FILE_NAME
    if 'SPLUNK_HOME' in os.environ:
        SPLUNK_HOME = os.environ['SPLUNK_HOME']

    log_format = "%(asctime)s %(levelname)-s\t%(module)s[%(process)d]:%(lineno)d - %(message)s"
    logger = logging.getLogger('v')
    logger.setLevel(LOG_LEVEL)

    l = logging.handlers.RotatingFileHandler(os.path.join(SPLUNK_HOME, 'var', 'log', 'splunk', LOG_FILE_NAME), mode='a', maxBytes=1000000, backupCount=2)
    l.setFormatter(logging.Formatter(log_format))
    logger.addHandler(l)

    # ..and (optionally) output to console
    logH = logging.StreamHandler()
    logH.setFormatter(logging.Formatter(fmt=log_format))
    # logger.addHandler(logH)

    logger.propagate = False
    return logger

logger = setup_logger()


def list_apps(acs_url,stackname,headers,experience,logger):
    returnVal = {}
    apps_url = acs_url+stackname+"/adminconfig/v2/apps"
    if experience in ["victoria"] :
        apps_url = apps_url+"/victoria"

    apps_url = apps_url + "?count=0"
    response = requests.get(apps_url, headers=headers)
    if response.status_code == 200 :
        resp = json.loads(response.text)
        returnVal["Private"] = resp
    
    logger.info(json.dumps(convertResponse_to_json(response, stackname)))
    return {'payload': returnVal, 'status': response.status_code}

def delete_app(acs_url,stackname,headers,experience,app,logger):
    returnVal = {}
    apps_url = acs_url+stackname+"/adminconfig/v2/apps"
    if experience in ["victoria"] :
        apps_url = apps_url+"/victoria"

    apps_url = apps_url+"/"+app
    response = requests.delete(apps_url, headers=headers)
    
    while True:
        if response.status_code == 424 :
            # retry ..
            time.sleep(10)
            response = requests.delete(apps_url, headers=headers)
        else :
            break
    
    logger.info(json.dumps(convertResponse_to_json(response, stackname)))

    return {'payload': {"message":response.text, "delete_status":response.status_code}, 'status': 200}

def list_files(rest_url,stackname,headers,app, logger):
    returnVal = {}
    response = requests.get("https://"+stackname+rest_url+"/servicesNS/-/"+app+"/properties?output_mode=json&count=0",headers=headers, verify=False)
    if response.status_code == 200 :
        returnVal = json.loads(response.text)
        for file in returnVal['entry'] :
            if file['name'] == 'views' :
                response = requests.get("https://"+stackname+rest_url+"/servicesNS/-/"+app+"/configs/conf-views?output_mode=json&count=0",headers=headers, verify=False)
                file['dashboards'] = [d for d in json.loads(response.text)['entry'] if d['content']['eai:appName']== app]

    logger.info(json.dumps(convertResponse_to_json(response, stackname)))
    return {'payload': returnVal, 'status': response.status_code}

def list_content(rest_url,stackname,headers,app,file, logger):
    returnVal = {}
    if ".xml" in file :
        response = requests.get("https://"+stackname+rest_url+"/servicesNS/-/"+app+"/data/ui/views/"+file.replace(".xml","")+"?output_mode=json",headers=headers, verify=False)
        if response.status_code == 200 :
            r = json.loads(response.text)
            for d in r['entry'] :
                if d['acl']['app'] == app :
                    returnVal =  {"content": d['content']['eai:data']}

    elif 'nav.' in file :
        response = requests.get("https://"+stackname+rest_url+"/servicesNS/-/"+app+"/data/ui/nav?output_mode=json",headers=headers, verify=False)
        if response.status_code == 200 :
            r = json.loads(response.text)
            for n in r['entry'] :
                if n['acl']['app'] == app :
                    returnVal =  {"content" : d['content']['eai:data']}

    else :
        response = requests.get("https://"+stackname+rest_url+"/servicesNS/-/"+app+"/configs/conf-"+file+"?output_mode=json&count=0",headers=headers, verify=False)
        if response.status_code == 200 :
            returnVal = [d for d in json.loads(response.text)['entry'] if d['content']['eai:appName']== app]
    
    

    logger.info(json.dumps(convertResponse_to_json(response, stackname)))
    return {'payload': returnVal, 'status': response.status_code}


class Stack_Helper(PersistentServerConnectionApplication):
    def __init__(self, _command_line, _command_arg):
        super(PersistentServerConnectionApplication, self).__init__()

    # Handle a syncronous from splunkd.
    def handle(self, in_string):
        """
        Called for a simple synchronous request.
        @param in_string: request data passed in
        @rtype: string or dict
        @return: String to return in response.  If a dict was passed in,
                 it will automatically be JSON encoded before being returned.
        """
        #dbg.set_breakpoint()

        if 'slim.__main__' in sys.modules.keys() :
            sys.modules.pop('slim.__main__')
            sys.modules.pop('slim.partition')
            sys.modules.pop('slim.package')
            sys.modules.pop('slim.validate')
        
        import slim.__main__
        slimmain = slim.__main__.main

        acs_url = "https://admin.splunk.com/"
        rest_url = ".splunkcloud.com:8089"

        # Parse the arguments
        args = self.parse_in_string(in_string)
        
        global user
        user = args['session']['user']
        
        token = ""
        if "token" in args['form_parameters'] :
            token = args['form_parameters']['token']

        app = ""
        if "app" in args['form_parameters'] :
            app = args['form_parameters']['app']

        experience = ""
        if "experience" in args['form_parameters'] :
            experience = args['form_parameters']['experience']

        action = ""
        if "action" in args['form_parameters'] :
            action = args['form_parameters']['action']
        
        file = ""
        if "file" in args['form_parameters'] :
            file = args['form_parameters']['file']
        
        stackname = ""
        if "stackname" in args['form_parameters'] :
            stackname = args['form_parameters']['stackname']
            if "stg-" in stackname :
                acs_url = "https://staging.admin.splunk.com/"
                rest_url = ".stg.splunkcloud.com:8089"
            
            if "-shw" in stackname :
                acs_url = "https://staging.admin.splunk.com/"
                rest_url = ".stg.splunkcloud.com:8089"

            if ".stg" in stackname :
                acs_url = "https://staging.admin.splunk.com/"
                stackname = stackname.replace(".stg","")
                rest_url = ".stg.splunkcloud.com:8089"
        
        headers = {
                'Authorization': 'Bearer '+ token,
                'User-Agent': 'ACS-Helper'
            }
        
        packages_folder = os.path.join(os.environ['SPLUNK_HOME'],'etc','apps','appcontentmanager','appserver','static','packages')

        if action == "list_apps" :
            return list_apps(acs_url,stackname,headers, experience,logger)
        elif action == "list_files" :
            return list_files(rest_url,stackname,headers,app,logger)
        elif action == "list_content" :
            return list_content(rest_url,stackname,headers,app,file,logger)
        elif action == "delete_app" :
            return delete_app(acs_url,stackname,headers, experience,app,logger)
        elif action == "download_app" :
            res = list_files(rest_url,stackname,headers,app,logger)
            if res['status'] == 200 :
                with tempfile.TemporaryDirectory() as tempdir:
                    os.mkdir(os.path.join(tempdir,app))
                    os.mkdir(os.path.join(tempdir,app,"default"))
                    os.mkdir(os.path.join(tempdir,app,"metadata"))
                    data_folder_created = False

                    files = res['payload']['entry']
                    for file in files :
                        if file['name'] == 'views' :
                            if not data_folder_created :
                                os.mkdir(os.path.join(tempdir,app,"default","data"))
                                os.mkdir(os.path.join(tempdir,app,"default","data","ui"))
                                os.mkdir(os.path.join(tempdir,app,"default","data","ui","views"))
                                os.mkdir(os.path.join(tempdir,app,"default","data","ui","nav"))
                                data_folder_created = True

                            res = list_content(rest_url,stackname,headers,app,file['name'],logger)
                            if res['status'] == 200 :
                                confs = res['payload']
                                for confFile in confs :
                                    # get dashboard difinition
                                    response = requests.get("https://"+stackname+rest_url+"/servicesNS/-/"+app+"/data/ui/views/"+confFile["name"]+"?output_mode=json",headers=headers, verify=False)
                                    if response.status_code == 200 :
                                        with open(os.path.join(tempdir,app,"default","data","ui","views",confFile["name"]+".xml"), "w+") as dash:
                                            r = json.loads(response.text)
                                            for d in r['entry'] :
                                                if d['acl']['app'] == app :
                                                    dash.write(d['content']['eai:data'])

                        elif file['name'] == 'nav' :
                            if not data_folder_created :
                                os.mkdir(os.path.join(tempdir,app,"default","data"))
                                os.mkdir(os.path.join(tempdir,app,"default","data","ui"))
                                os.mkdir(os.path.join(tempdir,app,"default","data","ui","views"))
                                os.mkdir(os.path.join(tempdir,app,"default","data","ui","nav"))
                                data_folder_created = True

                        else :
                            res = list_content(rest_url,stackname,headers,app,file['name'],logger)
                            if res['status'] == 200 :
                                confs = res['payload']
                                c = {}
                                for confFile in confs :
                                    #c[confFile['name']] = confFile['content']
                                    c[confFile['name']] = {k: v for k, v in confFile['content'].items() if not k.startswith('eai:') and k != "install_source_checksum"}
                                splunk.clilib.cli_common.writeConfFile(os.path.join(tempdir,app,"default",file['name']+".conf"),c)
                    
                    # generate nav 
                    if not data_folder_created :
                        os.mkdir(os.path.join(tempdir,app,"default","data"))
                        os.mkdir(os.path.join(tempdir,app,"default","data","ui"))
                        os.mkdir(os.path.join(tempdir,app,"default","data","ui","views"))
                        os.mkdir(os.path.join(tempdir,app,"default","data","ui","nav"))
                        data_folder_created = True
                    
                    response = requests.get("https://"+stackname+rest_url+"/servicesNS/-/"+app+"/data/ui/nav?output_mode=json",headers=headers, verify=False)
                    if response.status_code == 200 :
                        with open(os.path.join(tempdir,app,"default","data","ui","nav","default.xml"), "w+") as nav:
                            r = json.loads(response.text)
                            for n in r['entry'] :
                                if n['acl']['app'] == app :
                                    nav.write(n['content']['eai:data'])

                    # generate a default.meta conf file
                    meta = {}
                    meta[''] = {}
                    meta['']["access"] = 'read : [ * ], write : [ admin ]'
                    meta['']["export"] = 'system'
                    splunk.clilib.cli_common.writeConfFile(os.path.join(tempdir,app,"metadata","default.meta"),meta)

                    # package the app to appserver/static/packages folder
                
                    argv = []
                    argv.append(re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]))
                    argv.append("package")

                    argv.append(os.path.join(tempdir,app))
                    argv.append("-o")
                    argv.append(packages_folder)
                    results = slimmain(argv)

                    apppath = ""
                    apppath = results[-1]["msg"].replace('Source package exported to  "','').replace('"','')
                    os.rename(apppath,apppath.replace("tar.gz","spl"))
                    return {'payload': {'path' : apppath.replace("tar.gz","spl")}, 'status': 200}
                            

            
        return {'payload': "", 'status': 200}

    def handleStream(self, handle, in_string):
        """
        For future use
        """
        raise NotImplementedError(
            "PersistentServerConnectionApplication.handleStream")

    def done(self):
        """
        Virtual method which can be optionally overridden to receive a
        callback after the request completes.
        """
        pass

    def convert_to_dict(self, query):
        """
        Create a dictionary containing the parameters.
        """
        parameters = {}

        for key, val in query:

            # If the key is already in the list, but the existing entry isn't a list then make the
            # existing entry a list and add thi one
            if key in parameters and not isinstance(parameters[key], list):
                parameters[key] = [parameters[key], val]

            # If the entry is already included as a list, then just add the entry
            elif key in parameters:
                parameters[key].append(val)

            # Otherwise, just add the entry
            else:
                parameters[key] = val

        return parameters

    def parse_in_string(self, in_string):
        """
        Parse the in_string
        """

        params = json.loads(in_string)

        params['method'] = params['method'].lower()

        params['form_parameters'] = self.convert_to_dict(params.get('form', []))
        params['query_parameters'] = self.convert_to_dict(params.get('query', []))

        return params