You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

518 lines
24 KiB

#!/usr/bin/python
import sys
from splunk.appserver.mrsparkle.lib.util import make_splunkhome_path
def add_to_sys_path(paths, prepend=False):
for path in paths:
if prepend:
if path in sys.path:
sys.path.remove(path)
sys.path.insert(0, path)
elif not path in sys.path:
sys.path.append(path)
add_to_sys_path([make_splunkhome_path(['etc', 'apps', 'Splunk_Security_Essentials', 'lib', 'py23', 'splunklib'])], prepend=True)
# We should not rely on core enterprise packages:
add_to_sys_path([make_splunkhome_path(['etc', 'apps', 'Splunk_Security_Essentials', 'lib', 'py3'])], prepend=True)
# Common libraries like future
add_to_sys_path([make_splunkhome_path(['etc', 'apps', 'Splunk_Security_Essentials', 'lib', 'py23'])], prepend=True)
from six.moves import reload_module
try:
if 'future' in sys.modules:
import future
reload_module(future)
except Exception:
'''noop: future was not loaded yet'''
import json, csv, re, os
import six.moves.urllib.request, six.moves.urllib.parse, six.moves.urllib.error, six.moves.urllib.request, six.moves.urllib.error, six.moves.urllib.parse
import time
from splunk.clilib.cli_common import getConfKeyValue, getConfStanza
import splunklib.client as client
import splunk.entity, splunk.Intersplunk
from io import open
from six.moves import range
def strip_non_ascii(string):
''' Returns the string without non ASCII characters'''
stripped = (c for c in string if 0 < ord(c) < 127)
return ''.join(stripped)
def errorOut(msg):
errorMessage = msg
#print "_time,status,message"
#print "%s,%s,\"%s\"" % (time.time(), "Error!", json.dumps(msg).replace('"', '""'))
which_killchain = "attack"
prettyPrint = True
debug = ""
(isgetinfo, sys.argv) = splunk.Intersplunk.isGetInfo(sys.argv)
args, kwargs = splunk.Intersplunk.getKeywordsAndOptions()
if isgetinfo:
# streaming, generating, retevs, reqsop, preop
splunk.Intersplunk.outputInfo(False, False, False, False, None)
results,dummyresults,settings = splunk.Intersplunk.getOrganizedResults()
sessionKey = settings['sessionKey']
owner = settings['owner']
app = "Splunk_Security_Essentials"
mitre_attack_blob = {}
mitre_preattack_blob = {}
# setattr( "search_name", "search_name")
# setattr( "mitre_technique", "mitre_technique")
search_name = "search_name"
mitre_technique = "mitre_technique"
delim = ""
m = re.search("mitremaplookup[^\|]*mitre_technique=\"*\s*([^ \|\"]*)", settings['search'])
if m:
mitre_technique = m.group(1)
m = re.search("mitremaplookup[^\|]*search_name=\"*\s*([^ \|\"]*)", settings['search'])
if m:
search_name = m.group(1)
m = re.search("mitremaplookup[^\|]*delim=\"*\s*([^ \"]*)", settings['search'])
if m:
delim = m.group(1)
try:
# Getting configurations
base_url = "https://" + getConfKeyValue('web', 'settings', 'mgmtHostPort')
except:
errorOut({"response": "Error getting configurations!"})
try:
# Getting configurations
request = six.moves.urllib.request.Request(base_url + '/services/pullJSON?config=mitreattack',
headers = { 'Authorization': ('Splunk %s' % sessionKey)})
search_results = six.moves.urllib.request.urlopen(request)
mitre_attack_blob = json.loads(search_results.read())
debug += "Here we go... " + " - ".join(list(mitre_attack_blob.keys()))
except Exception as e:
errorOut({"status": "ERROR", "description": "Error occurred while grabbing mitre attack", "message": str(e)})
searchTitleToSummaryName = {}
try:
# Getting configurations
request = six.moves.urllib.request.Request(base_url + '/services/SSEShowcaseInfo?fields=mini',
headers = { 'Authorization': ('Splunk %s' % sessionKey)})
search_results = six.moves.urllib.request.urlopen(request)
ShowcaseInfo = json.loads(search_results.read())
for summaryName in ShowcaseInfo['summaries']:
searchTitles = ShowcaseInfo['summaries'][summaryName]['search_title'].split("|")
for searchTitle in searchTitles:
if searchTitle != "":
searchTitleToSummaryName[searchTitle] = summaryName
except Exception as e:
errorOut({"status": "ERROR", "description": "Error occurred while grabbing SSE ShowcaseInfo", "message": str(e)})
try:
# Getting configurations
request = six.moves.urllib.request.Request(base_url + '/services/pullJSON?config=mitreattack',
headers = { 'Authorization': ('Splunk %s' % sessionKey)})
search_results = six.moves.urllib.request.urlopen(request)
mitre_preattack_blob = json.loads(search_results.read())
except Exception as e:
errorOut({"status": "ERROR", "description": "Error occurred while grabbing mitre preattack", "message": str(e)})
prettyPrint = True
which_killchain = "attack"
mitre_names = {}
phase_short_names_to_tactics = {}
mitre_tactics = {}
mitre_tactics_to_pretty_names = {}
try:
if which_killchain == "attack":
with open("../appserver/static/vendor/mitre/enterprise-attack.json") as f:
if len(list(mitre_attack_blob.keys())) == 0:
mitre_attack_blob = json.load(f)
debug += "Error -- no mitre attack data found, so failing back to file"
for obj in mitre_attack_blob['objects']:
if "name" in obj:
obj['name'] = obj['name'].replace(u'\xe4', "a")
obj['name'] = strip_non_ascii(obj['name'])
if "external_references" in obj:
for reference in obj['external_references']:
if "url" in reference and (obj["type"] == "attack-pattern" or obj["type"] == "x-mitre-tactic") and ( "https://attack.mitre.org/techniques/" in reference['url'] or "https://attack.mitre.org/tactics/" in reference['url'] ):
mitre_names[ reference['external_id'] ] = obj['name']
if "url" in reference and (obj["type"] == "attack-pattern" or obj["type"] == "x-mitre-tactic") and "https://attack.mitre.org/tactics/" in reference['url']:
mitre_tactics[ reference['external_id'] ] = []
mitre_tactics_to_pretty_names[ reference['external_id'] ] = obj['name']
phase_short_names_to_tactics[ obj['x_mitre_shortname'] ] = reference['external_id']
for obj in mitre_attack_blob['objects']:
if "external_references" in obj:
for reference in obj['external_references']:
if "url" in reference and (obj["type"] == "attack-pattern" or obj["type"] == "x-mitre-tactic") and "https://attack.mitre.org/techniques/" in reference['url'] :
if "kill_chain_phases" in obj:
for phase in obj['kill_chain_phases']:
if phase['kill_chain_name'] == "mitre-pre-attack" or phase['kill_chain_name'] == "mitre-attack":
if phase['phase_name'] in phase_short_names_to_tactics:
mitre_tactics[ phase_short_names_to_tactics[ phase['phase_name'] ] ].append( reference['external_id'] )
elif which_killchain == "preattack":
with open("../appserver/static/vendor/mitre/pre-attack.json") as f:
if len(list(mitre_preattack_blob.keys())) == 0:
mitre_attack_blob = json.load(f)
else:
mitre_attack_blob = mitre_preattack_blob #lazy
mitre_attack_blob = json.load(f)
for obj in mitre_attack_blob['objects']:
if "external_references" in obj:
for reference in obj['external_references']:
if "url" in reference and (obj["type"] == "attack-pattern" or obj["type"] == "x-mitre-tactic") and ( "https://attack.mitre.org/techniques/" in reference['url'] or "https://attack.mitre.org/tactics/" in reference['url'] ):
mitre_names[reference['external_id']] = obj['name']
if "url" in reference and (obj["type"] == "attack-pattern" or obj["type"] == "x-mitre-tactic") and "https://attack.mitre.org/tactics/" in reference['url']:
mitre_tactics[reference['external_id']] = []
mitre_tactics_to_pretty_names[ reference['external_id'] ] = obj['name']
phase_short_names_to_tactics[ obj['x_mitre_shortname'] ] = reference['external_id']
for obj in mitre_attack_blob['objects']:
if "external_references" in obj:
for reference in obj['external_references']:
if "url" in reference and (obj["type"] == "attack-pattern" or obj["type"] == "x-mitre-tactic") and "https://attack.mitre.org/techniques/" in reference['url'] :
if "kill_chain_phases" in obj:
for phase in obj['kill_chain_phases']:
if phase['kill_chain_name'] == "mitre-pre-attack" or phase['kill_chain_name'] == "mitre-attack":
if phase['phase_name'] in phase_short_names_to_tactics:
mitre_tactics[ phase_short_names_to_tactics[ phase['phase_name'] ] ].append( reference['external_id'] )
except Exception as e:
debug += str(e)
listOfTechniques = []
for record in results:
if mitre_technique in record and record[mitre_technique] != "":
mitres = []
if isinstance(record[mitre_technique], list):
mitres = record[mitre_technique]
else:
if delim != "":
mitres = record[mitre_technique].split(delim)
else:
mitres = [record['mitre_technique']]
# mitres = record[mitre_technique].split(delim)
for mitre in mitres:
mitre = re.sub(r'[^tT0-9].*', '', mitre)
if mitre!="" and mitre!="None":
listOfTechniques.append(mitre)
if search_name in record and record[search_name] in searchTitleToSummaryName:
regex = '"'
summaryName = searchTitleToSummaryName[ record[search_name] ]
if "mitre_technique" in ShowcaseInfo['summaries'][summaryName]:
mitres = ShowcaseInfo['summaries'][summaryName]['mitre_technique'].split("|")
for mitre in mitres:
if mitre!="" and mitre!="None":
listOfTechniques.append(mitre)
columns = sorted(list(mitre_tactics.keys()))
for column in columns:
mitre_tactics[column].sort()
newColumn = []
for technique in mitre_tactics[column]:
if technique in listOfTechniques:
newColumn.append(technique)
mitre_tactics[column] = newColumn
longest_key = len(mitre_tactics[max(mitre_tactics, key= lambda x: len(set(mitre_tactics[x])))])
returnObj = []
columns = sorted(list(mitre_tactics.keys()))
w = csv.writer(sys.stdout)
if prettyPrint:
pretty_columns = []
for column in columns:
mitre_tactics[column].sort()
pretty_columns.append(mitre_tactics_to_pretty_names[column])
w.writerow(pretty_columns)
else:
w.writerow(columns)
# longest_key = len(mitre_tactics[max(mitre_tactics, key= lambda x: len(set(mitre_tactics[x])))])
for i in range(0, longest_key):
currentRow = []
for tactic in columns:
if i < len(mitre_tactics[tactic]) and mitre_tactics[tactic][i] in mitre_names:
# currentRow.append(debug)
currentRow.append(mitre_names[mitre_tactics[tactic][i]])
else:
currentRow.append("")
w.writerow(currentRow)
# sys.exit()
# def strip_non_ascii(string):
# ''' Returns the string without non ASCII characters'''
# stripped = (c for c in string if 0 < ord(c) < 127)
# return ''.join(stripped)
# def errorOut(obj):
# print "Error!"
# print '"' + json.dumps(obj).replace('"', '""') + '"'
# sys.exit()
# mitre_names = {}
# phase_short_names_to_tactics = {}
# mitre_tactics = {}
# mitre_tactics_to_pretty_names = {}
# mitre_attack_blob = {}
# mitre_preattack_blob = {}
# mitre_groups = {}
# mitre_techniques_to_groups = {}
# mitre_refs_to_refs = {}
# mitre_refs_to_names = {}
# ShowcaseInfo = {"summaries": {}}
# inScopeTechniques = {}
# popularTechniques = {}
# try:
# # Getting configurations
# base_url = "https://" + getConfKeyValue('web', 'settings', 'mgmtHostPort')
# except:
# errorOut({"response": "Error getting configurations!"})
# try:
# # Getting configurations
# request = urllib2.Request(base_url + '/services/pullJSON?config=mitreattack',
# headers = { 'Authorization': ('Splunk %s' % sessionKey)})
# search_results = urllib2.urlopen(request)
# mitre_attack_blob = json.loads(search_results.read())
# except Exception as e:
# errorOut({"status": "ERROR", "description": "Error occurred while grabbing mitre attack", "message": str(e)})
# try:
# # Getting configurations
# request = urllib2.Request(base_url + '/services/SSEShowcaseInfo',
# headers = { 'Authorization': ('Splunk %s' % sessionKey)})
# search_results = urllib2.urlopen(request)
# ShowcaseInfo = json.loads(search_results.read())
# for summaryName in ShowcaseInfo['summaries']:
# if "mitre_technique" in ShowcaseInfo['summaries'][summaryName] and ShowcaseInfo['summaries'][summaryName]["mitre_technique"] != "":
# mitres = ShowcaseInfo['summaries'][summaryName]["mitre_technique"].split("|")
# for mitre in mitres:
# if mitre != "" and mitre != "None":
# if mitre not in inScopeTechniques:
# inScopeTechniques[mitre] = 1
# else:
# inScopeTechniques[mitre] += 1
# except Exception as e:
# errorOut({"status": "ERROR", "description": "Error occurred while grabbing SSE ShowcaseInfo", "message": str(e)})
# try:
# # Getting configurations
# request = urllib2.Request(base_url + '/services/pullJSON?config=mitrepreattack',
# headers = { 'Authorization': ('Splunk %s' % sessionKey)})
# search_results = urllib2.urlopen(request)
# mitre_preattack_blob = json.loads(search_results.read())
# except Exception as e:
# errorOut({"status": "ERROR", "description": "Error occurred while grabbing mitre preattack", "message": str(e)})
# if which_killchain == "attack":
# with open("../appserver/static/vendor/mitre/enterprise-attack.json") as f:
# if len(mitre_attack_blob.keys()) == 0:
# mitre_attack_blob = json.load(f)
# debug.append("Using the file version of the attack blob")
# else:
# debug.append("Found the rest endpoint version of the attack blob")
# debug.append("Attack count" + str(len(mitre_attack_blob['objects'])))
# for obj in mitre_attack_blob['objects']:
# if "name" in obj:
# obj['name'] = obj['name'].replace(u'\xe4', "a")
# obj['name'] = strip_non_ascii(obj['name'])
# if "external_references" in obj:
# for reference in obj['external_references']:
# if "url" in reference and "type" in obj and (obj["type"] == "attack-pattern" or obj["type"] == "x-mitre-tactic") and ( "https://attack.mitre.org/techniques/" in reference['url'] or "https://attack.mitre.org/tactics/" in reference['url'] ):
# mitre_names[ reference['external_id'] ] = obj['name']
# mitre_refs_to_names[obj['id']] = reference['external_id']
# if "url" in reference and "type" in obj and (obj["type"] == "attack-pattern" or obj["type"] == "x-mitre-tactic") and "https://attack.mitre.org/tactics/" in reference['url']:
# mitre_tactics[ reference['external_id'] ] = []
# mitre_tactics_to_pretty_names[ reference['external_id'] ] = obj['name']
# phase_short_names_to_tactics[ obj['x_mitre_shortname'] ] = reference['external_id']
# if "type" in obj and obj["type"] == "intrusion-set":
# mitre_refs_to_names[obj['id']] = obj['name']
# for reference in obj['external_references']:
# if "url" in reference and "https://attack.mitre.org/groups" in reference['url']:
# mitre_groups[reference['external_id']] = {
# "url": reference['url'],
# "name": obj["name"]
# }
# if "type" in obj and obj['type'] == "relationship":
# if "intrusion-set" in obj['source_ref'] and "attack-pattern" in obj['target_ref']:
# if obj['target_ref'] not in mitre_refs_to_refs:
# mitre_refs_to_refs[obj['target_ref']] = []
# mitre_refs_to_refs[obj['target_ref']].append(obj['source_ref'])
# if "intrusion-set" in obj['target_ref'] and "attack-pattern" in obj['source_ref']:
# if obj['source_ref'] not in mitre_refs_to_refs:
# mitre_refs_to_refs[obj['source_ref']] = []
# mitre_refs_to_refs[obj['source_ref']].append(obj['target_ref'])
# for ref in mitre_refs_to_refs:
# for refvalue in mitre_refs_to_refs[ref]:
# if mitre_refs_to_names[ref] not in popularTechniques:
# popularTechniques[mitre_refs_to_names[ref]] = 1
# else:
# popularTechniques[mitre_refs_to_names[ref]] += 1
# if ref in mitre_refs_to_names and refvalue in mitre_refs_to_names and mitre_refs_to_names[refvalue] in groups:
# if mitre_refs_to_names[ref] not in mitre_techniques_to_groups:
# mitre_techniques_to_groups[mitre_refs_to_names[ref]] = []
# if mitre_refs_to_names[refvalue] not in mitre_techniques_to_groups[mitre_refs_to_names[ref]]:
# mitre_techniques_to_groups[mitre_refs_to_names[ref]].append(mitre_refs_to_names[refvalue])
# for obj in mitre_attack_blob['objects']:
# if "external_references" in obj:
# for reference in obj['external_references']:
# if "url" in reference and "https://attack.mitre.org/techniques/" in reference['url'] :
# if "kill_chain_phases" in obj:
# for phase in obj['kill_chain_phases']:
# if phase['kill_chain_name'] == "mitre-pre-attack" or phase['kill_chain_name'] == "mitre-attack":
# if phase['phase_name'] in phase_short_names_to_tactics:
# num_content_string = ""
# if popularOnly and (reference['external_id'] not in popularTechniques or popularTechniques[reference['external_id']] < popularity_threshold):
# continue
# if inScopeOnly and reference['external_id'] not in inScopeTechniques:
# continue
# # if inScopeOnly:
# # if inScopeTechniques[reference['external_id']] > 1:
# # num_content_string = " (" + str(inScopeTechniques[reference['external_id']]) + " items)"
# # else:
# # num_content_string = " (" + str(inScopeTechniques[reference['external_id']]) + " item)"
# if reference['external_id'] in mitre_techniques_to_groups:
# if prettyPrint:
# mitre_tactics[ phase_short_names_to_tactics[ phase['phase_name'] ] ].append(obj['name'] + " (" + ", ".join(mitre_techniques_to_groups[ reference['external_id'] ]) + ")" + num_content_string)
# else:
# mitre_tactics[ phase_short_names_to_tactics[ phase['phase_name'] ] ].append( reference['external_id'] )
# else:
# if forGroupOnly:
# continue
# if prettyPrint:
# mitre_tactics[ phase_short_names_to_tactics[ phase['phase_name'] ] ].append(obj['name'] + num_content_string)
# else:
# mitre_tactics[ phase_short_names_to_tactics[ phase['phase_name'] ] ].append( reference['external_id'] )
# elif which_killchain == "preattack":
# with open("../appserver/static/vendor/mitre/pre-attack.json") as f:
# if len(mitre_preattack_blob.keys()) == 0:
# mitre_attack_blob = json.load(f)
# debug.append("Using the file version of the preattack blob")
# else:
# mitre_attack_blob = mitre_preattack_blob #lazy
# debug.append("Found the rest endpoint version of the preattack blob")
# debug.append("Preattack count" + str(len(mitre_attack_blob['objects'])))
# for obj in mitre_attack_blob['objects']:
# if "external_references" in obj:
# for reference in obj['external_references']:
# if "url" in reference and "type" in obj and (obj["type"] == "attack-pattern" or obj["type"] == "x-mitre-tactic") and ( "https://attack.mitre.org/techniques/" in reference['url'] or "https://attack.mitre.org/tactics/" in reference['url'] ):
# mitre_names[reference['external_id']] = obj['name']
# if "url" in reference and "type" in obj and (obj["type"] == "attack-pattern" or obj["type"] == "x-mitre-tactic") and "https://attack.mitre.org/tactics/" in reference['url']:
# mitre_tactics[reference['external_id']] = []
# mitre_tactics_to_pretty_names[ reference['external_id'] ] = obj['name']
# phase_short_names_to_tactics[ obj['x_mitre_shortname'] ] = reference['external_id']
# for obj in mitre_attack_blob['objects']:
# if "external_references" in obj:
# for reference in obj['external_references']:
# if "url" in reference and "https://attack.mitre.org/techniques/" in reference['url'] :
# if "kill_chain_phases" in obj:
# for phase in obj['kill_chain_phases']:
# if phase['kill_chain_name'] == "mitre-pre-attack" or phase['kill_chain_name'] == "mitre-attack":
# if phase['phase_name'] in phase_short_names_to_tactics:
# num_content_string = ""
# if inScopeOnly and reference['external_id'] not in inScopeTechniques:
# continue
# # if inScopeOnly:
# # if inScopeTechniques[reference['external_id']] > 1:
# # num_content_string = " (" + str(inScopeTechniques[reference['external_id']]) + " items)"
# # else:
# # num_content_string = " (" + str(inScopeTechniques[reference['external_id']]) + " item)"
# if prettyPrint:
# mitre_tactics[ phase_short_names_to_tactics[ phase['phase_name'] ] ].append(obj['name'] + num_content_string)
# else:
# mitre_tactics[ phase_short_names_to_tactics[ phase['phase_name'] ] ].append( reference['external_id'] )
# else:
# print "Error!"
# print '"' + "Could not find an attack phase called: " + which_killchain.replace('"', '""') + '"'
# sys.exit()
# #print json.dumps(mitre_tactics, sort_keys = True, indent=4)
# # w = csv.DictWriter(sys.stdout, mitre_tactics.keys())
# # w.writeheader()
# # w.writerow(mitre_tactics)
# # w = csv.writer(sys.stdout)
# # w.writerows(mitre_tactics.items())
# columns = mitre_tactics.keys()
# columns.sort()
# w = csv.writer(sys.stdout)
# if prettyPrint:
# pretty_columns = []
# for column in columns:
# mitre_tactics[column].sort()
# pretty_columns.append(mitre_tactics_to_pretty_names[column])
# w.writerow(pretty_columns)
# else:
# w.writerow(columns)
# longest_key = len(mitre_tactics[max(mitre_tactics, key= lambda x: len(set(mitre_tactics[x])))])
# for i in range(0, longest_key):
# currentRow = []
# for tactic in columns:
# if i < len(mitre_tactics[tactic]):
# if len(groups)>0:
# if tactic in mitre_techniques_to_groups:
# currentRow.append(", ".join(mitre_techniques_to_groups[tactic]))
# else:
# currentRow.append(mitre_tactics[tactic][i])
# else:
# currentRow.append(mitre_tactics[tactic][i])
# else:
# currentRow.append("")
# w.writerow(currentRow)