").attr("id", "div_outputServers" + newNum)
div.append('
')
div.append(
'
"
)
div.insertAfter("#div_outputServers" + maxNum)
// console.log("Just added ", div, "to go after item", maxNum, "element", "#div_outputServers" + maxNum)
}
/*
window.addEventListener('popstate', function(event) {
if (event.state) {
// console.log("Got a state request!", event.state)
if(typeof event.state.myDS != "undefined" && event.state.myDS != window.myDS){ // the second is for the intra-page links.
SetDataSource(event.state.myDS)
}else if(typeof event.state.myTechnology != "undefined" && event.state.myDS != window.myTechnology){ // the second is for the intra-page links.
$(".row, .contents-body").remove()
SetTechnology(event.state.myTechnology)
}else{
ClearTokens()
}
}
}, false);
*/
function ClearTokens(tech) {
require([
"jquery",
"splunkjs/mvc",
"splunkjs/mvc/utils",
"splunkjs/mvc/tokenutils",
"splunkjs/mvc/simplexml",
"splunkjs/ready!",
], function ($, mvc, utils, TokenUtils, DashboardController, Ready) {
var unsubmittedTokens = mvc.Components.getInstance("default")
var submittedTokens = mvc.Components.getInstance("submitted")
unsubmittedTokens.unset("technology")
unsubmittedTokens.unset("datasource")
unsubmittedTokens.unset("dataSourcesTextRow")
submittedTokens.unset("technology")
submittedTokens.unset("datasource")
submittedTokens.unset("dataSourcesTextRow")
})
ApplyTokens()
}
function SetTechnology(tech) {
require([
"jquery",
"splunkjs/mvc",
"splunkjs/mvc/utils",
"splunkjs/mvc/tokenutils",
"splunkjs/mvc/simplexml",
"splunkjs/ready!",
], function ($, mvc, utils, TokenUtils, DashboardController, Ready) {
var unsubmittedTokens = mvc.Components.getInstance("default")
var submittedTokens = mvc.Components.getInstance("submitted")
unsubmittedTokens.set("technology", tech)
unsubmittedTokens.unset("datasource")
unsubmittedTokens.unset("dataSourcesTextRow")
submittedTokens.set(unsubmittedTokens.toJSON())
submittedTokens.unset("datasource")
submittedTokens.unset("dataSourcesTextRow")
})
ApplyTokens()
}
function SetDataSource(DS) {
require([
"jquery",
"splunkjs/mvc",
"splunkjs/mvc/utils",
"splunkjs/mvc/tokenutils",
"splunkjs/mvc/simplexml",
"splunkjs/ready!",
], function ($, mvc, utils, TokenUtils, DashboardController, Ready) {
var unsubmittedTokens = mvc.Components.getInstance("default")
var submittedTokens = mvc.Components.getInstance("submitted")
unsubmittedTokens.set("datasource", DS)
unsubmittedTokens.unset("technology")
//unsubmittedTokens.unset("dataSourcesTextRow");
submittedTokens.set(unsubmittedTokens.toJSON())
submittedTokens.unset("technology")
//submittedTokens.unset("dataSourcesTextRow");
})
ApplyTokens()
}
function descriptionForDataSource(dataSourceName, disclaimerText) {
var ReadyIt = "
Description "
ReadyIt += validDataSources[dataSourceName]["description"]
if (typeof disclaimerText != "undefined") ReadyIt += disclaimerText
ReadyIt += "
Technologies "
ReadyIt +=
'
Technology Guide Available Difficulty Description '
validDataSources[dataSourceName]["technologies"].forEach(function (
dataSourceName
) {
if (
typeof validTechnologies[dataSourceName] != "undefined" &&
typeof validTechnologies[dataSourceName]["guide"] != "undefined" &&
validTechnologies[dataSourceName]["guide"].length > 0
) {
var difficulty = "Easy"
if (
typeof validTechnologies[dataSourceName]["difficulty"] !=
"undefined" &&
validTechnologies[dataSourceName]["difficulty"] != ""
)
difficulty = validTechnologies[dataSourceName]["difficulty"]
ReadyIt +=
'' +
dataSourceName +
" Yes " +
difficulty +
" " +
validTechnologies[dataSourceName]["description"] +
" \n"
} else if (
typeof validTechnologies[dataSourceName] != "undefined" &&
typeof validTechnologies[dataSourceName]["link"] != "undefined" &&
validTechnologies[dataSourceName]["link"] != ""
) {
ReadyIt +=
'' +
dataSourceName +
" " +
validTechnologies[dataSourceName]["description"] +
" \n"
} else {
ReadyIt +=
'' +
dataSourceName +
" "
if (
typeof validTechnologies[dataSourceName] != "undefined" &&
typeof validTechnologies[dataSourceName]["description"] !=
"undefined"
)
ReadyIt +=
"" +
validTechnologies[dataSourceName]["description"] +
" "
else ReadyIt += " "
ReadyIt += " \n"
}
})
ReadyIt += "
"
if (typeof UseCasesBySource[dataSourceName] != "undefined") {
// console.log("Prepping, ", dataSourceName, UseCasesBySource[dataSourceName])
ReadyIt +=
"
Number of Examples " +
UseCasesBySource[dataSourceName]["count"] +
" Examples
Sample Examples "
for (
var i = 0;
i < UseCasesBySource[dataSourceName]["shownList"].length;
i++
) {
ReadyIt +=
"" +
UseCasesBySource[dataSourceName]["shownList"][i] +
" "
}
ReadyIt += " "
}
return ReadyIt
}
function ApplyTokens() {
require([
"jquery",
"splunkjs/mvc",
"splunkjs/mvc/utils",
"splunkjs/mvc/tokenutils",
"splunkjs/mvc/simplexml",
"splunkjs/ready!",
], function ($, mvc, utils, TokenUtils, DashboardController, Ready) {
// console.log("Kicking off!")
var unsubmittedTokens = mvc.Components.getInstance("default")
var submittedTokens = mvc.Components.getInstance("submitted")
var myTechnology = unsubmittedTokens.get("technology")
var myDS = unsubmittedTokens.get("datasource")
if (
typeof myTechnology != "undefined" &&
myTechnology != null &&
myTechnology.indexOf("#") >= 0
) {
myTechnology = myTechnology.replace(/#.*/, "")
SetTechnology(myTechnology)
return false
}
if (
typeof myDS != "undefined" &&
myDS != null &&
myDS.indexOf("#") >= 0
) {
myDS = myDS.replace(/#.*/, "")
SetDataSource(myDS)
return false
}
var currentURL = window.location.href.replace(/[\?#].*/, "")
if (typeof myDS != "undefined") {
// console.log("Hey, I have a datasource!", validDataSources[myDS])
if (typeof validDataSources[myDS] == "undefined") myDS = "Other"
if (validDataSources[myDS]["technologies"].length == 1) {
myTechnology = validDataSources[myDS]["technologies"][0]
} else {
var ReadyIt = ""
$(".dashboard-header-title").text(myDS)
ReadyIt += descriptionForDataSource(
myDS,
"
Splunk accepts data from any sources you have, and isn't limited to any particular vendors. One of our fundamental technologies is what is called Schema on the Fly (or Schema at Read) which allows us to ingest anything that is text, or can be converted to text, and get the same value out of it without implementation headaches. So while we don't play favorites, to make your life easier, we've included documentation on a some of common data sources that we get asked about the most.
\n"
)
ReadyIt += "\n"
// console.log("Setting Data Source", ReadyIt)
unsubmittedTokens.set("dataSourcesTextRow", "Test")
submittedTokens.set(unsubmittedTokens.toJSON())
$("#dataSourceToTechnology").html(ReadyIt)
$("#introTextRow").hide()
$(".contents-body").hide()
}
currentURL = currentURL + "?datasource=" + myDS
}
if (typeof myTechnology != "undefined") {
// console.log("Hey, I have a technology!", myTechnology)
require([
"components/data/sendTelemetry",
"json!" +
$C["SPLUNKD_PATH"] +
"/servicesNS/nobody/Splunk_Security_Essentials/storage/collections/data/sse_app_config",
], function (Telemetry, appConfig) {
let record = { status: "docLoaded", pageName: myTechnology }
for (let i = 0; i < appConfig.length; i++) {
if (
appConfig[i].param == "demoMode" &&
appConfig[i].value == "true"
) {
record.demoMode = true
}
}
Telemetry.SendTelemetryToSplunk("PageStatus", record)
})
if (typeof validTechnologies[myTechnology] == "undefined")
myTechnology = "Other"
// console.log("Going to load", validTechnologies[myTechnology])
$(".dashboard-header-title").text(myTechnology)
if (
typeof validTechnologies[myTechnology]["last_updated"] !=
"undefined"
) {
$(".dashboard-header-description").text(
"Last Updated: " +
validTechnologies[myTechnology]["last_updated"]
)
}
var myDocs = []
var collectedReferences = new Object()
var currentSection = ""
if (
validTechnologies[myTechnology] &&
validTechnologies[myTechnology]["guide"]
) {
validTechnologies[myTechnology]["guide"]?.forEach(function (
step
) {
allDocItems.forEach(function (guide) {
if (guide["id"] == step) {
if (currentSection == "")
currentSection = guide["category"][0]
if (
currentSection != guide["category"][0] &&
Object.keys(collectedReferences).length > 0
) {
if (
currentSection !=
"Data Source Onboarding Guide Overview"
) {
//id, title, short-description, category
var mystring = ""
for (var reference in collectedReferences) {
if (
typeof collectedReferences[
reference
]["title"] != "undefined" &&
collectedReferences[reference][
"title"
] != ""
) {
mystring +=
'
' +
collectedReferences[reference][
"title"
] +
" "
} else {
mystring +=
'
' +
collectedReferences[reference][
"href"
] +
" "
}
}
var newElement = {
id:
currentSection.replace(
/[^a-zA-Z0-9]/g,
""
) + "_references",
title: currentSection + " References",
"short-description":
"
Here are links from this section:\n
",
category: [currentSection],
}
// console.log("Adding references for", currentSection, newElement)
myDocs.push(newElement)
}
collectedReferences = new Object()
currentSection = guide["category"][0]
}
$(guide["short-description"])
.find("a")
.each(function (num, obj) {
if (
typeof $(obj).attr("href") !=
"undefined" &&
$(obj).attr("href").indexOf("ttp") >=
0 &&
typeof $(obj).attr("noreferences") ==
"undefined"
) {
// console.log("going to add the following link, ", $(obj).attr("href"), $(obj).attr("title"), "to ", collectedReferences)
if (
typeof collectedReferences[
$(obj).attr("href")
] == "undefined"
) {
collectedReferences[
$(obj).attr("href")
] = new Object()
collectedReferences[
$(obj).attr("href")
]["title"] = ""
collectedReferences[
$(obj).attr("href")
]["href"] = $(obj).attr("href") // duplicate I know, but it's prettier.
}
if (
typeof $(obj).attr("title") !=
"undefined" &&
$(obj).attr("title").length >
collectedReferences[
$(obj).attr("href")
]["title"].length
) {
collectedReferences[
$(obj).attr("href")
]["title"] = $(obj).attr("title")
}
}
})
if (guide["title"] != "References")
myDocs.push(guide)
}
})
})
}
if (
currentSection != "Data Source Onboarding Guide Overview" &&
Object.keys(collectedReferences).length > 0
) {
var mystring = ""
for (var reference in collectedReferences) {
if (
typeof collectedReferences[reference]["title"] !=
"undefined" &&
collectedReferences[reference]["title"] != ""
) {
mystring +=
'
' +
collectedReferences[reference]["title"] +
" "
} else {
mystring +=
'
' +
collectedReferences[reference]["href"] +
" "
}
}
var newElement = {
id:
currentSection.replace(/[^a-zA-Z0-9]/g, "") +
"_references",
title: currentSection + " References",
"short-description":
"
Here are links from this section:\n
",
category: [currentSection],
}
// console.log("Adding references for", currentSection, newElement)
myDocs.push(newElement)
}
$("#introTextRow").hide()
//$("#dataSourcesTextRowID").hide()
currentURL = currentURL + "?technology=" + myTechnology
HandleItExternal(myDocs)
}
if (typeof myDS == "undefined" && typeof myTechnology == "undefined") {
$("#introTextRow").show()
$(".dashboard-header-description")
.removeClass("hidden")
.text(
"Splunk can ingest data from any type of product. The following examples show data that has been ingested into Splunk Security Essentials."
)
$("#layout1").css("margin-top", "10px")
var listOfGuides =
"
Below, you will find a detailed list of the Data Sources that commonly make Splunk for Security (and certainly Splunk Security Essentials) tick, along with some of the common products for each.
Among the full list, we have several products where we've built out full data onboarding guides, to help walk you through the process:"
Object.keys(validTechnologies).forEach(function (techName) {
if (
typeof validTechnologies[techName]["guide"] !=
"undefined" &&
validTechnologies[techName]["guide"].length > 0 &&
techName != "Other"
) {
listOfGuides +=
'
\n"
}
})
listOfGuides += ""
var ReadyIt =
listOfGuides +
'
Data Source # of Common Products # of Data Source Guides '
Object.keys(validDataSources).forEach(function (dataSourceName) {
if (dataSourceName != "Other") {
ReadyIt +=
' \n"
ReadyIt +=
"" +
validDataSources[dataSourceName]["technologies"]
.length +
" "
var NumSourcesWithGuides = 0
for (
var i = 0;
i <
validDataSources[dataSourceName]["technologies"].length;
i++
) {
if (
typeof validTechnologies[
validDataSources[dataSourceName][
"technologies"
][i]
] != "undefined" &&
typeof validTechnologies[
validDataSources[dataSourceName][
"technologies"
][i]
]["guide"] != "undefined"
)
NumSourcesWithGuides++
}
ReadyIt += "" + NumSourcesWithGuides + " "
ReadyIt += " "
ReadyIt +=
''
ReadyIt += descriptionForDataSource(dataSourceName)
ReadyIt += " "
}
})
ReadyIt += "
"
$("#introTextRow").find(".panel-body").html(ReadyIt)
$(".contents-body").hide()
$(".expandlink").each(function (a, obj) {
$(obj).click(function () {
$(
"#" + $(obj).attr("id").replace("_link", "") + "_detail"
).toggle()
if (
$(obj)
.find("i")
.attr("class")
.indexOf("icon-chevron-right") >= 0
) {
$(obj).find("i").attr("class", "icon-chevron-down")
} else {
$(obj).find("i").attr("class", "icon-chevron-right")
}
return false
})
})
$(".expandlink").each(function (num, obj) {
var linkobj = $(obj).parent().parent().find(".coredatasource")
linkobj.click(function () {
$(
"#" + $(obj).attr("id").replace("_link", "") + "_detail"
).toggle()
if (
$(obj)
.find("i")
.attr("class")
.indexOf("icon-chevron-right") >= 0
) {
$(obj).find("i").attr("class", "icon-chevron-down")
} else {
$(obj).find("i").attr("class", "icon-chevron-right")
}
return false
})
})
//$("#dataSourcesTextRowID").hide()
//$("#introTextRow").hide()
}
//window.history.pushState({"myDS":myDS,"myTechnology":myTechnology},"", currentURL);
// unsubmittedTokens.set(myDataset.replace(/\W/g,""),"Test");
// submittedTokens.set(unsubmittedTokens.toJSON());
window.myDS = myDS
window.myTechnology = myTechnology
/*HandleIt(exampleLayoutJSONforWindows)*/
})
}
ApplyTokens()
function triggerDocsRead(id) {
// console.log("Opening Trigger")
if (
typeof localStorage["sse-docs-" + id] == "undefined" ||
localStorage["sse-docs-" + id] != "complete"
) {
buttonText =
'
Section Completed'
$("#docsButton-" + id).html(buttonText)
// console.log("Running it", buttonText)
localStorage["sse-docs-" + id] = "complete"
navCheckmark =
'
'
$("#navLink-" + id).prepend(navCheckmark)
} else {
buttonText = "Mark Complete"
$("#docsButton-" + id).html(buttonText)
// console.log("Running it", buttonText)
localStorage["sse-docs-" + id] = "notcomplete"
$("#navLink-" + id).html(
$("#navLink-" + id)
.html()
.replace(/\
]*[^;]*;/, "")
)
}
}
function DoImageSubtitles(numLoops) {
if (typeof numLoops == "undefined") numLoops = 1
var doAnotherLoop = false
// console.log("Starting the Subtitle..")
$(".documentation")
.find("img")
.each(function (count, img) {
if (
typeof $(img).css("width") != "undefined" &&
parseInt($(img).css("width").replace("px")) > 10 &&
typeof $(img).attr("processed") == "undefined"
) {
var width = "width: " + $(img).css("width")
var myTitle = ""
if (
typeof $(img).attr("title") != "undefined" &&
$(img).attr("title") != ""
) {
myTitle =
'
' +
$(img).attr("title") +
" "
}
$(img).attr("processed", "true")
if (
typeof $(img).attr("zoomin") != "undefined" &&
$(img).attr("zoomin") != ""
) {
$(img).replaceWith(
'
"
)
} else {
$(img).replaceWith(
'
' +
img.outerHTML +
myTitle +
"
"
)
}
} else {
doAnotherLoop = true
// console.log("Analyzing image: ", $(img).css("width"), $(img).attr("processed"), $(img))
}
})
if (doAnotherLoop && numLoops < 30) {
numLoops++
setTimeout(function () {
DoImageSubtitles(numLoops)
}, 500)
}
}
var allDocItems = [
{
id: "GI-1-DSOG-overview",
title: "Overview",
"short-description":
"
Welcome to the Splunk Data Source Onboarding Guides (DSOGs)!
Splunk has lots of docs, so why are we creating more? The primary goal of the DSOGs is to provide you with a curated, easy-to-digest view of the most common ways that Splunk users ingest data from our most popular sources, including how to configure the systems that will send us data (such as turning on AWS logging or Windows Security's process-launch logs, for example). While these guides won't cover every single possible option for installation or configuration, they will give you the most common, easiest way forward.
How to use these docs: We've broken the docs out into different segments that get linked together. Many of them will be shared across multiple products. We suggest clicking the \"Mark Complete\" button above to remind yourself of those you've completed. Since this info will be stored locally in your browser, you won't have to worry about it affecting anyone else's view of the document. And when you're reading about ingesting Sysmon logs, for example, it's a convenient way to keep track of the fact that you already installed the forwarder in order to onboard your Windows Security logs.
So, go on and dive right in! And don't forget, Splunk is here to make sure you're successful. Feel free to ask questions of your Sales Engineer or Professional Services Engineer, if you run into trouble. You can also look for answers or post your questions on https://answers.splunk.com/ .
",
category: ["Data Source Onboarding Guide Overview"],
},
{
id: "GI-2-expectationsandscale",
title: "Instruction Expectations and Scaling",
"short-description":
'
Expectations This doc is intended to be an easy guide to onboarding data from Splunk, as opposed to comprehensive set of docs. We\'ve specifically chosen only straightforward technologies to implement here (avoiding ones that have lots of complications), but if at any point you feel like you need more traditional documentation for the deployment or usage of Splunk, Splunk Docs has you covered with over 10,000 pages of docs (let alone other languages!).
Because simpler is almost always better when getting started, we are also not worrying about more complicated capabilities like Search Head Clustering, Indexer Clustering, or anything else of a similar vein. If you do have those requirements, Splunk Docs is a great place to get started, and you can also always avail yourself of Splunk Professional Services so that you don't have to worry about any of the setup.
Scaling While Splunk scales to hundreds or thousands of indexers with ease, we usually have some pretty serious architecture conversation before ordering tons of hardware. That said, these docs aren't just for lab installs. We've found that they will work just fine with most customers in the 5 GB to 500 GB range, even some larger! Regardless of whether you have a single Splunk box doing everything, or a distributed install with a Search Head and a set of Indexers, you should be able to get the data and the value flowing quickly.
There's one important note: the first request we get for orchestration as customers scale, is to distribute configurations across many different universal forwarders. Imagine that you've just vetted out the Windows Process Launch Logs guide on a few test systems, and it's working great. Now you want to deploy it to 500, or 50,000 other Windows boxes. Well, there are a variety of ways to do this:
The standard Splunk answer is to use the Deployment Server. The deployment server is designed for exactly this task, and is free with Splunk. We aren't going to document it here, mostly because it's extremely well documented by our EDU and also docs.splunk.com, here . If you are a decent sized organization, you've probably already got a way to deploy configurations and code, like Puppet, Chef, SCCM, Ansible, etc. All of those tools are used to deploy splunk on a regular basis. Now, you might not want to go down this route if it requires onerous change control, or reliance on other teams, etc. -- many large Splunk environments with well developed software deployment systems prefer to use the Deployment Server because it can be owned by Splunk and is optimized for Splunk's needs. But many customers are very happy with using Puppet to distribute Splunk configurations. Ultimately, Splunk configurations are almost all just text files, so you can distribute the configurations with our packaged software, with your own favorite tools, or even by just copying configuration files around.",
category: ["General Infrastructure"],
},
{
id: "GI-3-indexesAndSourcetypes",
title: "Indexes and Sourcetypes Overview",
"short-description":
'
Overview The DSOGs talk a lot about indexes and sourcetypes. Here\'s a quick overview.
Splexicon (Splunk\'s Lexicon, a glossary of Splunk-specific terms) defines an index as the repository for data in Splunk Enterprise. When Splunk Enterprise indexes raw event data , it transforms the data into searchable events . Indexes are the collections of flat files on the Splunk Enterprise instance. That instance is known as an Indexer because it stores data. Splunk instances that users log into and run searches from are known as Search Heads . When you have a single instance, it takes on both the search head and indexer roles.
"Sourcetype" is defined as a default field that identifies the data structure of an event. A sourcetype determines how Splunk Enterprise formats the data during the indexing process. Example sourcetypes include access_combined and cisco_syslog.
In other words, an index is where we store data, and the sourcetype is a label given to similar types of data. All Windows Security Logs will have a sourcetype of WinEventLog:Security, which means you can always search for source=*wineventlog:security (when searching, the word sourcetype is case sensitive, the value is not).
Why is this important? We\'re going to guide you to use indexes that our professional services organization recommends to customers as an effective starting point. Using standardized sourcetypes (those shared by other customers) makes it much easier to use Splunk and avoid headaches down the road. Splunk will allow you to use any sourcetype you can imagine, which is great for custom log sources, but for common log sources, life is easier sticking with standard sourcetypes. These docs will walk you through standard sourcetypes.
Implementation Below is a sample indexes.conf that will prepare you for all of the data sources we use in these docs. You will note that we separate OS logs from Network logs and Security logs from Application logs. The idea here is to separate them for performance reasons, but also for isolation purposes-you may want to expose the application or system logs to people who shouldn\'t view security logs. Putting them in separate indexes prevents that.
To install this configuration, you should download the app below and put it in the apps directory.
For Windows systems, this will typically be: c:\\Program Files\\Splunk\\etc\\apps. Once you\'ve extracted the app there, you can restart Splunk via the Services Control Panel applet, or by running "c:\\Program Files\\Splunk\\bin\\splunk.exe" restart.
For Linux systems, this will typically be /opt/splunk/etc/apps/. Once you\'ve extracted the app there, you can restart Splunk by running /opt/splunk/bin/splunk restart.
You can view the indexes.conf below, but it\'s easiest to just click Click here to download a Splunk app with this indexes.conf , below.
Splunk Cloud Customers : You won\'t copy the files onto your Splunk servers because you don\'t have access. You could go one-by-one through the UI and create all of the indexes below, but it might be easiest if you download the app, and open a ticket with CloudOps to have it installed.
# Overview. Below you will find the basic indexes.conf settings for\n# setting up your indexes in Splunk. We separate into different indexes \n# to allow for performance (in some cases) or data isolation in others. \n# All indexes come preconfigured with a relatively short retention period \n# that should work for everyone, but if you have more disk space, we \n# encourage (and usually see) longer retention periods, particularly \n# for security customers.\n\n# Endpoint Indexes used for Splunk Security Essentials. \n# If you have the sources, other standard indexes we recommend include:\n# epproxy - Local Proxy Activity\n\n[epav]\ncoldPath = $SPLUNK_DB/epav/colddb\nhomePath = $SPLUNK_DB/epav/db\nthawedPath = $SPLUNK_DB/epav/thaweddb\nfrozenTimePeriodInSecs = 2592000\n#30 days\n\n[epfw]\ncoldPath = $SPLUNK_DB/epnet/colddb\nhomePath = $SPLUNK_DB/epnet/db\nthawedPath = $SPLUNK_DB/epnet/thaweddb\nfrozenTimePeriodInSecs = 2592000\n#30 days\n\n[ephids]\ncoldPath = $SPLUNK_DB/epmon/colddb\nhomePath = $SPLUNK_DB/epmon/db\nthawedPath = $SPLUNK_DB/epmon/thaweddb\nfrozenTimePeriodInSecs = 2592000\n#30 days\n\n[epintel]\ncoldPath = $SPLUNK_DB/epweb/colddb\nhomePath = $SPLUNK_DB/epweb/db\nthawedPath = $SPLUNK_DB/epweb/thaweddb\nfrozenTimePeriodInSecs = 2592000\n#30 days\n\n[oswin]\ncoldPath = $SPLUNK_DB/oswin/colddb\nhomePath = $SPLUNK_DB/oswin/db\nthawedPath = $SPLUNK_DB/oswin/thaweddb\nfrozenTimePeriodInSecs = 2592000\n#30 days\n\n[oswinsec]\ncoldPath = $SPLUNK_DB/oswinsec/colddb\nhomePath = $SPLUNK_DB/oswinsec/db\nthawedPath = $SPLUNK_DB/oswinsec/thaweddb\nfrozenTimePeriodInSecs = 2592000\n#30 days\n\n[oswinscript]\ncoldPath = $SPLUNK_DB/oswinscript/colddb\nhomePath = $SPLUNK_DB/oswinscript/db\nthawedPath = $SPLUNK_DB/oswinscript/thaweddb\nfrozenTimePeriodInSecs = 2592000\n#30 days\n\n[oswinperf]\ncoldPath = $SPLUNK_DB/oswinperf/colddb\nhomePath = $SPLUNK_DB/oswinperf/db\nthawedPath = $SPLUNK_DB/oswinperf/thaweddb\nfrozenTimePeriodInSecs = 604800 \n#7 days\n\n[osnix]\ncoldPath = $SPLUNK_DB/osnix/colddb\nhomePath = $SPLUNK_DB/osnix/db\nthawedPath = $SPLUNK_DB/osnix/thaweddb\nfrozenTimePeriodInSecs = 2592000\n#30 days\n\n[osnixsec]\ncoldPath = $SPLUNK_DB/osnixsec/colddb\nhomePath = $SPLUNK_DB/osnixsec/db\nthawedPath = $SPLUNK_DB/osnixsec/thaweddb\nfrozenTimePeriodInSecs = 2592000\n#30 days\n\n[osnixscript]\ncoldPath = $SPLUNK_DB/osnixscript/colddb\nhomePath = $SPLUNK_DB/osnixscript/db\nthawedPath = $SPLUNK_DB/osnixscript/thaweddb\nfrozenTimePeriodInSecs = 2592000\n#30 days\n\n[osnixperf]\ncoldPath = $SPLUNK_DB/osnixperf/colddb\nhomePath = $SPLUNK_DB/osnixperf/db\nthawedPath = $SPLUNK_DB/osnixperf/thaweddb\nfrozenTimePeriodInSecs = 604800 \n#7 days\n\n# Network Indexes used for Splunk Security Essentials\n# If you have the sources, other standard indexes we recommend include:\n# netauth - for network authentication sources\n# netflow - for netflow data\n# netids - for dedicated IPS environments\n# netipam - for IPAM systems\n# netnlb - for non-web server load balancer data (e.g., DNS, SMTP, SIP, etc.)\n# netops - for general network system data (such as Cisco iOS non-netflow logs)\n# netvuln - for Network Vulnerability Data\n\n[netdns]\ncoldPath = $SPLUNK_DB/netdns/colddb\nhomePath = $SPLUNK_DB/netdns/db\nthawedPath = $SPLUNK_DB/netdns/thaweddb\nfrozenTimePeriodInSecs = 2592000\n#30 days\n\n[mail]\ncoldPath = $SPLUNK_DB/mail/colddb\nhomePath = $SPLUNK_DB/mail/db\nthawedPath = $SPLUNK_DB/mail/thaweddb\nfrozenTimePeriodInSecs = 2592000\n#30 days\n\n[netfw]\ncoldPath = $SPLUNK_DB/netfw/colddb\nhomePath = $SPLUNK_DB/netfw/db\nthawedPath = $SPLUNK_DB/netfw/thaweddb\nfrozenTimePeriodInSecs = 2592000\n#30 days\n\n[netops]\ncoldPath = $SPLUNK_DB/netops/colddb\nhomePath = $SPLUNK_DB/netops/db\nthawedPath = $SPLUNK_DB/netops/thaweddb\nfrozenTimePeriodInSecs = 2592000\n#30 days\n\n[netproxy]\ncoldPath = $SPLUNK_DB/netproxy/colddb\nhomePath = $SPLUNK_DB/netproxy/db\nthawedPath = $SPLUNK_DB/netproxy/thaweddb\nfrozenTimePeriodInSecs = 2592000\n#30 days\n\n[netvpn]\ncoldPath = $SPLUNK_DB/netvpn/colddb\nhomePath = $SPLUNK_DB/netvpn/db\nthawedPath = $SPLUNK_DB/netvpn/thaweddb\nfrozenTimePeriodInSecs = 2592000\n#30 days\n\n\n# Splunk Security Essentials doesn\'t have examples of Application Security, \n# but if you want to ingest those logs, here are the recommended indexes:\n# appwebint - Internal WebApp Access Logs\n# appwebext - External WebApp Access Logs\n# appwebintrp - Internal-facing Web App Load Balancers\n# appwebextrp - External-facing Web App Load Balancers\n# appwebcdn - CDN logs for your website\n# appdbserver - Database Servers\n# appmsgserver - Messaging Servers\n# appint - App Servers for internal-facing apps \n# appext - App Servers for external-facing apps Validation Once this is complete, you will be able to find the list of indexes that the system is aware of by logging into Splunk, and going into Settings -> Indexes.
',
category: ["General Infrastructure"],
},
{
id: "GI-4-forwarderOnWindows",
title: "Forwarder on Windows Systems",
"short-description":
'
Overview \nInstalling the Windows forwarder is a straightforward process, similar to installing any Windows program. These instructions will walk you through a manual instruction for getting started (perfect for a lab, a few laptops, or when you\'re just getting started on domain controllers). \n
Implementation \n
Note for larger environments: When you want to automatically roll out the forwarder to hundreds (or thousands or hundreds of thousands) of systems, you will want to leverage your traditional software-deployment techniques. The Splunk forwarder is an MSI package and we have docs on recommended ways to deploy it:
\n
\n
Of course, you can also deploy it with traditional system-configuration management software. This can vary a lot from environment to environment. For this doc we\'ll just walk you through the installation so that you know what\'s coming.
\n
The first thing to do is download the Universal Forwarder from Splunk\'s website (https://www.splunk.com/en_us/download/universal-forwarder.html ). This is a separate download from the main Splunk installer, as the universal forwarder is lightweight, so it can be installed on all of the systems in your environment. Most users today will download the x64 version as an MSI installer.
\n
When you double click the downloaded file, the standard MSI installer will appear.
\n
Don\'t worry about the Cloud checkbox -- we will use the same settings for both.
\n
While you can click "Customize Settings" here and manually insert the address of your indexers or manually choose the log sources you would like to index, etc., we generally don\'t recommend that, unless you\'re never going to move beyond the one source you\'re looking at. (Harder to go find those settings and then apply them to other systems.) Ignore "Customize Options" and click on "Next."\nThe setup will now go through its process, and you\'ll be finished with a freshly installed forwarder. There are three more steps you\'ll want to take before you can see the data in Splunk:
\n
You will need an outputs.conf to tell the forwarder where to send data (next section) \nYou will need an inputs.conf to tell the forwarder what data to send (below, in the Splunk Configuration for Data Source) \nYou will need an indexes.conf on the indexers to tell them where to put the data that is received (Previous section) \n
Validation \n
You can now check Task Manager and you should see Splunk running. Alternatively, check under Services in the Control Panel. You will see Splunk listed and started.
\n\n\n',
category: ["General Infrastructure"],
},
{
id: "GI-5-outputsconf",
title: "Sending Data from Forwarders to Indexers",
"short-description":
'
Overview \n
For any Splunk system in the environment, whether it\'s a Universal Forwarder on a Windows host, a Linux Heavy-Weight Forwarder pulling the more difficult AWS logs, or even a dedicated Search Head that dispatches searches to your indexers, every system in the environment that is not an indexers (i.e., any system that doesn\'t store its data locally) should have an outputs.conf that points to your indexers.
\n
Implementation \n
Fortunately the outputs.conf will be the same across the entire environment, and is fairly simple. There are three steps:
\n
Create the app using the button below (SplunkCloud customers: use the app you received from SplunkCloud). \nExtract the file (it will download a zip file). \nPlace in the etc/apps directory. \n\n\n
For Windows systems, this will typically be: c:\\Program Files\\Splunk\\etc\\apps. Once you\'ve extracted the app there, you can restart Splunk via the Services Control Panel applet, or by running "c:\\Program Files\\Splunk\\bin\\splunk.exe" restart.
\n\n
For Linux systems, this will typically be /opt/splunkforwarder/etc/apps/. Once you\'ve extracted the app there, you can restart Splunk by running /opt/splunk/bin/splunk restart.
\n\n
For customers not using SplunkCloud:
\n[tcpout]\ndefaultGroup = default-autolb-group\n\n[tcpout:default-autolb-group]\nserver = MySplunkServer.mycompany.local:9997\n\n[tcpout-server://MySplunkServer.mycompany.local:9997] \n
\n\n
Validation \n
Run a search in the Splunk environment for the host you\'ve installed the forwarder on. E.g., index=* host=mywinsystem1*
\n
You can also review all hosts that are sending data from via | metadata index=* type=hosts
\n\n',
category: ["General Infrastructure"],
},
{
id: "SC-WinSecurity-1-SizingEstimate",
title: "Sizing Estimate",
"short-description":
'
Windows event volume can vary greatly based on the type of host. At a very high level, common ranges we’ve seen are:
\n
Workstation: 4-6 MB/day (Including Application, System, and Security Logs) \nApplication Servers: 25-50 MB/day \nDomain Controllers: 50-500 MB/day depending on the number of users \n\n
Obviously, these ranges can vary dramatically. For high horsepower AD controllers with thousands of simultaneous users, you could see more volume.
\n
A common follow-on question we often get is what the expected volume is just for the Process Launch Logs (Event ID 4688) – this can vary based on how many new processes spin up, of course, but it is usually a rounding error on event volume. It’s often said that Event ID 4688 is the best bang for the buck in all of security logging!
The last question we get when having these discussions is: "what first?" If you have a formal risk assessment process, it\'s always best to there, but generally speaking we see most customers start with Domain Controllers as they have the most sensitive information, move on to member servers, and reach the desktops last.
',
category: ["Splunk Configuration for Data Source"],
},
{
id: "SC-WinSecurity-2-InstallTA",
title: "Install the Technology Add-On -- TA",
"short-description":
'
Overview \n
Splunk has a detailed Technology Add-on that supports ingesting all manner of Windows logs. Like all Splunk Technology Add-ons, it also includes everything needed in order to parse out the fields, and give them names that are compliant with Splunk’s Common Information Model, so they can easily be used by the searches in Splunk Security Essentials, along with searches you will find in other community supported and premium apps.
\n
Implementation \n
Find the TA along with all your other Splunk apps / needs on SplunkBase. You could go to https://splunkbase.splunk.com/ and search for it, or you could just follow the direct link here: https://splunkbase.splunk.com/app/742/ .
\n\n
As with all Splunk TAs, we recommend you deploy it to all parts of your Splunk environment for simplicity and uniformity, so plan to install the TA on your Search Head, Indexers, and any Windows Forwarders in your environment.
To install the app, start by downloading the file from the SplunkBase just shown, and then extract it. Note: The app itself is a tgz file, or a gzipped tarball. If you’re a pure Windows environment, this means that you will need a third party program to extract it – fortunately tgz is the most common format in the world behind zip, so virtually any extraction program you have (WinZip, 7z, WinRAR, etc.) will all extract it. Once you have the extracted folder, move it into %SPLUNK_HOME%/etc/apps/ folder. For most modern Splunk environments, that will be C:\\Program Files\\SplunkUniversalForwarder\\etc\\apps. Once you’ve extracted the app, you can restart Splunk via the Services Control Panel applet, or by just running "c:\\Program Files\\SplunkUniversalForwarder\\bin\\splunk.exe" restart. \n\n
Validation \n
You can make sure that Splunk has picked up the presence of the app by running: "c:\\Program Files\\SplunkUniversalForwarder\\bin\\splunk.exe" display app, which after asking you to log in, will provide you with a list of installed apps. Usually though if you see the folder listed alongside the other apps (learned, search, splunk_httpinput, etc.) you will know that it’s there successfully.
Splunk Cloud Customers : you won\'t be copying any files or folders to your indexers or search heads, but good news! The Splunk Add-on for Microsoft Windows is Cloud Self-Service Enabled. So you can just got to Find Apps, and be up and running in seconds.
\n\n\n',
category: ["Splunk Configuration for Data Source"],
},
{
id: "SC-WinSecurity-3-IndexesAndSourcetypes",
title: "General Windows Indexes and Sourcetypes",
"short-description":
'
Overview \n
Amongst Splunk’s 15000+ customers, we’ve done a lot of implementations, and we’ve learned a few things along the way. While you can use any sourcetypes or indexes that you want in the land of Splunk, we’ve found that the most successful customers follow specific patterns, as it sets them up for success moving forward.
\n
Implementation \n
The most common Windows data types are the Security Log, System Log, and Application Log, but there are a few others as well including Microsoft Sysmon. Here are our most commonly used Windows Data Types and the recommended indexes and sourcetypes.
\n
Data Type Input (inputs.conf, below) Source Index Notes \nWindows Security Logs WinEventLog://Security wineventlog:security oswinsec We leverage a blacklist for common “noise” events, below. \nWindows Application Logs WinEventLog://Application wineventlog:application oswin \nWindows System Logs WinEventLog://System wineventlog:system oswin \nWindows Update Log monitor://$WINDIR\\WindowsUpdate.log WindowsUpdateLog oswinsec \nMicrosoft Sysmon Logs WinEventLog://Microsoft-Windows-Sysmon/Operational XmlWinEventLog:Microsoft-Windows-Sysmon/Operational epintel Based on the sysmon sysinternals tool, not out of the box. \n
\n
If you have already started ingesting the data sources into another index, then you can usually proceed (though consider if you should separate Windows Security logs from Process Launch Logs and both from Application and System logs, based on who likely will need access or be prohibited access). If you have already started ingesting data with a different sourcetype, we would recommend you switch over to the standardized sourcetypes if at all possible. If you\'re not using the Splunk TA for Windows to ingest data, then keep in mind you may need to go through extra work to align field names to get value out of Splunk Security Essentials, and other Splunk content.
\n
To support your Windows sources, follow the procedure mentioned above in General Infrastructure - Indexes and Sourcetypes to add the new indexes for the data you will be bringing in (generally it’s easiest if you just create oswin, oswinsec, epintel).
\n
For the sourcetypes and monitor statements, we will show those next in the Configuration Files.\n',
category: ["Splunk Configuration for Data Source"],
},
{
id: "SC-WinSecurity-4-inputsconf",
title: "Configuration Files",
"short-description":
'
Overview \n
Configuration files for Windows inputs tends to be pretty simple. In this case, we just have a single inputs.conf file that will go on the Windows Hosts you will be monitoring. As detailed above in Instruction Expectations and Scaling , you will need some mechanism to distribute these files to the hosts you’re monitoring. For initial tests, or deployments to just your most sensitive systems, it is easy to copy the files to the hosts. For larger distributions, you can use the Splunk Deployment Server, or use another code distribution system such as SCCM, Puppet, Chef, Ansible, or others.
\n
Implementation \n
Distribute the below inputs.conf file to all of the hosts that you will be monitoring in the %SPLUNK_HOME%/etc/apps/Splunk_TA_windows/local folder. If the folder doesn’t exist, you will need to create it. For most customers, the path to this file will end up being: C:/Program Files/SplunkUniversalForwarder/etc/apps/Splunk_TA_windows/local/inputs.conf.
\n
\n[WinEventLog://Security]\ndisabled = 0\nevt_resolve_ad_obj = 1\ncheckpointInterval = 5\nblacklist1 = EventCode="4662" Message="Object Type:\\s+(?!groupPolicyContainer)"\nblacklist2 = EventCode="566" Message="Object Type:\\s+(?!groupPolicyContainer)"\nblacklist3 = EventCode="4688" Message="New Process Name: (?i)(?:[C-F]:\\Program Files\\Splunk(?:UniversalForwarder)?\\bin\\(?:btool|splunkd|splunk|splunk-(?:MonitorNoHandle|admon|netmon|perfmon|powershell|regmon|winevtlog|winhostinfo|winprintmon|wmi)).exe)"\nindex = oswinsec\n\n[WinEventLog://Application]\ndisabled = 0\ncheckpointInterval = 5\nindex = oswin\n\n[WinEventLog://System]\ndisabled = 0\ncheckpointInterval = 5\nindex = oswin\n\n[monitor://$WINDIR\\WindowsUpdate.log]\ndisabled = 0\nsourcetype = WindowsUpdateLog\nindex = oswinsec\n\n[WinHostMon://Service]\ninterval = 3600\ndisabled = 0\ntype = Service\nindex = oswinscript\n \n',
category: ["Splunk Configuration for Data Source"],
},
{
id: "VC-WinSecurity-1-EnablingLog",
title: "Enabling Windows Security Log",
"short-description":
'
Overview \n
To maintain a good Security Posture, and to leverage the examples provided in Splunk Security Essentials, we recommend following Microsoft’s official guidance for “Stronger” security visibility. The Audit Policy Recommendations page from Microsoft TechNet provides very detailed configuration settings per operating system from Windows 7 / Server 2008 and up here: https://docs.microsoft.com/en-us/windows-server/identity/ad-ds/plan/security-best-practices/audit-policy-recommendations \n\n
Implementation \n\n
Important Note: Splunk is a monitoring product, and not an Active Directory system, so while we’re working hard to centralize some of the recommendations you should follow in one place to make your life easy, we cannot offer support for the actual configuration of anything other than Splunk itself, and strongly recommend that you leverage trained Microsoft resources when making any changes. That’s in large part why we’re pointing you to Microsoft docs for the nitty gritty details!
\n\n
If you are new to configuring auditing on Microsoft systems, there are two primary ways in which you can go about configuring auditing: a one-off (typically lab) system via the Local Security Policy, or a managed system via Group Policy. Virtually all Splunk customers will configure their Windows audit logging via Group Policy, but you absolutely can use Local Security Policy if you only have a small number of machines, or you are trialing on a few systems.
\n\n
If you do want to configure via Local Security Policy, you can click Start (or highlight Cortana Search) and then type in “Local Security Policy” to open the policy editor. Finding the right configuration settings is straightforward, just expand “System Audit Policies – Local Group Policy” at the bottom of the list, and then the next item with the same name. If you compare these items to the link above (also included under “References”), you will find that they map directly and you can proceed to mirror what Microsoft recommends – use the Stronger column for adequate security visibility.
\n\n
To configure via Group Policy, you should open the Group Policy Editor for a group policy that covers any computer accounts that are in scope for monitoring. Most medium to large organizations that we work have a separate admin or group for configuring these types of Group Policy Settings, so it’s usually easiest to send the quoted paragraph below over to that group to apply the settings.
\n\n
If you do manage Group Policy as well, here’s how you can make the changes on your own – note that Microsoft has different recommendations for servers versus workstations, so if possible it’s best to apply separate policies to each (when in doubt, we usually opt for more visibility):
\nOpen the Group Policy Manager by opening the Microsoft Management Console (mmc.exe) Add the Group Policy Management snap-in (File -> Add) \nChoose or create a policy that is applied to your in-scope systems, and right-click to Edit the policy. \nFind the policy settings that match the Microsoft Document under: Computer Configuration -> Policies -> Windows Settings -> Security Settings -> Advanced Audit Policy Configuration -> Audit Policy \nGo through the Microsoft Doc to implement their recommendations for the stronger audit policy. \nYou might have noticed a warning about one other key that you need to have set in order for these audit policies to take effect – don’t worry, it’s been default to on since Vista, but you might as well configure it as well, under Configuration -> Policies -> Windows Settings -> Security Settings -> Local Settings -> Audit: Force audit policy subcategory setting (Windows Vista and later) to override Audit Policy Category Settings \n \nSettings take effect as soon as the systems refresh their global policy, which is usually within 15-60 minutes depending on your environment.\n\n
If you can’t (or don’t want to) run through the above on your own, here’s a paragraph you can just send to whoever in your organization manages Active Directory.
\n
To conform to the Microsoft recommendations for strong security auditing, we would like to have the following changes made to Group Policy objects that cover our servers and our workstations. Microsoft has made separate recommendations for workstation configurations, and for server configurations. The settings are:
\nTurn on the Advanced Audit Policy (defaults to enabled) at:\nConfiguration -> Policies -> Windows Settings -> Security Settings -> Local Settings -> Audit: Force audit policy subcategory setting (Windows Vista and later) to override Audit Policy Category Settings \nFind the Advanced Audit Policy Configuration at:\nComputer Configuration -> Policies -> Windows Settings -> Security Settings -> Advanced Audit Policy Configuration -> Audit Policy \nWalk through the Microsoft “Stronger” column to apply the settings: https://docs.microsoft.com/en-us/windows-server/identity/ad-ds/plan/security-best-practices/audit-policy-recommendations \n
Thank you for your help!
\n\n
Validation \n
Usually the first thing people will see when deploying audit policies is either new systems showing up in Splunk, or at least an increase in system log messages. If you already have some logs coming in and want to validate that you’re getting the new ones, look for the delta between your old policy and your new one, and google “Windows Event ID” – that will usually give you something specific to search for (though you may have to go take the action that gets logged, if it’s less common). An easy example is “Windows Process Creation Event ID” which quickly nets you “Event ID 4688” as the first result.
',
category: ["Windows Configuration"],
},
{
id: "VC-WinSecurity-2-turnon4688",
title: "Turning On Process Launch Logs, Event 4688",
"short-description":
"
Overview \n
Fortunately, if you followed through the steps above, you already have Process Logging turned on! If you already had an audit policy configured and skipped the above, give it another read through – we recommend that people follow the official Microsoft Recommendation for a “Stronger” audit policy which turns on many things that would likely be valuable to you, without being too much.
\n\n
Implementation \n
Important Note: Splunk is a monitoring product, and not an Active Directory system, so while we’re working hard to centralize some of the recommendations you should follow in one place to make your life easy, we cannot offer support for the actual configuration of anything other than Splunk itself, and strongly recommend that you leverage trained Microsoft resources when making any changes. That’s in large part why we’re pointing you to Microsoft docs for the nitty gritty details!
\n\n
If you are persistent and want to go your own road though, we are happy to point you in the right path. You will use the same Local Security Policy or Group Policy configuration that is referenced above, but there are two different places where you might configure this setting.
\n
If you are using the older Windows Audit Policies defined at the category level, you can turn Process Logging in either of the following locations:
\n
\nLocal Security Policy: Local Policies -> Audit Policy -> Audit Process Tracking (Success Only) \nGroup Policy: Computer Configuration -> Policies -> Windows Settings -> Security Settings -> Local Policies -> Audit Policy -> Audit Process Tracking (Success Only) \n \n
Settings take effect as soon as the systems refresh their global policy, which is usually within 15-60 minutes depending on your environment.
\n
If you are using the newer Advanced Audit Policies defined at the subcategory level, you can turn on Process Logging in either of the following locations:
\n
Local Security Policy: System Audit Policies – Local Group Policy -> System Audit Policies – Local Group Policy -> Detailed Tracking -> Audit Process Creation (Success and Failure) \nGroup Policy: Computer Configuration -> Policies -> Windows Settings -> Security Settings -> Advanced Audit Policy Configuration -> Audit Policy -> Detailed Tracking -> Audit Process Creation (Success and Failure) \n\n
Validation \n
Validation for this step is simple. Open a program, such as Outlook and then search for the filename in Splunk: outlook.exe.
",
category: ["Windows Configuration"],
},
{
id: "VC-WinSecurity-3-CommandLineAuditing",
title: "(Optional) Turn on Command Line Auditing - Part One",
"short-description":
'
Overview \n
So far we’ve turned on Process Launch logging, which will tell you what processes a user has run. This is a great start, and provides fantastic visibility! You’ll now know every time that a user launches a PDF Reader, or some other software that could be exploited, and you’ll know about suspicious process launches that could be malware!
\n
But how great would it be if you could not only know that a PDF Reader was launched, but know what file it actually opened? (You’d probably respond differently to something in the Outlook temp directory versus something on a corporate file share, right?) And what if rather than just knowing that cmd.exe was launched, you could actually know what commands were passed to it? Fortunately Command Line Auditing gets us all that information, with almost no extra log volume.
\n
Implementation \n
Things get a bit more complicated here, and vary based on what version of Windows is used in your environment, but it’s not too bad. You’ll start by wanting to apply one more group policy setting (or Registry Key, for those on individual machines):
\n
Computer Configurations -> Policies -> Administrative Templates -> System -> Audit Process Configuration -> Include command line in process creation events \n
Here is the Microsoft Doc so you’ll know we’re not making this up: https://support.microsoft.com/en-us/help/3004375/microsoft-security-advisory-update-to-improve-windows-command-line-aud
\n
If you are on an individual machine, there is a registry key that makes the same change:
\n
Go under: HKEY_LOCAL_MACHIN\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\policies\\system\\Audit \nCreate a new REG_DWORD called ProcessCreationIncludeCmdLine_Enabled \nSet the value to 1 \n\n
It’s mostly documented in Digital Forensics and Windows Logging Cheatsheet sites, though you can find it here at Microsoft’s Azure Security site: https://docs.microsoft.com/sl-si/azure/security-center/security-center-alert-validation
\n
Verification \n
Verification here is also very simple. Search through your Splunk logs for source=*WinEventLog:Security EventCode=4688 and look for the CLI parameters.
\n\n',
category: ["Windows Configuration"],
},
{
id: "VC-WinSecurity-4-MaybePatch",
title: "(Optional) Turn on Command Line Auditing - Part Two",
"short-description":
'
There’s one final step that you will need to take if you have machines that are older than Windows 10 / Server 2016 (you probably do, right?).
\n
The same KB Article we linked to above is tied to a patch, that needs to be applied to your systems in order to get the command line auditing. This patch came out in Feb 10, 2015, and was bundled in a critical Security update, so it is likely already installed in your environment. However, if it is not there, then you may need to distribute this patch via your normal Microsoft Software Update approaches.
\n
Here are the full details: https://support.microsoft.com/en-us/help/3004375/microsoft-security-advisory-update-to-improve-windows-command-line-aud
',
category: ["Windows Configuration"],
},
{
id: "VC-WinSecurity-5-References",
title: "References",
"short-description":
'
\n',
category: ["Windows Configuration"],
},
{
id: "GI-4-forwarderOnLinux",
title: "Forwarder on Linux Systems",
"short-description":
'
Overview \n
Installing the Windows forwarder is a straightforward process, similar to installing any Linux program. These instructions will walk you through a manual instruction for getting started (perfect for a lab, a few laptops, or when you\'re just getting started on domain controllers). You will have three options for how to proceed -- using an RPM package (easiest for any Red Hat or similar system with rpm), using a DEB package (easiest for any Ubuntu or similiar system with dpkg), or using just the compressed .tgz file (will work across Linux platforms).
\n
Note: For full and latest information on installing a forwarder, please follow the instructions in the Linux installation manual: \n http://docs.splunk.com/Documentation/Forwarder/latest/Forwarder/Installanixuniversalforwarder
\n
Implementation \n
Prerequisites \n
You will need to have elevated permissions to install the software and configure correctly \n \n \n
Installation using an RPM file: \n
Make sure you have downloaded the universal forwarder package from Splunk’s website: https://www.splunk.com/en_us/download/universal-forwarder.html and have it on the system you want to install Splunk on.
\n
Run: rpm -i splunkforwarder<version>.rpm
\n
This will install the Splunk forwarder into the default directory of /opt/splunkforwarder
\n
To enable Splunk to run each time your server is restarted use the following command: \n /opt/splunkforwarder/bin/splunk enable boot-start
\n\n\n
Installation using an DEB file: \n
Make sure you have downloaded the universal forwarder package from Splunk’s website: https://www.splunk.com/en_us/download/universal-forwarder.html and have it on the system on which you want to install Splunk.\n
\n
Run: dpkg -i splunkforwarder<version>.rpm
\n
This will install the Splunk forwarder into the default directory of /opt/splunkforwarder
\n
To enable Splunk to run each time your server is restarted use the following command: \n /opt/splunkforwarder/bin/splunk enable boot-start
\n
Installation using the .tgz file:
\n
Make sure you have copied the tarball (or appropriate package for your system) and extract or install it into the /opt directory.
\n
Run: tar zxvf <splunk_tarball_file.tgz> -C /opt\n\n
[root@ip-172-31-94-210 ~]# tar zxvf splunkforwarder-7.0.1-2b5b15c4ee89-Linux-x86_64.tgz -C /opt\nsplunkforwarder/\nsplunkforwarder/etc/\nsplunkforwarder/etc/deployment-apps/\nsplunkforwarder/etc/deployment-apps/README\nsplunkforwarder/etc/apps/ \n\n
Check your extraction:
\n
Run: ls -l /opt
\n
[root@ip-172-31-94-210 apps]# ls -l /opt\ntotal 8\ndrwxr-xr-x 8 splunk splunk 4096 Nov 29 20:21 splunkforwarder \n
If you would like Splunk to run at startup then execute the following command \n /opt/splunkforwarder/bin/splunk enable boot-start
\n
Wrap Up \n
After following any of the above three options, you will have a fully installed Splunk forwarder. There are three more steps you’ll want to take before you can see the data in Splunk:
\n
\n You will need an outputs.conf to tell the forwarder where to send data (next section) \n You will need an inputs.conf to tell the forwarder what data to send (below, in the "Splunk Configuration for Data Source") \n You will need an indexes.conf on the indexers to tell them where to put the data received. (You just passed that section.) \n \n',
category: ["General Infrastructure"],
},
{
id: "SC-LinuxSecurity-1-SizingEstimate",
title: "Sizing Estimate",
"short-description":
"
Linux event volume can vary greatly based on the type of host. At a very high level, common ranges we’ve seen are:\n
Workstation: 4-6 MB/day \nApplication Servers: 25-50 MB/day \n
Obviously, these ranges can vary dramatically. For high horsepower Linux servers with thousands of simultaneous users, you may see more dramatically volume.
\n
A common follow-on question we often get is regarding the expected volume for the Process Launch Logs alone, pulled from auditd. This can vary based on how many new processes spin up, of course, but it is usually a rounding error on event volume. It’s often said that process auditing is the best bang for the buck in all of security logging!
\n \n
The last question we get when having these discussions is: "What first?" If you have a formal risk-assessment process or information-security audit policy or standards documentation, it's always best to start there. However, we generally see most customers start with servers that contain the most sensitive information and then move on from there. Importantly, remember that some organizations also run Linux desktops too, so don’t miss those.
",
category: ["Splunk Configuration for Data Source"],
},
{
id: "SC-LinuxSecurity-2-InstallTA",
title: "Install the Technology Add-On -- TA",
"short-description":
'
Overview \n
Splunk has a detailed technology add-on (Splunk add-on for Unix and Linux) that supports ingesting all manner of Linux logs. Like all Splunk technology add-ons, it also includes everything needed in order to parse out the fields and give them names that are compliant with Splunk’s Common Information Model (Common Information Model Overview ), so they can easily be used by the searches in Splunk Security Essentials. It also includes searches you will find in other community-supported and premium apps.
\n\n
Implementation \n
Find the TA along with all your other Splunk apps/needs on SplunkBase. You can go to https://splunkbase.splunk.com/ and search for it or follow the direct link here: https://splunkbase.splunk.com/app/833/ . \n As with all Splunk TAs, we recommend you deploy it to all parts of your Splunk environment for simplicity and uniformity. So plan to install the TA on your search head, indexers, and any Linux forwarders in your environment.
\n
To install the app, start by downloading the file from the SplunkBase mentioned above and extract it. \n Note: The app itself is a .tgz file, or a gzipped tarball. Fortunately, Linux systems make extraction easy, using the tar command tar (zxvf <filename.tgz>) installed on most distributions. \n Once you have the extracted folder, move it into $SPLUNK_HOME/etc/apps/ folder. If you follow the default installation path, this will be in /opt/splunkforwarder/etc/apps \n
Here is an example of the Linux TA extracted into the proper location (note the path) on a Linux universal forwarder.
\n
[root@ip-172-31-94-210 apps]# cd /opt/splunkforwarder/etc/apps/\n[root@ip-172-31-94-210 apps]# ls -l \ntotal 24\ndrwxr-xr-x 4 root root 4096 Nov 29 20:14 introspection_generator_addon\ndrwxr-xr-x 4 root root 4096 Nov 29 20:14 learned \ndrwxr-xr-x 4 root root 4096 Nov 29 20:14 search \ndrwxr-xr-x 3 root root 4096 Nov 29 20:14 splunk_httpinput \ndrwxr-xr-x 9 root root 4096 Jan 25 13:34 Splunk_TA_nix \ndrwxr-xr-x 4 root root 4096 Nov 29 20:14 SplunkUniversalForwarder \n
Once you’ve extracted the app, you can restart Splunk using the command $SPLUNK_HOME/bin/splunk restart ($SPLUNK_HOME is /opt/splunkforwarder if using our default location).
\n
Validation \n
You can make sure that Splunk has picked up the presence of the app by running $SPLUNK_HOME/bin/splunk display app, will, after asking you to log in, provide you with a list of installed apps. Usually if you see the folder listed alongside the other apps (learned, search, splunk_httpinput, etc.) you will know that it’s there successfully.
Splunk Cloud Customers : you won\'t be copying any files or folders to your indexers or search heads, but good news! The Splunk Add-on for Unix and Linux is Cloud Self-Service Enabled. So you can just got to Find Apps, and be up and running in seconds.
\n',
category: ["Splunk Configuration for Data Source"],
},
{
id: "SC-LinuxSecurity-3-IndexesAndSourcetypes",
title: "Linux Indexes and Sourcetypes",
"short-description":
'
Overview \n
Amongst Splunk’s 15,000+ customers, we’ve done a lot of implementations, and we’ve learned a few things along the way. While you can use any sourcetypes or indexes that you want in the "land of Splunk," we’ve found that the most successful customers follow specific patterns, as it sets them up for success moving forward.
\n
Implementation \n
The most common Linux data types are listed below, along with the recommended indexes and sourcetypes. Other inputs may be included in your configuration by enabling them in the local/inputs.conf (as described in "Configuration Files" section below.
\n
\n \n Data Type
\n Input (inputs.conf, below)
\n Sourcetype
\n Index
\n Notes
\n \n \n Running processes
\n script://./bin/ps.sh
\n ps
\n osnixscript
\n Scripted input, for running processes sampled every 30s
\n \n \n Network Ports
\n script://./bin/netstat.sh
\n netstat
\n osnixscript
\n Network(s) port status sampled every 60s
\n \n \n Open Files
\n script://./bin/lsof.sh
\n lsof
\n osnixperf
\n Open files to process ID map sampled every 10m
\n \n \n Audited Events
\n script://./bin/rlog.sh
\n auditd
\n osnixsec
\n System events (adds / moves and changes)/privilege escalation, etc.
\n \n \n System Log Directory
\n monitor:///var/log
\n syslog
\n osnix
\n DHCP leases, scheduled tasks, service information, etc.
\n \n \n
\n monitor:///var/log/secure
\n syslog
\n osnixsec
\n
\n \n \n System Connections
\n monitor:///var/log/auth.log
\n Syslog
\n osnixsec
\n Multiple protocols (sshd, logind, cron, sudo, etc.)
\n \n \n Command History
\n monitor:///root/.bash_history \n monitor:///home/.../.bash_history
\n bash_history
\n osnixbash
\n All commands typed in the bash shell
\n \n \n
\n
\n
\n
\n
\n \n
\n
If you have already started ingesting the data sources into another index, then you can usually proceed (though consider if you should separate logs, based on who likely will need access or be prohibited access). If you have already started ingesting data with a different sourcetype, we recommend you switch over to the standardized sourcetypes, if possible. If you\'re not using the Splunk TA for Linux to ingest data, keep in mind you may need to go through extra work to align field names to get value out of Splunk Security Essentials and other Splunk content.
\n
To support your Linux sources, follow the procedure mentioned above in General Infrastructure - Indexes and Sourcetypes to add the new indexes for the data you will be bringing in.
\n
We will show the sourcetypes and monitor statements next in the configuration files.
',
category: ["Splunk Configuration for Data Source"],
},
{
id: "SC-LinuxSecurity-4-inputsconf",
title: "Configuration Files",
"short-description":
'
Overview \n
Configuration files for Linux inputs tend to be relatively simple. In this case, we just have a single inputs.conf file that will go on the Linux hosts you will be monitoring. As detailed above in Instruction Expectations and Scaling , you will need some mechanism to distribute these files to the hosts you’re monitoring. For initial tests or deployments to only your most sensitive systems, it is easy to copy the files to the hosts. For larger distributions, you can use the Splunk deployment server or use another code-distribution system, such as Puppet, Chef, Ansible, or others.
\n
Implementation \n
\n Download the Splunk_TA_nix app and extract it, as described earlier \n Create a folder called "local" in the app \n Download the inputs.conf from below, and place it into the local folder. It should now be stored at Splunk_TA_nix/local/inputs.conf \n Distribute the Splunk_TA_nix directory (and its contents) to all of the hosts you will be monitoring. \n \n
\n### bash history\n[monitor:///root/.bash_history]\nsourcetype = bash_history\nindex = osnixbash\ndisabled = 0\n\n[monitor:///home/.../.bash_history]\nsourcetype = bash_history\nindex = osnixbash\ndisabled = 0\n\n[script://$SPLUNK_HOME/etc/apps/Splunk_TA_nix/bin/netstat.sh]\ninterval = 120\nsourcetype = netstat\nsource = netstat\nindex = osnixscript\ndisabled = 0\n\n[script://$SPLUNK_HOME/etc/apps/Splunk_TA_nix/bin/lsof.sh]\ninterval = 300\nsourcetype = lsof\nsource = lsof\nindex = osnixperf\ndisabled = 0\n\n[monitor:///var/log]\nwhitelist=(log$|messages|mesg$|cron$|acpid$|\\.out)\nblacklist=(\\.gz$|\\.zip$|\\.bz2$|auth\\.log|lastlog|secure|anaconda\\.syslog)\nindex=osnix\nsourcetype=syslog\ndisabled = 0\n\n[monitor:///var/log/secure]\nblacklist=(\\.gz$|\\.zip$|\\.bz2$)\nindex=osnixsec\nsourcetype=syslog\nsource=secure\ndisabled = 0\n\n[monitor:///var/log/auth.log*]\nblacklist=(\\.gz$|\\.zip$|\\.bz2$)\nindex=osnixsec\nsourcetype=syslog\ndisabled = 0\n\n# This script reads the auditd logs translated with ausearch\n[script://./bin/rlog.sh]\nsourcetype = auditd\nsource = auditd\ninterval = 60\nindex = osnixsec\ndisabled = 0\n\n[script://./bin/ps.sh]\ninterval = 30\nsourcetype = ps\nsource = ps\nindex = osnixscript\ndisabled = 0\n \n ',
category: ["Splunk Configuration for Data Source"],
},
{
id: "VC-LinuxSecurity-1-EnablingLog",
title: "Enabling Monitoring",
"short-description":
'\n
Overview \n
To maintain a good security posture and to leverage the examples provided in Splunk Security Essentials, we recommend following your own security or audit policy. In the absence of that, there are a number of industry standard guides to help.
\n
Implementation \n
Important Note: Splunk is a monitoring product, so while we’re working hard to centralize some of the recommendations you should follow in one place to make your life easier, we cannot offer support for the actual configuration of anything other than Splunk itself. We strongly recommend that you leverage trained expert resources when making any changes. That’s in large part why we’re pointing you to the documentation for the nitty-gritty details!
\n
Fortunately, on most Linux distributions, the monitored inputs we typically see at Splunk are already logged into log files, ready for you to monitor. Typically, some of the audit controls maybe resident in different files, but using the inputs.conf above, you’ve already got that covered!
\n
If after reviewing your company’s policies or industry standard guide you need a finer-grained level of monitoring, you should look to configure the auditd daemon. On most distributions, auditd is installed by default. For those of you with Ubuntu 14.04, you would need to install it. Here’s a document that explains how to do that. As a sneak preview "apt-get install auditd" will be your friend! Once installed (if it is not already), you may configure it to monitor a number of events, such as user and process tracking. The Ubuntu configuration guide is here . The daemon is pretty universal across distributions. Once you have amended your audit policy, these events will be reported in the audit.log that we are monitoring as part of our auditd sourcetype in inputs.conf.
\n
Important Note: As with all system-administration tasks and auditing controls, make sure you seek the appropriate authorization and go through the testing first, to ensure the right level of monitoring and system resource.
\n
Validation \n
Usually the first thing people will see when deploying audit policies is either new systems showing up in Splunk or, at least, an increase in system log messages. If you already have some logs coming in and want to validate that you’re getting the new ones, look for the delta between your old policy and your new one.
',
category: ["System Configuration"],
},
{
id: "SC-LinuxSecurity-5-leastprivilege",
title: "(Optional) Deploy Least Permission",
"short-description":
'
Overview \n
At Splunk we don’t advise you to run our software with any more privilege than necessary. However, on monitoring systems, there is an argument to do just that. Consider this scenario: an attacker gains unauthorized access to your systems. The attacker\'s next objective is to gain privilege escalation to a system administrator. Once achieved, you can manipulate the monitored files\' (the ones that alert you of an intruder presence) permissions, so that your monitoring tool can no longer read and is effectively blind.
\n
Implementation \n
If you do wish to restrict the user, you may wish to look at the following solution:
\n
Change the group permission set to allow the "Splunk" group to read and ensure that only the Splunk user belongs to that group. (We have a monitoring solution for that!) \nIf you followed our best practice for installing Splunk on a Linux system, you will by now have a user and group called "Splunk." If not, create them.\n[root@ip-172-31-94-210 ec2-user]# groupadd splunk\n[root@ip-172-31-94-210 ec2-user]# useradd -g splunk splunk\n[root@ip-172-31-94-210 ec2-user]# \nAmend the permission set on the files you need to monitor to allow users in the "Splunk" group to read the files. \nProvided Splunk is running as the splunk user and is a member of the "splunk" group, you will now be able to successfully monitor. \n\n
One "gotcha" to this solution is that the files will often be automatically rolled and (alas) your changes will be lost. To avoid this situation, you should amend the logrotate configuration to generate the new file with the permission set we have just configured. As another sneak preview, the "create" statement will help. However, please see the man page and (like always) be sure to thoroughly test and seek the appropriate approvals.
',
category: ["Splunk Configuration for Data Source"],
},
{
id: "SC-Sysmon-1-Overview",
title: "Sysmon Overview",
"short-description":
'\n
Sysmon, a component of Microsoft’s Sysinternals suite of Windows utilities, is a very powerful host-level tool that can assist you in detecting advanced threats on your network by providing intricate host-operation details in real time. In contrast to common Antivirus/Host-Based Intrusion-detection (HIDS) solutions, Sysmon performs system activity deep monitoring and logs high-confidence indicators of advanced attacks. Read more about Sysmon in Microsoft’s documentation .
\n
A fantastic way to collect detailed information about your Windows endpoints in Splunk, Sysmon is free of charge, installs painlessly on many variants of Windows, and integrates well with Splunk deployments. In fact, Mark Russinovich, Sysmon’s author, has spoken about Sysmon at the past two RSA conferences and showcases Splunk as an excellent mechanism for the collection and analysis of Sysmon data. Check out his slide deck .
\n
As of the most recent version, Sysmon is capable of producing extensive details on the following一much of it very useful in the early detection of malicious code execution or other nefarious behavior:
\n
\n \n Process executions, including parent/child relationships, user that launched process, and hash data \n \n \n File creations \n \n \n File creation time changes \n \n \n Network activity, down to the process level \n \n \n Image loads \n \n \n Creation of remote threads \n \n \n Interprocess accesses \n \n \n Windows registry modifications \n \n \n NTFS alternate data stream (ADS) creations \n \n \n Pipe creations and connections \n \n \n WMI event monitoring \n \n \n
Assuming you have already installed a recent version of the Splunk universal forwarder (UF) for Windows, follow these high-level steps to configure Splunk to properly ingest Sysmon data:
\n
\n \n Install Sysmon on the endpoint running the UF and configure it to collect the information you require. \n \n \n Configure Splunk to properly parse the Sysmon event logs. \n \n \n Configure Splunk to collect the Sysmon event log from the endpoint. \n \n \n ',
category: ["Splunk Configuration for Data Source"],
},
{
id: "SC-Sysmon-2-SizingEstimate",
title: "Sysmon Sizing Estimate",
"short-description":
'
Sizing can vary significantly, depending on the Sysmon configuration, but the SwiftOnSecurity configuration linked below reduces the information returned by Sysmon to the minimum needed to be useful. Since the UF operation can also result in significant process-execution events, we also recommend specific configurations to filter out this data here. We will cover this in the Installing Sysmon section below.
\n
A properly configured Windows endpoint running Sysmon will result in 2-4MB of Sysmon data ingested in Splunk daily—sometimes much less. Of course, particularly busy or compromised machines may generate more data. You may also select “critical” machines or machines owned by “most likely targets” and voluntarily increase the verbosity of logging on these systems. For example, the NetworkConnect (Event Code 3) and Image Load (Event Code 7) logging may be increased for these systems.
\n
For a recent success story involving the deployment of Sysmon (plus other Windows endpoint data sources) that resulted in 10-12MB of data per endpoint ingested, review the .conf2017 presentation delivered by Splunk customer TransAlta .
\n',
category: ["Splunk Configuration for Data Source"],
},
{
id: "SC-Sysmon-3-InstallTA",
title: "Install the Technology Add-On -- TA",
"short-description":
'
The Sysmon TA is a Splunk app that configures Splunk to understand the Sysmon data format. It is located on SplunkBase .
\n
Splunk recommends that you install TAs (particularly the files props.conf and transforms.conf) on all Splunk installs, including the universal forwarder. This isn’t necessarily required for Sysmon, but we recommend you stick with the best practices across the board.
\n
Installing on Search Heads, Indexers, or on a Single Splunk Environment
\n
While you can opt to install the TA by copying the files into your directory structure in the same way explained for forwarders below, you can take a simpler approach for a Splunk system with UI. In fact, it’s the same approach you probably took for installing Splunk Security Essentials in the first place—via the web interface, as documented here . The Sysmon TA does not have a user interface, so it will not appear in the list of apps. However, once it’s installed, you will be able to see it under Manage Apps.
\n
Installing on forwarders
\n
\n \n To install the app, start by downloading the file from the SplunkBase , then extract it. Note: The app itself is a .tgz file, or a gzipped tarball. If you’re running a pure Windows environment, this means that you will need a third-party program to extract it. Fortunately, .tgz is the most common format in the world after .zip, so virtually any extraction program you have (WinZip, 7z, WinRAR, etc.) will all extract it.
\n \n \n Once you have the extracted folder, move it into
%SPLUNK_HOME%/etc/apps/ folder. For most modern Splunk environments, that will be C:\\Program Files\\SplunkUniversalForwarder\\etc\\apps .\n \n \n Once you’ve extracted the app, you must restart Splunk, either via the Services Control Panel applet or by running
C:\\Program Files\\SplunkUniversalForwarder\\bin\\splunk.exe restart .\n \n \n
Validation \n
You can make sure that Splunk has picked up the presence of the app by running:
C:\\Program Files\\SplunkUniversalForwarder\\bin\\splunk.exe display app, will, after asking you to log in, provide you with a list of installed apps. Usually, however, if you see the folder listed alongside the other apps (learned, search, splunk_httpinput, etc.), you can be assured that it’s there.
Splunk Cloud Customers : you won\'t be copying any files or folders to your indexers or search heads, but good news! Even though the Sysmon TA is not Cloud Self-Service Enabled, you will still be able to open a ticket with Cloud Ops and be ready to go in short order.
\n\n',
category: ["Splunk Configuration for Data Source"],
},
{
id: "SC-Sysmon-4-configurationfiles",
title: "Configuration Files",
"short-description":
'
Overview \n
Configuration files for Sysmon are pretty simple. In this case, we just have a single inputs.conf file that will go on the Windows hosts you will be monitoring. You’ll need to collect another Windows Event Log to collect the information from Sysmon running on the endpoint. While Splunk supports a few mechanisms for collecting event logs, almost all customers follow our Professional Services’ best practices and pull them in via the universal forwarder (UF).
\n
As detailed in Instruction Expectations and Scaling, you will need some mechanism to distribute these files to the hosts you’re monitoring. For initial tests or deployments to just your most sensitive systems, it is easy to copy the files to the hosts. For larger distributions, you can use the Splunk deployment server or use another code-distribution system, such as SCCM, Puppet, Chef, Ansible, or others.
\n
Implementation \n
Distribute the below inputs.conf file to your hosts in the %SPLUNK_HOME%\\etc\\apps\\TA-microsoft-sysmon\\local folder. If the folder doesn’t exist, you will need to create it. For most customers, the path to this file will end up being C:\\Program Files\\SplunkUniversalForwarder\\etc\\apps\\TA-microsoft-sysmon\\local\\inputs.conf.
\n\n
[WinEventLog://Microsoft-Windows-Sysmon/Operational]\ndisabled = false\nrenderXml = 1\nindex = epintel\n \n
Verification \n
If all is properly configured, you should be able to find your data using the following Splunk search:\n
index=* source=XmlWinEventLog:Microsoft-Windows-Sysmon/Operational \n
You should see data coming into Splunk and then verify correct timestamps, event breaking, and field extraction.
\n',
category: ["Splunk Configuration for Data Source"],
},
{
id: "VC-Sysmon-1-SingleServer",
title: "Single System Sysmon Installation",
"short-description":
'
Implementation \n
Download Sysmon .
\n
Sysmon can generate large amounts of logs. You will want to tune your collection configuration. We recommend starting with this Sysmon configuration: https://github.com/SwiftOnSecurity/sysmon-config . Advanced users may also want to look at https://github.com/MHaggis/sysmon-dfir .
\n\n
In either case, be sure to include configuration lines to exclude Splunk forwarder execution activity, especially for process-creation events .
\n\n
To install on your endpoint, run the following with administrator rights:\n
sysmon.exe -accepteula -i sysmonconfig-export.xml \n\n
Update existing configuration, again, with administrator rights:\n
sysmon.exe -c sysmonconfig-export.xml \n\n
Upon installation, Sysmon will begin logging events to the operational event log. \n
C:\\Windows\\System32\\winevt\\Logs\\Microsoft-Windows-Sysmon%4Operational.evtx \n
Verify your installation \n
After successful installation, you should be able to see the Sysmon log in your Windows Event Viewer.
\n',
category: ["System Configuration"],
},
{
id: "VC-Sysmon-1-LargeScale",
title: "(Optional) Large Scale Sysmon Installation",
"short-description":
'
Large-scale deployments of Sysmon can be automated using Group Policy Objects (GPO), as described here .
',
category: ["System Configuration"],
},
{
id: "SC-Symantec-1-Overview",
title: "Overview",
"short-description":
"
Most Symantec environments will have a Windows Server running the Symantec Endpoint Protection Management (SEPM) server, where you install the Universal Forwarder (UF), the Symantec Technology Add-In (TA), and the inputs.conf. You don't need to apply this configuration to all of the devices in your environment.
\n
SEPM creates log files, called dump files, in the local file system. The UF is configured to ingest these log files through a deployed configuration file called inputs.conf.
\n",
category: ["Splunk Configuration for Data Source"],
},
{
id: "SC-Symantec-2-SizingEstimate",
title: "Sizing Estimate",
"short-description":
'
Sizing of the SEPM logs depend on policy, activity and number of clients. In table 1-6 (page 22) of the Symantec Endpoint Protection 14 Sizing and Scalability Best Practices White Paper , Symantec gives an example of average events per log. Based on this example, the daily ingest into Splunk for Viruses logs could be 0.5MB per 1,000 clients per day.
\n',
category: ["Splunk Configuration for Data Source"],
},
{
id: "SC-Symantec-3-InstallTA",
title: "Install the Technology Add-On -- TA",
"short-description":
'
Overview \n
Splunk has a detailed TA that supports ingesting all the different data types generated by your Symantec Endpoint Protection Manager. Like all Splunk TAs, it also includes everything needed to parse out the fields and give them names that are compliant with Splunk’s Common Information Model (CIM), so they can easily be used by the searches in Splunk Security Essentials (SSE), along with searches you will find in other community-supported and premium apps.
\n
Implementation \n
Find the TA, along with all your other Splunk apps needs, on SplunkBase. You can https://splunkbase.splunk.com/ and search for it or follow the direct link, here .
\n\n
As with all Splunk TAs, we recommend you deploy it to all parts of your Splunk environment, for simplicity and uniformity. To install the app, start by downloading the file from the SplunkBase URL just shown and then extract it into %SPLUNK_HOME%/etc/apps/ folder. For most modern Splunk environments, that will be C:\\Program Files\\SplunkUniversalForwarder\\etc\\apps.
\n
Note: The app itself is a .tgz file, or a gzipped tarball. If you’re running in a pure Windows environment, this means that you will need a third-party program to extract it. Fortunately, .tgz is the most common format in the world after .zip, so virtually any extraction program you have (WinZip, 7z, WinRAR, etc.) will all extract it.
\n
Once you’ve extracted the app, you can restart Splunk via the Services Control Panel applet, or by just running:
c:\\Program Files\\SplunkUniversalForwarder\\bin\\splunk.exe restart \n\n
Validation \n
You can make sure that Splunk has picked up the presence of the app by running:
"C:\\program files\\splunk\\bin\\splunk.exe" display app , will, after asking you to log in, provide you with a list of installed apps. Usually, if you see the folder listed alongside the other apps (learned, search, splunk_httpinput, etc.) you will know that it’s there successfully.
Splunk Cloud Customers : you won\'t be copying any files or folders to your indexers or search heads, but good news! Even though the Splunk Add-on for Symantec Endpoint Protection is not Cloud Self-Service Enabled, you will still be able to open a ticket with Cloud Ops and be ready to go in short order.
',
category: ["Splunk Configuration for Data Source"],
},
{
id: "SC-Symantec-4-IndexesAndSourcetypes",
title: "Symantec Indexes and Sourcetypes",
"short-description":
'
Overview \n
Amongst Splunk’s 15000+ customers, we’ve done a lot of implementations and we’ve learned a few things along the way. While you can use any sourcetypes or indexes that you want in “Splunk land,” we’ve found that the most successful customers follow specific patterns, as it sets them up for success moving forward.
\n
Implementation \n
The most common SEPM data types are the Security Log, System Log, and Application Log, but there are a few others as well. Here is a list of the recommended indexes and sourcetypes.
\n
\n \n \n Data Type \n Input (inputs.conf, below)
\n Sourcetype
\n Index
\n
\n \n \n Client scan data
\n agt_scan.tmp
\n symantec:ep:scan:file
\n epav
\n \n \n \n Client risk data
\n agt_risk.tmp
\n symantec:ep:risk:file
\n epav
\n \n \n \n Client proactive threat data
\n agt_proactive.tmp
\n symantec:ep:proactive:file
\n epav
\n \n \n \n Application and device control data
\n Agt_behavior.tmp
\n symantec:ep:behavior:file
\n ephids
\n \n \n \n Client security data
\n Agt_security.tmp
\n Symantec:ep:security:file
\n ephids
\n \n \n \n Server client data
\n Scm_agent_act.tmp
\n Symantec:ep:agent:file
\n ephids
\n
\n \n \n Client traffic data
\n Agt_traffic.tmp
\n Symantec:ep:traffic:file
\n epfw
\n
\n \n \n Client packet data
\n Agt_packet.tmp
\n Symantec:ep:packet:file
\n epfw
\n
\n \n \n Server system data
\n Scm_system.tmp
\n Symantec:ep:scm_system:file
\n epav
\n
\n \n \n Client system data
\n Agt_system.tmp
\n Symantec:ep:agt_system:file
\n epav
\n
\n \n \n Server policy data
\n Scm_policy.tmp
\n Symantec:ep:scm_policy:file
\n epav
\n
\n \n \n Server administration data
\n Scm_admin.tmp
\n Symantec:ep:scm_admin:file
\n epav
\n
\n \n \n
\n
\n
\n
\n
\n \n
\n
If you have already started ingesting the data sources into another index, you can usually proceed (do consider, however, whether you should separate security logs from administration logs, application, and system logs, based on who likely will need access or be prohibited access). If you have already started ingesting data with a different sourcetype, we recommend you switch over to the standardized sourcetypes, if possible. If you\'re not using the Splunk TA for SEPM to ingest data, keep in mind you may need to go through extra work to align field names to get value out of Splunk Security Essentials and other Splunk content.
\n
To support your SEPM sources, follow the procedure mentioned above in “General Infrastructure--Indexes and Sourcetypes ” to add the new indexes for the data you will be bringing in.
\n
For the sourcetypes and monitor statements, we will show those next in “Symantec Configuration Files.”
\n',
category: ["Splunk Configuration for Data Source"],
},
{
id: "SC-Symantec-5-configurationfiles",
title: "Symantec Configuration Files",
"short-description":
'
Overview \n
Configuration files for SEP Manager inputs tend to be pretty simple. In this case, we just have a single inputs.conf file that will go on the Windows SEPM hosts you will be monitoring. As detailed above in Instruction Expectations and Scaling , you will need some mechanism to distribute these files to the hosts you\'re monitoring. For initial tests or deployments to just your most sensitive systems, it is easy to copy the files to the hosts. For larger distributions, you can use the Splunk Deployment Server, or use another code distribution system such as SCCM, Puppet, Chef, Ansible, or others.
\n
Implementation \n
Distribute the below inputs.conf file to your hosts in the %SPLUNK_HOME%\\etc\\apps\\Splunk_TA_symantec_ep\\localfolder. If the folder doesn\'t exist, you will need to create it. For most customers, the path to this file will end up being C:\\Program Files\\SplunkUniversalForwarder\\etc\\apps\\Splunk_TA_symantec_ep\\local\\inputs.conf.
\n\n
Example of inputs.conf that can be deployed to the Splunk UF on the SEP Manager system:
\n
[monitor://C:\\Program Files (x86)\\Symantec\\Symantec Endpoint Protection Manager\\data\\dump\\agt_scan.tmp]\ndisabled = false\nindex = epav\nsourcetype = symantec:ep:scan:file\n\n[monitor://C:\\Program Files (x86)\\Symantec\\Symantec Endpoint Protection Manager\\data\\dump\\scm_admin.tmp]\nindex = epav\nsourcetype = symantec:ep:admin:file\ndisabled = false\n\n[monitor://C:\\Program Files (x86)\\Symantec\\Symantec Endpoint Protection Manager\\data\\dump\\agt_behavior.tmp]\nindex = ephids\nsourcetype = symantec:ep:behavior:file\ndisabled = false\n\n[monitor://C:\\Program Files (x86)\\Symantec\\Symantec Endpoint Protection Manager\\data\\dump\\scm_agent_act.tmp]\nindex = ephids\nsourcetype = symantec:ep:agent:file\ndisabled = false\n\n[monitor://C:\\Program Files (x86)\\Symantec\\Symantec Endpoint Protection Manager\\data\\dump\\scm_policy.tmp]\nindex = epav\nsourcetype = symantec:ep:policy:file\ndisabled = false\n\n[monitor://C:\\Program Files (x86)\\Symantec\\Symantec Endpoint Protection Manager\\data\\dump\\scm_system.tmp]\nindex = epav\nsourcetype = symantec:ep:scm_system:file\ndisabled = false\n\n[monitor://C:\\Program Files (x86)\\Symantec\\Symantec Endpoint Protection Manager\\data\\dump\\agt_packet.tmp]\nindex = epfw\nsourcetype = symantec:ep:packet:file\ndisabled = false\n\n[monitor://C:\\Program Files (x86)\\Symantec\\Symantec Endpoint Protection Manager\\data\\dump\\agt_proactive.tmp]\nindex = epav\nsourcetype = symantec:ep:proactive:file\ndisabled = false\n\n[monitor://C:\\Program Files (x86)\\Symantec\\Symantec Endpoint Protection Manager\\data\\dump\\agt_risk.tmp]\nindex = epav\nsourcetype = symantec:ep:risk:file\ndisabled = false\n\n[monitor://C:\\Program Files (x86)\\Symantec\\Symantec Endpoint Protection Manager\\data\\dump\\agt_security.tmp]\nindex = ephids\nsourcetype = symantec:ep:security:file\ndisabled = false\n\n[monitor://C:\\Program Files (x86)\\Symantec\\Symantec Endpoint Protection Manager\\data\\dump\\agt_system.tmp]\nindex = epav\nsourcetype = symantec:ep:agt_system:file\ndisabled = false\n\n[monitor://C:\\Program Files (x86)\\Symantec\\Symantec Endpoint Protection Manager\\data\\dump\\agt_traffic.tmp]\nindex = epfw\nsourcetype = symantec:ep:traffic:file\ndisabled = false\n \n',
category: ["Splunk Configuration for Data Source"],
},
{
id: "VC-Symantec-1-Configuration",
title: "Enabling Logging in Symantec Endpoint Protection Manager",
"short-description":
'
Overview \n
To maintain a good security posture and to leverage the examples provided in SSE, we recommend following logging configured in the SEPM.
\n
Implementation \n
Step One \n
Open the admin console on Symantec Endpoint Protection Manager (SEPM). \nOpen the admin panel, click Servers, select your site and select "Configure External Logging." \nEnable export logs to a dump file. \n
\n
Step Two \n
Click on the Log Filter tab and select the logs to export to a file. \nSeverity level Info should be enabled as well, as it gives for example information on privileged account access (into SEPM), files submitted to Symantec, or functions enabled/disabled.\n
\n
Step Three \n
The last step (often overlooked!) is to verify that log handling is configured, so that all events on the endpoint are sent to the SEPM.
\n
Open the Policies panel, select the Virus and Spyware Protection Policy that is in use, and open it. \nOpen Miscellaneous and select the Log Handling tab. \nSelect show all virus and spyware protection events and select all event types. \n
\n
Validation \n
Once all configured correctly, the events should flow into Splunk. Below an example of how the source, sourcetype, and index should look. Search example:
index=ep* | stats count by source, sourcetype, index \n
',
category: ["System Configuration"],
},
{
id: "VC-o365-1-Overview",
title: "Office 365 Overview",
"short-description":
"
The Office365 Reporting Add-on lets you collect Exchange message-tracking logs by querying the Office 365 Reporting web service API and indexing the results.
Exchange message-tracking logs record email message activity as they flow through the transport pipeline on Exchange mail servers. These are particularly helpful not only for exchange troubleshooting and diagnosing, but also from a security-operations perspective.
They can help you:
Find out what happened to a message sent by a specific sender. Find out if a transport rule acted on a message. Find out if a message sent from an Internet sender made it into your Exchange organization. Correlate sender domains against threat intelligence or look for non-standard senders. \n",
category: ["System Configuration"],
},
{
id: "VC-o365-2-configureo365",
title: "Validate Office 365 Permissions",
"short-description":
'
The Office365 Reporting Add-on requires an Exchange admin account to query the message trace APIs to retrieve data.
To validate that the account you are using has sufficient access:
Login to https://portal.office.com Access the Exchange Admin Center Select mail flow , then message trace . If you\'re able to successfully run a message trace, the account will suffice. \n',
category: ["System Configuration"],
},
{
id: "SC-o365-1-SizingEstimate",
title: "Sizing Estimate",
"short-description":
"There is a large amount of variability in the volume of O365 logs. There are several areas that impact volumes:
Subscription type and Workloads (Apps) used Size of organisation O365 adoption inside of the organization Kinds of federation / ADsync / ExpressRoute, and etc. Message trace events tend to be about 650 bytes each, with multiple events per email. Management logs tend to be about 1200 bytes each, and Azure Audit logs tend to be north of 3000 bytes each.
If you're trying to use this information to accurately size your environment, unfortunately there's not an easy way to size this information without just ingesting it. The way that most customers will determine their data ingest is to just start ingesting data, and then look at how much data has been ingested.
\n",
category: ["Splunk Configuration for Data Source"],
},
{
id: "SC-o365-2-WhereToCollect",
title: "Where to Collect",
"short-description":
'Pulling logs from Office 365 requires a web service. It is functionally very different from grabbing data from local logs or events, since it must be configured via the Office 365 Reporting Add-on on a Splunk box with a web UI. It\'s deployed in one of the following two ways:
Single instance: Splunk customers who have a smaller Splunk load that fits on a single system often add the Technology Add-on (TA) to the same system. Sizing here is environment specific, so you will want to ensure adequate performance (although this setup is usually quite workable in smaller environments). If you need to, you can always redo the configuration later, using a dedicated heavy forwarder. Heavy forwarder: In most environments, customers will install the TA on a dedicated heavy forwarder. A heavy forwarder is just like a normal Splunk install (in effect, not a universal forwarder), but its only role is to pull in data from special sources and send it to indexers. The Office 365 Reporting Add-on requires Internet connectivity to run REST API queries to the reporting web service.
\nWhile, generally speaking, it is Splunk best practice to install TAs across all parts of your Splunk environment (particularly props and transforms), in the case of the Office 365 Reporting Add-on, we will be reaching out to a cloud service, which makes the configuration slightly different. We separate out installing the TA from configuring the inputs. \n
Configuring the inputs: You will only configure the inputs on one system in your environment, such as a heavy forwarder or a single instance. (See "Overview" for more detail.)
\n Installing the TA: The TA itself should reside wherever you configure the inputs (since the TA is the mechanism that allows you to configure the inputs). If you have a larger or more advanced environment where you configure the inputs on a heavy forwarder, you should also install the TA on your search heads, so you can see the Office 365 field extractions.
\n Advanced tip: Hide the app on your search heads, so you don’t accidentally reconfigure and duplicate your data later. To do this, click the app dropdown on the upper left-hand corner of the screen, then select Manage Apps. then Edit Properties next to the Office 365 Reporting Add-on. Next, click Visible: No and then save.
\n The following table provides a reference for installing this specific add-on to a distributed deployment of Splunk Enterprise:
\n\n \n Splunk Platform Component \n Supported?
\n Required
\n \n \n Search heads
\n Yes
\n Yes
\n \n \n Heavy forwarders
\n Yes
\n Depends on size
\n \n \n Indexers
\n Yes
\n No
\n \n \n Universal forwarders
\n No
\n No
\n \n
\n',
category: ["Splunk Configuration for Data Source"],
},
{
id: "SC-o365-3-InstallTA",
title: "Install the Technology Add-On -- TA",
"short-description":
'Log into Splunk and click Splunk Apps .
Search for "Office 365 Reporting." Click the Install button.
After installation, click Restart Now.
Log back into Splunk and select the Microsoft Office 365 Reporting Add-on app.
Splunk Cloud Customers : you won\'t be copying any files or folders to your indexers or search heads, but good news! Even though the Office 365 Reporting Add-on is not Cloud Self-Service Enabled, you will still be able to open a ticket with Cloud Ops and be ready to go in short order.
\n',
category: ["Splunk Configuration for Data Source"],
},
{
id: "SC-o365-4-IndexesAndSourcetypes",
title: "O365 Indexes and Sourcetypes",
"short-description":
'Overview \nAmongst Splunk’s 15,000+ customers, we’ve done a lot of implementations, and we’ve learned a few things along the way. While you can use any sourcetypes or indexes that you want in the "land of Splunk," we’ve found that the most successful customers follow specific patterns, as it sets them up for success moving forward.
\nImplementation \nHere is a table of sourcetypes and indexes we recommend. If you have already followed the recommended indexes.conf setup above, then the index is already configured for you and everything will run automatically. If you are blazing your own path, we strongly recommend creating an index called “mail” now. (on premise link , cloud link )
\n\n \n Data source
\n Description
\n Sourcetype
\n Index
\n \n \n ms_o365_message_trace
\n REST API data from the O365 reporting web service
\n ms:o365:reporting:messagetrace
\n mail
\n \n
\n',
category: ["Splunk Configuration for Data Source"],
},
{
id: "SC-o365-5-configuration",
title: "Configuration",
"short-description":
'Implementation In the Microsoft Office 365 Reporting Add-on for Splunk, select Configuration in the navigation, and then Add . Enter Name, Username, and Password. Select Add. Now that we\'ve configured our account, select the Inputs tab, then Create New Input . Enter Name and Interval. Select Index and Office365 Account. Enter Start date/time and select Add. Note: Depending on the size of the environment, you may run into issues with Azure limits when trying to retrieve too many previous events. If historical data is not essential, set the start date/time as the current day. Validation Validate the input and confirm the data is being ingested by running the following search: index=mail sourcetype=ms:o365:reporting:messagetrace
\n',
category: ["Splunk Configuration for Data Source"],
},
{
id: "VC-AWS-1-Overview",
title: "AWS Setup Overview",
"short-description":
'Amazon Web Services (AWS) has become an integral part of several organizations’ IT infrastructure. Splunk offers an easy method to ingest various data sources from the AWS platform, which Splunk Security Essentials (SSE) uses to enhance your overall security posture. This overview will provide step-by-step guidance on setting up this integration—specifically for the CloudTrail and VPC Flow Logs\' data sources. CloudTrail provides a comprehensive trail of account activity related to actions across your AWS infrastructure. VPC Flow Logs contain a comprehensive record of network traffic in and out of your AWS environment.
\n To set up this integration on your supported platform , follow these steps:
\n \nConfigure your AWS accounts and services or confirm your existing configurations. \nConfigure accounts or EC2 roles with IAM permissions to match those required by the add-on. \nInstall the add-on . \nOn your data-collection node, configure the AWS accounts you want to use to collect data with the add-on. \nConfigure your inputs to get your AWS data into the Splunk platform. \n \n Comprehensive documentation for configuring both your AWS and Splunk environments can be found at https://docs.splunk.com/Documentation/AddOns/released/AWS .
\n Prerequisite: Performing all the steps below requires administrator access to your AWS account. If you do not have the required permissions to perform all the actions yourself, work with an AWS admin to complete all steps, including creating the accounts or EC2 IAM roles, with the permissions that the Splunk Add-on for AWS uses to connect.
',
category: ["System Configuration"],
},
{
id: "VC-AWS-2-awsiam",
title: "Set Up AWS Identity Access Management -- IAM",
"short-description":
'Overview Correctly configuring the AWS IAM policy is required for ingesting the subsequent data streams into your Splunk environment.
Implementation Splunk documentation contains comprehensive information on how to setup IAM roles in AWS, either for individual data sources or globally, for all AWS data sources. Please see http://docs.splunk.com/Documentation/AddOns/released/AWS/ConfigureAWSpermissions for detailed information.
Create Splunk Access user Within the AWS IAM configuration menu, create a new user, splunk_access. Attach the SplunkAccess policy created in the previous step and grant the user only programmatic access. Once complete, download the user credentials, as shown in the screenshot below.
',
category: ["System Configuration"],
},
{
id: "VC-AWS-3-awssns",
title: "Set Up AWS Simple Notification Service -- SNS",
"short-description":
'Overview You need to grant permissions to the AWS accounts or EC2 IAM roles that the add-on uses to connect to the Amazon SNS API. See Configure AWS permissions for details.
Implementation If you plan to use the Simple Queue Service (SQS) -based S3 input, you must enable Amazon S3 bucket events to send notification messages to an SQS queue whenever the events occur. For instructions on setting up S3 bucket event notifications, see the following AWS documentation pages:
https://docs.aws.amazon.com/AmazonS3/latest/UG/SettingBucketNotifications.html http://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html ',
category: ["System Configuration"],
},
{
id: "VC-AWS-4-awssqs",
title: "Set up AWS Simple Queueing Service - SQS",
"short-description":
'Overview \nYou need to grant permissions to the AWS accounts or EC2 IAM roles that the add-on uses to connect to the Amazon SQS API. See Configure AWS permissions for details.
\nImplementation \nIf you plan to use the SQS-based S3 input, you must perform the following:
\n\n Set up a dead-letter queue for the SQS queue to be used for the input for storing invalid messages. Read more about SQS dead-letter queues and how to configure them . \n Configure the SQS visibility timeout to prevent multiple inputs from receiving and processing messages in a queue more than once. We recommend that you set your SQS visibility timeout to 5 minutes or longer. If the visibility timeout for a message is reached before the message has been fully processed by the SQS-based S3 input, the message will reappear in the queue and will be retrieved and processed again, resulting in duplicate data. Read more about SQS visibility timeout and how to configure it . \n ',
category: ["System Configuration"],
},
{
id: "VC-AWS-5-cloudtrail",
title: "Set up AWS CloudTrail",
"short-description":
'Overview> \nEnabling AWS CloudTrail inputs into Splunk will allows you to record AWS API calls for your account and ingest the resulting dataset. This data can then be used for searching, visualization, and correlation.
\nImplementation \nThe Splunk Add-on for AWS collects events from an SQS that subscribes to the SNS notification events from CloudTrail. Configure CloudTrail to produce these notifications, then create an SQS in each region for the add-on to access them.
\n\nEnable CloudTrail. Follow the instructions in the AWS documentation . \nCreate an S3 Bucket in which to store the CloudTrail events. Follow the AWS documentation to ensure the permissions for this bucket are correct. \nEnable SNS Notifications. See the AWS documentation for instructions. \nCreate a new SQS. \nIf you are in the China region, explicitly grant DeleteMessage and SendMessage permissions to the SQS you just created. This step is not necessary in commercial regions. \nSubscribe the SQS to the SNS Notifications that you enabled in step 3. \nGrant IAM permissions to access the S3 bucket and SQS to the AWS account that the add-on uses to connect to your AWS environment. See Configure AWS Permissions for details. \n ',
category: ["System Configuration"],
},
{
id: "VC-AWS-5-vpcflow",
title: "Set up AWS VPC Flow",
"short-description":
'Overview> \nVPC Flow Logs provide a complete record of traffic into and out of your AWS environment. Several Splunk Security Essentials (SSE) searches leverage this data to give you deeper insights into the security posture of your environment.
\n Implementation \nThe CloudWatch service is automatically enabled to collect free metrics for your AWS services and requires no additional configuration for the Splunk Add-on for AWS. However, you do need to grant permissions to the AWS account(s) that the add-on uses to connect to the CloudWatch API. See Configure AWS Permissions for details.
\nIngesting data from CloudWatch Logs requires no additional configuration beyond that described in the AWS documentation. Similarly, VPC Flow Logs require no additional configuration for the Splunk Add-on for AWS, other than enabling them for your VPCs. However, you do need to grant permissions to the AWS account(s) that the add-on uses to connect to the VPC Flow Log groups and streams. See Configure AWS Permissions for details.
\nSee the AWS documentation for information on how to enable flow logs for your VPCs and configure an IAM role for them.
',
category: ["System Configuration"],
},
{
id: "SC-AWS-1-WhereToCollect",
title: "Where to Collect Logs From",
"short-description":
'Pulling logs from AWS requires a web service. It is functionally very different from grabbing data from local logs or events, since it must be configured via the Splunk Add-on for Amazon Web Services on a Splunk box with a web UI. It\'s deployed in one of the following two ways:
Single instance: Splunk customers who have a smaller Splunk load that fits on a single system often add the Technology Add-on (TA) to the same system. Sizing here is environment specific, so you will want to ensure adequate performance (although this setup is usually quite workable in smaller environments). If you need to, you can always redo the configuration later, using a dedicated heavy forwarder. Heavy forwarder: In most environments, customers will install the TA on a dedicated heavy forwarder. A heavy forwarder is just like a normal Splunk install (in effect, not a universal forwarder), but its only role is to pull in data from special sources and send it to indexers. The Splunk Add-on for Amazon Web Services requires Internet connectivity to send queries to the AWS APIs.
\nWhile, generally speaking, it is Splunk best practice to install TAs across all parts of your Splunk environment (particularly props and transforms), in the case of the Splunk Add-on for Amazon Web Services, we will be reaching out to a cloud service, which makes the configuration slightly different. We separate out installing the TA from configuring the inputs. \n
Configuring the inputs: You will only configure the inputs on one system in your environment, such as a heavy forwarder or a single instance. (See "Overview" for more detail.)
\n Installing the TA: The TA itself should reside wherever you configure the inputs (since the TA is the mechanism that allows you to configure the inputs). If you have a larger or more advanced environment where you configure the inputs on a heavy forwarder, you should also install the TA on your search heads, so you can see the Splunk Add-on for Amazon Web Services field extractions.
\n Advanced tip: Hide the app on your search heads, so you don’t accidentally reconfigure and duplicate your data later. To do this, click the app dropdown on the upper left-hand corner of the screen, then select Manage Apps. then Edit Properties next to the Splunk Add-on for Amazon Web Services. Next, click Visible: No and then save.
\n The following table provides a reference for installing this specific add-on to a distributed deployment of Splunk Enterprise:
\n \n \n Splunk Platform Component \n Supported? \n Required \n \n \n Search heads
\n Yes
\n Yes
\n \n \n Heavy forwarders
\n Yes
\n Depends on size
\n \n \n Indexers
\n Yes
\n No
\n \n \n Universal forwarders
\n No
\n No
\n \n
',
category: ["Splunk Configuration for Data Source"],
},
{
id: "SC-AWS-2-InstallTA",
title: "Installing the Technology Add-on -- TA",
"short-description":
'Log into Splunk and click Splunk Apps .
Search for "Amazon Web Services Add-on." Click the Install button.
After installation, click Restart Now.
Log back into Splunk and select the Splunk Add-on for Amazon Web Services app.
Splunk Cloud Customers : you won\'t be copying any files or folders to your indexers or search heads, but good news! Even though the Amazon Web Services Add-on is not Cloud Self-Service Enabled, you will still be able to open a ticket with Cloud Ops and be ready to go in short order.
',
category: ["Splunk Configuration for Data Source"],
},
{
id: "SC-AWS-3-IndexesAndSourcetypes",
title: "AWS Indexes and Sourcetypes",
"short-description":
'Overview \n Amongst Splunk’s 15,000+ customers, we’ve done a lot of implementations, and we’ve learned a few things along the way. While you can use any sourcetypes or indices that you want in “Splunk land,”, we’ve found that the most successful customers follow specific patterns, as it sets them up for success moving forward.
\nImplementation \n The most common relevant AWS data types to Splunk Security Essentials are CloudTrail and VPC Flow Logs, but there are many others available to you. The following is an overview of SSE-relevant AWS data types and the recommended indices and sourcetypes. Other AWS data sources are outlined in more detail later.
\n\n \n Data source
\n Description
\n Sourcetype
\n Index
\n \n \n CloudTrail
\n AWS API call history from the AWS CloudTrail service.
\n aws:cloudtrail
\n awscloudtrail
\n \n \n CloudWatch Logs
\n VPC Flow Logs from the CloudWatch Logs service.
\n aws:cloudwatchlogs:vpcflow
\n awsflow
\n \n
\nTo support your AWS data sources, follow the procedure mentioned above in <a href="#AWS-IndexesAndSourcetypes">AWS Indexes and Sourcetypes</a> to add the new indices for the data you will be bringing in (it’s generally easiest if you just create awscloudtrail, awsflow, awscloudwatch, etc).
\nAs mentioned, there are several other AWS data sources you could opt to bring in. If you wish to ingest those, here are our recommended sourcetypes for that data:
\n\n \n Data source
\n Description
\n Sourcetype
\n \n \n Config
\n Configuration snapshots and historical configuration data from the AWS Config service
\n aws:config
\n \n \n Configuration change notifications from the AWS Config service
\n aws:config:notification
\n \n \n Description
\n Descriptions of your AWS EC2 instances, reserved instances, and EBS snapshots. Used to improve dashboard readability
\n aws:description
\n \n \n Config Rules
\n Compliance details, compliance summary, and evaluation status of your AWS Config rules
\n aws:config:rule
\n \n \n Inspector
\n Assessment runs and findings data from the Amazon Inspector service
\n aws:inspector
\n \n \n CloudTrail
\n AWS API call history from the AWS CloudTrail service
\n aws:cloudtrail
\n \n \n CloudWatch Logs
\n Data from the CloudWatch Logs service
\n aws:cloudwatchlogs
\n \n \n VPC Flow Logs from the CloudWatch Logs service
\n aws:cloudwatchlogs:vpcflow
\n \n \n CloudWatch
\n Performance and billing metrics from the AWS CloudWatch service
\n aws:cloudwatch
\n \n \n Billing
\n Billing reports that you have configured in AWS
\n aws:billing
\n \n \n S3
\n Generic log data from your S3 buckets
\n aws:s3
\n \n \n S3 access logs
\n aws:s3:accesslogs
\n \n \n CloudFront access logs
\n aws:cloudfront:accesslogs
\n \n \n ELB access logs
\n aws:elb:accesslogs
\n \n \n CloudTrail data
\n aws:cloudtrail
\n \n \n Kinesis
\n Data from Kinesis streams
\n aws:kinesis
\n \n \n SQS
\n Generic data from SQS
\n aws:sqs
\n \n
',
category: ["Splunk Configuration for Data Source"],
},
{
id: "SC-AWS-4-accountconfig",
title: "Configure AWS Account Information",
"short-description":
'The first step to configure the Splunk add-on for AWS to ingest data is to add the account created above. Go to Configuration > Account, and then click Add . Supply the information, and click Add . ',
category: ["Splunk Configuration for Data Source"],
},
{
id: "SC-AWS-5-cloudtrailinput",
title: "Configure CloudTrail Input",
"short-description":
'Overview After completing the above steps, configuring the data inputs within the Splunk interface is simple. The steps are outlined in more detail here.
Implementation In the Splunk add-on for AWS, go to data inputs, and select Create new input > CloudTrail .
Select the desired configuration.
The Splunk add-on for AWS will subsequently ingest data from the configured AWS API endpoints, and perform the relevant field extractions and CIM data model mappings for use by the SSE app. You can always find more detail on configuration in our docs .
Validation You can make sure that Splunk has begun ingesting the data from AWS by running Splunk searches. The Splunk add-on for AWS also has a built-in health-overview dashboard that will provide initial troubleshooting information.
',
category: ["Splunk Configuration for Data Source"],
},
{
id: "SC-AWS-5-vpcflowinput",
title: "Configure VPC Flow Input",
"short-description":
'Overview After completing the above steps, configuring the data inputs within the Splunk interface is simple. The steps are outlined in more detail here.
Note: we are following the non-Kinesis input, which works for small to medium environments. If you have a large AWS environment, we recommend configuring the (somewhat more complicated) Kinesis Firehose configuration. For more detail: Click Here .
Implementation In the Splunk add-on for AWS, go to data inputs, and select Create New Input > VPC Flow Logs > CloudWatch Logs.
Choose the relevant regions and configurations for your environment.
The Splunk add-on for AWS will subsequently ingest data from the configured AWS API endpoints, and perform the relevant field extractions and CIM data model mappings for use by the SSE app. You can always find more detail on configuration in our docs .
Validation You can make sure that Splunk has begun ingesting the data from AWS by running Splunk searches. The Splunk add-on for AWS also has a built-in health-overview dashboard that will provide initial troubleshooting information.
',
category: ["Splunk Configuration for Data Source"],
},
{
id: "SC-AWS-6-updateAWSApp",
title: "(Optional) Update AWS App, If Used",
"short-description":
'By default, the Splunk App for AWS and add-on send the data into the Splunk Main (default) index. From there, a saved search will run to populate the summary indices. The summary indices are used to populate the dashboards in the Splunk App for AWS. Sending data to custom indices will require making changes to the macros supporting this app. To modify this to meet Splunk best practices, follow these steps:
Create a new index for AWS data. Update the proper macros and make sure the saved searches are running to populate the summary indices. (See http://docs.splunk.com/Documentation/AWS/5.0.2/Installation/Macros ) Make sure the saved searches are running properly. (See http://docs.splunk.com/Documentation/AWS/5.0.2/Installation/Savedsearches ) \n',
category: ["Splunk Configuration for Data Source"],
},
{
id: "VC-General-1-syslogoverview",
title: "Syslog Overview",
"short-description":
"While Splunk can receive syslog messages directly from syslog devices like Cisco ASA, Palo Alto Networks, and others, this is not a best practice for production deployments. Using a separate syslog server allows for less impact to Splunk configuration reloads (which take longer and are more frequent), along with leveraging best of breed tools. We recommend either utilizing an existing syslog server or deploying one, such as rsyslog or syslog-ng on Linux, or Kiwi Syslog Server on Windows. There are many example configurations available for ingesting data with any of these technologies, but for convenience we will provide detailed setup instructions for setting up rsyslog on Linux to ingest data for Splunk in line with our best practices.
Rsyslog server that writes files per source-type to disk, place file .conf in /etc/rsyslog.d/ with a config to receive logs over UDP. Logrotate should be configured on this server to prevent logs from flooding the disk. We will also walk through configuring the Splunk Universal Forwarder on this rsyslog server with to forward logs to Splunk.
If you are setting this up in a lab and just want to get started quickly, you can follow the online documentation about ingesting data directly into Splunk via syslog.
",
category: ["System Configuration"],
},
{
id: "VC-PAN-2-rsyslog-config",
title: "Configuring rsyslog",
"short-description":
'Overview Before we configure the Palo Alto Networks device, we need to setup a rsyslog server. The supplied config assumes a "vanilla" install of rsyslog. If you already have a rsyslog server in place, you need to validate this config for your deployment.
Implementation Many customers have experienced issues, as the version of rsyslog shipped with RHEL and Ubuntu is out of date and no longer supported. Run rsyslogd -version and validate that it is running a version higher than or equal to 8.32.0:
If the version is not higher or equal to 8.32.0-even after updating your distribution-please add the rsyslog repositories to your distribution and update again. These repos can be found here:
Install rsyslog server on your choice of Linux flavors using the appropriate command, i.e., Yum install rsyslog or apt-get install rsyslog. Some distributions come with rsyslog preinstalled. Make sure that your /etc/rsyslog.conf has a rule that reads $IncludeConfig /etc/rsyslog.d/*.conf.
Copy the following two files into /etc/rsyslog.d/ -- we recommend calling them file splunk.conf and splunk-pan.conf. splunk.conf will contain all of the global rsyslog configurations, splunk-pan.conf contains all of the Palo Alto Networks specific configurations.
# Provides UDP syslog reception, leave these out if your server is already listening to a network port for receiving syslog.\n# for parameters see http://www.rsyslog.com/doc/imudp.html\n# alternatively these can be commented out from /etc/rsyslog.conf\nmodule(load="imudp") # needs to be done just once\ninput(type="imudp" port="514")\n \n# Provides TCP syslog reception Optional; in case you would like to use TCP as a preferred transport mechanism.\n# for parameters see http://www.rsyslog.com/doc/imtcp.html\n# module(load="imtcp") # needs to be done just once\n# input(type="imtcp" port="514")\n # The purpose of this config is to receive data from a Palo Alto Networks device on a vanilla rsyslog environment\n# and stage it for a Splunk universal forwarder\n# For an existing server read the comments\n# 2018-26-01 Filip Wijnholds fwijnholds@splunk.com\nmodule(load="builtin:omfile")\n$Umask 0022\n# If you are running Splunk in limited privilege mode, make sure to configure the file ownership:\n# $FileOwner splunk\n# $FileGroup splunk\n \n \n#Filters Data and writes to files per Sourcetype:\n#Palo alto knows different sourcetypes, this creates file a per sourcetype per host per day.\n$template threat,"/var/log/rsyslog/pan/threat/%HOSTNAME%-%timegenerated:::date-strftime(%S)%.log"\n$template traffic,"/var/log/rsyslog/pan/traffic/%HOSTNAME%-%timegenerated:::date-strftime(%S)%.log"\n$template system,"/var/log/rsyslog/pan/system/%HOSTNAME%.log"\n$template config,"/var/log/rsyslog/pan/config/%HOSTNAME%.log"\n$template hipmatch,"/var/log/rsyslog/pan/hipmatch/%HOSTNAME%.log"\n$template endpoint,"/var/log/rsyslog/pan/endpoint/%HOSTNAME%.log"\n$template wildfire,"/var/log/rsyslog/pan/wildfire/%HOSTNAME%.log"\n$template correlation,"/var/log/rsyslog/pan/correlation/%HOSTNAME%.log"\n$template aperture,"/var/log/rsyslog/pan/aperture/%HOSTNAME%.log"\n \n#Looks for sourcetypes in the data, to\n:msg, contains, "THREAT" ?threat\n:msg, contains, "TRAFFIC" ?traffic\n:msg, contains, "SYSTEM" ?system\n:msg, contains, "CONFIG" ?config\n:msg, contains, "HIPMATCH" ?hipmatch\n:msg, contains, "ENDPOINT" ?endpoint\n:msg, contains, "WILDFIRE" ?wildfire\n:msg, contains, "CORRELATION" ?correlation Restart the rsyslog service by running: sudo service rsyslog restart
Validation After the daemon is restarted and traffic is sent to rsyslog, you should see at least directories being created:
/var/logs/rsyslog/pan//traffic/ /var/logs/rsyslog/pan//threat/ ',
category: ["System Configuration"],
},
{
id: "VC-PAN-3-logrotate",
title: "Configuring logrotate",
"short-description":
'Overview With the above configuration, we will be able to receive data via UDP syslog from our PAN device. However, left to its own whims, rsyslog would fill up all the disk space available to it -- not desireable. The most common way to handle this is to use logrotate, which is ubiquitous on Linux. The below configuration will automatically rotate all of your PAN log files every day, compress the older ones, and then delete the oldest files.
Implementation Create a file called splunk-pan in the logrotate directory /etc/logrotate.d/ with the following contents, or just click download below.
/var/log/rsyslog/pan/*/*.log\n{\n daily\n compress\n delaycompress\n rotate 3\n ifempty\n maxage 4\n nocreate\n missingok\n sharedscripts\n postrotate\n /bin/kill -HUP `cat /var/run/syslogd.pid 2> /dev/null` 2> /dev/null || true\n endscript\n} For 99.9% of environmemts, you should be set now, as logrotate will run regularly. If you don\'t see the log files being rotated (see Validation below), you may fall into the 0.01% of environments where you need to configure logrotate to run. The easy way to do this is using crontab (another of those ubiquitous Linux tools).
To edit your crontab, from the terminal run: crontab -e. For a daily schedule that rolls the log at midnight, the following cron job will do:
0 0 * * * root logrotate -f /etc/logrotate.d/splunk-pan If you are unfamiliar with cron and want a different schedule, visit this website: https://crontab.guru/#0_0_*_*_* .
Implementation After 24 hours you should see a .1 appended to the log files when you run: ls -l /var/log/rsyslog/pan//traffic/:
',
category: ["System Configuration"],
},
{
id: "VC-PAN-4-enablesyslog",
title: "Enable Syslog in PAN Management",
"short-description":
'Overview To leverage uses cases in Splunk Security Essentials (SSE), we must first get the data from the firewall to our syslog server.
Implementation First, we must create a server syslog profile. Do so by navigating to Device -> Server Profiles -> syslog and click add. Give your server a name and add the IP address or FQDN in the Syslog Server field and change the port (if it differs on your deployment).
Under Device -> Log Settings, find the system box and select every topic of your interest. Next, and add the syslog profile for the configured syslog server.
Now find the HIP Match box, and click the gear icon: Select the server profile you configured for syslog, per the screenshot below.
Do the same for config.
Now that we\'ve configured the high level syslog destination, we need to configure the actual profile. You can do this under Object -> Log Forwarding and click add.
You\'re almost there! You\'ve now got the device all set up to send syslog out, all you have to do now is actually configure the rules so that the device will know when to send. Navigate to policies -> Security and select the policy you are interested in. You\'ll have to configure the syslog profile for each rule, per the screenshot below.
Great! You\'ve got everything configured. As with all PAN policy changes though, don\'t forget to commit before you sip your victory coffee.
Validation Perform the following search on your Splunk instance:
index=* sourcetype=pan* | stats count by sourcetype index This should return something similar to the screenshot below.
',
category: ["System Configuration"],
},
{
id: "SC-PAN-1-SizingEstimate",
title: "PAN Sizing Estimate",
"short-description":
"There is a wild amount of variability in the size for Palo Alto Networks logs. Each message is typically around 850 bytes and we typically see one message per connection (as we recommend logging allows, along with denies). The volume then depends on the size of your PAN device and can be in the hundreds of MB/day for a branch office, all the way to north of 250 GB/day for a main datacenter cluster.
Using only Palo Alto's built-in tools, the show session info command will tell you how many connections there have been since bootup. So, one way of estimating event volume is to check that number at the same time on subsequent days and then calculate the number of connections you typically see per day. When multiplied by the general 850 byte number, you will get a decent expectation for data size.
Another common approach is to implement the rsyslog config referenced above and then track the size of the files created on disk, to determine volume.
",
category: ["Splunk Configuration for Data Source"],
},
{
id: "SC-PAN-2-InstallTA",
title: "Installing the Technology Addon -- TA",
"short-description":
'Overview Splunk has a detailed technology add-on that supports ingesting all the different data types generated by your Palo Alto Next-Generation Firewall. Like all Splunk technology add-ons, it also includes everything needed to parse out the fields and give them names that are compliant with Splunk\'s Common Information Model, so they can easily be used by the searches in Splunk Security Essentials-along with searches you will find in other community-supported and premium apps.
Implementation Find the TA along with all your other Splunk apps and needs on SplunkBase. You can visit https://splunkbase.splunk.com/ and search for it or you could just follow the direct link: https://splunkbase.splunk.com/app/2757/
As with all Splunk TAs, we recommend you deploy it to all parts of your Splunk environment, for simplicity and uniformity. To install the app, start by downloading the file from the SplunkBase location just shown, and then extract it into the apps folder. On Windows systems, this will be %SPLUNK_HOME%\\etc\\apps folder, or usually C:\\Program Files\\Splunk\\etc\\apps. On Linux systems, this will be $SPLUNK_HOME/etc/apps, or usually /opt/splunk/etc/apps.
Validation You can make sure that Splunk has picked up the presence of the app by running $SPLUNK_HOME/bin/splunk display app (or on Windows, %SPLUNK_HOME%\\bin\\splunk display app), will, after asking you to log in, provide you with a list of installed apps. Most likely, if you see the folder listed alongside the other apps (learned, search, splunk_httpinput, etc.) you will know that it\'s there successfully.
Splunk Cloud Customers : you won\'t be copying any files or folders to your indexers or search heads, but good news! The Palo Alto Networks add-on is Cloud Self-Service Enabled. So you can just got to Find Apps, and be up and running in seconds.
',
category: ["Splunk Configuration for Data Source"],
},
{
id: "SC-PAN-3-IndexesAndSourcetypes",
title: "PAN Indexes and Sourcetypes",
"short-description":
'Overview \n Amongst Splunk’s 15,000+ customers, we’ve done a lot of implementations, and we’ve learned a few things along the way. While you can use any sourcetypes or indices that you want in “Splunk land,”, we’ve found that the most successful customers follow specific patterns, as it sets them up for success moving forward.
\nImplementation The PAN Firewall uses six different sourcetypes. Best practice dictates that sourcetypes that fit a specific data model go in to a separate index, per the table below.
\n \n pan:threat
\n netproxy
\n \n \n pan:traffic
\n netfw
\n \n \n pan:system
\n netops
\n \n \n pan:config
\n netops
\n \n \n pan:hipmatch
\n epintel
\n \n \n pan:endpoint
\n epintel
\n \n \n pan:correlation
\n netintel
\n \n \n pan:aperture
\n netintel
\n \n \n pan:wildfire
\n epintel
\n \n
',
category: ["Splunk Configuration for Data Source"],
},
{
id: "SC-PAN-4-inputsconf",
title: "PAN Inputs",
"short-description":
'Overview Configuration files for PAN inputs tend to be pretty simple. In this case, we just have a single inputs.conf file that will go on the syslog server you will be monitoring. As detailed above in Instruction Expectations and Scaling , you will need some mechanism to distribute these files to the hosts you\'re monitoring. For initial tests or deployments to only your most sensitive systems, it is easy to copy the files to the hosts. For larger distributions, you can use the Splunk deployment server or another code-distribution system, such as SCCM, Puppet, Chef, Ansible, or others.
Implementation Distribute the below inputs.conf file to the Universal Forwarder installed on your syslog server (only where you actually have the rsyslog information). You should create a "local" folder inside of the TA folder. For most customers, the path to this file will end up being /opt/splunk/etc/apps/Splunk_TA_paloalto/local/inputs.conf (or on Windows, C:\\Program Files\\Splunk\\etc\\apps\\Splunk_TA_paloalto\\local\\inputs.conf. You can click Download File below to grab the file.
[monitor:///var/log/rsyslog/pan/traffic/*.log]\nhost_regex = traffic/(.*?)-\\d+.log$\nsourcetype = pan:traffic\nindex = netfw\ndisabled = 0\n\n[monitor:///var/log/rsyslog/pan/threat/*.log]\nhost_regex = threat/(.*?)-\\d+.log$\nsourcetype = pan:threat\nindex = netproxy\ndisabled = 0\n\n[monitor:///var/log/rsyslog/pan/system/*.log]\nhost_regex = system/(.*?)-\\d+.log$\nsourcetype = pan:system\nindex = netops\ndisabled = 0\n\n\n[monitor:///var/log/rsyslog/pan/config/*.log]\nhost_regex = config/(.*?)-\\d+.log$\nsourcetype = pan:config\nindex = netops\ndisabled = 0\n\n[monitor:///var/log/rsyslog/pan/hipmatch/*.log]\nhost_regex = hipmatch/(.*?)-\\d+.log$\nsourcetype = pan:hipmatch\nindex = epintel\ndisabled = 0\n\n[monitor:///var/log/rsyslog/pan/endpoint/*.log]\nhost_regex = endpoint/(.*?)-\\d+.log$\nsourcetype = pan:endpoint\nindex = epintel\ndisabled = 0\n\n[monitor:///var/log/rsyslog/pan/correlation/*.log]\nhost_regex = correlation/(.*?)-\\d+.log$\nsourcetype = pan:correlation\nindex = pan\ndisabled = 0\n\n[monitor:///var/log/rsyslog/pan/aperture/*.log]\nhost_regex = aperture/(.*?)-\\d+.log$\nsourcetype = pan:aperture\nindex = pan\ndisabled = 0\n\n[monitor:///var/log/rsyslog/pan/wildfire/*.log]\nhost_regex = wildfire/(.*?)-\\d+.log$\nsourcetype = pan:wildfire\nindex = epintel\ndisabled = 0 ',
category: ["Splunk Configuration for Data Source"],
},
{
id: "VC-ASA-2-rsyslog-config",
title: "Configuring rsyslog",
"short-description":
'Overview Before we configure the Cisco ASA device, we need to setup a rsyslog server. The supplied config assumes a "vanilla" install of rsyslog. If you already have a rsyslog server in place, you need to validate this config for your deployment.
Implementation Many customers have experienced issues, as the version of rsyslog shipped with RHEL and Ubuntu is out of date and no longer supported. Run rsyslogd -version and validate that it is running a version higher than or equal to 8.32.0:
If the version is not higher or equal to 8.32.0-even after updating your distribution-please add the rsyslog repositories to your distribution and update again. These repos can be found here:
Install rsyslog server on your choice of Linux flavors using the appropriate command, i.e., Yum install rsyslog or apt-get install rsyslog. Some distributions come with rsyslog preinstalled. Make sure that your /etc/rsyslog.conf has a rule that reads $IncludeConfig /etc/rsyslog.d/*.conf.
Copy the following two files into /etc/rsyslog.d/ -- we recommend calling them file splunk.conf and splunk-cisco_asa.conf. splunk.conf will contain all of the global rsyslog configurations, splunk-cisco_asa.conf contains all of the Cisco ASA specific configurations.
# Provides UDP syslog reception, leave these out if your server is already listening to a network port for receiving syslog.\n# for parameters see http://www.rsyslog.com/doc/imudp.html\n# alternatively these can be commented out from /etc/rsyslog.conf\nmodule(load="imudp") # needs to be done just once\ninput(type="imudp" port="514")\n \n# Provides TCP syslog reception Optional; in case you would like to use TCP as a preferred transport mechanism.\n# for parameters see http://www.rsyslog.com/doc/imtcp.html\n# module(load="imtcp") # needs to be done just once\n# input(type="imtcp" port="514")\n # Rsyslog configuration file\n# The purpose of this config is to receive data from a Cisco ASA firewall on a vanilla rsyslog environment\n# and stage it for a Splunk universal forwarder\n# For an existing server read the comments\n# 2018-26-01 Filip Wijnholds fwijnholds@splunk.com\n# 2018-30-01 Modified for Cisco ASA - Kyle Champlin kchamplin@splunk.com\n\n\nmodule(load="builtin:omfile")\n$Umask 0022\n# If you are running Splunk in limited privilege mode, make sure to configure the file ownership:\n# $FileOwner splunk\n# $FileGroup splunk\n \n#Filters Data and writes to files per Sourcetype:\n$template asa,"/var/log/rsyslog/cisco/asa/%HOSTNAME%-%$MINUTE%.log"\n\n\n#From splunk transforms.conf in Cisco ASA TA\n#[force_sourcetype_for_cisco_asa]\n#DEST_KEY = MetaData:Sourcetype\n#REGEX = %ASA-\\d-\\d{6}\n#FORMAT = sourcetype::cisco:asa\n\n#Looks for sourcetypes in the data, to\n:msg, regex, "%ASA-\\d-\\d{6}" ?asa Restart the rsyslog service by running: sudo service rsyslog restart
Validation After the daemon is restarted and traffic is sent to rsyslog, you should see at least this directory being created:
/var/logs/rsyslog/cisco/asa/ ',
category: ["System Configuration"],
},
{
id: "VC-ASA-3-logrotate",
title: "Configuring logrotate",
"short-description":
'Overview With the above configuration, we will be able to receive data via UDP syslog from our Cisco ASA device. However, left to its own whims, rsyslog would fill up all the disk space available to it -- not desireable. The most common way to handle this is to use logrotate, which is ubiquitous on Linux. The below configuration will automatically rotate all of your Cisco ASA log files every day, compress the older ones, and then delete the oldest files.
Implementation Create a file called splunk-ciscoasa in the logrotate directory /etc/logrotate.d/ with the following contents, or just click download below.
/var/log/rsyslog/cisco/asa/*.log\n{\n daily\n compress\n delaycompress\n rotate 3\n ifempty\n maxage 4\n nocreate\n missingok\n sharedscripts\n postrotate\n /bin/kill -HUP `cat /var/run/syslogd.pid 2> /dev/null` 2> /dev/null || true\n endscript\n} For 99.9% of environmemts, you should be set now, as logrotate will run regularly. If you don\'t see the log files being rotated (see Validation below), you may fall into the 0.01% of environments where you need to configure logrotate to run. The easy way to do this is using crontab (another of those ubiquitous Linux tools).
To edit your crontab, from the terminal run: crontab -e. For a daily schedule that rolls the log at midnight, the following cron job will do:
0 0 * * * root logrotate -f /etc/logrotate.d/splunk-ciscoasa If you are unfamiliar with cron and want a different schedule, visit this website: https://crontab.guru/#0_0_*_*_* .
Implementation After 24 hours you should see a .1 appended to the log files when you run: ls -l /var/log/rsyslog/cisco/asa/
',
category: ["System Configuration"],
},
{
id: "VC-ASA-4-enablesyslog",
title: "General Logging Configuration for Cisco ASA",
"short-description":
'The Cisco ASA should utilize TCP as the syslog transport and will maintain an open TCP port with the rsyslog server. Note: A load balancer must not be placed between the ASA and the syslog server.
DNS Prerequisites:
For each IP address assigned for management of the ASA, ensure both A and R records exist and match For each egress NAT address assigned to the device, ensure A and R records exist and match For each ingress NAT address assigned to the device, ensure the R record matches the internal destination A. The A record for this IP is not required (might cause communication issues) Update the ASA configuration to direct syslog messages to the rsyslog server. You can do this either via the command line or via the Cisco Adaptive Security Device Manager (ASDM)-IPS Device Manager (IDM) GUI.
Here is an example of the Cisco Logging Configuration -- click to zoom in:
Here is an example of the Cisco Syslog Server Configuration -- click to zoom in:
\n',
category: ["System Configuration"],
},
{
id: "VC-ASA-5-ruleconfiguration",
title: "Enable Logging of Events",
"short-description":
'Cisco ASA provides many configuration options for logging and thus can dictate how much visibility you have on your network. Generally speaking, the following are best practices for balancing between load on the system, logging fidelity, and data volume.
All \n
\n \n Review and implement Cisco Firewall Best Practices Guide \'s critical sections \n Ensure network time protocol is synchronized with your company\'s standard server. If a standard does not exist, create one before continuing \n If Terminal Access Controller Access Control System+ is utilized, ensure that command accounting is enabled. \n Logging Level: \n \n \n \n \nSpecific Subsystem Logging Levels \n
\n Edge Firewall: \n \n Ensure the default deny rule from external interface is not configured to log \n Ensure the inbound allow rule for well-known servers (public web, SMTP, VPN, DNS) is not configured to log \n \n \n Border Firewall: \n \n Ensure the default deny rule is configured to log in all directions \n Configure rule logging \n Ensure ICMP egress is logged \n Ensure egress for the following protocols are not logged from specifically authorized servers : \n \n HTTP(s) from web proxy servers \n SMTP from email gateway servers \n DNS from internal recursion servers \n \n \n Ensure ingress for the following protocols are not logged to specifically authorized servers: \n \n HTTP(s) web servers \n DNS authoritative servers \n SMTP to email gateway servers \n \n \n VPN \n \n Ensure all authentication and tunnel status events are logged for accept and failure \n Ensure all user names are logged without masking \n Ensure ingress/egress rules are logged except the following: \n \n Traffic to the web proxy \n Traffic to the internal DNS servers \n \n \n \n Here is an example of the Cisco Logging Filters Configuration -- click to zoom in:
Here is an example of the Cisco Access Rules Logging Configuration -- click to zoom in:
\n',
category: ["System Configuration"],
},
{
id: "SC-ASA-1-SizingEstimate",
title: "Sizing Estimate",
"short-description":
'There is a wide amount of variability in the size of Cisco ASA logs. Each firewall message is typically around 230 bytes. We typically see one message per connection (we recommend logging allowed connections, along with denied connections). The volume depends on the size of your ASA device. It can be +/- 10MB/day for a branch office to north of 50 GB/day for a main datacenter cluster.
Using only Cisco\'s built-in tools, the show ip inspect statistics command will tell you how many connections there have been since last reset. So, one way of estimating event volume is to check that number at the same time on subsequent days and then calculate the number of connections you typically see per day. When multiplied by the general 230 byte number, you will get a decent expectation for data size.
Another common approach is to implement the rsyslog configuration referenced above and then track the size of the files created on disk to determine volume.
These estimates are predicated on logging configuration of "level 6 (informational)," which is detailed later in "Cisco ASA Configuration:"
Edge firewall: Negligible Zone-based firewall: 230 bytes per event VPN Services: 10 kb per session, plus firewall activity Operational: Variable, but typically < 200 MB per day, per Cisco ASA \n',
category: ["Splunk Configuration for Data Source"],
},
{
id: "SC-ASA-2-InstallTA",
title: "Install the Cisco ASA Technology Addon -- TA",
"short-description":
'Overview Splunk has a detailed technology add-on that supports ingesting all the different data types generated by your Cisco ASA Firewall. Like all Splunk technology add-ons, it also includes everything needed to parse out the fields and give them names that are compliant with Splunk\'s Common Information Model, so they can easily be used by the searches in Splunk Security Essentials-along with searches you will find in other community-supported and premium apps.
Implementation Find the TA along with all your other Splunk apps and needs on SplunkBase. You can visit https://splunkbase.splunk.com/ and search for it or you could just follow the direct link: https://splunkbase.splunk.com/app/1620/
As with all Splunk TAs, we recommend you deploy it to all parts of your Splunk environment, for simplicity and uniformity. To install the app, start by downloading the file from the SplunkBase location just shown, and then extract it into the apps folder. On Windows systems, this will be %SPLUNK_HOME%\\etc\\apps folder, or usually C:\\Program Files\\Splunk\\etc\\apps. On Linux systems, this will be $SPLUNK_HOME/etc/apps, or usually /opt/splunk/etc/apps.
Validation You can make sure that Splunk has picked up the presence of the app by running $SPLUNK_HOME/bin/splunk display app (or on Windows, %SPLUNK_HOME%\\bin\\splunk display app), will, after asking you to log in, provide you with a list of installed apps. Most likely, if you see the folder listed alongside the other apps (learned, search, splunk_httpinput, etc.) you will know that it\'s there successfully.
Splunk Cloud Customers : you won\'t be copying any files or folders to your indexers or search heads, but good news! The Splunk Add-on for Cisco ASA is Cloud Self-Service Enabled. So you can just got to Find Apps, and be up and running in seconds.
\n',
category: ["Splunk Configuration for Data Source"],
},
{
id: "SC-ASA-3-IndexesAndSourcetypes",
title: "Indexes and Sourcetypes",
"short-description":
'Overview \nAmongst Splunk’s 15,000+ customers, we’ve done a lot of implementations, and we’ve learned a few things along the way. While you can use any sourcetypes or indexes that you want in the "land of Splunk," we’ve found that the most successful customers follow specific patterns, as it sets them up for success moving forward.
\nImplementation \nThe following is a table of sourcetypes from Splunk documentation. The Splunk Add-on for Cisco ASA provides the index-time and search-time knowledge for the Cisco ASA, Cisco PIX, and Cisco Firewall Services Module (FWSM) devices, using the following sourcetypes. We will be focused on the cisco:asa sourcetype.
\n\n \n Sourcetype
\n Index
\n Description
\n Common Information Model (CIM) Data Model
\n \n \n cisco:asa
\n netfw
\n The system logs of Cisco ASA record user authentication, user session, VPN and intrusion messages.
\n Authentication , Change Analysis , Network Sessions , Network Traffic , Malware
\n \n \n cisco:fwsm
\n netfw
\n The system logs of Cisco FWSM record user authentication, user session, and firewall messages.
\n Authentication , Network Sessions , Network Traffic
\n \n \n cisco:pix
\n netfw
\n The system logs of Cisco PIX record user authentication, user session, and intrusion messages.
\n Authentication , Network Sessions , Network Traffic
\n \n
\nFor our index, we will standardize on the netfw index to store all firewall logs. If you went through the "Indexes Configuration" above in "Indexes and Sourcetypes," you already have this index configured across your environment. If you are forging your own path, we recommend creating the netfw index on any search heads or indexers in your environment or on your single-instance server.
',
category: ["Splunk Configuration for Data Source"],
},
{
id: "SC-ASA-4-inputsconf",
title: "Cisco ASA inputs.conf",
"short-description":
'Overview Configuration files for Cisco ASA inputs tend to be pretty simple. In this case, we just have a single inputs.conf file that will go on the syslog server you will be monitoring. As detailed above in Instruction Expectations and Scaling , you will need some mechanism to distribute these files to the hosts you\'re monitoring. For initial tests or deployments to only your most sensitive systems, it is easy to copy the files to the hosts. For larger distributions, you can use the Splunk deployment server or another code-distribution system, such as SCCM, Puppet, Chef, Ansible, or others.
Implementation Distribute the below inputs.conf file to the Universal Forwarder installed on your syslog server (only where you actually have the rsyslog information). You should create a "local" folder inside of the TA folder. For most customers, the path to this file will end up being /opt/splunk/etc/apps/Splunk_TA_cisco-asa/local/inputs.conf (or on Windows, C:\\Program Files\\Splunk\\etc\\apps\\Splunk_TA_cisco-asa\\local\\inputs.conf. You can click Download File below to grab the file.
[monitor:///var/log/rsyslog/cisco/asa/*.log]\nhost_regex = asa/(.*?)-\\d+.log$\nsourcetype = cisco:asa\nindex = netfw\ndisabled = 0\n\n# search to check\n# index=netfw sourcetype=cisco:asa\n\n ',
category: ["Splunk Configuration for Data Source"],
},
{
id: "SC-Stream-1-Overview",
title: "Stream Overview",
"short-description":
'Splunk Stream is great way to monitor network traffic from a host or via a network tap or span port. The software acts as a network traffic "sniffer." The web GUI interface allows you to choose individual metadata fields that are specific to a network protocol and write that metadata to your Splunk indexers for searching.
This means that you can capture all kinds of useful metadata through Splunk Stream, and even do limited full packet capture! The top use cases for Stream are DNS and DHCP (both protocols where logging is notoriously weak), but many people use Stream to capture HTTP transactions, database queries, emails, and more. Check out all the protocols that Stream can handle!
The simplest way to get Stream set up is as a full, standalone Splunk instance with the Stream app installed. While this initially will act as an indexer, you will add an output configuration to direct Splunk to send the data out to indexers in the Splunk environment. This converts the instance into what is known as a "heavy-weight forwarder" and is the most common way to set up a new Stream environment.
The most common scenario for stream is to just install it on the host that\'s generating the traffic you want to capture, frequently a Windows domain Controller serving DHCP and DNS Server roles. The next most common model is to install Stream on a SPAN port or a network tap, allowing you to have an out-of-band Stream host monitoring the network. The Splunk configuration for that setup is identical, you only need to lean on your network team to help make it happen. Finally, advanced users might configure stream on a universal forwarder (see the Multi-Forwarder Environments section at the bottom).
\n',
category: ["Splunk Configuration for Data Source"],
},
{
id: "SC-Stream-2-SizingEstimate",
title: "Sizing for Splunk Stream",
"short-description":
"Stream is perhaps the most difficult product to properly estimate sizing for inside of the Splunk world, for two reasons. The first is that you can choose what protocols you want to capture, and even then you can apply filters so that you only get a certain percentage of that traffic (for example, only HTTP transactions headed out to the internet). The second reason is that even once you've decided to capture a data stream (such as DNS logs), you can then decide what individual fields you're looking for -- just the query, type, and response? Or everything?
For this reason, the easiest way to size your stream ingest is to just give it a shot. Don't put an outputs.conf on your heavy-weight forwarder, and just see what volume it brings in. Or send it into your Splunk environment and closely monitor the volume.
\n",
category: ["Splunk Configuration for Data Source"],
},
{
id: "SC-Stream-3-IndexesAndSourcetypes",
title: "Stream Indexes and Sourcetypes",
"short-description":
'Overview \nAmongst Splunk’s 15,000+ customers, we’ve done a lot of implementations, and we’ve learned a few things along the way. While you can use any sourcetypes or indexes that you want in the "land of Splunk," we’ve found that the most successful customers follow specific patterns, as it sets them up for success moving forward.
\nImplementation Below is a list of the Stream data types from Splunk docs, along with the recommended sourcetypes and indexes.\n\n\n \n \n Protocol \n Description \n Sourcetype \n Index \n \n \n \n \n AMQP \n Advanced Messaging Queuing Protocol \n stream:amqp \n netfw \n \n \n DHCP \n Dynamic Host Configuration Protocol \n stream:dhcp \n netipam \n \n \n DIAMETER \n Authentication Protocol \n stream:diameter \n streamsec \n \n \n DNS \n Domain Name Service \n stream:dns \n netdns \n \n \n FTP \n File Transfer Protocol \n stream:ftp \n stream \n \n \n HTTP \n Hypertext Transfer Protocol \n stream:http \n netproxy / appwebint / appwebext as appropriate \n \n \n ICMP \n Internet Control Message Protocol \n stream:icmp \n stream \n \n \n IMAP \n Internet Message Access Protocol \n stream:imap \n mail \n \n \n IP \n Internet Protocol \n stream:ip \n stream \n \n \n IRC \n Internet Relay Chat \n stream:irc \n streamsec \n \n \n LDAP \n Lightweight Directory Access Protocol \n stream:ldap \n stream \n \n \n MAPI \n Messaging Application Programming Interface \n stream:mapi \n mail \n \n \n MySQL \n MySQL client/server protocol \n stream:mysql \n db \n \n \n NetBIOS \n Network Basic Input/Output System \n stream:netbios \n stream \n \n \n NFS \n Network File System \n stream:nfs \n stream \n \n \n POP3 \n Post Office Protocol v3 \n stream:pop3 \n mail \n \n \n Postgres \n PostgreSQL \n stream:postgres \n db \n \n \n RADIUS \n Remote Authentication Dial In User Service \n stream:radius \n streamsec \n \n \n RTP \n Real-time Transport Protocol \n stream:rtp \n stream \n \n \n SIP \n Session Initiation Protocol \n stream:sip \n stream \n \n \n SMB \n Server Message Block \n stream:smb \n stream \n \n \n SMPP \n Short Message Peer to Peer \n stream:smpp \n stream \n \n \n SNMP \n Simple Network Management Protocol \n stream:snmp \n stream \n \n \n TCP \n Transmission Control Protocol \n stream:tcp \n stream \n \n \n TDS \n Tabular Data Stream - Sybase/MSSQL \n stream:tds \n db \n \n \n TNS \n Transparent Network Substrate (Oracle) \n stream:tns \n db \n \n \n UDP \n User Datagram Protocol \n stream:udp \n stream \n \n \n XMPP \n Extensible Messaging and Presence Protocol \n stream:xmpp \n streamsec \n \n \n
\n',
category: ["Splunk Configuration for Data Source"],
},
{
id: "SC-Stream-4-InstallStream",
title: "Install the Stream App",
"short-description":
'Log into Splunk and click Splunk Apps .
Search for "Splunk Stream." Click the Install button.
After installation, click Restart Now.
Log back into Splunk and select the Splunk Stream app. Accept the defaults and click Let\'s Get Started .
Now you\'re ready to configure Stream to monitor the relevant network interface on your Windows server and forward the resulting DNS metadata to your Splunk indexers.
\n',
category: ["Splunk Configuration for Data Source"],
},
{
id: "SC-Stream-5-DNS",
title: "Configure a New DNS Stream",
"short-description":
'Implementation Within the Splunk Stream app, select Configuration > Configure Streams .
The Configure Streams dashboard will display the default settings for protocol information to be collected. You\'ll want to disable the defaults, then select the protocol and details to create your new stream.
You can select all of the available protocols and disable them all at once, by clicking the checkbox next to Name on the title bar.
After selecting all of the protocols, click the Disable option.
Now that you\'ve disabled the defaults, create a new stream for collecting the DNS details that you\'d like to capture. Start by selecting the New Stream button, then Metadata Stream .
This will bring you into a workflow that allows you to configure the stream.
Select DNS as the protocol in the first step.
Once DNS is selected, give it a name and description with some context to help you to identify the data. Click Next.
On the aggregation step, ensure that No is selected for aggregation, then click Next. (You don\'t want aggregation because you want to see the individual DNS records.)
On the Fields screen, you\'ll select the fields (specific to DNS) that you want to collect and store in Splunk. Note that some-but not all-fields are selected by default.
Once you\'ve selected the DNS fields that you\'d like to collect, click Next .
You define filtering of the collected data on the Filters screen. The filters are based on the fields you selected on the previous screen. For instance, if you only wanted Stream to capture data from type "A" queries, you could define that here.
Filters are something that you may want to go back and tweak later, once you\'ve collected data for a while and know what you have and what you\'d like to keep (or discard).
After defining filters, select the Next button again to go to the Settings screen, where you\'ll define the destination index for your DNS data.
Select the destination index from the dropdown menu. You should select netdns in this dropdown. If you don\'t see netdns listed here, it is because you missed a step in the Indexes and Sourcetypes section. We recommend installing the standard set of indexes on any Heavy-weight Forwarder running Splunk Stream not because it will actually store data in those indexes, but because it will show up in the dropdown here. If you missed that step, don\'t fret -- you\'re almost done. Go ahead and finish this section, then go up to Indexes and Sourcetypes Overview to install the indexes. You can come back here and edit your stream afterwards.
After selecting the destination index, you can choose to save the configuration in Disabled mode, if you\'re not quite ready to begin collecting data. You can also put it into Estimate mode to get an idea of how much data you\'ll be collecting once the configuration is enabled.
On the Groups screen, you\'ll have the ability to select a group with which to associate the Stream configuration. In this case, because you are not configuring distributed forwarders, you have not created separate groups-so leave defaultgroup selected. Finally, click Create Stream to save your configuration! You\'re done!
Validation If you\'ve enabled the configuration, you should now be collecting DNS data. You can validate this by searching for:
index=netdns sourcetype=stream:dns You should able to see beautiful JSON blobs of DNS transactions, with fields available on the left.
\n',
category: ["Splunk Configuration for Data Source"],
},
{
id: "SC-Stream-6-MultiServer",
title: "(Optional) Multi-Forwarder Environments",
"short-description":
'Overview If you only have a small handful of stream hosts, it is by far easiest to just install the Heavy-weight forwarder and manually configure it. But if you are planning to roll out a fleet of Stream sensors throughout your network, you will want to centrally monitor them. While Stream can be deployed via the Deployment Server, the actual stream configuration is managed via a different model. We will walk through that model below, but the high level summary is that you can deploy the Stream Technology Add-on (TA) onto Universal Forwarders (no requirement for heavy-weight forwarders for the TA), and tell them to all point to a central Stream configuration server over your standard Splunk port (default http over 8000). See below for the full setup.
Implementation Note that there are two primary components in Splunk Stream. First is the Splunk Stream app, which provides the web interface and allows stream configuration. This component exposes the configuration you build to clients. The client (Splunk_TA_stream) gets its configuration from the Splunk Stream app via REST API. In the above example of a standalone configuration, both of these components are installed (Splunk_TA_stream comes as part of the Stream app that you download from Splunkbase). In a standalone configuration, the request and transfer of configuration information from server to client takes place on the local network stack. In a distributed configuration, the request and transfer of configuration takes place over the wire.
The location of the Splunk Stream management server is stored in inputs.conf.
You\'ll need the Splunk_TA_stream app for a forwarder configuration. The custom inputs.conf that resides in that app should point to your remote Stream server, as below.
[streamfwd://streamfwd]\nsplunk_stream_app_location = http://remote_stream_server:8000/en-us/custom/splunk_app_stream/\nstream_forwarder_id = \ndisabled = 0 Don\'t forget to modify the protocol if you\'re using if you\'re utilizing SSL/TLS on your Stream server.
Final Notes: when using this configuration, don\'t forget that your stream forwarders will need to connect home to the Splunk Stream server, so network access will be required. You will also need to adjust the frequency that they call home if you deploy a large number (hundreds or thousands), which you can do by adding the "pingInterval" setting on the streamfwd.conf. The default value is 5 seconds, but in larger environments an interval of many minutes is usually more than sufficient.
\n',
category: ["Splunk Configuration for Data Source"],
},
{
id: "VC-ZSCALER-1-overview",
title: "Zscaler Overview",
"short-description":
"Overview Zscaler can stream logs into the customer environments. This is facilitated via Zscaler supplied virtual machines which execute in a customer’s (or partner’s) hosted compute environment. These virtual machines attach to the Zscaler cloud via outbound connections and receive encrypted and tokenized logs to stream into customer log collection and SIEM platforms.
For full details on how to configure Zscaler data collection and onboarding, visit the Splunk Lantern site for Zscaler. https://lantern.splunk.com/Data_Descriptors/Data_Sources/Zscaler/Zscaler
",
category: ["System Configuration"],
},
{
id: "SC-Placeholder",
title: "Placeholder",
"short-description":
"Placeholder -- we do not have content for this technology yet.
",
category: ["Splunk Configuration for Data Source"],
},
{
id: "SC-Other",
title: "Other Data Source",
"short-description":
'We weren\'t able to find that data source. This likely means that we haven\'t built out documentation for this data source yet. Please check the Splunkbase page for the app.
\n',
category: ["System Configuration"],
},
]