This is a very basic notebook to demonstrate using Jupyter notebooks to analyze data stored in Elastic Search rather than Kibana. It uses an R kernel, however the same approach will work with a Python or other kernel as well.
If you were actually querying ES, you'd run this stuff
### USER EDITABLE STATIC VARIABLES
host <- "###.###.###.###" # IP address of ES host
port <- 5601 # ES Host port
index <- NULL # The indexes you want to query
### Load libraries
library("elastic")
library("tidyverse")
library("lubridate")
library("ggthemes")
library("viridis")
### Set Constants
## change search default values to return something else from ES
## These are similar to a basic default query against something like ES or Splunk
formals(Search)$index <- index
formals(Search)$size <- 10000 # the max query size
formals(Search)$q <- "*" # give us everything (this is the main search thing)
formals(Search)$asdf <- TRUE # make it into a table
# formals(elastic::Search)$sort <- "@timestamp:desc"
formals(Search)$time_scroll <- "5m" # we're going to scroll so we get more than the max query size
### Define Functions
## This takes care of some of the basics of joining multiple scrolls on the queries
ES <- function(...) {
# search
res <- Search(...)
# scroll through search. Open 1m
df <- tibble::tibble()
hits <- 1
while(hits != 0){
res <- elastic::scroll(res$`_scroll_id`, asdf=TRUE)
hits <- length(res$hits$hits)
if(hits > 0)
df <- dplyr::bind_rows(df, res$hits$hits)
}
# return(out) # DEBUG
# clean up
names(df) <- gsub("^_(.*)", "\\1", names(df))
df$timestamp <- lubridate::ymd_hms(df$`source.@timestamp`)
df
}
### Set up initial connection
elastic::connect(es_host = host, port = port)
elastic::index_get()
elastic::cat_indices()
## after this we'll just use the 'ES()' function to query elastic search
### For demonstration purposes, we'll just load a pre-generated query
load("THE PATH TO elk_jupyter_r_blog.Rda", verbose=TRUE) # verbose=TRUE means it'll tell you the name of the dataframe loaded. Hint, it's "df".
### basic search
df <- ES()
### For demonstration purposes, we'll just load a pre-generated query
load("/THE/PATH/TO/elk_jupyter_r_blog.Rda", verbose=TRUE) # verbose=TRUE means it'll tell you the name of the dataframe loaded. Hint, it's "df".
### Lets take a quick look at what our dataframe looks like
glimpse(df)
Observations: 24,966 Variables: 394 $ index <chr> "filebeat-20... $ type <chr> "log", "log"... $ id <chr> "AVzxqgKZG5k... $ score <dbl> 1, 1, 1, 1, ... $ `source.@timestamp` <chr> "2017-12-02T... $ source.input_type <chr> "log", "log"... $ source.message <chr> "Dec 1 19:0... $ source.offset <int> 2908116, 290... $ source.source <chr> "/var/log/au... $ source.type <chr> "log", "log"... $ source.beat.hostname <chr> "blueteam-vi... $ source.beat.name <chr> "blueteam-vi... $ source.beat.version <chr> "5.4.2", "5.... $ source.computer_name <chr> NA, NA, NA, ... $ source.event_id <int> NA, NA, NA, ... $ source.keywords <list> [NULL, NULL... $ source.level <chr> NA, NA, NA, ... $ source.log_name <chr> NA, NA, NA, ... $ source.process_id <int> NA, NA, NA, ... $ source.provider_guid <chr> NA, NA, NA, ... $ source.record_number <chr> NA, NA, NA, ... $ source.source_name <chr> NA, NA, NA, ... $ source.thread_id <int> NA, NA, NA, ... $ source.opcode <chr> NA, NA, NA, ... $ source.task <chr> NA, NA, NA, ... $ source.version <int> NA, NA, NA, ... $ source.activity_id <chr> NA, NA, NA, ... $ source.event_data.Binary <chr> NA, NA, NA, ... $ source.event_data.param1 <chr> NA, NA, NA, ... $ source.event_data.param2 <chr> NA, NA, NA, ... $ source.event_data.StopTime <chr> NA, NA, NA, ... $ source.event_data.ShutdownActionType <chr> NA, NA, NA, ... $ source.event_data.ShutdownEventCode <chr> NA, NA, NA, ... $ source.event_data.ShutdownReason <chr> NA, NA, NA, ... $ source.event_data.param4 <chr> NA, NA, NA, ... $ source.event_data.param5 <chr> NA, NA, NA, ... $ source.event_data.BootType <chr> NA, NA, NA, ... $ source.event_data.DeviceName <chr> NA, NA, NA, ... $ source.event_data.DeviceNameLength <chr> NA, NA, NA, ... $ source.event_data.DeviceTime <chr> NA, NA, NA, ... $ source.event_data.DeviceVersionMajor <chr> NA, NA, NA, ... $ source.event_data.DeviceVersionMinor <chr> NA, NA, NA, ... $ source.event_data.FinalStatus <chr> NA, NA, NA, ... $ source.event_data.Group <chr> NA, NA, NA, ... $ source.event_data.IdleImplementation <chr> NA, NA, NA, ... $ source.event_data.IdleStateCount <chr> NA, NA, NA, ... $ source.event_data.MaximumPerformancePercent <chr> NA, NA, NA, ... $ source.event_data.MinimumPerformancePercent <chr> NA, NA, NA, ... $ source.event_data.MinimumThrottlePercent <chr> NA, NA, NA, ... $ source.event_data.NominalFrequency <chr> NA, NA, NA, ... $ source.event_data.Number <chr> NA, NA, NA, ... $ source.event_data.PerformanceImplementation <chr> NA, NA, NA, ... $ source.event_data.DirtyPages <chr> NA, NA, NA, ... $ source.event_data.HiveName <chr> NA, NA, NA, ... $ source.event_data.HiveNameLength <chr> NA, NA, NA, ... $ source.event_data.KeysUpdated <chr> NA, NA, NA, ... $ source.event_data.IpAddress <chr> NA, NA, NA, ... $ source.event_data.IpPort <chr> NA, NA, NA, ... $ source.event_data.LogonGuid <chr> NA, NA, NA, ... $ source.event_data.ProcessId <chr> NA, NA, NA, ... $ source.event_data.ProcessName <chr> NA, NA, NA, ... $ source.event_data.SubjectDomainName <chr> NA, NA, NA, ... $ source.event_data.SubjectLogonId <chr> NA, NA, NA, ... $ source.event_data.SubjectUserName <chr> NA, NA, NA, ... $ source.event_data.SubjectUserSid <chr> NA, NA, NA, ... $ source.event_data.TargetDomainName <chr> NA, NA, NA, ... $ source.event_data.TargetInfo <chr> NA, NA, NA, ... $ source.event_data.TargetLogonGuid <chr> NA, NA, NA, ... $ source.event_data.TargetServerName <chr> NA, NA, NA, ... $ source.event_data.TargetUserName <chr> NA, NA, NA, ... $ source.event_data.PrivilegeList <chr> NA, NA, NA, ... $ source.event_data.AuthenticationPackageName <chr> NA, NA, NA, ... $ source.event_data.ImpersonationLevel <chr> NA, NA, NA, ... $ source.event_data.KeyLength <chr> NA, NA, NA, ... $ source.event_data.LmPackageName <chr> NA, NA, NA, ... $ source.event_data.LogonProcessName <chr> NA, NA, NA, ... $ source.event_data.LogonType <chr> NA, NA, NA, ... $ source.event_data.TargetLogonId <chr> NA, NA, NA, ... $ source.event_data.TargetUserSid <chr> NA, NA, NA, ... $ source.event_data.TransmittedServices <chr> NA, NA, NA, ... $ source.event_data.HandleId <chr> NA, NA, NA, ... $ source.event_data.NewSd <chr> NA, NA, NA, ... $ source.event_data.ObjectName <chr> NA, NA, NA, ... $ source.event_data.ObjectServer <chr> NA, NA, NA, ... $ source.event_data.ObjectType <chr> NA, NA, NA, ... $ source.event_data.SamAccountName <chr> NA, NA, NA, ... $ source.event_data.SidHistory <chr> NA, NA, NA, ... $ source.event_data.TargetSid <chr> NA, NA, NA, ... $ source.event_data.NewTargetUserName <chr> NA, NA, NA, ... $ source.event_data.OldTargetUserName <chr> NA, NA, NA, ... $ source.event_data.DwordVal <chr> NA, NA, NA, ... $ source.event_data.AccountExpires <chr> NA, NA, NA, ... $ source.event_data.AllowedToDelegateTo <chr> NA, NA, NA, ... $ source.event_data.DisplayName <chr> NA, NA, NA, ... $ source.event_data.Dummy <chr> NA, NA, NA, ... $ source.event_data.HomeDirectory <chr> NA, NA, NA, ... $ source.event_data.HomePath <chr> NA, NA, NA, ... $ source.event_data.LogonHours <chr> NA, NA, NA, ... $ source.event_data.NewUacValue <chr> NA, NA, NA, ... $ source.event_data.OldUacValue <chr> NA, NA, NA, ... $ source.event_data.PasswordLastSet <chr> NA, NA, NA, ... $ source.event_data.PrimaryGroupId <chr> NA, NA, NA, ... $ source.event_data.ProfilePath <chr> NA, NA, NA, ... $ source.event_data.ScriptPath <chr> NA, NA, NA, ... $ source.event_data.UserAccountControl <chr> NA, NA, NA, ... $ source.event_data.UserParameters <chr> NA, NA, NA, ... $ source.event_data.UserPrincipalName <chr> NA, NA, NA, ... $ source.event_data.UserWorkstations <chr> NA, NA, NA, ... $ source.event_data.ObjectCollectionName <chr> NA, NA, NA, ... $ source.event_data.ObjectIdentifyingProperties <chr> NA, NA, NA, ... $ source.event_data.ObjectProperties <chr> NA, NA, NA, ... $ source.event_data.SubjectUserDomainName <chr> NA, NA, NA, ... $ source.event_data.WorkstationName <chr> NA, NA, NA, ... $ source.event_data.NewTime <chr> NA, NA, NA, ... $ source.event_data.PreviousTime <chr> NA, NA, NA, ... $ source.event_data.param3 <chr> NA, NA, NA, ... $ source.event_data.AccountName <chr> NA, NA, NA, ... $ source.event_data.ImagePath <chr> NA, NA, NA, ... $ source.event_data.ServiceName <chr> NA, NA, NA, ... $ source.event_data.ServiceType <chr> NA, NA, NA, ... $ source.event_data.StartType <chr> NA, NA, NA, ... $ source.event_data.OldTime <chr> NA, NA, NA, ... $ source.event_data.Reason <chr> NA, NA, NA, ... $ source.event_data.TSId <chr> NA, NA, NA, ... $ source.event_data.UserSid <chr> NA, NA, NA, ... $ source.event_data.BitlockerUserInputTime <chr> NA, NA, NA, ... $ source.event_data.DomainPeer <chr> NA, NA, NA, ... $ source.event_data.ErrorMessage <chr> NA, NA, NA, ... $ source.event_data.RetryMinutes <chr> NA, NA, NA, ... $ source.event_data.MemberName <chr> NA, NA, NA, ... $ source.event_data.MemberSid <chr> NA, NA, NA, ... $ source.event_data.AuditSourceName <chr> NA, NA, NA, ... $ source.event_data.EventSourceId <chr> NA, NA, NA, ... $ source.event_data.BootMode <chr> NA, NA, NA, ... $ source.event_data.BuildVersion <chr> NA, NA, NA, ... $ source.event_data.MajorVersion <chr> NA, NA, NA, ... $ source.event_data.MinorVersion <chr> NA, NA, NA, ... $ source.event_data.QfeVersion <chr> NA, NA, NA, ... $ source.event_data.ServiceVersion <chr> NA, NA, NA, ... $ source.event_data.StartTime <chr> NA, NA, NA, ... $ source.event_data.IdleState <chr> NA, NA, NA, ... $ source.event_data.PerfStateCount <chr> NA, NA, NA, ... $ source.event_data.ThrottleStateCount <chr> NA, NA, NA, ... $ source.event_data.param7 <chr> NA, NA, NA, ... $ source.event_data.DeviceObject <chr> NA, NA, NA, ... $ source.event_data.Url <chr> NA, NA, NA, ... $ `source.event_data.OS EditionID` <chr> NA, NA, NA, ... $ `source.event_data.OS Name` <chr> NA, NA, NA, ... $ `source.event_data.OS build version` <chr> NA, NA, NA, ... $ `source.event_data.OS major version` <chr> NA, NA, NA, ... $ `source.event_data.OS minor version` <chr> NA, NA, NA, ... $ `source.event_data.OS service pack major version` <chr> NA, NA, NA, ... $ `source.event_data.OS service pack minor version` <chr> NA, NA, NA, ... $ source.event_data.param6 <chr> NA, NA, NA, ... $ source.event_data.NewDate <chr> NA, NA, NA, ... $ source.event_data.PreviousDate <chr> NA, NA, NA, ... $ source.event_data.PuaCount <chr> NA, NA, NA, ... $ source.event_data.PuaPolicyId <chr> NA, NA, NA, ... $ source.event_data.ErrorCode <chr> NA, NA, NA, ... $ source.event_data.PackageName <chr> NA, NA, NA, ... $ source.event_data.Status <chr> NA, NA, NA, ... $ source.event_data.Workstation <chr> NA, NA, NA, ... $ source.event_data.Address <chr> NA, NA, NA, ... $ source.event_data.Interface <chr> NA, NA, NA, ... $ source.event_data.ProtocolType <chr> NA, NA, NA, ... $ source.event_data.Attributes <chr> NA, NA, NA, ... $ source.event_data.BiosInitDuration <chr> NA, NA, NA, ... $ source.event_data.DriverInitDuration <chr> NA, NA, NA, ... $ source.event_data.EffectiveState <chr> NA, NA, NA, ... $ source.event_data.HiberPagesWritten <chr> NA, NA, NA, ... $ source.event_data.HiberReadDuration <chr> NA, NA, NA, ... $ source.event_data.HiberWriteDuration <chr> NA, NA, NA, ... $ source.event_data.SleepDuration <chr> NA, NA, NA, ... $ source.event_data.SleepTime <chr> NA, NA, NA, ... $ source.event_data.TargetState <chr> NA, NA, NA, ... $ source.event_data.WakeDuration <chr> NA, NA, NA, ... $ source.event_data.WakeSourceTextLength <chr> NA, NA, NA, ... $ source.event_data.WakeSourceType <chr> NA, NA, NA, ... $ source.event_data.WakeTime <chr> NA, NA, NA, ... $ source.event_data.WakeTimerContextLength <chr> NA, NA, NA, ... $ source.event_data.WakeTimerOwnerLength <chr> NA, NA, NA, ... $ source.event_data.Processor <chr> NA, NA, NA, ... $ source.event_data.MandatoryLabel <chr> NA, NA, NA, ... $ source.event_data.NewProcessId <chr> NA, NA, NA, ... $ source.event_data.NewProcessName <chr> NA, NA, NA, ... $ source.event_data.ParentProcessName <chr> NA, NA, NA, ... $ source.event_data.TokenElevationType <chr> NA, NA, NA, ... $ source.event_data.ElevatedToken <chr> NA, NA, NA, ... $ source.event_data.RestrictedAdminMode <chr> NA, NA, NA, ... $ source.event_data.TargetLinkedLogonId <chr> NA, NA, NA, ... $ source.event_data.TargetOutboundDomainName <chr> NA, NA, NA, ... $ source.event_data.TargetOutboundUserName <chr> NA, NA, NA, ... $ source.event_data.VirtualAccount <chr> NA, NA, NA, ... $ source.event_data.CorruptionActionState <chr> NA, NA, NA, ... $ source.event_data.DriveName <chr> NA, NA, NA, ... $ source.event_data.ExtraInfoLength <chr> NA, NA, NA, ... $ source.event_data.ExtraInfoString <chr> NA, NA, NA, ... $ source.event_data.FilterID <chr> NA, NA, NA, ... $ source.event_data.Turn <chr> NA, NA, NA, ... $ source.event_data.EntryCount <chr> NA, NA, NA, ... $ source.event_data.AlgorithmName <chr> NA, NA, NA, ... $ source.event_data.KeyFilePath <chr> NA, NA, NA, ... $ source.event_data.KeyName <chr> NA, NA, NA, ... $ source.event_data.KeyType <chr> NA, NA, NA, ... $ source.event_data.Operation <chr> NA, NA, NA, ... $ source.event_data.ProviderName <chr> NA, NA, NA, ... $ source.event_data.ReturnCode <chr> NA, NA, NA, ... $ source.event_data.Flags <chr> NA, NA, NA, ... $ source.event_data.FailureReason <chr> NA, NA, NA, ... $ source.event_data.SubStatus <chr> NA, NA, NA, ... $ source.event_data.TimeSource <chr> NA, NA, NA, ... $ source.user.domain <chr> NA, NA, NA, ... $ source.user.identifier <chr> NA, NA, NA, ... $ source.user.name <chr> NA, NA, NA, ... $ source.user.type <chr> NA, NA, NA, ... $ source.user_data.DeviceInstanceID <chr> NA, NA, NA, ... $ source.user_data.DriverDescription <chr> NA, NA, NA, ... $ source.user_data.DriverName <chr> NA, NA, NA, ... $ source.user_data.DriverProvider <chr> NA, NA, NA, ... $ source.user_data.DriverVersion <chr> NA, NA, NA, ... $ source.user_data.InstallStatus <chr> NA, NA, NA, ... $ source.user_data.IsDriverOEM <chr> NA, NA, NA, ... $ source.user_data.RebootOption <chr> NA, NA, NA, ... $ source.user_data.SetupClass <chr> NA, NA, NA, ... $ source.user_data.UpgradeDevice <chr> NA, NA, NA, ... $ source.user_data.xml_name <chr> NA, NA, NA, ... $ source.user_data.AddServiceStatus <chr> NA, NA, NA, ... $ source.user_data.DriverFileName <chr> NA, NA, NA, ... $ source.user_data.PrimaryService <chr> NA, NA, NA, ... $ source.user_data.ServiceName <chr> NA, NA, NA, ... $ source.user_data.UpdateService <chr> NA, NA, NA, ... $ source.user_data.CachingSubsystemState <chr> NA, NA, NA, ... $ source.user_data.InstallSubsystemState <chr> NA, NA, NA, ... $ source.user_data.ErrorCode <chr> NA, NA, NA, ... $ source.user_data.Operation <chr> NA, NA, NA, ... $ source.user_data.OperationCompleted <chr> NA, NA, NA, ... $ source.user_data.PackageAssembly <chr> NA, NA, NA, ... $ source.user_data.PackageIdentifier <chr> NA, NA, NA, ... $ source.user_data.PackageState <chr> NA, NA, NA, ... $ source.event_data.ProcessingMode <chr> NA, NA, NA, ... $ source.event_data.ProcessingTimeInMilliseconds <chr> NA, NA, NA, ... $ source.event_data.SupportInfo1 <chr> NA, NA, NA, ... $ source.event_data.SupportInfo2 <chr> NA, NA, NA, ... $ source.event_data.CallerProcessId <chr> NA, NA, NA, ... $ source.event_data.CallerProcessName <chr> NA, NA, NA, ... $ source.event_data.param10 <chr> NA, NA, NA, ... $ source.event_data.param11 <chr> NA, NA, NA, ... $ source.event_data.param8 <chr> NA, NA, NA, ... $ source.event_data.param9 <chr> NA, NA, NA, ... $ source.event_data.updateGuid <chr> NA, NA, NA, ... $ source.event_data.updateRevisionNumber <chr> NA, NA, NA, ... $ source.event_data.updateTitle <chr> NA, NA, NA, ... $ source.event_data.serviceGuid <chr> NA, NA, NA, ... $ source.event_data.NewSchemeGuid <chr> NA, NA, NA, ... $ source.event_data.OldSchemeGuid <chr> NA, NA, NA, ... $ source.event_data.ProcessPath <chr> NA, NA, NA, ... $ source.event_data.ProcessPid <chr> NA, NA, NA, ... $ source.event_data.OldSd <chr> NA, NA, NA, ... $ source.event_data.UnsynchronizedTimeSeconds <chr> NA, NA, NA, ... $ source.event_data.AddressLength <chr> NA, NA, NA, ... $ source.event_data.QueryName <chr> NA, NA, NA, ... $ source.event_data.LastBootGood <chr> NA, NA, NA, ... $ source.event_data.LastShutdownGood <chr> NA, NA, NA, ... $ source.event_data.ProgrammedWakeTimeAc <chr> NA, NA, NA, ... $ source.event_data.ProgrammedWakeTimeDc <chr> NA, NA, NA, ... $ source.event_data.WakeFromState <chr> NA, NA, NA, ... $ source.event_data.WakeRequesterTypeAc <chr> NA, NA, NA, ... $ source.event_data.WakeRequesterTypeDc <chr> NA, NA, NA, ... $ source.event_data.ModifiedObjectProperties <chr> NA, NA, NA, ... $ source.event_data.ConfigurationReader <chr> NA, NA, NA, ... $ source.event_data.RunningMode <chr> NA, NA, NA, ... $ source.event_data.Endpoint <chr> NA, NA, NA, ... $ source.event_data.AccessGranted <chr> NA, NA, NA, ... $ source.event_data.AccessList <chr> NA, NA, NA, ... $ source.event_data.AccessMask <chr> NA, NA, NA, ... $ source.event_data.AdditionalInfo <chr> NA, NA, NA, ... $ source.event_data.OperationType <chr> NA, NA, NA, ... $ source.event_data.Properties <chr> NA, NA, NA, ... $ source.event_data.ServiceSid <chr> NA, NA, NA, ... $ source.event_data.TicketEncryptionType <chr> NA, NA, NA, ... $ source.event_data.TicketOptions <chr> NA, NA, NA, ... $ source.event_data.PreAuthType <chr> NA, NA, NA, ... $ source.event_data.SecurityPackage <chr> NA, NA, NA, ... $ source.event_data.DCName <chr> NA, NA, NA, ... $ source.event_data.NumberOfGroupPolicyObjects <chr> NA, NA, NA, ... $ source.event_data.DomainBehaviorVersion <chr> NA, NA, NA, ... $ source.event_data.DomainName <chr> NA, NA, NA, ... $ source.event_data.DomainPolicyChanged <chr> NA, NA, NA, ... $ source.event_data.DomainSid <chr> NA, NA, NA, ... $ source.event_data.ForceLogoff <chr> NA, NA, NA, ... $ source.event_data.LockoutDuration <chr> NA, NA, NA, ... $ source.event_data.LockoutObservationWindow <chr> NA, NA, NA, ... $ source.event_data.MachineAccountQuota <chr> NA, NA, NA, ... $ source.event_data.MinPasswordAge <chr> NA, NA, NA, ... $ source.event_data.MinPasswordLength <chr> NA, NA, NA, ... $ source.event_data.MixedDomainMode <chr> NA, NA, NA, ... $ source.event_data.OemInformation <chr> NA, NA, NA, ... $ source.event_data.PasswordHistoryLength <chr> NA, NA, NA, ... $ source.event_data.PasswordProperties <chr> NA, NA, NA, ... $ source.event_data.NoMultiStageResumeReason <chr> NA, NA, NA, ... $ source.event_data.ComputerName <chr> NA, NA, NA, ... $ source.event_data.AdapterName <chr> NA, NA, NA, ... $ source.event_data.AdapterSuffixName <chr> NA, NA, NA, ... $ source.event_data.DnsServerList <chr> NA, NA, NA, ... $ source.event_data.HostName <chr> NA, NA, NA, ... $ source.event_data.Ipaddress <chr> NA, NA, NA, ... $ `source.event_data.Sent UpdateServer` <chr> NA, NA, NA, ... $ source.event_data.IfGuid <chr> NA, NA, NA, ... $ source.event_data.IfIndex <chr> NA, NA, NA, ... $ source.event_data.IfLuid <chr> NA, NA, NA, ... $ source.event_data.ResetCount <chr> NA, NA, NA, ... $ source.event_data.ResetReason <chr> NA, NA, NA, ... $ source.event_data.AlertDesc <chr> NA, NA, NA, ... $ source.event_data.ErrorState <chr> NA, NA, NA, ... $ source.event_data.Type <chr> NA, NA, NA, ... $ source.event_data.NewSize <chr> NA, NA, NA, ... $ source.event_data.OriginalSize <chr> NA, NA, NA, ... $ source.user_data.Client <chr> NA, NA, NA, ... $ source.user_data.InitialPackageState <chr> NA, NA, NA, ... $ source.user_data.IntendedPackageState <chr> NA, NA, NA, ... $ source.user_data.ReleaseType <chr> NA, NA, NA, ... $ source.user_data.SupportInformation <chr> NA, NA, NA, ... $ source.user_data.Argument <chr> NA, NA, NA, ... $ source.user_data.UpdateDisplayName <chr> NA, NA, NA, ... $ source.user_data.UpdateName <chr> NA, NA, NA, ... $ source.user_data.UpdateState <chr> NA, NA, NA, ... $ source.event_data.AdvancedOptions <chr> NA, NA, NA, ... $ source.event_data.ConfigAccessPolicy <chr> NA, NA, NA, ... $ source.event_data.DisableIntegrityChecks <chr> NA, NA, NA, ... $ source.event_data.FlightSigning <chr> NA, NA, NA, ... $ source.event_data.HypervisorDebug <chr> NA, NA, NA, ... $ source.event_data.HypervisorLaunchType <chr> NA, NA, NA, ... $ source.event_data.HypervisorLoadOptions <chr> NA, NA, NA, ... $ source.event_data.KernelDebug <chr> NA, NA, NA, ... $ source.event_data.LoadOptions <chr> NA, NA, NA, ... $ source.event_data.RemoteEventLogging <chr> NA, NA, NA, ... $ source.event_data.TestSigning <chr> NA, NA, NA, ... $ source.event_data.VsmLaunchType <chr> NA, NA, NA, ... $ source.event_data.TimeOffsetSeconds <chr> NA, NA, NA, ... $ source.event_data.ErrorDescription <chr> NA, NA, NA, ... $ source.event_data.AppPoolID <chr> NA, NA, NA, ... $ source.event_data.Minutes <chr> NA, NA, NA, ... $ source.event_data.ProcessID <chr> NA, NA, NA, ... $ `source.event_data.Host OS Name` <chr> NA, NA, NA, ... $ `source.event_data.Host OS build version` <chr> NA, NA, NA, ... $ `source.event_data.Host OS major version` <chr> NA, NA, NA, ... $ `source.event_data.Host OS minor version` <chr> NA, NA, NA, ... $ `source.event_data.Host OS service pack major version` <chr> NA, NA, NA, ... $ `source.event_data.Host OS service pack minor version` <chr> NA, NA, NA, ... $ `source.event_data.Host OS was Windows PE` <chr> NA, NA, NA, ... $ `source.event_data.Install was an upgrade` <chr> NA, NA, NA, ... $ source.event_data.BootMenuPolicy <chr> NA, NA, NA, ... $ source.event_data.OSEditionID <chr> NA, NA, NA, ... $ source.event_data.OSName <chr> NA, NA, NA, ... $ source.event_data.OSbuildversion <chr> NA, NA, NA, ... $ source.event_data.OSmajorversion <chr> NA, NA, NA, ... $ source.event_data.OSminorversion <chr> NA, NA, NA, ... $ source.event_data.OSservicepackmajorversion <chr> NA, NA, NA, ... $ source.event_data.OSservicepackminorversion <chr> NA, NA, NA, ... $ source.event_data.AccessRemoved <chr> NA, NA, NA, ... $ source.event_data.AdditionalInfo2 <chr> NA, NA, NA, ... $ source.user_data.Reason <chr> NA, NA, NA, ... $ source.event_data.KDCRealm <chr> NA, NA, NA, ... $ source.event_data.Server <chr> NA, NA, NA, ... $ source.event_data.Parameter <chr> NA, NA, NA, ... $ source.event_data.TaskName <chr> NA, NA, NA, ... $ source.event_data.errorCode <chr> NA, NA, NA, ... $ source.event_data.KerberosPolicyChange <chr> NA, NA, NA, ... $ source.event_data.BootAppStatus <chr> NA, NA, NA, ... $ source.event_data.BugcheckCode <chr> NA, NA, NA, ... $ source.event_data.BugcheckParameter1 <chr> NA, NA, NA, ... $ source.event_data.BugcheckParameter2 <chr> NA, NA, NA, ... $ source.event_data.BugcheckParameter3 <chr> NA, NA, NA, ... $ source.event_data.BugcheckParameter4 <chr> NA, NA, NA, ... $ source.event_data.PowerButtonTimestamp <chr> NA, NA, NA, ... $ source.event_data.SleepInProgress <chr> NA, NA, NA, ... $ source.event_data.NetStatusCode <chr> NA, NA, NA, ... $ source.event_data.Target <chr> NA, NA, NA, ... $ source.event_data.FilePath <chr> NA, NA, NA, ... $ source.event_data.GPOCNName <chr> NA, NA, NA, ... $ source.event_data.ClientRealm <chr> NA, NA, NA, ... $ source.event_data.TargetRealm <chr> NA, NA, NA, ... $ source.event_data.Targetname <chr> NA, NA, NA, ... $ source.event_data.HostOSName <chr> NA, NA, NA, ... $ source.event_data.HostOSbuildversion <chr> NA, NA, NA, ... $ source.event_data.HostOSmajorversion <chr> NA, NA, NA, ... $ source.event_data.HostOSminorversion <chr> NA, NA, NA, ... $ source.event_data.HostOSservicepackmajorversion <chr> NA, NA, NA, ... $ source.event_data.HostOSservicepackminorversion <chr> NA, NA, NA, ... $ source.event_data.HostOSwasWindowsPE <chr> NA, NA, NA, ... $ source.event_data.Installwasanupgrade <chr> NA, NA, NA, ... $ source.event_data.TimeDifferenceMilliseconds <chr> NA, NA, NA, ... $ source.event_data.TimeSampleSeconds <chr> NA, NA, NA, ... $ timestamp <dttm> 2017-12-02 ...
### Lets quickly look at when things happened. I've filtered this to just the day of the mini blue-red CTF.
## It looks like a lot happened about 7 and then again right before noon
df %>%
ggplot(aes(x=timestamp)) +
geom_density()
df %>%
mutate(day = as.Date(timestamp)) %>%
count(day) %>%
arrange(-n)
day | n |
---|---|
2017-12-02 | 24966 |
### A quick list of names
dput(names(df))
### Lets see how many unique values there are in each field
purrr::map(df, n_distinct)
# Filebeat columns
names(df)[!is.na(df[grepl("filebeat.*", df$index), ][1, ])]
# Winbeat columns
names(df)[!is.na(df[grepl("winlogbeat.*", df$index), ][1, ])]
### Lets do a joy plot of the different servers
## Looks like Win2008-64v1, mysql, and BT-VM didn't get touched.
## Something happened on the rest of the windows servers a bit before noon
## The LAMP server did something about 7am
df %>%
## Next two lines as if we needed to zero in on a specific time period
filter(timestamp >= lubridate::ymd("2017-12-01")) %>%
filter(timestamp <= lubridate::ymd("2017-12-10")) %>%
## lets make the server names look better in the plot
mutate(source.beat.name = ifelse(source.beat.name == "elasticsearch", "ES", source.beat.name)) %>%
mutate(source.beat.name = ifelse(source.beat.name == "blueteam-virtual-machine", "BT-VM", source.beat.name)) %>%
mutate(source.beat.name = stringr::str_wrap(source.beat.name, 8)) %>%
## ggplot() starts a plot
ggplot() +
## geom_joy2 means we want a joy plot
## aes() sets the aesthetic, i.e. what's on the x & y axes, the line and fill colors, the alpha, etc.
ggjoy::geom_joy2(aes(x=timestamp, y=source.beat.name, fill=source.beat.name), alpha=0.8) +
## lets make the x axis labels look nicer
ggthemes::scale_fill_tableau(palette = "tableau20") +
scale_x_datetime(date_labels="%H") +
## some theme stuff to make it look nicer
ggjoy::theme_joy() +
theme(legend.position = "bottom",
legend.title=element_blank(),
axis.title.y=element_blank())
Picking joint bandwidth of 2650
### Lets do the same thing for the event source to see what sources generated events when
## We can see various logs being active at various times. Definitely not consistant the whole time.
df %>%
## Next two lines as if we needed to zero in on a specific time period
filter(timestamp >= lubridate::ymd("2017-12-01")) %>%
filter(timestamp <= lubridate::ymd("2017-12-10")) %>%
## Get rid of records without a source_name
mutate(source.source_name = factor(source.source_name)) %>%
## lets make the source names look better in the plot
mutate(source.source_name = gsub("Microsoft-Windows", "Win", source.source_name)) %>%
ggplot() +
# the geom and aesthetic
ggjoy::geom_joy2(aes(x=timestamp, y=source.source_name, fill=source.source_name), alpha=0.8) +
## the axis
ggthemes::scale_fill_tableau(palette = "tableau20", guide="none") +
scale_x_datetime(date_labels="%H") +
# the theme
ggjoy::theme_joy() +
theme(legend.position = "bottom",
legend.title=element_blank(),
axis.title.y=element_blank(),
panel.grid.major.y=element_blank(),
axis.text.y=element_text(size=6)
)
Picking joint bandwidth of 4150
### Lets look at the windows logs
## we can see some logs are happening almost constantly, some are periodic, while others happen just a few times.
## many events happen together such has 4720-4738
## I'm not a windows log person so can't tell you what these are but can see the patterns.
df %>%
## Next two lines as if we needed to zero in on a specific time period
filter(timestamp >= lubridate::ymd_h("2017-12-02 00")) %>%
filter(timestamp <= lubridate::ymd_h("2017-12-03 00")) %>%
## get rid of thigns without an event ID. (NA means "Not Available")
filter(!is.na(source.event_id)) %>%
## This isn't a number, it's an id so should be a character or factor
mutate(source.event_id = as.character(source.event_id)) %>%
## The figure
ggplot() +
## Set the geom and aesthetic
geom_point(aes(x=timestamp, y=source.event_id, color=source.event_id), alpha=0.2) +
## Set the axis (in this case, the color)
viridis::scale_color_viridis(discrete=TRUE, option="C", end=0.8) +
## Set the theme
theme(legend.position = "none",
legend.title=element_blank(),
axis.title.y=element_blank() ,
axis.text.y=element_text(size=6)
)
### Event 7036 has an interesting period of activity between 10 and 2, lets look at it
## This query just shows the columns
df %>%
# filter to the time period we want
filter(timestamp >= lubridate::ymd_h("2017-12-02 10")) %>%
filter(timestamp <= lubridate::ymd_h("2017-12-02 14")) %>%
# Filter to the event_id we want
filter(source.event_id == 7036) %>%
# select(one_of(grep("source", names(df), value=TRUE))) %>%
## The following line is super helpful in ES data as it gets rid of columns that are all NA
## These are common as some columns may be unix specific when we're just looking at a windows event.
select_if(colSums(!is.na(.)) > 0) %>%
glimpse()
Observations: 69 Variables: 24 $ index <chr> "winlogbeat-2017.12.02", "winlogbeat-2017.... $ type <chr> "wineventlog", "wineventlog", "wineventlog... $ id <chr> "AVzz0mGeG5kLSOhm7BBY", "AVzz3JQOG5kLSOhm7... $ score <dbl> 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... $ `source.@timestamp` <chr> "2017-12-02T10:04:52.327Z", "2017-12-02T10... $ source.message <chr> "The Windows Error Reporting Service servi... $ source.type <chr> "wineventlog", "wineventlog", "wineventlog... $ source.beat.hostname <chr> "win7-i64v2", "win7-i64v2", "win2012-i64v1... $ source.beat.name <chr> "win7-i64v2", "win7-i64v2", "win2012-i64v1... $ source.beat.version <chr> "5.4.2", "5.4.2", "5.4.2", "5.4.2", "5.4.2... $ source.computer_name <chr> "win7-i64v2.delta.net", "win7-i64v2.delta.... $ source.event_id <int> 7036, 7036, 7036, 7036, 7036, 7036, 7036, ... $ source.keywords <list> ["Classic", "Classic", "Classic", "Classi... $ source.level <chr> "Information", "Information", "Information... $ source.log_name <chr> "System", "System", "System", "System", "S... $ source.process_id <int> 492, 492, 500, 492, 492, 500, 500, 500, 49... $ source.provider_guid <chr> "{555908d1-a6d7-4695-8e1e-26931d2012f4}", ... $ source.record_number <chr> "2260", "2261", "11645", "2262", "2501", "... $ source.source_name <chr> "Service Control Manager", "Service Contro... $ source.thread_id <int> 3340, 4080, 3692, 4036, 3052, 4040, 3508, ... $ source.event_data.Binary <chr> "5700650072005300760063002F0031000000", "4... $ source.event_data.param1 <chr> "Windows Error Reporting Service", "Applic... $ source.event_data.param2 <chr> "stopped", "stopped", "running", "stopped"... $ timestamp <dttm> 2017-12-02 10:04:52, 2017-12-02 10:15:59,...
### Building ont he previous block about event 7036, lets visualize it
## Ah, now we can see various services starting and stopping on various servers
df %>%
# filter to the time period we want
filter(timestamp >= lubridate::ymd_h("2017-12-02 10")) %>%
filter(timestamp <= lubridate::ymd_h("2017-12-02 14")) %>%
# Filter to the event_id we want
filter(source.event_id == 7036) %>%
# select(one_of(grep("source", names(df), value=TRUE))) %>%
## The following line is super helpful in ES data as it gets rid of columns that are all NA
## These are common as some columns may be unix specific when we're just looking at a windows event.
select_if(colSums(!is.na(.)) > 0) %>%
mutate(source.beat.name = stringr::str_wrap(source.beat.name, 8)) %>%
# the figure
ggplot() +
# the geom and aesthetic
geom_point(aes(x=timestamp, y=source.event_data.param2)) +
# The axes
scale_x_datetime(date_labels = "%H") +
# makes a grid of figures
facet_grid(source.event_data.param1 ~ source.beat.name) +
# how it all looks
theme(
strip.text.y = element_text(angle=0)
)
### TESTING
## I always keep a cell at the bottom I use for stuff I want to test out.
## Normally it's just a glimpse, but it's helpful for various transient things
glimpse(df)
Observations: 478,605 Variables: 394 $ index <chr> "filebeat-20... $ type <chr> "log", "log"... $ id <chr> "AVzqpNxEG5k... $ score <dbl> 1, 1, 1, 1, ... $ `source.@timestamp` <chr> "2017-06-27T... $ source.input_type <chr> "log", "log"... $ source.message <chr> "Jun 26 18:3... $ source.offset <int> 1239818, 123... $ source.source <chr> "/var/log/au... $ source.type <chr> "log", "log"... $ source.beat.hostname <chr> "blueteam-vi... $ source.beat.name <chr> "blueteam-vi... $ source.beat.version <chr> "5.4.2", "5.... $ source.computer_name <chr> NA, NA, NA, ... $ source.event_id <int> NA, NA, NA, ... $ source.keywords <list> [NULL, NULL... $ source.level <chr> NA, NA, NA, ... $ source.log_name <chr> NA, NA, NA, ... $ source.process_id <int> NA, NA, NA, ... $ source.provider_guid <chr> NA, NA, NA, ... $ source.record_number <chr> NA, NA, NA, ... $ source.source_name <chr> NA, NA, NA, ... $ source.thread_id <int> NA, NA, NA, ... $ source.opcode <chr> NA, NA, NA, ... $ source.task <chr> NA, NA, NA, ... $ source.version <int> NA, NA, NA, ... $ source.activity_id <chr> NA, NA, NA, ... $ source.event_data.Binary <chr> NA, NA, NA, ... $ source.event_data.param1 <chr> NA, NA, NA, ... $ source.event_data.param2 <chr> NA, NA, NA, ... $ source.event_data.StopTime <chr> NA, NA, NA, ... $ source.event_data.ShutdownActionType <chr> NA, NA, NA, ... $ source.event_data.ShutdownEventCode <chr> NA, NA, NA, ... $ source.event_data.ShutdownReason <chr> NA, NA, NA, ... $ source.event_data.param4 <chr> NA, NA, NA, ... $ source.event_data.param5 <chr> NA, NA, NA, ... $ source.event_data.BootType <chr> NA, NA, NA, ... $ source.event_data.DeviceName <chr> NA, NA, NA, ... $ source.event_data.DeviceNameLength <chr> NA, NA, NA, ... $ source.event_data.DeviceTime <chr> NA, NA, NA, ... $ source.event_data.DeviceVersionMajor <chr> NA, NA, NA, ... $ source.event_data.DeviceVersionMinor <chr> NA, NA, NA, ... $ source.event_data.FinalStatus <chr> NA, NA, NA, ... $ source.event_data.Group <chr> NA, NA, NA, ... $ source.event_data.IdleImplementation <chr> NA, NA, NA, ... $ source.event_data.IdleStateCount <chr> NA, NA, NA, ... $ source.event_data.MaximumPerformancePercent <chr> NA, NA, NA, ... $ source.event_data.MinimumPerformancePercent <chr> NA, NA, NA, ... $ source.event_data.MinimumThrottlePercent <chr> NA, NA, NA, ... $ source.event_data.NominalFrequency <chr> NA, NA, NA, ... $ source.event_data.Number <chr> NA, NA, NA, ... $ source.event_data.PerformanceImplementation <chr> NA, NA, NA, ... $ source.event_data.DirtyPages <chr> NA, NA, NA, ... $ source.event_data.HiveName <chr> NA, NA, NA, ... $ source.event_data.HiveNameLength <chr> NA, NA, NA, ... $ source.event_data.KeysUpdated <chr> NA, NA, NA, ... $ source.event_data.IpAddress <chr> NA, NA, NA, ... $ source.event_data.IpPort <chr> NA, NA, NA, ... $ source.event_data.LogonGuid <chr> NA, NA, NA, ... $ source.event_data.ProcessId <chr> NA, NA, NA, ... $ source.event_data.ProcessName <chr> NA, NA, NA, ... $ source.event_data.SubjectDomainName <chr> NA, NA, NA, ... $ source.event_data.SubjectLogonId <chr> NA, NA, NA, ... $ source.event_data.SubjectUserName <chr> NA, NA, NA, ... $ source.event_data.SubjectUserSid <chr> NA, NA, NA, ... $ source.event_data.TargetDomainName <chr> NA, NA, NA, ... $ source.event_data.TargetInfo <chr> NA, NA, NA, ... $ source.event_data.TargetLogonGuid <chr> NA, NA, NA, ... $ source.event_data.TargetServerName <chr> NA, NA, NA, ... $ source.event_data.TargetUserName <chr> NA, NA, NA, ... $ source.event_data.PrivilegeList <chr> NA, NA, NA, ... $ source.event_data.AuthenticationPackageName <chr> NA, NA, NA, ... $ source.event_data.ImpersonationLevel <chr> NA, NA, NA, ... $ source.event_data.KeyLength <chr> NA, NA, NA, ... $ source.event_data.LmPackageName <chr> NA, NA, NA, ... $ source.event_data.LogonProcessName <chr> NA, NA, NA, ... $ source.event_data.LogonType <chr> NA, NA, NA, ... $ source.event_data.TargetLogonId <chr> NA, NA, NA, ... $ source.event_data.TargetUserSid <chr> NA, NA, NA, ... $ source.event_data.TransmittedServices <chr> NA, NA, NA, ... $ source.event_data.HandleId <chr> NA, NA, NA, ... $ source.event_data.NewSd <chr> NA, NA, NA, ... $ source.event_data.ObjectName <chr> NA, NA, NA, ... $ source.event_data.ObjectServer <chr> NA, NA, NA, ... $ source.event_data.ObjectType <chr> NA, NA, NA, ... $ source.event_data.SamAccountName <chr> NA, NA, NA, ... $ source.event_data.SidHistory <chr> NA, NA, NA, ... $ source.event_data.TargetSid <chr> NA, NA, NA, ... $ source.event_data.NewTargetUserName <chr> NA, NA, NA, ... $ source.event_data.OldTargetUserName <chr> NA, NA, NA, ... $ source.event_data.DwordVal <chr> NA, NA, NA, ... $ source.event_data.AccountExpires <chr> NA, NA, NA, ... $ source.event_data.AllowedToDelegateTo <chr> NA, NA, NA, ... $ source.event_data.DisplayName <chr> NA, NA, NA, ... $ source.event_data.Dummy <chr> NA, NA, NA, ... $ source.event_data.HomeDirectory <chr> NA, NA, NA, ... $ source.event_data.HomePath <chr> NA, NA, NA, ... $ source.event_data.LogonHours <chr> NA, NA, NA, ... $ source.event_data.NewUacValue <chr> NA, NA, NA, ... $ source.event_data.OldUacValue <chr> NA, NA, NA, ... $ source.event_data.PasswordLastSet <chr> NA, NA, NA, ... $ source.event_data.PrimaryGroupId <chr> NA, NA, NA, ... $ source.event_data.ProfilePath <chr> NA, NA, NA, ... $ source.event_data.ScriptPath <chr> NA, NA, NA, ... $ source.event_data.UserAccountControl <chr> NA, NA, NA, ... $ source.event_data.UserParameters <chr> NA, NA, NA, ... $ source.event_data.UserPrincipalName <chr> NA, NA, NA, ... $ source.event_data.UserWorkstations <chr> NA, NA, NA, ... $ source.event_data.ObjectCollectionName <chr> NA, NA, NA, ... $ source.event_data.ObjectIdentifyingProperties <chr> NA, NA, NA, ... $ source.event_data.ObjectProperties <chr> NA, NA, NA, ... $ source.event_data.SubjectUserDomainName <chr> NA, NA, NA, ... $ source.event_data.WorkstationName <chr> NA, NA, NA, ... $ source.event_data.NewTime <chr> NA, NA, NA, ... $ source.event_data.PreviousTime <chr> NA, NA, NA, ... $ source.event_data.param3 <chr> NA, NA, NA, ... $ source.event_data.AccountName <chr> NA, NA, NA, ... $ source.event_data.ImagePath <chr> NA, NA, NA, ... $ source.event_data.ServiceName <chr> NA, NA, NA, ... $ source.event_data.ServiceType <chr> NA, NA, NA, ... $ source.event_data.StartType <chr> NA, NA, NA, ... $ source.event_data.OldTime <chr> NA, NA, NA, ... $ source.event_data.Reason <chr> NA, NA, NA, ... $ source.event_data.TSId <chr> NA, NA, NA, ... $ source.event_data.UserSid <chr> NA, NA, NA, ... $ source.event_data.BitlockerUserInputTime <chr> NA, NA, NA, ... $ source.event_data.DomainPeer <chr> NA, NA, NA, ... $ source.event_data.ErrorMessage <chr> NA, NA, NA, ... $ source.event_data.RetryMinutes <chr> NA, NA, NA, ... $ source.event_data.MemberName <chr> NA, NA, NA, ... $ source.event_data.MemberSid <chr> NA, NA, NA, ... $ source.event_data.AuditSourceName <chr> NA, NA, NA, ... $ source.event_data.EventSourceId <chr> NA, NA, NA, ... $ source.event_data.BootMode <chr> NA, NA, NA, ... $ source.event_data.BuildVersion <chr> NA, NA, NA, ... $ source.event_data.MajorVersion <chr> NA, NA, NA, ... $ source.event_data.MinorVersion <chr> NA, NA, NA, ... $ source.event_data.QfeVersion <chr> NA, NA, NA, ... $ source.event_data.ServiceVersion <chr> NA, NA, NA, ... $ source.event_data.StartTime <chr> NA, NA, NA, ... $ source.event_data.IdleState <chr> NA, NA, NA, ... $ source.event_data.PerfStateCount <chr> NA, NA, NA, ... $ source.event_data.ThrottleStateCount <chr> NA, NA, NA, ... $ source.event_data.param7 <chr> NA, NA, NA, ... $ source.event_data.DeviceObject <chr> NA, NA, NA, ... $ source.event_data.Url <chr> NA, NA, NA, ... $ `source.event_data.OS EditionID` <chr> NA, NA, NA, ... $ `source.event_data.OS Name` <chr> NA, NA, NA, ... $ `source.event_data.OS build version` <chr> NA, NA, NA, ... $ `source.event_data.OS major version` <chr> NA, NA, NA, ... $ `source.event_data.OS minor version` <chr> NA, NA, NA, ... $ `source.event_data.OS service pack major version` <chr> NA, NA, NA, ... $ `source.event_data.OS service pack minor version` <chr> NA, NA, NA, ... $ source.event_data.param6 <chr> NA, NA, NA, ... $ source.event_data.NewDate <chr> NA, NA, NA, ... $ source.event_data.PreviousDate <chr> NA, NA, NA, ... $ source.event_data.PuaCount <chr> NA, NA, NA, ... $ source.event_data.PuaPolicyId <chr> NA, NA, NA, ... $ source.event_data.ErrorCode <chr> NA, NA, NA, ... $ source.event_data.PackageName <chr> NA, NA, NA, ... $ source.event_data.Status <chr> NA, NA, NA, ... $ source.event_data.Workstation <chr> NA, NA, NA, ... $ source.event_data.Address <chr> NA, NA, NA, ... $ source.event_data.Interface <chr> NA, NA, NA, ... $ source.event_data.ProtocolType <chr> NA, NA, NA, ... $ source.event_data.Attributes <chr> NA, NA, NA, ... $ source.event_data.BiosInitDuration <chr> NA, NA, NA, ... $ source.event_data.DriverInitDuration <chr> NA, NA, NA, ... $ source.event_data.EffectiveState <chr> NA, NA, NA, ... $ source.event_data.HiberPagesWritten <chr> NA, NA, NA, ... $ source.event_data.HiberReadDuration <chr> NA, NA, NA, ... $ source.event_data.HiberWriteDuration <chr> NA, NA, NA, ... $ source.event_data.SleepDuration <chr> NA, NA, NA, ... $ source.event_data.SleepTime <chr> NA, NA, NA, ... $ source.event_data.TargetState <chr> NA, NA, NA, ... $ source.event_data.WakeDuration <chr> NA, NA, NA, ... $ source.event_data.WakeSourceTextLength <chr> NA, NA, NA, ... $ source.event_data.WakeSourceType <chr> NA, NA, NA, ... $ source.event_data.WakeTime <chr> NA, NA, NA, ... $ source.event_data.WakeTimerContextLength <chr> NA, NA, NA, ... $ source.event_data.WakeTimerOwnerLength <chr> NA, NA, NA, ... $ source.event_data.Processor <chr> NA, NA, NA, ... $ source.event_data.MandatoryLabel <chr> NA, NA, NA, ... $ source.event_data.NewProcessId <chr> NA, NA, NA, ... $ source.event_data.NewProcessName <chr> NA, NA, NA, ... $ source.event_data.ParentProcessName <chr> NA, NA, NA, ... $ source.event_data.TokenElevationType <chr> NA, NA, NA, ... $ source.event_data.ElevatedToken <chr> NA, NA, NA, ... $ source.event_data.RestrictedAdminMode <chr> NA, NA, NA, ... $ source.event_data.TargetLinkedLogonId <chr> NA, NA, NA, ... $ source.event_data.TargetOutboundDomainName <chr> NA, NA, NA, ... $ source.event_data.TargetOutboundUserName <chr> NA, NA, NA, ... $ source.event_data.VirtualAccount <chr> NA, NA, NA, ... $ source.event_data.CorruptionActionState <chr> NA, NA, NA, ... $ source.event_data.DriveName <chr> NA, NA, NA, ... $ source.event_data.ExtraInfoLength <chr> NA, NA, NA, ... $ source.event_data.ExtraInfoString <chr> NA, NA, NA, ... $ source.event_data.FilterID <chr> NA, NA, NA, ... $ source.event_data.Turn <chr> NA, NA, NA, ... $ source.event_data.EntryCount <chr> NA, NA, NA, ... $ source.event_data.AlgorithmName <chr> NA, NA, NA, ... $ source.event_data.KeyFilePath <chr> NA, NA, NA, ... $ source.event_data.KeyName <chr> NA, NA, NA, ... $ source.event_data.KeyType <chr> NA, NA, NA, ... $ source.event_data.Operation <chr> NA, NA, NA, ... $ source.event_data.ProviderName <chr> NA, NA, NA, ... $ source.event_data.ReturnCode <chr> NA, NA, NA, ... $ source.event_data.Flags <chr> NA, NA, NA, ... $ source.event_data.FailureReason <chr> NA, NA, NA, ... $ source.event_data.SubStatus <chr> NA, NA, NA, ... $ source.event_data.TimeSource <chr> NA, NA, NA, ... $ source.user.domain <chr> NA, NA, NA, ... $ source.user.identifier <chr> NA, NA, NA, ... $ source.user.name <chr> NA, NA, NA, ... $ source.user.type <chr> NA, NA, NA, ... $ source.user_data.DeviceInstanceID <chr> NA, NA, NA, ... $ source.user_data.DriverDescription <chr> NA, NA, NA, ... $ source.user_data.DriverName <chr> NA, NA, NA, ... $ source.user_data.DriverProvider <chr> NA, NA, NA, ... $ source.user_data.DriverVersion <chr> NA, NA, NA, ... $ source.user_data.InstallStatus <chr> NA, NA, NA, ... $ source.user_data.IsDriverOEM <chr> NA, NA, NA, ... $ source.user_data.RebootOption <chr> NA, NA, NA, ... $ source.user_data.SetupClass <chr> NA, NA, NA, ... $ source.user_data.UpgradeDevice <chr> NA, NA, NA, ... $ source.user_data.xml_name <chr> NA, NA, NA, ... $ source.user_data.AddServiceStatus <chr> NA, NA, NA, ... $ source.user_data.DriverFileName <chr> NA, NA, NA, ... $ source.user_data.PrimaryService <chr> NA, NA, NA, ... $ source.user_data.ServiceName <chr> NA, NA, NA, ... $ source.user_data.UpdateService <chr> NA, NA, NA, ... $ source.user_data.CachingSubsystemState <chr> NA, NA, NA, ... $ source.user_data.InstallSubsystemState <chr> NA, NA, NA, ... $ source.user_data.ErrorCode <chr> NA, NA, NA, ... $ source.user_data.Operation <chr> NA, NA, NA, ... $ source.user_data.OperationCompleted <chr> NA, NA, NA, ... $ source.user_data.PackageAssembly <chr> NA, NA, NA, ... $ source.user_data.PackageIdentifier <chr> NA, NA, NA, ... $ source.user_data.PackageState <chr> NA, NA, NA, ... $ source.event_data.ProcessingMode <chr> NA, NA, NA, ... $ source.event_data.ProcessingTimeInMilliseconds <chr> NA, NA, NA, ... $ source.event_data.SupportInfo1 <chr> NA, NA, NA, ... $ source.event_data.SupportInfo2 <chr> NA, NA, NA, ... $ source.event_data.CallerProcessId <chr> NA, NA, NA, ... $ source.event_data.CallerProcessName <chr> NA, NA, NA, ... $ source.event_data.param10 <chr> NA, NA, NA, ... $ source.event_data.param11 <chr> NA, NA, NA, ... $ source.event_data.param8 <chr> NA, NA, NA, ... $ source.event_data.param9 <chr> NA, NA, NA, ... $ source.event_data.updateGuid <chr> NA, NA, NA, ... $ source.event_data.updateRevisionNumber <chr> NA, NA, NA, ... $ source.event_data.updateTitle <chr> NA, NA, NA, ... $ source.event_data.serviceGuid <chr> NA, NA, NA, ... $ source.event_data.NewSchemeGuid <chr> NA, NA, NA, ... $ source.event_data.OldSchemeGuid <chr> NA, NA, NA, ... $ source.event_data.ProcessPath <chr> NA, NA, NA, ... $ source.event_data.ProcessPid <chr> NA, NA, NA, ... $ source.event_data.OldSd <chr> NA, NA, NA, ... $ source.event_data.UnsynchronizedTimeSeconds <chr> NA, NA, NA, ... $ source.event_data.AddressLength <chr> NA, NA, NA, ... $ source.event_data.QueryName <chr> NA, NA, NA, ... $ source.event_data.LastBootGood <chr> NA, NA, NA, ... $ source.event_data.LastShutdownGood <chr> NA, NA, NA, ... $ source.event_data.ProgrammedWakeTimeAc <chr> NA, NA, NA, ... $ source.event_data.ProgrammedWakeTimeDc <chr> NA, NA, NA, ... $ source.event_data.WakeFromState <chr> NA, NA, NA, ... $ source.event_data.WakeRequesterTypeAc <chr> NA, NA, NA, ... $ source.event_data.WakeRequesterTypeDc <chr> NA, NA, NA, ... $ source.event_data.ModifiedObjectProperties <chr> NA, NA, NA, ... $ source.event_data.ConfigurationReader <chr> NA, NA, NA, ... $ source.event_data.RunningMode <chr> NA, NA, NA, ... $ source.event_data.Endpoint <chr> NA, NA, NA, ... $ source.event_data.AccessGranted <chr> NA, NA, NA, ... $ source.event_data.AccessList <chr> NA, NA, NA, ... $ source.event_data.AccessMask <chr> NA, NA, NA, ... $ source.event_data.AdditionalInfo <chr> NA, NA, NA, ... $ source.event_data.OperationType <chr> NA, NA, NA, ... $ source.event_data.Properties <chr> NA, NA, NA, ... $ source.event_data.ServiceSid <chr> NA, NA, NA, ... $ source.event_data.TicketEncryptionType <chr> NA, NA, NA, ... $ source.event_data.TicketOptions <chr> NA, NA, NA, ... $ source.event_data.PreAuthType <chr> NA, NA, NA, ... $ source.event_data.SecurityPackage <chr> NA, NA, NA, ... $ source.event_data.DCName <chr> NA, NA, NA, ... $ source.event_data.NumberOfGroupPolicyObjects <chr> NA, NA, NA, ... $ source.event_data.DomainBehaviorVersion <chr> NA, NA, NA, ... $ source.event_data.DomainName <chr> NA, NA, NA, ... $ source.event_data.DomainPolicyChanged <chr> NA, NA, NA, ... $ source.event_data.DomainSid <chr> NA, NA, NA, ... $ source.event_data.ForceLogoff <chr> NA, NA, NA, ... $ source.event_data.LockoutDuration <chr> NA, NA, NA, ... $ source.event_data.LockoutObservationWindow <chr> NA, NA, NA, ... $ source.event_data.MachineAccountQuota <chr> NA, NA, NA, ... $ source.event_data.MinPasswordAge <chr> NA, NA, NA, ... $ source.event_data.MinPasswordLength <chr> NA, NA, NA, ... $ source.event_data.MixedDomainMode <chr> NA, NA, NA, ... $ source.event_data.OemInformation <chr> NA, NA, NA, ... $ source.event_data.PasswordHistoryLength <chr> NA, NA, NA, ... $ source.event_data.PasswordProperties <chr> NA, NA, NA, ... $ source.event_data.NoMultiStageResumeReason <chr> NA, NA, NA, ... $ source.event_data.ComputerName <chr> NA, NA, NA, ... $ source.event_data.AdapterName <chr> NA, NA, NA, ... $ source.event_data.AdapterSuffixName <chr> NA, NA, NA, ... $ source.event_data.DnsServerList <chr> NA, NA, NA, ... $ source.event_data.HostName <chr> NA, NA, NA, ... $ source.event_data.Ipaddress <chr> NA, NA, NA, ... $ `source.event_data.Sent UpdateServer` <chr> NA, NA, NA, ... $ source.event_data.IfGuid <chr> NA, NA, NA, ... $ source.event_data.IfIndex <chr> NA, NA, NA, ... $ source.event_data.IfLuid <chr> NA, NA, NA, ... $ source.event_data.ResetCount <chr> NA, NA, NA, ... $ source.event_data.ResetReason <chr> NA, NA, NA, ... $ source.event_data.AlertDesc <chr> NA, NA, NA, ... $ source.event_data.ErrorState <chr> NA, NA, NA, ... $ source.event_data.Type <chr> NA, NA, NA, ... $ source.event_data.NewSize <chr> NA, NA, NA, ... $ source.event_data.OriginalSize <chr> NA, NA, NA, ... $ source.user_data.Client <chr> NA, NA, NA, ... $ source.user_data.InitialPackageState <chr> NA, NA, NA, ... $ source.user_data.IntendedPackageState <chr> NA, NA, NA, ... $ source.user_data.ReleaseType <chr> NA, NA, NA, ... $ source.user_data.SupportInformation <chr> NA, NA, NA, ... $ source.user_data.Argument <chr> NA, NA, NA, ... $ source.user_data.UpdateDisplayName <chr> NA, NA, NA, ... $ source.user_data.UpdateName <chr> NA, NA, NA, ... $ source.user_data.UpdateState <chr> NA, NA, NA, ... $ source.event_data.AdvancedOptions <chr> NA, NA, NA, ... $ source.event_data.ConfigAccessPolicy <chr> NA, NA, NA, ... $ source.event_data.DisableIntegrityChecks <chr> NA, NA, NA, ... $ source.event_data.FlightSigning <chr> NA, NA, NA, ... $ source.event_data.HypervisorDebug <chr> NA, NA, NA, ... $ source.event_data.HypervisorLaunchType <chr> NA, NA, NA, ... $ source.event_data.HypervisorLoadOptions <chr> NA, NA, NA, ... $ source.event_data.KernelDebug <chr> NA, NA, NA, ... $ source.event_data.LoadOptions <chr> NA, NA, NA, ... $ source.event_data.RemoteEventLogging <chr> NA, NA, NA, ... $ source.event_data.TestSigning <chr> NA, NA, NA, ... $ source.event_data.VsmLaunchType <chr> NA, NA, NA, ... $ source.event_data.TimeOffsetSeconds <chr> NA, NA, NA, ... $ source.event_data.ErrorDescription <chr> NA, NA, NA, ... $ source.event_data.AppPoolID <chr> NA, NA, NA, ... $ source.event_data.Minutes <chr> NA, NA, NA, ... $ source.event_data.ProcessID <chr> NA, NA, NA, ... $ `source.event_data.Host OS Name` <chr> NA, NA, NA, ... $ `source.event_data.Host OS build version` <chr> NA, NA, NA, ... $ `source.event_data.Host OS major version` <chr> NA, NA, NA, ... $ `source.event_data.Host OS minor version` <chr> NA, NA, NA, ... $ `source.event_data.Host OS service pack major version` <chr> NA, NA, NA, ... $ `source.event_data.Host OS service pack minor version` <chr> NA, NA, NA, ... $ `source.event_data.Host OS was Windows PE` <chr> NA, NA, NA, ... $ `source.event_data.Install was an upgrade` <chr> NA, NA, NA, ... $ source.event_data.BootMenuPolicy <chr> NA, NA, NA, ... $ source.event_data.OSEditionID <chr> NA, NA, NA, ... $ source.event_data.OSName <chr> NA, NA, NA, ... $ source.event_data.OSbuildversion <chr> NA, NA, NA, ... $ source.event_data.OSmajorversion <chr> NA, NA, NA, ... $ source.event_data.OSminorversion <chr> NA, NA, NA, ... $ source.event_data.OSservicepackmajorversion <chr> NA, NA, NA, ... $ source.event_data.OSservicepackminorversion <chr> NA, NA, NA, ... $ source.event_data.AccessRemoved <chr> NA, NA, NA, ... $ source.event_data.AdditionalInfo2 <chr> NA, NA, NA, ... $ source.user_data.Reason <chr> NA, NA, NA, ... $ source.event_data.KDCRealm <chr> NA, NA, NA, ... $ source.event_data.Server <chr> NA, NA, NA, ... $ source.event_data.Parameter <chr> NA, NA, NA, ... $ source.event_data.TaskName <chr> NA, NA, NA, ... $ source.event_data.errorCode <chr> NA, NA, NA, ... $ source.event_data.KerberosPolicyChange <chr> NA, NA, NA, ... $ source.event_data.BootAppStatus <chr> NA, NA, NA, ... $ source.event_data.BugcheckCode <chr> NA, NA, NA, ... $ source.event_data.BugcheckParameter1 <chr> NA, NA, NA, ... $ source.event_data.BugcheckParameter2 <chr> NA, NA, NA, ... $ source.event_data.BugcheckParameter3 <chr> NA, NA, NA, ... $ source.event_data.BugcheckParameter4 <chr> NA, NA, NA, ... $ source.event_data.PowerButtonTimestamp <chr> NA, NA, NA, ... $ source.event_data.SleepInProgress <chr> NA, NA, NA, ... $ source.event_data.NetStatusCode <chr> NA, NA, NA, ... $ source.event_data.Target <chr> NA, NA, NA, ... $ source.event_data.FilePath <chr> NA, NA, NA, ... $ source.event_data.GPOCNName <chr> NA, NA, NA, ... $ source.event_data.ClientRealm <chr> NA, NA, NA, ... $ source.event_data.TargetRealm <chr> NA, NA, NA, ... $ source.event_data.Targetname <chr> NA, NA, NA, ... $ source.event_data.HostOSName <chr> NA, NA, NA, ... $ source.event_data.HostOSbuildversion <chr> NA, NA, NA, ... $ source.event_data.HostOSmajorversion <chr> NA, NA, NA, ... $ source.event_data.HostOSminorversion <chr> NA, NA, NA, ... $ source.event_data.HostOSservicepackmajorversion <chr> NA, NA, NA, ... $ source.event_data.HostOSservicepackminorversion <chr> NA, NA, NA, ... $ source.event_data.HostOSwasWindowsPE <chr> NA, NA, NA, ... $ source.event_data.Installwasanupgrade <chr> NA, NA, NA, ... $ source.event_data.TimeDifferenceMilliseconds <chr> NA, NA, NA, ... $ source.event_data.TimeSampleSeconds <chr> NA, NA, NA, ... $ timestamp <dttm> 2017-06-27 ...