ryan/sagacity
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

initial commit of SVN release repo

Browse Source
This commit is contained in:
Ryan Prather
2018-05-07 10:51:08 -04:00
committed by Ryan Prather
parent 2c25d5e577
commit 8c38a6cdb9
4369 changed files with 728576 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
exec/Target Host Tools/Unix/OSE-Unix-MD5.tar.gz Normal file
View File

Binary file not shown.

182
exec/Target Host Tools/Unix/linux-data-collection.sh Normal file
View File

@ -0,0 +1,182 @@
#!/bin/bash
# Script to collect the major security configuration files on a Linux system
# RUN AS ROOT!
# tested on RHEL 5.2, SUSE 11
# Jeff A. Odegard, CISSP, CPT, CEH
# AFSPC SMC/GPEVA
# 20 Aug 13
# Rewritten 16 Sep 14
# Update 31 Mar 15: Use find -xdev to limit the ffile-permissions.txt to local filesystems only.
# Erik Wohlgemuth (Raytheon) and Jeff Odegard
# Add to this list as necessary (get copies of these files)
FILELIST="/.cshrc
/.profile
/etc/aide.conf
/etc/apache
/etc/apache2
/etc/audit/audit.rules
/etc/audit/auditd.conf
/etc/cron.allow
/etc/cron.d
/etc/cron.deny
/etc/crontab
/etc/default
/etc/ftpusers
/etc/gshadow
/etc/hosts
/etc/hosts.allow
/etc/hosts.deny
/etc/hosts.equiv
/etc/httpd
/etc/inetd.conf
/etc/inittab
/etc/motd
/etc/newsyslog.conf
/etc/nsswitch.conf
/etc/ntp.conf
/etc/ntp.conf
/etc/pam.conf
/etc/pam.d
/etc/passwd
/etc/profile
/etc/redhat-release
/etc/resolv.conf
/etc/securetty
/etc/security
/etc/shells
/etc/ssh_config
/etc/sshd_config
/etc/ssh/ssh_config
/etc/ssh/sshd_config
/etc/SuSE-brand
/etc/SuSE-release
/etc/syslog-ng
/etc/sysconfig/apache2
/etc/sysconfig/selinux
/etc/sysctl.conf
/etc/syslog.conf
/etc/syslog-ng
/etc/xinetd.conf
/etc/xinetd.d
/proc/cmdline
/root/.cshrc
/root/.profile"
#HOSTNAME=`uname -a | cut -d" " -f2`
HOSTNAME=`hostname`
DIR="$HOSTNAME-baseline"
echo ""
echo "Results will be in ./$DIR"
mkdir -p $DIR
cd $DIR
FILEDIR="system-files"
echo "System files will be in ./$DIR/system-files"
mkdir -p system-files
rm -f $HOSTNAME-errors.txt
echo "Linux Data collection started on `date`" >> $HOSTNAME-errors.txt
echo ""
echo "Collecting some system information..."
echo " uname -a"
uname -a > uname.txt
echo " ifconfig -a"
ifconfig -a > ifconfig.txt
echo " netstat -nr"
netstat -nr > netstat-nr.txt
echo " netstat -nap"
netstat -nap > netstat-nap.txt
echo " ps aux"
ps aux > ps-aux.txt
echo " last -a"
last -a -i > last-a-i.txt
echo " who -a"
who -a > who-a.txt
echo " df -ak"
df -ak > df-ak.txt
echo " mount"
mount > mount.txt
echo " rpcinfo -p"
rpcinfo -p > rpcinfo-p.txt
if [ `grep "nfs" rpcinfo-p.txt` ] ; then
echo " showmount"
showmount 2>showmount.txt > showmount.txt
echo " showmount -e"
showmount -e 2>showmount.txt > showmount-e.txt
else
echo " Skipping showmount. NFS does not appear in rpcinfo."
echo " NFS does not appear in rpcinfo. Skipping showmount." >> $HOSTNAME-errors.txt
fi
echo " rpm -qa -last"
rpm -qa -last > rpm-qa-last.txt
echo " crontab -l"
crontab -l 2>crontab-l.txt > crontab-l.txt
echo " pwck -r"
pwck -r > pwck-r.txt
echo ""
echo "Gathering file listing/permissions for STIG checks"
echo " NOTE: find errors are normal"
rm -f file-permissions.txt
FSTYPE=`mount | egrep "on \/ type" | awk '{print $5}'`
for MOUNTPT in `mount | grep $FSTYPE | awk '{print $3}'`; do
find $MOUNTPT -xdev -fstype $FSTYPE -ls >> file-permissions.txt
done
FILESIZE=`ls -sh file-permissions.txt | cut -d" " -f1`
if [ $FILESIZE -eq "0" ]; then # SuSE Linux
echo " Hmmm, might be a SuSE Linux system"
find / -fstype rootfs -ls > file-permissions.txt
fi
ls -sh file-permissions.txt
echo ""
echo "Collecting some security configuration files and folders."
echo " NOTE: Inability to find some files is normal":
for FILE in $FILELIST ; do
if [ -f $FILE -o -d $FILE ] ; then
DEST=`echo $FILE | sed "s/\//\-/g" | sed "s/^\-//"`
echo " cp -af $FILE ./$FILEDIR/$DEST"
cp -af $FILE ./$FILEDIR/$DEST
else
#egrep "\/passwd$" ehud-baseline/file-permissions.txt | awk '{print $11}'
echo " Could not find $FILE" >> $HOSTNAME-errors.txt
echo " Could not find $FILE"
fi
done
# We don't want to collect password hashes, but need to know if the accounts are locked.
# Note: this "for LINE in" hack only works because there are no spaces in /etc/shadow... :o}
rm -f shadow-trimmed
echo ""
echo "Trimming /etc/shadow for safety..."
for LINE in `cat /etc/shadow` ; do
HASH=`echo $LINE | cut -d":" -f2`
# Typical password hash is 34 characters
if [ ${#HASH} -lt 13 ] ; then
echo $LINE >> shadow-trimmed.txt
elif [ ${#HASH} -lt 34 ] ; then
echo $LINE | awk -F':' 'BEGIN{ OFS=":"; } { print $1,"SHORT/WEAK HASH",$3,$4,$5,$6,$7,$8,$9 }' >> shadow-trimmed.txt
else
echo $LINE | awk -F':' 'BEGIN{ OFS=":"; } { print $1,"FILTERED",$3,$4,$5,$6,$7,$8,$9 }' >> shadow-trimmed.txt
fi
done
echo ""
echo "Please review to ensure hashes are filtered"
echo ""
cat shadow-trimmed.txt
echo ""
echo "Linux Data collection ended on `date`" >> $HOSTNAME-errors.txt
cd ..
echo "Tarring and Gzipping the results"
tar -zcvf $DIR.tgz ./$DIR
echo ""
echo "All packaged up and ready to go in $DIR.tgz"
ls -sh $DIR.tgz
echo "Have a nice day!"
echo ""

165
exec/Target Host Tools/Unix/solaris-data-collection.sh Normal file
View File

@ -0,0 +1,165 @@
#!/bin/ksh
# Script to collect the major security configuration files on a Solaris system
# RUN AS ROOT!
# tested on Solaris 10
# Jeff A. Odegard, CISSP
# AFSPC SMC/GPEA
# 20 Aug 13
# Updated 4 Sep 14
# Add to this list as necessary (get copies of these files)
FILELIST="/.cshrc
/.profile
/etc/access.conf
/etc/apache
/etc/apache2
/etc/cron.allow
/etc/cron.d
/etc/cron.deny
/etc/default
/etc/dfs
/etc/ftpd
/etc/ftpusers
/etc/hosts
/etc/hosts.allow
/etc/hosts.deny
/etc/hosts.equiv
/etc/httpd
/etc/inet/inetd.conf
/etc/inet/ntp.conf
/etc/inetd.conf
/etc/issue
/etc/issue.net
/etc/motd
/etc/nsswitch.conf
/etc/ntp.conf
/etc/pam.conf
/etc/passwd
/etc/passwd
/etc/profile
/etc/resolv.conf
/etc/securetty
/etc/security
/etc/shells
/etc/snmp/conf/snmpd.conf
/etc/snmpd.conf
/etc/syslog.conf
/etc/system
/noshell"
#HOSTNAME=`uname -a | cut -d" " -f2`
HOSTNAME=`hostname`
DIR="$HOSTNAME-baseline"
echo ""
echo "Results will be in ./$DIR"
mkdir -p $DIR
cd $DIR
FILEDIR="system-files"
echo "System files will be in ./$DIR/system-files"
mkdir -p system-files
rm -f $HOSTNAME-errors
echo ""
echo "Collecting some system information..."
echo " uname -a"
uname -a > uname.txt
echo " ifconfig -a"
ifconfig -a > ifconfig.txt
echo " netstat -nr"
netstat -nr > netstat-nr.txt
echo " netstat -nap"
netstat -nap > netstat-nap.txt
echo " ps -eaf"
ps -eaf > ps-eaf.txt
echo " last -a"
last -a > last-a.txt
echo " who -a"
who -a > who-a.txt
echo " df -ak"
df -ak > df-ak.txt
echo " mount -p"
mount -p > mount-p.txt
echo " rpcinfo -p"
rpcinfo -p >rpcinfo-p.txt
if [ `grep "nfs" rpcinfo-p.txt` ] ; then
echo " showmount"
showmount 2>&1 > showmount.txt
echo " showmount -e"
showmount -e 2>&1 > showmount-e.txt
else
echo " Skipping showmount. NFS does not appear in rpcinfo."
echo " NFS does not appear in rpcinfo. Skipping showmount." >> $HOSTNAME-errors.log
fi
echo " pkginfo -l"
pkginfo -l > pkginfo-l.txt
echo " crontab -l"
crontab -l > crontab-l.txt
echo " showrev -a"
showrev -a > showrev-a.txt
echo " xhost"
xhost 2>&1 1>xhost.txt
echo " eeprom security-mode"
eeprom security-mode 2>&1 1>eeprom-security-mode.txt
echo " prtconf -D"
prtconf -D 2>&1 1>prtconf-D.txt
echo ""
echo "Gathering file listing/permissions for STIG checks"
echo " NOTE: find errors are normal"
rm -f file-permissions.txt
# Get FStype for /
#FSTYPE=`mount -p | egrep " \/ [a-z]+" | awk '{print $4}'`
find / -local -ls > file-permissions.txt
ls -sh file-permissions.txt
echo ""
echo "Collecting some security configuration files and folders."
echo " NOTE: Inability to find some files is normal":
# use cp -R - cron.d has a named pipe
for FILE in $FILELIST ; do
if [ -f $FILE -o -d $FILE ] ; then
DEST=`echo $FILE | sed "s/\//\-/g" | sed "s/^\-//"`
echo "cp -R $FILE ./$FILEDIR/$DEST"
cp -R $FILE ./$FILEDIR/$DEST
else
echo " Could not find $FILE" >> $HOSTNAME-errors.log
echo " Could not find $FILE"
fi
done
# We don't want to collect password hashes, but need to know if the accounts are locked.
# Note: this "for LINE in" hack only works because there are no spaces in /etc/shadow... :o}
rm -f shadow-trimmed
echo ""
echo "Trimming /etc/shadow for safety..."
for LINE in `cat /etc/shadow` ; do
HASH=`echo $LINE | cut -d":" -f2`
# Typical password hash is 34 characters
if [ ${#HASH} -lt 13 ] ; then
echo $LINE >> shadow-trimmed.txt
elif [ ${#HASH} -lt 34 ] ; then
echo $LINE | awk -F':' 'BEGIN{ OFS=":"; } { print $1,"SHORT/WEAK HASH",$3,$4,$5,$6,$7,$8,$9 }' >> shadow-trimmed.txt
else
echo $LINE | awk -F':' 'BEGIN{ OFS=":"; } { print $1,"FILTERED",$3,$4,$5,$6,$7,$8,$9 }' >> shadow-trimmed.txt
fi
done
echo ""
echo "Please review to ensure hashes are filtered"
echo ""
cat shadow-trimmed.txt
echo ""
cd ..
echo "Tarring and Gzipping the results"
tar -cvf $DIR.tar ./$DIR
gzip $DIR.tar
echo ""
echo "All packaged up and ready to go in $DIR.tar.gz"
ls -sh $DIR.tar.gz
echo "Have a nice day!"
echo ""

BIN
exec/Target Host Tools/Windows/Windows Data Collection/7z.dll Normal file
View File

Binary file not shown.

BIN
exec/Target Host Tools/Windows/Windows Data Collection/7z.exe Normal file
View File

Binary file not shown.

BIN
exec/Target Host Tools/Windows/Windows Data Collection/DUMPSEC.exe Normal file
View File

Binary file not shown.

BIN
exec/Target Host Tools/Windows/Windows Data Collection/fciv.exe Normal file
View File

Binary file not shown.

4
exec/Target Host Tools/Windows/Windows Data Collection/files-to-version.txt Normal file
View File

@ -0,0 +1,4 @@
C:\Windows\system32\notepad.exe
C:\Program Files (x86)\Notepad++\notepad++.exe
c:\Windows\system32\EnPasFltV2x86.dll
c:\Windows\system32\EnPasFltV2x64.dll

10
exec/Target Host Tools/Windows/Windows Data Collection/filever.vbs Normal file
View File

@ -0,0 +1,10 @@
Set fso = WScript.CreateObject("Scripting.FileSystemObject")
Set objArgs = WScript.Arguments
if objArgs.Count<>1 then WScript.Quit()
if NOT fso.FileExists(objArgs(0)) then WScript.Echo("File " & objArgs(0) & " does not exist") & WScript.Quit()
set f = fso.GetFile(objArgs(0))
WScript.Echo objArgs(0) & "," & fso.GetFileVersion(objArgs(0)) & "," & f.DateLastModified
rem to call the script just use
rem cscript -nologo filever.vbs "c:\WINNT\system32\notepad.exe"

2
exec/Target Host Tools/Windows/Windows Data Collection/reg-perms-to-check.txt Normal file
View File

@ -0,0 +1,2 @@
HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Winlogon
HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion\IniFileMapping\Autorun.inf

175
exec/Target Host Tools/Windows/Windows Data Collection/reg-values-to-check.txt Normal file
View File

@ -0,0 +1,175 @@
HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\Lsa,RestrictAnonymous
HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\Lsa,RestrictAnonymousSAM
HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\Lsa,LmCompatibilityLevel
HEKY_LOCAL_MACHINE\System\CurrentControlSet\Control\Lsa,DisableDomainCreds
HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Lsa,ForceGuest
HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Lsa,SCENoApplyLegacyAuditPolicy
hkey_local_machine\system\CurrentControlSet\control\lsa,usemachineid
hkey_local_machine\system\CurrentControlSet\control\lsa\msv1_0,allownullsessionfallback
HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Lsa\MSV1_0,NTLMMinClientSec
hkey_local_machine\system\CurrentControlSet\Control\lsa\pku2u,AllowOnlineID
HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\SecurePipeServers\Winreg\AllowedExactPaths,Machine
HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager,SafeDllSearchMode
HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\SecurePipeServers\Winreg\AllowedPaths,Machine
HKEY_LOCAL_MACHINE\System\CurrentControlSet\Services\Eventlog\Security,WarningLevel
HKEY_LOCAL_MACHINE\System\CurrentControlSet\Services\IPSEC,NoDefaultExempt
HKEY_LOCAL_MACHINE\System\CurrentControlSet\Services\Netlogon\Parameters,DisablePasswordChange
HKEY_LOCAL_MACHINE\System\CurrentControlSet\Services\Netlogon\Parameters,MaximumPasswordAge
HKEY_LOCAL_MACHINE\System\CurrentControlSet\Services\LanManServer\Parameters,NullSessionShares
HKEY_LOCAL_MACHINE\System\CurrentControlSet\Services\LanManServer\Parameters,NullSessionPipes
HKEY_LOCAL_MACHINE\System\CurrentControlSet\Services\LanManServer\Parameters,RequireSecuritySignature
HKEY_LOCAL_MACHINE\System\CurrentControlSet\Services\Lanmanserver\Parameters,Hidden
HKEY_LOCAL_MACHINE\System\CurrentControlSet\Services\LanmanServer\Parameters,SmbServerNameHardeningLevel
HKEY_LOCAL_MACHINE\System\CurrentControlSet\Services\LanManServer\Parameters,AutoDisconnect
HKEY_LOCAL_MACHINE\System\CurrentControlSet\Services\LDAP,LDAPClientIntegrity
HKEY_LOCAL_MACHINE\System\CurrentControlSet\Services\Netbt\Parameters,NoNameReleaseOnDemand
HKEY_LOCAL_MACHINE\System\CurrentControlSet\Services\Tcpip\Parameters,DisableIPSourceRouting
HKEY_LOCAL_MACHINE\System\CurrentControlSet\Services\Tcpip\Parameters,EnableICMPRedirect
HKEY_LOCAL_MACHINE\System\CurrentControlSet\Services\Tcpip\Parameters,KeepAliveTime
HKEY_LOCAL_MACHINE\System\CurrentControlSet\Services\Tcpip\Parameters,PerformRouterDiscovery
HKEY_LOCAL_MACHINE\System\CurrentControlSet\Services\Tcpip\Parameters,TcpMaxDataRetransmissions
HKEY_LOCAL_MACHINE\System\CurrentControlSet\Services\Tcpip6\Parameters,DisableIpSourceRouting
HKEY_LOCAL_MACHINE\System\CurrentControlSet\Services\Tcpip6\Parameters,TcpMaxDataRetransmissions
HKEY_LOCAL_MACHINE\System\CurrentControlSet\Services\Tcpip6\Parameters,DisabledComponents
HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Winlogon,AutoAdminLogon
HKEY_LOCAL_MACHINE\Software\Microsoft\Windows NT\CurrentVersion\Winlogon,ScreenSaverGracePeriod
HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Setup\RecoveryConsole,SecurityLevel
HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion\IniFileMapping\Autorun.inf,(Default)
hkey_local_machine\Software\Policies\Microsoft\EMET\SysSettings,ASLR
hkey_local_machine\Software\Policies\Microsoft\EMET\SysSettings,DEP
hkey_local_machine\Software\Policies\Microsoft\EMET\SysSettings,SEHOP
hkey_local_machine\Software\Policies\Microsoft\EMET\Defaults,IE
hkey_local_machine\Software\Policies\Microsoft\EMET\Defaults
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Messenger\Client,CEIP
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\SearchCompanion,DisableContentFileUpdates
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows NT\Printers,DisbleHTTPPrinting
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows NT\Printers,DisableWebPnPDownload
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows NT\Rpc,RestrictRemoteClients
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows NT\Rpc,EnableAuthEpResolution
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows NT\Terminal Services,fAllowToGetHelp
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows NT\Terminal Services,fAllowUnsolicited
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows NT\Terminal Services,fPromptForPassword
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows NT\Terminal Services,fDenyTSConnections
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows NT\Terminal Services,fDisableCdm
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows NT\Terminal Services,MinEncryptionLevel
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows NT\Terminal Services,PerSessionTempDir
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows NT\Terminal Services,DeleteTempDirsOnExit
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows NT\Terminal Services,MaxDisconnectionTime
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows NT\Terminal Services,MaxIdleTime
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows NT\Terminal Services,DisablePasswordSaving
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\PCHealth\ErrorReporting,DoReport
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Peernet,Disabled
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows\DeviceInstall\Settings,AllowRemoteRPC
hkey_local_machine\Software\Policies\Microsoft\Windows\Explorer,NoDataExecutionPrevention
hkey_local_machine\Software\Policies\Microsoft\Windows\EventLog\Application,MaxSize
hkey_local_machine\Software\Policies\Microsoft\Windows\EventLog\Security,MaxSize
hkey_local_machine\Software\Policies\Microsoft\Windows\EventLog\Setup,MaxSize
hkey_local_machine\Software\Policies\Microsoft\Windows\EventLog\System,MaxSize
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows\LLTD,AllowLLTDIOONdomain
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows\LLTD,AllowLLTDIOOnPublicNet
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows\LLTD,EnableLLTDIO
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows\LLTD,ProhibitLLTDIOOnPrivateNet
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows\LLTD,AllowRspndrOndomain
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows\LLTD,AllowRspndrOnPublicNet
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows\LLTD,EnableRspndr
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows\LLTD,ProhibitRspndrOnPrivateNet
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows\Network Connections,NC_AllowNetBridge_NLA
hkey_local_machine\Software\Policies\Microsoft\Windows\TCPIP\v6Transition,6to4_State
hkey_local_machine\Software\Policies\Microsoft\Windows\TCPIP\v6Transition,ISATAP_State
hkey_local_machine\Software\Policies\Microsoft\Windows\TCPIP\v6Transition,Teredo_State
hkey_local_machine\Software\Policies\Microsoft\Windows\TCPIP\v6Transition\IPHTTPS\IPHTTPSInterface,IPHTTPS_ClientState
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows\WCN\Registrars,DisableFlashConfigRegistrar
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows\WCN\Registrars,DisableInBand802DOT11Registrar
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows\WCN\Registrars,DisableUPnPRegistrar
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows\WCN\Registrars,DisableWPDRegistrar
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows\WCN\Registrars,EnableRegistrars
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows\WCN\UI,DisableWcnUi
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows\Group Policy\{35378EAC-683F-11D2-A89A-00C04FBBCFA2},NoGPOListChanges
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\WindowsMediaPlayer,DisableAutoupdate
HKEY_LOCAL_MACHINE\Software\Microsoft\Windows\CurrentVersion\Policies\CredUI,EnumerateAdministrators
HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\CurrentVersion\policies\Explorer,NoDriveTypeAutorun
HKEY_LOCAL_MACHINE\Software\Microsoft\Windows\CurrentVersion\Policies\Explorer,NoPublishingWizard
HKEY_LOCAL_MACHINE\Software\Microsoft\Windows\CurrentVersion\Policies\Explorer,NoWebServices
HKEY_LOCAL_MACHINE\Software\Microsoft\Windows\CurrentVersion\Policies\Explorer,NoInternetOpenWith
HKEY_LOCAL_MACHINE\Software\Microsoft\Windows\CurrentVersion\Policies\Explorer,NoOnlinePrintsWizard
hkey_local_machine\Software\Microsoft\Windows\CurrentVersion\Policies\Explorer,PreXPSP2ShellProtocolBehavior
hkey_local_machine\SOFTWARE\Microsoft\Windows\CurrentVersion\Policies\Explorer,NoAutorun
hkey_local_machine\Software\Microsoft\Windows\CurrentVersion\Policies\Explorer\WAU,Disabled
HKEY_LOCAL_MACHINE\Software\Microsoft\Windows\CurrentVersion\Policies\System,FilterAdministratorToken
HKEY_LOCAL_MACHINE\Software\Microsoft\Windows\CurrentVersion\Policies\System,ConsentPromptBehaviorAdmin
HKEY_LOCAL_MACHINE\Software\Microsoft\Windows\CurrentVersion\Policies\System,EnableSecurityUIAPaths
HKEY_LOCAL_MACHINE\Software\Microsoft\Windows\CurrentVersion\Policies\System,PromptOnSecureDesktop
HKEY_LOCAL_MACHINE\Software\Microsoft\Windows\CurrentVersion\Policies\System,ValidateAdminCodeSignatures
HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\CurrentVersion\Policies\system,DisableBkGndGroupPolicy
HKEY_LOCAL_MACHINE\Software\Microsoft\Windows\CurrentVersion\Policies\system,LogonType
hkey_local_machine\Software\Microsoft\Windows\CurrentVersion\Policies\System,ReportControllerMissing
hkey_local_machine\SOFTWARE\Microsoft\Windows\CurrentVersion\Policies\System,LocalAccountTokenFilterPolicy
HKEY_LOCAL_MACHINE\Software\Microsoft\Windows\CurrentVersion\Policies\System\Kerberos\Parameters,SupportedEncryptionTypes
hkey_local_machine\Software\Microsoft\Windows\CurrentVersion\Policies\Windows\Sidebar,TurnOffUnsignedGadgets
hkey_local_machine\Software\Microsoft\Windows\CurrentVersion\Policies\Windows\Sidebar,OverrideMoreGadgetsLink
hkey_local_machine\Software\Microsoft\Windows\CurrentVersion\Policies\Windows\Sidebar,TurnOffUserInstalledGadgets
hkey_local_machine\Software\Microsoft\Windows NT\CurrentVersion\Winlogon,AllocateCDRoms
HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Active Setup\Installed Components
HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\Active Setup\Installed Components
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\EventViewer,MicrosoftEventVwrDisableLinks
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Internet Explorer\Feeds,DisableEnclosureDownload
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Power\PowerSettings\0e796bdb-100d-47d6-a2d5-f7d2daa51f51,DCSettingIndex
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Power\PowerSettings\0e796bdb-100d-47d6-a2d5-f7d2daa51f51,ACSettingIndex
hkey_local_machine\Software\Policies\Microsoft\SQMClient\Windows,CEIPEnable
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\SystemCertificates\AuthRoot,DisableRootAutoUpdate
hkey_local_machine\Software\Policies\Microsoft\WMDRM,DisableOnline
hkey_local_machine\Software\Policies\Microsoft\Windows\AppCompat,DisableInventory
HKEY_LOCAL_MACHINE\SOFTWARE\Policies\Microsoft\Windows\CurrentVersion\Internet Settings,WarnOnBadCertRecving
HKEY_LOCAL_MACHINE\SOFTWARE\Policies\Microsoft\Windows\CurrentVersion\Internet Settings\Zones\3,1E05
HKEY_LOCAL_MACHINE\SOFTWARE\Policies\Microsoft\Windows\CurrentVersion\Internet Settings\Zones\4,1E05
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows\DriverSearching,DontSearchWindowsUpdate
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows\DriverSearching,DontPromptForWindowsUpdate
hkey_local_machine\Software\Policies\Microsoft\Windows\DriverSearching,SearchOrderConfig
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows\DeviceInstall\Settings,DisableSystemRestore
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows\DeviceInstall\Settings,DisableSendGenericDriverNotFoundToWER
hkey_local_machine\Software\Policies\Microsoft\Windows\Device Metadata,PreventDeviceMetadataFromNetwork
hkey_local_machine\Software\Policies\Microsoft\Windows\Explorer,NoHeapTerminationOnCorruption
hkey_local_machine\Software\Policies\Microsoft\Windows\Explorer,NoAutoplayfornonVolume
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows\GameUX,DownloadGameInfo
hkey_local_machine\Software\Policies\Microsoft\Windows\GameUX,GameUpdateOptions
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows\HandwritingErrorReports,PreventHandwritingErrorReports
hkey_local_machine\Software\Policies\Microsoft\Windows\Homegroup,DisableHomeGroup
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows\Internet Connection Wizard,ExitOnMSICW
HKEY_LOCAL_MACHINE\SOFTWARE\Policies\Microsoft\Windows\Installer,AlwaysInstallElevated
hkey_local_machine\Software\Policies\Microsoft\Windows\Installer,SafeForScripting
hkey_local_machine\Software\Policies\Microsoft\Windows\Installer,EnableUserControl
hkey_local_machine\Software\Policies\Microsoft\Windows\Installer,DisableLUAPatching
hkey_local_machine\Software\Policies\Microsoft\Windows\Network Connections,NC_StdDomainUserSetInstaller
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows\Registration Wizard Control,NoRegistration
hkey_local_machine\Software\Policies\Microsoft\Windows\ScriptedDiagnosticsProvider\Policy,DisableQueryRemoteServer
hkey_local_machine\Software\Policies\Microsoft\Windows\ScriptedDiagnosticsProvider\Policy,EnableQueryRemoteServer
hkey_local_machine\Software\Policies\Microsoft\Windows\TabletPC,PreventHandwritingDataSharing
hkey_local_machine\Software\Policies\Microsoft\Windows\TCPIP\v6Transition,Force_Tunneling
hkey_local_machine\Software\Policies\Microsoft\Windows\WDI\{9c5a40da-b965-4fc3-8781-88dd50a6299d},ScenarioExecutionEnabled
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows\Windows Error Reporting,LoggingDisabled
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows\Windows Error Reporting,Disabled
hkey_local_machine\Software\Policies\Microsoft\Windows\Windows Error Reporting,DontSendAdditionalData
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows\Windows Search,AllowIndexingEncryptedStoresOrItems
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows\Windows Search,PreventIndexingUncachedExchangeFolders
HKEY_LOCAL_MACHINE\SOFTWARE\Policies\Microsoft\Windows\WindowsUpdate\AU,NoAutoUpdate
HKEY_LOCAL_MACHINE\SOFTWARE\Policies\Microsoft\Windows\WindowsUpdate,WUServer
hkey_local_machine\Software\Policies\Microsoft\WindowsMediaPlayer,GroupPrivacyAcceptance
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows Defender\Spynet,SpyNetReporting
hkey_local_machine\Software\Policies\Microsoft\Windows NT\Printers,DoNotInstallCompatibleDriverFromWindowsUpdate
HKEY_LOCAL_MACHINE\Software\Policies\Microsoft\Windows NT\Terminal Services,LoggingEnabled
HKEY_LOCAL_MACHINE\SOFTWARE\Policies\Microsoft\W32Time\Parameters,Type
HKEY_LOCAL_MACHINE\SOFTWARE\Policies\Microsoft\W32Time\Parameters,NTPServer
HKEY_CURRENT_USER\Software\Policies\Microsoft\Internet Explorer\Main,Use FormSuggest
HKEY_CURRENT_USER\Software\Policies\Microsoft\Internet Explorer\Restrictions,NoExternalBranding
HKEY_CURRENT_USER\Software\Policies\Microsoft\Internet Explorer\Main,FormSuggest Passwords
HKEY_CURRENT_USER\Software\Policies\Microsoft\Windows\Control Panel\Desktop,ScreenSaveActive
HKEY_CURRENT_USER\Software\Policies\Microsoft\Windows\Control Panel\Desktop,ScreenSaverIsSecure
HKEY_CURRENT_USER\Software\Policies\Microsoft\Windows\Control Panel\Desktop,ScreenSaveTimeOut
HKEY_CURRENT_USER\Software\Policies\Microsoft\Assistance\Client\1.0,NoImplicitFeedback
HKEY_CURRENT_USER\Software\Policies\Microsoft\Assistance\Client\1.0,NoExplicitFeedback
HKEY_CURRENT_USER\Software\Microsoft\Windows\CurrentVersion\WinTrust\Trust Providers\Software Publishing
HKEY_CURRENT_USER\Software\Microsoft\Windows\CurrentVersion\Policies\Attachments,SaveZoneInformation
HKEY_CURRENT_USER\Software\Microsoft\Windows\CurrentVersion\Policies\Attachments,HideZoneInfoOnProperties
HKEY_CURRENT_USER\Software\Microsoft\Windows\CurrentVersion\Policies\Attachments,ScanWithAntiVirus
HKEY_CURRENT_USER\Software\Microsoft\Windows\CurrentVersion\Policies\Explorer,NoInPlaceSharing
HKEY_CURRENT_USER\Software\Microsoft\Windows\CurrentVersion\Internet Settings,SecureProtocols

27
exec/Target Host Tools/Windows/Windows Data Collection/tee.cmd Normal file
View File

@ -0,0 +1,27 @@
::tee.cmd, by Ken Henderson
:: http://blogs.msdn.com/khen1234/archive/2005/10/27/486031.aspx
:: modified to append the file rather than over-write
@echo off
IF (%1)==() GOTO help
::Overwrite the file (W2K/XP require /Y)
:: SET slash_y=
:: ver <20> find "Windows NT" >nul
:: if ERRORLEVEL 1 set slash_y=/Y
:: ::Overwrite the file
:: copy %slash_y% nul %1 >nul 2>&1
for /f "tokens=1* delims=]" %%A in ('find /V /N ""') do (
>con echo.%%B
>>%1 echo.%%B
)
GOTO :eof
:help
ECHO.
ECHO Pipe text to the console and redirect to a file simultaneously
ECHO.
ECHO Usage: command | tee filename

299
exec/Target Host Tools/Windows/Windows Data Collection/windows-data-collection.bat Normal file
View File

@ -0,0 +1,299 @@
@echo off
REM ########################################################################
REM windows-data-collection.bat - collects ST&E relevant data
REM Jeff A. Odegard, CISSP 15 May 13
REM
REM ########################################################################
HOSTNAME >HOSTNAME.txt
for /F "eol=; tokens=1* delims= " %%i in (HOSTNAME.txt) do @set HOSTNAME=%%i
set OUTDIR=C:\temp\%HOSTNAME%
mkdir %OUTDIR% >nul 2>&1
set originaldir=%cd%
echo.
echo Results will be in %OUTDIR%
echo.
echo Admin privilege required. Checking access
net session >nul 2>&1
if %errorlevel% NEQ 0 (
echo.
echo. ERROR: Script must run with administrative privileges!
goto end_of_script
)
echo. - Success
echo.
set VERSION="0.1, 15 May 13"
echo. | date /t >C:\temp\%HOSTNAME%\date.txt
echo. | time /t >C:\temp\%HOSTNAME%\time.txt
set dt=
set tm=
for /F "eol=; tokens=* delims= " %%i in (C:\temp\%HOSTNAME%\date.txt) do @set dt=%%i
for /F "eol=; tokens=5 delims= " %%i in (C:\temp\%HOSTNAME%\time.txt) do @set tm=%%i
rem This is the way you do string substitution in Batch...
set dt=%dt:/=-%
set dt=%dt: =%
REM set tm=%tm =%
REM set tm=%tm: =%
del /F C:\temp\%HOSTNAME%\date.txt C:\temp\%HOSTNAME%\time.txt HOSTNAME.txt
set OUTBASE=%HOSTNAME%.txt
set SUMMARYFILE=%OUTDIR%\%HOSTNAME%-data-collection-summary-%dt%.txt
set CHECKSUMS=%OUTDIR%\%HOSTNAME%-checksums-%dt%.txt
setlocal enableextensions enabledelayedexpansion
REM ########################################################################
REM Data Gathering Starts Here
REM ########################################################################
del /f /q c:\temp\%HOSTNAME%\*
echo User List | tee.cmd %SUMMARYFILE%
wmic useraccount get name,sid 2>&1 > %OUTDIR%\user_list.txt
echo Windows Registry Permissions
REM This produces a 60Meg file. Need to modify to only pull perms required for STIG
echo * DUMPSEC.exe /rpt=registry=HKEY_LOCAL_MACHINE /outfile=%OUTDIR%\HKLM-permissions.csv /saveas=csv >> %SUMMARYFILE%
echo * dumpsec.exe HKEY_LOCAL_MACHINE to HKLM-permissions.csv
DUMPSEC.exe /rpt=registry=HKEY_LOCAL_MACHINE /outfile=%OUTDIR%\HKLM-permissions.csv /saveas=csv
echo * DUMPSEC.exe /rpt=registry=HKEY_USERS /outfile=%OUTDIR%\HKU-permissions.csv /saveas=csv >> %SUMMARYFILE%
echo * dumpsec.exe HKEY_USERS to HKU-permissions.csv
DUMPSEC.exe /rpt=registry=HKEY_USERS /outfile=%OUTDIR%\HKU-permissions.csv /saveas=csv
echo.
echo * 1.006, net localgroup "Administrators" | tee.cmd %OUTDIR%\admin_group.txt
echo -- net localgroup "Administrators" | tee.cmd %SUMMARYFILE%
net localgroup "Administrators" > %OUTDIR%\admin_group.txt
echo.
echo * 1.007, Backup Operators Group | tee.cmd %OUTDIR%\backup_group.txt
echo -- net localgroup "Backup Operators" | tee.cmd %SUMMARYFILE%
net localgroup "Backup Operators" > %OUTDIR%\backup_group.txt
echo.
echo * 2.001, Log File Permissions | tee.cmd %OUTDIR%\log_permissions.txt
echo -- icacls C:\Windows\System32\winevt\Logs\Application.evtx | tee.cmd %SUMMARYFILE%
icacls C:\Windows\System32\winevt\Logs\Application.evtx > %OUTDIR%\log_permissions.txt
echo -- icacls C:\Windows\System32\winevt\Logs\Security.evtx | tee.cmd %SUMMARYFILE%
icacls C:\Windows\System32\winevt\Logs\Security.evtx >> %OUTDIR%\log_permissions.txt
echo -- icacls C:\Windows\System32\winevt\Logs\System.evtx | tee.cmd %SUMMARYFILE%
icacls C:\Windows\System32\winevt\Logs\System.evtx >> %OUTDIR%\log_permissions.txt
echo.
echo * 2.008 NTFS Requirement | tee.cmd %OUTDIR%\disk_partitions.txt
echo list volume > listvol.scr
echo -- diskpart /s listvol.scr | tee.cmd %SUMMARYFILE%
diskpart /s listvol.scr > %OUTDIR%\disk_partitions.txt
del listvol.scr
echo * 2.015 File Share ACLs | tee.cmd %OUTDIR%\net_shares.txt
echo -- net share | tee.cmd %SUMMARYFILE%
net share > %OUTDIR%\net_shares.txt
for /F "eol=; tokens=1 delims= " %%i in (%OUTDIR%\net_shares.txt) do (
set mytest=foo
if %%i == "2.015" set mytest=bar
if "%%i" == "Share" set mytest=bar
if "%%i" == "The" set mytest=bar
if "%%i" == "-------------------------------------------------------------------------------" set mytest=bar
if '%%i' == '*' set mytest =bar
if "!mytest!"=="foo" (
echo. - Permissions for %%i
echo - net share %%i >> %SUMMARYFILE%
net share %%i >> %OUTDIR%\net_shares.txt 2>&1
)
)
echo.
echo * 2.005, Unsupported Service Packs | tee.cmd %OUTDIR%\os_info.txt
echo -- systeminfo OS Name, Version, Type, Domain, Logon Server | tee.cmd %SUMMARYFILE%
systeminfo | findstr /B /C:"OS Name" /C:"OS Version" /C:"System Type" /C:"Domain" /C:"Logon Server" > %OUTDIR%\os_info.txt
echo.
echo * 2.019 Security Related Software Patches | tee.cmd %OUTDIR%\hotfixes.txt
echo -- wmic /output:hotfixes.txt qfe list | tee.cmd %SUMMARYFILE%
wmic qfe list > %OUTDIR%\hotfixes.txt
echo.
echo * 2.021, Software Certificate Installation Files | tee.cmd %OUTDIR%\hotfixes.txt
echo -- dir /s /b *.p12 *.pfs (C:\) | tee.cmd %SUMMARYFILE%
cd C:\
dir /s /b *.p12 *.pfs > %OUTDIR%\hotfixes.txt
cd %originaldir%
echo.
REM Miscellaneous info
echo Miscellaneous Information | tee.cmd %SUMMARYFILE%
echo * tasklist.exe - process list | tee.cmd %SUMMARYFILE%
set OUTFILE=%OUTDIR%\tasklist.txt
echo -- tasklist.exe | tee.cmd %SUMMARYFILE%
tasklist.exe > %OUTFILE%
fciv.exe -both "%OUTFILE%" >> %CHECKSUMS%
echo * net.exe start - Running Windows Services | tee.cmd %SUMMARYFILE%
set OUTFILE=%OUTDIR%\net-start.txt
echo -- net.exe start | tee.cmd %SUMMARYFILE%
net.exe start > %OUTFILE%
fciv.exe -both "%OUTFILE%" >> %CHECKSUMS%
echo * tasklist /svc - Services Associated with Processes | tee.cmd %SUMMARYFILE%
set OUTFILE=%OUTDIR%\tasklist-svc.txt
echo -- tasklist.exe /svc | tee.cmd %SUMMARYFILE%
tasklist.exe /svc > %OUTFILE%
fciv.exe -both "%OUTFILE%" >> %CHECKSUMS%
echo * wmic process list full - detailed process information | tee.cmd %SUMMARYFILE%
set OUTFILE=%OUTDIR%\wmic-process-list-full.txt
echo -- wmic.exe process list full | tee.cmd %SUMMARYFILE%
wmic.exe process list full > %OUTFILE%
fciv.exe -both "%OUTFILE%" >> %CHECKSUMS%
echo * wmic startup list full - List all startup tasks | tee.cmd %SUMMARYFILE%
set OUTFILE=%OUTDIR%\wmic-startup-list-full.txt
echo -- wmic.exe startup list full | tee.cmd %SUMMARYFILE%
wmic.exe startup list full > %OUTFILE%
fciv.exe -both "%OUTFILE%" >> %CHECKSUMS%
echo * reg query - list contents of startup registry keys | tee.cmd %SUMMARYFILE%
echo - reg query HKLM\Software\Microsoft\Windows\CurrentVersion\Run
echo - reg query HKLM\Software\Microsoft\Windows\CurrentVersion\Run >> %SUMMARYFILE%
set OUTFILE=%OUTDIR%\reg-query-Run.txt
echo -- reg.exe query HKLM\Software\Microsoft\Windows\CurrentVersion\Run | tee.cmd %SUMMARYFILE%
reg.exe query HKLM\Software\Microsoft\Windows\CurrentVersion\Run 2>&1 > %OUTFILE%
fciv.exe -both "%OUTFILE%" >> %CHECKSUMS%
echo - reg query HKLM\Software\Microsoft\Windows\CurrentVersion\Runonce | tee.cmd %SUMMARYFILE%
set OUTFILE=%OUTDIR%\reg-query-Runonce.txt
echo -- reg.exe query HKLM\Software\Microsoft\Windows\CurrentVersion\Runonce | tee.cmd %SUMMARYFILE%
reg.exe query HKLM\Software\Microsoft\Windows\CurrentVersion\Runonce 2>&1 > %OUTFILE%
fciv.exe -both "%OUTFILE%" >> %CHECKSUMS%
echo - reg query HKLM\Software\Microsoft\Windows\CurrentVersion\RunonceEx | tee.cmd %SUMMARYFILE%
set OUTFILE=%OUTDIR%\reg-query-Runonce-Ex.txt
echo -- reg.exe query HKLM\Software\Microsoft\Windows\CurrentVersion\RunonceEx | tee.cmd %SUMMARYFILE%
reg.exe query HKLM\Software\Microsoft\Windows\CurrentVersion\RunonceEx 2>&1 > %OUTFILE%
fciv.exe -both "%OUTFILE%" >> %CHECKSUMS%
echo * netstat -naob - list network services, connections and processes | tee.cmd %SUMMARYFILE%
set OUTFILE=%OUTDIR%\netstat-naob.txt
echo -- netstat -naob | tee.cmd %SUMMARYFILE%
netstat -naob > %OUTFILE%
fciv.exe -both "%OUTFILE%" >> %CHECKSUMS%
echo * nbtstat -S - record active NetBIOS connections | tee.cmd %SUMMARYFILE%
set OUTFILE=%OUTDIR%\nbtstat-S.txt
echo -- nbtstat -S | tee.cmd %SUMMARYFILE%
nbtstat -S > %OUTFILE%
fciv.exe -both "%OUTFILE%" >> %CHECKSUMS%
echo * nbtstat -c - record cached NetBIOS connections | tee.cmd %SUMMARYFILE%
set OUTFILE=%OUTDIR%\nbtstat-c.txt
echo -- nbtstat -c | tee.cmd %SUMMARYFILE%
nbtstat -c > %OUTFILE%
fciv.exe -both "%OUTFILE%" >> %CHECKSUMS%
echo * arp -a - record Arp Table | tee.cmd %SUMMARYFILE%
set OUTFILE=%OUTDIR%\arp-a.txt
echo -- arp -a | tee.cmd %SUMMARYFILE%
arp -a > %OUTFILE%
fciv.exe -both "%OUTFILE%" >> %CHECKSUMS%
echo * ipconfig /all - List all network devices | tee.cmd %SUMMARYFILE%
set OUTFILE=%OUTDIR%\ipconfig-all.txt
echo -- ipconfig /all | tee.cmd %SUMMARYFILE%
ipconfig /all > %OUTFILE%
fciv.exe -both "%OUTFILE%" >> %CHECKSUMS%
echo * net view \\127.0.0.1 - list file shares | tee.cmd %SUMMARYFILE%
set OUTFILE=%OUTDIR%\file-shares.txt
echo -- net view \\127.0.0.1 | tee.cmd %SUMMARYFILE%
net view \\127.0.0.1 > %OUTFILE%
fciv.exe -both "%OUTFILE%" >> %CHECKSUMS%
echo * net sessions - list open NetBIOS sessions | tee.cmd %SUMMARYFILE%
set OUTFILE=%OUTDIR%\net-sessions.txt
echo -- net sessions | tee.cmd %SUMMARYFILE%
net sessions > %OUTFILE%
fciv.exe -both "%OUTFILE%" >> %CHECKSUMS%
echo * netsh firewall show config - display firewall configuration | tee.cmd %SUMMARYFILE%
set OUTFILE=%OUTDIR%\netsh-firewall-show-config.txt
echo -- netsh firewall show config | tee.cmd %SUMMARYFILE%
netsh firewall show config > %OUTFILE%
fciv.exe -both "%OUTFILE%" >> %CHECKSUMS%
echo * net user - list system users | tee.cmd %SUMMARYFILE%
set OUTFILE=%OUTDIR%\system-users.txt
echo -- net user | tee.cmd %SUMMARYFILE%
net user > %OUTFILE%
fciv.exe -both "%OUTFILE%" >> %CHECKSUMS%
echo * net localgroup administrators - list local system administrator accounts | tee.cmd %SUMMARYFILE%
set OUTFILE=%OUTDIR%\local-administrators.txt
echo -- net localgroup administrator | tee.cmd %SUMMARYFILE%
net localgroup administrators > %OUTFILE%
fciv.exe -both "%OUTFILE%" >> %CHECKSUMS%
echo * Installed Software | tee.cmd %SUMMARYFILE%
set OUTFILE=%OUTDIR%\installed-software.csv
echo -- wmic product /format:csv get name,version | tee.cmd %SUMMARYFILE%
wmic product get /format:csv name,version > %OUTFILE%
echo * Query the registry for values | tee.cmd %SUMMARYFILE%
for /F "eol=; tokens=1,2 delims=," %%i in (reg-values-to-check.txt) do (
echo - reg query "%%i" | tee.cmd %SUMMARYFILE%
if "%%j" NEQ "" (
if "%%j" EQU "(Default)" (reg query "%%i" /v 2>&1 >> %OUTDIR%\registry-values.txt
) else (reg query "%%i" /v "%%j" 2>&1 >> %OUTDIR%\registry-values.txt)
) else (reg query "%%i" 2>&1 >> %OUTDIR%\registry-values.txt)
)
echo * Gather file version information | tee.cmd %SUMMARYFILE%
for /F "eol=; tokens=* delims= " %%i in (files-to-version.txt) do (
echo -- filever.vbs %%i | tee.cmd %SUMMARYFILE%
cscript -nologo "%originaldir%\filever.vbs" "%%i" >> %OUTDIR%\file-version-results.txt
)
echo * Gather auditing information | tee.cmd %SUMMARYFILE%
set OUTFILE=%OUTDIR%\audit_information.csv
echo -- auditpol /get /category:* /r
auditpol /get /category:* /r > %OUTFILE%
echo * Gather Security Policy Information | tee.cmd %SUMMARYFILE%
echo -- secedit /export /cfg security_policy.inf /areas SECURITYPOLICY
secedit /export /cfg %OUTDIR%\security_policy.inf /areas SECURITYPOLICY /quiet
REM Do this last, so they can save off the policy, zip everything up and finish the script.
REM echo Security Policy Checks | tee.cmd %SUMMARYFILE%
REM echo gpedit.msc will open.
REM echo Expand: Computer Configuration
REM echo - Windows Settings
REM echo -- Security Settings
REM echo --- Local Policies
REM echo.
REM echo * Audit Policy - single click, right click - export list,
REM echo -- save as audit_policy.txt
REM echo * User Rights - single click, right click - export list,
REM echo -- save as user_rights.txt
REM echo * Security Options - single click, right click - export list,
REM echo -- save as security_options.txt
REM echo * Under Account Policies, save Password Policies and Account Lockout Policies
REM echo -- save as password_policy.txt
REM echo -- save as account_policy.txt
REM echo.
REM echo Close the Policy editor when you're finished.
REM echo * gpedit.msc | tee.cmd %SUMMARYFILE%
REM gpedit.msc
REM echo.
REM ########################################################################
REM End Data Gathering
REM ########################################################################
echo Windows data collection complete. Normality has been restored...
REM echo Zip and copy %OUTDIR% to the PTL for further analysis.
REM dir %OUTDIR%
REM PAUSE
7z u -y -tzip DataCollection.zip %OUTDIR%
cd %originaldir%
:end_of_script
cd !originaldir!

38
exec/Target Host Tools/Windows/Windows-Manual-Tools/Fport-readme.txt Normal file
View File

@ -0,0 +1,38 @@
Readme for fport v2.0
fport supports Windows NT4, Windows 2000 and Windows XP
fport reports all open TCP/IP and UDP ports and maps them to the owning application.
This is the same information you would see using the 'netstat -an' command, but it also
maps those ports to running processes with the PID, process name and path. Fport can be
used to quickly identify unknown open ports and their associated applications.
Usage:
C:\>fport
FPort v2.0 - TCP/IP Process to Port Mapper
Copyright 2000 by Foundstone, Inc.
http://www.foundstone.com
Pid Process Port Proto Path
392 svchost -> 135 TCP C:\WINNT\system32\svchost.exe
8 System -> 139 TCP
8 System -> 445 TCP
508 MSTask -> 1025 TCP C:\WINNT\system32\MSTask.exe
392 svchost -> 135 UDP C:\WINNT\system32\svchost.exe
8 System -> 137 UDP
8 System -> 138 UDP
8 System -> 445 UDP
224 lsass -> 500 UDP C:\WINNT\system32\lsass.exe
212 services -> 1026 UDP C:\WINNT\system32\services.exe
The program contains five (5) switches. The switches may be utilized using either a '/'
or a '-' preceding the switch. The switches are;
Usage:
/? usage help
/p sort by port
/a sort by application
/i sort by pid
/ap sort by application path
For updates visit: www.foundstone.com

BIN
exec/Target Host Tools/Windows/Windows-Manual-Tools/Fport.exe Normal file
View File

Binary file not shown.

28
exec/Target Host Tools/Windows/Windows-Manual-Tools/tee.cmd Normal file
View File

@ -0,0 +1,28 @@
::tee.cmd, by Ken Henderson
:: http://blogs.msdn.com/khen1234/archive/2005/10/27/486031.aspx
:: modified to append the file rather than over-write
@echo off
IF (%1)==() GOTO help
IF (%1)==(-a) (SET FILE=%2) || (SET FILE=%1)
::Overwrite the file (W2K/XP require /Y)
:: SET slash_y=
:: ver <20> find "Windows NT" >nul
:: if ERRORLEVEL 1 set slash_y=/Y
:: ::Overwrite the file
:: copy %slash_y% nul %1 >nul 2>&1
for /f "tokens=1* delims=]" %%A in ('find /V /N ""') do (
>con echo.%%B
>>%FILE% echo.%%B
)
GOTO :eof
:help
ECHO.
ECHO Pipe text to the console and redirect to a file simultaneously
ECHO.
ECHO Usage: command | tee filename

BIN
exec/Target Host Tools/Windows/Windows-Manual-Tools/u_active-directory-utilities.zip Normal file
View File

Binary file not shown.

BIN
exec/Target Host Tools/Windows/Windows-Manual-Tools/u_dumpsec.zip Normal file
View File

Binary file not shown.

BIN
exec/Target Host Tools/Windows/Windows-Manual-Tools/u_net-framework-tools.zip Normal file
View File

Binary file not shown.

347
exec/background_results.php Normal file
View File

@ -0,0 +1,347 @@
<?php
/**
* File: background_results.php
* Author: Ryan Prather
* Purpose: Background script file that will call appropriate function for files found
* Created: Feb 26, 2014
*
* Portions Copyright 2016-2017: Cyber Perspectives, LLC, All rights reserved
* Released under the Apache v2.0 License
*
* Portions Copyright (c) 2012-2015, Salient Federal Solutions
* Portions Copyright (c) 2008-2011, Science Applications International Corporation (SAIC)
* Released under Modified BSD License
*
* See license.txt for details
*
* Change Log:
* - Feb 26, 2014 - File created
* - May 05, 2014 - Converted parsing functions to classes for threading
* - Sep 1, 2016 - Copyright updated, added CWD parameter option,
* Converted to constants, made script execution platform independent
* - Oct 24, 2016 - Added debug output and cleaned up script string generation
* - Nov 7, 2016 - If it ain't broke, don't fix it! Had to revert to a previous version because intended improvements broke it
* - Dec 7, 2016 - Fixed bug where Windows threading was not being started,
* Changed PHP constant to PHP_BIN, and make sure that script continues running until last result file is done.
* - Jan 30, 2017 - Converted script to use parse_config.ini file instead of command line parameters and set script to remove config file when all files are completely parsed
* - Feb 15, 2017 - Converted file_types constants to defined constants and removed unnecessary parameters from parse_* scripts string creation
* - Feb 21, 2017 - Fixed path issues with scripts not running
* - Oct 23, 2017 - Conditionally delete parse_config.ini only if not in DEBUG log level
* - Oct 27, 2017 - Fix to remove desktop.ini files if found
*/
error_reporting(E_ALL);
$cmd = getopt("t::", ["help::"]);
$conf = parse_ini_file("parse_config.ini", false);
if (isset($cmd['help']) || !is_numeric($conf['ste']) || !isset($conf['doc_root'])) {
die(usage());
}
elseif (!file_exists($conf['doc_root'])) {
die("Folder {$conf['doc_root']} doesn't exist" . PHP_EOL);
}
chdir($conf['doc_root']);
set_time_limit(0);
include_once 'config.inc';
include_once 'database.inc';
include_once 'helper.inc';
include_once 'vendor/autoload.php';
$debug = (LOG_LEVEL == E_DEBUG ? true : false);
check_path(TMP . "/echecklist");
check_path(TMP . "/nessus");
check_path(TMP . "/nmap");
check_path(TMP . "/scc");
check_path(TMP . "/stig_viewer");
check_path(TMP . "/terminated");
check_path(TMP . "/unsupported");
chdir(TMP);
$dbh = new db();
$files = glob("*.*");
$stack = [];
$running = [];
$time = 0;
$threads = [];
foreach ($files as $file) {
$res = FileDetection($file);
if ($debug) {
Sagacity_Error::err_handler(print_r($res, true), E_DEBUG);
}
switch ($res['type']) {
case NESSUS:
$stack[] = array(
'exec' => 'nessus',
'file' => $file,
'ste' => $conf['ste'],
'source' => 'nessus'
);
break;
case SCC_XCCDF:
$stack[] = array(
'exec' => 'scc_xccdf',
'file' => $file,
'ste' => $conf['ste'],
'source' => 'scc_xccdf'
);
break;
case STIG_VIEWER_CKL:
$stack[] = array(
'exec' => 'stig_viewer',
'file' => $file,
'ste' => $conf['ste'],
'source' => 'stig_viewer'
);
break;
case TECH_ECHECKLIST_EXCEL:
$ignore = false;
if (isset($conf['ignore'])) {
$ignore = true;
}
$stack[] = array(
'exec' => 'excel_echecklist',
'file' => $file,
'ste' => $conf['ste'],
'ignore_hidden' => $ignore,
'source' => 'echecklist'
);
break;
case ECHECKLIST_CSV:
$stack[] = array(
'exec' => 'csv_echecklist',
'file' => $file,
'ste' => $conf['ste'],
'source' => 'echecklist'
);
break;
case PROC_ECHECKLIST_EXCEL:
$stack[] = array(
'exec' => 'proc_echecklist',
'file' => $file,
'ste' => $conf['ste']
);
break;
case HOST_DATA_COLLECTION:
$stack[] = array(
'exec' => 'data_collection',
'file' => $file,
'ste' => $conf['ste'],
'target' => $cmd['t'],
'source' => 'data_collection'
);
break;
case NMAP_GREPABLE:
case NMAP_TEXT:
case NMAP_XML:
$stack[] = array(
'exec' => 'nmap',
'file' => $file,
'ste' => $conf['ste'],
'source' => 'nmap'
);
break;
case MBSA_TEXT:
case MBSA_XML:
$stack[] = array(
'exec' => 'mbsa',
'file' => $file,
'ste' => $conf['ste'],
'source' => 'mbsa'
);
break;
case MSSQL_XML:
$stack[] = array(
'exec' => 'mssql',
'file' => $file,
'ste' => $conf['ste'],
'source' => 'mssql'
);
break;
case DIRECTORY:
break;
case strpos("UNSUPPORTED", $file) !== false:
rename($file, realpath(TMP . "/unsupported/" . basename($file)));
break;
default:
error_log("Do not have a parser for " . $file);
}
}
if ($debug) {
Sagacity_Error::err_handler(print_r($stack, true), E_DEBUG);
}
foreach ($stack as $key => $s) {
$existing = $dbh->get_Running_Script_Status($s['ste'], $s['file']);
if (isset($existing['status']) && $existing['status'] == 'RUNNING') {
unset($stack[$key]);
continue;
}
$ignore = '';
if ($s['source'] == 'echecklist' && $s['ignore_hidden']) {
$ignore = " -i=1";
}
$stack[$key]['script'] = realpath(defined('PHP_BIN') ? PHP_BIN : PHP) .
" -c " . realpath(PHP_CONF) . " " .
" -f " . realpath(DOC_ROOT . "/exec/parse_{$s['exec']}.php") . " --" .
" -f=\"{$s['file']}\"" .
$ignore .
($debug ? " --debug" : "");
$dbh->add_Running_Script(basename($s['file']), $s['ste'], $s['source'], $conf['location']);
}
$proc = array();
$count = 0;
chdir(realpath(DOC_ROOT . "/exec"));
foreach ($stack as $s) {
$threads[] = new Cocur\BackgroundProcess\BackgroundProcess($s['script']);
end($threads)->run();
sleep(3);
$count++;
while ($count >= MAX_RESULTS) {
sleep(1);
$count = $dbh->get_Running_Script_Count($conf['ste']);
}
}
do {
sleep(1);
}
while ($dbh->get_Running_Script_Count($conf['ste']));
if (!$debug) {
unlink(DOC_ROOT . "/exec/parse_config.ini");
}
/**
* Function to import SCC Oval XML Result files
*
* @param string $file
*/
function import_SCC_OVAL($file) {
if (preg_match('/.*Results\_iavm\_(2009|2010)|Results\_USGCB/i', $file)) {
return;
}
$target_data = array();
$db = new db();
$match = array();
preg_match('/\_SCC-(\d\.?)+\_(\d{4}\-\d{2}\-\d{2}\_\d{6})\_OVAL/', $file, $match);
$time_stamp = $match[2];
$dt = DateTime::createFromFormat('Y-m-d_His', $time_stamp);
$source = $db->get_Sources('SCC');
$dom = new DOMDocument();
$dom->load($file);
$csv = fopen("scc/" . substr(basename($file), 0, -3) . "csv", 'w');
$ste = $db->get_STE($GLOBALS['opt']['s'])[0];
$scan = new scan(null, $source, $ste, 1, basename($file), $dt->format('Y-m-d H:i:s'));
$scan->set_ID($db->save_Scan($scan));
$x = new DOMXPath($dom);
$sysinfo = $x->query('/oval-res:oval_results/oval-res:results/oval-res:system/oval-sc:oval_system_characteristics/oval-sc:system_info')->item(0);
$target_data['os_name'] = $x->query('oval-sc:os_name', $sysinfo)->item(0)->textContent;
$target_data['os_ver'] = $x->query('oval-sc:os_version', $sysinfo)->item(0)->textContent;
$target_data['host_name'] = $x->query('oval-sc:primary_host_name', $sysinfo)->item(0)->textContent;
$interfaces = $x->query('oval-sc:interfaces/oval-sc:interface', $sysinfo);
$int_count = 0;
foreach ($interfaces as $node) {
$target_data['interface_name' . $int_count] = $x->query('oval-sc:interface_name', $node)->item(0)->textContent;
$target_data['ip' . $int_count] = $x->query('oval-sc:ip_address', $node)->item(0)->textContent;
$target_data['mac' . $int_count] = $x->query('oval-sc:mac_address', $node)->item(0)->textContent;
$int_count++;
}
$defs = $x->query('/oval-res:oval_results/oval-def:oval_definitions/oval-def:definitions/oval-def:definition');
foreach ($defs as $node) {
$id = $node->getAttribute('id');
print "Checking oval id: $id" . PHP_EOL;
//$meta = $x->query('oval-def:metadata', $node)->item(0);
$title = $x->query('oval-def:metadata/oval-def:title', $node)->item(0)->textContent;
$desc = $x->query('oval-def:metadata/oval-def:description', $node)->item(0)->textContent;
$plat = $x->query('oval-def:metadata/oval-def:affected/oval-def:platform', $node)->item(0)->textContent;
$ext = $x->query('oval-def:criteria/oval-def:extend_definition', $node);
if ($ext->length > 0) {
$ext_def = $ext->item(0)->getAttribute('definition_ref');
$ext_def_op = $x->query('oval-def:criteria', $node)->item(0)->getAttribute('operator');
}
else {
$ext_def = '';
$ext_def_op = '';
}
$ref = $x->query('oval-def:metadata/oval-def:reference', $node);
$oval = $db->get_Oval($id);
if ($oval->get_PDI_ID()) {
print "current oval: " . print_r($oval, true);
$oval->clear_References();
}
else {
$oval = new oval(null, $id, $title, $desc, $plat, $ext_def, $ext_def_op);
}
foreach ($ref as $ref_node) {
$source = $ref_node->getAttribute('source') == 'http://cce.mitre.org' ? 'CCE' : $ref_node->getAttribute('source');
$url = $ref_node->hasAttribute('ref_url') ? $ref_node->getAttribute('ref_url') : '';
$ref_id = $ref_node->getAttribute('ref_id');
$oval->add_Reference(new oval_ref($id, $source, $url, $ref_id));
if (is_null($oval->get_PDI_ID()) && $source == 'CCE') {
$cce = $db->get_CCE($ref_id);
if (!is_null($cce)) {
$oval->set_PDI_ID($cce->get_PDI_ID());
}
}
}
if ($db->save_Oval($oval)) {
error_log("Saved oval id: " . $oval->get_Oval_ID());
}
else {
error_log("Error saving oval id: " . $oval->get_Oval_ID());
}
}
}
function usage() {
print <<<EOO
Purpose: This program was written to look at all files in the /tmp directory, determine what parser is needed, then call that parser with the appropriate flags.
Usage: background_results.php -s={ste_id} [-i=1] [-t=1] [--help]
-s={STE ID} The ID of the ST&E to know what to assign the results to
-i=1 Ignore hidden Excel worksheets (only used on Excel eChecklist files) (defaulted to false)
-t={Target Name} The name of the target to evaluate (only used on host data collection)
--help This screen
EOO;
}

292
exec/background_stigs.php Normal file
View File

@ -0,0 +1,292 @@
<?php
/**
* File: background_stigs.php
* Author: Ryan Prather
* Purpose: To allow scripts to run in the background
* Currently only implements the STIG XML importing
* Created: Jul 18, 2014
*
* Portions Copyright (c) 2016-2017: Cyber Perspectives, LLC All rights reserved
* Released under the Apache v2.0 License
*
* Portions Copyright (c) 2012-2015, Salient Federal Solutions
* Portions Copyright (c) 2008-2011, Science Applications International Corporation (SAIC)
* Released under Modified BSD License
*
* See license.txt for details
*
* Change Log:
* - Jul 18, 2014 - File created
* - Dec 7, 2016 - Changed PHP constant to PHP_BIN and added Cyber Perspectives copyright
* - Dec 12, 2016 - Revised text for to run parse_stig script, delete files only if --delete parameter is set
* - Feb 15, 2017 - Formatting, revised the printed messages throughout the script, and converted file_types constants where required
* - Feb 21, 2017 - Fixed paths and revised progress output, Revised directories and fixed output
* - Mar 3, 2017 - Now shuffling the STIG files to prevent duplicate STIG creation and fixed bug with scripts not being updated to complete when done
* - Mar 8, 2017 - Fixed typo with catalog_scripts table and added update to $count value when waiting for all script to complete
* - Apr 5, 2017 - Hard coded parsing 20 STIGs instead of using MAX_RESULTS constant
* - Jun 27, 2017 - Cleanup
* - Jul 13, 2017 - Changed STIG parsing to serial instead of parallel to fix issue with duplicate STIGs from race conditions
*/
$cmd = getopt("x::h::d::", ["debug::", "delete::", "ia::", "extract::", "help::"]);
if (isset($cmd['help']) || isset($cmd['h'])) {
die(usage());
}
set_time_limit(0);
require_once 'config.inc';
require_once 'helper.inc';
require_once 'database.inc';
require_once 'vendor/autoload.php';
use Monolog\Logger;
use Monolog\Handler\StreamHandler;
use Monolog\Formatter\LineFormatter;
$log_level = Logger::ERROR;
switch(LOG_LEVEL) {
case E_WARNING:
$log_level = Logger::WARNING;
break;
case E_NOTICE:
$log_level = Logger::NOTICE;
break;
case E_DEBUG:
$log_level = Logger::DEBUG;
}
$stream = new StreamHandler("php://output", Logger::INFO);
$stream->setFormatter(new LineFormatter("%datetime% %level_name% %message%" . PHP_EOL, "H:i:s.u"));
$log = new Logger("stig_parser");
$log->pushHandler(new StreamHandler(LOG_PATH . "/stig_parser.log", $log_level));
$log->pushHandler($stream);
$path = realpath(TMP . "/stigs");
if(isset($cmd['d']) && $cmd['d']) {
$path = $cmd['d'];
}
chdir($path);
check_path(TMP . "/stigs");
check_path(TMP . "/stigs/zip");
check_path(TMP . "/stigs/checklist");
check_path(TMP . "/stigs/xml");
check_path(DOC_ROOT . "/reference/stigs");
$db = new db();
$stack = [];
$zip_files = glob("*.zip");
$zip = new ZipArchive();
// Find the .zip files that were uploaded
foreach ($zip_files as $file) {
$ft = FileDetection($file);
if ($ft['type'] == DISA_STIG_LIBRARY_ZIP) {
$log->info("Extracting $file");
$zip->open($file);
$zip->extractTo(realpath(TMP . "/stigs/checklist"));
$zip->close();
if (isset($cmd['delete'])) {
unlink($file);
}
}
}
// traverse the checklist directory to find all the zip files and extract those
for ($x = 0; $x < 2; $x++) {
$dir = new RecursiveDirectoryIterator(realpath(TMP . "/stigs/checklist"));
$files = new RecursiveIteratorIterator($dir);
directory_crawl($files);
}
// traverse the zip directory, and extract the xml, xsl, jpg, or gif files.
for ($x = 0; $x < 3; $x++) {
$dir = new RecursiveDirectoryIterator(realpath(TMP . "/stigs/zip"));
$files = new RecursiveIteratorIterator($dir);
directory_crawl($files);
}
if (isset($cmd['x']) || isset($cmd['extract'])) {
$log->info("Extract only complete");
die;
}
// find all the xml files in the directory
chdir(TMP . "/stigs/xml");
$xml_files = glob("*.xml");
// change back to the document root directory
chdir(DOC_ROOT);
$count = 0;
$db->help->update("settings", ['meta_value' => 0], [
[
'field' => 'meta_key',
'op' => '=',
'value' => 'stig-progress'
]
]);
$db->help->execute();
foreach ($xml_files as $key => $file) {
// if the file has a space in the file name we need to replace it because it will cause parsing errors
if (strpos($file, ' ') !== false) {
$new_file = str_replace(' ', '_', $file);
rename(realpath(TMP . "/stigs/xml/$file"), TMP . "/stigs/xml/$new_file");
$xml_files[$key] = $file = $new_file;
copy(realpath(TMP . "/stigs/xml/$file"), realpath(DOC_ROOT . "/reference/stigs") . "/$file");
}
// determine the file type
$ft = FileDetection(TMP . "/stigs/xml/$file");
// add the file to the stack if it is of the proper type
// can add additional types as the parser are created
if ($ft['type'] == DISA_STIG_XML) {
$log->info("Parsing STIG file: $file");
$script = realpath(defined('PHP_BIN') ? PHP_BIN : PHP) .
" -c " . realpath(PHP_CONF) .
" -f " . realpath(DOC_ROOT . "/exec/parse_stig.php") . " --" .
" -f=\"" . realpath(TMP . "/stigs/xml/{$file}") . "\"" .
(isset($cmd['debug']) ? " --debug" : "");
$db->add_Catalog_Script(basename($file));
passthru($script);
}
else {
$log->debug("Skipping $file");
continue;
}
$count++;
$db->help->update("settings", ['meta_value' => number_format(($count / count($xml_files) * 100), 2)], [
[
'field' => 'meta_key',
'op' => '=',
'value' => 'stig-progress'
]
]);
$db->help->execute();
}
$db->help->update("catalog_scripts", ['status' => 'COMPLETE'], [
[
'field' => 'perc_comp',
'op' => '=',
'value' => 100
],
[
'field' => 'status',
'op' => '=',
'value' => 'RUNNING',
'sql_op' => 'AND'
]
]);
$db->help->execute();
$db->help->update("settings", ['meta_value' => 100], [
[
'field' => 'meta_key',
'op' => IN,
'value' => ['stig-dl-progress', 'stig-progress']
]
]);
$db->help->execute();
if (isset($cmd['delete'])) {
if (strtolower(substr(PHP_OS, 0, 3)) == 'win') {
exec("del /S /Q /F " . realpath(TMP . "/stigs/checklist") . "\\*");
exec("del /S /Q /F " . realpath(TMP . "/stigs/zip") . "\\*");
}
else {
exec("rm -rf " . realpath(TMP . "/stigs/checklist") . "/*");
exec("rm -rf " . realpath(TMP . "/stigs/zip") . "/*");
}
}
/**
* Function to crawl directory structure to find zip, xml, xsl, gif, and jpg files
*
* @param RecursiveIteratorIterator $files
*/
function directory_crawl($files)
{
global $zip;
foreach ($files as $file) {
if (preg_match('/\.zip/', $file)) {
if ($zip->open($file) === true) {
for ($i = 0; $i < $zip->numFiles; $i++) {
$contents = '';
$in_Skips = false;
$path = '';
$filename = str_replace('\\', '/', $zip->getNameIndex($i));
$fileinfo = pathinfo($filename);
if (isset($fileinfo['extension']) && !$in_Skips) {
switch (strtolower($fileinfo['extension'])) {
case 'zip':
$path = TMP . "/stigs/zip/";
break;
case 'xml':
if (!preg_match('/xccdf/i', $fileinfo['basename'])) {
continue;
}
elseif (strpos($fileinfo['basename'], "$") !== false) {
continue;
}
$path = TMP . "/stigs/xml/";
break;
case 'xsl':
case 'gif':
case 'jpg':
$path = DOC_ROOT . "/reference/stigs/";
break;
}
if ($path) {
$fp = $zip->getStream($filename);
if (!$fp) {
error_log("Couldn't get zip file stream for file $filename in $file");
}
else {
while (!feof($fp)) {
$contents .= fread($fp, 1024);
}
fclose($fp);
if (file_put_contents($path . $fileinfo['basename'], $contents) === false) {
die;
}
}
}
}
}
$zip->close();
}
}
}
}
function usage()
{
print <<<EOO
Purpose: This program was written to look at all files in the {doc_root}/tmp directory, determine what parser is needed, then call that parser with the appropriate flags.
Usage: background_stigs.php [-x|--extract] [-d="directory"] [--debug] [--delete] [--ia] [-h|--help]
-x|--extract Simply extract the contents of a .zip file (STIG library) to it's proper places, do not parse the contents
-d="directory" Directory to search for the zip and xml files in (optional, defaults to {doc_root}/tmp)
--ia Override any IA controls in the DB to use only the ones that are in the STIG file
--delete Delete any files once complete
--debug Debugging output
--help This screen
EOO;
}

133
exec/create_risk_assessment.php Normal file
View File

@ -0,0 +1,133 @@
<?php
/**
* File: create_risk_assessment.php
* Author: Ryan Prather
* Purpose: File to create a final risk assessment output file
* Created: Oct 20, 2014
*
* Portions Copyright (c) 2012-2015, Salient Federal Solutions
* Portions Copyright (c) 2008-2011, Science Applications International Corporation (SAIC)
* Released under Modified BSD License
*
* See license.txt for details
*
* Change Log:
* - Oct 20, 2014 - File created
*/
$cmd = getopt("s:", array("debug::"));
if (isset($cmd['debug'])) {
$path = realpath("/Users/pratherr/workspace/web/exec");
}
else {
$path = realpath("/xampp/www/exec");
}
chdir($path);
set_time_limit(0);
set_include_path(get_include_path());
include_once 'database.inc';
include_once 'helper.inc';
$db = new db();
$log = new Sagacity_Error("Create_Risk_Assessment.log");
$ste = $db->get_STE($cmd['s'])[0];
$sys = $db->get_System_By_STE_ID($cmd['s']);
$site = $db->get_Site_By_STE_ID($cmd['s']);
$tgts = $db->get_Target_Details($cmd['s']);
$xml = new DOMDocument();
$pi = $xml->createProcessingInstruction("xml-stylesheet", 'type="text/xsl" href="diacap.xsl"');
$xml->appendChild($pi);
$xml->appendChild($report = xml_helper($xml, 'report'));
$report->appendChild($ste_node = xml_helper($xml, "ste", null, false, array(
'start_date' => $ste->get_Eval_Start_Date()->format("Y-m-d"),
'end_date' => $ste->get_Eval_End_Date()->format("Y-m-d"),
'status' => $ste->get_Status(),
'ao' => $ste->get_AO(),
'proc_checklist_fname' => "Procedural-eChecklist-" . $ste->get_ID() . ".xlsx"
)));
$ste_node->appendChild(xml_helper($xml, "recommendations", $ste->get_Recommendations(), true));
$ste_node->appendChild(xml_helper($xml, "conclusion", $ste->get_Conclusions(), true));
$ste_node->appendChild(xml_helper($xml, "constraints", $ste->get_Constraints(), true));
$ste_node->appendChild(xml_helper($xml, "assumptions", $ste->get_Assumptions(), true));
$ste_node->appendChild(xml_helper($xml, "residual_risk", $ste->get_Residual_Risk(), true));
$ste_node->appendChild(xml_helper($xml, "deviations", $ste->get_Deviations(), true));
$ste_node->appendChild(xml_helper($xml, "scope", $ste->get_Scope(), true));
$ste_node->appendChild($team = xml_helper($xml, "ste_team"));
foreach ($ste->get_STE_Team() as $key => $people) {
$team->appendChild(xml_helper($xml, "member", null, false, array(
'name' => $people->name,
'org' => $people->org,
'phone' => $people->phone,
'position' => $people->position
)));
}
$report->appendChild($sys_node = xml_helper($xml, "system", null, false, array(
'name' => $sys->get_Name(),
'class' => $sys->get_Classification(),
'mac' => $sys->get_MAC()
)));
$sys_node->appendChild(xml_helper($xml, "description", $sys->get_Description(), true));
$sys_node->appendChild(xml_helper($xml, "executive_summary", $sys->get(), true));
$report->appendchild(xml_helper($xml, "site", null, false, array(
'name' => $site->get_Name(),
'address' => $site->get_Address(),
'city' => $site->get_City(),
'state' => $site->get_State(),
'zip' => $site->get_Zip(),
'country' => $site->get_Country(),
'poc_name' => $site->get_POC_Name(),
'poc_email' => $site->get_POC_Email(),
'poc_phone' => $site->get_POC_Phone()
)));
$report->appendChild($targets = xml_helper($xml, "targets"));
foreach ($tgts as $key => $tgt) {
$os = $db->get_Software($tgt->get_OS_ID())[0];
$targets->appendChild($tgt_node = xml_helper($xml, "target", null, false, array(
'name' => $tgt->get_Name(),
'os' => $os->get_Man() . " " . $os->get_Name() . " " . $os->get_Version()
)));
foreach ($tgt->interfaces as $key => $int) {
if (false) {
$int = new interfaces();
}
$tgt_node->appendChild(xml_helper($xml, "interface", null, false, array(
'name' => $int->get_Name(),
'hostname' => $int->get_Hostname(),
'ipv4' => $int->get_IPv4(),
'ipv6' => $int->get_IPv6(),
'fqdn' => $int->get_FQDN()
)));
}
}
$report->appendChild($ia_node = xml_helper($xml, "ia_controls"));
$proc = $db->get_Proc_IA_Controls($ste);
foreach ($proc as $key => $ia) {
$ia_node->appendChild($node = xml_helper($xml, "ia_control", null, false, array(
'id' => $ia->get_Control_ID(),
'name' => $ia->get_Name(),
'status' => $ia->get_Worst_Status_String()
)));
$node->appendChild(xml_helper($xml, "vuln_desc", $ia->finding->vul_desc, true));
$node->appendChild(xml_helper($xml, "mitigations", $ia->finding->mitigations, true));
$node->appendChild(xml_helper($xml, "references", $ia->finding->reference, true));
$node->appendChild(xml_helper($xml, "notes", $ia->finding->notes, true));
}
$xml->formatOutput = true;
$xml->save("../report/" . $sys->get_Name() . "_" . $site->get_Name() . "_" . $ste->get_Eval_Start_Date()->format("Y_m_d") . ".xml");

BIN
exec/eMASS-CCI.rmf.xlsx Normal file
View File

Binary file not shown.

434
exec/export-ckl.php Normal file
View File

@ -0,0 +1,434 @@
<?php
/**
* File: export-ckl.php
* Author: Ryan Prather <ryan.prather@cyberperspectives.com>
* Purpose:
* Created: Feb 20, 2017
*
* Copyright 2017: Cyber Perspective, LLC, All rights reserved
* Released under the Apache v2.0 License
*
* See license.txt for details
*
* Change Log:
* - Feb 20, 2017 - File created
* - Mar 8, 2017 - Completed preliminary functionality
* - May 13, 2017 - Set default export path to TMP/ckl
* Only exporting manual checklists and not export orphan findings
* - Oct 23, 2017 - Added a few more fields and added data to some fields that didn't have a value
* - Nov 25, 2017 - Fixed notice bug #346
* - Jan 6, 2018 - Bug fix #337 and formatting
*/
include_once 'config.inc';
include_once 'database.inc';
include_once 'helper.inc';
include_once 'array2xml.inc';
$db = new db();
$dt = new DateTime();
$cmd = getopt('s:t::c::h::d::', ['help::', 'debug::']);
if (!isset($cmd['s']) || !is_numeric($cmd['s'])) {
die(usage());
}
if (isset($cmd['t'])) {
$tgts = $db->get_Target_Details($cmd['s'], $cmd['t']);
}
elseif (isset($cmd['c'])) {
$tgts = $db->get_Target_By_Category($cmd['c']);
}
else {
$tgts = $db->get_Target_Details($cmd['s']);
}
if (isset($cmd['d'])) {
if (file_exists($cmd['d'])) {
$dest = realpath($cmd['d']) . "/";
}
else {
die("Could not find destination path {$cmd['d']}");
}
}
else {
check_path(TMP . "/ckl", true);
$dest = realpath(TMP . "/ckl") . "/";
}
print "Destination: $dest" . PHP_EOL;
$xml = new Array2XML();
Array2XML::$all_caps = true;
$xml->standalone = true;
$xml->formatOutput = true;
$chk_comp_count = 0;
$tgt_comp_count = 0;
$total_chk_count = 0;
$total_stigs = 0;
if ($tgt_count = count($tgts)) {
print "Total Targets: $tgt_count" . PHP_EOL;
foreach ($tgts as $tgt) {
$host_ip = (is_array($tgt->interfaces) && count($tgt->interfaces) ? current($tgt->interfaces)->get_IPv4() : null);
$host_fqdn = (is_array($tgt->interfaces) && count($tgt->interfaces) ? current($tgt->interfaces)->get_FQDN() : null);
$host_mac = (is_array($tgt->interfaces) && count($tgt->interfaces) ? current($tgt->interfaces)->get_MAC() : null);
//$host_mac = (count($tgt->interfaces) ? current($tgt->interfaces)->get_Mac() : null);
print "Target: {$tgt->get_Name()}" . PHP_EOL;
foreach ($tgt->checklists as $key => $chk) {
if ($chk->name == 'Orphan' || $chk->type != 'manual') {
unset($tgt->checklists[$key]);
}
}
$total_chk_count += $chk_count = (is_array($tgt->checklists) ? count($tgt->checklists) : 0);
print "Total Checklists: $chk_count" . PHP_EOL;
foreach ($tgt->checklists as $chk) {
print "Type: {$chk->type}\tChecklist: {$chk->name}" . PHP_EOL;
$class = '';
$stig_class = '';
switch ($chk->get_Classification()) {
case 'U':
$class = 'UNCLASSIFIED';
$stig_class = "Unclass";
break;
case 'FOUO':
$class = 'UNCLASSIFIED//FOUO';
$stig_class = "FOUO";
break;
case 'S':
$class = 'SECRET';
$stig_class = "Secret";
break;
}
$arr = [
'ASSET' => [
'ASSET_TYPE' => 'Computing',
'HOST_NAME' => $tgt->get_Name(),
'HOST_IP' => $host_ip,
'HOST_MAC' => $host_mac,
'HOST_GUID' => '',
'HOST_FQDN' => $host_fqdn,
'TECH_AREA' => '',
'TARGET_KEY' => '',
'WEB_OR_DATABASE' => false,
'WEB_DB_SITE' => '',
'WEB_DB_INSTANCE' => ''
],
'STIGS' => [
'iSTIG' => [
'STIG_INFO' => [
'SI_DATA' => [
[
'SID_NAME' => 'version',
'SID_DATA' => $chk->get_Version()
],
[
'SID_NAME' => 'classification',
'SID_DATA' => $class
],
[
'SID_NAME' => 'customname'
],
[
'SID_NAME' => 'stigid',
'SID_DATA' => $chk->get_Checklist_ID()
],
[
'SID_NAME' => 'description',
'SID_DATA' => $chk->get_Description()
],
[
'SID_NAME' => 'filename',
'SID_DATA' => $chk->get_File_Name()
],
[
'SID_NAME' => 'releaseinfo',
'SID_DATA' => "Release: {$chk->get_Release()} Benchmark Date: {$chk->get_Date()->format("j M Y")}"
],
[
'SID_NAME' => 'title',
'SID_DATA' => $chk->get_Name()
],
[
'SID_NAME' => 'uuid',
'SID_DATA' => UUID::v4()
],
[
'SID_NAME' => 'notice',
'SID_DATA' => 'terms-of-use'
],
[
'SID_NAME' => 'source',
'SID_DATA' => 'STIG.DOD.MIL'
]
]
]
]
]
];
$pdis = get_checklist_data($tgt, $chk);
$stig_data = [];
$total_stigs += $pdi_count = (is_array($pdis) ? count($pdis) : 0);
$count = 0;
foreach ($pdis as $pdi) {
$find = $db->get_Finding($tgt, new stig($pdi['pdi_id'], $pdi['STIG_ID'], null));
if (is_array($find) && count($find) && isset($find[0]) && is_a($find[0], 'finding')) {
$find = $find[0];
}
$sev = 'low';
if ($pdi['CAT'] == 'I') {
$sev = 'high';
}
elseif ($pdi['CAT'] == 'II') {
$sev = 'medium';
}
$ccis = preg_grep("/CCI\-/", explode(" ", $pdi['IA_Controls']));
$cci_list = [];
if (is_array($ccis) && count($ccis)) {
foreach ($ccis as $cci) {
$cci_list[] = [
'VULN_ATTRIBUTE' => 'CCI_REF',
'ATTRIBUTE_DATA' => $cci
];
}
}
// decoding because check contents are already encoded
$cc = str_replace("\\n", "<br />", htmlentities(html_entity_decode($pdi['check_contents'])));
$stig_data = array_merge([
[
'VULN_ATTRIBUTE' => 'Vuln_Num',
'ATTRIBUTE_DATA' => $pdi['VMS_ID']
],
[
'VULN_ATTRIBUTE' => 'Severity',
'ATTRIBUTE_DATA' => $sev
],
[
'VULN_ATTRIBUTE' => 'Group_Title',
'ATTRIBUTE_DATA' => $pdi['group_title']
],
[
'VULN_ATTRIBUTE' => 'Rule_ID',
'ATTRIBUTE_DATA' => $pdi['SCAP_Rule']
],
[
'VULN_ATTRIBUTE' => 'Rule_Ver',
'ATTRIBUTE_DATA' => $pdi['STIG_ID']
],
[
'VULN_ATTRIBUTE' => 'Rule_Title',
'ATTRIBUTE_DATA' => $pdi['short_title']
],
[
'VULN_ATTRIBUTE' => 'Vuln_Discuss',
'ATTRIBUTE_DATA' => $pdi['Description']
],
[
'VULN_ATTRIBUTE' => 'IA_Controls',
'ATTRIBUTE_DATA' => $pdi['IA_Controls']
],
[
'VULN_ATTRIBUTE' => 'Check_Content',
'ATTRIBUTE_DATA' => $cc
],
[
'VULN_ATTRIBUTE' => 'Fix_Text',
'ATTRIBUTE_DATA' => htmlentities($pdi['fix_text'])
],
[
'VULN_ATTRIBUTE' => 'False_Positives',
'ATTRIBUTE_DATA' => ''
],
[
'VULN_ATTRIBUTE' => 'False_Negatives',
'ATTRIBUTE_DATA' => ''
],
[
'VULN_ATTRIBUTE' => 'Documentable',
'ATTRIBUTE_DATA' => 'false'
],
[
'VULN_ATTRIBUTE' => 'Mitigations',
'ATTRIBUTE_DATA' => ''
],
[
'VULN_ATTRIBUTE' => 'Potential_Impact',
'ATTRIBUTE_DATA' => ''
],
[
'VULN_ATTRIBUTE' => 'Third_Party_Tools',
'ATTRIBUTE_DATA' => ''
],
[
'VULN_ATTRIBUTE' => 'Mitigation_Control',
'ATTRIBUTE_DATA' => ''
],
[
'VULN_ATTRIBUTE' => 'Responsibility',
'ATTRIBUTE_DATA' => ''
],
[
'VULN_ATTRIBUTE' => 'Security_Override_Guidance',
'ATTRIBUTE_DATA' => ''
],
[
'VULN_ATTRIBUTE' => 'Check_Content_Ref',
'ATTRIBUTE_DATA' => ''
],
[
'VULN_ATTRIBUTE' => 'Class',
'ATTRIBUTE_DATA' => $stig_class
],
[
'VULN_ATTRIBUTE' => 'STIGRef',
'ATTRIBUTE_DATA' => "{$chk->get_Name()} :: Release: {$chk->get_Release()} Benchmark Date: {$chk->get_Date()->format("j M Y")}"
],
[
'VULN_ATTRIBUTE' => 'TargetKey',
'ATTRIBUTE_DATA' => ''
]
], $cci_list);
$status = "Not_Reviewed";
$notes = '';
if (is_a($find, 'finding')) {
$status = $find->get_Finding_Status_String();
if ($status == 'Not a Finding' || $status == 'False Positive') {
$status = "NotAFinding";
}
elseif($status == 'Exception') {
$status = 'Open';
}
else {
$status = str_replace(" ", "_", $status);
}
$notes = $find->get_Notes();
}
$arr['STIGS']['iSTIG']['VULN'][] = [
'STIG_DATA' => $stig_data,
'STATUS' => $status,
'FINDING_DETAILS' => $notes,
'COMMENTS' => '',
'SEVERITY_OVERRIDE' => '',
'SEVERITY_JUSTIFICATION' => ''
];
$count++;
printf("\r%.2f%%", ($count / $pdi_count) * 100);
}
print PHP_EOL;
$file = $xml->createXML('CHECKLIST', $arr);
$file->save("{$dest}{$tgt->get_Name()}_{$chk->get_Checklist_ID()}_{$chk->get_type()}_{$dt->format("Ymd")}.ckl");
}
}
}
print <<<EOO
Total Targets: $tgt_count
Total Checklists: $total_chk_count
Total STIGs: $total_stigs
EOO;
/**
*
* @global db $db
*
* @param target $tgt
* @param checklist $chk
*
* @return mixed
*/
function get_checklist_data($tgt, $chk) {
if (!is_a($tgt, 'target') || !is_a($chk, 'checklist')) {
return;
}
global $db;
$db->help->select("sagacity.pdi", ["pdi.*", "pcl.*", "s.description AS 'Description'"], [
[
'field' => 'tc.tgt_id',
'op' => '=',
'value' => $tgt->get_ID()
],
[
'field' => 'tc.chk_id',
'op' => '=',
'value' => $chk->id,
'sql_op' => 'AND'
]
], [
'table_joins' => [
"JOIN sagacity.pdi_checklist_lookup pcl ON pcl.pdi_id = pdi.pdi_id",
"JOIN sagacity.target_checklist tc ON tc.chk_id = pcl.checklist_id",
"JOIN sagacity.stigs s ON s.pdi_id = pdi.pdi_id"
]
]);
$pdis = $db->help->execute();
return $pdis;
}
/**
* Function retrieve
*
* @global db $db
*
* @param target $tgt
* @param checklist $chk
*
* @return mixed
*/
function get_finding_data($tgt, $chk) {
global $db;
$ret = [];
return $ret;
}
/**
* Usage output
*/
function usage() {
print <<<EOO
Purpose: This script was written to be able to export CKL files from the data contained in the database.
Usage: php export-ckl.php [-d={destination}] -s={ste id} [-c={category id}] [-t={target id}] [-h|--help]
-s={STE ID} Export a CKL for each assigned checklist for ALL targets in this ST&E
-c={Category ID} Export CKL files for all targets contained in this Category
-t={Target ID} Export CKL file for each assigned checklist for this target
-d={destination} Location of where you want the files saved
-h|--help This screen
--debug Debugging output
EOO;
}

489
exec/installer.php Normal file
View File

@ -0,0 +1,489 @@
<?php
/**
* File: installer.php
* Author: Ryan Prather <ryan.prather@cyberperspectives.com>
* Purpose: This script runs the installer processes
* Created: Nov 28, 2017
*
* Copyright 2017: Cyber Perspective, LLC, All rights reserved
* Released under the Apache v2.0 License
*
* See license.txt for details
*
* Change Log:
* - Nov 28, 2017 - File created
* - Dec 27, 2017 - Fixed bug with SCG showing empty, and added download progress meta keys
* - Jan 2, 2018 - Add sleep to fix bug #357 race condition
* - Jan 10, 2018 - Formatting
*/
include_once 'helper.inc';
include_once 'vendor/autoload.php';
use Cocur\BackgroundProcess\BackgroundProcess;
set_time_limit(0);
$params = [
'filter' => FILTER_SANITIZE_STRING,
'flag' => FILTER_NULL_ON_FAILURE
];
$db_step = [
'doc-root' => $params,
'pwd-file' => $params,
'tmp-path' => $params,
'log-path' => $params,
'log-level' => $params,
'db-server' => $params,
'root-uname' => $params,
'root-pwd' => $params,
'conf-root-pwd' => $params,
'web-pwd' => $params,
'local-path' => $params,
'action' => $params,
'sample-data' => ['filter' => FILTER_VALIDATE_BOOLEAN],
'cpe' => ['filter' => FILTER_VALIDATE_BOOLEAN],
'cve' => ['filter' => FILTER_VALIDATE_BOOLEAN],
'stig' => ['filter' => FILTER_VALIDATE_BOOLEAN]
];
$company_step = [
'company' => $params,
'comp-add' => $params,
'last-modified' => $params,
'creator' => $params,
'system-class' => $params,
'classified-by' => $params,
'scg' => $params,
'derived-on' => $params,
'declassify-on' => $params
];
$options_step = [
'flatten' => ['filter' => FILTER_VALIDATE_BOOLEAN],
'wrap-text' => ['filter' => FILTER_VALIDATE_BOOLEAN],
'notifications' => ['filter' => FILTER_VALIDATE_BOOLEAN],
'port-limit' => [
'filter' => FILTER_VALIDATE_INT,
'flag' => FILTER_REQUIRE_ARRAY,
'options' => ['max_range' => 10000]
],
'max-results' => [
'filter' => FILTER_VALIDATE_INT,
'flag' => FILTER_REQUIRE_ARRAY,
'options' => ['min_range' => 1, 'max_range' => 20]
],
'output-format' => [
'filter' => FILTER_VALIDATE_REGEXP,
'flag' => FILTER_NULL_ON_FAILURE,
'options' => ['regexp' => "/xlsx|xls|html|csv|pdf|ods/"]
]
];
$step = filter_input(INPUT_POST, 'step', FILTER_VALIDATE_INT);
if ($step == 0) {
$fields = filter_input_array(INPUT_POST, $db_step);
save_Database($fields);
}
elseif ($step == 1) {
$fields = filter_input_array(INPUT_POST, $company_step);
save_Company($fields);
}
elseif ($step == 2) {
$fields = filter_input_array(INPUT_POST, $options_step);
save_Options($fields);
}
/**
* Function to save database details and load data
*
* @param array $params
*/
function save_Database($params)
{
$config = file_get_contents("config.inc", FILE_USE_INCLUDE_PATH);
$php = null;
$mysql = null;
if (strtolower(substr(PHP_OS, 0, 3)) == 'lin') {
if (file_exists('/bin/php')) {
$php = realpath("/bin/php");
}
else {
die(json_encode(['error' => 'Cannot find the PHP executable']));
}
if (file_exists('/bin/mysql')) {
$mysql = realpath('/bin/mysql');
}
else {
die(json_encode(['error' => 'Cannot find the MySQL executable']));
}
}
else {
if (file_exists("c:/xampp/php/php.exe")) {
$php = realpath("c:/xampp/php/php.exe");
}
else {
die(json_encode(['error' => 'Cannot find the PHP executable']));
}
if (file_exists("c:/xampp/mysql/bin/mysql.exe")) {
$mysql = realpath("c:/xampp/mysql/bin/mysql.exe");
}
else {
die(json_encode(['error' => 'Cannot find the MySQL executable']));
}
}
my_str_replace("{DOC_ROOT}", realpath($params['doc-root']), $config);
my_str_replace("{PWD_FILE}", $params['pwd-file'], $config);
my_str_replace("'{E_ERROR}'", "E_{$params['log-level']}", $config);
my_str_replace("{PHP_BIN}", $php, $config);
my_str_replace("{PHP_CONF}", realpath(php_ini_loaded_file()), $config);
my_str_replace("{DB_SERVER}", $params['db-server'], $config);
my_str_replace("{DB_BIN}", $mysql, $config);
my_str_replace("@new", "@step1", $config);
if (!file_exists($params['tmp-path'])) {
if (!mkdir($params['tmp-path'])) {
die(json_encode(['error' => 'Temporary path is not available. Please create and give Apache user write permissions']));
}
}
elseif (!is_dir($params['tmp-path']) || !is_writable($params['tmp-path'])) {
die(json_encode(['error' => 'TMP path is not a writable directory to Apache']));
}
my_str_replace("{TMP_PATH}", $params['tmp-path'], $config);
if (!file_exists($params['log-path'])) {
if (!mkdir($params['log-path'])) {
die(json_encode(['error' => 'Log path is not available. Please create and give Apache user write permissions']));
}
}
elseif (!is_dir($params['log-path']) || !is_writable($params['log-path'])) {
die(json_encode(['error' => 'Log path is not a writable directory by Apache']));
}
my_str_replace("{LOG_PATH}", $params['log-path'], $config);
file_put_contents("{$params['doc-root']}/config.inc", $config);
include_once 'config.inc';
include_once 'database.inc';
/* ---------------------------------
* CREATE DB PASSWORD FILE
* --------------------------------- */
$enc_pwd = my_encrypt($params['web-pwd']);
file_put_contents(DOC_ROOT . "/" . PWD_FILE, $enc_pwd);
if (isset($params['conf-root-pwd']) && $params['conf-root-pwd'] == $params['root-pwd']) {
$db = new mysqli(DB_SERVER, $params['root-uname'], '', 'mysql');
if (!$db->real_query("UPDATE user SET Password=PASSWORD('{$db->real_escape_string($params['root-pwd'])}') WHERE User='root'")) {
error_log($db->error);
die(json_encode(['error' => "Could not set the root users password, manually set it and try this again"]));
}
$db->real_query("FLUSH PRIVILEGES");
unset($db);
}
$successful = true;
$zip = new ZipArchive();
$db = new mysqli(DB_SERVER, $params['root-uname'], $params['root-pwd'], 'mysql');
if ($db->connect_errno && $db->connect_errno == 1045) {
die(json_encode(['error' => 'There was a problem with the user/password combination, please go back and try again']));
}
elseif ($db->connect_errno) {
die(json_encode(['error' => "There was an error connecting to the database on " . DB_SERVER . " with user {$params['root-uname']} and {$params['root-pwd']}"]));
}
$help = new db_helper($db);
$svr_ver = (int) $db->server_version;
$maj = (int) ($svr_ver / 10000);
$svr_ver -= ($maj * 10000);
$min = (int) ($svr_ver / 100);
$svr_ver -= ($min * 100);
$update = $svr_ver;
if (version_compare("{$maj}.{$min}.{$update}", "5.5", "<=")) {
die(json_encode(['error' => "The current version of MySQL needs to be at least 5.5"]));
}
// set the character set and default database
$db->set_charset("utf8");
/* --------------------------------
* USER MANAGEMENT
* -------------------------------- */
$help->delete("mysql.user", null, [
[
'field' => 'User',
'op' => '=',
'value' => ''
]
]);
$help->execute();
$errors = [];
/* --------------------------------
* SCHEMA MANAGEMENT
* -------------------------------- */
if (!$db->real_query("CREATE DATABASE IF NOT EXISTS `rmf`")) {
$errors[] = $db->error;
}
if (!$db->real_query("CREATE DATABASE IF NOT EXISTS `sagacity`")) {
$errors[] = $db->error;
}
$db->real_query("DROP DATABASE IF EXISTS cdcol");
$db->real_query("DROP DATABASE IF EXISTS phpmyadmin");
$db->real_query("DROP DATABASE IF EXISTS test");
/* --------------------------------
* SET SCHEMA PERMISSIONS
* -------------------------------- */
$host = '%';
if (in_array(strtolower(DB_SERVER), ["localhost", "127.0.0.1"])) {
$host = 'localhost';
}
$help->select("mysql.user", ["COALESCE(COUNT(1), 0) AS 'count'"], [
[
'field' => 'User',
'op' => '=',
'value' => 'web'
]
]);
if (!$help->execute()['count']) {
if (!$db->real_query("CREATE USER 'web'@'$host' IDENTIFIED BY '{$db->real_escape_string($params['web-pwd'])}'")) {
$errors[] = $db->error;
}
}
else {
if (!$db->real_query("SET PASSWORD FOR 'web'@'$host' = PASSWORD('{$db->real_escape_string($params['web-pwd'])}')")) {
$errors[] = $db->error;
}
}
if (!$db->real_query("GRANT CREATE TEMPORARY TABLES, INSERT, DELETE, UPDATE, SELECT, TRIGGER ON `rmf`.* TO 'web'@'$host'")) {
$errors[] = $db->error;
}
if (!$db->real_query("GRANT CREATE TEMPORARY TABLES, INSERT, DELETE, UPDATE, SELECT, TRIGGER ON `sagacity`.* TO 'web'@'$host'")) {
$errors[] = $db->error;
}
if (count($errors)) {
die(json_encode(['errors' => implode("<br />", $errors)]));
}
$db->real_query("FLUSH PRIVILEGES");
chdir(realpath(DOC_ROOT));
$json = json_decode(file_get_contents("db_schema.json"));
foreach ($json->tables as $table) {
Sagacity_Error::err_handler("Creating {$table->schema}.{$table->name}");
$help->create_table_json($table);
if (isset($table->triggers)) {
// see if the first entry is a drop statement, run it and remove for subsequent statements
if (substr($table->triggers[0], 0, 4) == 'DROP') {
$db->real_query($table->triggers[0]);
unset($table->triggers[0]);
}
// concatenate the trigger into one string
$trig = implode(" ", $table->triggers);
if (!$db->real_query(str_replace("{host}", $host, $trig))) {
die($db->error);
}
}
$help->insert("sagacity.settings", [
'meta_key' => "{$table->schema}.{$table->name}",
'db_data' => json_encode($table)
], true);
if (!$help->execute()) {
$help->debug(E_WARNING, "JSON for {$table->schema}.{$table->name} table was not pushed to database");
}
}
$help->extended_insert("settings", ["meta_key", "meta_value"], [
['cpe-load-date', new DateTime('1970-01-01')],
['cpe-progress', 0],
['cpe-dl-progress', 0],
['cve-load-date', new DateTime('1970-01-01')],
['cve-progress', 0],
['cve-dl-progress', 0],
['stig-load-date', new DateTime('1970-01-01')],
['stig-progress', 0],
['stig-dl-progress', 0],
['nasl-load-date', new DateTime('1970-01-01')],
['nasl-progress', 0],
['nasl-dl-progress', 0]
], true);
$help->execute();
/*
* ***********************************************************
* Load table data
* ***********************************************************
*/
chdir(DOC_ROOT);
$zip->open("Database_Baseline.zip");
$zip->extractTo("Database_Baseline");
chdir("Database_Baseline");
$sql_files = glob("*.sql");
$zip->close();
if (!$params['sample-data']) {
if (($key = array_search("sample_data.sql", $sql_files)) !== false) {
unset($sql_files[$key]);
unlink("sample_data.sql");
}
}
$defaults = <<<EOO
[client]
password="{$params['root-pwd']}"
port=3306
EOO;
file_put_contents(realpath(TMP) . "/defaults.tmp", $defaults);
$routines = glob("*routines.sql");
foreach ($routines as $file) {
if (($key = array_search($file, $sql_files)) !== false) {
unset($sql_files[$key]);
}
}
if (count($sql_files)) {
sort($sql_files);
foreach ($sql_files as $file) {
$output = [];
$cmd = realpath(DB_BIN) . " --defaults-file=\"" . realpath(TMP . "/defaults.tmp") . "\"" .
" --user={$params['root-uname']}" .
" --host=" . DB_SERVER .
" --default-character-set=utf8 < \"$file\"";
exec($cmd, $output);
if (preg_grep("/Access Denied/i", $output)) {
$errors[] = $output;
$successful = false;
}
else {
unlink($file);
}
}
foreach ($routines as $file) {
$str = file_get_contents($file);
my_str_replace("{host}", $host, $str);
file_put_contents($file, $str);
$cmd = realpath(DB_BIN) . " --defaults-file=\"" . realpath(TMP . "/defaults.tmp") . "\"" .
" --user={$params['root-uname']}" .
" --host=" . DB_SERVER .
" --default-character-set=utf8 < \"$file\"";
exec($cmd);
unlink($file);
flush();
}
}
if (count($errors)) {
print json_encode(['errors' => implode("<br />", $errors)]);
return;
}
unlink(realpath(TMP . "/defaults.tmp"));
rmdir(realpath(DOC_ROOT . "/Database_Baseline"));
$cpe = null;
$cve = null;
$stig = null;
$action = null;
if ($params['cpe']) {
$cpe = " --cpe";
}
if ($params['cve']) {
$cve = " --cve";
}
if ($params['stig']) {
$stig = " --stig";
}
$msg = null;
if ($params['action'] == 'do' || $params['action'] == 'po') {
$action = " --{$params['action']}";
$msg = "Files need to be placed in {doc_root}/tmp for parsing to work correctly";
}
print json_encode(['success' => true, 'msg' => $msg]);
if (!is_null($cpe) || !is_null($cve) || !is_null($stig)) {
$script = realpath(PHP_BIN) .
" -c " . realpath(PHP_CONF) .
" -f " . realpath(DOC_ROOT . "/exec/update_db.php") .
" --{$cpe}{$cve}{$stig}{$action}";
$process = new BackgroundProcess($script);
$process->run();
}
}
/**
* Function to save company information
*
* @param array $fields
*/
function save_Company($fields)
{
$config = file_get_contents("config.inc", FILE_USE_INCLUDE_PATH);
$scg_date = new DateTime($fields['derived-on']);
$declass_date = new DateTime($fields['declassify-on']);
if (!is_a($scg_date, "DateTime") || !is_a($declass_date, "DateTime")) {
print json_encode(['error' => 'Error parsing the dates']);
return;
}
my_str_replace("{COMPANY}", $fields['company'], $config);
my_str_replace("{COMP_ADD}", $fields['comp-add'], $config);
my_str_replace("{LAST_MODIFIED_BY}", $fields['last-modified'], $config);
my_str_replace("{CREATOR}", $fields['creator'], $config);
my_str_replace("{SYSTEM_CLASS}", $fields['system-class'], $config);
my_str_replace("{CLASSIFIED_BY}", $fields['classified-by'], $config);
my_str_replace("{SCG}", $fields['scg'], $config);
my_str_replace("{DERIVED_ON}", $scg_date->format("Y-m-d"), $config);
my_str_replace("{DECLASSIFY_ON}", $declass_date->format("Y-m-d"), $config);
my_str_replace("@step1", "@step2", $config);
file_put_contents(dirname(dirname(__FILE__)) . "/config.inc", $config);
print json_encode(['success' => true]);
}
/**
* Function to save Sagacity options
*
* @param array $fields
*/
function save_Options($fields)
{
$config = file_get_contents("config.inc", FILE_USE_INCLUDE_PATH);
my_str_replace("'{FLATTEN}'", ($fields['flatten'] ? 'true' : 'false'), $config);
my_str_replace("'{WRAP_TEXT}'", ($fields['wrap-text'] ? 'true' : 'false'), $config);
my_str_replace("'{NOTIFICATIONS}'", ($fields['notifications'] ? 'true' : 'false'), $config);
my_str_replace("'{PORT_LIMIT}'", $fields['port-limit'], $config);
my_str_replace("'{MAX_RESULTS}'", $fields['max-results'], $config);
my_str_replace("{ECHECKLIST_FORMAT}", $fields['output-format'], $config);
my_str_replace("@step2", "", $config);
file_put_contents(dirname(dirname(__FILE__)) . "/config.inc", $config);
print json_encode(['success' => true]);
}

125
exec/nessus-plugin-import.php Normal file
View File

@ -0,0 +1,125 @@
<?php
/**
* File: nessus-plugin-import.php
* Author: Ryan Prather
* Purpose: Script to import all Nessus plugins from *.nasl files
* Created: Jan 5, 2015
*
* Portions Copyright 2016: Cyber Perspectives, All rights reserved
* Released under the Apache v2.0 License
*
* Portions Copyright (c) 2012-2015, Salient Federal Solutions
* Portions Copyright (c) 2008-2011, Science Applications International Corporation (SAIC)
* Released under Modified BSD License
*
* See license.txt for details
*
* Change Log:
* - Jan 5, 2015 - File created
* - Sep 1, 2016 - Copyright updated, converted to constants, and added file header
* - Jan 30, 2017 - Updated for platform independence and formatting
* - Jan 31, 2017 - Completed testing, ready for prime time
* - Feb 15, 2017 - Store existing plugin IDs in memory for evaluation to check if we should actually run the script,
* Fixed error with PHP_BIN not being defined for some weird reason
*/
include_once 'config.inc';
include_once "database.inc";
include_once "helper.inc";
$cmd = getopt("h::", array("help::"));
if (isset($cmd['h']) || isset($cmd['help'])) {
die(usage());
}
$db = new db();
if (!file_exists(TMP . "/nessus_plugins")) {
mkdir(TMP . "/nessus_plugins");
}
$nasl_ids = array();
$db->help->select("sagacity.nessus_plugins", array('plugin_id', 'file_date'));
if ($rows = $db->help->execute()) {
foreach ($rows as $row) {
$nasl_ids[$row['plugin_id']] = DateTime::createFromFormat("U", $row['file_date']);
}
}
chdir(TMP . '/nessus_plugins');
$files = glob("*.nasl");
$start_time = new DateTime();
print "Found " . count($files) . " NASL files\nStarted at {$start_time->format("Y-m-d H:i:s")}\n";
chdir(DOC_ROOT . '/exec');
$x = 0;
foreach ($files as $file) {
$db->help->select("nessus_plugins", array('plugin_id', 'file_date'), [
[
'field' => 'file_name',
'op' => '=',
'value' => basename($file)
]
]);
$row = $db->help->execute();
if (!isset($row['file_name']) || is_null($row['file_date']) || filemtime(TMP . "/nessus_plugins/$file") > $row['file_date']) {
$comp = number_format(($x / count($files)) * 100, 2) . "%";
print "\r$comp";
$script = realpath(defined('PHP_BIN') ? PHP_BIN : PHP) .
" -c " . realpath(PHP_CONF) .
" -f " . realpath(DOC_ROOT . "/exec/nessus-plugin-to-database.php") . " --" .
" -f=\"" . realpath(TMP . "/nessus_plugins/$file") . "\"";
if (substr(strtolower(PHP_OS), 0, 3) == "win") {
$shell = new COM("WScript.Shell");
$shell->CurrentDirectory = DOC_ROOT . "/exec";
$shell->run($script, 0, false);
}
elseif (substr(strtolower(PHP_OS), 0, 3) == 'lin') {
exec("$script > /dev/null &");
$output = array();
exec("netstat -an | grep TIME_WAIT | wc -l", $output);
if ($output[0] > 1200) {
do {
sleep(1);
exec("netstat -an | grep TIME_WAIT | wc -l", $output);
}
while ($output[0] > 100);
}
}
$x++;
}
}
$db->help->update("settings", ['meta_value' => 100], [
[
'field' => 'meta_key',
'op' => IN,
'value' => ['nasl-dl-progress', 'nasl-progress']
]
]);
$db->help->execute();
$end_time = new DateTime();
$diff = $end_time->diff($start_time);
print "\nFinished at {$end_time->format("Y-m-d H:i:s")}\nTotal Time: {$diff->format("%H:%I:%S")}\n";
function usage() {
print <<<EOF
Purpose: The purpose of this script is to update the CVE, CPE, and CCE databases. Script will sleep for 3 seconds between actions to allow you review the results.
Usage: php nessus-plugin-import.php [-h|--help]
-h|--help This screen
EOF;
}

192
exec/nessus-plugin-to-database.php Normal file
View File

@ -0,0 +1,192 @@
<?php
/**
* File: nessus-plugin-to-database.php
* Author: Ryan Prather
* Purpose: Script to read .NASL files and import them to the database
* Created: Jan 15, 2017
*
* Copyright 2017: Cyber Perspectives, All rights reserved
* Released under the Apache v2.0 License
*
* See license.txt for details
*
* Change Log:
* - Jan 15, 2017 - File created
* - Jan 31, 2017 - Competed testing, ready for prime time
* - Apr 5, 2017 - Delete file if error in parsing, check for TMP/nessus_plugins and LOG_PATH/nessus_plugins.log
*/
error_reporting(E_ALL);
include_once 'config.inc';
include_once "database.inc";
include_once "helper.inc";
$cmd = getopt("f:h::", array("help::", "debug::"));
if (isset($cmd['h']) || isset($cmd['help']) || !isset($cmd['f'])) {
die(usage());
}
elseif (!file_exists($cmd['f'])) {
die("Could not find file specified {$cmd['f']}\n");
}
check_path(TMP . "/nessus_plugins", true);
check_path(LOG_PATH . "/nessus_plugins.log");
$db = new db();
file_put_contents("check.log", "checking plugin file {$cmd['f']}");
$nasl = new nasl($cmd['f']);
if (!isset($nasl->{'id'})) {
unlink($cmd['f']);
die;
}
if (isset($cmd['debug'])) {
print_r($nasl);
}
$plugin_id = 0;
$file_date = null;
$db->help->select("sagacity.nessus_plugins", array('plugin_id', 'file_date'), [
[
'field' => 'plugin_id',
'op' => '=',
'value' => $nasl->id
]
]);
if ($row = $db->help->execute()) {
$plugin_id = $row['plugin_id'];
$file_date = DateTime::createFromFormat("U", $row['file_date']);
}
if (($plugin_id && !is_a($file_date, "DateTime")) ||
(is_a($file_date, "DateTime") && isset($nasl->last_modification) && is_a($nasl->last_modification, "DateTime") &&
$file_date->format("U") < $nasl->last_modification->format("U"))) {
file_put_contents(LOG_PATH . "/nessus_plugins.log", "Updating {$nasl->id}\n", FILE_APPEND);
$db->help->update("sagacity.nessus_plugins", [
'file_name' => basename($cmd['f']),
'file_date' => (is_a($file_date, "DateTime") ? $file_date->format("U") : filemtime($cmd['f']))], [
[
'field' => 'plugin_id',
'op' => '=',
'value' => $nasl->id
]
]);
if (!isset($cmd['debug'])) {
if (!$db->help->execute()) {
throw(new Exception("Failed to update the plugin {$nasl->id}", E_WARNING));
}
}
else {
print "$db->help->sql\n";
}
}
elseif (!$plugin_id) {
file_put_contents(LOG_PATH . "/nessus_plugins.log", "Inserting {$nasl->id}\n", FILE_APPEND);
$params = [
'plugin_id' => $nasl->id,
'oid' => isset($nasl->oid) ? $nasl->oid : null,
'name' => isset($nasl->name) ? $nasl->name : null,
'copyright' => isset($nasl->copyright) ? $nasl->copyright : null,
'version' => isset($nasl->rev) ? $nasl->rev : null,
'file_name' => basename($cmd['f']),
'file_date' => isset($nasl->last_modification) && is_a($nasl->last_modification, "DateTime") ?
$nasl->last_modification->format("U") : null
];
$db->help->insert("sagacity.nessus_plugins", $params, true);
if (!isset($cmd['debug'])) {
if (!$db->help->execute()) {
throw(new Exception("Failed to insert a new plugin {$nasl->id}", E_WARNING));
}
}
else {
print "$db->help->sql\n";
}
}
else {
file_put_contents(LOG_PATH . "/nessus_plugins.log", "No changes to plugin {$nasl->id}\n", FILE_APPEND);
}
$params = array();
if (isset($nasl->ref)) {
foreach ($nasl->ref as $key => $refs) {
if (is_array($refs)) {
foreach ($refs as $ref) {
$params[] = [
$nasl->id,
$key,
$ref
];
}
}
else {
$params[] = [
$nasl->id,
$key,
$refs
];
}
}
}
unset($nasl->ref);
unset($nasl->oid);
unset($nasl->name);
unset($nasl->copyright);
unset($nasl->rev);
unset($nasl->last_modification);
foreach ((array) $nasl as $field => $val) {
if (($field == 'id') || (is_array($val) && count($val) > 1)) {
continue;
}
elseif (is_array($val) && count($val) == 1 && isset($val[0])) {
$val = $val[0];
}
$params[] = [
$nasl->id,
$field,
$val
];
}
if (count($params)) {
$db->help->extended_insert("sagacity.nessus_meta", [
'plugin_id', 'type', 'val'
], $params, true);
}
if (!isset($cmd['debug'])) {
$db->help->execute();
}
else {
print $db->help->sql . PHP_EOL;
}
if (!isset($cmd['debug'])) {
unlink($cmd['f']);
}
function usage() {
print <<<EOL
Purpose: This script is for reading NASL files and adding them to the database
Usage: php nessus-plugin-to-database.php -f={NASL file to parse} [--debug]
NOTE: This will create a file for any CVE's not found in the database. An update of CVE's should be done first.
-f={NASL file} The .nasl file to parse
--debug This will output what was parsed by the script and NOT add anything to the database
EOL;
}

301
exec/nvd_xml_parser.php Normal file
View File

@ -0,0 +1,301 @@
<?php
/* XML parser for the NIST's National Vulnerability Database (NVD)
* Open Source license, NCSA/U. of Illinois
* Version 1.01, September 2, 2005
* Pascal Meunier
* Purdue University CERIAS
* Contact: pmeunier
* Uses the basic XML parser included with PHP
* that doesn't perform schema validation
* see http://us2.php.net/xml
*
* The NVD's XML schema doesn't have conflicting name tags
* at various nesting levels, so its parsing is relatively
* simple. Therefore, a flat namespace (independent of nesting level)
* works fine. This simple approach doesn't work, for example, with Microsoft's
* XML export of vulnerabilities.
* Note that this parser does not have cases for all tags in the NVD schema.
* Simply add "case" statements for the additional tags that matter to you.
*
* The data from parsing needs further validation or escaping if it is
* to be inserted into another database.
* -Replace the first line with the path to your installation of PHP
* for command line execution. Delete the path to PHP on the first line for
* execution with an Apache web server.
* -Choose a source for the XML file (see below)
*ChangeLog:
* 1.0 first version August 29, 2005
* 1.01 September 2, 2005
-Changed the name nvdcve-1999-2002.xml to nvdcve-2002.xml to match nvd changes
-Removed comment "from Apache" regarding https locations, as they work from the command line too, depending on how PHP was installed
-Now uses command line argument if present as XML source
-Now captures the content of <REF> tags
*/
error_reporting (E_ALL);
$debug = true; // will print messages about every field found if true
$count_entries = 0; // count number of new CVE entries found today
// Choose a source for parsing
// Check if a source was passed as argument
if (isset($argv[1])) {
parse_this($argv[1]);
} else {
parse_this("nvdcve-2004.xml");
// other possibilities:
//parse_this("http://nvd.nist.gov/download/nvdcve-2002.xml");
//parse_this("http://nvd.nist.gov/download/nvdcve-2003.xml");
//parse_this("http://nvd.nist.gov/download/nvdcve-2004.xml");
//parse_this("http://nvd.nist.gov/download/nvdcve-2005.xml");
//parse_this("http://nvd.nist.gov/download/nvdcve-2006.xml");
//
// The following do not work with vanilla PHP installs
// without the wrapper "https" (e.g., MacOS X default install)
//parse_this("https://nvd.nist.gov/download/nvdcve-2002.xml");
//parse_this("https://nvd.nist.gov/download/nvdcve-2003.xml");
//parse_this("https://nvd.nist.gov/download/nvdcve-2004.xml");
//parse_this("https://nvd.nist.gov/download/nvdcve-2005.xml");
//parse_this("https://nvd.nist.gov/download/nvdcve-2006.xml");
}
echo "Found $count_entries entries in this NVD location".PHP_EOL;
die; // not necessary, added for clarity
function startElement($parser, $name, $attrs) {
global $all_data, $CVE, $debug, $insert, $vendor, $product, $count_entries, $desc_type;
if ($debug) {
echo "processing tag named '$name'".PHP_EOL;
}
//
// the switch statement should enumerate all of the possible name tags.
// State is kept using global variables, so that nested tags can
// access their context.
//
switch ($name) {
case 'ENTRY':
$count_entries++;
// get name and publication date
$CVE = validate_CVE($attrs['NAME']);
$published = validate_date($attrs['PUBLISHED']);
// other attributes, such as severity, are available
// see http://nvd.nist.gov/download/nvdcve-xmldoc.cfm
if ($debug) {
echo "found CVE entry $CVE with publication date {$attrs['PUBLISHED']}".PHP_EOL;
}
break;
case 'DESCRIPT':
// if there ever was more than one type of description
// remember which source with a global
// NVD or CVE
$desc_type = $attrs['SOURCE'];
// reset data accumulator
$all_data = '';
// need to wait for end tag to get contents
break;
case 'PROD':
$vendor = $attrs['VENDOR'];
$product = $attrs['NAME'];
if ($CVE == '') {
echo "error, no CVE number";
die;
}
if ($debug) {
echo "found vendor $vendor";
echo " found product $product".PHP_EOL;
}
if ($vendor == "") {
// this happens
echo "NVD integrity alert: no vendor for product $product, CVE is $CVE".PHP_EOL;
}
if ($product == "") {
echo "NVD integrity alert: no product, CVE is $CVE".PHP_EOL;
}
break;
case 'LOSS_TYPES':
break;
case 'VULNS_TYPES':
break;
case 'REF':
// source, url, etc...
if ($debug) {
echo " reference from {$attrs['SOURCE']}";
echo " is available at {$attrs['URL']}";
echo PHP_EOL;
}
$all_data = '';
// need to wait for end tag to get contents
break;
case 'VERS':
if ($debug) {
echo " version {$attrs['NUM']}";
echo " of product $product is vulnerable".PHP_EOL;
}
break;
case '':
break;
}
}
function endElement($parser, $name) {
global $all_data, $CVE, $debug;
switch ($name) {
case 'ENTRY':
$CVE = '';
break;
case 'DESCRIPT':
if ($CVE == '') {
echo "error, no CVE number";
die;
}
if ($debug) {
echo "found description $all_data".PHP_EOL." for CVE entry $CVE".PHP_EOL;
}
// reset data accumulator
$all_data = '';
break;
case 'REF':
if ($debug) {
echo "found reference content '$all_data'".PHP_EOL." for CVE entry $CVE".PHP_EOL;
}
// reset data accumulator
$all_data = '';
break;
case 'AVAIL':
if ($debug) {
echo "loss type is availability".PHP_EOL;
}
break;
case 'CONF':
if ($debug) {
echo "loss type is confidentiality".PHP_EOL;
}
break;
case 'INT':
if ($debug) {
echo "loss type is integrity".PHP_EOL;
}
break;
case 'SEC_PROT':
if ($debug) {
echo "loss type is security protection".PHP_EOL;
}
break;
case 'ACCESS':
if ($debug) {
echo "vulnerability type is access".PHP_EOL;
}
break;
case 'INPUT':
if ($debug) {
echo "vulnerability type is input".PHP_EOL;
}
break;
case 'DESIGN':
if ($debug) {
echo "vulnerability type is design".PHP_EOL;
}
break;
case 'EXCEPTION':
if ($debug) {
echo "vulnerability type is exception".PHP_EOL;
}
break;
case 'ENV':
if ($debug) {
echo "vulnerability type is environment".PHP_EOL;
}
break;
case 'CONFIG':
if ($debug) {
echo "vulnerability type is configuration".PHP_EOL;
}
break;
case 'RACE':
if ($debug) {
echo "vulnerability type is race".PHP_EOL;
}
break;
case 'OTHER':
if ($debug) {
echo "vulnerability type is other".PHP_EOL;
}
break;
case 'REMOTE':
if ($debug) {
echo "vulnerability range is remote".PHP_EOL;
}
break;
case 'LOCAL':
if ($debug) {
echo "vulnerability range is local".PHP_EOL;
}
break;
case 'USER_INIT':
if ($debug) {
echo "vulnerability range is through a user".PHP_EOL;
}
break;
case '':
break;
}
}
function characterData($parser, $data) {
global $all_data;
// concatenate due to bug with & according to a user entry in PHP docs
// entry by sam at cwa dot co dot nz, Sept 2000
// This issue might have been fixed later. The fix can't hurt though.
$all_data .= $data;
}
function parse_this($url) {
// code for this function is from http://us2.php.net/xml
$xml_parser = xml_parser_create();
// use case-folding so we are sure to find the tag in $map_array
xml_parser_set_option($xml_parser, XML_OPTION_CASE_FOLDING, true);
xml_set_element_handler($xml_parser, "startElement", "endElement");
xml_set_character_data_handler($xml_parser, "characterData");
if (!($fp = fopen($url, "r"))) {
die("could not open XML input");
}
while ($data = fread($fp, 4096)) {
if (!xml_parse($xml_parser, $data, feof($fp))) {
die(sprintf("XML error: %s at line %d",
xml_error_string(xml_get_error_code($xml_parser)),
xml_get_current_line_number($xml_parser)));
}
}
xml_parser_free($xml_parser);
}
function validate_CVE($input) {
// expecting something like CAN-2004-0002
// NVD documentation suggests this reg exp,
// (http://nvd.nist.gov/download/nvdcve-xmldoc.cfm)
// (CAN|CVE)-/d/d/d/d-/d/d/d/d
// which doesn't work. See below
if (preg_match('/(CAN|CVE)\-\d\d\d\d\-\d+/', $input, $matches)) {
return $matches[0];
} else {
// if there was an error (false) or if 0 matches
die ("Invalid CVE number encountered: $input");
}
}
function validate_Date($input) {
// expecting something like 2004-03-03
if (preg_match('/\d{4}\-\d{2}\-\d{2}/', $input, $matches)) {
return $matches[0];
} else {
// if there was an error (false) or if 0 matches
die ("Invalid date encountered: $input");
}
}
?>

72
exec/parse_cce.php Normal file
View File

@ -0,0 +1,72 @@
<?php
/**
* File: parse_cce.php
* Author: Ryan Prather
* Purpose: Read the CCE file from NIST and import all CCE's to database
* Created: Mar 17, 2015
*
* Portions Copyright 2016: Cyber Perspectives, All rights reserved
* Released under the Apache v2.0 License
*
* Portions Copyright (c) 2012-2015, Salient Federal Solutions
* Portions Copyright (c) 2008-2011, Science Applications International Corporation (SAIC)
* Released under Modified BSD License
*
* See license.txt for details
*
* Change Log:
* - Mar 17, 2015 - File created
* - Sep 1, 2016 - Copyright Updated, added file purpose, and converted to use constants
*/
$cmd = getopt("f:", array("debug::", "help::"));
if (!isset($cmd['f']) || isset($cmd['help'])) {
usage();
exit;
}
include_once 'config.inc';
include_once 'database.inc';
include_once 'helper.inc';
include_once 'xml_parser.inc';
chdir(DOC_ROOT . "/tmp");
set_time_limit(0);
class cce_parser extends basic_xml_parser {
var $sql;
var $count;
var $cce;
public function __construct($xml_fname) {
parent::__construct($this, $xml_fname);
$this->sql = "INSERT IGNORE INTO `targets`.`software` (`cpe`,`cpe23`,`sw_string`) VALUES ";
$this->count = 0;
$this->db = new mysqli(DB_SERVER, "web", db::decrypt_pwd());
}
public function cce_cce_list_cces_cce($attrs) {
}
}
$parser = new cce_parser($cmd['f']);
$parser->debug = (isset($cmd['debug']) ? true : false);
$parser->parse();
function usage() {
print <<<EOO
Purpose: To read a CCE list, parse it, and populate/update the database
Usage: php parse_cce.php -f={cce list file} [--debug] [--help]
-f={cce file} The CCE list file retrieved from http://static.nvd.nist.gov/feeds/xml/cce/cce-COMBINED-5.20130214.xml
--debug Debugging output
--help This screen
EOO;
}

101
exec/parse_cci.php Normal file
View File

@ -0,0 +1,101 @@
<?php
/**
* File: parse_cci.php
* Author: Ryan Prather
* Purpose: Script to parse CCI library
* Created: Jul 28, 2014
*
* Portions Copyright (c) 2012-2015, Salient Federal Solutions
* Portions Copyright (c) 2008-2011, Science Applications International Corporation (SAIC)
* Released under Modified BSD License
*
* See license.txt for details
*
* Change Log:
* - Jul 28, 2014 - File created
*/
$cmd = getopt("f:", array('debug::', "help::"));
if (!isset($cmd['f']) || isset($cmd['help'])) {
usage();
exit;
}
include_once 'config.inc';
include_once 'database.inc';
include_once 'helper.inc';
chdir(DOC_ROOT);
set_time_limit(0);
$base_name = basename($cmd['f']);
$db = new db();
$err = new Sagacity_Error($cmd['f']);
$match = array();
if (!file_exists($cmd['f'])) {
$err->script_log("File not found", E_ERROR);
}
$xml = new DOMDocument();
if (!$xml->load($cmd['f'])) {
$err->script_log("There was an issue loading XML document", E_ERROR);
}
$cci_list = getValue($xml, "/root/cci", null, true);
print "Reading $cci_list->length CCIs" . PHP_EOL;
$all_ccis = array();
foreach ($cci_list as $cci_node) {
$cci = new cci();
$cci->cci_id = preg_replace("/CCI\-[0]+/", "", $cci_node->getAttribute("id"));
if ($cci->cci_id > 3391) {
break;
}
$control = getValue($xml, "control", $cci_node);
$cci->control_id = preg_replace("/ \(\d+\)/", "", $control);
$cci->definition = getValue($xml, "definition", $cci_node);
$cci->guidance = getValue($xml, "guidance", $cci_node);
$cci->procedure = getValue($xml, "procedure", $cci_node);
if (preg_match("/ \(([\d]+)\)/", getValue($xml, "control", $cci_node), $match)) {
$cci->enh_id = $match[1];
}
else {
$cci->enh_id = null;
}
$all_ccis[] = $cci;
print "$cci->cci_id" . PHP_EOL;
if (count($all_ccis) == 100) {
print "Saving 100 CCI's" . PHP_EOL;
$db->save_CCI($all_ccis);
$all_ccis = array();
}
}
//die;
print "Saving..." . PHP_EOL;
$db->save_CCI($all_ccis);
print "Done saving " . count($all_ccis) . " CCIs" . PHP_EOL;
function usage() {
print <<<EOO
Purpose: To parse the NIST CCI list
Usage: php parse_cci.php -f={CCI list file} [--debug] [--help]
-f={CCI file} The CCI file to parse
--debug Debugging output
--help This screen
EOO;
}

454
exec/parse_cpe.php Normal file
View File

@ -0,0 +1,454 @@
<?php
/**
* File: parse_cpe.php
* Author: Ryan Prather
* Purpose: Script to parse CPE library
* Created: Jul 28, 2014
*
* Portions Copyright 2016: Cyber Perspectives, All rights reserved
* Released under the Apache v2.0 License
*
* Portions Copyright (c) 2012-2015, Salient Federal Solutions
* Portions Copyright (c) 2008-2011, Science Applications International Corporation (SAIC)
* Released under Modified BSD License
*
* See license.txt for details
*
* Change Log:
* - Jul 28, 2014 - File created
* - Sep 1, 2016 - Copyright Updated and converted to constants
* - Nov 11, 2016 - Comments added by Ryan Prather and Matt Shuter
* - Nov 21, 2016 - Added print out to display the number of new CPEs imported
* - Jan 30, 2017 - Added short string for software and conversions to translate some of the more popular software (MS Windows, RedHat ELS, and OpenSuSE)
* - Feb 15, 2017 - Formatting and migrated to use the new db_helper functions
* - Apr 5, 2017 - Removed MS manufacture name from Microsoft owned software for shortened software string
*/
$cmd = getopt("f:d:", ['debug::', 'help::']);
if (!isset($cmd['f']) || isset($cmd['help'])) {
die(usage());
}
elseif (!file_exists($cmd['f'])) {
die("Could not find {$cmd['f']}\n");
}
include_once 'config.inc';
include_once 'database.inc';
include_once 'helper.inc';
include_once 'xml_parser.inc';
chdir(TMP);
set_time_limit(0);
class cpe_parser extends basic_xml_parser {
/**
* The CPE that is currently being parsed
*
* @var string
*/
var $cpe;
/**
* The CPE v2.3 formatted string of the CPE that is currently being parsed
*
* @var string
*/
var $cpe23;
/**
* An array to represent the CPE
*
* @var array
*/
var $cpe_arr;
/**
* The software title string associated with the CPE that is currently being parsed
*
* @var string
*/
var $sw_string;
/**
* Variable to store a short software string
*
* @var string
*/
var $short_string;
/**
* The counter that tracks how many cpe_items we've processed
*
* @var int
*/
var $count;
/**
* Counter for the number of new CPEs
*
* @var number
*/
var $new;
/**
* Counter for the number of deleted CPEs
*
* @var number
*/
var $deleted;
/**
* Variable to store existing CPEs
*
* @var array
*/
var $existing_cpes;
/**
* Array to store list of CPEs to delete from database
*
* @var array
*/
var $cpes_to_remove;
/**
* Array to store new CPEs to add to DB
*
* @var array
*/
var $new_cpes;
/**
* Variable to store the total number of CPEs to parse
*
* @var int
*/
var $total_cpes;
/**
* Constructor
*
* @param string $xml_fname
* @param string $date
*/
public function __construct($xml_fname, $date) {
$cpe = file($xml_fname);
$this->total_cpes = count(preg_grep("/<cpe\-item/", $cpe));
unset($cpe);
parent::__construct($this, $xml_fname);
$this->count = 0;
$conn = new mysqli(DB_SERVER, "web", db::decrypt_pwd(), 'sagacity');
$this->db = new db_helper($conn);
$this->db->update("settings", ['meta_value' => new DateTime($date)], [
[
'field' => 'meta_key',
'op' => '=',
'value' => 'cpe-load-date'
]
]);
$this->db->execute();
$this->db->select("software", ['cpe']);
$cpes = $this->db->execute();
if (!is_null($cpes) && is_array($cpes) && count($cpes)) {
foreach ($cpes as $cpe) {
if (isset($cpe['cpe'])) {
$this->existing_cpes["{$cpe['cpe']}"] = 1;
}
}
}
if (!isset($this->existing_cpes["cpe:/o:generic:generic:-"])) {
$this->new_cpes[] = [
"cpe:/o:generic:generic:-",
"cpe:2.3:o:generic:generic:*:*:*:*:*:*:*",
"Generic Generic OS",
"Generic"
];
}
if (!isset($this->existing_cpes["cpe:/a:generic:generic:-"])) {
$this->new_cpes[] = [
"cpe:/a:generic:generic:-",
"cpe:2.3:a:generic:generic:*:*:*:*:*:*:*",
"Generic Generic",
"Generic"
];
}
}
/**
* Start function for &lt;cpe-list&gt;/&lt;cpe-item&gt; element
*
* @param array $attrs
*/
public function cpe_list_cpe_item($attrs) {
if (isset($attrs['deprecated']) && $attrs['deprecated'] == 'true') {
$this->skip = true;
if (isset($attrs['name'])) {
$this->cpe = $attrs['name'];
}
return;
}
$match = [];
if (isset($attrs['name'])) {
$this->cpe = $attrs['name'];
$this->cpe_arr = explode(':', $attrs['name']);
}
switch ($this->cpe_arr[2]) {
case 'microsoft':
$this->short_string = '';
break;
case 'redhat':
$this->short_string = 'RH';
break;
case 'opensuse_project':
$this->short_string = 'openSuSE';
break;
default:
$this->short_string = ucfirst($this->cpe_arr[2]);
}
switch ($this->cpe_arr[3]) {
case 'windows':
case 'windows_nt':
$this->short_string .= 'Win';
break;
case (preg_match("/windows_([\d\.]+)(_server)?$/", $this->cpe_arr[3], $match) ? true : false):
if (isset($match[2]) && $match[2]) {
$this->short_string .= "Win Server {$match[1]}";
}
else {
$this->short_string .= "Win {$match[1]}";
}
break;
case (preg_match("/windows_server_([\d]+)$/", $this->cpe_arr[3], $match) ? true : false):
$this->short_string .= "Win Server {$match[1]}";
break;
case (preg_match("/windows_(vista|xp)$/", $this->cpe_arr[3], $match) ? true : false):
$this->short_string .= "Win {$match[1]}";
break;
case 'pocket_ie':
case 'pocket_internet_explorer':
case 'internet_explorer':
case 'ie':
$this->short_string .= "IE";
break;
case 'enterprise_linux_server':
$this->short_string .= " EL";
break;
case 'enterprise_linux_workstation':
$this->short_string .= " EL";
break;
default:
$this->short_string .= " " . ucfirst(str_replace(array('-', '_'), ' ', $this->cpe_arr[3]));
}
if (isset($this->cpe_arr[4]) && ($this->cpe_arr[4] != '-' || $this->cpe_arr[4] != '*')) {
switch ($this->cpe_arr[4]) {
case (preg_match("/([R\d\.z]+)/", $this->cpe_arr[4], $match) ? true : false):
$this->short_string .= " {$match[1]}";
break;
default:
$this->short_string .= " " . $this->cpe_arr[4];
}
}
if (isset($this->cpe_arr[6]) && $this->cpe_arr[6]) {
$this->short_string .= " " . str_replace('~', '', $this->cpe_arr[6]);
}
if (isset($this->cpe_arr[5]) && !empty($this->cpe_arr[5]) && $this->cpe_arr[5] != '-') {
//die(print_r($this->cpe_arr, true));
switch ($this->cpe_arr[5]) {
case (preg_match("/sp([\d]+)/", $this->cpe_arr[5], $match) ? true : false):
$this->short_string .= " SP{$match[1]}";
break;
default:
$this->short_string .= " " . strtoupper($this->cpe_arr[5]);
}
}
}
/**
* Start function for &lt;cpe-list&gt;/&lt;cpe-item&gt;/&lt;title&gt; element
*
* @param array $attrs
* Name/value pair of attributes
*/
public function cpe_list_cpe_item_title($attrs) {
if (isset($attrs['xml:lang']) && $attrs['xml:lang'] != 'en-US') {
$this->skip = true;
}
}
/**
* Character data function for &lt;cpe-list&gt;/&lt;cpe-item&gt;/&lt;title&gt; element
*
* @param string $data
* The value within the tags
*/
public function cpe_list_cpe_item_title_data($data) {
$this->sw_string = trim($data);
}
/**
* Start function for &lt;cpe-list&gt;/&lt;cpe-item&gt;/&lt;cpe-23:cpe23-item&gt; element
*
* @param array $attrs
* Name/value pairs of attributes
*/
public function cpe_list_cpe_item_cpe_23_cpe23_item($attrs) {
if (isset($attrs['name'])) {
$this->cpe23 = $attrs['name'];
}
}
/**
* End function for &lt;cpe-list&gt;/&lt;cpe-item&gt; element
*/
public function cpe_list_cpe_item_end() {
// if we are supposed to skip this CPE (because of deprecation or the title is not english) then delete it from the database
if ($this->skip) {
$this->cpes_to_remove[] = $this->cpe;
$this->skip = false;
PHP_SAPI == "cli" ? print "-" : null;
}
// look for current item in the existing list
elseif (!isset($this->existing_cpes["{$this->cpe}"])) {
$this->new_cpes[] = [
$this->cpe,
$this->cpe23,
$this->sw_string,
$this->short_string
];
PHP_SAPI == "cli" ? print "*" : null;
}
else { // current cpe is already in the database, so just print "."
PHP_SAPI == 'cli' ? print "." : null;
}
$this->count++;
// every 100 CPEs, print the count and execute the SQL.
if ($this->count % 100 == 0) {
print "\t$this->count completed" . PHP_EOL;
$this->db->update("settings", ['meta_value' => number_format(($this->count / $this->total_cpes * 100), 2)], [
[
'field' => 'meta_key',
'op' => '=',
'value' => 'cpe-progress'
]
]);
$this->db->execute();
if (is_array($this->new_cpes) && count($this->new_cpes)) {
$this->db->extended_insert('software', ['cpe', 'cpe23', 'sw_string', 'short_sw_string'], $this->new_cpes, true);
$this->new += $this->db->execute();
unset($this->new_cpes);
$this->{'new_cpes'} = [];
}
if (is_array($this->cpes_to_remove) && count($this->cpes_to_remove)) {
$this->db->delete("software", null, [
[
'field' => 'cpe',
'op' => IN,
'value' => $this->cpes_to_remove
]
]);
$this->deleted += $this->db->execute();
unset($this->cpes_to_remove);
$this->{'cpes_to_remove'} = [];
}
}
// reset cpe, cpe23, and sw_string for next cpe item
$this->cpe = '';
$this->cpe23 = '';
$this->sw_string = '';
$this->short_string = '';
}
/**
* End function for &lt;cpe-list&gt; element
*/
public function cpe_list_end() {
// execute what is left in the SQL just incase there are some leftover
if (is_array($this->new_cpes) && count($this->new_cpes)) {
$this->db->extended_insert('software', ['cpe', 'cpe23', 'sw_string', 'short_sw_string'], $this->new_cpes, true);
$this->db->execute();
}
if (is_array($this->cpes_to_remove) && count($this->cpes_to_remove)) {
$this->db->delete("software", null, [
[
'field' => 'cpe',
'op' => IN,
'value' => $this->cpes_to_remove
]
]);
$this->deleted += $this->db->execute();
}
$this->db->update("settings", ['meta_value' => 100], [
[
'field' => 'meta_key',
'op' => IN,
'value' => ['cpe-dl-progress', 'cpe-progress']
]
]);
$this->db->execute();
}
}
$xml = new cpe_parser($cmd['f'], $cmd['d']);
$xml->debug = false;
if (isset($cmd['debug'])) {
$xml->debug = true;
}
elseif (LOG_LEVEL == E_DEBUG) {
$xml->debug = true;
}
//Enter xml code here
$xml->parse();
$unchanged = $xml->count - $xml->new - $xml->deleted;
print <<<EOO
Unchanged CPEs: $unchanged
New CPEs: $xml->new
Deleted CPEs: $xml->deleted
EOO;
function usage() {
print <<<EOO
Purpose: To parse the NIST CPE list
Output: You will see either a . (dot), * (asterisk), or - (hyphen) for each CPE.
. - CPE was already in the DB
* - CPE was added to the DB
- - CPE was removed from the DB (CPE deprecated)
Usage: php parse_cpe.php -f={CPE list file} [--debug] [--help]
-f={CPE file} The CPE file to parse retrieved from http://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml
--debug Debugging output
--help This screen
EOO;
}

226
exec/parse_csv_echecklist.php Normal file
View File

@ -0,0 +1,226 @@
<?php
/**
* File: parse_csv_echecklist.php
* Author: Ryan Prather
* Purpose: Background script to parse CSV eChecklist files
* Created: May 9, 2014
*
* Portions Copyright 2016: Cyber Perspectives, All rights reserved
* Released under the Apache v2.0 License
*
* Portions Copyright (c) 2012-2015, Salient Federal Solutions
* Portions Copyright (c) 2008-2011, Science Applications International Corporation (SAIC)
* Released under Modified BSD License
*
* See license.txt for details
*
* Change Log:
* - May 9, 2014 - File created
* - Sep 1, 2016 - Copyright Updated, added CWD parameter, and
* Updated to use correct functions after class mergers
* - Jan 30, 2017 - Updated to use parse_config.ini file
* - Mar 4, 2017 - Removed Thread class calls
*/
$start = new DateTime();
$cmd = getopt("f:", array('debug::', 'help::'));
if (!isset($cmd['f']) || isset($cmd['help'])) {
die(usage());
}
$conf = parse_ini_file("parse_config.ini");
if (!$conf) {
die("Could not find parse_config.ini configuration file");
}
chdir($conf['doc_root']);
print "Start Time: " . $start->format("H:i:s") . PHP_EOL;
include_once 'config.inc';
include_once "database.inc";
include_once 'helper.inc';
chdir(TMP);
set_time_limit(0);
$dbh = new db();
$base_name = basename($cmd['f']);
$err = new Sagacity_Error($base_name);
if (!file_exists($cmd['f'])) {
$dbh->update_Running_Scan($base_name, array('name' => 'status', 'value' => 'ERROR'));
$err->script_log("File not found", E_ERROR);
}
$dbh->update_Running_Scan($base_name, array('name' => 'pid', 'value' => getmypid()));
$error = array();
$success = array();
$tgts = array();
$finding_count = array();
$sum = '';
$host_list = array();
$src = $dbh->get_Sources('eChecklist');
$handle = fopen($cmd['f'], "r");
// Get the headers (might be one extra line)
$eol = true;
$header_lines = 0;
while ($data = fgetcsv($handle)) {
$header_lines++;
if (preg_match('/STIG[\_ ]ID/i', $data[0])) {
$eol = false;
break;
}
}
if ($eol) {
$err->script_log("Got to end of " . $cmd['f'] . " without finding 'STIG ID'", E_ERROR);
}
// add a new scan for this E-Checklist
if (count($data) >= 9) {
for ($x = echecklist::HOST_COL_START; $data[$x] != 'Notes'; $x++) {
if (!preg_match("/^(\d{1,3}\.){3}(\d{1,3})$/", $data[$x])) {
if ($pos = strpos($data[$x], '.')) {
$data[$x] = substr($data[$x], 0, $pos);
}
}
if ($id = $dbh->check_Target($conf['ste'], $data[$x])) {
$tgt = $dbh->get_Target_Details($conf['ste'], $id)[0];
$tgts[] = $tgt;
//print "Identified: ".$data[$x].PHP_EOL;
}
else {
$tgt = new target($data[$x]);
$tgt->set_STE_ID($conf['ste']);
$tgt->set_Location("New Target");
$tgt->set_Notes("New Target");
//print "Added target: ".$data[$x].PHP_EOL;
if (preg_match('/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/', $data[$x])) {
$int = new interfaces(null, $tgt_id, null, $data[$x], null, null, null, null);
$tgt->interfaces[] = $int;
}
$tgt_id = $dbh->save_Target($tgt);
$tgts[] = $tgt;
}
$host_list[$tgt->get_Name()] = array('target' => $tgt, 'count' => 0);
}
//$host_list = substr($host_list, 0, -1);
$dt = new DateTime();
$existing_scan = $dbh->get_ScanData($conf['ste'], $cmd['f']);
if (!count($existing_scan)) {
$ste = $db->get_STE($conf['ste'])[0];
$scan = new scan(null, $src, $ste, 1, $base_name, $dt->format('Y-m-d'));
// $scan->add_Target_Array_to_Host_List($host_list);
if (!$scan_id = $dbh->save_Scan($scan)) {
$err->script_log("Failure to add scan for " . $cmd['f'], E_ERROR);
}
else {
$scan->set_ID($scan_id);
}
}
else {
$scan = $existing_scan[0];
}
}
else {
fclose($handle);
$err->script_log("There were less than 9 columns in " . $cmd['f'], E_ERROR);
}
$line = 0;
$line_count = 0;
while ($data = fgetcsv($handle)) {
$line_count++;
}
// rewind after counting the lines
rewind($handle);
// skip the header lines
for ($x = 0; $x < $header_lines; $x++) {
$data = fgetcsv($handle);
}
// loop through all the findings and add a new finding item
while ($data = fgetcsv($handle)) {
if (empty($data[0])) {
continue;
}
if (!$dbh->add_Finding($scan, $tgts, $data)) {
$sum .= "Failure adding finding: $data[0]<br />";
$err->script_log("Error adding STIG ID: " . $data[0], E_WARNING);
}
else {
$err->script_log("Added STIG ID: " . $data[0]);
}
$dbh->update_Running_Scan($base_name, array('name' => 'perc_comp', 'value' => ($line / $line_count) * 100));
$line++;
}
foreach ($tgts as $key => $tgt) {
$host_list[$tgt->get_Name()]['count'] = $line;
}
$dbh->update_Scan_Host_List($scan, $host_list);
$count = 0;
while (is_resource($handle)) {
if (fclose($handle)) {
$err->script_log("File closed");
break;
}
$count++;
if ($count == 3) {
$err->script_log("File didn't close, forcing");
unset($handle);
break;
}
sleep(1);
}
if (!isset($cmd['debug'])) {
rename($cmd['f'], DOC_ROOT . "/tmp/echecklist/$base_name");
}
$dbh->update_Running_Scan($base_name, array('name' => 'perc_comp', 'value' => 100, 'complete' => 1));
$end = new DateTime();
print "End Time: " . $end->format("H:i:s") . PHP_EOL;
$diff = $end->diff($start);
print "Total Time: " . $diff->format("%H:%I:%S") . PHP_EOL;
function usage() {
print <<<EOO
Purpose: To parse a CSV E-Checklist file
Usage: php parse_csv_echecklist.php -s={ST&E ID} -f={CSV eChecklist} [--debug] [--help]
-s={ST&E ID} The ST&E ID that this result file is going to be imported for
-f={eChecklist file} The eChecklist file to import
--debug Debugging output
--help This screen
EOO;
}

252
exec/parse_cve.php Normal file
View File

@ -0,0 +1,252 @@
<?php
/**
* File: parse_cve.php
* Author: Ryan Prather
* Purpose: To parse CVE repository retrieved from http://cve.mitre.org/data/downloads/index.html
* Created: Jul 9, 2014
*
* Portions Copyright 2016: Cyber Perspectives, All rights reserved
* Released under the Apache v2.0 License
*
* Portions Copyright (c) 2012-2015, Salient Federal Solutions
* Portions Copyright (c) 2008-2011, Science Applications International Corporation (SAIC)
* Released under Modified BSD License
*
* See license.txt for details
*
* Change Log:
* - Jul 9, 2014 - File created
* - Sep 1, 2016 - Copyright Updated and converted to constants
* - Nov 21, 2016 - Added print out to display the number of new CVEs imported
* - Feb 15, 2017 - Retrieve and store all existing CVEs in memory to use to evaluate if one needs to be imported
* This will now not update existing content.
* - Nov 25, 2017 - Fixed bug #342
* - Dec 27, 2017 - Syntax updates, and added update to load date
*/
$cmd = getopt("f:", ['debug::', 'help::']);
if (!isset($cmd['f']) || isset($cmd['help'])) {
die(usage());
}
elseif (!file_exists($cmd['f'])) {
die("Could not find file {$cmd['f']}\n");
}
include_once 'config.inc';
include_once "database.inc";
include_once 'helper.inc';
chdir(TMP);
set_time_limit(0);
$sys = new db();
$err = new Sagacity_Error($cmd['f']);
$existing_cves = [];
$sys->help->select("cve_db", ['cve_id']);
$cves = $sys->help->execute();
if (!is_null($cves) && is_array($cves) && count($cves)) {
foreach ($cves as $cve) {
$existing_cves["{$cve['cve_id']}"] = 1;
}
}
print "Currently " . count($existing_cves) . " in the DB" . PHP_EOL;
$sys->help->update("settings", ['meta_value' => new DateTime()], [
[
'field' => 'meta_key',
'op' => '=',
'value' => 'cve-load-date'
]
]);
$sys->help->execute();
$doc = new DOMDocument();
$doc->load($cmd['f']);
$items = getValue($doc, "/x:cve/x:item", null, true);
$existing = 0;
$new = 0;
$count = 0;
print "Total of {$items->length} CVEs to parse" . PHP_EOL;
if ($items->length) {
$new_cves = [];
$new_cve_refs = [];
$new_cve_web = [];
$cve_fields = [
'cve_id', 'seq', 'status', 'phase', 'phase_date', 'desc'
];
$ref_fields = [
'cve_seq', 'source', 'url', 'val'
];
$web_fields = [
'cve_id', 'xml'
];
foreach ($items as $node) {
$name = getValue($doc, '@name', $node);
if (!isset($existing_cves["$name"])) {
$count++;
$cve_xml = new DOMDocument();
$cve_node = $cve_xml->importNode($node, true);
$cve_xml->appendChild($cve_node);
$pi = $cve_xml->createProcessingInstruction('xml-stylesheet', 'type="text/xsl" href="cve.xsl"');
$cve_xml->insertBefore($pi, $cve_xml->getElementsByTagName("item")->item(0));
$cve_xml->xmlStandalone = true;
$tmp = str_replace(" xmlns=\"http://cve.mitre.org/cve/downloads\"", "", $cve_xml->saveXML());
$cve_xml->loadXML($tmp);
$refs = [];
$type = getValue($doc, '@type', $node);
$seq = getValue($doc, '@seq', $node);
$desc = getValue($doc, 'x:desc', $node);
$status = getValue($doc, 'x:status', $node);
$phase = getValue($doc, 'x:phase', $node);
$phase_dt_str = getValue($doc, 'x:phase/@date', $node);
$phase_dt = new DateTime($phase_dt_str);
$new_cves[] = [
$name, $seq, $status, $phase, $phase_dt, $desc
];
$tmp_refs = getValue($doc, 'x:refs/x:ref', $node, true);
for ($x = 0; $x < $tmp_refs->length; $x++) {
$refs[$x]['val'] = getValue($doc, '.', $tmp_refs->item($x));
$refs[$x]['src'] = getValue($doc, '@source', $tmp_refs->item($x));
$refs[$x]['url'] = getValue($doc, '@url', $tmp_refs->item($x));
}
foreach ($refs as $key => $ref) {
$new_cve_refs[] = [
$name, $ref['src'], $ref['url'], $ref['val']
];
}
$new_cve_web[] = [$name, $cve_xml->saveXML()];
$err->script_log("new cve $name");
$new++;
print "*";
}
else {
/*
$sql = "REPLACE INTO `reference`.`cve_db` (`type`, `seq`, `status`, `phase`, `phase_date`, `desc`) VALUES (" .
"'$type'," .
"'$seq'," .
"'{$sys->help->real_escape_string($status)}'," .
"'{$sys->help->real_escape_string($phase)}'," .
"'{$phase_dt->format('Y-m-d')}'," .
"'{$sys->help->real_escape_string($desc)}') " .
"WHERE cve_id='{$sys->help->real_escape_string($name)}'";
$sys->help->real_query($sql);
foreach ($refs as $key => $ref) {
$sql = "SELECT `id` FROM `reference`.`cve_references` WHERE `cve_seq`='" .
$sys->help->real_escape_string($name) . "' AND `source`='" .
$sys->help->real_escape_string($ref['src']) . "'";
$res2 = $sys->help->query($sql);
$row2 = $res2->fetch_array(MYSQLI_ASSOC);
if ($row2['id']) {
$sql = "UPDATE `reference`.`cve_references` SET `url`='" . $sys->help->real_escape_string($ref['url']) .
"', val='" . $sys->help->real_escape_string($ref['val']) . "' WHERE `id`=" . $row2['id'];
$sys->help->real_query($sql);
}
else {
$sql = "INSERT INTO `reference`.`cve_references` (`cve_seq`,`source`,`url`,`val`) VALUES ('" .
$sys->help->real_escape_string($name) . "','" . $sys->help->real_escape_string($ref['src']) .
"','" . $sys->help->real_escape_string($ref['url']) . "','" .
$sys->help->real_escape_string($ref['val']) . "')";
$sys->help->real_query($sql);
}
}
$err->script_log("existing cve $name");
*/
$existing++;
print ".";
}
if (($existing + $new) % 100 == 0) {
if (count($new_cves)) {
$sys->help->extended_insert("cve_db", $cve_fields, $new_cves, true);
$sys->help->execute();
}
if (count($new_cve_refs)) {
$sys->help->extended_insert("cve_references", $ref_fields, $new_cve_refs, true);
$sys->help->execute();
}
if (count($new_cve_web)) {
$sys->help->extended_replace("cve_web", $web_fields, $new_cve_web);
$sys->help->execute();
}
$new_cves = [];
$new_cve_refs = [];
$new_cve_web = [];
print "\t" . ($existing + $new) . " completed" . PHP_EOL;
$sys->help->update("settings", ['meta_value' => number_format((($existing + $new) / $items->length) * 100, 2)], [
[
'field' => 'meta_key',
'op' => '=',
'value' => 'cve-progress'
]
]);
$sys->help->execute();
}
}
if (is_array($new_cves) && count($new_cves)) {
$sys->help->extended_insert("cve_db", $cve_fields, $new_cves, true);
$sys->help->execute();
}
if (is_array($new_cve_refs) && count($new_cve_refs)) {
$sys->help->extended_insert("cve_references", $ref_fields, $new_cve_refs, true);
$sys->help->execute();
}
if (is_array($new_cve_web) && count($new_cve_web)) {
$sys->help->extended_replace("cve_web", $web_fields, $new_cve_web);
$sys->help->execute();
}
$sys->help->update("settings", ['meta_value' => 100], [
[
'field' => 'meta_key',
'op' => IN,
'value' => ['cve-dl-progress', 'cve-progress']
]
]);
$sys->help->execute();
}
function usage() {
print <<<EOO
Purpose: Parse the CVE file (allitems.xml) retrieved from http://cve.mitre.org/data/downloads/allitems.xml
Usage: php parse_cve.php -f={CVE filename} [--debug] [--help]
-f={CVE filename} The file to be parsed (allitems.xml). Can be absolute or relative path.
--debug Debugging output
--help This screen
EOO;
}

93
exec/parse_emass_control_map.php Normal file
View File

@ -0,0 +1,93 @@
<?php
/**
* File: parse_emass_control_map.php
* Author: Matt Shuter
* Purpose: Parse the Excel mapping of eMASS controls (.xlsx or .xls) into the database
* Created: Jul 6, 2017
*
* Portions Copyright 2016-2017: Cyber Perspectives, LLC, All rights reserved
* Released under the Apache v2.0 License
*
* Portions Copyright (c) 2012-2015, Salient Federal Solutions
* Portions Copyright (c) 2008-2011, Science Applications International Corporation (SAIC)
* Released under Modified BSD License
*
* See license.txt for details
*
* Change Log:
* - Jul 6, 2017 - File created
*/
$cmd = getopt("f:", ['debug::', 'help::']);
set_time_limit(0);
//use \PhpOffice\PhpSpreadsheet\Cell;
if (!isset($cmd['f']) || isset($cmd['help'])) {
die(usage());
}
elseif (!file_exists($cmd['f'])) {
die("File {$cmd['f']} not found" . PHP_EOL);
}
include_once 'config.inc';
require_once "database.inc";
require_once 'helper.inc';
require_once 'vendor/autoload.php';
check_path(TMP . "/rmf");
chdir(TMP);
$base_name = basename($cmd['f']);
$err = new Sagacity_Error($cmd['f']);
$db = new db();
// Create reader for file
$Reader = \PhpOffice\PhpSpreadsheet\IOFactory::createReaderForFile($cmd['f']);
$Reader->setReadDataOnly(true);
$objSS = $Reader->load($cmd['f']);
// Get the first and only worksheet
$wksht = $objSS->getSheet(0);
$col1 = true; // Bool to determine which column we're in within the loop
$ccis = array(); // Array to hold cci-control mappings
// Main loop to read in the Excel spreadsheet
foreach ($wksht->getRowIterator() as $row) {
foreach ($row->getCellIterator() as $cell) {
if ($col1) { // First column is the control
$ctrl = $cell->getValue();
}
else { // Second column is the cci
$cci = $cell->getValue();
}
// Change the col1 to iterate between the two columns
$col1 = !$col1;
}
$cci_id = substr($cci, strpos($cci, "-") + 1);
array_push($ccis, array($cci_id, $ctrl));
}
$db->save_EMASS_CCIs($ccis);
unset($objSS);
if (!isset($cmd['debug'])) {
rename($cmd['f'], TMP . "/rmf/$base_name");
}
function usage() {
print <<<EOO
Purpose: To import an Excel eMASS-CCI control mapping spreadsheet.
Usage: php parse_emass_control_map.php -f={Excel control map file} [--debug] [--help]
-f={Excel filename} File to import
--debug Debugging output
--help This screen
EOO;
}

363
exec/parse_excel_echecklist.php Normal file
View File

@ -0,0 +1,363 @@
<?php
/**
* File: parse_excel_echecklist.php
* Author: Ryan Prather
* Purpose: Parse the Excel version (.xlsx or .xls) of an eChecklist
* Created: May 9, 2014
*
* Portions Copyright 2016-2017: Cyber Perspectives, LLC, All rights reserved
* Released under the Apache v2.0 License
*
* Portions Copyright (c) 2012-2015, Salient Federal Solutions
* Portions Copyright (c) 2008-2011, Science Applications International Corporation (SAIC)
* Released under Modified BSD License
*
* See license.txt for details
*
* Change Log:
* - May 9, 2014 - File created
* - Sep 1, 2016 - Copyright Updated, added CWD parameter,
* Updated file purpose, and functions after class merger
* - Jan 30, 2017 - Updated to use parse_config.ini file
* - Mar 3, 2017 - Converted to getmypid() method instead of using Thread class
* - Mar 8, 2017 - Added check for existence of {TMP}/echecklist directory and revised directories to use TMP constant
* - May 26, 2017 - Migrated to PHPSpreadsheet library
* - Aug 28, 2017 - Fixed couple minor bugs
* - Jan 15, 2018 - Formatting, reorganized use statements, and cleaned up
*/
$cmd = getopt("f:", ['debug::', 'help::']);
set_time_limit(0);
if (!isset($cmd['f']) || isset($cmd['help'])) {
die(usage());
}
if (!file_exists("parse_config.ini")) {
die("Could not find parse_config.ini configuration file");
}
$conf = parse_ini_file("parse_config.ini");
chdir($conf['doc_root']);
include_once 'config.inc';
require_once "database.inc";
require_once 'helper.inc';
require_once 'vendor/autoload.php';
include_once 'excelConditionalStyles.inc';
use PhpOffice\PhpSpreadsheet\Cell\Coordinate;
use PhpOffice\PhpSpreadsheet\Worksheet\Worksheet;
check_path(TMP . "/echecklist");
chdir(TMP);
$db = new db();
$base_name = basename($cmd['f']);
$log = new Sagacity_Error($cmd['f']);
if (!file_exists($cmd['f'])) {
$db->update_Running_Scan($base_name, ['name' => 'status', 'value' => 'ERROR']);
$log->script_log("File not found", E_ERROR);
}
$db->update_Running_Scan($base_name, ['name' => 'pid', 'value' => getmypid()]);
$src = $db->get_Sources("eChecklist");
if (is_array($src) && count($src) && isset($src[0]) && is_a($src[0], 'source')) {
$src = $src[0];
}
else {
$log->script_log("Could not find the source", E_ERROR);
}
/*
$cacheMethod = \PhpOffice\PhpSpreadsheet\Collection\CellsFactory::cache_to_sqlite;
$cacheSettings = [
'memoryCacheSize' => '512MB'
];
\PhpOffice\PhpSpreadsheet\Settings::setCacheStorageMethod($cacheMethod, $cacheSettings);
*/
$host_list = [];
$Reader = \PhpOffice\PhpSpreadsheet\IOFactory::createReaderForFile($cmd['f']);
$Reader->setReadDataOnly(true);
$objSS = $Reader->load($cmd['f']);
$dt = new DateTime();
$existing_scan = $db->get_ScanData($conf['ste'], $base_name);
if (is_array($existing_scan) && count($existing_scan) && isset($existing_scan[0]) && is_a($existing_scan[0], 'scan')) {
$scan = $existing_scan[0];
}
else {
$ste = $db->get_STE($conf['ste']);
if (is_array($ste) && count($ste) && isset($ste[0]) && is_a($ste[0], 'ste')) {
$ste = $ste[0];
}
else {
$log->script_log("Could not retrieve the ST&E", E_ERROR);
}
$scan = new scan(null, $src, $ste, 1, $base_name, $dt->format('Y-m-d'));
if (!$scan_id = $db->save_Scan($scan)) {
$log->script_log("Failed to add scan for file: {$cmd['f']}", E_ERROR);
}
$scan->set_ID($scan_id);
}
$gen_os = $db->get_Software("cpe:/o:generic:generic:-", true);
if (is_array($gen_os) && count($gen_os) && isset($gen_os[0]) && is_a($gen_os[0], 'software')) {
$gen_os = $gen_os[0];
}
foreach ($objSS->getWorksheetIterator() as $wksht) {
if (preg_match('/Instruction|Cover Sheet/i', $wksht->getTitle())) {
$log->script_log("Skipping instruction and cover sheet", E_DEBUG);
continue;
}
elseif (isset($conf['ignore']) && $wksht->getSheetState() == Worksheet::SHEETSTATE_HIDDEN) {
$log->script_log("Skipping hidden worksheet {$wksht->getTitle()}");
continue;
}
$db->help->select("scans", ['status'], [
[
'field' => 'id',
'op' => '=',
'value' => $scan->get_ID()
]
]);
$thread_status = $db->help->execute();
if ($thread_status['status'] == 'TERMINATED') {
unset($objSS);
rename(realpath(TMP . "/{$scan->get_File_Name()}"), TMP . "/terminated/{$scan->get_File_Name()}");
$log->script_log("File parsing terminated by user");
}
$log->script_log("Reading from {$wksht->getTitle()} worksheet");
if (!preg_match('/STIG ID/i', $wksht->getCell("A10")->getValue()) &&
!preg_match('/VMS ID/i', $wksht->getCell("B10")->getValue()) &&
!preg_match('/CAT/i', $wksht->getCell("C10")->getValue()) &&
!preg_match('/IA Controls/i', $wksht->getCell("D10")->getValue()) &&
!preg_match('/Short Title/i', $wksht->getCell("E10")->getValue())) {
$log->script_log("Invalid headers in {$wksht->getTitle()}", E_WARNING);
continue;
}
$idx = [
'stig_id' => 1,
'vms_id' => 2,
'cat_lvl' => 3,
'ia_controls' => 4,
'short_title' => 5,
'target' => 6,
'overall' => 7, // min col
'consistent' => 8,
'notes' => 9,
'check_contents' => 10
];
$finding_count = [];
$tgts = [];
$short_title_col = Coordinate::stringFromColumnIndex($idx['short_title']);
$row_count = $wksht->getHighestDataRow() - 10;
foreach ($wksht->getRowIterator(10) as $row) {
foreach ($row->getCellIterator() as $cell) {
$ip = null;
$db->help->select("scans", ['status'], [
[
'field' => 'id',
'op' => '=',
'value' => $scan->get_ID()
]
]);
$thread_status = $db->help->execute();
if ($thread_status['status'] == 'TERMINATED') {
unset($objSS);
rename(realpath(TMP . "/{$scan->get_File_Name()}"), TMP . "/terminated/{$scan->get_File_Name()}");
$log->script_log("File parsing terminated by user");
die;
}
if ($cell->getColumn() > $short_title_col && !preg_match('/Overall/i', $cell->getValue())) {
if (preg_match('/status/i', $cell->getValue())) {
$log->script_log("Error: Invalid host name ('status') in {$wksht->getTitle()}", E_WARNING);
break;
}
if ($tgt_id = $db->check_Target($conf['ste'], $cell->getValue())) {
$tgt = $db->get_Target_Details($conf['ste'], $tgt_id);
if (is_array($tgt) && count($tgt) && isset($tgt[0]) && is_a($tgt[0], 'target')) {
$tgt = $tgt[0];
}
else {
$log->script_log("Could not find host {$cell->getValue()}", E_ERROR);
}
}
else {
$tgt = new target($cell->getValue());
$tgt->set_OS_ID($gen_os->get_ID());
$tgt->set_STE_ID($conf['ste']);
$tgt->set_Location($conf['location']);
$tgt->set_Notes('New Target');
if (preg_match('/((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.|$)){4}/', $cell->getValue())) {
$ip = $cell->getValue();
$int = new interfaces(null, null, null, $ip, null, null, null, null);
$tgt->interfaces["{$ip}"] = $int;
}
$tgt->set_ID($db->save_Target($tgt));
}
$tgts[] = $tgt;
$hl = new host_list();
$hl->setFindingCount($row_count);
$hl->setTargetId($tgt->get_ID());
$hl->setTargetName($tgt->get_Name());
if ($ip) {
$hl->setTargetIp($ip);
}
elseif (is_array($tgt->interfaces) && count($tgt->interfaces)) {
foreach ($tgt->interfaces as $int) {
if (!in_array($int->get_IPv4(), ['0.0.0.0', '127.0.0.1'])) {
$ip = $int->get_IPv4();
break;
}
}
$hl->setTargetIp($ip);
}
$scan->add_Target_to_Host_List($hl);
}
if (preg_match('/Overall/i', $cell->getValue())) {
break;
}
}
break;
}
$db->update_Running_Scan($base_name, ['name' => 'host_count', 'value' => count($tgts)]);
// increment the column indexes for notes, check contents, and missing PDI
if (is_array($tgts) && count($tgts) > 1) {
$idx['overall'] += count($tgts);
$idx['consistent'] += count($tgts);
$idx['notes'] += count($tgts);
$idx['check_contents'] += count($tgts);
}
elseif (empty($tgts)) {
$log->script_log("Failed to identify targets in worksheet {$wksht->getTitle()}", E_WARNING);
continue;
}
$stig_col = Coordinate::stringFromColumnIndex($idx['stig_id']);
$vms_col = Coordinate::stringFromColumnIndex($idx['vms_id']);
$cat_col = Coordinate::stringFromColumnIndex($idx['cat_lvl']);
$ia_col = Coordinate::stringFromColumnIndex($idx['ia_controls']);
$title_col = Coordinate::stringFromColumnIndex($idx['short_title']);
$notes_col = Coordinate::stringFromColumnIndex($idx['notes']);
$new_findings = [];
$updated_findings = [];
foreach ($wksht->getRowIterator(11) as $row) {
$stig_id = $wksht->getCell("{$stig_col}{$row->getRowIndex()}")->getValue();
$cat_lvl = substr_count($wksht->getCell("{$cat_col}{$row->getRowIndex()}")->getValue(), "I");
$short_title = $wksht->getCell("{$title_col}{$row->getRowIndex()}")->getValue();
$notes = $wksht->getCell("{$notes_col}{$row->getRowIndex()}")->getValue();
$stig = $db->get_Stig($stig_id);
if (is_array($stig) && count($stig) && isset($stig[0]) && is_a($stig[0], 'stig')) {
$stig = $stig[0];
}
else {
$pdi = new pdi(null, $cat_lvl, $dt->format("Y-m-d"));
$pdi->set_Short_Title($short_title);
$pdi->set_Group_Title($short_title);
if (!($pdi_id = $db->save_PDI($pdi))) {
$log->script_log("Failed to add a new PDI for STIG ID $stig_id", E_ERROR);
}
$stig = new stig($pdi_id, $stig_id, $short_title);
$db->add_Stig($stig);
}
$x = 0;
foreach ($tgts as $tgt) {
$status = $wksht->getCell(Coordinate::stringFromColumnIndex($idx['target'] + $x) . $row->getRowIndex())
->getValue();
$log->script_log("{$tgt->get_Name()} {$stig->get_ID()} ($status)\n", E_DEBUG);
$finding = $db->get_Finding($tgt, $stig);
if (is_array($finding) && count($finding) && isset($finding[0]) && is_a($finding[0], 'finding')) {
$tmp = $finding[0];
$tmp->set_Finding_Status_By_String($status);
$tmp->set_Notes($notes);
$tmp->set_Category($cat_lvl);
$updated_findings[] = $tmp;
}
else {
$tmp = new finding(null, $tgt->get_ID(), $stig->get_PDI_ID(), $scan->get_ID(), $status, $notes, null, null, null);
$tmp->set_Category($cat_lvl);
$new_findings[] = $tmp;
}
$x++;
}
if (PHP_SAPI == 'cli') {
print "\r" . sprintf("%.2f%%", (($row->getRowIndex() - 10) / $row_count) * 100);
}
else {
$db->update_Running_Scan($base_name, ['name' => 'perc_comp', 'value' => (($row->getRowIndex() - 10) / $row_count) * 100]);
}
}
if (!$db->add_Findings_By_Target($updated_findings, $new_findings)) {
print "Error adding finding" . PHP_EOL;
}
}
unset($objSS);
$db->update_Scan_Host_List($scan, $host_list);
if (!isset($cmd['debug'])) {
rename($cmd['f'], TMP . "/echecklist/$base_name");
}
$db->update_Running_Scan($base_name, ['name' => 'perc_comp', 'value' => 100, 'complete' => 1]);
function usage()
{
print <<<EOO
Purpose: To import an Excel E-Checklist file.
Usage: php parse_excel_echecklist.php -f={eChecklist File} [-i] [--debug] [--help]
-f={eChecklist File} The file to import
-i Ignore hidden worksheets. This run by default when run through Sagacity
--debug Debugging output
--help This screen
EOO;
}
/**
* Function to validate and make sure spreadsheet is as it should be
*
* @param \PhpOffice\PhpSpreadsheet\Worksheet\Worksheet $wksht
*/
function check_worksheet(\PhpOffice\PhpSpreadsheet\Worksheet\Worksheet &$wksht)
{
}

189
exec/parse_host_data_collection.php Normal file
View File

@ -0,0 +1,189 @@
<?php
/**
* File: parse_host_data_collection.php
* Author: Ryan Prather
* Purpose: This script will parse the configuration files that are passed to it and update the finding details
* Created: May 29, 2014
*
* Portions Copyright 2016: Cyber Perspectives, All rights reserved
* Released under the Apache v2.0 License
*
* Portions Copyright (c) 2012-2015, Salient Federal Solutions
* Portions Copyright (c) 2008-2011, Science Applications International Corporation (SAIC)
* Released under Modified BSD License
*
* See license.txt for details
*
* Change Log:
* - May 29, 2014 - File created
* - Sep 1, 2016 - Copyright Updated, added CWD parameter, and functions after class merger
* - Jan 30, 2017 - Updated to use parse_config.ini file
* - Feb 15, 2017 - Moved MYSQL_DT_FROMAT constant to inc/helper.inc with other constants
*/
$conf = parse_ini_file("parse_config.ini", false);
$cmd = getopt(array('debug::', 'help::'));
if (isset($cmd['help'])) {
usage();
exit;
}
chdir($conf['doc_root']);
include_once 'config.inc';
include_once "database.inc";
include_once 'helper.inc';
set_time_limit(0);
$db = new db();
if ($tgt_id = $db->check_Target($conf['ste'], $conf['target'])) {
$tgt = $db->get_Target_Details($conf['ste'], $tgt_id)[0];
}
else {
Sagacity_Error::err_handler("Could not find host ID " . $conf['target']);
die;
}
$err = new Sagacity_Error($tgt->get_Name() . "_data_col.log");
chdir(TMP . "/data_collection/" . $tgt->get_Name());
$os = $db->get_Software($tgt->get_OS_ID())[0];
$sw_arr = $db->get_Target_Software($tgt->get_ID());
$sw_str = '';
foreach ($sw_arr as $key => $sw) {
$sw_str .= $sw->get_Man() . " " . $sw->get_Name() . " " . $sw->get_Version() . PHP_EOL;
}
$answer_files = glob("*-answers.txt");
if (isset($conf['overwrite'])) {
foreach ($answer_files as $file) {
unlink($file);
}
$answer_files = array();
}
if (!count($answer_files)) {
if ($tgt->get_Cat_ID()) {
$qa = $db->get_Interview_Answers($tgt->get_Cat_ID());
$cat = $db->get_Category($tgt->get_Cat_ID())[0];
$qa_handle = fopen($cat->get_Name() . "-answers.txt", "w");
foreach ($qa as $key => $ans) {
fwrite($qa_handle, $ans->key . "=" . ($ans->answer ? "y" : "n") . PHP_EOL);
}
fclose($qa_handle);
}
}
$findings = $db->get_Finding($tgt, null, null, false, "Not Reviewed");
$run_stigs = array();
$files = glob("*.*");
$hostfiles = array();
$filepermsSize = 0;
$minfilepermsSize = 15000000;
foreach ($files as $file) {
if (preg_match("/summary|error|checksum/i", $file)) {
continue;
}
if (preg_match("/\-answers\.txt/i", $file)) {
$fname = "answerfile";
}
elseif (preg_match("/\-config\.txt/", $file)) {
$fname = "cisco_config";
}
elseif ($file == "file_permissions.txt") {
$filepermsSize = filesize($file);
$fname = "file_permissions";
}
else {
$fname = preg_replace("/[\.][^\.]+$/", '', basename($file));
}
$hostfiles["$fname"] = preg_replace("/[^[:print:]]/", "", file($file));
}
// print "finding count: ".count($findings).PHP_EOL;
$scan_id = 0;
foreach ($findings as $key => $find) {
if (false) {
$find = new finding();
}
$ret = array();
if ($find->get_Scan_ID()) {
$scan_id = $find->get_Scan_ID();
}
elseif ($scan_id) {
$find->set_Scan_ID($scan_id);
}
else {
$existing_scan = $db->get_ScanData($tgt->get_STE_ID(), $tgt->get_Name() . " data collection");
if (count($existing_scan)) {
$scan = $existing_scan[0];
$scan_id = $scan->get_ID();
}
else {
$src = $db->get_Sources("Data Collection");
$dt = new DateTime();
$ste = $db->get_STE($tgt->get_STE_ID())[0];
$scan_id = $db->save_Scan(new scan(null, $src, $ste, 1, $tgt->get_Name() . " data collection", $dt->format('Y-m-d H:i:s')));
}
$find->set_Scan_ID($scan_id);
}
$stig = $db->get_STIG_By_PDI($find->get_PDI_ID());
if (!is_a($stig, 'stig')) {
continue;
}
$function = $db->get_STIG_Function($stig, $tgt);
if (empty($function)) {
continue;
}
$stig_function = preg_replace("/[\.\-\ ]+/", "", $stig->get_ID());
if (is_numeric(substr($stig_function, 0, 1))) {
$stig_function = "S" . $stig_function;
}
if (!in_array($stig_function, $run_stigs)) {
eval($function);
$run_stigs[] = $stig_function;
}
$start = new DateTime();
$ret = call_user_func($stig_function);
$end = new DateTime();
$diff = $end->diff($start);
$err->script_log("Tweak function: $stig_function" . PHP_EOL . "Result: " . print_r($ret, true));
if ($diff->format("%s") > 3) {
$err->script_log("Tweak function execution exceeded 3 seconds.");
}
$find->set_Finding_Status_By_String($ret['status']);
$find->prepend_Notes("(Script) " . $ret['notes']);
$db->update_Finding($find);
}
function usage() {
print <<<EOO
Purpose: To execute tweak function to limit the number of manual checks that need to be accomplished.
Usage: php parse_host_data_collection.php -s={ST&E ID} -t={target name} [--debug] [--overwrite] [--help]
-s={ST&E ID} The ST&E ID the results are going to imported for
-t={target name} The name of the target this data is for (can be formal name, hostname, or IP address)
--overwrite Will create a new answer file for this target using what is in the database
--debug Debugging output
EOO;
}

337
exec/parse_iavm.php Normal file
View File

@ -0,0 +1,337 @@
<?php
/**
* File: parse_iavm.php
* Author: Ryan Prather
* Purpose: To parse IAVM files retrieved from https://iavm.csd.disa.mil
* Created: Jul 9, 2014
*
* Portions Copyright 2016: Cyber Perspectives, All rights reserved
* Released under the Apache v2.0 License
*
* Portions Copyright (c) 2012-2015, Salient Federal Solutions
* Portions Copyright (c) 2008-2011, Science Applications International Corporation (SAIC)
* Released under Modified BSD License
*
* See license.txt for details
*
* Change Log:
* - Jul 9, 2014 - File created
* - Sep 1, 2016 - Copyright Updated and converted to constants
*/
$cmd = getopt("d:f::", array('debug::', 'help::'));
if (!isset($cmd['f']) || !isset($cmd['s']) || isset($cmd['help'])) {
usage();
exit;
}
include_once 'config.inc';
include_once "database.inc";
include_once 'helper.inc';
chdir(DOC_ROOT . "/tmp");
set_time_limit(0);
$sys = new db();
$db = new mysqli(DB_SERVER, 'web', db::decrypt_pwd(), 'sagacity');
if ($db->connect_errno) {
die($db->connect_error);
}
if (!isset($cmd['d'])) {
die("Did not include the directory the files are in");
}
chdir($cmd['d']);
if (isset($cmd['f'])) {
$files = array(0 => $cmd['f']);
}
else {
$files = glob("*.xml");
}
foreach ($files as $file) {
print $file . PHP_EOL;
$doc = new DOMDocument();
$doc->load($file);
$pi = $doc->createProcessingInstruction('xml-stylesheet', 'type="text/xsl" href="iavm.xsl"');
$doc->insertBefore($pi, $doc->getElementsByTagName("iavmNotice")->item(0));
$doc->xmlStandalone = true;
$tmp = $doc->saveXML();
$tmp = str_replace(" xmlns=\"http://iavm.csd.disa.mil/schemas/IavmNoticeSchema/1.2\"", "", $tmp);
$doc->loadXML($tmp);
// root node values (iavm_notice table)
$id = getValue($doc, '/iavmNotice/@noticeId');
$xmlurl = getValue($doc, '/iavmNotice/xmlUrl');
$htmlurl = getValue($doc, '/iavmNotice/htmlUrl');
$noticeNumber = getValue($doc, '/iavmNotice/iavmNoticeNumber');
$title = getValue($doc, '/iavmNotice/title');
$type = getValue($doc, '/iavmNotice/type');
$state = getValue($doc, '/iavmNotice/state');
$lastUpdated = getValue($doc, '/iavmNotice/lastUpdated');
$releaseDate = getValue($doc, '/iavmNotice/releaseDate');
$supersedes = getValue($doc, '/iavmNotice/supersedes');
$execSummary = getValue($doc, '/iavmNotice/executiveSummary');
$fixAction = getValue($doc, '/iavmNotice/fixAction');
$note = getValue($doc, '/iavmNotice/note');
$vulnAppsSysAndCntrmsrs = getValue($doc, '/iavmNotice/vulnAppsSysAndCntrmsrs');
$knownExploits = getValue($doc, '/iavmNotice/knownExploits');
$stigFindingSeverity = getValue($doc, '/iavmNotice/vms/stigFindingSeverity');
// iavm_tech_overview
$techOverview = getValue($doc, '/iavmNotice/techOverview', null, true);
// iavm_references
$references = getValue($doc, '/iavmNotice/references/reference', null, true);
// iavm_bids
$bids = getValue($doc, '/iavmNotice/deepSightBids/bid', null, true);
// iavm_patches
$patches = getValue($doc, '/iavmNotice/patches/patch', null, true);
// iavm_mitigations
$mitHeader = getValue($doc, '/iavmNotice/tempMitStrat/header');
$mitBody = getValue($doc, '/iavmNotice/tempMitStrat/body');
$doc->formatOutput = true;
$doc->preserveWhiteSpace = true;
//print $doc->saveXML();
if (is_array($supersedes)) {
$supersedes = implode(',', $supersedes);
}
$stig = $sys->get_Stig($noticeNumber);
if (is_array($stig) && count($stig) && isset($stig[0]) && is_a($stig[0], 'stig')) {
$stig = $stig[0];
$pdi_id = $stig->get_PDI_ID();
}
else {
$pdi = new pdi(null, $stigFindingSeverity, $lastUpdated);
$pdi->set_Short_Title($title);
$pdi->set_Group_Title($title);
$pdi->set_Description($execSummary);
// print_r($pdi);
$pdi_id = $sys->save_PDI($pdi);
$stig = new stig($pdi_id, $noticeNumber, $execSummary);
// print_r($stig);
$sys->add_Stig($stig);
}
$last_updated_dt = new DateTime($lastUpdated);
$release_date_dt = new DateTime($releaseDate);
$iavm = $sys->get_IAVM($noticeNumber);
if (is_null($iavm)) {
$sys->help->insert("sagacity.iavm_notices", [
'pdi_id' => $pdi_id,
'noticeId' => $id,
'xmlUrl' => $xmlurl,
'htmlUrl' => $htmlurl,
'file_name' => basename($file),
'iavmNoticeNumber' => $noticeNumber,
'title' => $title,
'type' => $type,
'state' => $state,
'lastUpdated' => $last_updated_dt->format(MYSQL_D_FORMAT),
'releaseDate' => $release_date_dt->format(MYSQL_D_FORMAT),
'supersedes' => $supersedes,
'executiveSummary' => $execSummary,
'fixAction' => $fixAction,
'note' => $note,
'vulnAppsSysAndCntrmsrs' => $vulnAppsSysAndCntrmsrs,
'stigFindingSeverity' => $stigFindingSeverity,
'knownExploits' => $knownExploits
], true);
if (!$sys->help->execute()) {
error_log("file: $file" . PHP_EOL . $db->error . PHP_EOL . $ins_sql);
continue;
//die;
}
}
if ($bids->length) {
foreach ($bids as $bid) {
if ($res = $db->query(
"SELECT COUNT(1) as 'cnt' FROM sagacity.iavm_bids WHERE iavm_notice_id = $id AND bid = '" . $bid->textContent .
"'")) {
if ($res->num_rows) {
$row = $res->fetch_array(MYSQLI_ASSOC);
print "existing bid: $bid->textContent" . PHP_EOL;
}
else {
$sql = "INSERT INTO sagacity.iavm_bids (iavm_notice_id, bid) VALUES ($id, $bid->textContent)";
$db->real_query($sql);
print "new bid: $bid->textContent" . PHP_EOL;
}
}
else {
error_log($db->error);
}
}
}
if ($references->length) {
foreach ($references as $ref) {
$url = getValue($doc, "url", $ref);
$title = getValue($doc, "title", $ref);
$type = getValue($doc, "type", $ref);
$res = $db->query(
"SELECT id FROM sagacity.iavm_references WHERE iavm_notice_id = $id AND url='" .
$db->real_escape_string($url) . "'");
if ($res->num_rows) {
$row = $res->fetch_array(MYSQLI_ASSOC);
$sql = "UPDATE sagacity.iavm_references SET title = '" . $db->real_escape_string($title) .
"' WHERE id = " . $row['id'];
$db->real_query($sql);
print "existing reference: " . $title . " (" . $row['id'] . ")" . PHP_EOL;
}
else {
$sql = "INSERT INTO sagacity.iavm_references (iavm_notice_id, title, url) VALUES ($id, '" .
$db->real_escape_string($title) . "','" . $db->real_escape_string($url) . "')";
$db->real_query($sql);
// print "db error: ".$db->error.PHP_EOL;
print "new reference: " . $title . PHP_EOL;
}
$matches = array();
if (preg_match("/microsoft\.com.*\/([\d]+)/i", $url, $matches) ||
preg_match("/(MS[\d]+\-[\d]+)/", $title, $matches)) {
if (is_numeric($matches[1])) {
$matches[1] = "KB" . $matches[1];
}
$adv = new advisory($pdi_id, $matches[1], "", "", $url);
$sys->save_Advisory(array(
0 => $adv
));
}
}
}
if ($techOverview->length) {
foreach ($techOverview as $to) {
$details = getValue($doc, "details", $to);
if ($details) {
$res = $db->query(
"SELECT id FROM sagacity.iavm_tech_overview WHERE iavm_notice_id = $id AND details='" .
$db->real_escape_string($details) . "'");
$row = $res->fetch_array(MYSQLI_ASSOC);
if ($row['id']) {
print "existing overview for $id" . PHP_EOL;
}
else {
$sql = "INSERT INTO sagacity.iavm_tech_overview (iavm_notice_id, details) VALUES ($id, '" .
$db->real_escape_string($details) . "')";
$db->real_query($sql);
print "new overview" . PHP_EOL;
}
}
$entries = getValue($doc, "entry", $to, true);
if ($entries->length) {
foreach ($entries as $entry) {
$entry_title = getValue($doc, "title", $entry);
$entry_desc = getValue($doc, "description", $entry);
if (substr($entry_title, 0, 3) == 'CVE') {
print "CVE: $entry_title" . PHP_EOL;
$sql = "REPLACE INTO sagacity.iavm_to_cve (noticeId, cve_id) VALUES (" . $id . "," . "'" .
$db->real_escape_string($entry_title) . "')";
$db->real_query($sql);
}
else {
Sagacity_Error::err_handler("Entry title: $entry_title in file $file");
}
}
}
}
}
if ($patches->length) {
foreach ($patches as $patch) {
$title = getValue($doc, "title", $patch);
$type = getValue($doc, "type", $patch);
$url = getValue($doc, "url", $patch);
$res = $db->query(
"SELECT id FROM sagacity.iavm_patches WHERE iavm_notice_id = $id AND url = '" .
$db->real_escape_string($url) . "'");
$row = $res->fetch_array(MYSQLI_ASSOC);
if ($row['id']) {
$sql = "UPDATE sagacity.iavm_patches SET `type` = '" . $db->real_escape_string($type) .
"', `title` = '" . $db->real_escape_string($title) . "' WHERE id = " . $row['id'];
$db->real_query($sql);
print "existing patch: $title (" . $row['id'] . ")" . PHP_EOL;
}
else {
$sql = "INSERT INTO sagacity.iavm_patches (iavm_notice_id, `type`, title, url) VALUES ($id, '" .
$db->real_escape_string($type) . "','" . $db->real_escape_string($title) . "','" .
$db->real_escape_string($url) . "')";
$db->real_query($sql);
print "new patch: $title" . PHP_EOL;
}
$matches = array();
if (preg_match("/(KB[\d]+)|(MS[\d]+\-[\d]+)/i", $title, $matches)) {
$adv = new advisory($pdi_id, $matches[1], "", "", $url);
$sys->save_Advisory(array(
0 => $adv
));
}
}
}
if ($mitHeader) {
$res = $db->query(
"SELECT id FROM sagacity.iavm_mitigations WHERE iavm_notice_id = $id AND header = '" .
$db->real_escape_string($mitHeader) . "'");
$row = $res->fetch_array(MYSQLI_ASSOC);
if ($row['id']) {
$sql = "UPDATE sagacity.iavm_mitigations SET body = '" . $db->real_escape_string($mitBody) .
"' WHERE id = " . $row['id'];
$db->real_query($sql);
print "existing mitigation: " . $row['id'] . PHP_EOL;
}
else {
$sql = "INSERT INTO sagacity.iavm_mitigations (iavm_notice_id, header, body) VALUES ($id, '" .
$db->real_escape_string($mitHeader) . "','" . $db->real_escape_string($mitBody) . "')";
$db->real_query($sql);
print "new mitigation: $mitHeader" . PHP_EOL;
}
}
if (!$doc->save(DOC_ROOT . "/reference/iavms/$file")) {
print "error saving" . PHP_EOL;
die;
}
}
function usage() {
print <<<EOO
Purpose: To import an IAVM file and populate/update the database
Usage: php parse_iavm.php -d={IAVM Directory} [-f={XCCDF result file}] [--debug] [--help]
-d={IAVM directory} The directory to import the files from. This will crawl the directory and import all the IAVMs
-f={XCCDF file} The IAVM file specifically
--debug Debugging output
--help This screen
EOO;
}

119
exec/parse_iavm_cve.php Normal file
View File

@ -0,0 +1,119 @@
<?php
/**
* File: parse_iavm_cve.php
* Author: Ryan Prather
* Purpose: Script to parse iavm_to_cve(u).xml file received from DISA
* Created: Jul 9, 2014
*
* Portions Copyright 2016: Cyber Perspectives, All rights reserved
* Released under the Apache v2.0 License
*
* Portions Copyright (c) 2012-2015, Salient Federal Solutions
* Portions Copyright (c) 2008-2011, Science Applications International Corporation (SAIC)
* Released under Modified BSD License
*
* See license.txt for details
*
* Change Log:
* - Jul 9, 2014 - File created
* - Sep 1, 2016 - Copyright Updated and converted to constants
*/
$cmd = getopt("f:", array('debug::', 'help::'));
if (!isset($cmd['f']) || isset($cmd['help'])) {
usage();
exit;
}
include_once 'config.inc';
include_once "database.inc";
include_once 'helper.inc';
chdir(DOC_ROOT . "/tmp");
set_time_limit(0);
$sys = new db();
$db = new mysqli(DB_SERVER, 'web', db::decrypt_pwd(), 'sagacity');
if ($db->connect_errno) {
die($db->connect_error);
}
$doc = new DOMDocument();
$doc->load($cmd['f']);
$items = getValue($doc, 'IAVM', null, true);
foreach ($items as $node) {
$pdi_id = 0;
$vms = getValue($doc, 'S/@VMSKey', $node);
$vms = preg_replace("/V0{1,6}/", "V-", $vms);
$iavm_id = getValue($doc, 'S/@IAVM', $node);
$title = getValue($doc, 'S/@Title', $node);
$cat = substr_count(getValue($doc, 'S/@Severity', $node), 'I', 7);
$rel_date = getValue($doc, 'S/@ReleaseDate', $node);
$rel_dt = new DateTime($rel_date);
$cves = getValue($doc, 'CVEs/CVENumber', $node, true);
$stig = $sys->get_Stig($iavm_id);
$iavm = $sys->get_IAVM($iavm_id);
print $iavm_id . PHP_EOL;
if (is_array($stig) && count($stig) && isset($stig[0]) && is_a($stig[0], 'stig')) {
$stig = $stig[0];
$pdi_id = $stig->get_PDI_ID();
}
else {
if ($iavm) {
$stig = new stig($iavm->get_PDI_ID(), $iavm_id, $title);
$sys->add_Stig($stig);
}
else {
$pdi = new pdi(null, $cat, $rel_dt->format("Y-m-d"));
$pdi->set_Short_Title($title);
$pdi->set_Group_Title($title);
$pdi_id = $sys->save_PDI($pdi);
$stig = new stig($pdi_id, $iavm_id, $title);
$sys->add_Stig($stig);
}
}
if ($iavm) {
if ($cves->length) {
foreach ($cves as $cve_node) {
if (substr($cve_node->textContent, 0, 3) == 'CAN') {
$cve = 'CVE' . substr($cve_node->textContent, 3);
}
else {
$cve = $cve_node->textContent;
}
if (!in_array($cve, $iavm->get_CVE())) {
$iavm->add_CVE($cve);
}
}
$sys->save_IAVM($iavm);
}
}
}
function usage() {
print <<<EOO
Purpose: To import the cve-to-iavm(u).xml file retrieved from http://iasecontent.disa.mil/stigs/xml/iavm-to-cve%28u%29.xml
Usage: php parse_iavm_cve.php -f={file} [--debug] [--help]
-f={file} The file to import
--debug Debugging output
--help This screen
EOO;
}

284
exec/parse_mbsa.php Normal file
View File

@ -0,0 +1,284 @@
<?php
/**
* File: parse_mbsa.php
* Author: Ryan Prather
* Purpose: Background script to parse MBSA files
* Created: July 3, 2014
*
* Portions Copyright 2016: Cyber Perspectives, All rights reserved
* Released under the Apache v2.0 License
*
* Portions Copyright (c) 2012-2015, Salient Federal Solutions
* Portions Copyright (c) 2008-2011, Science Applications International Corporation (SAIC)
* Released under Modified BSD License
*
* See license.txt for details
*
* Change Log:
* - July 3, 2014 - File created
* - Sep 1, 2016 - Copyright Updated, added CWD parameter, and updated functions after class merger
* - Mar 4, 2017 - Removed Thread class calls
* - Jun 3, 2017 - Changed to check thread status and die if changed to TERMINATED
*/
$cmd = getopt("f:s:d:", array('debug::', 'help::'));
if (!isset($cmd['f']) || !isset($cmd['s']) || isset($cmd['help'])) {
usage();
exit;
}
include_once 'config.inc';
include_once 'helper.inc';
include_once 'database.inc';
check_path(TMP . "/mbsa");
chdir(TMP);
set_time_limit(0);
$err = new Sagacity_Error($cmd['f']);
if (!file_exists($cmd['f'])) {
$db->update_Running_Scan(basename($cmd['f']), array('name' => 'status', 'value' => 'ERROR'));
$err->script_log("File not found", E_ERROR);
}
$db = new db();
$base_name = basename($cmd['f']);
$host_list = array();
$db->update_Running_Scan($base_name, array('name' => 'pid', 'value' => getmypid()));
$src = $db->get_Sources("MBSA");
$existing_scan = $db->get_ScanData($cmd['s'], $base_name);
if (is_array($existing_scan) && count($existing_scan)) {
$scan = $existing_scan[0];
}
else {
$dt = new DateTime();
$ste = $db->get_STE($cmd['s'])[0];
$scan = new scan(null, $src, $ste, 0, $base_name, $dt->format("Y-m-d"));
$scan_id = $db->save_Scan($scan);
$scan->set_ID($scan_id);
}
if (substr($base_name, -3) == 'xml') {
$match = array();
if (preg_match('/([^\\\\]+)\-mbsa\.xml/i', $cmd['f'], $match)) {
$tgt_id = get_a_tgt_id($db, $cmd['s'], $match[1], $match[1]);
$tgt = $db->get_Target_Details($cmd['s'], $tgt_id)[0];
}
else {
$err->script_log("File name is not the correct format (hostname)-mbsa.xml required! (" . $base_name . ")", E_ERROR);
}
$host_list = array(
'target' => $tgt,
'count' => 0
);
$xml = new DOMDocument();
$xml->load($cmd['f']);
$checks = getValue($xml, "/x:XMLOut/x:Check/x:Detail/x:UpdateData", null, true);
foreach ($checks as $check) {
$db->help->select("sagacity.scans", array('status'), array(
array(
'field' => 'id',
'op' => '=',
'value' => $scan->get_ID()
)
));
$thread_status = $db->help->execute();
if ($thread_status['status'] == 'TERMINATED') {
unset($xml);
rename(realpath(TMP . "/{$scan->get_File_Name()}"), TMP . "/terminated/{$scan->get_File_Name()}");
$err->script_log("File parsing terminated by user");
die();
}
$ms = getValue($xml, "@BulletinID", $check);
$kb = getValue($xml, "@KBID", $check) ? "KB" . getValue($xml, "@KBID", $check) : "";
$installed = (getValue($xml, "@IsInstalled", $check) == 'true' ? true : false);
if ($ms) {
$adv = $db->get_Advisory($ms);
$iavm = $db->get_IAVM_From_External($ms);
if (is_null($iavm) && $kb) {
$iavm = $db->get_IAVM_From_External($kb);
}
}
elseif ($kb) {
$adv = $db->get_Advisory($kb);
$iavm = $db->get_IAVM_From_External($kb);
}
else {
$adv = null;
$iavm = null;
}
$err->script_log("$ms/$kb");
if (is_array($adv) && count($adv)) {
$adv = $adv[0];
}
if ($iavm && $iavm->get_PDI_ID()) {
$status = 'Open';
if ($installed) {
$status = 'Not a Finding';
}
$stig = $db->get_Stig($iavm->get_Notice_Number());
if (is_array($stig) && count($stig) && isset($stig[0]) && is_a($stig[0], 'stig')) {
$stig = $stig[0];
}
$err->script_log("pdi\t=\t" . $stig->get_PDI_ID() . "\tstig\t=\t" . $stig->get_ID());
$pdi = $db->get_PDI($stig->get_PDI_ID());
$vms = $db->get_GoldDisk_By_PDI($pdi->get_ID());
$ias = $db->get_IA_Controls_By_PDI($pdi->get_ID());
if (is_array($vms) && count($vms) && isset($vms[0]) && is_a($vms[0], 'stig')) {
$vms = $vms[0];
}
else {
$vms = '';
}
$finding = $db->get_Finding($tgt, $iavm, $scan);
if (!count($finding)) {
$ia_str = '';
foreach ($ias as $key => $ia):$ia_str .= $ia->get_Type() . "-" . $ia->get_Type_ID() . " ";
endforeach;
$finding = array(
0 => $stig->get_ID(),
1 => (is_a($vms, 'golddisk') ? $vms->get_ID() : $vms),
2 => $pdi->get_Category_Level_String(),
3 => $ia_str,
4 => $pdi->get_Short_Title(),
5 => $status,
6 => '(MBSA)',
7 => $pdi->get_Check_Contents(),
8 => ''
);
$host_list['count'] ++;
if (!$db->add_Finding($scan, $tgt, $finding)) {
$err->script_log("add finding failure");
}/**/
}
else {
// need to code update for MBSA
}
}
else {
$err->script_log("don't have iavm");
$cves = getValue($xml, "x:OtherIDs/x:OtherID[@Type='CVE']", $check, true);
if ($cves->length) {
foreach ($cves as $cve) {
$db_cve = $db->get_CVE($cve->textContent);
if ($db_cve) {
if (count($db_cve->get_IAVM())) {
$iavm = $db->get_IAVM($db_cve->get_IAVM()[0]);
}
break;
}
}
if ($iavm && $iavm->get_PDI_ID()) {
$err->script_log("found one");
$status = 'Open';
if ($installed) {
$status = 'Not a Finding';
}
$pdi = $db->get_PDI($iavm->get_PDI_ID());
$vms = $db->get_GoldDisk_By_PDI($pdi->get_ID());
$ias = $db->get_IA_Controls_By_PDI($pdi->get_ID());
if (is_array($vms) && count($vms)) {
$vms = $vms[0];
}
else {
$vms = '';
}
$finding = $db->get_Finding($tgt, $iavm, $scan);
if (!count($finding)) {
$ia_str = '';
foreach ($ias as $key => $ia):$ia_str .= $ia->get_Type() . "-" . $ia->get_Type_ID() . " ";
endforeach;
$finding = array(
0 => $iavm->get_Notice_Number(),
1 => (is_a($vms, 'golddisk') ? $vms->get_ID() : $vms),
2 => $pdi->get_Category_Level_String(),
3 => $ia_str,
4 => $pdi->get_Short_Title(),
5 => $status,
6 => '',
7 => $pdi->get_Check_Contents(),
8 => ''
);
$host_list['count'] ++;
if (!$db->add_Finding($scan, $tgt, $finding)) {
$err->script_log("add finding failure");
}/**/
}
else {
}
}
}
else {
$cve = $db->get_CVE_From_External(substr($kb, 2));
if ($cve && $cve->get_PDI_ID()) {
$err->script_log("found one");
$err->script_log("pdi: " . $cve->get_PDI_ID());
}
else {
$err->script_log("still don't have it");
}
}
}
}
}
elseif (substr($base_name, -3) == 'txt') {
}
$db->update_Scan_Host_List($scan, array(0 => $host_list));
if (!isset($cmd['debug'])) {
rename($cmd['f'], DOC_ROOT . "/tmp/mbsa/$base_name");
}
$db->update_Running_Scan($base_name, array('name' => 'perc_comp', 'value' => 100, 'complete' => 1));
function usage() {
print <<<EOO
Purpose: To import an MBSA result file
Usage: php parse_mbsa.php -s={ST&E ID} -f={result file} [--debug] [--help]
-s={ST&E ID} The ST&E ID this result file is being imported for
-f={result file} The result file to import
--debug Debugging output
--help This screen
EOO;
}

193
exec/parse_ms_kb_to_cve.php Normal file
View File

@ -0,0 +1,193 @@
<?php
/**
* File: parse_ms_kb_to_cve.php
* Author: Ryan Prather
* Purpose: To parse Excel MS/KB to CVE file retrieved from https://technet.microsoft.com/en-us/security/bulletin
* Released on the second Tues of the month
* Created: Jul 9, 2014
*
* Portions Copyright 2016: Cyber Perspectives, All rights reserved
* Released under the Apache v2.0 License
*
* Portions Copyright (c) 2012-2015, Salient Federal Solutions
* Portions Copyright (c) 2008-2011, Science Applications International Corporation (SAIC)
* Released under Modified BSD License
*
* See license.txt for details
*
* Change Log:
* - Jul 9, 2014 - File created
* - Sep 1, 2016 - Copyright Updated and updated functions after class merger
*/
$cmd = getopt("f:", array('debug::'));
include_once 'config.inc';
include_once 'database.inc';
include_once 'helper.inc';
chdir(DOC_ROOT . "/tmp");
set_time_limit(0);
$db = new db();
$base_name = basename($cmd['f']);
$err = new Sagacity_Error($cmd['f']);
if (!file_exists($cmd['f'])) {
$db->update_Running_Scan($base_name, array('name' => 'status', 'value' => 'ERROR'));
$err->script_log("File not found", E_ERROR);
}
$start = new DateTime();
$fh = fopen($cmd['f'], "r");
$row = fegetcsv($fh);
while ($row = fgetcsv($fh)) {
print ".";
$adv = array();
$ms_url = '';
$kb_url = '';
$sev = 'II';
$pdi_id = 0;
$date = $row[0];
$ms = $row[1];
switch ($row[3]) {
case 'Critical':
case 'Important':
$sev = 'I';
break;
case 'Low':
$sev = 'III';
break;
}
$impact = $row[4];
$title = $row[5];
$prod = $row[6];
$kb = "KB" . (isset($row[7]) && !empty($row[7]) ? $row[7] : $row[2]);
$comp = $row[8];
$cves = explode(",", $row[13]);
if (isset($cmd['debug'])) {
$err->script_log("$ms/$kb/$date");
}
$db_cve = null;
$has_cve = false;
if (is_array($cves) && count($cves)) {
foreach ($cves as $cve) {
$db_cve = $db->get_CVE($cve);
if (!is_null($db_cve)) {
if ($db_cve->get_PDI_ID()) {
$has_cve = true;
}
$dt = DateTime::createFromFormat("m-d-y", $date);
$db_cve->set_Phase_Date($dt->format("Y-m-d"));
if ($ms && !$db_cve->ref_Exists($ms)) {
$db_cve->add_Reference(new cve_reference(null, 'MS', $ms_url, $ms));
}
if ($kb && !$db_cve->ref_Exists($kb)) {
$db_cve->add_Reference(new cve_reference(null, 'MS', $kb_url, $kb));
}
$db->save_CVE(array(0 => $db_cve));
}
}
}
$stig = $db->get_Stig($ms);
if (!$iavm = $db->get_IAVM_From_External($ms)) {
$iavm = $db->get_IAVM_From_External($kb);
}
if (is_array($stig) && count($stig) && isset($stig[0]) && is_a($stig[0], 'stig')) {
$stig = $stig[0];
}
else {
$err->script_log("Failed to identify the correct STIG", E_WARNING);
}
// insert pdi and advisory
if ($iavm) {
$pdi_id = $iavm->get_PDI_ID();
}
elseif ($has_cve && $db_cve->get_PDI_ID()) {
$pdi_id = $db_cve->get_PDI_ID();
}
elseif ($stig) {
$pdi_id = $stig->get_PDI_ID();
}
else {
// insert PDI, STIG, advisory, and CVE's
$pdi = new pdi(null, $sev, null);
$pdi->set_Short_Title($title);
$pdi->set_Group_Title($title);
$pdi_id = $db->save_PDI($pdi);
$stig = new stig($pdi_id, $ms, $title);
$db->add_Stig($stig);
}
if ($tmp = $db->get_Advisory($ms)) {
$adv[] = $tmp[0];
}
if ($tmp = $db->get_Advisory("$kb")) {
$adv[] = $tmp[0];
}
if (is_array($adv) && count($adv)) {
foreach ($adv as $key => $ad) {
//if(!$ad->get_PDI_ID()) {
$ad->set_PDI_ID($pdi_id);
//}
$ad->set_Title($title);
$ad->set_Impact($impact);
}
}
else {
if ($ms && $kb) {
$adv = array(
0 => new advisory($pdi_id, $ms, "", "", $ms_url),
1 => new advisory($pdi_id, $kb, "", "", $kb_url)
);
$adv[0]->set_Title($title);
$adv[0]->set_Impact($impact);
$adv[1]->set_Title($title);
$adv[1]->set_Impact($impact);
}
elseif ($ms) {
$adv = array(
0 => new advisory($pdi_id, $ms, "", "", $ms_url)
);
$adv[0]->set_Title($title);
$adv[0]->set_Impact($impact);
}
elseif ($kb) {
$adv = array(
0 => new advisory($pdi_id, $kb, "", "", $kb_url)
);
$adv[0]->set_Title($title);
$adv[0]->set_Impact($impact);
}
}
$db->save_Advisory($adv);
}
$end = new DateTime();
$diff = $start->diff($end);
$err->script_log($diff->format("%H:%I:%S"));

240
exec/parse_mssql.php Normal file
View File

@ -0,0 +1,240 @@
<?php
/**
* File: parse_mssql.php
* Author: Ryan Prather
* Purpose: Parse MSSQL SRR result files
* Created: Jan 15, 2015
*
* Portions Copyright 2016: Cyber Perspectives, All rights reserved
* Released under the Apache v2.0 License
*
* Portions Copyright (c) 2012-2015, Salient Federal Solutions
* Portions Copyright (c) 2008-2011, Science Applications International Corporation (SAIC)
* Released under Modified BSD License
*
* See license.txt for details
*
* Change Log:
* - Jan 15, 2015 - File created
* - Sep 1, 2016 - Copyright Updated, added CWD parameter,
* added file purpose, and updated functions after class merger
* - Oct 24, 2016 - Updated class extension after rename from XMLParser to scan_xml_parser
* - Mar 4, 2017 - Removed Thread class calls
*/
$cmd = getopt("f:s:d:", array('debug::', 'help::'));
if (!isset($cmd['f']) || !isset($cmd['s']) || isset($cmd['help'])) {
usage();
exit;
}
chdir($cmd['d']);
include_once 'config.inc';
include_once 'helper.inc';
include_once 'xml_parser.inc';
chdir(DOC_ROOT . "/tmp");
set_time_limit(0);
$base_name = basename($cmd['f']);
$db = new db();
class mssql_parser extends scan_xml_parser {
var $tgt;
var $tag;
var $schema;
var $vms;
var $stig;
var $status;
var $notes;
public function __construct($ste_id_in, $fname_in) {
parent::__construct($this, $ste_id_in, $fname_in);
$this->host_list = array();
$this->count = 0;
$this->db->update_Running_Scan($this->scan->get_File_Name(), array('name' => 'pid', 'value' => getmypid()));
}
public function IMPORT_FILE_ASSET($attrs) {
$this->tag = array();
}
public function IMPORT_FILE_ASSET_ASSET_ID($attrs) {
if (isset($attrs['TYPE'])) {
$this->tag_id = str_replace(" ", "_", $attrs['TYPE']);
}
else {
$this->tag_id = null;
}
}
public function IMPORT_FILE_ASSET_ASSET_ID_data($data) {
if ($this->tag_id == "IP_ADDRESS") {
if (preg_match("/\./", $data)) {
$this->tag_id = "IP4_ADDRESS";
}
elseif (preg_match("/\:/", $data)) {
$this->tag_id = "IP6_ADDRESS";
}
}
$this->tag[$this->tag_id] = $data;
}
public function IMPORT_FILE_ASSET_ELEMENT($attrs) {
// finished ingesting the target data...need to check for target existence
if ($tgt_id = $this->db->check_Target($this->ste_id, $this->tag['HOST_NAME'])) {
$this->tgt = $this->db->get_Target_Details($this->ste_id, $tgt_id)[0];
}
elseif (isset($this->tag['IP4_ADDRESS']) && $tgt_id = $this->db->check_Target($this->ste_id, $this->tag['IP4_ADDRESS'])) {
$this->tgt = $this->db->get_Target_Details($this->ste_id, $tgt_id)[0];
}
else {
$this->tgt = new target($this->tag['HOST_NAME']);
$this->tgt->set_STE_ID($this->ste_id);
$sw = new software("cpe:/o:generic:generic:-");
$os = $this->db->get_Software($sw)[0];
$this->tgt->set_OS_ID($os->get_ID());
if (isset($this->tag['IP4_ADDRESS'])) {
$int = new interfaces(null, null, null, $this->tag['IP4_ADDRESS'], null, $this->tag['HOST_NAME'], $this->tag['HOST_NAME'], null);
$this->tgt->interfaces[] = $int;
}
if (isset($this->tag['IP6_ADDRESS'])) {
$int = new interfaces(null, null, null, null, $this->tag['IP6_ADDRESS'], $this->tag['HOST_NAME'], $this->tag['HOST_NAME'], null);
$this->tgt->interfaces[] = $int;
}
$this->db->save_Target($this->tgt);
}
}
public function IMPORT_FILE_ASSET_TARGET($attrs) {
$this->updated_findings = array();
$this->new_findings = array();
$this->schema = null;
}
public function IMPORT_FILE_ASSET_TARGET_IDENTIFIER_data($data) {
$this->schema = $data;
}
public function IMPORT_FILE_ASSET_TARGET_FINDING($attrs) {
$this->vms = null;
$this->stig = null;
$this->notes = null;
$this->status = "Not Reviewed";
}
public function IMPORT_FILE_ASSET_TARGET_FINDING_FINDING_ID_data($data) {
$vms_id = preg_replace("/V0+/", "V-", $data);
$vms = $this->db->get_GoldDisk($vms_id);
if (is_array($vms) && count($vms) && isset($vms[0]) && is_a($vms[0], 'golddisk')) {
$this->vms = $vms[0];
}
else {
$this->log->script_log("VMS $vms_id not found", E_WARNING);
$this->skip = true;
return;
}
if (!empty($this->vms)) {
$this->stig = $this->db->get_STIG_By_PDI($this->vms->get_PDI_ID());
}
}
public function IMPORT_FILE_ASSET_TARGET_FINDING_FINDING_STATUS_data($data) {
if ($data == "O") {
$this->status = "Open";
}
elseif ($data == "NF") {
$this->status = "Not a Finding";
}
else {
$this->status = "Not Reviewed";
}
}
public function IMPORT_FILE_ASSET_TARGET_FINDING_FINDING_DETAILS_data($data) {
if (!empty($this->schema)) {
$this->notes = $this->schema . " - " . $data;
}
else {
$this->notes = $data;
}
}
public function IMPORT_FILE_ASSET_TARGET_FINDING_end() {
if ($this->skip) {
$this->skip = false;
return;
}
// check for finding
$finding = $this->db->get_Finding($this->tgt, $this->stig);
if (is_array($finding) && count($finding)) {
$finding = $finding[0];
if (false) {
$finding = new finding();
}
$finding->prepend_Notes("(MSSQL) " . $this->notes);
if ($finding->get_Finding_Status_String() != "Not Reviewed" && $finding->get_Finding_Status_String() != $this->status) {
$finding->set_Finding_Status_By_String(
$finding->get_Deconflicted_Status($this->status)
);
}
else {
$finding->set_Finding_Status_By_String($this->status);
}
$this->updated_findings[$finding->get_PDI_ID()] = $finding;
}
else {
$finding = new finding(null, $this->tgt->get_ID(), $this->stig->get_PDI_ID(), $this->scan->get - ID(), $this->status, $this->notes, finding::NC, "MSSQL", 1);
$this->new_findings[$this->stig->get_PDI_ID()] = $finding;
}
// if present in $new_findings, append notes
// if present in $updated_findings, ? check for notes?
}
public function IMPORT_FILE_ASSET_TARGET_end() {
$this->db->add_Findings_By_Target($this->updated_findings, $this->new_findings);
}
}
$xml = new mssql_parser($cmd['s'], $cmd['f']);
$xml->debug = (isset($cmd['debug']) ? true : false);
//Enter xml code here
$xml->parse();
if (!$xml->debug) {
rename($cmd['f'], DOC_ROOT . "/tmp/mssql/" . $base_name);
}
$db->update_Running_Scan($base_name, array("name" => "perc_comp", "value" => 100, "complete" => 1));
function usage() {
print <<<EOO
Purpose: To import the MSSQL SRR result file from DISA
Usage: php parse_mssql.php -s={ST&E ID} -f={result file} [--debug] [--help]
-s={ST&E ID} The ST&E ID this result file is being imported for
-f={result file} The result file to import
--debug Debugging output
--help This screen
EOO;
}

3419
exec/parse_nessus.php Normal file
View File

File diff suppressed because it is too large Load Diff

430
exec/parse_nmap.php Normal file
View File

@ -0,0 +1,430 @@
<?php
/**
* File: parse_nmap.php
* Author: Ryan Prather
* Purpose: Parse an nmap result file
* Created: Jul 3, 2014
*
* Portions Copyright 2016: Cyber Perspectives, All rights reserved
* Released under the Apache v2.0 License
*
* Portions Copyright (c) 2012-2015, Salient Federal Solutions
* Portions Copyright (c) 2008-2011, Science Applications International Corporation (SAIC)
* Released under Modified BSD License
*
* See license.txt for details
*
* Change Log:
* - Jul 3, 2014 - File created
* - Sep 1, 2016 - Copyright Updated, added CWD parameter, and
* updated function calls after class merger
* - Oct 24, 2016 - Fixed bug (#6) when parsing port data, and added exclusion for "[host down]"
* - Nov 7, 2016 - Added d parameter documentation
* - Dec 7, 2016 - Added check for "Interesting ports on {IP}" line
* - Jan 30, 2017 - Updated to use parse_config.ini file, and added populating new targets with shortened os software string if available.
*/
$cmd = getopt("f:", ['debug::', 'help::']);
if (!isset($cmd['f']) || isset($cmd['help'])) {
die(usage());
}
$conf = parse_ini_file("parse_config.ini");
if (!$conf) {
die("Could not find parse_config.ini configuration file");
}
chdir($conf['doc_root']);
include_once 'config.inc';
include_once 'database.inc';
include_once 'helper.inc';
chdir(TMP);
set_time_limit(0);
$db = new db();
$base_name = basename($cmd['f']);
$err = new Sagacity_Error($cmd['f']);
if (!file_exists($cmd['f'])) {
$db->update_Running_Scan($base_name, ['name' => 'status', 'value' => 'ERROR']);
$err->script_log("File not found", E_ERROR);
}
$db->update_Running_Scan($base_name, ['name' => 'pid', 'value' => getmypid()]);
$src = $db->get_Sources("NMAP");
$existing_scan = $db->get_ScanData($conf['ste'], $base_name);
if (is_array($existing_scan) && count($existing_scan)) {
$scan = $existing_scan[0];
}
else {
$mtime = filemtime($cmd['f']);
$dt = DateTime::createFromFormat("U", $mtime);
$ste = $db->get_STE($conf['ste'])[0];
$scan = new scan(null, $src, $ste, 1, $base_name, $dt->format("Y-m-d"));
$scan_id = $db->save_Scan($scan);
$scan->set_ID($scan_id);
}
//echo "\$steid ($steid) will be used later\n";
# file is cool - reads the whole file into an array with one command...
$lines = file($cmd['f']);
$target = [];
foreach ($lines as $line_num => $line) {
$db->help->select("scans", ['status'], [
[
'field' => 'id',
'op' => '=',
'value' => $scan->get_ID()
]
]);
$thread_status = $db->help->execute();
if ($thread_status['status'] == 'TERMINATED') {
rename(realpath(TMP . "/{$scan->get_File_Name()}"), TMP . "/terminated/{$scan->get_File_Name()}");
$err->script_log("File parsing terminated by user");
die();
}
if (preg_match('/^[\r\n]+$/', $line)) {
continue;
} # skip blank lines
$line = trim($line, "\t\n\r"); # chomp would be nice...
if (!isset($filetype)) {
if (preg_match('/Starting|\-oN/', $line)) {
$filetype = "text";
}
elseif (preg_match('/\-oG/', $line)) {
$filetype = "grep";
}
elseif (preg_match('/xml version/', $line)) {
$filetype = "xml";
break;
}
if ($line_num >= 1 && !isset($filetype)) {
$err->script_log("ERROR File Type not found");
}
}
if ($filetype == "text") {
//echo "Text:$line_num: $line\n";
if (preg_match("/Host is up|Not shown|PORT\s+STATE|\[host down\]/", $line)) {
continue;
}
if (preg_match("/Interesting ports on ([\d\.]+)/", $line, $matches)) {
$ip = $matches[1];
$target[$ip] = [];
$target[$ip]['hostname'] = $ip;
}
elseif (preg_match("/Nmap scan report for ([^ ]+) \(([0-9\.]+)\)/", $line, $matches)) {
$ip = $matches[2];
if (preg_match('/\./', $matches[1])) {
$name = explode('.', $matches[1]);
$target[$ip]['hostname'] = $name[0];
$target[$ip]['fqdn'] = $matches[1];
}
else {
$target[$ip]['hostname'] = $matches[1];
}
$err->script_log($target[$ip]['hostname'] . ":$ip");
}
elseif (preg_match("/Nmap scan report for ([0-9\.]+)/", $line, $matches)) {
$ip = $matches[1];
$target[$ip]['hostname'] = "";
$err->script_log($target[$ip]['hostname'] . ":$ip");
}
elseif (preg_match("/^Discovered ([a-z]+) port (\d+)\/([udtcp]+) on (\d\.]+)$/", $line, $matches)) {
$state = $matches[1];
$port = $matches[2];
$proto = $matches[3];
$ip = $matches[4];
$target[$ip][$proto][$port]['state'] = $state;
$err->script_log("\t$ip:$port:$proto:$state");
}
elseif (preg_match("/Other addresses.*: ([0-9\. ]+)/", $line, $matches)) {
$target[$ip]['otherips'] = $matches[1];
$err->script_log("\tOther:" . $matches[1]);
}
elseif (preg_match('/(\d+)\/([udtcp]+)\s+(\S+)\s+(\S+)/', $line, $matches)) {
if ($matches[3] == 'unknown') {
continue;
}
$port = $matches[1];
$proto = $matches[2];
if (!empty($ip)) {
$target[$ip][$proto][$port]['state'] = $matches[3];
$target[$ip][$proto][$port]['iana'] = $matches[4];
}
$err->script_log("\t$port:$proto:$matches[3]:$matches[4]");
if (preg_match("/\d+\/[udtcp]+\s+\S+\s+\S+\s+(.*)/", $line, $matches)) {
$target[$ip][$proto][$port]['banner'] = $matches[1];
$err->script_log("\tBanner:$matches[1]");
}
}
elseif (preg_match('/MAC Address: ([A-F0-9:]+)/', $line, $matches)) {
$target[$ip]['mac'] = $matches[1];
if (preg_match('/MAC Address: [A-F0-9:]+\s+\((.*)\)/', $line, $matches)) {
$target[$ip]['description'] = $matches[1];
$err->script_log("\t" . $target[$ip]['mac'] . ": Interface:$matches[1]");
}
}
elseif (preg_match('/Service Info: OS: (\S+);/', $line, $matches)) {
$target[$ip]['OS'] = $matches[1];
if (preg_match('/Service Info: OS: \S+; CPE: (.+)/', $line, $matches)) {
$target[$ip]['cpe'] = $matches[1];
}
else {
$target[$ip]['cpe'] = null;
}
$err->script_log("\t" . $target[$ip]['OS'] . ", " . $target[$ip]['cpe']);
}
}
elseif ($filetype == "grep") {
$err->script_log("Grep:$line_num: $line" . PHP_EOL);
# -oG grep format is not recommended - it discards helpful information like hostname
if (preg_match("/Host: ([0-9\.]+) \((.*)\)\s+(Ports|Protocols):(.*)/", $line, $matches)) {
$ip = $matches[1];
if (preg_match('/\./', $matches[2])) {
$name = explode('.', $matches[2]);
$target[$ip]['hostname'] = $name[0];
$target[$ip]['fqdn'] = $matches[2];
}
else {
$target[$ip]['hostname'] = $matches[2];
}
$err->script_log("$ip:" . $matches[2]);
$type = $matches[3]; # will be used later when we support IP protocol scans
$ports_string = $matches[4];
$ports_list = explode(",", $ports_string);
foreach ($ports_list as $port_num => $port_str) {
# fields: port, state, owner, service/sunRPC/banner
# need to read the manual for grepable!
$port_info = explode("/", $port_str);
$port = $port_info[0];
$proto = $port_info[2];
$target[$ip][$proto][$port]['state'] = $port_info[1];
$target[$ip][$proto][$port]['iana'] = $port_info[4];
$target[$ip][$proto][$port]['banner'] = $port_info[6];
$err->script_log("\t$port:$proto:" . $port_info[1] . $port_info[4]);
$err->script_log("\tBanner: " . $port_info[6]);
}
}
} # end Grep parsing
} # end foreach line in file
if ($filetype == "xml") {
$err->script_log("Parsing XML");
$xml = new DOMDocument();
$xml->load($cmd['f']);
$hosts = getValue($xml, "/nmaprun/host", null, true);
$count = 0;
foreach ($hosts as $host) {
$addrs = getValue($xml, "address", $host, true);
foreach ($addrs as $addr) {
$addrtype = $addr->getAttribute("addrtype");
if ($addrtype == "ipv4") {
$ip = $addr->getAttribute("addr");
}
elseif ($addrtype == "mac") {
$vendor = $addr->getAttribute("vendor");
$mac = $addr->getAttribute("addr");
}
}
$target[$ip]['hostname'] = getValue($xml, "hostnames/hostname[@type='user']/@name", $host);
$target[$ip]['mac'] = $mac;
$target[$ip]['description'] = $vendor;
# Iterate through ports
$ports = getValue($xml, "ports/port", $host, true);
$tcp_ports = [];
$udp_ports = [];
foreach ($ports as $portxml) {
$portid = $portxml->getAttribute("portid");
$proto = $portxml->getAttribute("protocol");
if ($proto == 'tcp') {
$port = $db->get_TCP_Ports($portid)[0];
}
else {
$port = $db->get_UDP_Ports($portid)[0];
}
$target[$ip][$proto][$portid]['state'] = getValue($xml, "state/@state", $portxml);
$target[$ip][$proto][$portid]['iana'] = getValue($xml, "service/@name", $portxml);
$product = getValue($xml, "service/@product", $portxml);
$version = getValue($xml, "service/@version", $portxml);
$extrainfo = getValue($xml, "service/@extrainfo", $portxml);
$target[$ip][$proto][$portid]['banner'] = "$product $version $extrainfo";
$port->set_Banner("$product $version $extrainfo");
$port->set_IANA_Name(getValue($xml, "service/@name", $portxml));
if ($proto == 'tcp') {
$tcp_ports[] = $port;
}
else {
$udp_ports[] = $port;
}
//echo "$portid, $proto, " .$target[$ip][$proto][$portid]['banner'] ."\n";
} # end foreach port
$target[$ip]['OS'] = getValue($xml, "os/osmatch/@name", $host);
$err->script_log($target[$ip]['OS']);
} # end foreach host
} # end XML parsing
###################################
$db->update_Running_Scan($base_name, ['name' => 'host_count', 'value' => count($target)]);
$count = 0;
$tgt_ip = null;
foreach ($target as $ip => $tgt) {
# get target ID
$tgt_id = 0;
if (!in_array($ip, ['0.0.0.0', '127.0.0.1', '::0'])) {
$tgt_ip = $ip;
}
if ($tgt['hostname']) {
$tgt_id = $db->check_Target($conf['ste'], $tgt['hostname']);
}
if (!$tgt_id) {
$tgt_id = $db->check_Target($conf['ste'], $ip);
}
if (!$tgt_id) { # insert
$sw = $db->get_Software("cpe:/o:generic:generic:-")[0];
$tgt_obj = new target(($tgt['hostname'] ? $tgt['hostname'] : $ip));
$tgt_obj->set_STE_ID($conf['ste']);
//$tgt_obj->set_Notes("New target found by NMap");
$tgt_obj->set_OS_ID($sw->get_ID());
if ($sw->get_Shortened_SW_String()) {
$tgt_obj->set_OS_String($sw->get_Shortened_SW_String());
}
else {
$tgt_obj->set_OS_String($sw->get_SW_String());
}
$tgt_obj->set_Location(($conf['location'] ? $conf['location'] : ''));
$tgt_obj->interfaces["{$ip}"] = new interfaces(null, null, null, $ip, null, $tgt_obj->get_Name(), (isset($tgt['fqdn']) ? $tgt['fqdn'] : $tgt_obj->get_Name()), (isset($tgt['description']) ? $tgt['description'] : ""));
if (isset($tgt['tcp'])) {
foreach ($tgt['tcp'] as $port_num => $port) {
if ($port['state'] != 'open') {
continue;
}
$tcp = $db->get_TCP_Ports($port_num)[0];
if (!empty($port['banner'])) {
$tcp->set_Banner($port['banner']);
}
$tcp->set_IANA_Name($port['iana']);
//$tcp->set_Notes("Found in scan file " . $scan->get_File_Name());
$tgt_obj->interfaces["{$ip}"]->add_TCP_Ports($tcp);
}
}
if (isset($tgt['udp'])) {
foreach ($tgt['udp'] as $port_num => $port) {
if ($port['state'] != 'open') {
continue;
}
$udp = $db->get_UDP_Ports($port_num)[0];
if (!empty($port['banner'])) {
$udp->set_Banner($port['banner']);
}
$udp->set_IANA_Name($port['iana']);
//$udp->set_Notes("Found in scan file " . $scan->get_File_Name());
$tgt_obj->interfaces["{$ip}"]->add_UDP_Ports($udp);
}
}
$tgt_obj->set_ID($tgt_id = $db->save_Target($tgt_obj));
}
else { #Update
$db_tgt = $db->get_Target_Details($conf['ste'], $tgt_id)[0];
if (isset($tgt['tcp'])) {
foreach ($tgt['tcp'] as $port_num => $port) {
if ($port['state'] != 'open') {
continue;
}
$tcp = new tcp_ports(null, $port_num, $port['iana'], (isset($port['banner']) ? $port['banner'] : ""), "");
if (!isset($db_tgt->interfaces["{$ip}"])) {
$db_tgt->interfaces["{$ip}"] = new interfaces(null, null, null, $ip, null, $tgt['hostname'], $tgt['hostname'], (isset($tgt['description']) ? $tgt['description'] : ""));
}
if ($db_tgt->interfaces["{$ip}"]->is_TCP_Port_Open($port_num)) {
$db_tgt->interfaces["{$ip}"]->update_TCP_Port($tcp);
}
else {
$db_tgt->interfaces["{$ip}"]->add_TCP_Ports($tcp);
}
}
}
if (isset($tgt['udp'])) {
foreach ($tgt['udp'] as $port_num => $port) {
if ($port['state'] != 'open') {
continue;
}
$udp = new udp_ports(null, $port_num, $port['iana'], (isset($port['banner']) ? $port['banner'] : ""), "");
if (!isset($db_tgt->interfaces["{$ip}"])) {
$interface = new interfaces(null, $tgt_id, null, $ip, null, $tgt['hostname'], $tgt['hostname'], (isset($tgt['description']) ? $tgt['description'] : ""));
$db_tgt->interfaces["{$ip}"] = $interface;
}
if ($db_tgt->interfaces["{$ip}"]->is_UDP_Port_Open($port_num)) {
$db_tgt->interfaces["{$ip}"]->update_UDP_Port($udp);
}
else {
$db_tgt->interfaces["{$ip}"]->add_UDP_Ports($udp);
}
}
}
$db->save_Target($db_tgt);
}
$count++;
$db_tgt = $db->get_Target_Details($conf['ste'], $tgt_id)[0];
$hl = new host_list();
$hl->setTargetId($db_tgt->get_ID());
$hl->setTargetName($db_tgt->get_Name());
$hl->setTargetIp($tgt_ip);
$hl->setFindingCount(0);
$hl->setScanError(false);
$scan->add_Target_to_Host_List($hl);
$db->update_Running_Scan($base_name, ['name' => 'perc_comp', 'value' => ($count / count($target) * 100)]);
$db->update_Running_Scan($base_name, ['name' => 'last_host', 'value' => $db_tgt->get_Name()]);
}
$db->update_Scan_Host_List($scan);
$db->update_Running_Scan($base_name, ['name' => 'perc_comp', 'value' => 100, 'complete' => 1]);
if (!isset($cmd['debug'])) {
rename($cmd['f'], TMP . "/nmap/" . $base_name);
}
function usage()
{
print <<<EOO
Purpose: To import an NMap result file
Usage: php parse_nmap.php -s={ST&E ID} -f={NMap result file} -d={Document root} [--debug] [--help]
-s={ST&E ID} The ST&E ID this result file is being imported for
-f={NMap file} The result file to import (will import text, XML, and grepable files)
-d={Document Root} The document root of the web server
--debug Debugging output
--help This screen
EOO;
}

241
exec/parse_nvd_json_cve.php Normal file
View File

@ -0,0 +1,241 @@
<?php
/**
* File: parse_nvd_json_cve
* Author: Ryan Prather <ryan.prather@cyberperspectives.com>
* Purpose:
* Created: Dec 30, 2017
*
* Copyright 2017: Cyber Perspective, LLC, All rights reserved
* Released under the Apache v2.0 License
*
* See license.txt for details
*
* Change Log:
* - Dec 30, 2017 - File created
*/
include_once 'config.inc';
include_once 'helper.inc';
include_once 'database.inc';
include_once 'vendor/autoload.php';
use Monolog\Logger;
use Monolog\Handler\StreamHandler;
use Monolog\Formatter\LineFormatter;
ini_set('memory_limit', '2G');
$cmd = getopt("f:");
if (!isset($cmd['f']) || isset($cmd['h'])) {
die(usage());
}
$log_level = Logger::ERROR;
switch (LOG_LEVEL) {
case E_WARNING:
$log_level = Logger::WARNING;
break;
case E_NOTICE:
$log_level = Logger::NOTICE;
break;
case E_DEBUG:
$log_level = Logger::DEBUG;
break;
}
$stream = new StreamHandler("php://output", $log_level);
$stream->setFormatter(new LineFormatter("%datetime% %level_name% %message%", "H:i:s.u"));
$log = new Logger("nvd_cve");
$log->pushHandler($stream);
$log->pushHandler(new StreamHandler(LOG_PATH . "/nvd_cve.log", $log_level));
$db = new db();
$json = json_decode(file_get_contents($cmd['f']));
$existing_cves = [];
$db->help->select("cve_db", ['cve_id']);
$cves = $db->help->execute();
if (is_array($cves) && count($cves)) {
foreach ($cves as $cve) {
$existing_cves["{$cve['cve_id']}"] = 1;
}
}
print "Currently " . count($existing_cves) . " in DB" . PHP_EOL . "Parsing: " . count($json->CVE_Items) . " items" . PHP_EOL;
$new_cves = [];
$new_cve_refs = [];
$new_cve_web = [];
$sw_rows = [];
$new = 0;
$existing = 0;
$cve_fields = [
'cve_id', 'seq', 'status', 'phase', 'phase_date', 'desc'
];
$ref_fields = [
'cve_seq', 'source', 'url', 'val'
];
$web_fields = [
'cve_id', 'xml'
];
foreach ($json->CVE_Items as $cve) {
if (!isset($existing_cves["{$cve->cve->CVE_data_meta->ID}"])) {
$log->debug("Adding {$cve->cve->CVE_data_meta->ID}");
$new++;
$desc = [];
$status = null;
$phase = null;
$cpes = [];
$name = $cve->cve->CVE_data_meta->ID;
$type = $cve->cve->data_type;
$seq = $cve->cve->CVE_data_meta->ID;
$pd = new DateTime($cve->publishedDate);
$lmd = new DateTime($cve->lastModifiedDate);
if (is_array($cve->cve->description->description_data) && count($cve->cve->description->description_data)) {
foreach ($cve->cve->description->description_data as $d) {
$desc[] = $d->value;
}
}
$new_cves[] = [
$name, $seq, $status, $phase, $pd, implode(PHP_EOL, $desc)
];
if (is_array($cve->cve->references->reference_data) && count($cve->cve->references->reference_data)) {
foreach ($cve->cve->references->reference_data as $ref) {
$log->debug("Adding reference {$ref->url}");
$new_cve_refs[] = [
$name, null, $ref->url, null
];
}
}
if (is_array($cve->configurations->nodes) && count($cve->configurations->nodes)) {
foreach ($cve->configurations->nodes as $n) {
if (isset($n->cpe) && is_array($n->cpe) && count($n->cpe)) {
foreach ($n->cpe as $cpe) {
if (isset($cpe->cpe22Uri)) {
$cpes[] = $cpe->cpe22Uri;
}
elseif (isset($cpe->cpeMatchString)) {
$cpes[] = $cpe->cpeMatchString;
}
}
}
}
}
if (count($cpes)) {
$sw_ids = $db->get_Software_Ids($cpes);
if (is_array($sw_ids) && count($sw_ids) && isset($sw_ids['id'])) {
$sw_ids = [0 => $sw_ids];
}
if (is_array($sw_ids) && count($sw_ids) && isset($sw_ids[0])) {
foreach ($sw_ids as $sw) {
$sw_rows[] = [$name, $sw];
}
}
}
print "*";
}
else {
$existing++;
print ".";
}
if (($new + $existing) % 100 == 0) {
if (count($new_cves)) {
$db->help->extended_insert("cve_db", $cve_fields, $new_cves, true);
$db->help->execute();
}
if (count($new_cve_refs)) {
$db->help->extended_insert("cve_references", $ref_fields, $new_cve_refs, true);
$db->help->execute();
}
if (count($sw_rows)) {
$db->help->extended_insert("cve_sw_lookup", ['cve_id', 'sw_id'], $sw_rows, true);
$db->help->execute();
}
$new_cves = [];
$new_cve_refs = [];
$new_cve_web = [];
$sw_rows = [];
print "\t" . ($existing + $new) . " completed" . PHP_EOL;
$db->help->update("settings", ['meta_value' => number_format((($existing + $new) / count($json->CVE_Items)) * 100, 2)], [
[
'field' => 'meta_key',
'op' => '=',
'value' => 'nvd-cve-progress'
]
]);
$db->help->execute();
}
}
if (count($new_cves)) {
$db->help->extended_insert("cve_db", $cve_fields, $new_cves, true);
$db->help->execute();
}
if (count($new_cve_refs)) {
$db->help->extended_insert("cve_references", $ref_fields, $new_cve_refs, true);
$db->help->execute();
}
if (count($sw_rows)) {
$db->help->extended_insert("cve_sw_lookup", ['cve_id', 'sw_id'], $sw_rows, true);
$db->help->execute();
}
$db->help->update("settings", ['meta_value' => 100], [
[
'field' => 'meta_key',
'op' => '=',
'value' => 'nvd-cve-progress'
]
]);
$db->help->execute();
$db->help->update("settings", ['meta_value' => 100], [
[
'field' => 'meta_key',
'op' => IN,
'value' => ['cve-dl-progress', 'cve-progress']
]
]);
$db->help->execute();
$db->help->update("settings", ['meta_value' => new DateTime()], [
[
'field' => 'meta_key',
'op' => IN,
'value' => ['cve-load-date', 'nvd-cve-load-date']
]
]);
$db->help->execute();
unlink($cmd['f']);
print PHP_EOL;
function usage() {
print <<<EOF
Purpose: To import the National Vulnerability Database (NVD) CVE JSON files
Usage: php parse_nvd_json_cve.php -f={JSON file} [-h]
-f={JSON file} The CVE file to import
-h This screen
EOF;
}

42
exec/parse_proc_echecklist.php Normal file
View File

@ -0,0 +1,42 @@
<?php
/**
* File: parse_proc_echecklist.php
* Author: Ryan Prather
* Purpose: Script to import a procedural eChecklist
* Created: Feb 23, 2015
*
* Portions Copyright 2016: Cyber Perspectives, All rights reserved
* Released under the Apache v2.0 License
*
* Portions Copyright (c) 2012-2015, Salient Federal Solutions
* Portions Copyright (c) 2008-2011, Science Applications International Corporation (SAIC)
* Released under Modified BSD License
*
* See license.txt for details
*
* Change Log:
* - Feb 23, 2015 - File created
* - Sep 1, 2016 - Copyright and file purpose updated
*/
$cmd = getopt("s:f:", array("debug::", "help::"));
if (!isset($cmd['s']) || !isset($cmd['f']) || isset($cmd['help'])) {
usage();
exit;
}
function usage() {
print <<<EOO
Purpose: To import a procedural eChecklist file that is filled out
Usage: php parse_proc_echecklist.php -s={ST&E ID} -f={Procedural eChecklist File} [--debug] [--help]
-s={ST&E ID} The ST&E ID this result file is being imported for
-f={file} The file to import
--debug Debugging output
--help This screen
EOO;
}

139
exec/parse_procedurals.php Normal file
View File

@ -0,0 +1,139 @@
<?php
/**
* File: parse_procedurals.php
* Purpose: Script to populate procedural database tables from Excel file
* Author: Ryan Prather
* Created: Jan 5, 2015
*
* Portions Copyright 2016: Cyber Perspectives, All rights reserved
* Released under the Apache v2.0 License
*
* Portions Copyright (c) 2012-2015, Salient Federal Solutions
* Portions Copyright (c) 2008-2011, Science Applications International Corporation (SAIC)
* Released under Modified BSD License
*
* See license.txt for details
*
* Change Log:
* - Jan 5, 2015 - File created
* - Sep 1, 2016 - Copyright Updated, added file header, and
* converted to constants
*/
$cmd = getopt("", array('debug::', "help::"));
if (isset($cmd['help'])) {
usage();
exit;
}
include_once 'config.inc';
require_once 'PHPExcel.php';
include_once 'helper.inc';
$db = new mysqli(DB_SERVER, 'web', db::decrypt_pwd());
if ($db->connect_error) {
print $db->connect_error;
die;
}
$cacheMethod = PHPExcel_CachedObjectStorageFactory::cache_to_sqlite;
$cacheSettings = array(
'memoryCacheSize' => '512MB'
);
PHPExcel_Settings::setCacheStorageMethod($cacheMethod, $cacheSettings);
$Reader = PHPExcel_IOFactory::createReaderForFile("8500.2_IA_Controls_and_Validation_Procedures.xls");
$Reader->setReadDataOnly(true);
$objPHPExcel = $Reader->load("8500.2_IA_Controls_and_Validation_Procedures.xls");
if (false) {
$objPHPExcel = PHPExcel_IOFactory::load("8500.2_IA_Controls_and_Validation_Procedures.xls");
}
$wksht = $objPHPExcel->getSheetByName("All 8500.2 IA Controls");
$lastrow = $wksht->getHighestDataRow();
for ($row = 2; $row <= $lastrow; $row++) {
$ia_id = $wksht->getCell("A$row")->getValue();
$name = htmlentities($wksht->getCell("B$row")->getValue());
$sub = $wksht->getCell("C$row")->getValue();
$desc = htmlentities($wksht->getCell("D$row")->getValue());
$tvcm = htmlentities($wksht->getCell("E$row")->getValue());
$gen_guide = htmlentities($wksht->getCell("F$row")->getValue());
$sys_spec = htmlentities($wksht->getCell("G$row")->getValue());
$impact = strtolower($wksht->getCell("H$row")->getValue());
$sql = "REPLACE INTO sagacity.proc_ia_controls (control_id, `name`, subject_area, description, threat_vul_cm, gen_imp_guide, guide_resource, impact) VALUES (?, ?, ?, ?, ?, ?, ?, ?)";
if (!$sth = $db->prepare($sql)) {
error_log($db->error);
continue;
}
$sth->bind_param('ssssssss', $ia_id, $name, $sub, $desc, $tvcm, $gen_guide, $sys_spec, $impact);
$sth->execute();
$sth->close();
$sql = "REPLACE INTO sagacity.proc_level_type (proc_control, `type`, level, class) VALUES (?, ?, ?, ?)";
if (!$sth = $db->prepare($sql)) {
error_log($db->error);
continue;
}
$control_type = 'diacap';
for ($idx = 8; ($col = PHPExcel_Cell::stringFromColumnIndex($idx)) < 'R'; $idx++) {
$val = $wksht->getCell($col . $row)->getValue();
$head = $wksht->getCell($col . "1")->getValue();
if ($val == 'Y') {
$type = explode(' - ', $head);
$lvl = substr_count($type[0], 'I');
switch ($type[1]) {
case 'CL':
$class = 'cl';
break;
case 'S':
$class = 'sen';
break;
case 'P':
$class = 'pub';
break;
}
$sth->bind_param('ssss', $ia_id, $control_type, $lvl, $class);
$sth->execute();
}
}
$sth->close();
}
$wksht = $objPHPExcel->getSheetByName("All Validation Procedures");
$lastrow = $wksht->getHighestDataRow();
for ($row = 2; $row <= $lastrow; $row++) {
$parent_id = $wksht->getCell("A$row")->getValue();
$sub_id = $wksht->getCell("B$row")->getValue();
$name = htmlentities($wksht->getCell("C$row")->getValue());
$obj = htmlentities($wksht->getCell("D$row")->getValue());
$prep = htmlentities($wksht->getCell("E$row")->getValue());
$script = htmlentities($wksht->getCell("F$row")->getValue());
$exp = htmlentities($wksht->getCell("G$row")->getValue());
$sql = "REPLACE INTO sagacity.proc_ia_sub_controls (sub_control_id, parent_control_id, `name`, objective, prep, `script`, exp_result) VALUES (?, ?, ?, ?, ?, ?, ?)";
if (!$sth = $db->prepare($sql)) {
error_log($db->error);
continue;
}
$sth->bind_param('sssssss', $sub_id, $parent_id, $name, $obj, $prep, $script, $exp);
$sth->execute();
$sth->close();
}
function usage() {
print <<<EOO
Purpose: To import the DIACAP IA control library
Usage: php parse_procedurals.php [--help]
--help This screen
EOO;
}

675
exec/parse_scc_xccdf.php Normal file
View File

@ -0,0 +1,675 @@
<?php
/**
* File: parse_scc_xccdf.php
* Author: Ryan Prather
* Purpose: Background script to parse SCC XCCDF result files
* Created: Feb 26, 2014
*
* Portions Copyright 2016-2017: Cyber Perspectives, LLC, All rights reserved
* Released under the Apache v2.0 License
*
* Portions Copyright (c) 2012-2015, Salient Federal Solutions
* Portions Copyright (c) 2008-2011, Science Applications International Corporation (SAIC)
* Released under Modified BSD License
*
* See license.txt for details
*
* Change Log:
* - Feb 26, 2014 - File created
* - Jun 3, 2015 - Copyright updated, added CWD parameter, and
* updated function calls after class merger
* - Oct 24, 2016 - Converted XMLParser to scan_xml_parser class
* - Jan 30, 2017 - Updated to use parse_config.ini file
* - May 13, 2017 - Fixed error when trying to delete a USGCB scan file (not supported)
* - Oct 23, 2017 - Fixed error of finding statuses being overwritten
*/
$cmd = getopt("f:", ['debug::', 'help::']);
if (!isset($cmd['f']) || isset($cmd['help'])) {
die(usage());
}
$conf = parse_ini_file("parse_config.ini");
if (!$conf) {
die("Could not find parse_config.ini configuration file");
}
chdir($conf['doc_root']);
set_time_limit(0);
include_once 'config.inc';
include_once 'xml_parser.inc';
include_once 'database.inc';
include_once 'helper.inc';
chdir(TMP);
$db = new db();
$base_name = basename($cmd['f']);
$host_list = array();
$err = new Sagacity_Error($cmd['f']);
if (!file_exists($cmd['f'])) {
$db->update_Running_Scan($base_name, ['name' => 'status', 'value' => 'ERROR']);
$err->script_log("File not found", E_ERROR);
}
elseif (preg_match('/.*Results\_iavm\_(2009|2010)|Results\_USGCB/i', $cmd['f'])) {
$scan = $db->get_ScanData($conf['ste'], $cmd['f']);
if (is_array($scan) && count($scan) && isset($scan[0]) && is_a($scan[0], 'scan')) {
$db->delete_Scan($scan[0]->get_ID(), false);
}
$err->script_log("Cannot parse these types of files", E_ERROR);
}
class scc_parser extends scan_xml_parser {
var $values;
var $value_id;
var $getvalue = false;
var $groups;
var $group_id;
var $vms_id;
var $vms = null;
var $sv_rule;
var $tgt;
var $tag;
var $int_count = 0;
var $found_rule = false;
/**
* Constructor
*
* @param int $ste_id_in
* @param string $fname_in
*/
public function __construct($ste_id_in, $fname_in) {
$this->values = array();
$this->groups = array();
$this->tag = array();
parent::__construct($this, $ste_id_in, $fname_in);
$this->db->update_Running_Scan($this->scan->get_File_Name(), ['name' => 'pid', 'value' => getmypid()]);
}
/**
* Function to parse \cdf:Benchmark\cdf:Value tag
*
* @param array $attrs
*/
public function cdf_Benchmark_cdf_Value($attrs) {
$this->values[$attrs['id']] = null;
$this->value_id = $attrs['id'];
}
/**
* Function to parse \cdf:Benchmark\cdf:Value\cdf:value tag
*
* @param array $attrs
*/
public function cdf_Benchmark_cdf_Value_cdf_value($attrs) {
if (!isset($attrs['selector'])) {
$this->getvalue = true;
}
else {
$this->getvalue = false;
}
}
/**
* Function to parse \cdf:Benchmark\cdf:value\cdf:value character data
*
* @param string $data
*/
public function cdf_Benchmark_cdf_Value_cdf_value_data($data) {
if ($this->getvalue) {
$this->values[$this->value_id] = $data;
}
}
/**
* Function to parse \cdf:Benchmark\cdf:Group tag
*
* @param array $attrs
*/
public function cdf_Benchmark_cdf_Group($attrs) {
$this->vms = $this->db->get_GoldDisk($attrs['id']);
if (is_array($this->vms) && count($this->vms) && isset($this->vms[0]) && is_a($this->vms[0], 'golddisk')) {
$this->group_id = $this->vms[0]->get_PDI_ID();
}
else {
$this->group_id = $attrs['id'];
$this->vms = null;
}
$this->vms_id = $attrs['id'];
$this->groups[$this->group_id] = array();
$this->found_rule = false;
}
/**
* Function to parse \cdf:Benchmark\cdf:Group\cdf:Rule tag
*
* @param array $attrs
*/
public function cdf_Benchmark_cdf_Group_cdf_Rule($attrs) {
$sv_rule = $this->db->get_SV_Rule(null, $attrs['id']);
if (is_array($sv_rule) && count($sv_rule) && isset($sv_rule[0]) && is_a($sv_rule[0], 'sv_rule')) {
$this->found_rule = true;
$this->sv_rule = $sv_rule[0];
unset($this->groups[$this->group_id]);
$this->group_id = $this->sv_rule->get_PDI_ID();
$this->groups[$this->group_id] = [
'sv_rule' => $this->sv_rule,
'stig' => null,
'version' => null,
'title' => null,
'vms_id' => $this->vms_id,
'oval_id' => null,
'val_id' => null,
'value' => null,
'cce' => null,
'fix' => null,
'desc' => null,
'status' => "Not Reviewed",
'cat' => 2
];
}
else {
return;
}
$stig = $this->db->get_STIG_By_PDI($this->sv_rule->get_PDI_ID());
if (is_a($stig, 'stig')) {
$this->groups[$this->group_id]['stig'] = $stig;
$this->groups[$this->group_id]['version'] = $stig->get_ID();
}
}
/**
* Function to parse \cdf:Benchmark\cdf:Group\cdf:Rule\cdf:version character data (STIG id)
*
* @param string $data
*/
public function cdf_Benchmark_cdf_Group_cdf_Rule_cdf_version_data($data) {
$stig = $this->db->get_Stig($data);
if (is_array($stig) && count($stig) && isset($stig[0]) && is_a($stig[0], 'stig')) {
$this->found_rule = true;
$stig = $stig[0];
unset($this->groups[$this->group_id]);
$this->group_id = $stig->get_PDI_ID();
$this->groups[$this->group_id] = [
'sv_rule' => (is_a($this->sv_rule, 'sv_rule') ? $this->sv_rule : null),
'stig' => $stig,
'version' => $stig->get_ID(),
'title' => null,
'vms_id' => $this->vms_id,
'oval_id' => null,
'val_id' => null,
'value' => null,
'cce' => null,
'fix' => null,
'desc' => null,
'status' => "Not Reviewed",
'cat' => 2
];
}
}
/**
* Function to parse \cdf:Benchmark\cdf:Group\cdf:Rule\cdf:title character data (short title)
*
* @param string $data
*/
public function cdf_Benchmark_cdf_Group_cdf_Rule_cdf_title_data($data) {
if (empty($this->groups[$this->group_id]['title'])) {
$this->groups[$this->group_id]['title'] = $data;
}
else {
//error_log(print_r($this->group_id, true));
}
}
/**
* Function to parse \cdf:Benchmark\cdf:Group\cdf:Rule\cdf:description character data (description)
*
* @param string $data
*/
public function cdf_Benchmark_cdf_Group_cdf_Rule_cdf_description_data($data) {
if (!isset($this->groups[$this->group_id])) {
$this->groups[$this->group_id] = array();
}
if (isset($this->groups[$this->group_id]['desc'])) {
$this->groups[$this->group_id]['desc'] .= $data;
}
else {
$this->groups[$this->group_id]['desc'] = $data;
}
}
/**
* Function to parse \cdf:Benchmark\cdf:Group\cdf:Rule\cdf:ident character data (CCI,CCE,etc)
*
* @param string $data
*/
public function cdf_Benchmark_cdf_Group_cdf_Rule_cdf_ident_data($data) {
if (empty($this->groups[$this->group_id]['cce']) && preg_match("/CCE/", $data)) {
$this->groups[$this->group_id]['cce'] = $data;
}
}
/**
* Function to parse \cdf:Benchmark\cdf:Group\cdf:Rule\cdf:fixtext character data
*
* @param string $data
*/
public function cdf_Benchmark_cdf_Group_cdf_Rule_cdf_fixtext_data($data) {
if (empty($this->groups[$this->group_id]['fix'])) {
$this->groups[$this->group_id]['fix'] = htmlentities($data);
}
}
/**
* Function to parse \cdf:Benchmark\cdf:Group\cdf:Rule\cdf:check\cdf:check-export tag
*
* @param array $attrs
*/
public function cdf_Benchmark_cdf_Group_cdf_Rule_cdf_check_cdf_check_export($attrs) {
if (empty($this->groups[$this->group_id]['val_id'])) {
$this->groups[$this->group_id]['val_id'] = $attrs['value-id'];
$this->groups[$this->group_id]['value'] = $this->values[$attrs['value-id']];
$this->groups[$this->group_id]['oval_id'] = $attrs['export-name'];
}
}
/**
* Function to parse \cdf:Benchmark\cdf:Group end tag and store content parsed from previous functions
*/
public function cdf_Benchmark_cdf_Group_end() {
if (!$this->found_rule) {
$this->log->script_log("Rule tag was not present for " . $this->group_id);
unset($this->groups[$this->group_id]);
return;
}
if (empty($this->groups[$this->group_id]['stig'])) {
$ia_controls = array();
$this->log->script_log("STIG ID " . $this->groups[$this->group_id]['version'] . " is not in the database, adding", E_WARNING);
$pdi = new pdi(null, '', 'NOW');
$pdi->set_Short_Title($this->groups[$this->group_id]['title']);
$pdi->set_Group_Title($this->groups[$this->group_id]['title']);
$pdi->set_Description($this->groups[$this->group_id]['desc']);
$pdi_id = $this->db->save_PDI($pdi);
$stig = new stig($pdi_id, $this->groups[$this->group_id]['version'], $this->groups[$this->group_id]['title']);
$this->db->add_Stig($stig);
$this->groups[$this->group_id]['stig'] = $stig;
if (!empty($this->groups[$this->group_id]['desc'])) {
$match = array();
if (preg_match("/\<IAControls\>(.*)\<\/IAControls\>/", $this->groups[$this->group_id]['desc'], $match)) {
$ias = explode(", ", $match[1]);
if (is_array($ias) && count($ias)) {
foreach ($ias as $ia) {
$ia_controls[] = new ia_control($pdi_id, substr($ia, 0, 4), substr($ia, -1));
}
}
else {
$ia_controls[] = new ia_control($pdi_id, "ECSC", 1);
}
}
}
else {
$ia_controls[] = new ia_control($pdi_id, 'ECSC', 1);
}
$this->db->save_IA_Control($ia_controls);
}
if (empty($this->vms)) {
$this->vms = new golddisk($this->groups[$this->group_id]['stig']->get_PDI_ID(), $this->vms_id, $this->groups[$this->group_id]['title']);
$this->db->save_GoldDisk($this->vms);
}
}
/**
* Function to parse \cdf:Benchmark\cdf:TestResult\cdf:target-facts\cdf:fact tag
*
* @param array $attrs
*/
public function cdf_Benchmark_cdf_TestResult_cdf_target_facts_cdf_fact($attrs) {
$tmp = explode(":", $attrs['name']);
$this->tag_id = end($tmp);
if (isset($this->tag[$this->tag_id])) {
if ($this->tag_id == 'interface_name') {
$this->int_count++;
}
$this->tag_id .= $this->int_count;
}
}
/**
* Function to parse \cdf:Benchmark\cdf:TestResult\cdf:target-facts\cdf:fact character data
*
* @param string $data
*/
public function cdf_Benchmark_cdf_TestResult_cdf_target_facts_cdf_fact_data($data) {
$this->tag[$this->tag_id] = str_replace("\n", "", $data);
}
/**
* Function to parse \cdf:Benchmark\cdf:TestResult\cdf:target-facts end tag and store results
*/
public function cdf_Benchmark_cdf_TestResult_cdf_target_facts_end() {
//error_log(print_r($this->tag, true));
$host_name = $this->tag['host_name'];
if (preg_match("/\./", $host_name)) {
$host_name = preg_replace("/^([^\.]+)\./i", "$1", $host_name);
}
if (!($tgt_id = $this->db->check_Target($this->ste_id, $host_name))) {
$this->log->script_log("Creating new target with hostname $host_name", E_DEBUG);
$os = array();
if (isset($this->tag['os_name']) && isset($this->tag['os_version']) && is_numeric($this->tag['os_version'])) {
$this->tag['os_name'] .= " {$this->tag['os_version']}";
}
if (isset($this->tag['os_name'])) {
$os_regex = $this->db->get_Regex_Array("os");
$os = software::identify_Software($os_regex, $this->tag['os_name']);
$os = $this->db->get_Software($os);
}
$this->log->script_log("Identified OS " . print_r($os, true), E_DEBUG);
if (is_array($os) && count($os) && isset($os[0]) && is_a($os[0], 'software')) {
$os = $os[0];
}
else {
$os = $this->db->get_Software("cpe:/o:generic:generic:-")[0];
}
if (!is_a($os, 'software')) {
$this->log->script_log("Failed to identify the OS", E_ERROR);
}
$tgt = new target($host_name);
$tgt->set_STE_ID($this->ste_id);
$tgt->set_Notes("New target found by SCC");
if (is_a($os, "software")) {
$this->log->script_log("Assigning OS {$os->get_CPE()}", E_DEBUG);
$tgt->set_OS_ID($os->get_ID());
$tgt->set_OS_String($os->get_Shortened_SW_String());
}
$tgt_id = $this->db->save_Target($tgt);
}
$this->tgt = $this->db->get_Target_Details($this->ste_id, $tgt_id)[0];
$int_keys = preg_grep("/interface_name/", array_keys($this->tag));
$match = array();
foreach ($int_keys as $key) {
$idx = '';
if (preg_match("/interface_name(\d+)/", $key, $match)) {
$idx = $match[1];
}
if (isset($this->tag["ipv4$idx"])) {
$ip = explode(",", $this->tag["ipv4$idx"]);
$ipv4 = null;
$ipv6 = null;
if (is_array($ip) && count($ip) == 1) {
if (preg_match("/\d+\./", $ip[0])) {
$ipv4 = $ip[0];
}
elseif (preg_match("/[a-f0-9]+/", $ip[0])) {
$ipv6 = $ip[0];
}
}
elseif (is_array($ip) && count($ip) == 2) {
$ipv4 = $ip[0];
$ipv6 = $ip[1];
}
if ($ipv4) {
$int = new interfaces(null, $tgt_id, $this->tag["interface_name$idx"], $ipv4, null, (isset($this->tag['host_name']) ? $this->tag['host_name'] : ""), (isset($this->tag['fqdn']) ? $this->tag['fqdn'] : ""), null);
if (isset($this->tag["mac$idx"])) {
$int->set_MAC($this->tag["mac$idx"]);
}
$this->db->save_Interface($int);
}
if ($ipv6) {
$int = new interfaces(null, $tgt_id, $this->tag["interface_name$idx"], null, $ipv6, (isset($this->tag['host_name']) ? $this->tag['host_name'] : ""), (isset($this->tag['fqdn']) ? $this->tag['fqdn'] : ""), null);
if (isset($this->tag["mac$idx"])) {
$int->set_MAC($this->tag["mac$idx"]);
}
$this->db->save_Interface($int);
}
}
}
}
/**
* Function to parse \cdf:Benchmark\cdf:TestResult\cdf:platform tag (stores CPE)
*
* @param array $attrs
*/
public function cdf_Benchmark_cdf_TestResult_cdf_platform($attrs) {
if (isset($attrs['idref']) && substr($attrs['idref'], 0, 3) == 'cpe') {
$cpe = $attrs['idref'];
$sw = $this->db->get_Software($cpe);
if (is_array($sw) && count($sw) && is_a($this->tgt, 'target')) {
$sw = $sw[0];
if ($sw->is_OS() && $this->tgt->get_OS_ID() != $sw->get_ID()) {
$this->log->script_log("Update OS " . $sw->get_CPE());
$this->tgt->set_OS_ID($sw->get_ID());
$this->tgt->set_OS_String($sw->get_Shortened_SW_String());
}
elseif (!$sw->is_OS() && !in_array($sw, $this->tgt->software)) {
$this->log->script_log("Assigning software " . $sw->get_CPE());
$this->tgt->software[] = $sw;
}
}
$this->db->save_Target($this->tgt);
}
}
/**
* Function to parse \cdf:Benchmark\cdf:TestResult\cdf:rule-result tag
*
* @param array $attrs
*/
public function cdf_Benchmark_cdf_TestResult_cdf_rule_result($attrs) {
$stig = $this->db->get_Stig($attrs['version']);
$sv_rule = $this->db->get_SV_Rule(null, $attrs['idref']);
$this->log->script_log("Version: {$attrs['version']}", E_DEBUG);
$this->log->script_log("STIG data: " . print_r($stig, true), E_DEBUG);
if (is_array($stig) && count($stig) && isset($stig[0]) && is_a($stig[0], 'stig')) {
$stig = $stig[0];
$this->group_id = $stig->get_PDI_ID();
}
elseif (is_array($sv_rule) && count($sv_rule) && isset($sv_rule[0]) && is_a($sv_rule[0], 'sv_rule') && !$this->group_id) {
$sv_rule = $sv_rule[0];
$this->group_id = $sv_rule->get_PDI_ID();
}
else {
$this->log->script_log("Cannot find PDI ID (" . $attrs['version'] . "/" . $attrs['idref'] . ") CREATING", E_WARNING);
$this->group_id = null;
return;
/*
$level = 1;
if ($attrs['severity'] == 'medium') {
$level = 2;
}
elseif ($attrs['severity'] == 'low') {
$level = 3;
}
$pdi = new pdi(null, $level, new DateTime);
$pdi_id = $this->db->save_PDI($pdi);
$this->group_id = $pdi_id;
if (!empty($attrs['version'])) {
$stig = new stig($pdi_id, $attrs['version'], null, null);
$this->db->add_Stig($stig);
}
if (!empty($attrs['idref'])) {
$sv_rule = new sv_rule($pdi_id, $attrs['idref']);
$this->db->save_SV_Rule($sv_rule);
}
return;
*/
}
if (empty($this->groups[$this->group_id]['sv_rule']) && is_a($sv_rule, "sv_rule")) {
$this->groups[$this->group_id]['sv_rule'] = $sv_rule;
}
if (empty($this->groups[$this->group_id]['stig']) && is_a($stig, "stig")) {
$this->groups[$this->group_id]['stig'] = $stig;
}
if (isset($attrs['severity'])) {
switch ($attrs['severity']) {
case 'low':
$this->groups[$this->group_id]['cat'] = 3;
break;
case 'high':
$this->groups[$this->group_id]['cat'] = 1;
break;
}
}
}
/**
* Function to parse \cdf:Benchmark\cdf:TestResult\cdf:rule-result\cdf:result character data
*
* @param string $data
*/
public function cdf_Benchmark_cdf_TestResult_cdf_rule_result_cdf_result_data($data) {
if (preg_match("/pass|true/i", $data)) {
$this->groups[$this->group_id]['status'] = "Not a Finding";
}
elseif (preg_match("/fail|false/i", $data)) {
$this->groups[$this->group_id]['status'] = "Open";
}
$this->log->script_log("{$this->group_id} {$this->groups[$this->group_id]['status']}", E_DEBUG);
}
/**
* Function to parse \cdf:Benchmark\cdf:TestResult\cdf:rule-result\cdf:ident character data
*
* @param string $data
*/
public function cdf_Benchmark_cdf_TestResult_cdf_rule_result_cdf_ident_data($data) {
}
/**
* Function to parse \cdf:Benchmark\cdf:TestResult end tag and store all results
*/
public function cdf_Benchmark_cdf_TestResult_end() {
$new_findings = [];
$update_findings = [];
foreach ($this->groups as $pdi_id => $group) {
if (!empty($group['val_id'])) {
$note = "(SCC) " . $group['val_id'] . "\nRequired: " . $group['value'] . "\nActual: " . $this->values[$group['val_id']];
}
else {
$note = "(SCC) ";
}
if (isset($group['stig']) && is_a($group['stig'], 'stig')) {
$ref = $group['stig'];
}
elseif (!empty($group['vms_id'])) {
$vms = $this->db->get_GoldDisk($group['vms_id']);
if (is_array($vms) && count($vms) && isset($vms[0]) && is_a($vms[0], 'golddisk')) {
$ref = $vms[0];
}
}
elseif (isset($group['sv_rule']) && is_a($group['sv_rule'], 'sv_rule')) {
$ref = $group['sv_rule'];
}
else {
$this->log->script_log("Error finding reference to search for PDI $pdi_id\n" . print_r($group, true), E_WARNING);
continue;
}
$existing_finding = $this->db->get_Finding($this->tgt, $ref);
if (is_array($existing_finding) && count($existing_finding) && isset($existing_finding[0])) {
$finding = $existing_finding[0];
$finding->set_Finding_Status_By_String(
$finding->get_Deconflicted_Status($group['status'])
);
$finding->prepend_Notes($note);
$update_findings[$finding->get_PDI_ID()] = $finding;
}
else {
$new_findings[$pdi_id] = new finding(null, $this->tgt->get_ID(), $pdi_id, $this->scan->get_ID(), $group['status'], $note, finding::NC, null, 1);
}
}
$this->db->add_Findings_By_Target($update_findings, $new_findings);
$hl = new host_list();
$hl->setTargetId($this->tgt->get_ID());
$hl->setTargetName($this->tgt->get_Name());
$hl->setFindingCount(count($new_findings) + count($update_findings));
$hl->setScanError(false);
$this->scan->add_Target_to_Host_List($hl);
}
}
$xml = new scc_parser($conf['ste'], $cmd['f']);
$xml->debug = (isset($cmd['debug']) ? true : false);
$xml->parse();
if (!$xml->debug) {
rename($cmd['f'], TMP . "/scc/" . $base_name);
}
$db->update_Running_Scan($base_name, ["name" => "perc_comp", "value" => 100, "complete" => 1]);
function usage() {
print <<<EOO
Purpose: To import an XCCDF result file from Security Compliance Checker 3.1+
Usage: php parse_scc_xccdf.php -s={ST&E ID} -f={XCCDF result file} [--debug] [--help]
-s={ST&E ID} The ST&E ID this result file is being imported for
-f={XCCDF file} The result file to import (will not import oval, dictionary, or other files)
--debug Debugging output
--help This screen
EOO;
}

709
exec/parse_stig.php Normal file
View File

@ -0,0 +1,709 @@
<?php
/**
* File: parse_stig.php
* Author: Ryan Prather
* Purpose: To parse a STIG file
* Created: Jul 9, 2014
*
* Portions Copyright 2016-2017: Cyber Perspectives, LLC, All rights reserved
* Released under the Apache v2.0 License
*
* Portions Copyright (c) 2012-2015, Salient Federal Solutions
* Portions Copyright (c) 2008-2011, Science Applications International Corporation (SAIC)
* Released under Modified BSD License
*
* See license.txt for details
*
* Change Log:
* - Jul 9, 2014 - File created
* - Jun 3, 2015 - Copyright Updated and converted to constants
* - Oct 24, 2016 - Updated E_DEBUG constant
* - Nov 7, 2016 - Make sure that /reference/stigs directory is present
* - Feb 15, 2017 - Formatting and migrated some SQL to db_helper
* - Mar 3, 2017 - Fixed a few bugs to get the code processing the latest STIGs
* - Apr 5, 2017 - Fixed bug parsing software correctly
* - Jun 27, 2017 - Fixed bug when parsing VVoIP file and cleanup
* - Jun 29, 2017 - Refactored a few lines of code and fixed bug #262. Also, changed default CCI to 002613 if there is no other control link
* - Jul 13, 2017 - Fixed bug #273/4
* - Jul 23, 2017 - MAS Added comments
* - Aug 28, 2017 - Added die for draft stigs
* - Dec 27, 2017 - Added up date for load date
*/
$cmd = getopt("f:", ['debug::', 'ia_reset::', 'draft::', 'help::']);
if (!isset($cmd['f']) || isset($cmd['help'])) {
die(usage());
}
set_time_limit(0);
include_once 'config.inc';
include_once 'database.inc';
include_once 'helper.inc';
require_once 'vendor/autoload.php';
use Monolog\Logger;
use Monolog\Handler\StreamHandler;
use Monolog\Formatter\LineFormatter;
$stream = new StreamHandler("php://output", Logger::INFO);
$stream->setFormatter(new LineFormatter("%datetime% %level_name% %message%", "H:i:s.u"));
/*
$log = new Logger("parse_stig");
$log->pushHandler(new StreamHandler(LOG_PATH . "/" . basename($cmd['f']) . ".log", LOG_LEVEL));
$log->pushHandler($stream);
*/
chdir(DOC_ROOT . "/exec");
// Capture start time for performance metrics
$start = new DateTime();
// Check to make sure file argument exists and is an XCCDF file
if (!file_exists($cmd['f'])) {
Sagacity_Error::err_handler("XML file not found {$cmd['f']}", E_ERROR);
}
elseif (strpos(strtolower($cmd['f']), "xccdf") === false) {
Sagacity_Error::err_handler("Only compatible with XCCDF file formats", E_ERROR);
}
// Verify our STIG reference directory exists
check_path(DOC_ROOT . "/reference/stigs");
// open db connection
$db = new db();
$content = str_replace(["’", "–", "“", "â€"], ["'", "-", '"', '"'], file_get_contents($cmd['f']));
file_put_contents($cmd['f'], $content);
// open xml file
$base_name = basename($cmd['f']);
$perc_comp = 0;
$new_count = 0;
$updated_count = 0;
$log = new Sagacity_Error($base_name);
// Create and update parse job details
$db->help->select_count("sagacity.catalog_scripts", [
[
'field' => 'file_name',
'op' => '=',
'value' => $base_name
]
]);
$exists = $db->help->execute();
if (!$exists) {
$db->add_Catalog_Script($base_name);
}
$db->update_Catalog_Script($base_name, ['name' => 'pid', 'value' => getmypid()]);
$db->help->update("sagacity.settings", ['meta_value' => new DateTime()], [
[
'field' => 'meta_key',
'op' => '=',
'value' => 'stig-load-date'
]
]);
$db->help->execute();
$tmp = $db->get_Stig();
$stigs = [];
foreach ($tmp as $s) {
$stigs["{$s->get_ID()}"] = $s;
}
print "Currently " . count($stigs) . " in the DB" . PHP_EOL;
// Load XML into DOMDocument
$xml = new DOMDocument();
if (!$xml->load($cmd['f'])) {
$log->script_log("Error opening file", E_ERROR);
}
// Get regexes used to assess the STIG for known applicable software products
$regex_arr = $db->get_Regex_Array("checklist");
if (is_array($regex_arr) && !count($regex_arr)) {
die("There are no regular expressions to detect checklist software");
}
$csv_file = substr($cmd['f'], 0, -3) . "csv";
$csv = fopen($csv_file, "w");
fputcsv($csv, ["STIG_ID", "VMS_ID", "CAT", "IA_Controls", "Short_Title", "Status", "Notes", "Check_Contents", "SV_Rule_ID", "Oval_ID"]);
// get checklist data
$checklist = [];
$checklist['id'] = str_replace("-", '.', getValue($xml, '@id'));
$checklist['status'] = getValue($xml, "/x:Benchmark/x:status");
// Skip draft STIGs if debug flag is not set. @Ryan: Shouldn't this be checking the draft flag instead of debug?
if (!isset($cmd['draft'])) {
if (strtolower($checklist['status']) == 'draft') {
$db->update_Catalog_Script($base_name, ["name" => "status", "value" => "SKIPPED"]);
fclose($csv);
unset($xml);
unlink($cmd['f']);
$log->script_log("Skipping since this is a draft STIG" . PHP_EOL, E_NOTICE);
die();
}
}
$checklist['status_date'] = getValue($xml, "/x:Benchmark/x:status", null, true)->item(0)->getAttribute('date');
$checklist['status_date'] = new DateTime($checklist['status_date']);
$checklist['ver'] = getValue($xml, "/x:Benchmark/x:version");
$checklist['plain_text'] = getValue($xml, "/x:Benchmark/x:plain-text");
// Attempt to identify the software product referenced by the STIG
$checklist['software'] = software::identify_Software($regex_arr, $checklist['id']);
if (isset($cmd['debug'])) {
$log->script_log(print_r($checklist['software'], true), E_DEBUG);
}
// If no matching software is found, default to "generic"
if (!count($checklist['software'])) {
$log->script_log("Could not identify software, setting as Generic/Generic", E_NOTICE);
$checklist['software'][] = [
'man' => 'Generic',
'name' => 'Generic',
'ver' => '-',
'sp' => null,
'type' => false
];
}
// Convert identified software to a software object.
$sw_arr = software::toSoftwareFromArray($checklist['software']);
if (isset($cmd['debug'])) {
$log->script_log(print_r($sw_arr, true), E_DEBUG);
}
foreach ($sw_arr as $key => $sw) {
do {
$cpe = "cpe:/" . ($sw->is_OS() ? "o" : "a") . ":{$sw->get_Man()}:{$sw->get_Name()}" .
($sw->get_Version() != '-' ? ":{$sw->get_Version()}" : "");
$cpe = str_replace(
[" ", "(", ")"], ["_", "%28", "%29"], strtolower($cpe)
);
$db_sw = $db->get_Software($cpe);
if (!count($db_sw) && !count($checklist['software'])) {
$sw->reduce_CPE();
}
elseif (is_array($db_sw) && count($db_sw) == 1 && $db_sw[0]->get_Version() == '-' && !preg_match("/generic/", $sw->get_CPE())) {
$checklist['software'] = array_merge($checklist['software'], $db_sw);
$sw->reduce_CPE();
$db_sw = [];
}
else {
break;
}
if (isset($cmd['debug'])) {
$log->script_log("$cpe found " . count($db_sw), E_DEBUG);
}
}
while (!count($db_sw));
$checklist['software'] = array_merge($checklist['software'], $db_sw);
}
foreach ($checklist['software'] as $key => $sw) {
if (!is_a($sw, 'software')) {
unset($checklist['software'][$key]);
}
}
$match = [];
if (preg_match('/Release: (\d+\.\d+|\d+)/', $checklist['plain_text'], $match)) {
$checklist['rel'] = $match[1];
}
else {
$checklist['rel'] = '';
}
// Get the date of the benchmark in the 'plain-text' element or set to 'status-date' if match fails
if (preg_match('/Benchmark Date: (.*)$/', $checklist['plain_text'], $match)) {
$checklist['benchmark_date'] = new DateTime($match[1]);
}
else {
$checklist['benchmark_date'] = $checklist['status_date'];
}
// Get the STIG title and convert common acronyms (STIG, IAVM, and SRG)
$checklist['title'] = getValue($xml, "/x:Benchmark/x:title");
$checklist['title'] = preg_replace("/Security Technical Implementation Guide/i", "STIG", $checklist['title']);
$checklist['title'] = preg_replace("/STIG \(?STIG\)?/i", "STIG", $checklist['title']);
$checklist['title'] = preg_replace("/Information Assurance Vulnerabilities/i", "IAVM", $checklist['title']);
$checklist['title'] = preg_replace("/Security Requirements Guide/i", "SRG", $checklist['title']);
$checklist['desc'] = getValue($xml, "/x:Benchmark/x:description");
// Set checklist type to benchmark, iavm, policy, or manual based on file name
$checklist['type'] = 'benchmark';
if (preg_match('/IAVM/i', $base_name)) {
$checklist['type'] = 'iavm';
}
elseif (preg_match('/policy|srg/i', $base_name)) {
$checklist['type'] = 'policy';
}
elseif (preg_match('/manual/i', $base_name)) {
$checklist['type'] = 'manual';
}
// Capture version release in filename as sometimes it doesn't match the plain_text element
if (preg_match('/V(\d+)R/', $base_name, $match)) {
$checklist['file_ver'] = $match[1];
}
else {
$checklist['file_ver'] = 0;
}
if (preg_match('/V\d+R(\d+|\d+\.\d+)/', $base_name, $match)) {
$checklist['file_rel'] = $match[1];
}
else {
$checklist['file_rel'] = 0;
}
// Assign ver and rel to whichever value is greater (filename or xml)
$checklist['ver'] = $checklist['file_ver'] > $checklist['ver'] ? $checklist['file_ver'] : $checklist['ver'];
$checklist['rel'] = $checklist['file_rel'] > $checklist['rel'] ? $checklist['file_rel'] : $checklist['rel'];
if (isset($cmd['debug'])) {
$log->script_log("Checklist:" . PHP_EOL . print_r($checklist, true), E_DEBUG);
}
// Query the db to see if the checklist is already in there
$db->help->select("sagacity.checklist", ['id'], [
[
'field' => 'checklist_id',
'op' => '=',
'value' => $checklist['id']
],
[
'field' => 'release',
'op' => '=',
'value' => $checklist['rel'],
'sql_op' => 'AND'
],
[
'field' => 'ver',
'op' => '=',
'value' => $checklist['ver'],
'sql_op' => 'AND'
],
[
'field' => 'type',
'op' => '=',
'value' => $checklist['type'],
'sql_op' => 'AND'
]
]);
$chk = $db->help->execute();
// If checklist is found, retrieve it
if ($chk) {
$chk = $db->get_Checklist($chk['id']);
if (count($chk) && is_a($chk[0], 'checklist')) {
$chk = $chk[0];
}
// Update software products associated with this checklist
$sw_arr = [];
foreach ($checklist['software'] as $sw) {
$sw_arr[] = [$chk->get_ID(), $sw->get_ID()];
}
if (is_array($sw_arr) && count($sw_arr)) {
$db->help->extended_insert("sagacity.checklist_software_lookup", ['chk_id', 'sw_id'], $sw_arr, true);
if (!$db->help->execute()) {
$db->debug(E_WARNING);
}
}
if (isset($cmd['debug'])) {
$log->script_log(print_r($chk, true), E_DEBUG);
}
}
else {
// If checklist is not found, add checklist to DB
$chk = new checklist(
null, $checklist['id'], $checklist['title'], $checklist['desc'], $checklist['status_date'], $base_name, $checklist['ver'], $checklist['rel'], ($checklist['type'] == 'iavm' ? 'IAVM' : ucfirst($checklist['type'])), null
);
$chk->add_SW($checklist['software']);
if (!($chk->id = $db->save_Checklist($chk))) {
$log->script_log("Failed to save new checklist ({$chk->get_Name()})", E_ERROR);
}
}
if (!$chk->id) {
$log->script_log("Could not find or create checklist", E_ERROR);
}
if (isset($cmd['debug'])) {
$log->script_log("Found checklist:" . PHP_EOL . print_r($chk, true), E_DEBUG);
}
// Get the collection of STIG rules i.e., <Group> elements
$groups = getValue($xml, '/x:Benchmark/x:Group', null, true);
$log->script_log("$groups->length STIGs to run", E_DEBUG);
$db->update_Catalog_Script($base_name, ['name' => 'stig_count', 'value' => $groups->length]);
print "File: $base_name" . PHP_EOL;
print "Total: $groups->length" . PHP_EOL;
// Iterate over each group element processing the attributes/children
foreach ($groups as $group) {
// Initialize local variables to hold parsed data
$new = false;
$references = [];
$ias = [];
$ia_controls = '';
$perc_comp++;
$vms_id = $group->getAttribute('id');
// the ".//" indicates that we are starting at the current node ($group) and looking in all child nodes for the "title" and "description" nodes
$group_title = getValue($xml, './/x:title', $group, true)->item(0)->nodeValue;
$group_desc = getValue($xml, './/x:description', $group);
// Get the Rule DOMElement
$group_rule = getValue($xml, 'x:Rule', $group, true)->item(0);
$sv_rule = $group_rule->getAttribute('id');
// Get the severity category and convert to an integer from one to three
$cat = 0;
if ($group_rule->getAttribute('severity') == 'high') {
$cat = 1;
}
elseif ($group_rule->getAttribute('severity') == 'medium') {
$cat = 2;
}
elseif ($group_rule->getAttribute('severity') == 'low') {
$cat = 3;
}
$rule_check_content = '';
$rule_ident = getValue($xml, ".//x:ident", $group_rule, true);
$rule_stig_id = getValue($xml, './/x:version', $group_rule);
$rule_title = textCleanup(getValue($xml, './/x:title', $group_rule));
$rule_desc = textCleanup(getValue($xml, './/x:description', $group_rule));
$check_content_nodes = getValue($xml, './/x:check-content', $group_rule, true);
$rule_check_ref = getValue($xml, './/x:check-content-ref', $group_rule, true);
$fix_text = getValue($xml, './/x:fixtext', $group_rule);
if ($rule_check_ref->length) {
$oval_id = $rule_check_ref->item(0)->getAttribute('name');
}
else {
$oval_id = '';
}
$match = [];
$discussion = "";
if (preg_match("/<VulnDiscussion>(.*)<\/VulnDiscussion>/", html_entity_decode($rule_desc), $match)) {
$discussion = $match[1];
}
// Remove unnecessary whitespace from and concatenate check content
if ($check_content_nodes->length > 0) {
for ($x = 0; $x < $check_content_nodes->length; $x++) {
$rule_check_content .= ($x + 1) . ") " . textCleanup($check_content_nodes->item($x)->textContent) . PHP_EOL;
}
$rule_check_content = trim($rule_check_content, PHP_EOL);
}
//$log->script_log("STIG ID: $rule_stig_id", E_DEBUG);
// Assign default category if not provided and add comment indicating such to rule description
if (!$cat) {
$cat = 2;
$discussion .= " :CAT SET BY SCRIPT";
}
// Extract and append potential impacts tag content from/to rule description
if (preg_match('/<PotentialImpacts>(.*)<\/PotentialImpacts>/', $rule_desc, $match)) {
$discussion .= "\n{$match[1]}";
}
if (!$rule_stig_id) {
if ($vms_id == 'V0001073' || $vms_id == 'V-1073') {
$rule_stig_id = '2.005';
}
elseif ($vms_id == 'V0001103' || $vms_id == 'V-1103') {
$rule_stig_id = '4.010';
}
}
// Check if rule is an IAVM
$is_iavm = false;
if (preg_match('/([\d]+\-[ABT]\-[\d]+)/', $rule_title, $match)) {
$references[] = $match[1];
if (!$rule_stig_id) {
$rule_stig_id = $match[1];
$is_iavm = true;
}
}
// Check if rule is an MS bulletin
if (preg_match('/(MS[\d]\-[\d]+)/', $rule_title, $match)) {
$references[] = $match[1];
if (!$rule_stig_id) {
$rule_stig_id = $match[1];
}
}
// If no STIG ID found, set to "No Reference"
if (!$rule_stig_id) {
error_log("Could not find stig id for group id $vms_id");
$rule_stig_id = 'No Reference';
}
$searchstring = [
'MS[\d]+\-[\d]+',
'CVE\-[\d\-]+',
'[^E]CAN\-[\d\-]+'
];
foreach ($searchstring as $string) {
if (preg_match_all("/($string)/", $rule_desc, $match)) {
for ($x = 0; $x < count($match[0]); $x++) {
if (!in_array($match[0][$x], $references)) {
$references[] = $match[0][$x];
}
}
}
if (preg_match_all("/($string)/", $rule_check_content, $match)) {
for ($x = 0; $x < count($match[0]); $x++) {
if (!in_array($match[0][$x], $references)) {
$references[] = $match[0][$x];
}
}
}
}
$safe_rule_title = preg_replace('/[\(\)\[\]\.\+\*]/', '', $rule_title);
$ia_ctrl = [];
if (isset($stigs["$rule_stig_id"])) {
print ".";
$updated_count++;
$db_stig = $stigs["$rule_stig_id"];
$db_pdi = $db->get_PDI($db_stig->get_PDI_ID(), $chk->get_ID());
$db_pdi->set_Group_Title($group_title);
$db_pdi->set_Short_Title($rule_title);
$db_pdi->set_Check_Contents($rule_check_content);
$db_pdi->set_Fix_Text($fix_text);
$pdi_id = $db_pdi->get_ID();
if ($db_pdi->get_Category_Level() != $cat) {
$db_pdi->set_Catetgory_Level($cat);
$db_pdi->set_Update($checklist['benchmark_date']->format("Y-m-d"));
}
$db->save_PDI($db_pdi, $chk);
}
else {
print "*";
// add pdi
$new_count++;
$db_pdi = new pdi(null, $cat, $checklist['benchmark_date']->format('Y-m-d'));
$db_pdi->set_Group_Title($group_title);
$db_pdi->set_Short_Title($rule_title);
$db_pdi->set_Check_Contents($rule_check_content);
$db_pdi->set_Fix_Text($fix_text);
$pdi_id = $db->save_PDI($db_pdi, $chk);
$db_pdi->set_ID($pdi_id);
// add stig
$db_stig = new stig($pdi_id, $rule_stig_id, $discussion);
$db->add_Stig($db_stig);
$new = true;
}
if (!empty($vms_id)) {
$vms_id = preg_replace("/^V0+/", "V-", $vms_id);
$gd = $db->get_GoldDisk($vms_id);
if (empty($gd)) {
$gd = new golddisk($pdi_id, $vms_id, $rule_title);
$db->save_GoldDisk($gd);
}
}
/*
if (!$db->save_Check_Contents($db_pdi, $chk, $rule_check_content, $fix_text)) {
$log->script_log("Couldn't save check contents for STIG ID: {$db_stig->get_ID()} in checklist {$chk->get_Checklist_ID()} ({$chk->get_File_Name()})\n", E_ERROR);
}
*/
$new_controls = [];
$control_fields = ['pdi_id', 'type', 'type_id'];
if (preg_match("/<IAControls>(.*)<\/IAControls>/i", $rule_desc, $match)) {
$ia_controls = (isset($match[1]) && !empty($match[1]) ? $match[1] : null);
if (preg_match("/DCSQ|ECMT/i", $ia_controls)) {
$new_controls[] = [
$pdi_id,
'VIVM',
'1'
];
$ias[] = "VIVM-1";
}
elseif ($ia_controls) {
$split_ias = preg_split('/\, ?/', $ia_controls);
foreach ($split_ias as $ia) {
$split_ia = explode("-", $ia);
if (isset($split_ia[0]) && $split_ia[1]) {
$ias[] = "{$split_ia[0]}-{$split_ia[1]}";
$new_controls[] = [
$pdi_id,
$split_ia[0],
$split_ia[1]
];
}
}
}
}
elseif ($rule_ident->length) {
for ($x = 0; $x < $rule_ident->length; $x++) {
if (substr($rule_ident->item($x)->textContent, 0, 3) == 'CCI') {
$split_ia = explode("-", $rule_ident->item($x)->textContent);
if (isset($split_ia[0]) && isset($split_ia[1])) {
$ias[] = "{$split_ia[0]}-{$split_ia[1]}";
$new_controls[] = [
$pdi_id,
$split_ia[0],
$split_ia[1]
];
}
}
}
}
else {
if ($is_iavm) {
$ias[] = "CCI-002613";
$new_controls[] = [
$pdi_id,
"CCI",
"002613"
];
}
else {
$ias[] = "CCI-000366";
$new_controls[] = [
$pdi_id,
"CCI",
"000366"
];
}
}
//$db_ia = $db->get_IA_Controls_By_PDI($db_pdi->get_ID());
if (isset($cmd['ia_reset']) && !$new) {
// delete ia controls
$db->help->delete("sagacity.ia_controls", [
[
'field' => 'pdi_id',
'op' => '=',
'value' => $pdi_id
]
]);
$db->help->execute();
}
if (count($new_controls)) {
$db->help->extended_replace("sagacity.ia_controls", $control_fields, $new_controls);
if (!$db->help->execute()) {
$db->help->debug(E_ERROR);
}
}
$sv = new sv_rule($pdi_id, $sv_rule);
$db->save_SV_Rule(array(0 => $sv));
if ($rule_ident->length) {
foreach ($rule_ident as $ident_node) {
if (!in_array($ident_node->textContent, $references) && $ident_node->textContent != 'CCI') {
$references[] = $ident_node->textContent;
}
}
}
if (count($references)) {
foreach ($references as $key => $ref) {
$tmp = null;
if (substr($ref, 0, 3) == 'CVE' || substr($ref, 0, 3) == 'CAN') {
$tmp[] = new cve($pdi_id, $ref);
$db->save_CVE($tmp);
}
elseif (substr($ref, 0, 3) == 'CCE') {
$tmp[] = new cce($pdi_id, $ref);
$db->save_CCE($tmp);
}
elseif (substr($ref, 0, 2) == 'KB') {
}
elseif (substr($ref, 0, 2) == 'MS') {
$tmp[] = new advisory($pdi_id, $ref, '', 'MS', '');
$db->save_Advisory($tmp);
}
//print_r($tmp[0]);
unset($tmp);
}
}
if ($perc_comp % 100 == 0) {
print "\t$perc_comp completed" . PHP_EOL;
}
// Output the CSV contents
fputcsv($csv, [$rule_stig_id, $vms_id, implode("", array_fill(0, $cat, "I")), implode(" ", $ias), $rule_title, "Not Reviewed", "", $rule_check_content, $sv_rule, $oval_id]);
unset($references);
$db->update_Catalog_Script($base_name, ['name' => 'perc_comp', 'value' => ($perc_comp / $groups->length) * 100]);
}
$end = new DateTime();
$diff = $end->diff($start);
print PHP_EOL . "Start Time: {$start->format("H:i:s")}" . PHP_EOL;
print "End Time: {$end->format("H:i:s")}" . PHP_EOL;
print "Execution time: {$diff->format("%H:%I:%S")}" . PHP_EOL . PHP_EOL;
print "New STIGs: $new_count" . PHP_EOL;
print "Updated STIGs: $updated_count" . PHP_EOL;
print "Total STIGs: " . ($new_count + $updated_count) . PHP_EOL . PHP_EOL . PHP_EOL;
$log->script_log("$groups->length complete");
fclose($csv);
if (!isset($cmd['debug'])) {
rename($cmd['f'], DOC_ROOT . "/reference/stigs/$base_name");
}
rename($csv_file, DOC_ROOT . "/reference/stigs/" . basename($csv_file));
$db->update_Catalog_Script($base_name, ['name' => 'perc_comp', 'value' => 100, 'complete' => 1]);
/**
* Usage output
*/
function usage() {
print <<<EOO
Purpose: To parse a STIG XCCDF checklist file and populate/update the database
Usage: php parse_stig.php -f={STIG file} [--debug] [--ia_reset] [--draft] [--help]
-f={STIG file} The file to be parsed
--debug Debugging output
--ia_reset To delete any existing mapped IA controls and repopulate with what is in the checklist file
--draft This will allow the importing of a draft STIG file (normally excluded)
--help This screen
EOO;
}

318
exec/parse_stig_viewer.php Normal file
View File

@ -0,0 +1,318 @@
<?php
/**
* File: parse_stig_viewer.php
* Author: Ryan Prather
* Purpose: Read STIG Viewer checklist files
* Created: Apr 10, 2014
*
* Portions Copyright 2016-2017: Cyber Perspectives, LLC, All rights reserved
* Released under the Apache v2.0 License
*
* Portions Copyright (c) 2012-2015, Salient Federal Solutions
* Portions Copyright (c) 2008-2011, Science Applications International Corporation (SAIC)
* Released under Modified BSD License
*
* See license.txt for details
*
* Change Log:
* - Apr 10, 2014 - File created
* - Jun 3, 2015 - Copyright updated, added CWD parameter, and
* updated function calls after class merger
* - Mar 4, 2017 - Removed Thread class calls
* - May 22, 2017 - Migrated to use parse_config.ini file and bug fixed to get working. Added CLI progress report
* - Jun 3, 2017 - Fixed bug #237
*/
$cmd = getopt("f:", ['debug::', 'help::']);
if (!isset($cmd['f']) || isset($cmd['help'])) {
die(usage());
}
if (!file_exists("parse_config.ini")) {
die("You must create parse_config.ini file with required parameters");
}
$conf = parse_ini_file("parse_config.ini");
if (!$conf) {
die("Could not find parse_config.ini configuration file");
}
chdir($conf['doc_root']);
include_once 'config.inc';
include_once 'helper.inc';
include_once 'database.inc';
check_path(TMP . "/stig_viewer");
chdir(TMP);
$db = new db();
$base_name = basename($cmd['f']);
$host_list = [];
$err = new Sagacity_Error($cmd['f']);
if (!file_exists($cmd['f'])) {
$db->update_Running_Scan($base_name, ['name' => 'status', 'value' => 'ERROR']);
$err->script_log("File not found", E_ERROR);
}
$db->update_Running_Scan($base_name, ['name' => 'pid', 'value' => getmypid()]);
$xml = new DOMDocument();
$xml->load($cmd['f']);
$root = $xml->getElementsByTagName('CHECKLIST')->item(0);
$xmlns = $xml->createAttribute('xmlns');
$xmlns->value = "http://www.w3.org/2001/XMLSchema-instance";
$root->appendChild($xmlns);
$host_name = getValue($xml, '//HOST_NAME');
$host_ip = getValue($xml, '//HOST_IP');
$host_mac = getValue($xml, '//HOST_MAC');
if (!$host_name) {
$db->update_Running_Scan($base_name, ['name' => 'status', 'value' => 'TERMINATED']);
unset($xml);
rename($cmd['f'], TMP . "/terminated/{$base_name}");
$err->script_log("File parsing terminated because host name was absent", E_ERROR);
die;
}
if ($tgt_id = $db->check_Target($conf['ste'], $host_name)) {
$tgt = $db->get_Target_Details($conf['ste'], $tgt_id)[0];
}
elseif ($tgt_id = $db->check_Target($conf['ste'], $host_ip)) {
$tgt = $db->get_Target_Details($conf['ste'], $tgt_id)[0];
}
else {
$tgt = new target($host_name);
$tgt->set_STE_ID($conf['ste']);
$sw = $db->get_Software("cpe:/o:generic:generic");
if (is_array($sw) && count($sw) && isset($sw[0]) && is_a($sw[0], 'software')) {
$sw = $sw[0];
$tgt->set_OS_ID($sw->get_ID());
$tgt->set_OS_String($sw->get_Shortened_SW_String());
}
$tgt_id = $db->save_Target($tgt);
$tgt->set_ID($tgt_id);
}
$source = $db->get_Sources('STIG Viewer');
if (is_array($source) && count($source) && isset($source[0]) && is_a($source[0], 'source')) {
$source = $source[0];
}
else {
die("Could not find source 'STIG Viewer' in DB");
}
$scan = $db->get_ScanData($conf['ste'], $base_name);
$vulns = getValue($xml, '//VULN', null, true);
if (!count($scan)) {
$fmt = filemtime($cmd['f']);
$fdt = DateTime::createFromFormat('U', $fmt);
$ste = $db->get_STE($conf['ste'])[0];
$scan = new scan(null, $source, $ste, 1, $base_name, $fdt->format('Y-m-d H:i:s'));
$hl = new host_list();
$hl->setTargetId($tgt->get_ID());
$hl->setTargetName($tgt->get_Name());
$hl->setFindingCount($vulns->length);
$scan->add_Target_to_Host_List($hl);
$scan_id = $db->save_Scan($scan);
$scan->set_ID($scan_id);
}
else {
$scan = $scan[0];
$hl = new host_list();
$hl->setTargetId($tgt->get_ID());
$hl->setTargetName($tgt->get_Name());
$hl->setFindingCount($vulns->length);
$scan->add_Target_to_Host_List($hl);
$db->update_Scan_Host_List($scan);
}
$vuln_count = 1;
foreach ($vulns as $vul) {
$stig_data = getValue($xml, "STIG_DATA", $vul, true);
$arr = [];
foreach ($stig_data as $node) {
$db->help->select("sagacity.scans", ['status'], [
[
'field' => 'id',
'op' => '=',
'value' => $scan->get_ID()
]
]);
$thread_status = $db->help->execute();
if ($thread_status == 'TERMINATED') {
unset($xml);
$source = strtolower($scan->get_Source()->get_Name());
rename(realpath(TMP . "/{$scan->get_File_Name()}"), realpath(TMP . "/scc/{$scan->get_File_Name()}"));
$err->script_log("File parsing terminated by user");
die();
}
$attr = getValue($xml, "VULN_ATTRIBUTE", $node);
$data = getValue($xml, "ATTRIBUTE_DATA", $node);
switch ($attr) {
case 'Vuln_Num':
$arr['vms_id'] = $data;
break;
case 'Severity':
if ($data == 'high') {
$arr['cat'] = 1;
}
elseif ($data == 'medium') {
$arr['cat'] = 2;
}
elseif ($data == 'low') {
$arr['cat'] = 3;
}
else {
$arr['cat'] = 2;
}
break;
case 'Rule_ID':
$arr['sv_rule'] = explode(' ', $data);
break;
case 'Rule_Ver':
$arr['stig_id'] = $data;
break;
case 'IA_Controls':
$arr['ia_controls'] = explode(", ", $data);
break;
case 'Check_Content_Ref':
$arr['ref'] = substr($data, 0, strpos($data, ' :: '));
break;
case 'Rule_Title':
$arr['short_title'] = $data;
break;
case 'Vuln_Discuss':
$arr['desc'] = $data;
break;
case 'Check_Content':
$arr['check_content'] = $data;
break;
}
}
if (isset($arr['stig_id'])) {
$stig = $db->get_Stig($arr['stig_id']);
if (is_array($stig) && count($stig) && isset($stig[0]) && is_a($stig[0], 'stig')) {
$stig = $stig[0];
}
else {
$pdi = new pdi(null, $arr['cat'], null, $arr['short_title'], $arr['desc']);
$pdi->set_Short_Title($arr['short_title']);
$pdi->set_Group_Title($arr['short_title']);
$pdi->set_Description($arr['desc']);
$pdi_id = $db->save_PDI($pdi);
$stig = new stig($pdi_id, $arr['stig_id'], $arr['desc']);
$db->add_Stig($stig);
// add stig
}
}
else {
print_r($arr);
}
$status = getValue($xml, 'STATUS', $vul);
switch ($status) {
case "Not_Reviewed":
$status = "Not Reviewed";
break;
case "NotAFinding":
$status = "Not a Finding";
break;
case "Not_Applicable":
$status = "Not Applicable";
}
$comments = "(STIG Viewer) " . getValue($xml, 'COMMENTS', $vul);
$vms = $db->get_GoldDisk($arr['vms_id']);
if (empty($vms)) {
$db->save_GoldDisk(new golddisk($stig->get_PDI_ID(), $arr['vms_id'], $arr['short_title']));
}
foreach ($arr['sv_rule'] as $key => $sv_rule) {
$sv = $db->get_SV_Rule($stig->get_PDI_ID(), $sv_rule);
if (!count($sv)) {
$db->save_SV_Rule(array(0 => new sv_rule($stig->get_PDI_ID(), $sv_rule)));
}
}
if (!$oval = $db->get_Oval($arr['ref']) || $oval->get_PDI_ID() != $stig->get_PDI_ID()) {
$db->add_Oval($oval = new oval($stig->get_PDI_ID(), $arr['ref'], $arr['short_title'], $arr['desc'], null, null, null));
}
$tmp = [];
foreach ($arr['ia_controls'] as $ia) {
if ($ia) {
$tmp[] = new ia_control($stig->get_PDI_ID(), substr($ia, 0, 4), substr($ia, 5));
}
}
if (count($tmp)) {
$db->save_IA_Control($tmp);
}
if (!$db->add_Finding($scan, $tgt, [
$arr['stig_id'],
$arr['vms_id'],
implode("", array_fill(0, $arr['cat'], 'I')),
implode(" ", $arr['ia_controls']),
$arr['short_title'],
$status,
$comments,
$arr['check_content'],
''
]
)) {
}
if (php_sapi_name() == 'cli') {
print "\r" . sprintf("%.02f%%", ($vuln_count / $vulns->length) * 100);
}
$db->update_Running_Scan($base_name, ['name' => 'perc_comp', 'value' => ($vuln_count / $vulns->length) * 100]);
$vuln_count++;
}
unset($xml);
if (!isset($cmd['debug'])) {
rename($cmd['f'], TMP . "/stig_viewer/$base_name");
}
$db->update_Running_Scan($base_name, ['name' => 'perc_comp', 'value' => 100, 'complete' => 1]);
function usage()
{
print <<<EOO
Purpose: To parse a STIG Viewer output result file
Usage: php parse_stig_viewer.php -f={STIG Viewer file} [--debug] [--help]
-f={STIG Viewer file} The STIG Viewer result file that is being imported
--debug Debugging output
--help This screen
EOO;
}

55
exec/populate_cve_web.php Normal file
View File

@ -0,0 +1,55 @@
<?php
/**
* File: populate_cve_web.php
* Author: Ryan Prather
* Purpose: Find all CVE files, read entire file and add to database for later retrieval
* Created: Sep 1, 2016
*
* Portions Copyright 2016: Cyber Perspectives, All rights reserved
* Released under the Apache v2.0 License
*
* See license.txt for details
*
* Change Log:
* - Sep 1, 2016 - File created
*/
include_once 'config.inc';
include_once 'database.inc';
include_once 'helper.inc';
$db = new mysqli(DB_SERVER, "web", db::decrypt_pwd());
set_time_limit(0);
$files = glob("../reference/cve/*.xml");
$count = 0;
$start_sql = $sql = "INSERT IGNORE INTO `reference`.`cve_web` (`cve_id`,`xml`) VALUES ";
foreach ($files as $file) {
print ".";
$xml = new DOMDocument();
$xml->load($file);
$cve_id = getValue($xml, "/item/@name");
if (empty($cve_id)) {
die;
}
$sql .= "(" .
"'" . $db->real_escape_string($cve_id) . "'," .
"'" . $db->real_escape_string($xml->saveXML()) . "'),";
$count++;
if ($count % 100 == 0 || strlen($sql) > 5000000) {
if (!$db->real_query(substr($sql, -1))) {
error_log($db->error);
Sagacity_Error::sql_handler($sql);
die;
}
print "\t$count\t" . strlen($sql) . PHP_EOL;
$sql = $start_sql;
}
}

56
exec/post_process_all.php Normal file
View File

@ -0,0 +1,56 @@
<?php
/**
* File: post_process_all.php
* Author: Jeff Odegard
* Purpose: Script to perform bulk post-processing on all targets or all targets in a specific ST&E
* Created: May 19,2015
*
* Portions Copyright 2016: Cyber Perspectives, All rights reserved
* Released under the Apache v2.0 License
*
* See license.txt for details
*
* Change Log:
* - May 19, 2015 - File created
*/
include_once 'config.inc';
include_once 'helper.inc';
include_once 'database.inc';
$cmd = getopt("h::", array("help::", "ste::", "debug::"));
if (isset($cmd['h']) || isset($cmd['help'])) {
usage();
exit;
}
$db = new db();
if (isset($cmd['ste'])) {
$tgts = $db->get_Target_Details($cmd['ste']);
foreach ($tgts as $key => $tgt) {
print "Post-processing " . $tgt->get_Name() . PHP_EOL;
$db->post_Processing($tgt->get_ID());
}
}
else {
$db->post_Processing();
}
function usage() {
print <<<EOO
Purpose: Perform bulk post-processing
Usage: php post_process_all.php [--ste={ST&E ID}] [--help|-h] [--debug]
NOTE: If no ST&E specified then will get all targets that have the pp_flag field in the database set to '1'
--ste={ST&E ID} The ST&E ID to evaluate targets
--debug Debugging output
--help | -h This screen
EOO;
}

18
exec/run_nmap.php Normal file
View File

@ -0,0 +1,18 @@
<?php
/**
* File: run_scan.php
* Author: Ryan Prather
* Purpose: Execute a scan in the background
* Created: Sep 10, 2014
*
* Portions Copyright (c) 2012-2015, Salient Federal Solutions
* Portions Copyright (c) 2008-2011, Science Applications International Corporation (SAIC)
* Released under Modified BSD License
*
* See license.txt for details
*
* Change Log:
* - Sep 10, 2014 - File created
*/
?>

8
exec/script_footer.inc Normal file
View File

@ -0,0 +1,8 @@
<?php
$end = new DateTime();
$diff = $start->diff($end);
print $diff->format("%H:%I:%S");
$log->script_log("Script runtiime: ".$diff->format("%H:%I:%S"));

43
exec/script_header.inc Normal file
View File

@ -0,0 +1,43 @@
<?php
/**
* File: script_header.inc
* Author: Ryan Prather
* Purpose: To standardize the header for any script that has to parse files
* Created: ?
*
* Portions Copyright 2016-2017: Cyber Perspectives, All rights reserved
* Released under the Apache v2.0 License
*
* Portions Copyright (c) 2012-2015, Salient Federal Solutions
* Portions Copyright (c) 2008-2011, Science Applications International Corporation (SAIC)
* Released under Modified BSD License
*
* See license.txt for details
*
* Change Log:
* - ? - File created
* - Sep 1, 2016 - Copyright updated and added file header
*/
$start = new DateTime();
$cmd = getopt("f:s::", array('debug::'));
set_time_limit(0);
include_once 'config.inc';
include_once 'helper.inc';
include_once 'database.inc';
chdir(TMP);
$db = new db();
$base_name = basename($cmd['f']);
$log = new Sagacity_Error($base_name);
if (!file_exists($cmd['f'])) {
$db->update_Running_Scan($base_name, array('name' => 'status', 'value' => 'ERROR'));
$err->script_log("File not found", E_ERROR);
}
$db->update_Running_Scan($base_name, array('name' => 'pid', 'value' => getmypid()));

626
exec/update_db.php Normal file
View File

@ -0,0 +1,626 @@
<?php
/**
* File: update_db.php
* Purpose: Script to download updated versions of the online files and update the database
* Author: Ryan Prather
* Created: Jan 5, 2015
*
* Portions Copyright 2016-2017: Cyber Perspectives, All rights reserved
* Released under the Apache v2.0 License
*
* Portions Copyright (c) 2012-2015, Salient Federal Solutions
* Portions Copyright (c) 2008-2011, Science Applications International Corporation (SAIC)
* Released under Modified BSD License
*
* See license.txt for details
*
* Change Log:
* - Jan 5, 2015 - File Created
* - Sep 1, 2016 - Copyright Updated and added file header
* - Oct 24, 2016 - Added check for presence of /tmp/cce, /tmp/cve, & /tmp/cpe directories before downloading
* - Nov 9, 2016 - Added command line parameters for each of the update types (CVE, CPE, CCE),
* Added detection if the file from same day already exists, added "download only" option,
* Added mkdir if temporary directories don't exist, added usage output,
* Changed URL for CCE and CPE to HTTPS paths
* - Nov 21, 2016 - Cleaned up code a little and added 3 second sleep between commands so user can see results
* - Dec 7, 2016 - Changed PHP constant to PHP_BIN
* - Jan 30, 2017 - Updated to parse Nessus NASL plugin update file
* - Jan 31, 2017 - Completed NASL plugin parsing
* - Feb 15, 2017 - Completed adding STIG download and update, added parse only (po) flag for offline systems,
* Added ping method to check for connection to internet. Added url_exists method to check for presence of file before attempting to download.
* - Feb 21, 2017 - Extended output for nasl on Windows hosts, store existing plugins in memory to speed up processing,
* Delete files if there are parsing errors, added --delete flag when parsing stigs,
* Added check to see if STIG file has been downloaded already today
* - Mar 3, 2017 - Fixed output of tar command like Jeff suggested, and clarified -u parameter
* - Mar 8, 2017 - Added check for presence of downloaded files before attempting to parse
* - Mar 13, 2017 - Cleaned up downloads of STIG compilation file, added check for *v2.zip file
* - Mar 17, 2017 - Added check for previous STIG file that contains '_v2' at the end of the file,
* Added check for any other *.zip files in the /tmp folder just in case user only wants to upgrade FOUO files or individual XML files
* - Mar 20, 2017 - Added check for previous STIG that includes '_v2' at end of filename, and added checks to fix issues when FOUO file is present
* - Mar 22, 2017 - Added check for extracted STIG files in /tmp/stig/xml/*.xml
* - Apr 5, 2017 - Added check for previous STIG compilation_v2, so will now check for 4 different naming possibilities,
* Extended TIME_WAIT ports to 2000, started time totaling process
* - Apr 7, 2017 - Fixed typo in printed tar command
* - May 13, 2017 - Clarified if it cannot access the internet instead of "cannot connect to server"
* Fixed confusion with Cygwin and Bash on Windows paths
* - Jun 27, 2017 - Matt Shuter: Fixed bug #262 & #270
* - Dec 27, 2017 - Added database field and download progress flag
*/
include_once 'config.inc';
include_once 'helper.inc';
include_once 'error.inc';
include_once 'database.inc';
include_once 'DateTimeDiff.php';
include_once 'vendor/autoload.php';
use Monolog\Logger;
use Monolog\Handler\StreamHandler;
use Monolog\Formatter\LineFormatter;
$current_date = new DateTime();
$total_time = null;
$total_diff = 0;
$summary_stats = [];
$cmd = getopt("h::u::p::", ['cpe::', 'cce::', 'cve::', 'nvd::', 'nasl::', 'stig::', 'do::', 'po::', 'help::']);
$db = new db();
$diff = new DateTimeDiff();
$log_level = Logger::ERROR;
switch (LOG_LEVEL) {
case E_WARNING:
$log_level = Logger::WARNING;
break;
case E_NOTICE:
$log_level = Logger::NOTICE;
break;
case E_DEBUG:
$log_level = Logger::DEBUG;
}
$stream = new StreamHandler("php://output", Logger::INFO);
$stream->setFormatter(new LineFormatter("%datetime% %level_name% %message%" . PHP_EOL, "H:i:s.u"));
$log = new Logger("update_db");
$log->pushHandler(new StreamHandler(LOG_PATH . "/update_db.log", $log_level));
$log->pushHandler($stream);
if (isset($cmd['h'], $cmd['help']) ||
(!isset($cmd['cpe']) && !isset($cmd['cve']) && !isset($cmd['nasl']) && !isset($cmd['stig']) && !isset($cmd['nvd']))) {
die(usage());
}
if (isset($cmd['do']) || !isset($cmd['po'])) {
if (!ping("cyberperspectives.com")) {
die("Cannot connect to internet" . PHP_EOL);
}
}
/**
* Update CPE content downloaded from NIST
*/
if (isset($cmd['cpe'])) {
$db->set_Setting('cpe-dl-progress', 0);
$db->set_Setting('cpe-progress', 0);
$path = TMP . "/cpe";
if (isset($cmd['p']) && $cmd['p']) {
$path = $cmd['p'];
}
check_path($path);
$diff->resetClock();
$log->info("Started CPE ingestion ({$diff->getStartClockTime()})");
// search for an unzip any zip files in the tmp directory
$zip_files = glob("{$path}/*cpe-dictionary*.zip");
if (count($zip_files)) {
$log->debug("Found a existing cpe-dictionary.zip file, unzipping then parsing");
$zip = new ZipArchive();
foreach ($zip_files as $file) {
$log->info("Unzipping {$file}");
$zip->open($file);
$zip->extractTo($path);
$zip->close();
unlink($file);
}
}
// search for any existing cpe-dictionary files in the /tmp directory
$tmp_files = glob(TMP . "/*cpe-dictionary*.xml");
if (count($tmp_files)) {
$log->debug("Found existing cpe-dictionary.xml file in TMP folder, moving to TMP/cpe then processing");
foreach ($tmp_files as $fname) {
$name = basename($fname);
if ($name == 'official-cpe-dictionary_v2.3.xml') {
$name = "cpe-dictionary-{$start_time->format("Ymd")}.xml";
}
rename($fname, "{$path}/{$name}");
}
}
$cpe_fname = realpath($path) . "/cpe-dictionary-{$current_date->format('Ymd')}.xml";
$cpe_url = "https://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml";
$cpe_parse_fname = null;
// download the file if the do flag is used even if it already exists
if (isset($cmd['do']) && ping("nist.gov") && !isset($cmd['po'])) {
download_file($cpe_url, $cpe_fname, $db, 'cpe-dl-progress');
}
// download the file only if it doesn't exist
elseif (!file_exists($cpe_fname) && ping("nist.gov") && !isset($cmd['po'])) {
download_file($cpe_url, $cpe_fname, $db, 'cpe-dl-progress');
}
elseif (!isset($cmd['po']) && !ping("nist.gov")) {
$log->error("Could not connect to nist.gov to download the CPE library");
die();
}
$dt = new DateTime();
if (!isset($cmd['do']) || isset($cmd['po'])) {
$cpe_files = glob("{$path}/*cpe-dictionary*.xml");
rsort($cpe_files, SORT_NATURAL);
if (count($cpe_files)) {
$match = [];
if (preg_match("/cpe\-dictionary\-([\d]+)\.xml/", $cpe_files[0], $match)) {
$dt = DateTime::createFromFormat("Ymd", $match[1]);
$seven_days_old = new DateTime();
$seven_days_old->sub(DateInterval::createFromDateString("7 days"));
if ($dt < $seven_days_old) {
$log->warning("The file that is being ingested is more than 7 days old ({$dt->format('Y-m-d')})");
}
$cpe_parse_fname = $cpe_files[0];
}
else {
$log->warning("Don't know when the file was downloaded, but parsing it anyway");
$cpe_parse_fname = $cpe_files[0];
}
}
if (is_null($cpe_parse_fname)) {
$log->warning("Coult not find a CPE file to parse");
}
$script = realpath(defined('PHP_BIN') ? PHP_BIN : PHP) .
" -c " . realpath(PHP_CONF) .
" -f " . realpath(DOC_ROOT . "/exec/parse_cpe.php") . " --" .
" -f=\"" . realpath($cpe_parse_fname) . "\"" .
" -d=\"{$dt->format("Y-m-d")}\"";
$log->info("Running parsing script");
passthru($script);
}
$diff->stopClock();
$log->info(PHP_EOL . "Finished at {$diff->getEndClockTime()}" . PHP_EOL .
"Total time: {$diff->getDiffString()}");
sleep(3);
}
/**
* Update CVE content
*/
if (isset($cmd['cve'])) {
$db->set_Setting('cve-dl-progress', 0);
$db->set_Setting('cve-progress', 0);
$path = TMP . "/cve";
if (isset($cmd['p']) && $cmd['p']) {
$path = $cmd['p'];
}
check_path($path);
$diff->resetClock();
$log->info("Started CVE ingestion {$diff->getStartClockTime()}");
$cve_files = glob(TMP . "/allitems.xml");
if (count($cve_files)) {
foreach ($cve_files as $file) {
rename($file, "{$path}/cve-all-{$start_time->format("Ymd")}.xml");
}
}
$tmp_files = glob("{$path}/cve*.xml");
if (count($tmp_files)) {
foreach ($tmp_files as $fname) {
rename($fname, "{$path}/" . basename($fname));
}
}
$cve_fname = realpath($path) . "/cve-all-{$current_date->format('Ymd')}.xml";
$cve_url = "http://cve.mitre.org/data/downloads/allitems.xml";
$cve_parse_fname = null;
if (isset($cmd['do']) && ping("cve.mitre.org") && !isset($cmd['po'])) {
download_file($cve_url, $cve_fname, $db, 'cve-dl-progress');
}
elseif (!file_exists($cve_fname) && ping("cve.mitre.org") && !isset($cmd['po'])) {
download_file($cve_url, $cve_fname, $db, 'cve-dl-progress');
}
elseif (!isset($cmd['po']) && !ping("cve.mitre.org")) {
Sagacity_Error::err_handler("Could not connect to cve.mitre.org to download the CVE library", E_ERROR);
}
if (!isset($cmd['do']) || isset($cmd['po'])) {
$cve_files = glob("{$path}/cve-all-*.xml");
rsort($cve_files, SORT_NATURAL);
if (count($cve_files)) {
$match = [];
if (preg_match("/cve\-all\-([\d]+)\.xml/", $cve_files[0], $match)) {
$dt = DateTime::createFromFormat("Ymd", $match[1]);
$seven_days_old = new DateTime();
$seven_days_old->sub(DateInterval::createFromDateString("7 days"));
if ($dt < $seven_days_old) {
$log->warning("The CVE file that is being ingested is more than 7 days old ({$dt->format('Y-m-d')})");
}
$cve_parse_fname = $cve_files[0];
}
}
if (is_null($cve_parse_fname)) {
$log->error("Coult not find a CVE file to parse");
die;
}
$script = realpath(defined('PHP_BIN') ? PHP_BIN : PHP) .
" -c " . realpath(PHP_CONF) .
" -f " . realpath(DOC_ROOT . "/exec/parse_cve.php") . " --" .
" -f=\"" . realpath($cve_parse_fname) . "\"" .
" -d=\"{$dt->format("Y-m-d")}\"";
$log->info("Script to run $script");
passthru($script);
}
$diff->stopClock();
$log->info("Finished at {$diff->getEndClockTime()}" . PHP_EOL .
"Total Time: {$diff->getDiffString()}");
sleep(3);
}
if (isset($cmd['nvd'])) {
$db->set_Setting('nvd-cve-dl-progress', 0);
$db->set_Setting('nvd-cve-progress', 0);
$path = TMP . "/nvd";
if (isset($cmd['p']) && $cmd['p']) {
$path = $cmd['p'];
}
check_path($path);
$diff->resetClock();
$log->info("Started NVD CVE ingestion ({$diff->getStartClockTime()})");
$nvd_years = [];
for ($x = 2002; $x <= $diff->getStartClock()->format("Y"); $x++) {
$nvd_years[] = $x;
}
$too_old = new DateTime();
$too_old->sub(DateInterval::createFromDateString("7 days"));
$load_date = new DateTime($db->get_Settings("nvd-cve-load-date"));
if ($load_date < $too_old) {
// More than 7 days old so have to do a full load
foreach ($nvd_years as $yr) {
download_file("https://static.nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-{$yr}.json.zip", TMP . "/nvd/nvdcve-{$yr}.json.zip");
$zip = new ZipArchive();
$zip->open(TMP . "/nvd/nvdcve-{$yr}.json.zip");
$zip->extractTo(TMP . "/nvd");
$zip->close();
unlink(TMP . "/nvd/nvdcve-{$yr}.json.zip");
}
}
else {
download_file("https://static.nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-modified.json.zip", TMP . "/nvd/nvdcve-modified.json.zip");
$zip = new ZipArchive();
$zip->open(TMP . "/nvd/nvdcve-modified.json.zip");
$zip->extractTo(TMP . "/nvd");
$zip->close();
unlink(TMP . "/nvd/nvdcve-modified.json.zip");
download_file("https://static.nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-recent.json.zip", TMP . "/nvd/nvdcve-recent.json.zip");
$zip->open(TMP . "/nvd/nvdcve-recent.json.zip");
$zip->extractTo(TMP . "/nvd");
$zip->close();
unlink(TMP . "/nvd/nvdcve-recent.json.zip");
}
chdir(DOC_ROOT . "/exec");
$json_files = glob(TMP . "/nvd/*.json");
foreach ($json_files as $j) {
$script = realpath(defined('PHP_BIN') ? PHP_BIN : PHP) .
" -c " . realpath(PHP_CONF) .
" -f " . realpath(DOC_ROOT . "/exec/parse_nvd_json_cve.php") . " --" .
" -f=\"" . realpath($j) . "\"";
$log->info("Running NVD CVE parsing script");
passthru($script);
}
$diff->stopClock();
$log->info("Finished at {$diff->getEndClockTime()}" . PHP_EOL .
"Total time {$diff->getTotalDiffString()}");
$db->set_Setting("nvd-cve-load-date", $diff->getEndClock()->format(MYSQL_DT_FORMAT));
}
/**
* Update CCE content
*/
if (isset($cmd['cce'])) {
check_path(TMP . "/cce");
$cce_fname = TMP . "/cce/cce-all-20130214.xml";
if (!file_exists($cce_fname) && ping("nist.gov") && !isset($cmd['po'])) {
download_file("https://static.nvd.nist.gov/feeds/xml/cce/cce-COMBINED-5.20130214.xml", $cce_fname);
}
if (!isset($cmd['do']) || isset($cmd['po'])) {
}
sleep(3);
}
/**
* Parse NASL content from NVT and/or Nessus
*/
if (isset($cmd['nasl'])) {
$db->set_Setting('nasl-dl-progress', 0);
$db->set_Setting('nasl-progress', 0);
check_path(TMP . "/nessus_plugins", true);
// Capture start time for performance monitoring
$diff->resetClock();
$log->info("Started NASL ingestion ({$diff->getStartClockTime()})");
// Generate a unique filename for the OpenVAS feed archive using the current date
$nasl_fname = TMP . "/nessus_plugins/nasl_plugins-{$current_date->format("Ymd")}.tar.bz2";
// Download OpenVAS feed if a) it doesn't exist, b) can reach openvas.org, and c) parse only flag not set
if (!file_exists($nasl_fname) && ping("openvas.org") && !isset($cmd['po'])) {
download_file("http://www.openvas.org/openvas-nvt-feed-current.tar.bz2", $nasl_fname, $db, 'nasl-dl-progress');
}
// Can only extract .tar.bz2 files on Linux so...
if (!isset($cmd['do']) || isset($cmd['po'])) {
if (file_exists($nasl_fname)) {
if (substr(strtolower(PHP_OS), 0, 3) == 'lin') {
passthru("tar xvf $nasl_fname -C " . realpath(TMP . "/nessus_plugins") .
" --wildcards --transform='s/.*\///' '*.nasl'");
}
}
// ...if there are no .nasl files in the directory, die and give instructions for unzipping in Windows
$files = glob("*.nasl");
if (!count($files)) {
die("Downloaded the OpenVAS NVT plugin repository, please extract *.nasl files to " . realpath(TMP . "/nessus_plugins") . PHP_EOL .
"If you have Bash on Windows ({path} = /mnt/c/xampp/www) or Cygwin ({path} = /cygdrive/c/xampp/www) installed you can run the following command on the downloaded file tweaking the paths" . PHP_EOL .
"tar xvf {path}/tmp/nessus_plugins/" . basename($nasl_fname) . " -C {path}/tmp/nessus_plugins --wildcards --transform='s/.*\///' '*.nasl'" . PHP_EOL);
}
// Report how many NASL files were found in the directory
$log->info("Found " . count($files) . " NASL files" . PHP_EOL .
"Started at {$start_time->format("Y-m-d H:i:s")}");
chdir(DOC_ROOT);
// Query database to build an array of existing plugins to compare against on import
$existing_plugins = [];
$db->help->select("nessus_plugins", ['plugin_id', 'file_date']);
$rows = $db->help->execute();
if (is_array($rows) && count($rows)) {
foreach ($rows as $row) {
$existing_plugins[$row['plugin_id']] = DateTime::createFromFormat("U", $row['file_date']);
}
}
// Sort the files and loop over them
$x = 0;
natsort($files);
foreach ($files as $file) {
$abs_file_path = realpath(TMP . "/nessus_plugins/$file");
// Read the current NASL file into a nasl object
$nasl = new nasl($abs_file_path);
// Report progress
$comp = number_format(($x / count($files)) * 100, 2) . "%";
print "\r$comp";
// If no plugin ID, delete file and continue to the next plugin
if (!isset($nasl->id)) {
unlink($abs_file_path);
continue;
}
// Only process if plugin doesn't already exist or has an older last_modificaiton date
if (!isset($existing_plugins[$nasl->id]) ||
(isset($nasl->last_modification) && $existing_plugins[$nasl->id] > $nasl->last_modification)) {
// define command line to call script to parse the file
$script = realpath(defined('PHP_BIN') ? PHP_BIN : PHP) .
" -c " . realpath(PHP_CONF) .
" -f " . realpath(DOC_ROOT . "/exec/nessus-plugin-to-database.php") . " --" .
" -f=\"" . $abs_file_path . "\"";
$process = new \Cocur\BackgroundProcess\BackgroundProcess($script);
$process->run();
// Call the script w/ shell or exec depending on platform
if (substr(strtolower(PHP_OS), 0, 3) == 'lin') {
$output = [];
exec("netstat -an | grep TIME_WAIT | wc -l", $output);
if ($output[0] > 2000) {
do {
$log->notice("\r$comp Sleeping till connections get below 100 {$output[0]}");
sleep(1);
$output = [];
exec("netstat -an | grep TIME_WAIT | wc -l", $output);
}
while ($output[0] > 100);
}
}
}
else {
unlink($abs_file_path);
}
$x++;
}
}
$diff->stopClock();
$log->info(PHP_EOL . "Finished at {$diff->getEndClockTime()}" . PHP_EOL .
"Total Time: {$diff->getDiffString()}");
sleep(3);
}
/**
* Update STIG library from DISA content
*/
if (isset($cmd['stig'])) {
$db->set_Setting('stig-dl-progress', 0);
$db->set_Setting('stig-progress', 0);
$path = TMP . "/stigs";
check_path($path);
$diff->resetClock();
$log->info("Started STIG ingestion ({$diff->getStartClockTime()})");
$mon = '01';
$prev_mon = '10';
$year = (int) $current_date->format("Y");
if (between($current_date->format("n"), 4, 6)) {
$mon = '04';
$prev_mon = '01';
}
elseif (between($current_date->format("n"), 7, 9)) {
$mon = '07';
$prev_mon = '04';
}
elseif (between($current_date->format("n"), 10, 12)) {
$mon = '10';
$prev_mon = '07';
}
$current_url = "http://iasecontent.disa.mil/stigs/zip/Compilations/U_SRG-STIG_Library_{$year}_{$mon}.zip";
$current_v2_url = "http://iasecontent.disa.mil/stigs/zip/Compilations/U_SRG-STIG_Library_{$year}_{$mon}_v2.zip";
$stig_fname = "{$path}/stig_library-{$year}_{$mon}.zip";
if (!file_exists($stig_fname) && ping("disa.mil") && !isset($cmd['po'])) {
if (isset($cmd['u'])) {
$url = $cmd['u'];
$log->info("Checking for $url");
if (url_exists($url)) {
download_file($url, $stig_fname, $db, 'stig-dl-progress');
}
}
else {
$log->info("Checking for $current_url");
if ($found = url_exists($current_url)) {
download_file($current_url, $stig_fname, $db, 'stig-dl-progress');
}
if (!$found) {
$log->info("Checking for $current_v2_url");
if ($found = url_exists($current_v2_url)) {
download_file($current_v2_url, $stig_fname, $db, 'stig-dl-progress');
}
}
if ($mon == '01') {
$year--;
}
$prev_url = "http://iasecontent.disa.mil/stigs/zip/Compilations/U_SRG-STIG_Library_{$year}_{$prev_mon}.zip";
$prev_v2_url = "http://iasecontent.disa.mil/stigs/zip/Compilations/U_SRG-STIG_Library_{$year}_{$prev_mon}_v2.zip";
if (!$found) {
$log->info("Checking for $prev_url");
if ($found = url_exists($prev_url)) {
download_file($prev_url, $stig_fname, $db, 'stig-dl-progress');
}
}
if (!$found) {
$log->info("Checking for $prev_v2_url");
if (url_exists($prev_v2_url)) {
download_file($prev_v2_url, $stig_fname, $db, 'stig-dl-progress');
}
}
}
}
if (!isset($cmd['do']) || isset($cmd['po'])) {
$stig_files = array_merge(
glob("{$path}/*.zip"), glob("{$path}/*.xml"), glob(TMP . "/*.zip"), glob(TMP . "/*.xml"), glob(TMP . "/stigs/xml/*.xml")
);
if (!file_exists($stig_fname) && !count($stig_files)) {
die("Could not locate $stig_fname or find any other zip files in " . realpath(TMP));
}
$script = realpath(defined('PHP_BIN') ? PHP_BIN : PHP) .
" -c " . realpath(PHP_CONF) .
" -f " . realpath(DOC_ROOT . "/exec/background_stigs.php") . " --" .
" --delete";
$log->info("Script to run $script");
passthru($script);
}
$diff->stopClock();
$log->info(PHP_EOL . "Finished at {$diff->getEndClockTime()}" . PHP_EOL .
"Total Time: {$diff->getDiffString()}");
sleep(3);
}
if (is_a($diff->getTotalDiff(), 'DateInterval')) {
$log->info("Total Script Time: {$diff->getTotalDiffString()}");
}
/**
*
*/
function usage()
{
$tmp = TMP;
print <<<EOO
Purpose: The purpose of this script is to update the CVE, CPE, and CCE databases. Script will sleep for 3 seconds between actions to allow you review the results.
Usage: php update_db.php [--cpe] [--cve] [--nasl] [--stig] [-u={URL}] [--do] [--po] [-h|--help]
--cpe To download and update the CPE catalog
--cve To download and update the CVE catalog
--nasl To download OpenVAS NVT library and update NASL files
You can also extract *.nasl files from the Nessus library to $tmp/nessus_plugins and it will include these in the update
--stig To download and update the STIG library
--do To download the files only...do not call the parsers will overwrite any existing files
--po To parse the downloaded files only, do not download
-u={url} [optional] Used only for STIGs because sometimes DISA will use a non-standard link which makes it difficult to download the file.
-h|--help This screen
EOO;
}