ryan/sagacity
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Revision of release v1.3.2

Browse Source
This commit is contained in:
CyberPerspectives
2018-07-26 08:33:50 -04:00
committed by Ryan Prather
parent 8c38a6cdb9
commit 750094e3b5
3231 changed files with 133590 additions and 135073 deletions
Download Patch File Download Diff File Expand all files Collapse all files

482
exec/background_results.php
View File

@ -1,12 +1,11 @@
<?php
/**
* File: background_results.php
* Author: Ryan Prather
* Purpose: Background script file that will call appropriate function for files found
* Created: Feb 26, 2014
*
* Portions Copyright 2016-2017: Cyber Perspectives, LLC, All rights reserved
* Portions Copyright 2016-2018: Cyber Perspectives, LLC, All rights reserved
* Released under the Apache v2.0 License
*
* Portions Copyright (c) 2012-2015, Salient Federal Solutions
@ -29,6 +28,7 @@
* - Feb 21, 2017 - Fixed path issues with scripts not running
* - Oct 23, 2017 - Conditionally delete parse_config.ini only if not in DEBUG log level
* - Oct 27, 2017 - Fix to remove desktop.ini files if found
* - May 24, 2018 - Moved a couple code blocks because of being out of order
*/
error_reporting(E_ALL);
@ -36,15 +36,6 @@ $cmd = getopt("t::", ["help::"]);
$conf = parse_ini_file("parse_config.ini", false);
if (isset($cmd['help']) || !is_numeric($conf['ste']) || !isset($conf['doc_root'])) {
die(usage());
}
elseif (!file_exists($conf['doc_root'])) {
die("Folder {$conf['doc_root']} doesn't exist" . PHP_EOL);
}
chdir($conf['doc_root']);
set_time_limit(0);
include_once 'config.inc';
@ -52,8 +43,31 @@ include_once 'database.inc';
include_once 'helper.inc';
include_once 'vendor/autoload.php';
use Monolog\Logger;
use Monolog\Handler\StreamHandler;
$log_level = Logger::ERROR;
switch (LOG_LEVEL) {
case E_WARNING:
$log_level = Logger::WARNING;
break;
case E_NOTICE:
$log_level = Logger::NOTICE;
break;
case E_DEBUG:
$log_level = Logger::DEBUG;
}
$log = new Logger('result_import');
$log->pushHandler(new StreamHandler(LOG_PATH . "/result_import.log", $log_level));
$debug = (LOG_LEVEL == E_DEBUG ? true : false);
if (isset($cmd['help']) || !is_numeric($conf['ste']) || !isset($conf['doc_root'])) {
die(usage());
}
chdir(TMP);
check_path(TMP . "/echecklist");
check_path(TMP . "/nessus");
check_path(TMP . "/nmap");
@ -62,171 +76,171 @@ check_path(TMP . "/stig_viewer");
check_path(TMP . "/terminated");
check_path(TMP . "/unsupported");
chdir(TMP);
$dbh = new db();
$files = glob("*.*");
$stack = [];
$files = glob("*.*");
$stack = [];
$running = [];
$time = 0;
$time = 0;
$threads = [];
foreach ($files as $file) {
$res = FileDetection($file);
if ($debug) {
Sagacity_Error::err_handler(print_r($res, true), E_DEBUG);
}
switch ($res['type']) {
case NESSUS:
$stack[] = array(
'exec' => 'nessus',
'file' => $file,
'ste' => $conf['ste'],
'source' => 'nessus'
);
break;
case SCC_XCCDF:
$stack[] = array(
'exec' => 'scc_xccdf',
'file' => $file,
'ste' => $conf['ste'],
'source' => 'scc_xccdf'
);
break;
case STIG_VIEWER_CKL:
$stack[] = array(
'exec' => 'stig_viewer',
'file' => $file,
'ste' => $conf['ste'],
'source' => 'stig_viewer'
);
break;
case TECH_ECHECKLIST_EXCEL:
$ignore = false;
if (isset($conf['ignore'])) {
$ignore = true;
}
$stack[] = array(
'exec' => 'excel_echecklist',
'file' => $file,
'ste' => $conf['ste'],
'ignore_hidden' => $ignore,
'source' => 'echecklist'
);
break;
case ECHECKLIST_CSV:
$stack[] = array(
'exec' => 'csv_echecklist',
'file' => $file,
'ste' => $conf['ste'],
'source' => 'echecklist'
);
break;
case PROC_ECHECKLIST_EXCEL:
$stack[] = array(
'exec' => 'proc_echecklist',
'file' => $file,
'ste' => $conf['ste']
);
break;
case HOST_DATA_COLLECTION:
$stack[] = array(
'exec' => 'data_collection',
'file' => $file,
'ste' => $conf['ste'],
'target' => $cmd['t'],
'source' => 'data_collection'
);
break;
case NMAP_GREPABLE:
case NMAP_TEXT:
case NMAP_XML:
$stack[] = array(
'exec' => 'nmap',
'file' => $file,
'ste' => $conf['ste'],
'source' => 'nmap'
);
break;
case MBSA_TEXT:
case MBSA_XML:
$stack[] = array(
'exec' => 'mbsa',
'file' => $file,
'ste' => $conf['ste'],
'source' => 'mbsa'
);
break;
case MSSQL_XML:
$stack[] = array(
'exec' => 'mssql',
'file' => $file,
'ste' => $conf['ste'],
'source' => 'mssql'
);
break;
case DIRECTORY:
break;
case strpos("UNSUPPORTED", $file) !== false:
rename($file, realpath(TMP . "/unsupported/" . basename($file)));
break;
default:
error_log("Do not have a parser for " . $file);
}
$res = FileDetection($file);
$log->debug("File detected", $res);
switch ($res['type']) {
case NESSUS:
$stack[] = [
'exec' => 'nessus',
'file' => $file,
'ste' => $conf['ste'],
'source' => 'nessus'
];
break;
case SCC_XCCDF:
$stack[] = [
'exec' => 'scc_xccdf',
'file' => $file,
'ste' => $conf['ste'],
'source' => 'scc_xccdf'
];
break;
case STIG_VIEWER_CKL:
$stack[] = [
'exec' => 'stig_viewer',
'file' => $file,
'ste' => $conf['ste'],
'source' => 'stig_viewer'
];
break;
case TECH_ECHECKLIST_EXCEL:
$ignore = false;
if (isset($conf['ignore'])) {
$ignore = true;
}
$stack[] = [
'exec' => 'excel_echecklist',
'file' => $file,
'ste' => $conf['ste'],
'ignore_hidden' => $ignore,
'source' => 'echecklist'
];
break;
case ECHECKLIST_CSV:
$stack[] = [
'exec' => 'csv_echecklist',
'file' => $file,
'ste' => $conf['ste'],
'source' => 'echecklist'
];
break;
case PROC_ECHECKLIST_EXCEL:
$stack[] = [
'exec' => 'proc_echecklist',
'file' => $file,
'ste' => $conf['ste']
];
break;
case HOST_DATA_COLLECTION:
$stack[] = [
'exec' => 'data_collection',
'file' => $file,
'ste' => $conf['ste'],
'target' => $cmd['t'],
'source' => 'data_collection'
];
break;
case NMAP_GREPABLE:
case NMAP_TEXT:
case NMAP_XML:
$stack[] = [
'exec' => 'nmap',
'file' => $file,
'ste' => $conf['ste'],
'source' => 'nmap'
];
break;
case MBSA_TEXT:
case MBSA_XML:
$stack[] = [
'exec' => 'mbsa',
'file' => $file,
'ste' => $conf['ste'],
'source' => 'mbsa'
];
break;
case MSSQL_XML:
$stack[] = [
'exec' => 'mssql',
'file' => $file,
'ste' => $conf['ste'],
'source' => 'mssql'
];
break;
case DIRECTORY:
break;
case strpos("UNSUPPORTED", $file) !== false:
rename($file, realpath(TMP . "/unsupported/" . basename($file)));
break;
default:
error_log("Do not have a parser for " . $file);
}
}
if ($debug) {
Sagacity_Error::err_handler(print_r($stack, true), E_DEBUG);
}
$log->debug("Current script stack", $stack);
foreach ($stack as $key => $s) {
$existing = $dbh->get_Running_Script_Status($s['ste'], $s['file']);
if (isset($existing['status']) && $existing['status'] == 'RUNNING') {
unset($stack[$key]);
continue;
}
$existing = $dbh->get_Running_Script_Status($s['ste'], $s['file']);
if (isset($existing['status']) && $existing['status'] == 'RUNNING') {
$log->warning("Script to parse " . basename($s['file']) . " is already running");
unset($stack[$key]);
continue;
}
$ignore = '';
if ($s['source'] == 'echecklist' && $s['ignore_hidden']) {
$ignore = " -i=1";
}
$ignore = '';
if ($s['source'] == 'echecklist' && $s['ignore_hidden']) {
$ignore = " -i=1";
}
$stack[$key]['script'] = realpath(defined('PHP_BIN') ? PHP_BIN : PHP) .
" -c " . realpath(PHP_CONF) . " " .
" -f " . realpath(DOC_ROOT . "/exec/parse_{$s['exec']}.php") . " --" .
" -f=\"{$s['file']}\"" .
$ignore .
($debug ? " --debug" : "");
$stack[$key]['script'] = realpath(defined('PHP_BIN') ? PHP_BIN : PHP) .
" -c " . realpath(PHP_CONF) . " " .
" -f " . realpath(DOC_ROOT . "/exec/parse_{$s['exec']}.php") . " --" .
" -f=\"{$s['file']}\"" .
$ignore .
($debug ? " --debug" : "");
$dbh->add_Running_Script(basename($s['file']), $s['ste'], $s['source'], $conf['location']);
$log->debug("Adding parser for " . basename($s['file']));
$dbh->add_Running_Script(basename($s['file']), $s['ste'], $s['source'], $conf['location']);
}
$proc = array();
$count = 0;
chdir(realpath(DOC_ROOT . "/exec"));
foreach ($stack as $s) {
$threads[] = new Cocur\BackgroundProcess\BackgroundProcess($s['script']);
end($threads)->run();
$threads[] = new Cocur\BackgroundProcess\BackgroundProcess($s['script']);
end($threads)->run();
sleep(3);
$count++;
$log->info("Starting parser script {$s['script']}");
while ($count >= MAX_RESULTS) {
sleep(1);
$count = $dbh->get_Running_Script_Count($conf['ste']);
}
sleep(3);
$count++;
while ($count >= MAX_RESULTS) {
$log->debug("Current MAX_RESULTS met at " . MAX_RESULTS);
sleep(1);
$count = $dbh->get_Running_Script_Count($conf['ste']);
}
}
do {
sleep(1);
sleep(1);
}
while ($dbh->get_Running_Script_Count($conf['ste']));
if (!$debug) {
unlink(DOC_ROOT . "/exec/parse_config.ini");
unlink(DOC_ROOT . "/exec/parse_config.ini");
}
/**
@ -234,105 +248,107 @@ if (!$debug) {
*
* @param string $file
*/
function import_SCC_OVAL($file) {
if (preg_match('/.*Results\_iavm\_(2009|2010)|Results\_USGCB/i', $file)) {
return;
}
$target_data = array();
$db = new db();
$match = array();
preg_match('/\_SCC-(\d\.?)+\_(\d{4}\-\d{2}\-\d{2}\_\d{6})\_OVAL/', $file, $match);
$time_stamp = $match[2];
$dt = DateTime::createFromFormat('Y-m-d_His', $time_stamp);
$source = $db->get_Sources('SCC');
$dom = new DOMDocument();
$dom->load($file);
$csv = fopen("scc/" . substr(basename($file), 0, -3) . "csv", 'w');
$ste = $db->get_STE($GLOBALS['opt']['s'])[0];
$scan = new scan(null, $source, $ste, 1, basename($file), $dt->format('Y-m-d H:i:s'));
$scan->set_ID($db->save_Scan($scan));
$x = new DOMXPath($dom);
$sysinfo = $x->query('/oval-res:oval_results/oval-res:results/oval-res:system/oval-sc:oval_system_characteristics/oval-sc:system_info')->item(0);
$target_data['os_name'] = $x->query('oval-sc:os_name', $sysinfo)->item(0)->textContent;
$target_data['os_ver'] = $x->query('oval-sc:os_version', $sysinfo)->item(0)->textContent;
$target_data['host_name'] = $x->query('oval-sc:primary_host_name', $sysinfo)->item(0)->textContent;
$interfaces = $x->query('oval-sc:interfaces/oval-sc:interface', $sysinfo);
$int_count = 0;
foreach ($interfaces as $node) {
$target_data['interface_name' . $int_count] = $x->query('oval-sc:interface_name', $node)->item(0)->textContent;
$target_data['ip' . $int_count] = $x->query('oval-sc:ip_address', $node)->item(0)->textContent;
$target_data['mac' . $int_count] = $x->query('oval-sc:mac_address', $node)->item(0)->textContent;
$int_count++;
}
$defs = $x->query('/oval-res:oval_results/oval-def:oval_definitions/oval-def:definitions/oval-def:definition');
foreach ($defs as $node) {
$id = $node->getAttribute('id');
print "Checking oval id: $id" . PHP_EOL;
//$meta = $x->query('oval-def:metadata', $node)->item(0);
$title = $x->query('oval-def:metadata/oval-def:title', $node)->item(0)->textContent;
$desc = $x->query('oval-def:metadata/oval-def:description', $node)->item(0)->textContent;
$plat = $x->query('oval-def:metadata/oval-def:affected/oval-def:platform', $node)->item(0)->textContent;
$ext = $x->query('oval-def:criteria/oval-def:extend_definition', $node);
if ($ext->length > 0) {
$ext_def = $ext->item(0)->getAttribute('definition_ref');
$ext_def_op = $x->query('oval-def:criteria', $node)->item(0)->getAttribute('operator');
}
else {
$ext_def = '';
$ext_def_op = '';
function import_SCC_OVAL($file)
{
if (preg_match('/.*Results\_iavm\_(2009|2010)|Results\_USGCB/i', $file)) {
return;
}
$ref = $x->query('oval-def:metadata/oval-def:reference', $node);
$oval = $db->get_Oval($id);
$target_data = array();
$db = new db();
$match = array();
preg_match('/\_SCC-(\d\.?)+\_(\d{4}\-\d{2}\-\d{2}\_\d{6})\_OVAL/', $file, $match);
$time_stamp = $match[2];
$dt = DateTime::createFromFormat('Y-m-d_His', $time_stamp);
if ($oval->get_PDI_ID()) {
print "current oval: " . print_r($oval, true);
$oval->clear_References();
}
else {
$oval = new oval(null, $id, $title, $desc, $plat, $ext_def, $ext_def_op);
$source = $db->get_Sources('SCC');
$dom = new DOMDocument();
$dom->load($file);
$csv = fopen("scc/" . substr(basename($file), 0, -3) . "csv", 'w');
$ste = $db->get_STE($GLOBALS['opt']['s'])[0];
$scan = new scan(null, $source, $ste, 1, basename($file), $dt->format('Y-m-d H:i:s'));
$scan->set_ID($db->save_Scan($scan));
$x = new DOMXPath($dom);
$sysinfo = $x->query('/oval-res:oval_results/oval-res:results/oval-res:system/oval-sc:oval_system_characteristics/oval-sc:system_info')->item(0);
$target_data['os_name'] = $x->query('oval-sc:os_name', $sysinfo)->item(0)->textContent;
$target_data['os_ver'] = $x->query('oval-sc:os_version', $sysinfo)->item(0)->textContent;
$target_data['host_name'] = $x->query('oval-sc:primary_host_name', $sysinfo)->item(0)->textContent;
$interfaces = $x->query('oval-sc:interfaces/oval-sc:interface', $sysinfo);
$int_count = 0;
foreach ($interfaces as $node) {
$target_data['interface_name' . $int_count] = $x->query('oval-sc:interface_name', $node)->item(0)->textContent;
$target_data['ip' . $int_count] = $x->query('oval-sc:ip_address', $node)->item(0)->textContent;
$target_data['mac' . $int_count] = $x->query('oval-sc:mac_address', $node)->item(0)->textContent;
$int_count++;
}
foreach ($ref as $ref_node) {
$source = $ref_node->getAttribute('source') == 'http://cce.mitre.org' ? 'CCE' : $ref_node->getAttribute('source');
$url = $ref_node->hasAttribute('ref_url') ? $ref_node->getAttribute('ref_url') : '';
$ref_id = $ref_node->getAttribute('ref_id');
$defs = $x->query('/oval-res:oval_results/oval-def:oval_definitions/oval-def:definitions/oval-def:definition');
$oval->add_Reference(new oval_ref($id, $source, $url, $ref_id));
foreach ($defs as $node) {
$id = $node->getAttribute('id');
print "Checking oval id: $id" . PHP_EOL;
//$meta = $x->query('oval-def:metadata', $node)->item(0);
if (is_null($oval->get_PDI_ID()) && $source == 'CCE') {
$cce = $db->get_CCE($ref_id);
$title = $x->query('oval-def:metadata/oval-def:title', $node)->item(0)->textContent;
$desc = $x->query('oval-def:metadata/oval-def:description', $node)->item(0)->textContent;
$plat = $x->query('oval-def:metadata/oval-def:affected/oval-def:platform', $node)->item(0)->textContent;
if (!is_null($cce)) {
$oval->set_PDI_ID($cce->get_PDI_ID());
$ext = $x->query('oval-def:criteria/oval-def:extend_definition', $node);
if ($ext->length > 0) {
$ext_def = $ext->item(0)->getAttribute('definition_ref');
$ext_def_op = $x->query('oval-def:criteria', $node)->item(0)->getAttribute('operator');
}
else {
$ext_def = '';
$ext_def_op = '';
}
}
}
if ($db->save_Oval($oval)) {
error_log("Saved oval id: " . $oval->get_Oval_ID());
$ref = $x->query('oval-def:metadata/oval-def:reference', $node);
$oval = $db->get_Oval($id);
if ($oval->get_PDI_ID()) {
print "current oval: " . print_r($oval, true);
$oval->clear_References();
}
else {
$oval = new oval(null, $id, $title, $desc, $plat, $ext_def, $ext_def_op);
}
foreach ($ref as $ref_node) {
$source = $ref_node->getAttribute('source') == 'http://cce.mitre.org' ? 'CCE' : $ref_node->getAttribute('source');
$url = $ref_node->hasAttribute('ref_url') ? $ref_node->getAttribute('ref_url') : '';
$ref_id = $ref_node->getAttribute('ref_id');
$oval->add_Reference(new oval_ref($id, $source, $url, $ref_id));
if (is_null($oval->get_PDI_ID()) && $source == 'CCE') {
$cce = $db->get_CCE($ref_id);
if (!is_null($cce)) {
$oval->set_PDI_ID($cce->get_PDI_ID());
}
}
}
if ($db->save_Oval($oval)) {
error_log("Saved oval id: " . $oval->get_Oval_ID());
}
else {
error_log("Error saving oval id: " . $oval->get_Oval_ID());
}
}
else {
error_log("Error saving oval id: " . $oval->get_Oval_ID());
}
}
}
function usage() {
print <<<EOO
function usage()
{
print <<<EOO
Purpose: This program was written to look at all files in the /tmp directory, determine what parser is needed, then call that parser with the appropriate flags.
Usage: background_results.php -s={ste_id} [-i=1] [-t=1] [--help]

50
exec/background_stigs.php
View File

@ -26,8 +26,10 @@
* - Apr 5, 2017 - Hard coded parsing 20 STIGs instead of using MAX_RESULTS constant
* - Jun 27, 2017 - Cleanup
* - Jul 13, 2017 - Changed STIG parsing to serial instead of parallel to fix issue with duplicate STIGs from race conditions
* - May 31, 2018 - Added deletion when files match exclusion
* - Jun 2, 2018 - Added code to check STIG_EXCLUSIONS constant to for permanently excluded STIGs
*/
$cmd = getopt("x::h::d::", ["debug::", "delete::", "ia::", "extract::", "help::"]);
$cmd = getopt("x::h::d::", ["debug::", "delete::", "ia::", "extract::", "help::", 'exclude::']);
if (isset($cmd['help']) || isset($cmd['h'])) {
die(usage());
@ -45,7 +47,7 @@ use Monolog\Handler\StreamHandler;
use Monolog\Formatter\LineFormatter;
$log_level = Logger::ERROR;
switch(LOG_LEVEL) {
switch (LOG_LEVEL) {
case E_WARNING:
$log_level = Logger::WARNING;
break;
@ -56,29 +58,32 @@ switch(LOG_LEVEL) {
$log_level = Logger::DEBUG;
}
$stream = new StreamHandler("php://output", Logger::INFO);
if (isset($cmd['debug'])) {
$log_level = Logger::DEBUG;
}
$stream = new StreamHandler("php://output", $log_level);
$stream->setFormatter(new LineFormatter("%datetime% %level_name% %message%" . PHP_EOL, "H:i:s.u"));
$log = new Logger("stig_parser");
$log->pushHandler(new StreamHandler(LOG_PATH . "/stig_parser.log", $log_level));
$log->pushHandler($stream);
$path = realpath(TMP . "/stigs");
if(isset($cmd['d']) && $cmd['d']) {
$path = $cmd['d'];
}
chdir($path);
check_path(TMP . "/stigs");
check_path(TMP . "/stigs/zip");
check_path(TMP . "/stigs/checklist");
check_path(TMP . "/stigs/xml");
check_path(DOC_ROOT . "/reference/stigs");
$db = new db();
$stack = [];
$path = realpath(TMP . "/stigs");
if (isset($cmd['d']) && $cmd['d']) {
$path = $cmd['d'];
}
chdir($path);
$db = new db();
$stack = [];
$zip_files = glob("*.zip");
$zip = new ZipArchive();
@ -125,12 +130,16 @@ $count = 0;
$db->help->update("settings", ['meta_value' => 0], [
[
'field' => 'meta_key',
'op' => '=',
'value' => 'stig-progress'
]
]);
$db->help->execute();
$regex = null;
if (isset($cmd['exclude'])) {
$regex = $cmd['exclude'];
}
foreach ($xml_files as $key => $file) {
// if the file has a space in the file name we need to replace it because it will cause parsing errors
if (strpos($file, ' ') !== false) {
@ -140,6 +149,17 @@ foreach ($xml_files as $key => $file) {
copy(realpath(TMP . "/stigs/xml/$file"), realpath(DOC_ROOT . "/reference/stigs") . "/$file");
}
if (!is_null($regex) && preg_match("/$regex/i", $file)) {
unlink($file);
$log->debug("Skipping $file due to matching regex");
continue;
}
elseif(!empty(STIG_EXCLUSIONS) && preg_match("/" . STIG_EXCLUSIONS . "/i", $file)) {
unlink($file);
$log->debug("Skipping $file due to matching STIG exclusion");
continue;
}
// determine the file type
$ft = FileDetection(TMP . "/stigs/xml/$file");
@ -278,11 +298,13 @@ function usage()
print <<<EOO
Purpose: This program was written to look at all files in the {doc_root}/tmp directory, determine what parser is needed, then call that parser with the appropriate flags.
Usage: background_stigs.php [-x|--extract] [-d="directory"] [--debug] [--delete] [--ia] [-h|--help]
Usage: background_stigs.php [-x|--extract] [-d="directory"] [--debug] [--regex="ex1|ex2"] [--delete] [--ia] [-h|--help]
-x|--extract Simply extract the contents of a .zip file (STIG library) to it's proper places, do not parse the contents
-d="directory" Directory to search for the zip and xml files in (optional, defaults to {doc_root}/tmp)
--regex="ex1|ex2" Insert a valid regex expression (properly escaped) to exclude specific STIGs from parsing
--ia Override any IA controls in the DB to use only the ones that are in the STIG file
--delete Delete any files once complete
--debug Debugging output

1
exec/export-ckl.php
View File

@ -60,7 +60,6 @@ else {
print "Destination: $dest" . PHP_EOL;
$xml = new Array2XML();
Array2XML::$all_caps = true;
$xml->standalone = true;
$xml->formatOutput = true;

32
exec/installer.php
View File

@ -5,7 +5,7 @@
* Purpose: This script runs the installer processes
* Created: Nov 28, 2017
*
* Copyright 2017: Cyber Perspective, LLC, All rights reserved
* Copyright 2017-2018: Cyber Perspective, LLC, All rights reserved
* Released under the Apache v2.0 License
*
* See license.txt for details
@ -15,12 +15,9 @@
* - Dec 27, 2017 - Fixed bug with SCG showing empty, and added download progress meta keys
* - Jan 2, 2018 - Add sleep to fix bug #357 race condition
* - Jan 10, 2018 - Formatting
* - Apr 29, 2018 - Removed settings to move to .sql file. Also, changed to pull CVEs from NVD instead of Mitre repo
*/
include_once 'helper.inc';
include_once 'vendor/autoload.php';
use Cocur\BackgroundProcess\BackgroundProcess;
set_time_limit(0);
$params = [
@ -263,10 +260,10 @@ function save_Database($params)
}
}
if (!$db->real_query("GRANT CREATE TEMPORARY TABLES, INSERT, DELETE, UPDATE, SELECT, TRIGGER ON `rmf`.* TO 'web'@'$host'")) {
if (!$db->real_query("GRANT ALL ON `rmf`.* TO 'web'@'$host'")) {
$errors[] = $db->error;
}
if (!$db->real_query("GRANT CREATE TEMPORARY TABLES, INSERT, DELETE, UPDATE, SELECT, TRIGGER ON `sagacity`.* TO 'web'@'$host'")) {
if (!$db->real_query("GRANT ALL ON `sagacity`.* TO 'web'@'$host'")) {
$errors[] = $db->error;
}
@ -306,22 +303,6 @@ function save_Database($params)
}
}
$help->extended_insert("settings", ["meta_key", "meta_value"], [
['cpe-load-date', new DateTime('1970-01-01')],
['cpe-progress', 0],
['cpe-dl-progress', 0],
['cve-load-date', new DateTime('1970-01-01')],
['cve-progress', 0],
['cve-dl-progress', 0],
['stig-load-date', new DateTime('1970-01-01')],
['stig-progress', 0],
['stig-dl-progress', 0],
['nasl-load-date', new DateTime('1970-01-01')],
['nasl-progress', 0],
['nasl-dl-progress', 0]
], true);
$help->execute();
/*
* ***********************************************************
* Load table data
@ -408,7 +389,7 @@ EOO;
}
if ($params['cve']) {
$cve = " --cve";
$cve = " --nvd";
}
if ($params['stig']) {
@ -424,11 +405,12 @@ EOO;
print json_encode(['success' => true, 'msg' => $msg]);
if (!is_null($cpe) || !is_null($cve) || !is_null($stig)) {
include_once 'vendor/autoload.php';
$script = realpath(PHP_BIN) .
" -c " . realpath(PHP_CONF) .
" -f " . realpath(DOC_ROOT . "/exec/update_db.php") .
" --{$cpe}{$cve}{$stig}{$action}";
$process = new BackgroundProcess($script);
$process = new Cocur\BackgroundProcess\BackgroundProcess($script);
$process->run();
}
}

159
exec/nessus-plugin-import.php
View File

@ -1,12 +1,11 @@
<?php
/**
* File: nessus-plugin-import.php
* Author: Ryan Prather
* Purpose: Script to import all Nessus plugins from *.nasl files
* Created: Jan 5, 2015
*
* Portions Copyright 2016: Cyber Perspectives, All rights reserved
* Portions Copyright 2016-2018: Cyber Perspectives, LLC, All rights reserved
* Released under the Apache v2.0 License
*
* Portions Copyright (c) 2012-2015, Salient Federal Solutions
@ -22,99 +21,125 @@
* - Jan 31, 2017 - Completed testing, ready for prime time
* - Feb 15, 2017 - Store existing plugin IDs in memory for evaluation to check if we should actually run the script,
* Fixed error with PHP_BIN not being defined for some weird reason
* - May 24, 2018 - Added parsing for plugins installed on the local machine
* Added DateTimeDiff helper class
*/
include_once 'config.inc';
include_once "database.inc";
include_once "helper.inc";
$cmd = getopt("h::", array("help::"));
$cmd = getopt("h::", ["help::"]);
if (isset($cmd['h']) || isset($cmd['help'])) {
die(usage());
die(usage());
}
$db = new db();
$time = new DateTimeDiff();
if (!file_exists(TMP . "/nessus_plugins")) {
mkdir(TMP . "/nessus_plugins");
mkdir(TMP . "/nessus_plugins");
}
$nasl_ids = array();
$db->help->select("sagacity.nessus_plugins", array('plugin_id', 'file_date'));
if ($rows = $db->help->execute()) {
foreach ($rows as $row) {
$nasl_ids[$row['plugin_id']] = DateTime::createFromFormat("U", $row['file_date']);
}
$nasl_ids = [];
$db->help->select("sagacity.nessus_plugins", ['plugin_id', 'file_date']);
if ($rows = $db->help->execute()) {
foreach ($rows as $row) {
$nasl_ids[$row['plugin_id']] = DateTime::createFromFormat("U", $row['file_date']);
}
}
chdir(TMP . '/nessus_plugins');
$files = glob("*.nasl");
$start_time = new DateTime();
print "Found " . count($files) . " NASL files\nStarted at {$start_time->format("Y-m-d H:i:s")}\n";
chdir(DOC_ROOT . '/exec');
$x = 0;
foreach ($files as $file) {
$db->help->select("nessus_plugins", array('plugin_id', 'file_date'), [
[
'field' => 'file_name',
'op' => '=',
'value' => basename($file)
]
]);
$row = $db->help->execute();
if (!isset($row['file_name']) || is_null($row['file_date']) || filemtime(TMP . "/nessus_plugins/$file") > $row['file_date']) {
$comp = number_format(($x / count($files)) * 100, 2) . "%";
print "\r$comp";
$script = realpath(defined('PHP_BIN') ? PHP_BIN : PHP) .
" -c " . realpath(PHP_CONF) .
" -f " . realpath(DOC_ROOT . "/exec/nessus-plugin-to-database.php") . " --" .
" -f=\"" . realpath(TMP . "/nessus_plugins/$file") . "\"";
if (substr(strtolower(PHP_OS), 0, 3) == "win") {
$shell = new COM("WScript.Shell");
$shell->CurrentDirectory = DOC_ROOT . "/exec";
$shell->run($script, 0, false);
if (strtolower(substr(PHP_OS, 0, 3)) == 'win') {
if (file_exists(getenv("%ProgramData%") . "/Tenable/Nessus/nessus/plugins")) {
chdir(getenv("%ProgramData%") . "/Tenable/Nessus/nessus/plugins");
$files = array_merge($files, glob("*.nasl"));
}
elseif (substr(strtolower(PHP_OS), 0, 3) == 'lin') {
exec("$script > /dev/null &");
$output = array();
exec("netstat -an | grep TIME_WAIT | wc -l", $output);
if ($output[0] > 1200) {
do {
sleep(1);
exec("netstat -an | grep TIME_WAIT | wc -l", $output);
}
while ($output[0] > 100);
}
}
elseif (strtolower(substr(PHP_OS, 0, 3)) == 'lin') {
if (file_exists("/opt/nessus/lib/nessus/plugins") && is_readable("/opt/nessus/lib/nessus/plugins")) {
chdir("/opt/nessus/lib/nessus/plugins");
$files = array_merge($files, glob("*.nasl"));
}
$x++;
}
if (file_exists("/opt/sc/data/nasl") && is_readable("/opt/sc/data/nasl")) {
chdir("/opt/sc/data/nasl");
$files = array_merge($files, glob("*.nasl"));
}
}
$db->help->update("settings", ['meta_value' => 100], [
[
'field' => 'meta_key',
'op' => IN,
'value' => ['nasl-dl-progress', 'nasl-progress']
]
]);
$db->help->execute();
$files = array_unique($files);
$end_time = new DateTime();
print "Found " . count($files) . " NASL files\nStarted at {$time->getStartClockTime()}\n";
$diff = $end_time->diff($start_time);
chdir(DOC_ROOT . "/exec");
print "\nFinished at {$end_time->format("Y-m-d H:i:s")}\nTotal Time: {$diff->format("%H:%I:%S")}\n";
// Query database to build an array of existing plugins to compare against on import
$existing_plugins = [];
$db->help->select("nessus_plugins", ['plugin_id', 'file_date']);
$rows = $db->help->execute();
if (is_array($rows) && count($rows)) {
foreach ($rows as $row) {
$existing_plugins[$row['plugin_id']] = DateTime::createFromFormat("U", $row['file_date']);
}
}
function usage() {
print <<<EOF
// Sort the files and loop over them
natsort($files);
$threads = [];
$count = 0;
$total_complete = 0;
foreach ($files as $file) {
$db->help->select("nessus_plugins", ['plugin_id', 'file_date'], [
[
'field' => 'file_name',
'value' => basename($file)
]
]);
$row = $db->help->execute();
if (!isset($row['file_name']) || is_null($row['file_date']) || filemtime(TMP . "/nessus_plugins/$file") > $row['file_date']) {
$comp = number_format(($x / count($files)) * 100, 2);
print "\r$comp%";
$script = realpath(defined('PHP_BIN') ? PHP_BIN : PHP) .
" -c " . realpath(PHP_CONF) .
" -f " . realpath(DOC_ROOT . "/exec/nessus-plugin-to-database.php") . " --" .
" -f=\"" . realpath(TMP . "/nessus_plugins/$file") . "\"";
$threads[] = new Cocur\BackgroundProcess\BackgroundProcess($script);
end($threads)->run();
//sleep(1);
$count++;
$total_complete++;
if ($count > 1000) {
$db->set_Setting("nasl-progress", $comp);
foreach ($threads as $k => $t) {
if (!$t->isRunning()) {
unset($threads[$k]);
$count--;
}
}
}
}
}
$db->set_Setting("nasl-dl-progress", 100);
$db->set_Setting("nasl-progress", 100);
$db->set_Setting("nasl-count", $total_complete);
$time->stopClock();
print "\nFinished at {$time->getEndClockTime()}\nTotal Time: {$time->getTotalDiffString()}\n";
function usage()
{
print <<<EOF
Purpose: The purpose of this script is to update the CVE, CPE, and CCE databases. Script will sleep for 3 seconds between actions to allow you review the results.
Usage: php nessus-plugin-import.php [-h|--help]

220
exec/nessus-plugin-to-database.php
View File

@ -1,12 +1,11 @@
<?php
/**
* File: nessus-plugin-to-database.php
* Author: Ryan Prather
* Purpose: Script to read .NASL files and import them to the database
* Created: Jan 15, 2017
*
* Copyright 2017: Cyber Perspectives, All rights reserved
* Copyright 2017-2018: Cyber Perspectives, LLC, All rights reserved
* Released under the Apache v2.0 License
*
* See license.txt for details
@ -15,127 +14,144 @@
* - Jan 15, 2017 - File created
* - Jan 31, 2017 - Competed testing, ready for prime time
* - Apr 5, 2017 - Delete file if error in parsing, check for TMP/nessus_plugins and LOG_PATH/nessus_plugins.log
* - Apr 29, 2018 - Updated to Monolog library and cleaned up script
*/
error_reporting(E_ALL);
include_once 'config.inc';
include_once "database.inc";
include_once "helper.inc";
include_once 'vendor/autoload.php';
$cmd = getopt("f:h::", array("help::", "debug::"));
use Monolog\Logger;
use Monolog\Handler\StreamHandler;
use Monolog\Formatter\LineFormatter;
$cmd = getopt("f:h::", ["help::", "debug::"]);
if (isset($cmd['h']) || isset($cmd['help']) || !isset($cmd['f'])) {
die(usage());
die(usage());
}
elseif (!file_exists($cmd['f'])) {
die("Could not find file specified {$cmd['f']}\n");
die("Could not find file specified {$cmd['f']}\n");
}
check_path(TMP . "/nessus_plugins", true);
check_path(LOG_PATH . "/nessus_plugins.log");
$db = new db();
file_put_contents("check.log", "checking plugin file {$cmd['f']}");
$nasl = new nasl($cmd['f']);
if (!isset($nasl->{'id'})) {
unlink($cmd['f']);
die;
$log_level = Logger::ERROR;
switch (LOG_LEVEL) {
case E_WARNING:
$log_level = Logger::WARNING;
break;
case E_NOTICE:
$log_level = Logger::NOTICE;
break;
case E_DEBUG:
$log_level = Logger::DEBUG;
}
if (isset($cmd['debug'])) {
print_r($nasl);
if (isset($cmd['debug']) && $cmd['debug']) {
$log_level = Logger::DEBUG;
}
$stream = new StreamHandler("php://output", $log_level);
$stream->setFormatter(new LineFormatter("%datetime% %level_name% %message%" . PHP_EOL, "H:i:s.u"));
$log = new Logger("nasl_plugin");
$log->pushHandler(new StreamHandler(LOG_PATH . "/nessus_plugins/{$cmd['f']}.log", $log_level));
$log->pushHandler($stream);
$db = new db();
$nasl = new nasl($cmd['f']);
$plugin_id = 0;
$file_date = null;
$db->help->select("sagacity.nessus_plugins", array('plugin_id', 'file_date'), [
[
'field' => 'plugin_id',
'op' => '=',
'value' => $nasl->id
]
if (!isset($nasl->{'id'})) {
//unlink($cmd['f']);
$log->critical("No ID available");
die;
}
if (isset($cmd['debug'])) {
$log->debug("", [$nasl]);
}
$db->help->select("sagacity.nessus_plugins", ['plugin_id', 'file_date'], [
[
'field' => 'plugin_id',
'op' => '=',
'value' => $nasl->id
]
]);
if ($row = $db->help->execute()) {
$plugin_id = $row['plugin_id'];
$file_date = DateTime::createFromFormat("U", $row['file_date']);
$plugin_id = $row['plugin_id'];
$file_date = DateTime::createFromFormat("U", $row['file_date']);
}
if (($plugin_id && !is_a($file_date, "DateTime")) ||
(is_a($file_date, "DateTime") && isset($nasl->last_modification) && is_a($nasl->last_modification, "DateTime") &&
$file_date->format("U") < $nasl->last_modification->format("U"))) {
file_put_contents(LOG_PATH . "/nessus_plugins.log", "Updating {$nasl->id}\n", FILE_APPEND);
$log->info("Updating {$nasl->id}");
$db->help->update("sagacity.nessus_plugins", [
'file_name' => basename($cmd['f']),
'file_date' => (is_a($file_date, "DateTime") ? $file_date->format("U") : filemtime($cmd['f']))], [
[
'field' => 'plugin_id',
'op' => '=',
'value' => $nasl->id
]
]);
if (!isset($cmd['debug'])) {
if (!$db->help->execute()) {
throw(new Exception("Failed to update the plugin {$nasl->id}", E_WARNING));
$db->help->update("sagacity.nessus_plugins", [
'file_name' => basename($cmd['f']),
'file_date' => (is_a($file_date, "DateTime") ? $file_date->format("U") : filemtime($cmd['f']))], [
[
'field' => 'plugin_id',
'op' => '=',
'value' => $nasl->id
]
]);
if (!isset($cmd['debug'])) {
if (!$db->help->execute()) {
throw(new Exception("Failed to update the plugin {$nasl->id}", E_WARNING));
}
}
}
else {
print "$db->help->sql\n";
}
}
elseif (!$plugin_id) {
file_put_contents(LOG_PATH . "/nessus_plugins.log", "Inserting {$nasl->id}\n", FILE_APPEND);
$log->info("Inserting {$nasl->id}");
$params = [
'plugin_id' => $nasl->id,
'oid' => isset($nasl->oid) ? $nasl->oid : null,
'name' => isset($nasl->name) ? $nasl->name : null,
'copyright' => isset($nasl->copyright) ? $nasl->copyright : null,
'version' => isset($nasl->rev) ? $nasl->rev : null,
'file_name' => basename($cmd['f']),
'file_date' => isset($nasl->last_modification) && is_a($nasl->last_modification, "DateTime") ?
$nasl->last_modification->format("U") : null
];
$params = [
'plugin_id' => $nasl->id,
'oid' => isset($nasl->oid) ? $nasl->oid : null,
'name' => isset($nasl->name) ? $nasl->name : null,
'copyright' => isset($nasl->copyright) ? $nasl->copyright : null,
'version' => isset($nasl->rev) ? $nasl->rev : null,
'file_name' => basename($cmd['f']),
'file_date' => isset($nasl->last_modification) && is_a($nasl->last_modification, "DateTime") ?
$nasl->last_modification->format("U") : null
];
$db->help->insert("sagacity.nessus_plugins", $params, true);
if (!isset($cmd['debug'])) {
if (!$db->help->execute()) {
throw(new Exception("Failed to insert a new plugin {$nasl->id}", E_WARNING));
$db->help->insert("sagacity.nessus_plugins", $params, true);
if (!isset($cmd['debug'])) {
if (!$db->help->execute()) {
throw(new Exception("Failed to insert a new plugin {$nasl->id}", E_WARNING));
}
}
}
else {
print "$db->help->sql\n";
}
}
else {
file_put_contents(LOG_PATH . "/nessus_plugins.log", "No changes to plugin {$nasl->id}\n", FILE_APPEND);
$log->info("No changes to plugin {$nasl->id}");
}
$params = array();
$params = [];
if (isset($nasl->ref)) {
foreach ($nasl->ref as $key => $refs) {
if (is_array($refs)) {
foreach ($refs as $ref) {
$params[] = [
$nasl->id,
$key,
$ref
];
}
foreach ($nasl->ref as $key => $refs) {
if (is_array($refs)) {
foreach ($refs as $ref) {
$params[] = [
$nasl->id,
$key,
$ref
];
}
}
else {
$params[] = [
$nasl->id,
$key,
$refs
];
}
}
else {
$params[] = [
$nasl->id,
$key,
$refs
];
}
}
}
unset($nasl->ref);
@ -146,38 +162,36 @@ unset($nasl->rev);
unset($nasl->last_modification);
foreach ((array) $nasl as $field => $val) {
if (($field == 'id') || (is_array($val) && count($val) > 1)) {
continue;
}
elseif (is_array($val) && count($val) == 1 && isset($val[0])) {
$val = $val[0];
}
$params[] = [
$nasl->id,
$field,
$val
];
if (($field == 'id') || (is_array($val) && count($val) > 1)) {
continue;
}
elseif (is_array($val) && count($val) == 1 && isset($val[0])) {
$val = $val[0];
}
$params[] = [
$nasl->id,
$field,
$val
];
}
if (count($params)) {
$db->help->extended_insert("sagacity.nessus_meta", [
'plugin_id', 'type', 'val'
], $params, true);
$db->help->extended_insert("sagacity.nessus_meta", [
'plugin_id', 'type', 'val'
], $params, true);
}
if (!isset($cmd['debug'])) {
$db->help->execute();
$db->help->execute();
}
else {
print $db->help->sql . PHP_EOL;
print $db->help->sql . PHP_EOL;
}
if (!isset($cmd['debug'])) {
unlink($cmd['f']);
}
function usage() {
print <<<EOL
function usage()
{
print <<<EOL
Purpose: This script is for reading NASL files and adding them to the database
Usage: php nessus-plugin-to-database.php -f={NASL file to parse} [--debug]

9
exec/parse_cve.php
View File

@ -225,15 +225,6 @@ if ($items->length) {
$sys->help->extended_replace("cve_web", $web_fields, $new_cve_web);
$sys->help->execute();
}
$sys->help->update("settings", ['meta_value' => 100], [
[
'field' => 'meta_key',
'op' => IN,
'value' => ['cve-dl-progress', 'cve-progress']
]
]);
$sys->help->execute();
}
function usage() {

49
exec/parse_excel_echecklist.php
View File

@ -5,7 +5,7 @@
* Purpose: Parse the Excel version (.xlsx or .xls) of an eChecklist
* Created: May 9, 2014
*
* Portions Copyright 2016-2017: Cyber Perspectives, LLC, All rights reserved
* Portions Copyright 2016-2018: Cyber Perspectives, LLC, All rights reserved
* Released under the Apache v2.0 License
*
* Portions Copyright (c) 2012-2015, Salient Federal Solutions
@ -24,6 +24,7 @@
* - May 26, 2017 - Migrated to PHPSpreadsheet library
* - Aug 28, 2017 - Fixed couple minor bugs
* - Jan 15, 2018 - Formatting, reorganized use statements, and cleaned up
* - May 24, 2018 - Attempt to fix bug #413
*/
$cmd = getopt("f:", ['debug::', 'help::']);
set_time_limit(0);
@ -48,17 +49,21 @@ include_once 'excelConditionalStyles.inc';
use PhpOffice\PhpSpreadsheet\Cell\Coordinate;
use PhpOffice\PhpSpreadsheet\Worksheet\Worksheet;
use Monolog\Logger;
use Monolog\Handler\StreamHandler;
check_path(TMP . "/echecklist");
chdir(TMP);
$log_level = convert_log_level();
$db = new db();
$base_name = basename($cmd['f']);
$log = new Sagacity_Error($cmd['f']);
$log = new Logger("excel-echecklist");
$log->pushHandler(new StreamHandler(logify($cmd['f']), $log_level));
if (!file_exists($cmd['f'])) {
$db->update_Running_Scan($base_name, ['name' => 'status', 'value' => 'ERROR']);
$log->script_log("File not found", E_ERROR);
die($log->emergency("File not found"));
}
$db->update_Running_Scan($base_name, ['name' => 'pid', 'value' => getmypid()]);
@ -68,7 +73,7 @@ if (is_array($src) && count($src) && isset($src[0]) && is_a($src[0], 'source'))
$src = $src[0];
}
else {
$log->script_log("Could not find the source", E_ERROR);
die($log->emergency("Could not find the source"));
}
/*
@ -94,13 +99,13 @@ else {
$ste = $ste[0];
}
else {
$log->script_log("Could not retrieve the ST&E", E_ERROR);
die($log->emergency("Could not retrieve ST&E"));
}
$scan = new scan(null, $src, $ste, 1, $base_name, $dt->format('Y-m-d'));
if (!$scan_id = $db->save_Scan($scan)) {
$log->script_log("Failed to add scan for file: {$cmd['f']}", E_ERROR);
die($log->error("Failed to add scan for file: {$cmd['f']}"));
}
$scan->set_ID($scan_id);
@ -113,11 +118,11 @@ if (is_array($gen_os) && count($gen_os) && isset($gen_os[0]) && is_a($gen_os[0],
foreach ($objSS->getWorksheetIterator() as $wksht) {
if (preg_match('/Instruction|Cover Sheet/i', $wksht->getTitle())) {
$log->script_log("Skipping instruction and cover sheet", E_DEBUG);
$log->debug("Skipping instruction and cover worksheet");
continue;
}
elseif (isset($conf['ignore']) && $wksht->getSheetState() == Worksheet::SHEETSTATE_HIDDEN) {
$log->script_log("Skipping hidden worksheet {$wksht->getTitle()}");
$log->info("Skipping hidden worksheet {$wksht->getTitle()}");
continue;
}
@ -132,17 +137,17 @@ foreach ($objSS->getWorksheetIterator() as $wksht) {
if ($thread_status['status'] == 'TERMINATED') {
unset($objSS);
rename(realpath(TMP . "/{$scan->get_File_Name()}"), TMP . "/terminated/{$scan->get_File_Name()}");
$log->script_log("File parsing terminated by user");
$log->notice("File parsing terminated by user");
}
$log->script_log("Reading from {$wksht->getTitle()} worksheet");
$log->notice("Reading from {$wksht->getTitle()}");
if (!preg_match('/STIG ID/i', $wksht->getCell("A10")->getValue()) &&
!preg_match('/VMS ID/i', $wksht->getCell("B10")->getValue()) &&
!preg_match('/CAT/i', $wksht->getCell("C10")->getValue()) &&
!preg_match('/IA Controls/i', $wksht->getCell("D10")->getValue()) &&
!preg_match('/Short Title/i', $wksht->getCell("E10")->getValue())) {
$log->script_log("Invalid headers in {$wksht->getTitle()}", E_WARNING);
$log->warning("Invalid headers in {$wksht->getTitle()}");
continue;
}
@ -177,13 +182,12 @@ foreach ($objSS->getWorksheetIterator() as $wksht) {
if ($thread_status['status'] == 'TERMINATED') {
unset($objSS);
rename(realpath(TMP . "/{$scan->get_File_Name()}"), TMP . "/terminated/{$scan->get_File_Name()}");
$log->script_log("File parsing terminated by user");
die;
die($log->notice("File parsing terminated by user"));
}
if ($cell->getColumn() > $short_title_col && !preg_match('/Overall/i', $cell->getValue())) {
if (preg_match('/status/i', $cell->getValue())) {
$log->script_log("Error: Invalid host name ('status') in {$wksht->getTitle()}", E_WARNING);
$log->error("Invalid host name ('status') in {$wksht->getTitle()}");
break;
}
@ -193,7 +197,7 @@ foreach ($objSS->getWorksheetIterator() as $wksht) {
$tgt = $tgt[0];
}
else {
$log->script_log("Could not find host {$cell->getValue()}", E_ERROR);
$log->error("Could not find host {$cell->getValue()}");
}
}
else {
@ -251,7 +255,7 @@ foreach ($objSS->getWorksheetIterator() as $wksht) {
$idx['check_contents'] += count($tgts);
}
elseif (empty($tgts)) {
$log->script_log("Failed to identify targets in worksheet {$wksht->getTitle()}", E_WARNING);
$log->warning("Failed to identify targets in worksheet {$wksht->getTitle()}");
continue;
}
@ -281,7 +285,7 @@ foreach ($objSS->getWorksheetIterator() as $wksht) {
$pdi->set_Short_Title($short_title);
$pdi->set_Group_Title($short_title);
if (!($pdi_id = $db->save_PDI($pdi))) {
$log->script_log("Failed to add a new PDI for STIG ID $stig_id", E_ERROR);
die($log->error("Failed to add new PDI for STIG ID {$stig_id}"));
}
$stig = new stig($pdi_id, $stig_id, $short_title);
@ -293,14 +297,21 @@ foreach ($objSS->getWorksheetIterator() as $wksht) {
$status = $wksht->getCell(Coordinate::stringFromColumnIndex($idx['target'] + $x) . $row->getRowIndex())
->getValue();
$log->script_log("{$tgt->get_Name()} {$stig->get_ID()} ($status)\n", E_DEBUG);
$log->debug("{$tgt->get_Name()} {$stig->get_ID()} ($status)");
$finding = $db->get_Finding($tgt, $stig);
if (is_array($finding) && count($finding) && isset($finding[0]) && is_a($finding[0], 'finding')) {
$tmp = $finding[0];
$tmp->set_Finding_Status_By_String($status);
if(preg_match("/Not a Finding|Not Applicable/i", $status)) {
$ds = $tmp->get_Deconflicted_Status($status);
$tmp->set_Finding_Status_By_String($ds);
}
else {
$tmp->set_Finding_Status_By_String($status);
}
$tmp->set_Notes($notes);
$tmp->set_Category($cat_lvl);

12
exec/parse_nessus.php
View File

@ -41,6 +41,7 @@
* - Oct 27, 2017 - Fix to convert '*' to '0.0.0.0' or '::' and validate IP's before making interface
* - Nov 25, 2017 - Fixed bug #345
* - Jan 16, 2018 - Updated to use host_list class
* - Jun 4, 2018 - Fixed bug #424 (IP address not pulled when name used for host)
*/
error_reporting(E_ALL);
@ -491,6 +492,16 @@ class nessus_parser extends scan_xml_parser
}
}
if (!empty($this->tag['host-ip']) && validation::valid_ip($this->tag['host-ip'])) {
if (!isset($this->tgt->interfaces[$this->tag['host-ip']])) {
$this->log->script_log("Adding new interface to target with IP: {$this->tag['host-ip']}");
$this->tgt->interfaces[$this->tag['host-ip']] = new interfaces(null, $this->tgt->get_ID(), null, $this->tag['host-ip'], null, $this->host->hostname, $this->host->fqdn, null);
}
else {
$this->log->script_log("Interface already exists for target: {$this->tag['host-ip']}");
}
}
$netstat_keys = preg_grep("/netstat\-established\-tcp/", array_keys($this->tag));
$this->log->script_log("Start established tcp conns...found " . count($netstat_keys) . " connections", E_DEBUG);
foreach (array_values($netstat_keys) as $key) {
@ -503,6 +514,7 @@ class nessus_parser extends scan_xml_parser
$this->log->script_log("Start listening tcp4 conns...found " . count($netstat_keys) . " connections", E_DEBUG);
if (between(count($netstat_keys), 1, PORT_LIMIT)) {
foreach (array_values($netstat_keys) as $key) {
// split into "ip:port" array
$ip_port = explode(":", $this->tag[$key]);
// skip this entry if it is not a valid IP

255
exec/parse_nvd_json_cve.php
View File

@ -1,18 +1,19 @@
<?php
/**
* File: parse_nvd_json_cve
* Author: Ryan Prather <ryan.prather@cyberperspectives.com>
* Purpose:
* Created: Dec 30, 2017
* Created: Apr 29, 2018
*
* Copyright 2017: Cyber Perspective, LLC, All rights reserved
* Copyright 2018: Cyber Perspective, LLC, All rights reserved
* Released under the Apache v2.0 License
*
* See license.txt for details
*
* Change Log:
* - Dec 30, 2017 - File created
* - Apr 29, 2018 - File created
* - May 10, 2018 - Formatting and fixed performance issue on Windows (bug #403)
* - Jun 5, 2018 - Fix for bug #425
*/
include_once 'config.inc';
include_once 'helper.inc';
@ -28,7 +29,7 @@ ini_set('memory_limit', '2G');
$cmd = getopt("f:");
if (!isset($cmd['f']) || isset($cmd['h'])) {
die(usage());
die(usage());
}
$log_level = Logger::ERROR;
@ -44,192 +45,166 @@ switch (LOG_LEVEL) {
break;
}
$stream = new StreamHandler("php://output", $log_level);
$stream->setFormatter(new LineFormatter("%datetime% %level_name% %message%", "H:i:s.u"));
$log = new Logger("nvd_cve");
$log->pushHandler($stream);
$log->pushHandler(new StreamHandler(LOG_PATH . "/nvd_cve.log", $log_level));
$db = new db();
$json = json_decode(file_get_contents($cmd['f']));
$db = new db();
$json = json_decode(file_get_contents($cmd['f']));
$existing_cves = [];
$db->help->select("cve_db", ['cve_id']);
$cves = $db->help->execute();
if (is_array($cves) && count($cves)) {
foreach ($cves as $cve) {
$existing_cves["{$cve['cve_id']}"] = 1;
}
foreach ($cves as $cve) {
$existing_cves["{$cve['cve_id']}"] = 1;
}
}
print "Currently " . count($existing_cves) . " in DB" . PHP_EOL . "Parsing: " . count($json->CVE_Items) . " items" . PHP_EOL;
$new_cves = [];
$db_cpes = [];
$new_cves = [];
$new_cve_refs = [];
$new_cve_web = [];
$sw_rows = [];
$new = 0;
$existing = 0;
$new_cve_web = [];
$sw_rows = [];
$new = 0;
$existing = 0;
$db->help->select("software", ['id', 'cpe']);
$rows = $db->help->execute();
foreach ($rows as $row) {
$db_cpes["{$row['cpe']}"] = $row['id'];
}
$cve_fields = [
'cve_id', 'seq', 'status', 'phase', 'phase_date', 'desc'
'cve_id', 'seq', 'status', 'phase', 'phase_date', 'desc'
];
$ref_fields = [
'cve_seq', 'source', 'url', 'val'
'cve_seq', 'source', 'url', 'val'
];
$web_fields = [
'cve_id', 'xml'
'cve_id', 'xml'
];
foreach ($json->CVE_Items as $cve) {
if (!isset($existing_cves["{$cve->cve->CVE_data_meta->ID}"])) {
$log->debug("Adding {$cve->cve->CVE_data_meta->ID}");
$new++;
if (!isset($existing_cves["{$cve->cve->CVE_data_meta->ID}"])) {
$log->debug("Adding {$cve->cve->CVE_data_meta->ID}");
$new++;
$desc = [];
$status = null;
$phase = null;
$cpes = [];
$name = $cve->cve->CVE_data_meta->ID;
$type = $cve->cve->data_type;
$seq = $cve->cve->CVE_data_meta->ID;
$pd = new DateTime($cve->publishedDate);
$lmd = new DateTime($cve->lastModifiedDate);
$desc = [];
$status = null;
$phase = null;
$cpes = [];
$name = $cve->cve->CVE_data_meta->ID;
$type = $cve->cve->data_type;
$seq = $cve->cve->CVE_data_meta->ID;
$pd = new DateTime($cve->publishedDate);
$lmd = new DateTime($cve->lastModifiedDate);
if (is_array($cve->cve->description->description_data) && count($cve->cve->description->description_data)) {
foreach ($cve->cve->description->description_data as $d) {
$desc[] = $d->value;
}
}
if (is_array($cve->cve->description->description_data) && count($cve->cve->description->description_data)) {
foreach ($cve->cve->description->description_data as $d) {
$desc[] = $d->value;
}
}
$new_cves[] = [
$name, $seq, $status, $phase, $pd, implode(PHP_EOL, $desc)
];
if (is_array($cve->cve->references->reference_data) && count($cve->cve->references->reference_data)) {
foreach ($cve->cve->references->reference_data as $ref) {
$log->debug("Adding reference {$ref->url}");
$new_cve_refs[] = [
$name, null, $ref->url, null
$new_cves[] = [
$name, $seq, $status, $phase, $pd, implode(PHP_EOL, $desc)
];
}
}
if (is_array($cve->configurations->nodes) && count($cve->configurations->nodes)) {
foreach ($cve->configurations->nodes as $n) {
if (isset($n->cpe) && is_array($n->cpe) && count($n->cpe)) {
foreach ($n->cpe as $cpe) {
if (isset($cpe->cpe22Uri)) {
$cpes[] = $cpe->cpe22Uri;
if (is_array($cve->cve->references->reference_data) && count($cve->cve->references->reference_data)) {
foreach ($cve->cve->references->reference_data as $ref) {
$log->debug("Adding reference {$ref->url}");
$new_cve_refs[] = [
$name, null, $ref->url, null
];
}
elseif (isset($cpe->cpeMatchString)) {
$cpes[] = $cpe->cpeMatchString;
}
if (is_array($cve->configurations->nodes) && count($cve->configurations->nodes)) {
foreach ($cve->configurations->nodes as $n) {
if (isset($n->cpe) && is_array($n->cpe) && count($n->cpe)) {
foreach ($n->cpe as $cpe) {
if (isset($cpe->cpe22Uri)) {
$cpes[] = $cpe->cpe22Uri;
}
elseif (isset($cpe->cpeMatchString)) {
$cpes[] = $cpe->cpeMatchString;
}
}
}
}
}
}
}
}
if (count($cpes)) {
$sw_ids = $db->get_Software_Ids($cpes);
if (is_array($sw_ids) && count($sw_ids) && isset($sw_ids['id'])) {
$sw_ids = [0 => $sw_ids];
}
if (is_array($sw_ids) && count($sw_ids) && isset($sw_ids[0])) {
foreach ($sw_ids as $sw) {
$sw_rows[] = [$name, $sw];
if (count($cpes)) {
foreach ($cpes as $cpe) {
if (isset($db_cpes["{$cpe}"])) {
$sw_rows[] = [$name, $db_cpes["{$cpe}"]];
}
}
}
}
print "*";
}
else {
$existing++;
print ".";
}
print "*";
}
else {
$existing++;
print ".";
}
if (($new + $existing) % 100 == 0) {
if (count($new_cves)) {
$db->help->extended_insert("cve_db", $cve_fields, $new_cves, true);
$db->help->execute();
}
if (($new + $existing) % 100 == 0) {
if (count($new_cves)) {
$db->help->extended_insert("cve_db", $cve_fields, $new_cves, true);
$db->help->execute();
if (count($new_cve_refs)) {
$db->help->extended_insert("cve_references", $ref_fields, $new_cve_refs, true);
$db->help->execute();
}
if (count($sw_rows)) {
$db->help->extended_insert("cve_sw_lookup", ['cve_id', 'sw_id'], $sw_rows, true);
$db->help->execute();
}
$new_cves = [];
$new_cve_refs = [];
$new_cve_web = [];
$sw_rows = [];
print "\t" . ($existing + $new) . " completed" . PHP_EOL;
$db->help->update("settings", ['meta_value' => number_format((($existing + $new) / count($json->CVE_Items)) * 100, 2)], [
[
'field' => 'meta_key',
'value' => 'nvd-cve-progress'
]
]);
$db->help->execute();
}
if (count($new_cve_refs)) {
$db->help->extended_insert("cve_references", $ref_fields, $new_cve_refs, true);
$db->help->execute();
}
if (count($sw_rows)) {
$db->help->extended_insert("cve_sw_lookup", ['cve_id', 'sw_id'], $sw_rows, true);
$db->help->execute();
}
$new_cves = [];
$new_cve_refs = [];
$new_cve_web = [];
$sw_rows = [];
print "\t" . ($existing + $new) . " completed" . PHP_EOL;
$db->help->update("settings", ['meta_value' => number_format((($existing + $new) / count($json->CVE_Items)) * 100, 2)], [
[
'field' => 'meta_key',
'op' => '=',
'value' => 'nvd-cve-progress'
]
]);
$db->help->execute();
}
}
if (count($new_cves)) {
$db->help->extended_insert("cve_db", $cve_fields, $new_cves, true);
$db->help->execute();
$db->help->extended_insert("cve_db", $cve_fields, $new_cves, true);
$db->help->execute();
}
if (count($new_cve_refs)) {
$db->help->extended_insert("cve_references", $ref_fields, $new_cve_refs, true);
$db->help->execute();
$db->help->extended_insert("cve_references", $ref_fields, $new_cve_refs, true);
$db->help->execute();
}
if (count($sw_rows)) {
$db->help->extended_insert("cve_sw_lookup", ['cve_id', 'sw_id'], $sw_rows, true);
$db->help->execute();
$db->help->extended_insert("cve_sw_lookup", ['cve_id', 'sw_id'], $sw_rows, true);
$db->help->execute();
}
$db->help->update("settings", ['meta_value' => 100], [
[
'field' => 'meta_key',
'op' => '=',
'value' => 'nvd-cve-progress'
]
]);
$db->help->execute();
$db->help->update("settings", ['meta_value' => 100], [
[
'field' => 'meta_key',
'op' => IN,
'value' => ['cve-dl-progress', 'cve-progress']
]
]);
$db->help->execute();
$db->help->update("settings", ['meta_value' => new DateTime()], [
[
'field' => 'meta_key',
'op' => IN,
'value' => ['cve-load-date', 'nvd-cve-load-date']
]
]);
$db->help->execute();
unlink($cmd['f']);
print PHP_EOL;
function usage() {
print <<<EOF
function usage()
{
print <<<EOF
Purpose: To import the National Vulnerability Database (NVD) CVE JSON files
Usage: php parse_nvd_json_cve.php -f={JSON file} [-h]

5
exec/parse_stig.php
View File

@ -29,6 +29,7 @@
* - Jul 23, 2017 - MAS Added comments
* - Aug 28, 2017 - Added die for draft stigs
* - Dec 27, 2017 - Added up date for load date
* - May 10, 2018 - Starting to migrate logging and fixed install status bar issues (#403)
*/
$cmd = getopt("f:", ['debug::', 'ia_reset::', 'draft::', 'help::']);
@ -670,6 +671,10 @@ foreach ($groups as $group) {
$db->update_Catalog_Script($base_name, ['name' => 'perc_comp', 'value' => ($perc_comp / $groups->length) * 100]);
}
$db->help->select_count("sagacity.stigs");
$stig_count = $db->help->execute();
$db->set_Setting('stig-count', $stig_count);
$end = new DateTime();
$diff = $end->diff($start);

377
exec/update_db.php
View File

@ -5,7 +5,7 @@
* Author: Ryan Prather
* Created: Jan 5, 2015
*
* Portions Copyright 2016-2017: Cyber Perspectives, All rights reserved
* Portions Copyright 2016-2018: Cyber Perspectives, LLC, All rights reserved
* Released under the Apache v2.0 License
*
* Portions Copyright (c) 2012-2015, Salient Federal Solutions
@ -45,6 +45,9 @@
* Fixed confusion with Cygwin and Bash on Windows paths
* - Jun 27, 2017 - Matt Shuter: Fixed bug #262 & #270
* - Dec 27, 2017 - Added database field and download progress flag
* - Apr 29, 2018 - Added extract parameter to only extract nasl archive file, fixed a couple bugs
* - May 10, 2018 - Removed ping of cve.mitre.org, and added 'po' and 'do' parameters for NVD CVE
* - Jun 5, 2018 - Fixed a couple setting updates
*/
include_once 'config.inc';
include_once 'helper.inc';
@ -62,7 +65,7 @@ $total_time = null;
$total_diff = 0;
$summary_stats = [];
$cmd = getopt("h::u::p::", ['cpe::', 'cce::', 'cve::', 'nvd::', 'nasl::', 'stig::', 'do::', 'po::', 'help::']);
$cmd = getopt("h::u::p::", ['cpe::', 'cce::', 'cve::', 'nvd::', 'nasl::', 'stig::', 'do::', 'po::', 'help::', 'debug::', 'extract::', 'exclude::']);
$db = new db();
$diff = new DateTimeDiff();
@ -79,7 +82,11 @@ switch (LOG_LEVEL) {
$log_level = Logger::DEBUG;
}
$stream = new StreamHandler("php://output", Logger::INFO);
if (isset($cmd['debug']) && $cmd['debug']) {
$log_level = Logger::DEBUG;
}
$stream = new StreamHandler("php://output", $log_level);
$stream->setFormatter(new LineFormatter("%datetime% %level_name% %message%" . PHP_EOL, "H:i:s.u"));
$log = new Logger("update_db");
@ -93,7 +100,7 @@ if (isset($cmd['h'], $cmd['help']) ||
if (isset($cmd['do']) || !isset($cmd['po'])) {
if (!ping("cyberperspectives.com")) {
die("Cannot connect to internet" . PHP_EOL);
$log->emergency("Cannot connect to internet");
}
}
@ -101,8 +108,11 @@ if (isset($cmd['do']) || !isset($cmd['po'])) {
* Update CPE content downloaded from NIST
*/
if (isset($cmd['cpe'])) {
$db->set_Setting('cpe-dl-progress', 0);
$db->set_Setting('cpe-progress', 0);
$db->set_Setting_Array([
'cpe-dl-progress' => 0,
'cpe-progress' => 0,
'cpe-count' => 0
]);
$path = TMP . "/cpe";
if (isset($cmd['p']) && $cmd['p']) {
@ -112,7 +122,7 @@ if (isset($cmd['cpe'])) {
check_path($path);
$diff->resetClock();
$log->info("Started CPE ingestion ({$diff->getStartClockTime()})");
print "Started CPE ingestion ({$diff->getStartClockTime()})" . PHP_EOL;
// search for an unzip any zip files in the tmp directory
$zip_files = glob("{$path}/*cpe-dictionary*.zip");
@ -120,7 +130,7 @@ if (isset($cmd['cpe'])) {
$log->debug("Found a existing cpe-dictionary.zip file, unzipping then parsing");
$zip = new ZipArchive();
foreach ($zip_files as $file) {
$log->info("Unzipping {$file}");
$log->debug("Unzipping {$file}");
$zip->open($file);
$zip->extractTo($path);
$zip->close();
@ -135,7 +145,7 @@ if (isset($cmd['cpe'])) {
foreach ($tmp_files as $fname) {
$name = basename($fname);
if ($name == 'official-cpe-dictionary_v2.3.xml') {
$name = "cpe-dictionary-{$start_time->format("Ymd")}.xml";
$name = "cpe-dictionary-{$diff->getStartClock()->format("Ymd")}.xml";
}
rename($fname, "{$path}/{$name}");
}
@ -146,14 +156,14 @@ if (isset($cmd['cpe'])) {
$cpe_parse_fname = null;
// download the file if the do flag is used even if it already exists
if (isset($cmd['do']) && ping("nist.gov") && !isset($cmd['po'])) {
download_file($cpe_url, $cpe_fname, $db, 'cpe-dl-progress');
if (isset($cmd['do']) && !isset($cmd['po'])) {
download_file($cpe_url, $cpe_fname, $db->help, 'cpe-dl-progress');
}
// download the file only if it doesn't exist
elseif (!file_exists($cpe_fname) && ping("nist.gov") && !isset($cmd['po'])) {
download_file($cpe_url, $cpe_fname, $db, 'cpe-dl-progress');
elseif (!file_exists($cpe_fname) && !isset($cmd['po'])) {
download_file($cpe_url, $cpe_fname, $db->help, 'cpe-dl-progress');
}
elseif (!isset($cmd['po']) && !ping("nist.gov")) {
elseif (!isset($cmd['po'])) {
$log->error("Could not connect to nist.gov to download the CPE library");
die();
}
@ -185,7 +195,7 @@ if (isset($cmd['cpe'])) {
}
if (is_null($cpe_parse_fname)) {
$log->warning("Coult not find a CPE file to parse");
$log->warning("Could not find a CPE file to parse");
}
$script = realpath(defined('PHP_BIN') ? PHP_BIN : PHP) .
@ -194,14 +204,19 @@ if (isset($cmd['cpe'])) {
" -f=\"" . realpath($cpe_parse_fname) . "\"" .
" -d=\"{$dt->format("Y-m-d")}\"";
$log->info("Running parsing script");
$log->debug("Running CPE parsing script on file: $cpe_parse_fname");
passthru($script);
}
$db->help->select_count("software");
$cpe_count = $db->help->execute();
$db->set_Setting("cpe-count", $cpe_count);
$diff->stopClock();
$log->info(PHP_EOL . "Finished at {$diff->getEndClockTime()}" . PHP_EOL .
"Total time: {$diff->getDiffString()}");
print PHP_EOL . "Finished at {$diff->getEndClockTime()}" . PHP_EOL .
"Total time: {$diff->getDiffString()}" . PHP_EOL;
sleep(3);
}
@ -210,8 +225,14 @@ if (isset($cmd['cpe'])) {
* Update CVE content
*/
if (isset($cmd['cve'])) {
$db->set_Setting('cve-dl-progress', 0);
$db->set_Setting('cve-progress', 0);
$db->set_Setting_Array([
'nvd-cve-dl-progress' => 0,
'nvd-cve-progress' => 0,
'nvd-cve-count' => 0,
'cve-dl-progress' => 0,
'cve-progress' => 0,
'cve-count' => 0
]);
$path = TMP . "/cve";
if (isset($cmd['p']) && $cmd['p']) {
$path = $cmd['p'];
@ -220,12 +241,12 @@ if (isset($cmd['cve'])) {
check_path($path);
$diff->resetClock();
$log->info("Started CVE ingestion {$diff->getStartClockTime()}");
print "Started CVE ingestion {$diff->getStartClockTime()}" . PHP_EOL;
$cve_files = glob(TMP . "/allitems.xml");
if (count($cve_files)) {
foreach ($cve_files as $file) {
rename($file, "{$path}/cve-all-{$start_time->format("Ymd")}.xml");
rename($file, "{$path}/cve-all-{$diff->getStartClock()->format("Ymd")}.xml");
}
}
@ -240,13 +261,13 @@ if (isset($cmd['cve'])) {
$cve_url = "http://cve.mitre.org/data/downloads/allitems.xml";
$cve_parse_fname = null;
if (isset($cmd['do']) && ping("cve.mitre.org") && !isset($cmd['po'])) {
download_file($cve_url, $cve_fname, $db, 'cve-dl-progress');
if (isset($cmd['do']) && !isset($cmd['po'])) {
download_file($cve_url, $cve_fname, $db->help, 'cve-dl-progress');
}
elseif (!file_exists($cve_fname) && ping("cve.mitre.org") && !isset($cmd['po'])) {
download_file($cve_url, $cve_fname, $db, 'cve-dl-progress');
elseif (!file_exists($cve_fname) && !isset($cmd['po'])) {
download_file($cve_url, $cve_fname, $db->help, 'cve-dl-progress');
}
elseif (!isset($cmd['po']) && !ping("cve.mitre.org")) {
elseif (!isset($cmd['po'])) {
Sagacity_Error::err_handler("Could not connect to cve.mitre.org to download the CVE library", E_ERROR);
}
@ -271,7 +292,7 @@ if (isset($cmd['cve'])) {
}
if (is_null($cve_parse_fname)) {
$log->error("Coult not find a CVE file to parse");
$log->error("Could not find a CVE file to parse");
die;
}
@ -281,21 +302,44 @@ if (isset($cmd['cve'])) {
" -f=\"" . realpath($cve_parse_fname) . "\"" .
" -d=\"{$dt->format("Y-m-d")}\"";
$log->info("Script to run $script");
$log->debug("Script to run $script");
passthru($script);
}
$db->help->select_count("sagacity.cve_db");
$cve_count = $db->help->execute();
$db->set_Setting_Array([
'cve-dl-progress' => 100,
'cve-progress' => 100,
'cve-count' => $cve_count,
'cve-load-date' => new DateTime(),
'nvd-cve-dl-progress' => 100,
'nvd-cve-progress' => 100,
'nvd-cve-count' => $cve_count,
'nvd-cve-load-date' => new DateTime()
]);
$diff->stopClock();
$log->info("Finished at {$diff->getEndClockTime()}" . PHP_EOL .
"Total Time: {$diff->getDiffString()}");
print "Finished at {$diff->getEndClockTime()}" . PHP_EOL .
"Total Time: {$diff->getDiffString()}" . PHP_EOL;
sleep(3);
}
/**
* Update to NVD CVE content
*/
if (isset($cmd['nvd'])) {
$db->set_Setting('nvd-cve-dl-progress', 0);
$db->set_Setting('nvd-cve-progress', 0);
$db->set_Setting_Array([
'nvd-cve-dl-progress' => 0,
'nvd-cve-progress' => 0,
'nvd-cve-count' => 0,
'cve-dl-progress' => 0,
'cve-progress' => 0,
'cve-count' => 0
]);
$path = TMP . "/nvd";
if (isset($cmd['p']) && $cmd['p']) {
$path = $cmd['p'];
@ -303,60 +347,87 @@ if (isset($cmd['nvd'])) {
check_path($path);
$diff->resetClock();
$log->info("Started NVD CVE ingestion ({$diff->getStartClockTime()})");
print "Started NVD CVE ingestion ({$diff->getStartClockTime()})" . PHP_EOL;
$nvd_years = [];
for ($x = 2002; $x <= $diff->getStartClock()->format("Y"); $x++) {
$nvd_years[] = $x;
}
$too_old = new DateTime();
$too_old->sub(DateInterval::createFromDateString("7 days"));
if (isset($cmd['do']) || !isset($cmd['po'])) {
$too_old = new DateTime();
$too_old->sub(DateInterval::createFromDateString("7 days"));
$load_date = new DateTime($db->get_Settings("nvd-cve-load-date"));
if ($load_date < $too_old) {
// More than 7 days old so have to do a full load
foreach ($nvd_years as $yr) {
download_file("https://static.nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-{$yr}.json.zip", TMP . "/nvd/nvdcve-{$yr}.json.zip");
$load_date = new DateTime($db->get_Settings("nvd-cve-load-date"));
if ($load_date < $too_old) {
// More than 7 days old so have to do a full load
foreach ($nvd_years as $yr) {
$db->set_Setting('nvd-year', $yr);
download_file("https://static.nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-{$yr}.json.zip", TMP . "/nvd/nvdcve-{$yr}.json.zip", $db->help, 'nvd-cve-dl-progress');
$zip = new ZipArchive();
$zip->open(TMP . "/nvd/nvdcve-{$yr}.json.zip");
$zip->extractTo(TMP . "/nvd");
$zip->close();
unlink(TMP . "/nvd/nvdcve-{$yr}.json.zip");
}
}
else {
$db->set_Setting('nvd-year', 'modified');
download_file("https://static.nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-modified.json.zip", TMP . "/nvd/nvdcve-modified.json.zip", $db->help, 'nvd-cve-dl-progress');
$zip = new ZipArchive();
$zip->open(TMP . "/nvd/nvdcve-{$yr}.json.zip");
$zip->open(TMP . "/nvd/nvdcve-modified.json.zip");
$zip->extractTo(TMP . "/nvd");
$zip->close();
unlink(TMP . "/nvd/nvdcve-{$yr}.json.zip");
}
}
else {
download_file("https://static.nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-modified.json.zip", TMP . "/nvd/nvdcve-modified.json.zip");
$zip = new ZipArchive();
$zip->open(TMP . "/nvd/nvdcve-modified.json.zip");
$zip->extractTo(TMP . "/nvd");
$zip->close();
unlink(TMP . "/nvd/nvdcve-modified.json.zip");
unlink(TMP . "/nvd/nvdcve-modified.json.zip");
download_file("https://static.nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-recent.json.zip", TMP . "/nvd/nvdcve-recent.json.zip");
$zip->open(TMP . "/nvd/nvdcve-recent.json.zip");
$zip->extractTo(TMP . "/nvd");
$zip->close();
unlink(TMP . "/nvd/nvdcve-recent.json.zip");
$db->set_Setting('nvd-year', 'recent');
download_file("https://static.nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-recent.json.zip", TMP . "/nvd/nvdcve-recent.json.zip", $db->help, 'nvd-cve-dl-progress');
$zip->open(TMP . "/nvd/nvdcve-recent.json.zip");
$zip->extractTo(TMP . "/nvd");
$zip->close();
unlink(TMP . "/nvd/nvdcve-recent.json.zip");
}
}
chdir(DOC_ROOT . "/exec");
$json_files = glob(TMP . "/nvd/*.json");
foreach ($json_files as $j) {
$script = realpath(defined('PHP_BIN') ? PHP_BIN : PHP) .
" -c " . realpath(PHP_CONF) .
" -f " . realpath(DOC_ROOT . "/exec/parse_nvd_json_cve.php") . " --" .
" -f=\"" . realpath($j) . "\"";
if (isset($cmd['po']) || !isset($cmd['do'])) {
$json_files = glob(TMP . "/nvd/*.json");
foreach ($json_files as $j) {
$match = [];
if (preg_match("/(\d{4}|recent|modified)/", basename($j), $match)) {
$db->set_Setting('nvd-year', $match[1]);
}
else {
$db->set_Setting('nvd-year', null);
}
$script = realpath(defined('PHP_BIN') ? PHP_BIN : PHP) .
" -c " . realpath(PHP_CONF) .
" -f " . realpath(DOC_ROOT . "/exec/parse_nvd_json_cve.php") . " --" .
" -f=\"" . realpath($j) . "\"";
$log->info("Running NVD CVE parsing script");
passthru($script);
$log->debug("Running NVD CVE parsing script on file: $j");
passthru($script);
}
}
$diff->stopClock();
$log->info("Finished at {$diff->getEndClockTime()}" . PHP_EOL .
"Total time {$diff->getTotalDiffString()}");
$db->help->select_count("sagacity.cve_db");
$nvd_count = $db->help->execute();
$db->set_Setting("nvd-cve-load-date", $diff->getEndClock()->format(MYSQL_DT_FORMAT));
$diff->stopClock();
print "Finished at {$diff->getEndClockTime()}" . PHP_EOL .
"Total time {$diff->getTotalDiffString()}" . PHP_EOL;
$db->set_Setting_Array([
'nvd-cve-load-date' => $diff->getEndClock()->format(MYSQL_DT_FORMAT),
'nvd-cve-count' => $nvd_count,
'nvd-cve-progress' => 100,
'nvd-cve-dl-progress' => 100,
'nvd-year' => null,
'cve-load-date' => $diff->getEndClock()->format(MYSQL_DT_FORMAT),
'cve-count' => $nvd_count,
'cve-progress' => 100,
'cve-dl-progress' => 100
]);
}
/**
@ -382,44 +453,73 @@ if (isset($cmd['cce'])) {
* Parse NASL content from NVT and/or Nessus
*/
if (isset($cmd['nasl'])) {
$db->set_Setting('nasl-dl-progress', 0);
$db->set_Setting('nasl-progress', 0);
check_path(TMP . "/nessus_plugins", true);
$db->set_Setting_Array([
'nasl-dl-progress' => 0,
'nasl-progress' => 0,
'nasl-count' => 0
]);
// Capture start time for performance monitoring
$diff->resetClock();
$log->info("Started NASL ingestion ({$diff->getStartClockTime()})");
print "Started NASL ingestion ({$diff->getStartClockTime()})" . PHP_EOL;
// Generate a unique filename for the OpenVAS feed archive using the current date
$nasl_fname = TMP . "/nessus_plugins/nasl_plugins-{$current_date->format("Ymd")}.tar.bz2";
// Download OpenVAS feed if a) it doesn't exist, b) can reach openvas.org, and c) parse only flag not set
if (!file_exists($nasl_fname) && ping("openvas.org") && !isset($cmd['po'])) {
download_file("http://www.openvas.org/openvas-nvt-feed-current.tar.bz2", $nasl_fname, $db, 'nasl-dl-progress');
$log->debug("Downloading new NASL library");
download_file("http://www.openvas.org/openvas-nvt-feed-current.tar.bz2", $nasl_fname, $db->help, 'nasl-dl-progress');
}
// Can only extract .tar.bz2 files on Linux so...
if (!isset($cmd['do']) || isset($cmd['po'])) {
if (file_exists($nasl_fname)) {
if (substr(strtolower(PHP_OS), 0, 3) == 'lin') {
$log->debug("Extracting NASL files from archive");
passthru("tar xvf $nasl_fname -C " . realpath(TMP . "/nessus_plugins") .
" --wildcards --transform='s/.*\///' '*.nasl'");
if (isset($cmd['extract'])) {
print "Completed extracting files from archive" . PHP_EOL;
}
}
}
if (isset($cmd['extract'])) {
die;
}
// ...if there are no .nasl files in the directory, die and give instructions for unzipping in Windows
$files = glob("*.nasl");
if (!count($files)) {
die("Downloaded the OpenVAS NVT plugin repository, please extract *.nasl files to " . realpath(TMP . "/nessus_plugins") . PHP_EOL .
$files = glob(TMP . "/nessus_plugins/*.nasl");
if (strtolower(substr(PHP_OS, 0, 3)) == 'win') {
if (file_exists(getenv("%ProgramData%") . "/Tenable/Nessus/nessus/plugins")) {
$log->debug("Importing local Nessus plugin files");
$files = array_merge($files, glob(getenv("%ProgramData%") . "/Tenable/Nessus/nessus/plugins/*.nasl"));
}
}
elseif (strtolower(substr(PHP_OS, 0, 3)) == 'lin') {
if (file_exists("/opt/nessus/lib/nessus/plugins") && is_readable("/opt/nessus/lib/nessus/plugins")) {
$log->debug("Importing local Nessus plugin files");
$files = array_merge($files, glob("/opt/nessus/lib/nessus/plugins/*.nasl"));
}
if (file_exists("/opt/sc/data/nasl") && is_readable("/opt/sc/data/nasl")) {
$log->debug("Importing local Nessus plugin files");
$files = array_merge($files, glob("/opt/sc/data/nasl/*.nasl"));
}
}
$files = array_unique($files);
if (!($file_count = count($files))) {
$log->emergency("Downloaded the OpenVAS NVT plugin repository, please extract *.nasl files to " . realpath(TMP . "/nessus_plugins") . PHP_EOL .
"If you have Bash on Windows ({path} = /mnt/c/xampp/www) or Cygwin ({path} = /cygdrive/c/xampp/www) installed you can run the following command on the downloaded file tweaking the paths" . PHP_EOL .
"tar xvf {path}/tmp/nessus_plugins/" . basename($nasl_fname) . " -C {path}/tmp/nessus_plugins --wildcards --transform='s/.*\///' '*.nasl'" . PHP_EOL);
die;
}
// Report how many NASL files were found in the directory
$log->info("Found " . count($files) . " NASL files" . PHP_EOL .
"Started at {$start_time->format("Y-m-d H:i:s")}");
chdir(DOC_ROOT);
print "Found {$file_count} NASL files" . PHP_EOL . "Started at {$diff->getStartClockTime()}" . PHP_EOL;
// Query database to build an array of existing plugins to compare against on import
$existing_plugins = [];
@ -430,65 +530,61 @@ if (isset($cmd['nasl'])) {
$existing_plugins[$row['plugin_id']] = DateTime::createFromFormat("U", $row['file_date']);
}
}
$log->debug("Count of existing plugins " . count($existing_plugins));
// Sort the files and loop over them
$x = 0;
natsort($files);
foreach ($files as $file) {
$abs_file_path = realpath(TMP . "/nessus_plugins/$file");
// Read the current NASL file into a nasl object
$nasl = new nasl($abs_file_path);
$nasl = new nasl($file);
// Report progress
$comp = number_format(($x / count($files)) * 100, 2) . "%";
print "\r$comp";
// calculate percent complete
$comp = number_format(($total_complete / $file_count) * 100, 2);
print "\r{$comp}%";
$log->debug("Parsing {$file} ({$comp}%)");
// If no plugin ID, delete file and continue to the next plugin
if (!isset($nasl->id)) {
unlink($abs_file_path);
$log->warning("Could not locate an ID in the plugin, skipping");
unlink($file);
continue;
}
// Only process if plugin doesn't already exist or has an older last_modificaiton date
if (!isset($existing_plugins[$nasl->id]) ||
(isset($nasl->last_modification) && $existing_plugins[$nasl->id] > $nasl->last_modification)) {
$log->info("Updating plugin {$nasl->id}");
// define command line to call script to parse the file
$script = realpath(defined('PHP_BIN') ? PHP_BIN : PHP) .
" -c " . realpath(PHP_CONF) .
" -f " . realpath(DOC_ROOT . "/exec/nessus-plugin-to-database.php") . " --" .
" -f=\"" . $abs_file_path . "\"";
" -f=\"{$file}\"";
$process = new \Cocur\BackgroundProcess\BackgroundProcess($script);
$process->run();
$threads[] = new \Cocur\BackgroundProcess\BackgroundProcess($script);
end($threads)->run();
// Call the script w/ shell or exec depending on platform
if (substr(strtolower(PHP_OS), 0, 3) == 'lin') {
$output = [];
exec("netstat -an | grep TIME_WAIT | wc -l", $output);
if ($output[0] > 2000) {
do {
$log->notice("\r$comp Sleeping till connections get below 100 {$output[0]}");
sleep(1);
$output = [];
exec("netstat -an | grep TIME_WAIT | wc -l", $output);
}
while ($output[0] > 100);
}
$count++;
$total_complete++;
if($total_complete % 100 == 0) {
$db->set_Setting('nasl-progress', $comp);
}
}
else {
unlink($abs_file_path);
}
$x++;
}
}
$db->set_Setting_Array([
'nasl-dl-progress' => 100,
'nasl-progress' => 100,
'nasl-count' => $total_complete,
'nasl-load-date' => new DateTime()
]);
$diff->stopClock();
$log->info(PHP_EOL . "Finished at {$diff->getEndClockTime()}" . PHP_EOL .
"Total Time: {$diff->getDiffString()}");
print PHP_EOL . "Finished at {$diff->getEndClockTime()}" . PHP_EOL .
"Total Time: {$diff->getDiffString()}" . PHP_EOL;
sleep(3);
}
@ -497,13 +593,16 @@ if (isset($cmd['nasl'])) {
* Update STIG library from DISA content
*/
if (isset($cmd['stig'])) {
$db->set_Setting('stig-dl-progress', 0);
$db->set_Setting('stig-progress', 0);
$db->set_Setting_Array([
'stig-dl-progress' => 0,
'stig-progress' => 0,
'stig-count' => 0
]);
$path = TMP . "/stigs";
check_path($path);
$diff->resetClock();
$log->info("Started STIG ingestion ({$diff->getStartClockTime()})");
print "Started STIG ingestion ({$diff->getStartClockTime()})" . PHP_EOL;
$mon = '01';
$prev_mon = '10';
@ -530,21 +629,21 @@ if (isset($cmd['stig'])) {
if (!file_exists($stig_fname) && ping("disa.mil") && !isset($cmd['po'])) {
if (isset($cmd['u'])) {
$url = $cmd['u'];
$log->info("Checking for $url");
$log->debug("Checking for $url");
if (url_exists($url)) {
download_file($url, $stig_fname, $db, 'stig-dl-progress');
download_file($url, $stig_fname, $db->help, 'stig-dl-progress');
}
}
else {
$log->info("Checking for $current_url");
$log->debug("Checking for $current_url");
if ($found = url_exists($current_url)) {
download_file($current_url, $stig_fname, $db, 'stig-dl-progress');
download_file($current_url, $stig_fname, $db->help, 'stig-dl-progress');
}
if (!$found) {
$log->info("Checking for $current_v2_url");
$log->debug("Checking for $current_v2_url");
if ($found = url_exists($current_v2_url)) {
download_file($current_v2_url, $stig_fname, $db, 'stig-dl-progress');
download_file($current_v2_url, $stig_fname, $db->help, 'stig-dl-progress');
}
}
@ -555,16 +654,16 @@ if (isset($cmd['stig'])) {
$prev_v2_url = "http://iasecontent.disa.mil/stigs/zip/Compilations/U_SRG-STIG_Library_{$year}_{$prev_mon}_v2.zip";
if (!$found) {
$log->info("Checking for $prev_url");
$log->debug("Checking for $prev_url");
if ($found = url_exists($prev_url)) {
download_file($prev_url, $stig_fname, $db, 'stig-dl-progress');
download_file($prev_url, $stig_fname, $db->help, 'stig-dl-progress');
}
}
if (!$found) {
$log->info("Checking for $prev_v2_url");
$log->debug("Checking for $prev_v2_url");
if (url_exists($prev_v2_url)) {
download_file($prev_v2_url, $stig_fname, $db, 'stig-dl-progress');
download_file($prev_v2_url, $stig_fname, $db->help, 'stig-dl-progress');
}
}
}
@ -581,46 +680,58 @@ if (isset($cmd['stig'])) {
$script = realpath(defined('PHP_BIN') ? PHP_BIN : PHP) .
" -c " . realpath(PHP_CONF) .
" -f " . realpath(DOC_ROOT . "/exec/background_stigs.php") . " --" .
(isset($cmd['exclude']) && $cmd['exclude'] ? " --exclude=\"{$cmd['exclude']}\"" : "") .
" --delete";
$log->info("Script to run $script");
$log->debug("Script to run $script");
passthru($script);
}
$db->help->select_count("sagacity.stigs");
$stig_count = $db->help->execute();
$db->set_Setting("stig-count", $stig_count);
$diff->stopClock();
$log->info(PHP_EOL . "Finished at {$diff->getEndClockTime()}" . PHP_EOL .
"Total Time: {$diff->getDiffString()}");
print PHP_EOL . "Finished at {$diff->getEndClockTime()}" . PHP_EOL .
"Total Time: {$diff->getDiffString()}" . PHP_EOL;
sleep(3);
}
if (is_a($diff->getTotalDiff(), 'DateInterval')) {
$log->info("Total Script Time: {$diff->getTotalDiffString()}");
print "Total Script Time: {$diff->getTotalDiffString()}" . PHP_EOL;
}
/**
*
* Usage information about the script
*/
function usage()
{
$tmp = TMP;
$tmp = realpath(TMP);
print <<<EOO
Purpose: The purpose of this script is to update the CVE, CPE, and CCE databases. Script will sleep for 3 seconds between actions to allow you review the results.
Usage: php update_db.php [--cpe] [--cve] [--nasl] [--stig] [-u={URL}] [--do] [--po] [-h|--help]
Usage: php update_db.php [--cpe] [--cve] [--nvd] [--nasl] [--stig] [-u={URL}] [--do] [--po] [-h|--help] [--debug] [--exclude="ex1"]
--cpe To download and update the CPE catalog
--cve To download and update the CVE catalog
--cve To download and update the CVE catalog using Mitre's database
--nvd To download and update the CVE catalog using the National Vulnerability Database (NVD) JSON library
--nasl To download OpenVAS NVT library and update NASL files
You can also extract *.nasl files from the Nessus library to $tmp/nessus_plugins and it will include these in the update
--stig To download and update the STIG library
--do To download the files only...do not call the parsers will overwrite any existing files
--po To parse the downloaded files only, do not download
-u={url} [optional] Used only for STIGs because sometimes DISA will use a non-standard link which makes it difficult to download the file.
--exclude="ex1" Insert a valid regex expression (properly escaped) to exclude specific STIGs from parsing (no '/' necessary)
--extract Used so script will download and extract files from archive and stop processing
-u={url} Used only for STIGs because sometimes DISA will use a non-standard link which makes it difficult to download the file.
-h|--help This screen
--debug To print verbose debugging messages to the console
EOO;
}