ryan/sagacity
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Revision of release v1.3.2

Browse Source
This commit is contained in:
CyberPerspectives
2018-07-26 08:33:50 -04:00
committed by Ryan Prather
parent 8c38a6cdb9
commit 750094e3b5
3231 changed files with 133590 additions and 135073 deletions
Download Patch File Download Diff File Expand all files Collapse all files

377
exec/update_db.php
View File

@ -5,7 +5,7 @@
* Author: Ryan Prather
* Created: Jan 5, 2015
*
* Portions Copyright 2016-2017: Cyber Perspectives, All rights reserved
* Portions Copyright 2016-2018: Cyber Perspectives, LLC, All rights reserved
* Released under the Apache v2.0 License
*
* Portions Copyright (c) 2012-2015, Salient Federal Solutions
@ -45,6 +45,9 @@
* Fixed confusion with Cygwin and Bash on Windows paths
* - Jun 27, 2017 - Matt Shuter: Fixed bug #262 & #270
* - Dec 27, 2017 - Added database field and download progress flag
* - Apr 29, 2018 - Added extract parameter to only extract nasl archive file, fixed a couple bugs
* - May 10, 2018 - Removed ping of cve.mitre.org, and added 'po' and 'do' parameters for NVD CVE
* - Jun 5, 2018 - Fixed a couple setting updates
*/
include_once 'config.inc';
include_once 'helper.inc';
@ -62,7 +65,7 @@ $total_time = null;
$total_diff = 0;
$summary_stats = [];
$cmd = getopt("h::u::p::", ['cpe::', 'cce::', 'cve::', 'nvd::', 'nasl::', 'stig::', 'do::', 'po::', 'help::']);
$cmd = getopt("h::u::p::", ['cpe::', 'cce::', 'cve::', 'nvd::', 'nasl::', 'stig::', 'do::', 'po::', 'help::', 'debug::', 'extract::', 'exclude::']);
$db = new db();
$diff = new DateTimeDiff();
@ -79,7 +82,11 @@ switch (LOG_LEVEL) {
$log_level = Logger::DEBUG;
}
$stream = new StreamHandler("php://output", Logger::INFO);
if (isset($cmd['debug']) && $cmd['debug']) {
$log_level = Logger::DEBUG;
}
$stream = new StreamHandler("php://output", $log_level);
$stream->setFormatter(new LineFormatter("%datetime% %level_name% %message%" . PHP_EOL, "H:i:s.u"));
$log = new Logger("update_db");
@ -93,7 +100,7 @@ if (isset($cmd['h'], $cmd['help']) ||
if (isset($cmd['do']) || !isset($cmd['po'])) {
if (!ping("cyberperspectives.com")) {
die("Cannot connect to internet" . PHP_EOL);
$log->emergency("Cannot connect to internet");
}
}
@ -101,8 +108,11 @@ if (isset($cmd['do']) || !isset($cmd['po'])) {
* Update CPE content downloaded from NIST
*/
if (isset($cmd['cpe'])) {
$db->set_Setting('cpe-dl-progress', 0);
$db->set_Setting('cpe-progress', 0);
$db->set_Setting_Array([
'cpe-dl-progress' => 0,
'cpe-progress' => 0,
'cpe-count' => 0
]);
$path = TMP . "/cpe";
if (isset($cmd['p']) && $cmd['p']) {
@ -112,7 +122,7 @@ if (isset($cmd['cpe'])) {
check_path($path);
$diff->resetClock();
$log->info("Started CPE ingestion ({$diff->getStartClockTime()})");
print "Started CPE ingestion ({$diff->getStartClockTime()})" . PHP_EOL;
// search for an unzip any zip files in the tmp directory
$zip_files = glob("{$path}/*cpe-dictionary*.zip");
@ -120,7 +130,7 @@ if (isset($cmd['cpe'])) {
$log->debug("Found a existing cpe-dictionary.zip file, unzipping then parsing");
$zip = new ZipArchive();
foreach ($zip_files as $file) {
$log->info("Unzipping {$file}");
$log->debug("Unzipping {$file}");
$zip->open($file);
$zip->extractTo($path);
$zip->close();
@ -135,7 +145,7 @@ if (isset($cmd['cpe'])) {
foreach ($tmp_files as $fname) {
$name = basename($fname);
if ($name == 'official-cpe-dictionary_v2.3.xml') {
$name = "cpe-dictionary-{$start_time->format("Ymd")}.xml";
$name = "cpe-dictionary-{$diff->getStartClock()->format("Ymd")}.xml";
}
rename($fname, "{$path}/{$name}");
}
@ -146,14 +156,14 @@ if (isset($cmd['cpe'])) {
$cpe_parse_fname = null;
// download the file if the do flag is used even if it already exists
if (isset($cmd['do']) && ping("nist.gov") && !isset($cmd['po'])) {
download_file($cpe_url, $cpe_fname, $db, 'cpe-dl-progress');
if (isset($cmd['do']) && !isset($cmd['po'])) {
download_file($cpe_url, $cpe_fname, $db->help, 'cpe-dl-progress');
}
// download the file only if it doesn't exist
elseif (!file_exists($cpe_fname) && ping("nist.gov") && !isset($cmd['po'])) {
download_file($cpe_url, $cpe_fname, $db, 'cpe-dl-progress');
elseif (!file_exists($cpe_fname) && !isset($cmd['po'])) {
download_file($cpe_url, $cpe_fname, $db->help, 'cpe-dl-progress');
}
elseif (!isset($cmd['po']) && !ping("nist.gov")) {
elseif (!isset($cmd['po'])) {
$log->error("Could not connect to nist.gov to download the CPE library");
die();
}
@ -185,7 +195,7 @@ if (isset($cmd['cpe'])) {
}
if (is_null($cpe_parse_fname)) {
$log->warning("Coult not find a CPE file to parse");
$log->warning("Could not find a CPE file to parse");
}
$script = realpath(defined('PHP_BIN') ? PHP_BIN : PHP) .
@ -194,14 +204,19 @@ if (isset($cmd['cpe'])) {
" -f=\"" . realpath($cpe_parse_fname) . "\"" .
" -d=\"{$dt->format("Y-m-d")}\"";
$log->info("Running parsing script");
$log->debug("Running CPE parsing script on file: $cpe_parse_fname");
passthru($script);
}
$db->help->select_count("software");
$cpe_count = $db->help->execute();
$db->set_Setting("cpe-count", $cpe_count);
$diff->stopClock();
$log->info(PHP_EOL . "Finished at {$diff->getEndClockTime()}" . PHP_EOL .
"Total time: {$diff->getDiffString()}");
print PHP_EOL . "Finished at {$diff->getEndClockTime()}" . PHP_EOL .
"Total time: {$diff->getDiffString()}" . PHP_EOL;
sleep(3);
}
@ -210,8 +225,14 @@ if (isset($cmd['cpe'])) {
* Update CVE content
*/
if (isset($cmd['cve'])) {
$db->set_Setting('cve-dl-progress', 0);
$db->set_Setting('cve-progress', 0);
$db->set_Setting_Array([
'nvd-cve-dl-progress' => 0,
'nvd-cve-progress' => 0,
'nvd-cve-count' => 0,
'cve-dl-progress' => 0,
'cve-progress' => 0,
'cve-count' => 0
]);
$path = TMP . "/cve";
if (isset($cmd['p']) && $cmd['p']) {
$path = $cmd['p'];
@ -220,12 +241,12 @@ if (isset($cmd['cve'])) {
check_path($path);
$diff->resetClock();
$log->info("Started CVE ingestion {$diff->getStartClockTime()}");
print "Started CVE ingestion {$diff->getStartClockTime()}" . PHP_EOL;
$cve_files = glob(TMP . "/allitems.xml");
if (count($cve_files)) {
foreach ($cve_files as $file) {
rename($file, "{$path}/cve-all-{$start_time->format("Ymd")}.xml");
rename($file, "{$path}/cve-all-{$diff->getStartClock()->format("Ymd")}.xml");
}
}
@ -240,13 +261,13 @@ if (isset($cmd['cve'])) {
$cve_url = "http://cve.mitre.org/data/downloads/allitems.xml";
$cve_parse_fname = null;
if (isset($cmd['do']) && ping("cve.mitre.org") && !isset($cmd['po'])) {
download_file($cve_url, $cve_fname, $db, 'cve-dl-progress');
if (isset($cmd['do']) && !isset($cmd['po'])) {
download_file($cve_url, $cve_fname, $db->help, 'cve-dl-progress');
}
elseif (!file_exists($cve_fname) && ping("cve.mitre.org") && !isset($cmd['po'])) {
download_file($cve_url, $cve_fname, $db, 'cve-dl-progress');
elseif (!file_exists($cve_fname) && !isset($cmd['po'])) {
download_file($cve_url, $cve_fname, $db->help, 'cve-dl-progress');
}
elseif (!isset($cmd['po']) && !ping("cve.mitre.org")) {
elseif (!isset($cmd['po'])) {
Sagacity_Error::err_handler("Could not connect to cve.mitre.org to download the CVE library", E_ERROR);
}
@ -271,7 +292,7 @@ if (isset($cmd['cve'])) {
}
if (is_null($cve_parse_fname)) {
$log->error("Coult not find a CVE file to parse");
$log->error("Could not find a CVE file to parse");
die;
}
@ -281,21 +302,44 @@ if (isset($cmd['cve'])) {
" -f=\"" . realpath($cve_parse_fname) . "\"" .
" -d=\"{$dt->format("Y-m-d")}\"";
$log->info("Script to run $script");
$log->debug("Script to run $script");
passthru($script);
}
$db->help->select_count("sagacity.cve_db");
$cve_count = $db->help->execute();
$db->set_Setting_Array([
'cve-dl-progress' => 100,
'cve-progress' => 100,
'cve-count' => $cve_count,
'cve-load-date' => new DateTime(),
'nvd-cve-dl-progress' => 100,
'nvd-cve-progress' => 100,
'nvd-cve-count' => $cve_count,
'nvd-cve-load-date' => new DateTime()
]);
$diff->stopClock();
$log->info("Finished at {$diff->getEndClockTime()}" . PHP_EOL .
"Total Time: {$diff->getDiffString()}");
print "Finished at {$diff->getEndClockTime()}" . PHP_EOL .
"Total Time: {$diff->getDiffString()}" . PHP_EOL;
sleep(3);
}
/**
* Update to NVD CVE content
*/
if (isset($cmd['nvd'])) {
$db->set_Setting('nvd-cve-dl-progress', 0);
$db->set_Setting('nvd-cve-progress', 0);
$db->set_Setting_Array([
'nvd-cve-dl-progress' => 0,
'nvd-cve-progress' => 0,
'nvd-cve-count' => 0,
'cve-dl-progress' => 0,
'cve-progress' => 0,
'cve-count' => 0
]);
$path = TMP . "/nvd";
if (isset($cmd['p']) && $cmd['p']) {
$path = $cmd['p'];
@ -303,60 +347,87 @@ if (isset($cmd['nvd'])) {
check_path($path);
$diff->resetClock();
$log->info("Started NVD CVE ingestion ({$diff->getStartClockTime()})");
print "Started NVD CVE ingestion ({$diff->getStartClockTime()})" . PHP_EOL;
$nvd_years = [];
for ($x = 2002; $x <= $diff->getStartClock()->format("Y"); $x++) {
$nvd_years[] = $x;
}
$too_old = new DateTime();
$too_old->sub(DateInterval::createFromDateString("7 days"));
if (isset($cmd['do']) || !isset($cmd['po'])) {
$too_old = new DateTime();
$too_old->sub(DateInterval::createFromDateString("7 days"));
$load_date = new DateTime($db->get_Settings("nvd-cve-load-date"));
if ($load_date < $too_old) {
// More than 7 days old so have to do a full load
foreach ($nvd_years as $yr) {
download_file("https://static.nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-{$yr}.json.zip", TMP . "/nvd/nvdcve-{$yr}.json.zip");
$load_date = new DateTime($db->get_Settings("nvd-cve-load-date"));
if ($load_date < $too_old) {
// More than 7 days old so have to do a full load
foreach ($nvd_years as $yr) {
$db->set_Setting('nvd-year', $yr);
download_file("https://static.nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-{$yr}.json.zip", TMP . "/nvd/nvdcve-{$yr}.json.zip", $db->help, 'nvd-cve-dl-progress');
$zip = new ZipArchive();
$zip->open(TMP . "/nvd/nvdcve-{$yr}.json.zip");
$zip->extractTo(TMP . "/nvd");
$zip->close();
unlink(TMP . "/nvd/nvdcve-{$yr}.json.zip");
}
}
else {
$db->set_Setting('nvd-year', 'modified');
download_file("https://static.nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-modified.json.zip", TMP . "/nvd/nvdcve-modified.json.zip", $db->help, 'nvd-cve-dl-progress');
$zip = new ZipArchive();
$zip->open(TMP . "/nvd/nvdcve-{$yr}.json.zip");
$zip->open(TMP . "/nvd/nvdcve-modified.json.zip");
$zip->extractTo(TMP . "/nvd");
$zip->close();
unlink(TMP . "/nvd/nvdcve-{$yr}.json.zip");
}
}
else {
download_file("https://static.nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-modified.json.zip", TMP . "/nvd/nvdcve-modified.json.zip");
$zip = new ZipArchive();
$zip->open(TMP . "/nvd/nvdcve-modified.json.zip");
$zip->extractTo(TMP . "/nvd");
$zip->close();
unlink(TMP . "/nvd/nvdcve-modified.json.zip");
unlink(TMP . "/nvd/nvdcve-modified.json.zip");
download_file("https://static.nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-recent.json.zip", TMP . "/nvd/nvdcve-recent.json.zip");
$zip->open(TMP . "/nvd/nvdcve-recent.json.zip");
$zip->extractTo(TMP . "/nvd");
$zip->close();
unlink(TMP . "/nvd/nvdcve-recent.json.zip");
$db->set_Setting('nvd-year', 'recent');
download_file("https://static.nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-recent.json.zip", TMP . "/nvd/nvdcve-recent.json.zip", $db->help, 'nvd-cve-dl-progress');
$zip->open(TMP . "/nvd/nvdcve-recent.json.zip");
$zip->extractTo(TMP . "/nvd");
$zip->close();
unlink(TMP . "/nvd/nvdcve-recent.json.zip");
}
}
chdir(DOC_ROOT . "/exec");
$json_files = glob(TMP . "/nvd/*.json");
foreach ($json_files as $j) {
$script = realpath(defined('PHP_BIN') ? PHP_BIN : PHP) .
" -c " . realpath(PHP_CONF) .
" -f " . realpath(DOC_ROOT . "/exec/parse_nvd_json_cve.php") . " --" .
" -f=\"" . realpath($j) . "\"";
if (isset($cmd['po']) || !isset($cmd['do'])) {
$json_files = glob(TMP . "/nvd/*.json");
foreach ($json_files as $j) {
$match = [];
if (preg_match("/(\d{4}|recent|modified)/", basename($j), $match)) {
$db->set_Setting('nvd-year', $match[1]);
}
else {
$db->set_Setting('nvd-year', null);
}
$script = realpath(defined('PHP_BIN') ? PHP_BIN : PHP) .
" -c " . realpath(PHP_CONF) .
" -f " . realpath(DOC_ROOT . "/exec/parse_nvd_json_cve.php") . " --" .
" -f=\"" . realpath($j) . "\"";
$log->info("Running NVD CVE parsing script");
passthru($script);
$log->debug("Running NVD CVE parsing script on file: $j");
passthru($script);
}
}
$diff->stopClock();
$log->info("Finished at {$diff->getEndClockTime()}" . PHP_EOL .
"Total time {$diff->getTotalDiffString()}");
$db->help->select_count("sagacity.cve_db");
$nvd_count = $db->help->execute();
$db->set_Setting("nvd-cve-load-date", $diff->getEndClock()->format(MYSQL_DT_FORMAT));
$diff->stopClock();
print "Finished at {$diff->getEndClockTime()}" . PHP_EOL .
"Total time {$diff->getTotalDiffString()}" . PHP_EOL;
$db->set_Setting_Array([
'nvd-cve-load-date' => $diff->getEndClock()->format(MYSQL_DT_FORMAT),
'nvd-cve-count' => $nvd_count,
'nvd-cve-progress' => 100,
'nvd-cve-dl-progress' => 100,
'nvd-year' => null,
'cve-load-date' => $diff->getEndClock()->format(MYSQL_DT_FORMAT),
'cve-count' => $nvd_count,
'cve-progress' => 100,
'cve-dl-progress' => 100
]);
}
/**
@ -382,44 +453,73 @@ if (isset($cmd['cce'])) {
* Parse NASL content from NVT and/or Nessus
*/
if (isset($cmd['nasl'])) {
$db->set_Setting('nasl-dl-progress', 0);
$db->set_Setting('nasl-progress', 0);
check_path(TMP . "/nessus_plugins", true);
$db->set_Setting_Array([
'nasl-dl-progress' => 0,
'nasl-progress' => 0,
'nasl-count' => 0
]);
// Capture start time for performance monitoring
$diff->resetClock();
$log->info("Started NASL ingestion ({$diff->getStartClockTime()})");
print "Started NASL ingestion ({$diff->getStartClockTime()})" . PHP_EOL;
// Generate a unique filename for the OpenVAS feed archive using the current date
$nasl_fname = TMP . "/nessus_plugins/nasl_plugins-{$current_date->format("Ymd")}.tar.bz2";
// Download OpenVAS feed if a) it doesn't exist, b) can reach openvas.org, and c) parse only flag not set
if (!file_exists($nasl_fname) && ping("openvas.org") && !isset($cmd['po'])) {
download_file("http://www.openvas.org/openvas-nvt-feed-current.tar.bz2", $nasl_fname, $db, 'nasl-dl-progress');
$log->debug("Downloading new NASL library");
download_file("http://www.openvas.org/openvas-nvt-feed-current.tar.bz2", $nasl_fname, $db->help, 'nasl-dl-progress');
}
// Can only extract .tar.bz2 files on Linux so...
if (!isset($cmd['do']) || isset($cmd['po'])) {
if (file_exists($nasl_fname)) {
if (substr(strtolower(PHP_OS), 0, 3) == 'lin') {
$log->debug("Extracting NASL files from archive");
passthru("tar xvf $nasl_fname -C " . realpath(TMP . "/nessus_plugins") .
" --wildcards --transform='s/.*\///' '*.nasl'");
if (isset($cmd['extract'])) {
print "Completed extracting files from archive" . PHP_EOL;
}
}
}
if (isset($cmd['extract'])) {
die;
}
// ...if there are no .nasl files in the directory, die and give instructions for unzipping in Windows
$files = glob("*.nasl");
if (!count($files)) {
die("Downloaded the OpenVAS NVT plugin repository, please extract *.nasl files to " . realpath(TMP . "/nessus_plugins") . PHP_EOL .
$files = glob(TMP . "/nessus_plugins/*.nasl");
if (strtolower(substr(PHP_OS, 0, 3)) == 'win') {
if (file_exists(getenv("%ProgramData%") . "/Tenable/Nessus/nessus/plugins")) {
$log->debug("Importing local Nessus plugin files");
$files = array_merge($files, glob(getenv("%ProgramData%") . "/Tenable/Nessus/nessus/plugins/*.nasl"));
}
}
elseif (strtolower(substr(PHP_OS, 0, 3)) == 'lin') {
if (file_exists("/opt/nessus/lib/nessus/plugins") && is_readable("/opt/nessus/lib/nessus/plugins")) {
$log->debug("Importing local Nessus plugin files");
$files = array_merge($files, glob("/opt/nessus/lib/nessus/plugins/*.nasl"));
}
if (file_exists("/opt/sc/data/nasl") && is_readable("/opt/sc/data/nasl")) {
$log->debug("Importing local Nessus plugin files");
$files = array_merge($files, glob("/opt/sc/data/nasl/*.nasl"));
}
}
$files = array_unique($files);
if (!($file_count = count($files))) {
$log->emergency("Downloaded the OpenVAS NVT plugin repository, please extract *.nasl files to " . realpath(TMP . "/nessus_plugins") . PHP_EOL .
"If you have Bash on Windows ({path} = /mnt/c/xampp/www) or Cygwin ({path} = /cygdrive/c/xampp/www) installed you can run the following command on the downloaded file tweaking the paths" . PHP_EOL .
"tar xvf {path}/tmp/nessus_plugins/" . basename($nasl_fname) . " -C {path}/tmp/nessus_plugins --wildcards --transform='s/.*\///' '*.nasl'" . PHP_EOL);
die;
}
// Report how many NASL files were found in the directory
$log->info("Found " . count($files) . " NASL files" . PHP_EOL .
"Started at {$start_time->format("Y-m-d H:i:s")}");
chdir(DOC_ROOT);
print "Found {$file_count} NASL files" . PHP_EOL . "Started at {$diff->getStartClockTime()}" . PHP_EOL;
// Query database to build an array of existing plugins to compare against on import
$existing_plugins = [];
@ -430,65 +530,61 @@ if (isset($cmd['nasl'])) {
$existing_plugins[$row['plugin_id']] = DateTime::createFromFormat("U", $row['file_date']);
}
}
$log->debug("Count of existing plugins " . count($existing_plugins));
// Sort the files and loop over them
$x = 0;
natsort($files);
foreach ($files as $file) {
$abs_file_path = realpath(TMP . "/nessus_plugins/$file");
// Read the current NASL file into a nasl object
$nasl = new nasl($abs_file_path);
$nasl = new nasl($file);
// Report progress
$comp = number_format(($x / count($files)) * 100, 2) . "%";
print "\r$comp";
// calculate percent complete
$comp = number_format(($total_complete / $file_count) * 100, 2);
print "\r{$comp}%";
$log->debug("Parsing {$file} ({$comp}%)");
// If no plugin ID, delete file and continue to the next plugin
if (!isset($nasl->id)) {
unlink($abs_file_path);
$log->warning("Could not locate an ID in the plugin, skipping");
unlink($file);
continue;
}
// Only process if plugin doesn't already exist or has an older last_modificaiton date
if (!isset($existing_plugins[$nasl->id]) ||
(isset($nasl->last_modification) && $existing_plugins[$nasl->id] > $nasl->last_modification)) {
$log->info("Updating plugin {$nasl->id}");
// define command line to call script to parse the file
$script = realpath(defined('PHP_BIN') ? PHP_BIN : PHP) .
" -c " . realpath(PHP_CONF) .
" -f " . realpath(DOC_ROOT . "/exec/nessus-plugin-to-database.php") . " --" .
" -f=\"" . $abs_file_path . "\"";
" -f=\"{$file}\"";
$process = new \Cocur\BackgroundProcess\BackgroundProcess($script);
$process->run();
$threads[] = new \Cocur\BackgroundProcess\BackgroundProcess($script);
end($threads)->run();
// Call the script w/ shell or exec depending on platform
if (substr(strtolower(PHP_OS), 0, 3) == 'lin') {
$output = [];
exec("netstat -an | grep TIME_WAIT | wc -l", $output);
if ($output[0] > 2000) {
do {
$log->notice("\r$comp Sleeping till connections get below 100 {$output[0]}");
sleep(1);
$output = [];
exec("netstat -an | grep TIME_WAIT | wc -l", $output);
}
while ($output[0] > 100);
}
$count++;
$total_complete++;
if($total_complete % 100 == 0) {
$db->set_Setting('nasl-progress', $comp);
}
}
else {
unlink($abs_file_path);
}
$x++;
}
}
$db->set_Setting_Array([
'nasl-dl-progress' => 100,
'nasl-progress' => 100,
'nasl-count' => $total_complete,
'nasl-load-date' => new DateTime()
]);
$diff->stopClock();
$log->info(PHP_EOL . "Finished at {$diff->getEndClockTime()}" . PHP_EOL .
"Total Time: {$diff->getDiffString()}");
print PHP_EOL . "Finished at {$diff->getEndClockTime()}" . PHP_EOL .
"Total Time: {$diff->getDiffString()}" . PHP_EOL;
sleep(3);
}
@ -497,13 +593,16 @@ if (isset($cmd['nasl'])) {
* Update STIG library from DISA content
*/
if (isset($cmd['stig'])) {
$db->set_Setting('stig-dl-progress', 0);
$db->set_Setting('stig-progress', 0);
$db->set_Setting_Array([
'stig-dl-progress' => 0,
'stig-progress' => 0,
'stig-count' => 0
]);
$path = TMP . "/stigs";
check_path($path);
$diff->resetClock();
$log->info("Started STIG ingestion ({$diff->getStartClockTime()})");
print "Started STIG ingestion ({$diff->getStartClockTime()})" . PHP_EOL;
$mon = '01';
$prev_mon = '10';
@ -530,21 +629,21 @@ if (isset($cmd['stig'])) {
if (!file_exists($stig_fname) && ping("disa.mil") && !isset($cmd['po'])) {
if (isset($cmd['u'])) {
$url = $cmd['u'];
$log->info("Checking for $url");
$log->debug("Checking for $url");
if (url_exists($url)) {
download_file($url, $stig_fname, $db, 'stig-dl-progress');
download_file($url, $stig_fname, $db->help, 'stig-dl-progress');
}
}
else {
$log->info("Checking for $current_url");
$log->debug("Checking for $current_url");
if ($found = url_exists($current_url)) {
download_file($current_url, $stig_fname, $db, 'stig-dl-progress');
download_file($current_url, $stig_fname, $db->help, 'stig-dl-progress');
}
if (!$found) {
$log->info("Checking for $current_v2_url");
$log->debug("Checking for $current_v2_url");
if ($found = url_exists($current_v2_url)) {
download_file($current_v2_url, $stig_fname, $db, 'stig-dl-progress');
download_file($current_v2_url, $stig_fname, $db->help, 'stig-dl-progress');
}
}
@ -555,16 +654,16 @@ if (isset($cmd['stig'])) {
$prev_v2_url = "http://iasecontent.disa.mil/stigs/zip/Compilations/U_SRG-STIG_Library_{$year}_{$prev_mon}_v2.zip";
if (!$found) {
$log->info("Checking for $prev_url");
$log->debug("Checking for $prev_url");
if ($found = url_exists($prev_url)) {
download_file($prev_url, $stig_fname, $db, 'stig-dl-progress');
download_file($prev_url, $stig_fname, $db->help, 'stig-dl-progress');
}
}
if (!$found) {
$log->info("Checking for $prev_v2_url");
$log->debug("Checking for $prev_v2_url");
if (url_exists($prev_v2_url)) {
download_file($prev_v2_url, $stig_fname, $db, 'stig-dl-progress');
download_file($prev_v2_url, $stig_fname, $db->help, 'stig-dl-progress');
}
}
}
@ -581,46 +680,58 @@ if (isset($cmd['stig'])) {
$script = realpath(defined('PHP_BIN') ? PHP_BIN : PHP) .
" -c " . realpath(PHP_CONF) .
" -f " . realpath(DOC_ROOT . "/exec/background_stigs.php") . " --" .
(isset($cmd['exclude']) && $cmd['exclude'] ? " --exclude=\"{$cmd['exclude']}\"" : "") .
" --delete";
$log->info("Script to run $script");
$log->debug("Script to run $script");
passthru($script);
}
$db->help->select_count("sagacity.stigs");
$stig_count = $db->help->execute();
$db->set_Setting("stig-count", $stig_count);
$diff->stopClock();
$log->info(PHP_EOL . "Finished at {$diff->getEndClockTime()}" . PHP_EOL .
"Total Time: {$diff->getDiffString()}");
print PHP_EOL . "Finished at {$diff->getEndClockTime()}" . PHP_EOL .
"Total Time: {$diff->getDiffString()}" . PHP_EOL;
sleep(3);
}
if (is_a($diff->getTotalDiff(), 'DateInterval')) {
$log->info("Total Script Time: {$diff->getTotalDiffString()}");
print "Total Script Time: {$diff->getTotalDiffString()}" . PHP_EOL;
}
/**
*
* Usage information about the script
*/
function usage()
{
$tmp = TMP;
$tmp = realpath(TMP);
print <<<EOO
Purpose: The purpose of this script is to update the CVE, CPE, and CCE databases. Script will sleep for 3 seconds between actions to allow you review the results.
Usage: php update_db.php [--cpe] [--cve] [--nasl] [--stig] [-u={URL}] [--do] [--po] [-h|--help]
Usage: php update_db.php [--cpe] [--cve] [--nvd] [--nasl] [--stig] [-u={URL}] [--do] [--po] [-h|--help] [--debug] [--exclude="ex1"]
--cpe To download and update the CPE catalog
--cve To download and update the CVE catalog
--cve To download and update the CVE catalog using Mitre's database
--nvd To download and update the CVE catalog using the National Vulnerability Database (NVD) JSON library
--nasl To download OpenVAS NVT library and update NASL files
You can also extract *.nasl files from the Nessus library to $tmp/nessus_plugins and it will include these in the update
--stig To download and update the STIG library
--do To download the files only...do not call the parsers will overwrite any existing files
--po To parse the downloaded files only, do not download
-u={url} [optional] Used only for STIGs because sometimes DISA will use a non-standard link which makes it difficult to download the file.
--exclude="ex1" Insert a valid regex expression (properly escaped) to exclude specific STIGs from parsing (no '/' necessary)
--extract Used so script will download and extract files from archive and stop processing
-u={url} Used only for STIGs because sometimes DISA will use a non-standard link which makes it difficult to download the file.
-h|--help This screen
--debug To print verbose debugging messages to the console
EOO;
}