Ryan Prather
17dbe134cc
installer.php - Fix bug when update_db.php - Converted STIG download to identify the zip files from the a-z master list and download them individually instead of downloading the compilation zip file. Also integrated the sunset list into the same process so ALL STIGs are downloaded and imported at the same time
791 lines
30 KiB
PHP
791 lines
30 KiB
PHP
<?php
|
|
/**
|
|
* File: update_db.php
|
|
* Purpose: Script to download updated versions of the online files and update the database
|
|
* Author: Ryan Prather
|
|
* Created: Jan 5, 2015
|
|
*
|
|
* Portions Copyright 2016-2018: Cyber Perspectives, LLC, All rights reserved
|
|
* Released under the Apache v2.0 License
|
|
*
|
|
* Portions Copyright (c) 2012-2015, Salient Federal Solutions
|
|
* Portions Copyright (c) 2008-2011, Science Applications International Corporation (SAIC)
|
|
* Released under Modified BSD License
|
|
*
|
|
* See license.txt for details
|
|
*
|
|
* Change Log:
|
|
* - Jan 5, 2015 - File Created
|
|
* - Sep 1, 2016 - Copyright Updated and added file header
|
|
* - Oct 24, 2016 - Added check for presence of /tmp/cce, /tmp/cve, & /tmp/cpe directories before downloading
|
|
* - Nov 9, 2016 - Added command line parameters for each of the update types (CVE, CPE, CCE),
|
|
* Added detection if the file from same day already exists, added "download only" option,
|
|
* Added mkdir if temporary directories don't exist, added usage output,
|
|
* Changed URL for CCE and CPE to HTTPS paths
|
|
* - Nov 21, 2016 - Cleaned up code a little and added 3 second sleep between commands so user can see results
|
|
* - Dec 7, 2016 - Changed PHP constant to PHP_BIN
|
|
* - Jan 30, 2017 - Updated to parse Nessus NASL plugin update file
|
|
* - Jan 31, 2017 - Completed NASL plugin parsing
|
|
* - Feb 15, 2017 - Completed adding STIG download and update, added parse only (po) flag for offline systems,
|
|
* Added ping method to check for connection to internet. Added url_exists method to check for presence of file before attempting to download.
|
|
* - Feb 21, 2017 - Extended output for nasl on Windows hosts, store existing plugins in memory to speed up processing,
|
|
* Delete files if there are parsing errors, added --delete flag when parsing stigs,
|
|
* Added check to see if STIG file has been downloaded already today
|
|
* - Mar 3, 2017 - Fixed output of tar command like Jeff suggested, and clarified -u parameter
|
|
* - Mar 8, 2017 - Added check for presence of downloaded files before attempting to parse
|
|
* - Mar 13, 2017 - Cleaned up downloads of STIG compilation file, added check for *v2.zip file
|
|
* - Mar 17, 2017 - Added check for previous STIG file that contains '_v2' at the end of the file,
|
|
* Added check for any other *.zip files in the /tmp folder just in case user only wants to upgrade FOUO files or individual XML files
|
|
* - Mar 20, 2017 - Added check for previous STIG that includes '_v2' at end of filename, and added checks to fix issues when FOUO file is present
|
|
* - Mar 22, 2017 - Added check for extracted STIG files in /tmp/stig/xml/*.xml
|
|
* - Apr 5, 2017 - Added check for previous STIG compilation_v2, so will now check for 4 different naming possibilities,
|
|
* Extended TIME_WAIT ports to 2000, started time totaling process
|
|
* - Apr 7, 2017 - Fixed typo in printed tar command
|
|
* - May 13, 2017 - Clarified if it cannot access the internet instead of "cannot connect to server"
|
|
* Fixed confusion with Cygwin and Bash on Windows paths
|
|
* - Jun 27, 2017 - Matt Shuter: Fixed bug #262 & #270
|
|
* - Dec 27, 2017 - Added database field and download progress flag
|
|
* - Apr 29, 2018 - Added extract parameter to only extract nasl archive file, fixed a couple bugs
|
|
* - May 10, 2018 - Removed ping of cve.mitre.org, and added 'po' and 'do' parameters for NVD CVE
|
|
* - Jun 5, 2018 - Fixed a couple setting updates
|
|
* - Sep 18, 2018 - Jeff - Added --sunset switch for Installing Sunset STIGs from https://iase.disa.mil/stigs/sunset/Pages/index.aspx
|
|
*/
|
|
include_once 'config.inc';
|
|
include_once 'helper.inc';
|
|
include_once 'error.inc';
|
|
include_once 'database.inc';
|
|
include_once 'DateTimeDiff.php';
|
|
include_once 'vendor/autoload.php';
|
|
|
|
use Monolog\Logger;
|
|
use Monolog\Handler\StreamHandler;
|
|
use Monolog\Formatter\LineFormatter;
|
|
|
|
$current_date = new DateTime();
|
|
$total_complete = 0;
|
|
$threads = [];
|
|
|
|
$cmd = getopt("h::u::p::", ['cpe::', 'cce::', 'cve::', 'nvd::', 'nasl::', 'stig::', 'sunset::', 'do::', 'po::', 'help::', 'debug::', 'extract::', 'exclude::']);
|
|
|
|
$db = new db();
|
|
$diff = new DateTimeDiff();
|
|
|
|
$log_level = Logger::ERROR;
|
|
switch (LOG_LEVEL) {
|
|
case E_WARNING:
|
|
$log_level = Logger::WARNING;
|
|
break;
|
|
case E_NOTICE:
|
|
$log_level = Logger::NOTICE;
|
|
break;
|
|
case E_DEBUG:
|
|
$log_level = Logger::DEBUG;
|
|
}
|
|
|
|
if (isset($cmd['debug']) && $cmd['debug']) {
|
|
$log_level = Logger::DEBUG;
|
|
}
|
|
|
|
$stream = new StreamHandler("php://output", $log_level);
|
|
$stream->setFormatter(new LineFormatter("%datetime% %level_name% %message%" . PHP_EOL, "H:i:s.u"));
|
|
|
|
$log = new Logger("update_db");
|
|
$log->pushHandler(new StreamHandler(LOG_PATH . "/update_db.log", $log_level));
|
|
$log->pushHandler($stream);
|
|
|
|
if (isset($cmd['h'], $cmd['help']) ||
|
|
(!isset($cmd['cpe']) && !isset($cmd['cve']) && !isset($cmd['nasl']) && !isset($cmd['stig']) && !isset($cmd['sunset']) && !isset($cmd['nvd']))) {
|
|
die(usage());
|
|
}
|
|
|
|
if (isset($cmd['do']) || !isset($cmd['po'])) {
|
|
if (!ping("cyberperspectives.com")) {
|
|
$log->emergency("Cannot connect to internet");
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Update CPE content downloaded from NIST
|
|
*/
|
|
if (isset($cmd['cpe'])) {
|
|
$db->set_Setting_Array([
|
|
'cpe-dl-progress' => 0,
|
|
'cpe-progress' => 0,
|
|
'cpe-count' => 0
|
|
]);
|
|
|
|
$path = TMP . "/cpe";
|
|
if (isset($cmd['p']) && $cmd['p']) {
|
|
$path = $cmd['p'];
|
|
}
|
|
|
|
check_path($path);
|
|
|
|
$diff->resetClock();
|
|
print "Started CPE ingestion ({$diff->getStartClockTime()})" . PHP_EOL;
|
|
|
|
// search for an unzip any zip files in the tmp directory
|
|
$zip_files = glob("{$path}/*cpe-dictionary*.zip");
|
|
if (count($zip_files)) {
|
|
$log->debug("Found a existing cpe-dictionary.zip file, unzipping then parsing");
|
|
$zip = new ZipArchive();
|
|
foreach ($zip_files as $file) {
|
|
$log->debug("Unzipping {$file}");
|
|
$zip->open($file);
|
|
$zip->extractTo($path);
|
|
$zip->close();
|
|
unlink($file);
|
|
}
|
|
}
|
|
|
|
// search for any existing cpe-dictionary files in the /tmp directory
|
|
$tmp_files = glob(TMP . "/*cpe-dictionary*.xml");
|
|
if (count($tmp_files)) {
|
|
$log->debug("Found existing cpe-dictionary.xml file in TMP folder, moving to TMP/cpe then processing");
|
|
foreach ($tmp_files as $fname) {
|
|
$name = basename($fname);
|
|
if ($name == 'official-cpe-dictionary_v2.3.xml') {
|
|
$name = "cpe-dictionary-{$diff->getStartClock()->format("Ymd")}.xml";
|
|
}
|
|
rename($fname, "{$path}/{$name}");
|
|
}
|
|
}
|
|
|
|
$cpe_fname = realpath($path) . "/cpe-dictionary-{$current_date->format('Ymd')}.xml";
|
|
$cpe_url = "https://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml";
|
|
$cpe_parse_fname = null;
|
|
|
|
// download the file if the do flag is used even if it already exists
|
|
if (isset($cmd['do']) && !isset($cmd['po'])) {
|
|
download_file($cpe_url, $cpe_fname, $db->help, 'cpe-dl-progress');
|
|
}
|
|
// download the file only if it doesn't exist
|
|
elseif (!file_exists($cpe_fname) && !isset($cmd['po'])) {
|
|
download_file($cpe_url, $cpe_fname, $db->help, 'cpe-dl-progress');
|
|
}
|
|
elseif (!isset($cmd['po'])) {
|
|
$log->error("Could not connect to nist.gov to download the CPE library");
|
|
die();
|
|
}
|
|
|
|
$dt = new DateTime();
|
|
|
|
if (!isset($cmd['do']) || isset($cmd['po'])) {
|
|
$cpe_files = glob("{$path}/*cpe-dictionary*.xml");
|
|
rsort($cpe_files, SORT_NATURAL);
|
|
|
|
if (count($cpe_files)) {
|
|
$match = [];
|
|
if (preg_match("/cpe\-dictionary\-([\d]+)\.xml/", $cpe_files[0], $match)) {
|
|
$dt = DateTime::createFromFormat("Ymd", $match[1]);
|
|
|
|
$seven_days_old = new DateTime();
|
|
$seven_days_old->sub(DateInterval::createFromDateString("7 days"));
|
|
|
|
if ($dt < $seven_days_old) {
|
|
$log->warning("The file that is being ingested is more than 7 days old ({$dt->format('Y-m-d')})");
|
|
}
|
|
|
|
$cpe_parse_fname = $cpe_files[0];
|
|
}
|
|
else {
|
|
$log->warning("Don't know when the file was downloaded, but parsing it anyway");
|
|
$cpe_parse_fname = $cpe_files[0];
|
|
}
|
|
}
|
|
|
|
if (is_null($cpe_parse_fname)) {
|
|
$log->warning("Could not find a CPE file to parse");
|
|
}
|
|
|
|
$script = realpath(defined('PHP_BIN') ? PHP_BIN : PHP) .
|
|
" -c " . realpath(PHP_CONF) .
|
|
" -f " . realpath(DOC_ROOT . "/exec/parse_cpe.php") . " --" .
|
|
" -f=\"" . realpath($cpe_parse_fname) . "\"" .
|
|
" -d=\"{$dt->format("Y-m-d")}\"";
|
|
|
|
$log->debug("Running CPE parsing script on file: $cpe_parse_fname");
|
|
passthru($script);
|
|
}
|
|
|
|
$db->help->select_count("software");
|
|
$cpe_count = $db->help->execute();
|
|
|
|
$db->set_Setting("cpe-count", $cpe_count);
|
|
|
|
$diff->stopClock();
|
|
|
|
print PHP_EOL . "Finished at {$diff->getEndClockTime()}" . PHP_EOL .
|
|
"Total time: {$diff->getDiffString()}" . PHP_EOL;
|
|
|
|
sleep(3);
|
|
}
|
|
|
|
/**
|
|
* Update CVE content
|
|
*/
|
|
if (isset($cmd['cve'])) {
|
|
$db->set_Setting_Array([
|
|
'nvd-cve-dl-progress' => 0,
|
|
'nvd-cve-progress' => 0,
|
|
'nvd-cve-count' => 0,
|
|
'cve-dl-progress' => 0,
|
|
'cve-progress' => 0,
|
|
'cve-count' => 0
|
|
]);
|
|
$path = TMP . "/cve";
|
|
if (isset($cmd['p']) && $cmd['p']) {
|
|
$path = $cmd['p'];
|
|
}
|
|
|
|
check_path($path);
|
|
|
|
$diff->resetClock();
|
|
print "Started CVE ingestion {$diff->getStartClockTime()}" . PHP_EOL;
|
|
|
|
$cve_files = glob(TMP . "/allitems.xml");
|
|
if (count($cve_files)) {
|
|
foreach ($cve_files as $file) {
|
|
rename($file, "{$path}/cve-all-{$diff->getStartClock()->format("Ymd")}.xml");
|
|
}
|
|
}
|
|
|
|
$tmp_files = glob("{$path}/cve*.xml");
|
|
if (count($tmp_files)) {
|
|
foreach ($tmp_files as $fname) {
|
|
rename($fname, "{$path}/" . basename($fname));
|
|
}
|
|
}
|
|
|
|
$cve_fname = realpath($path) . "/cve-all-{$current_date->format('Ymd')}.xml";
|
|
$cve_url = "http://cve.mitre.org/data/downloads/allitems.xml";
|
|
$cve_parse_fname = null;
|
|
|
|
if (isset($cmd['do']) && !isset($cmd['po'])) {
|
|
download_file($cve_url, $cve_fname, $db->help, 'cve-dl-progress');
|
|
}
|
|
elseif (!file_exists($cve_fname) && !isset($cmd['po'])) {
|
|
download_file($cve_url, $cve_fname, $db->help, 'cve-dl-progress');
|
|
}
|
|
elseif (!isset($cmd['po'])) {
|
|
Sagacity_Error::err_handler("Could not connect to cve.mitre.org to download the CVE library", E_ERROR);
|
|
}
|
|
|
|
if (!isset($cmd['do']) || isset($cmd['po'])) {
|
|
$cve_files = glob("{$path}/cve-all-*.xml");
|
|
rsort($cve_files, SORT_NATURAL);
|
|
|
|
if (count($cve_files)) {
|
|
$match = [];
|
|
if (preg_match("/cve\-all\-([\d]+)\.xml/", $cve_files[0], $match)) {
|
|
$dt = DateTime::createFromFormat("Ymd", $match[1]);
|
|
|
|
$seven_days_old = new DateTime();
|
|
$seven_days_old->sub(DateInterval::createFromDateString("7 days"));
|
|
|
|
if ($dt < $seven_days_old) {
|
|
$log->warning("The CVE file that is being ingested is more than 7 days old ({$dt->format('Y-m-d')})");
|
|
}
|
|
|
|
$cve_parse_fname = $cve_files[0];
|
|
}
|
|
}
|
|
|
|
if (is_null($cve_parse_fname)) {
|
|
$log->error("Could not find a CVE file to parse");
|
|
die;
|
|
}
|
|
|
|
$script = realpath(defined('PHP_BIN') ? PHP_BIN : PHP) .
|
|
" -c " . realpath(PHP_CONF) .
|
|
" -f " . realpath(DOC_ROOT . "/exec/parse_cve.php") . " --" .
|
|
" -f=\"" . realpath($cve_parse_fname) . "\"" .
|
|
" -d=\"{$dt->format("Y-m-d")}\"";
|
|
|
|
$log->debug("Script to run $script");
|
|
passthru($script);
|
|
}
|
|
|
|
$db->help->select_count("sagacity.cve_db");
|
|
$cve_count = $db->help->execute();
|
|
|
|
$db->set_Setting_Array([
|
|
'cve-dl-progress' => 100,
|
|
'cve-progress' => 100,
|
|
'cve-count' => $cve_count,
|
|
'cve-load-date' => new DateTime(),
|
|
'nvd-cve-dl-progress' => 100,
|
|
'nvd-cve-progress' => 100,
|
|
'nvd-cve-count' => $cve_count,
|
|
'nvd-cve-load-date' => new DateTime()
|
|
]);
|
|
|
|
$diff->stopClock();
|
|
|
|
print "Finished at {$diff->getEndClockTime()}" . PHP_EOL .
|
|
"Total Time: {$diff->getDiffString()}" . PHP_EOL;
|
|
|
|
sleep(3);
|
|
}
|
|
|
|
/**
|
|
* Update to NVD CVE content
|
|
*/
|
|
if (isset($cmd['nvd'])) {
|
|
$db->set_Setting_Array([
|
|
'nvd-cve-dl-progress' => 0,
|
|
'nvd-cve-progress' => 0,
|
|
'nvd-cve-count' => 0,
|
|
'cve-dl-progress' => 0,
|
|
'cve-progress' => 0,
|
|
'cve-count' => 0
|
|
]);
|
|
$path = TMP . "/nvd";
|
|
if (isset($cmd['p']) && $cmd['p']) {
|
|
$path = $cmd['p'];
|
|
}
|
|
check_path($path);
|
|
|
|
$diff->resetClock();
|
|
print "Started NVD CVE ingestion ({$diff->getStartClockTime()})" . PHP_EOL;
|
|
|
|
$nvd_years = [];
|
|
for ($x = 2002; $x <= $diff->getStartClock()->format("Y"); $x++) {
|
|
$nvd_years[] = $x;
|
|
}
|
|
|
|
if (isset($cmd['do']) || !isset($cmd['po'])) {
|
|
$too_old = new DateTime();
|
|
$too_old->sub(DateInterval::createFromDateString("7 days"));
|
|
|
|
$load_date = new DateTime($db->get_Settings("nvd-cve-load-date"));
|
|
if ($load_date < $too_old) {
|
|
// More than 7 days old so have to do a full load
|
|
foreach ($nvd_years as $x => $yr) {
|
|
$db->set_Setting('nvd-year', $yr);
|
|
$db->set_Setting('nvd-cve-dl-progress', (($x + 1) / count($nvd_years)) * 100);
|
|
download_file("https://static.nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-{$yr}.json.zip", TMP . "/nvd/nvdcve-{$yr}.json.zip");
|
|
$zip = new ZipArchive();
|
|
$zip->open(TMP . "/nvd/nvdcve-{$yr}.json.zip");
|
|
$zip->extractTo(TMP . "/nvd");
|
|
$zip->close();
|
|
unlink(TMP . "/nvd/nvdcve-{$yr}.json.zip");
|
|
}
|
|
}
|
|
else {
|
|
$db->set_Setting('nvd-year', 'modified');
|
|
download_file("https://static.nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-modified.json.zip", TMP . "/nvd/nvdcve-modified.json.zip", $db->help, 'nvd-cve-dl-progress');
|
|
$zip = new ZipArchive();
|
|
$zip->open(TMP . "/nvd/nvdcve-modified.json.zip");
|
|
$zip->extractTo(TMP . "/nvd");
|
|
$zip->close();
|
|
unlink(TMP . "/nvd/nvdcve-modified.json.zip");
|
|
|
|
$db->set_Setting('nvd-year', 'recent');
|
|
download_file("https://static.nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-recent.json.zip", TMP . "/nvd/nvdcve-recent.json.zip", $db->help, 'nvd-cve-dl-progress');
|
|
$zip->open(TMP . "/nvd/nvdcve-recent.json.zip");
|
|
$zip->extractTo(TMP . "/nvd");
|
|
$zip->close();
|
|
unlink(TMP . "/nvd/nvdcve-recent.json.zip");
|
|
}
|
|
}
|
|
|
|
chdir(DOC_ROOT . "/exec");
|
|
if (isset($cmd['po']) || !isset($cmd['do'])) {
|
|
$json_files = glob(TMP . "/nvd/*.json");
|
|
foreach ($json_files as $x => $j) {
|
|
$match = [];
|
|
if (preg_match("/(\d{4}|recent|modified)/", basename($j), $match)) {
|
|
$db->set_Setting('nvd-year', $match[1]);
|
|
}
|
|
else {
|
|
$db->set_Setting('nvd-year', null);
|
|
}
|
|
$script = realpath(defined('PHP_BIN') ? PHP_BIN : PHP) .
|
|
" -c " . realpath(PHP_CONF) .
|
|
" -f " . realpath(DOC_ROOT . "/exec/parse_nvd_json_cve.php") . " --" .
|
|
" -f=\"" . realpath($j) . "\"";
|
|
|
|
$log->debug("Running NVD CVE parsing script on file: $j");
|
|
passthru($script);
|
|
$db->set_Setting('nvd-cve-progress', (($x + 1) / count($json_files)) * 100);
|
|
}
|
|
}
|
|
|
|
$db->help->select_count("sagacity.cve_db");
|
|
$nvd_count = $db->help->execute();
|
|
|
|
$diff->stopClock();
|
|
print "Finished at {$diff->getEndClockTime()}" . PHP_EOL .
|
|
"Total time {$diff->getTotalDiffString()}" . PHP_EOL;
|
|
|
|
$db->set_Setting_Array([
|
|
'nvd-cve-load-date' => $diff->getEndClock()->format(MYSQL_DT_FORMAT),
|
|
'nvd-cve-count' => $nvd_count,
|
|
'nvd-cve-progress' => 100,
|
|
'nvd-cve-dl-progress' => 100,
|
|
'nvd-year' => null,
|
|
'cve-load-date' => $diff->getEndClock()->format(MYSQL_DT_FORMAT),
|
|
'cve-count' => $nvd_count,
|
|
'cve-progress' => 100,
|
|
'cve-dl-progress' => 100
|
|
]);
|
|
}
|
|
|
|
/**
|
|
* Update CCE content
|
|
*/
|
|
if (isset($cmd['cce'])) {
|
|
check_path(TMP . "/cce");
|
|
|
|
$cce_fname = TMP . "/cce/cce-all-20130214.xml";
|
|
|
|
if (!file_exists($cce_fname) && ping("nist.gov") && !isset($cmd['po'])) {
|
|
download_file("https://static.nvd.nist.gov/feeds/xml/cce/cce-COMBINED-5.20130214.xml", $cce_fname);
|
|
}
|
|
|
|
if (!isset($cmd['do']) || isset($cmd['po'])) {
|
|
|
|
}
|
|
|
|
sleep(3);
|
|
}
|
|
|
|
/**
|
|
* Parse NASL content from NVT and/or Nessus
|
|
*/
|
|
if (isset($cmd['nasl'])) {
|
|
$db->set_Setting_Array([
|
|
'nasl-dl-progress' => 0,
|
|
'nasl-progress' => 0,
|
|
'nasl-count' => 0
|
|
]);
|
|
$count = 0;
|
|
|
|
// Capture start time for performance monitoring
|
|
$diff->resetClock();
|
|
print "Started NASL ingestion ({$diff->getStartClockTime()})" . PHP_EOL;
|
|
|
|
// Generate a unique filename for the OpenVAS feed archive using the current date
|
|
$nasl_fname = TMP . "/nessus_plugins/nasl_plugins-{$current_date->format("Ymd")}.tar.bz2";
|
|
|
|
// Download OpenVAS feed if a) it doesn't exist, b) can reach openvas.org, and c) parse only flag not set
|
|
if (!file_exists($nasl_fname) && ping("openvas.org") && !isset($cmd['po'])) {
|
|
$log->debug("Downloading new NASL library");
|
|
download_file("http://www.openvas.org/openvas-nvt-feed-current.tar.bz2", $nasl_fname, $db->help, 'nasl-dl-progress');
|
|
}
|
|
|
|
// Can only extract .tar.bz2 files on Linux so...
|
|
if (!isset($cmd['do']) || isset($cmd['po'])) {
|
|
if (file_exists($nasl_fname)) {
|
|
if (substr(strtolower(PHP_OS), 0, 3) == 'lin') {
|
|
$log->debug("Extracting NASL files from archive");
|
|
passthru("tar xvf $nasl_fname -C " . realpath(TMP . "/nessus_plugins") .
|
|
" --wildcards --transform='s/.*\///' '*.nasl'");
|
|
|
|
if (isset($cmd['extract'])) {
|
|
print "Completed extracting files from archive" . PHP_EOL;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (isset($cmd['extract'])) {
|
|
die;
|
|
}
|
|
|
|
// ...if there are no .nasl files in the directory, die and give instructions for unzipping in Windows
|
|
$files = glob(TMP . "/nessus_plugins/*.nasl");
|
|
if (strtolower(substr(PHP_OS, 0, 3)) == 'win') {
|
|
if (file_exists(getenv("%ProgramData%") . "/Tenable/Nessus/nessus/plugins")) {
|
|
$log->debug("Importing local Nessus plugin files");
|
|
$files = array_merge($files, glob(getenv("%ProgramData%") . "/Tenable/Nessus/nessus/plugins/*.nasl"));
|
|
}
|
|
}
|
|
elseif (strtolower(substr(PHP_OS, 0, 3)) == 'lin') {
|
|
if (file_exists("/opt/nessus/lib/nessus/plugins") && is_readable("/opt/nessus/lib/nessus/plugins")) {
|
|
$log->debug("Importing local Nessus plugin files");
|
|
$files = array_merge($files, glob("/opt/nessus/lib/nessus/plugins/*.nasl"));
|
|
}
|
|
|
|
if (file_exists("/opt/sc/data/nasl") && is_readable("/opt/sc/data/nasl")) {
|
|
$log->debug("Importing local Nessus plugin files");
|
|
$files = array_merge($files, glob("/opt/sc/data/nasl/*.nasl"));
|
|
}
|
|
}
|
|
$files = array_unique($files);
|
|
|
|
if (!($file_count = count($files))) {
|
|
$log->emergency("Downloaded the OpenVAS NVT plugin repository, please extract *.nasl files to " . realpath(TMP . "/nessus_plugins") . PHP_EOL .
|
|
"If you have Bash on Windows ({path} = /mnt/c/xampp/www) or Cygwin ({path} = /cygdrive/c/xampp/www) installed you can run the following command on the downloaded file tweaking the paths" . PHP_EOL .
|
|
"tar xvf {path}/tmp/nessus_plugins/" . basename($nasl_fname) . " -C {path}/tmp/nessus_plugins --wildcards --transform='s/.*\///' '*.nasl'" . PHP_EOL);
|
|
die;
|
|
}
|
|
|
|
// Report how many NASL files were found in the directory
|
|
print "Found {$file_count} NASL files" . PHP_EOL . "Started at {$diff->getStartClockTime()}" . PHP_EOL;
|
|
|
|
// Query database to build an array of existing plugins to compare against on import
|
|
$existing_plugins = [];
|
|
$db->help->select("nessus_plugins", ['plugin_id', 'file_date']);
|
|
$rows = $db->help->execute();
|
|
if (is_array($rows) && count($rows)) {
|
|
foreach ($rows as $row) {
|
|
$existing_plugins[$row['plugin_id']] = DateTime::createFromFormat("U", $row['file_date']);
|
|
}
|
|
}
|
|
$log->debug("Count of existing plugins " . count($existing_plugins));
|
|
|
|
// Sort the files and loop over them
|
|
natsort($files);
|
|
foreach ($files as $file) {
|
|
// Read the current NASL file into a nasl object
|
|
$nasl = new nasl($file);
|
|
|
|
// calculate percent complete
|
|
$comp = number_format(($total_complete / $file_count) * 100, 2);
|
|
print "\r{$comp}%";
|
|
$log->debug("Parsing {$file} ({$comp}%)");
|
|
|
|
// If no plugin ID, delete file and continue to the next plugin
|
|
if (!isset($nasl->id)) {
|
|
$log->warning("Could not locate an ID in the plugin, skipping");
|
|
unlink($file);
|
|
continue;
|
|
}
|
|
|
|
// Only process if plugin doesn't already exist or has an older last_modificaiton date
|
|
if (!isset($existing_plugins[$nasl->id]) ||
|
|
(isset($nasl->last_modification) && $existing_plugins[$nasl->id] > $nasl->last_modification)) {
|
|
$log->info("Updating plugin {$nasl->id}");
|
|
|
|
// define command line to call script to parse the file
|
|
$script = realpath(defined('PHP_BIN') ? PHP_BIN : PHP) .
|
|
" -c " . realpath(PHP_CONF) .
|
|
" -f " . realpath(DOC_ROOT . "/exec/nessus-plugin-to-database.php") . " --" .
|
|
" -f=\"{$file}\"";
|
|
|
|
$threads[] = new \Cocur\BackgroundProcess\BackgroundProcess($script);
|
|
end($threads)->run();
|
|
|
|
$count++;
|
|
$total_complete++;
|
|
|
|
if($total_complete % 100 == 0) {
|
|
$db->set_Setting('nasl-progress', $comp);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
$db->set_Setting_Array([
|
|
'nasl-dl-progress' => 100,
|
|
'nasl-progress' => 100,
|
|
'nasl-count' => $total_complete,
|
|
'nasl-load-date' => new DateTime()
|
|
]);
|
|
|
|
$diff->stopClock();
|
|
|
|
print PHP_EOL . "Finished at {$diff->getEndClockTime()}" . PHP_EOL .
|
|
"Total Time: {$diff->getDiffString()}" . PHP_EOL;
|
|
|
|
sleep(3);
|
|
}
|
|
|
|
/**
|
|
* Update STIG library from DISA content
|
|
*/
|
|
if (isset($cmd['stig'])) {
|
|
$db->set_Setting_Array([
|
|
'stig-dl-progress' => 0,
|
|
'stig-progress' => 0,
|
|
'stig-count' => 0
|
|
]);
|
|
$path = TMP . "/stigs/zip";
|
|
check_path($path);
|
|
$stigUrlArray = [];
|
|
$tmp = [];
|
|
$tmp1 = [];
|
|
$tmp2 = [];
|
|
$tmp3 = [];
|
|
|
|
$diff->resetClock();
|
|
print "Started STIG ingestion ({$diff->getStartClockTime()})" . PHP_EOL;
|
|
|
|
$url_1 = "https://iase.disa.mil/stigs/Pages/a-z.aspx";
|
|
$url_2 = "https://iase.disa.mil";
|
|
$sunset_url = "https://iase.disa.mil/stigs/Lists/Sunset%20Master%20List/FinalView.aspx";
|
|
$regex = "/a href=\"([^ ]+zip\/U_[^ ]+STIG\.zip)[^\>]+\>([^\<]+)\<\/a\>/i";
|
|
|
|
if (!isset($cmd['po']) || isset($cmd['do'])) {
|
|
$log->debug("Checking url: $url_1");
|
|
$pg_contents = file_get_contents($url_1);
|
|
|
|
if(preg_match("/RefreshPageTo\(event, ([^\}]+\})/i", $pg_contents, $tmp)) {
|
|
$url_2 .= str_replace(["\u0026", '"'], ["&", ""], html_entity_decode($tmp[1]));
|
|
}
|
|
|
|
$log->debug("Checking url: $url_2");
|
|
$pg_contents2 = file_get_contents($url_2);
|
|
$log->debug("Checking url: $sunset_url");
|
|
$sunset_contents = file_get_contents($sunset_url);
|
|
|
|
$log->debug("Retrieving all matches");
|
|
preg_match_all($regex, $pg_contents, $tmp1);
|
|
preg_match_all($regex, $pg_contents2, $tmp2);
|
|
preg_match_all($regex, $sunset_contents, $tmp3);
|
|
|
|
$stigUrlArray = array_merge($tmp1[1], $tmp2[1], $tmp3[1]);
|
|
$log->debug("Match count: " . count($stigUrlArray));
|
|
|
|
print "Downloading " . count($stigUrlArray) . PHP_EOL;
|
|
if(is_array($stigUrlArray) && count($stigUrlArray)) {
|
|
foreach($stigUrlArray as $url) {
|
|
$stigFname = basename($url);
|
|
$log->debug("Downloading $stigFname");
|
|
download_file($url, "{$path}/$stigFname");
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!isset($cmd['do']) || isset($cmd['po'])) {
|
|
$stig_files = array_merge(
|
|
glob("{$path}/*.zip"), glob("{$path}/*.xml"), glob(TMP . "/*.zip"), glob(TMP . "/*.xml"), glob(TMP . "/stigs/xml/*.xml")
|
|
);
|
|
if (!count($stig_files)) {
|
|
die("Could not locate any XCCDF STIG libraries " . realpath(TMP));
|
|
}
|
|
|
|
$script = realpath(defined('PHP_BIN') ? PHP_BIN : PHP) .
|
|
" -c " . realpath(PHP_CONF) .
|
|
" -f " . realpath(DOC_ROOT . "/exec/background_stigs.php") . " --" .
|
|
(isset($cmd['exclude']) && $cmd['exclude'] ? " --exclude=\"{$cmd['exclude']}\"" : "") .
|
|
" --delete";
|
|
|
|
$log->debug("Script to run $script");
|
|
passthru($script);
|
|
}
|
|
|
|
$db->help->select_count("sagacity.stigs");
|
|
$stig_count = $db->help->execute();
|
|
|
|
$db->set_Setting("stig-count", $stig_count);
|
|
|
|
$diff->stopClock();
|
|
|
|
print PHP_EOL . "Finished at {$diff->getEndClockTime()}" . PHP_EOL .
|
|
"Total Time: {$diff->getDiffString()}" . PHP_EOL;
|
|
|
|
sleep(3);
|
|
}
|
|
|
|
if (is_a($diff->getTotalDiff(), 'DateInterval')) {
|
|
print "Total Script Time: {$diff->getTotalDiffString()}" . PHP_EOL;
|
|
}
|
|
|
|
/**
|
|
* Function to download the latest STIG compilation library zip file for extraction and updating
|
|
*/
|
|
function getStigLibrary()
|
|
{
|
|
global $current_date, $cmd, $log, $db;
|
|
$path = TMP;
|
|
|
|
$mon = '01';
|
|
$prev_mon = '10';
|
|
$year = (int) $current_date->format("Y");
|
|
|
|
if (between($current_date->format("n"), 4, 6)) {
|
|
$mon = '04';
|
|
$prev_mon = '01';
|
|
}
|
|
elseif (between($current_date->format("n"), 7, 9)) {
|
|
$mon = '07';
|
|
$prev_mon = '04';
|
|
}
|
|
elseif (between($current_date->format("n"), 10, 12)) {
|
|
$mon = '10';
|
|
$prev_mon = '07';
|
|
}
|
|
|
|
$current_url = "http://iasecontent.disa.mil/stigs/zip/Compilations/U_SRG-STIG_Library_{$year}_{$mon}.zip";
|
|
$current_v2_url = "http://iasecontent.disa.mil/stigs/zip/Compilations/U_SRG-STIG_Library_{$year}_{$mon}_v2.zip";
|
|
|
|
$stig_fname = "{$path}/stig_library-{$year}_{$mon}.zip";
|
|
|
|
if (!file_exists($stig_fname) && ping("disa.mil") && !isset($cmd['po'])) {
|
|
if (isset($cmd['u'])) {
|
|
$url = $cmd['u'];
|
|
$log->debug("Checking for $url");
|
|
if (url_exists($url)) {
|
|
download_file($url, $stig_fname, $db->help, 'stig-dl-progress');
|
|
}
|
|
}
|
|
else {
|
|
$log->debug("Checking for $current_url");
|
|
if ($found = url_exists($current_url)) {
|
|
download_file($current_url, $stig_fname, $db->help, 'stig-dl-progress');
|
|
}
|
|
|
|
if (!$found) {
|
|
$log->debug("Checking for $current_v2_url");
|
|
if ($found = url_exists($current_v2_url)) {
|
|
download_file($current_v2_url, $stig_fname, $db->help, 'stig-dl-progress');
|
|
}
|
|
}
|
|
|
|
if ($mon == '01') {
|
|
$year--;
|
|
}
|
|
$prev_url = "http://iasecontent.disa.mil/stigs/zip/Compilations/U_SRG-STIG_Library_{$year}_{$prev_mon}.zip";
|
|
$prev_v2_url = "http://iasecontent.disa.mil/stigs/zip/Compilations/U_SRG-STIG_Library_{$year}_{$prev_mon}_v2.zip";
|
|
|
|
if (!$found) {
|
|
$log->debug("Checking for $prev_url");
|
|
if ($found = url_exists($prev_url)) {
|
|
download_file($prev_url, $stig_fname, $db->help, 'stig-dl-progress');
|
|
}
|
|
}
|
|
|
|
if (!$found) {
|
|
$log->debug("Checking for $prev_v2_url");
|
|
if (url_exists($prev_v2_url)) {
|
|
download_file($prev_v2_url, $stig_fname, $db->help, 'stig-dl-progress');
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Usage information about the script
|
|
*/
|
|
function usage()
|
|
{
|
|
$tmp = realpath(TMP);
|
|
print <<<EOO
|
|
Purpose: The purpose of this script is to update the CVE, CPE, and CCE databases. Script will sleep for 3 seconds between actions to allow you review the results.
|
|
|
|
Usage: php update_db.php [--cpe] [--cve] [--nvd] [--nasl] [--stig] [-u={URL}] [--do] [--po] [-h|--help] [--debug] [--exclude="ex1"]
|
|
|
|
--cpe To download and update the CPE catalog
|
|
--cve To download and update the CVE catalog using Mitre's database
|
|
--nvd To download and update the CVE catalog using the National Vulnerability Database (NVD) JSON library
|
|
--nasl To download OpenVAS NVT library and update NASL files
|
|
You can also extract *.nasl files from the Nessus library to $tmp/nessus_plugins and it will include these in the update
|
|
--stig To download and update the STIG library
|
|
|
|
--do To download the files only...do not call the parsers will overwrite any existing files
|
|
--po To parse the downloaded files only, do not download
|
|
|
|
--exclude="ex1" Insert a valid regex expression (properly escaped) to exclude specific STIGs from parsing (no '/' necessary)
|
|
--extract Used so script will download and extract files from archive and stop processing
|
|
|
|
-u={url} Used only for STIGs because sometimes DISA will use a non-standard link which makes it difficult to download the file.
|
|
|
|
-h|--help This screen
|
|
--debug To print verbose debugging messages to the console
|
|
|
|
EOO;
|
|
}
|