Ryan Prather
21082c7513
finding.inc - removed ID property to prevent duplicate findings from being added to the table host_list.inc - deleted unused constructor import.inc - formatting db_schema.json - removed sagacity.findings.id field (making tgt_id and pdi_id new primary keys), and updated references Database_Baseline.zip - updated routines for above change background_results.php - fixed bug #19 export-ckl.php - performance adjustments parse_excel_echecklist.php - performance improvements, ensure duplicate findings are not created, make eChecklist true status, update for removing findings.id field parse_nvd_json_cve.php - convert reading json to array instead of object for reading CPEs (which were updated to CPE 2.3 instead of 2.2) parse_* - remove findings.id field database.inc - formatting, and update for removing findings.id field index.php - ensure user can't import a host list without uploading a host list file Fixed: #65, #51, #28, #27, #10
362 lines
12 KiB
PHP
362 lines
12 KiB
PHP
<?php
|
|
/**
|
|
* File: background_results.php
|
|
* Author: Ryan Prather
|
|
* Purpose: Background script file that will call appropriate function for files found
|
|
* Created: Feb 26, 2014
|
|
*
|
|
* Portions Copyright 2016-2018: Cyber Perspectives, LLC, All rights reserved
|
|
* Released under the Apache v2.0 License
|
|
*
|
|
* Portions Copyright (c) 2012-2015, Salient Federal Solutions
|
|
* Portions Copyright (c) 2008-2011, Science Applications International Corporation (SAIC)
|
|
* Released under Modified BSD License
|
|
*
|
|
* See license.txt for details
|
|
*
|
|
* Change Log:
|
|
* - Feb 26, 2014 - File created
|
|
* - May 05, 2014 - Converted parsing functions to classes for threading
|
|
* - Sep 1, 2016 - Copyright updated, added CWD parameter option,
|
|
* Converted to constants, made script execution platform independent
|
|
* - Oct 24, 2016 - Added debug output and cleaned up script string generation
|
|
* - Nov 7, 2016 - If it ain't broke, don't fix it! Had to revert to a previous version because intended improvements broke it
|
|
* - Dec 7, 2016 - Fixed bug where Windows threading was not being started,
|
|
* Changed PHP constant to PHP_BIN, and make sure that script continues running until last result file is done.
|
|
* - Jan 30, 2017 - Converted script to use parse_config.ini file instead of command line parameters and set script to remove config file when all files are completely parsed
|
|
* - Feb 15, 2017 - Converted file_types constants to defined constants and removed unnecessary parameters from parse_* scripts string creation
|
|
* - Feb 21, 2017 - Fixed path issues with scripts not running
|
|
* - Oct 23, 2017 - Conditionally delete parse_config.ini only if not in DEBUG log level
|
|
* - Oct 27, 2017 - Fix to remove desktop.ini files if found
|
|
* - May 24, 2018 - Moved a couple code blocks because of being out of order
|
|
*/
|
|
error_reporting(E_ALL);
|
|
|
|
$cmd = getopt("t::", ["help::"]);
|
|
|
|
$conf = parse_ini_file("parse_config.ini", false);
|
|
|
|
set_time_limit(0);
|
|
|
|
include_once 'config.inc';
|
|
include_once 'database.inc';
|
|
include_once 'helper.inc';
|
|
include_once 'vendor/autoload.php';
|
|
|
|
use Monolog\Logger;
|
|
use Monolog\Handler\StreamHandler;
|
|
|
|
$log_level = Logger::ERROR;
|
|
switch (LOG_LEVEL) {
|
|
case E_WARNING:
|
|
$log_level = Logger::WARNING;
|
|
break;
|
|
case E_NOTICE:
|
|
$log_level = Logger::NOTICE;
|
|
break;
|
|
case E_DEBUG:
|
|
$log_level = Logger::DEBUG;
|
|
}
|
|
|
|
$log = new Logger('result_import');
|
|
$log->pushHandler(new StreamHandler(LOG_PATH . "/result_import.log", $log_level));
|
|
|
|
$debug = (LOG_LEVEL == E_DEBUG ? true : false);
|
|
|
|
if (isset($cmd['help']) || !is_numeric($conf['ste']) || !isset($conf['doc_root'])) {
|
|
die(usage());
|
|
}
|
|
|
|
chdir(TMP);
|
|
check_path(TMP . "/echecklist");
|
|
check_path(TMP . "/nessus");
|
|
check_path(TMP . "/nmap");
|
|
check_path(TMP . "/scc");
|
|
check_path(TMP . "/stig_viewer");
|
|
check_path(TMP . "/terminated");
|
|
check_path(TMP . "/unsupported");
|
|
|
|
$dbh = new db();
|
|
|
|
$files = glob("*.*");
|
|
$stack = [];
|
|
$threads = [];
|
|
|
|
foreach ($files as $file) {
|
|
$res = FileDetection($file);
|
|
$log->debug("File detected", $res);
|
|
|
|
switch ($res['type']) {
|
|
case NESSUS:
|
|
$stack[] = [
|
|
'exec' => 'nessus',
|
|
'file' => $file,
|
|
'ste' => $conf['ste'],
|
|
'source' => 'nessus'
|
|
];
|
|
break;
|
|
case SCC_XCCDF:
|
|
$stack[] = [
|
|
'exec' => 'scc_xccdf',
|
|
'file' => $file,
|
|
'ste' => $conf['ste'],
|
|
'source' => 'scc_xccdf'
|
|
];
|
|
break;
|
|
case STIG_VIEWER_CKL:
|
|
$stack[] = [
|
|
'exec' => 'stig_viewer',
|
|
'file' => $file,
|
|
'ste' => $conf['ste'],
|
|
'source' => 'stig_viewer'
|
|
];
|
|
break;
|
|
case TECH_ECHECKLIST_EXCEL:
|
|
$ignore = false;
|
|
if (isset($conf['ignore'])) {
|
|
$ignore = true;
|
|
}
|
|
$stack[] = [
|
|
'exec' => 'excel_echecklist',
|
|
'file' => $file,
|
|
'ste' => $conf['ste'],
|
|
'ignore_hidden' => $ignore,
|
|
'source' => 'echecklist'
|
|
];
|
|
break;
|
|
case ECHECKLIST_CSV:
|
|
$stack[] = [
|
|
'exec' => 'csv_echecklist',
|
|
'file' => $file,
|
|
'ste' => $conf['ste'],
|
|
'source' => 'echecklist'
|
|
];
|
|
break;
|
|
case PROC_ECHECKLIST_EXCEL:
|
|
$stack[] = [
|
|
'exec' => 'proc_echecklist',
|
|
'file' => $file,
|
|
'ste' => $conf['ste']
|
|
];
|
|
break;
|
|
case HOST_DATA_COLLECTION:
|
|
$stack[] = [
|
|
'exec' => 'data_collection',
|
|
'file' => $file,
|
|
'ste' => $conf['ste'],
|
|
'target' => $cmd['t'],
|
|
'source' => 'data_collection'
|
|
];
|
|
break;
|
|
case NMAP_GREPABLE:
|
|
case NMAP_TEXT:
|
|
case NMAP_XML:
|
|
$stack[] = [
|
|
'exec' => 'nmap',
|
|
'file' => $file,
|
|
'ste' => $conf['ste'],
|
|
'source' => 'nmap'
|
|
];
|
|
break;
|
|
case MBSA_TEXT:
|
|
case MBSA_XML:
|
|
$stack[] = [
|
|
'exec' => 'mbsa',
|
|
'file' => $file,
|
|
'ste' => $conf['ste'],
|
|
'source' => 'mbsa'
|
|
];
|
|
break;
|
|
case MSSQL_XML:
|
|
$stack[] = [
|
|
'exec' => 'mssql',
|
|
'file' => $file,
|
|
'ste' => $conf['ste'],
|
|
'source' => 'mssql'
|
|
];
|
|
break;
|
|
case DIRECTORY:
|
|
break;
|
|
case strpos("UNSUPPORTED", $file) !== false:
|
|
rename($file, realpath(TMP . "/unsupported/" . basename($file)));
|
|
break;
|
|
default:
|
|
error_log("Do not have a parser for " . $file);
|
|
}
|
|
}
|
|
|
|
$log->debug("Current script stack", $stack);
|
|
|
|
foreach ($stack as $key => $s) {
|
|
$existing = $dbh->get_Running_Script_Status($s['ste'], $s['file']);
|
|
if (isset($existing['status']) && $existing['status'] == 'RUNNING') {
|
|
$log->warning("Script to parse " . basename($s['file']) . " is already running");
|
|
unset($stack[$key]);
|
|
continue;
|
|
}
|
|
|
|
$ignore = '';
|
|
if ($s['source'] == 'echecklist' && $s['ignore_hidden']) {
|
|
$ignore = " -i=1";
|
|
}
|
|
|
|
$stack[$key]['script'] = realpath(defined('PHP_BIN') ? PHP_BIN : PHP) .
|
|
" -c " . realpath(PHP_CONF) . " " .
|
|
" -f " . realpath(DOC_ROOT . "/exec/parse_{$s['exec']}.php") . " --" .
|
|
" -f=\"{$s['file']}\"" .
|
|
$ignore .
|
|
($debug ? " --debug" : "");
|
|
|
|
$log->debug("Adding parser for " . basename($s['file']));
|
|
|
|
$dbh->add_Running_Script(basename($s['file']), $s['ste'], $s['source'], $conf['location']);
|
|
}
|
|
|
|
$count = 0;
|
|
|
|
chdir(realpath(DOC_ROOT . "/exec"));
|
|
|
|
foreach ($stack as $s) {
|
|
$threads[] = new Cocur\BackgroundProcess\BackgroundProcess($s['script']);
|
|
end($threads)->run();
|
|
|
|
$log->info("Starting parser script {$s['script']}");
|
|
|
|
sleep(3);
|
|
$count++;
|
|
|
|
while ($count >= MAX_RESULTS) {
|
|
$log->debug("Current MAX_RESULTS met at " . MAX_RESULTS);
|
|
sleep(1);
|
|
$count = $dbh->get_Running_Script_Count($conf['ste']);
|
|
}
|
|
}
|
|
|
|
do {
|
|
sleep(1);
|
|
}
|
|
while ($dbh->get_Running_Script_Count($conf['ste']));
|
|
|
|
if (!$debug && file_exists(DOC_ROOT . "/exec/parse_config.ini")) {
|
|
unlink(DOC_ROOT . "/exec/parse_config.ini");
|
|
}
|
|
|
|
/**
|
|
* Function to import SCC Oval XML Result files
|
|
*
|
|
* @param string $file
|
|
*/
|
|
function import_SCC_OVAL($file)
|
|
{
|
|
if (preg_match('/.*Results\_iavm\_(2009|2010)|Results\_USGCB/i', $file)) {
|
|
return;
|
|
}
|
|
|
|
$target_data = array();
|
|
$db = new db();
|
|
$match = array();
|
|
preg_match('/\_SCC-(\d\.?)+\_(\d{4}\-\d{2}\-\d{2}\_\d{6})\_OVAL/', $file, $match);
|
|
$time_stamp = $match[2];
|
|
$dt = DateTime::createFromFormat('Y-m-d_His', $time_stamp);
|
|
|
|
$source = $db->get_Sources('SCC');
|
|
$dom = new DOMDocument();
|
|
$dom->load($file);
|
|
|
|
$csv = fopen("scc/" . substr(basename($file), 0, -3) . "csv", 'w');
|
|
$ste = $db->get_STE($GLOBALS['opt']['s'])[0];
|
|
$scan = new scan(null, $source, $ste, 1, basename($file), $dt->format('Y-m-d H:i:s'));
|
|
$scan->set_ID($db->save_Scan($scan));
|
|
|
|
$x = new DOMXPath($dom);
|
|
|
|
$sysinfo = $x->query('/oval-res:oval_results/oval-res:results/oval-res:system/oval-sc:oval_system_characteristics/oval-sc:system_info')->item(0);
|
|
|
|
$target_data['os_name'] = $x->query('oval-sc:os_name', $sysinfo)->item(0)->textContent;
|
|
$target_data['os_ver'] = $x->query('oval-sc:os_version', $sysinfo)->item(0)->textContent;
|
|
$target_data['host_name'] = $x->query('oval-sc:primary_host_name', $sysinfo)->item(0)->textContent;
|
|
$interfaces = $x->query('oval-sc:interfaces/oval-sc:interface', $sysinfo);
|
|
$int_count = 0;
|
|
|
|
foreach ($interfaces as $node) {
|
|
$target_data['interface_name' . $int_count] = $x->query('oval-sc:interface_name', $node)->item(0)->textContent;
|
|
$target_data['ip' . $int_count] = $x->query('oval-sc:ip_address', $node)->item(0)->textContent;
|
|
$target_data['mac' . $int_count] = $x->query('oval-sc:mac_address', $node)->item(0)->textContent;
|
|
|
|
$int_count++;
|
|
}
|
|
|
|
$defs = $x->query('/oval-res:oval_results/oval-def:oval_definitions/oval-def:definitions/oval-def:definition');
|
|
|
|
foreach ($defs as $node) {
|
|
$id = $node->getAttribute('id');
|
|
print "Checking oval id: $id" . PHP_EOL;
|
|
//$meta = $x->query('oval-def:metadata', $node)->item(0);
|
|
|
|
$title = $x->query('oval-def:metadata/oval-def:title', $node)->item(0)->textContent;
|
|
$desc = $x->query('oval-def:metadata/oval-def:description', $node)->item(0)->textContent;
|
|
$plat = $x->query('oval-def:metadata/oval-def:affected/oval-def:platform', $node)->item(0)->textContent;
|
|
|
|
$ext = $x->query('oval-def:criteria/oval-def:extend_definition', $node);
|
|
|
|
if ($ext->length > 0) {
|
|
$ext_def = $ext->item(0)->getAttribute('definition_ref');
|
|
$ext_def_op = $x->query('oval-def:criteria', $node)->item(0)->getAttribute('operator');
|
|
}
|
|
else {
|
|
$ext_def = '';
|
|
$ext_def_op = '';
|
|
}
|
|
|
|
$ref = $x->query('oval-def:metadata/oval-def:reference', $node);
|
|
$oval = $db->get_Oval($id);
|
|
|
|
if ($oval->get_PDI_ID()) {
|
|
print "current oval: " . print_r($oval, true);
|
|
$oval->clear_References();
|
|
}
|
|
else {
|
|
$oval = new oval(null, $id, $title, $desc, $plat, $ext_def, $ext_def_op);
|
|
}
|
|
|
|
foreach ($ref as $ref_node) {
|
|
$source = $ref_node->getAttribute('source') == 'http://cce.mitre.org' ? 'CCE' : $ref_node->getAttribute('source');
|
|
$url = $ref_node->hasAttribute('ref_url') ? $ref_node->getAttribute('ref_url') : '';
|
|
$ref_id = $ref_node->getAttribute('ref_id');
|
|
|
|
$oval->add_Reference(new oval_ref($id, $source, $url, $ref_id));
|
|
|
|
if (is_null($oval->get_PDI_ID()) && $source == 'CCE') {
|
|
$cce = $db->get_CCE($ref_id);
|
|
|
|
if (!is_null($cce)) {
|
|
$oval->set_PDI_ID($cce->get_PDI_ID());
|
|
}
|
|
}
|
|
}
|
|
|
|
if ($db->save_Oval($oval)) {
|
|
error_log("Saved oval id: " . $oval->get_Oval_ID());
|
|
}
|
|
else {
|
|
error_log("Error saving oval id: " . $oval->get_Oval_ID());
|
|
}
|
|
}
|
|
}
|
|
|
|
function usage()
|
|
{
|
|
print <<<EOO
|
|
Purpose: This program was written to look at all files in the /tmp directory, determine what parser is needed, then call that parser with the appropriate flags.
|
|
|
|
Usage: background_results.php -s={ste_id} [-i=1] [-t=1] [--help]
|
|
|
|
-s={STE ID} The ID of the ST&E to know what to assign the results to
|
|
-i=1 Ignore hidden Excel worksheets (only used on Excel eChecklist files) (defaulted to false)
|
|
-t={Target Name} The name of the target to evaluate (only used on host data collection)
|
|
|
|
--help This screen
|
|
|
|
EOO;
|
|
}
|