sagacity/exec/background_results.php

362 lines
12 KiB
PHP
Raw Normal View History

2018-05-07 10:51:08 -04:00
<?php
/**
* File: background_results.php
* Author: Ryan Prather
* Purpose: Background script file that will call appropriate function for files found
* Created: Feb 26, 2014
*
2018-07-26 08:33:50 -04:00
* Portions Copyright 2016-2018: Cyber Perspectives, LLC, All rights reserved
2018-05-07 10:51:08 -04:00
* Released under the Apache v2.0 License
*
* Portions Copyright (c) 2012-2015, Salient Federal Solutions
* Portions Copyright (c) 2008-2011, Science Applications International Corporation (SAIC)
* Released under Modified BSD License
*
* See license.txt for details
*
* Change Log:
* - Feb 26, 2014 - File created
* - May 05, 2014 - Converted parsing functions to classes for threading
* - Sep 1, 2016 - Copyright updated, added CWD parameter option,
* Converted to constants, made script execution platform independent
* - Oct 24, 2016 - Added debug output and cleaned up script string generation
* - Nov 7, 2016 - If it ain't broke, don't fix it! Had to revert to a previous version because intended improvements broke it
* - Dec 7, 2016 - Fixed bug where Windows threading was not being started,
* Changed PHP constant to PHP_BIN, and make sure that script continues running until last result file is done.
* - Jan 30, 2017 - Converted script to use parse_config.ini file instead of command line parameters and set script to remove config file when all files are completely parsed
* - Feb 15, 2017 - Converted file_types constants to defined constants and removed unnecessary parameters from parse_* scripts string creation
* - Feb 21, 2017 - Fixed path issues with scripts not running
* - Oct 23, 2017 - Conditionally delete parse_config.ini only if not in DEBUG log level
* - Oct 27, 2017 - Fix to remove desktop.ini files if found
2018-07-26 08:33:50 -04:00
* - May 24, 2018 - Moved a couple code blocks because of being out of order
2018-05-07 10:51:08 -04:00
*/
error_reporting(E_ALL);
$cmd = getopt("t::", ["help::"]);
$conf = parse_ini_file("parse_config.ini", false);
set_time_limit(0);
include_once 'config.inc';
include_once 'database.inc';
include_once 'helper.inc';
include_once 'vendor/autoload.php';
2018-07-26 08:33:50 -04:00
use Monolog\Logger;
use Monolog\Handler\StreamHandler;
$log_level = Logger::ERROR;
switch (LOG_LEVEL) {
case E_WARNING:
$log_level = Logger::WARNING;
break;
case E_NOTICE:
$log_level = Logger::NOTICE;
break;
case E_DEBUG:
$log_level = Logger::DEBUG;
}
$log = new Logger('result_import');
$log->pushHandler(new StreamHandler(LOG_PATH . "/result_import.log", $log_level));
2018-05-07 10:51:08 -04:00
$debug = (LOG_LEVEL == E_DEBUG ? true : false);
2018-07-26 08:33:50 -04:00
if (isset($cmd['help']) || !is_numeric($conf['ste']) || !isset($conf['doc_root'])) {
die(usage());
}
chdir(TMP);
2018-05-07 10:51:08 -04:00
check_path(TMP . "/echecklist");
check_path(TMP . "/nessus");
check_path(TMP . "/nmap");
check_path(TMP . "/scc");
check_path(TMP . "/stig_viewer");
check_path(TMP . "/terminated");
check_path(TMP . "/unsupported");
$dbh = new db();
2018-07-26 08:33:50 -04:00
$files = glob("*.*");
$stack = [];
2018-05-07 10:51:08 -04:00
$threads = [];
foreach ($files as $file) {
2018-07-26 08:33:50 -04:00
$res = FileDetection($file);
$log->debug("File detected", $res);
switch ($res['type']) {
case NESSUS:
$stack[] = [
'exec' => 'nessus',
'file' => $file,
'ste' => $conf['ste'],
'source' => 'nessus'
];
break;
case SCC_XCCDF:
$stack[] = [
'exec' => 'scc_xccdf',
'file' => $file,
'ste' => $conf['ste'],
'source' => 'scc_xccdf'
];
break;
case STIG_VIEWER_CKL:
$stack[] = [
'exec' => 'stig_viewer',
'file' => $file,
'ste' => $conf['ste'],
'source' => 'stig_viewer'
];
break;
case TECH_ECHECKLIST_EXCEL:
$ignore = false;
if (isset($conf['ignore'])) {
$ignore = true;
}
$stack[] = [
'exec' => 'excel_echecklist',
'file' => $file,
'ste' => $conf['ste'],
'ignore_hidden' => $ignore,
'source' => 'echecklist'
];
break;
case ECHECKLIST_CSV:
$stack[] = [
'exec' => 'csv_echecklist',
'file' => $file,
'ste' => $conf['ste'],
'source' => 'echecklist'
];
break;
case PROC_ECHECKLIST_EXCEL:
$stack[] = [
'exec' => 'proc_echecklist',
'file' => $file,
'ste' => $conf['ste']
];
break;
case HOST_DATA_COLLECTION:
$stack[] = [
'exec' => 'data_collection',
'file' => $file,
'ste' => $conf['ste'],
'target' => $cmd['t'],
'source' => 'data_collection'
];
break;
case NMAP_GREPABLE:
case NMAP_TEXT:
case NMAP_XML:
$stack[] = [
'exec' => 'nmap',
'file' => $file,
'ste' => $conf['ste'],
'source' => 'nmap'
];
break;
case MBSA_TEXT:
case MBSA_XML:
$stack[] = [
'exec' => 'mbsa',
'file' => $file,
'ste' => $conf['ste'],
'source' => 'mbsa'
];
break;
case MSSQL_XML:
$stack[] = [
'exec' => 'mssql',
'file' => $file,
'ste' => $conf['ste'],
'source' => 'mssql'
];
break;
case DIRECTORY:
break;
case strpos("UNSUPPORTED", $file) !== false:
rename($file, realpath(TMP . "/unsupported/" . basename($file)));
break;
default:
error_log("Do not have a parser for " . $file);
}
2018-05-07 10:51:08 -04:00
}
2018-07-26 08:33:50 -04:00
$log->debug("Current script stack", $stack);
2018-05-07 10:51:08 -04:00
foreach ($stack as $key => $s) {
2018-07-26 08:33:50 -04:00
$existing = $dbh->get_Running_Script_Status($s['ste'], $s['file']);
if (isset($existing['status']) && $existing['status'] == 'RUNNING') {
$log->warning("Script to parse " . basename($s['file']) . " is already running");
unset($stack[$key]);
continue;
}
$ignore = '';
if ($s['source'] == 'echecklist' && $s['ignore_hidden']) {
$ignore = " -i=1";
}
$stack[$key]['script'] = realpath(defined('PHP_BIN') ? PHP_BIN : PHP) .
" -c " . realpath(PHP_CONF) . " " .
" -f " . realpath(DOC_ROOT . "/exec/parse_{$s['exec']}.php") . " --" .
" -f=\"{$s['file']}\"" .
$ignore .
($debug ? " --debug" : "");
$log->debug("Adding parser for " . basename($s['file']));
$dbh->add_Running_Script(basename($s['file']), $s['ste'], $s['source'], $conf['location']);
2018-05-07 10:51:08 -04:00
}
$count = 0;
chdir(realpath(DOC_ROOT . "/exec"));
foreach ($stack as $s) {
2018-07-26 08:33:50 -04:00
$threads[] = new Cocur\BackgroundProcess\BackgroundProcess($s['script']);
end($threads)->run();
2018-05-07 10:51:08 -04:00
2018-07-26 08:33:50 -04:00
$log->info("Starting parser script {$s['script']}");
2018-05-07 10:51:08 -04:00
2018-07-26 08:33:50 -04:00
sleep(3);
$count++;
while ($count >= MAX_RESULTS) {
$log->debug("Current MAX_RESULTS met at " . MAX_RESULTS);
sleep(1);
$count = $dbh->get_Running_Script_Count($conf['ste']);
}
2018-05-07 10:51:08 -04:00
}
do {
2018-07-26 08:33:50 -04:00
sleep(1);
2018-05-07 10:51:08 -04:00
}
while ($dbh->get_Running_Script_Count($conf['ste']));
if (!$debug) {
2018-07-26 08:33:50 -04:00
unlink(DOC_ROOT . "/exec/parse_config.ini");
2018-05-07 10:51:08 -04:00
}
/**
* Function to import SCC Oval XML Result files
*
* @param string $file
*/
2018-07-26 08:33:50 -04:00
function import_SCC_OVAL($file)
{
if (preg_match('/.*Results\_iavm\_(2009|2010)|Results\_USGCB/i', $file)) {
return;
}
2018-05-07 10:51:08 -04:00
2018-07-26 08:33:50 -04:00
$target_data = array();
$db = new db();
$match = array();
preg_match('/\_SCC-(\d\.?)+\_(\d{4}\-\d{2}\-\d{2}\_\d{6})\_OVAL/', $file, $match);
$time_stamp = $match[2];
$dt = DateTime::createFromFormat('Y-m-d_His', $time_stamp);
2018-05-07 10:51:08 -04:00
2018-07-26 08:33:50 -04:00
$source = $db->get_Sources('SCC');
$dom = new DOMDocument();
$dom->load($file);
2018-05-07 10:51:08 -04:00
2018-07-26 08:33:50 -04:00
$csv = fopen("scc/" . substr(basename($file), 0, -3) . "csv", 'w');
$ste = $db->get_STE($GLOBALS['opt']['s'])[0];
$scan = new scan(null, $source, $ste, 1, basename($file), $dt->format('Y-m-d H:i:s'));
$scan->set_ID($db->save_Scan($scan));
2018-05-07 10:51:08 -04:00
2018-07-26 08:33:50 -04:00
$x = new DOMXPath($dom);
2018-05-07 10:51:08 -04:00
2018-07-26 08:33:50 -04:00
$sysinfo = $x->query('/oval-res:oval_results/oval-res:results/oval-res:system/oval-sc:oval_system_characteristics/oval-sc:system_info')->item(0);
2018-05-07 10:51:08 -04:00
2018-07-26 08:33:50 -04:00
$target_data['os_name'] = $x->query('oval-sc:os_name', $sysinfo)->item(0)->textContent;
$target_data['os_ver'] = $x->query('oval-sc:os_version', $sysinfo)->item(0)->textContent;
$target_data['host_name'] = $x->query('oval-sc:primary_host_name', $sysinfo)->item(0)->textContent;
$interfaces = $x->query('oval-sc:interfaces/oval-sc:interface', $sysinfo);
$int_count = 0;
2018-05-07 10:51:08 -04:00
2018-07-26 08:33:50 -04:00
foreach ($interfaces as $node) {
$target_data['interface_name' . $int_count] = $x->query('oval-sc:interface_name', $node)->item(0)->textContent;
$target_data['ip' . $int_count] = $x->query('oval-sc:ip_address', $node)->item(0)->textContent;
$target_data['mac' . $int_count] = $x->query('oval-sc:mac_address', $node)->item(0)->textContent;
2018-05-07 10:51:08 -04:00
2018-07-26 08:33:50 -04:00
$int_count++;
}
2018-05-07 10:51:08 -04:00
2018-07-26 08:33:50 -04:00
$defs = $x->query('/oval-res:oval_results/oval-def:oval_definitions/oval-def:definitions/oval-def:definition');
2018-05-07 10:51:08 -04:00
2018-07-26 08:33:50 -04:00
foreach ($defs as $node) {
$id = $node->getAttribute('id');
print "Checking oval id: $id" . PHP_EOL;
//$meta = $x->query('oval-def:metadata', $node)->item(0);
2018-05-07 10:51:08 -04:00
2018-07-26 08:33:50 -04:00
$title = $x->query('oval-def:metadata/oval-def:title', $node)->item(0)->textContent;
$desc = $x->query('oval-def:metadata/oval-def:description', $node)->item(0)->textContent;
$plat = $x->query('oval-def:metadata/oval-def:affected/oval-def:platform', $node)->item(0)->textContent;
2018-05-07 10:51:08 -04:00
2018-07-26 08:33:50 -04:00
$ext = $x->query('oval-def:criteria/oval-def:extend_definition', $node);
2018-05-07 10:51:08 -04:00
2018-07-26 08:33:50 -04:00
if ($ext->length > 0) {
$ext_def = $ext->item(0)->getAttribute('definition_ref');
$ext_def_op = $x->query('oval-def:criteria', $node)->item(0)->getAttribute('operator');
}
else {
$ext_def = '';
$ext_def_op = '';
}
2018-05-07 10:51:08 -04:00
2018-07-26 08:33:50 -04:00
$ref = $x->query('oval-def:metadata/oval-def:reference', $node);
$oval = $db->get_Oval($id);
2018-05-07 10:51:08 -04:00
2018-07-26 08:33:50 -04:00
if ($oval->get_PDI_ID()) {
print "current oval: " . print_r($oval, true);
$oval->clear_References();
}
else {
$oval = new oval(null, $id, $title, $desc, $plat, $ext_def, $ext_def_op);
}
2018-05-07 10:51:08 -04:00
2018-07-26 08:33:50 -04:00
foreach ($ref as $ref_node) {
$source = $ref_node->getAttribute('source') == 'http://cce.mitre.org' ? 'CCE' : $ref_node->getAttribute('source');
$url = $ref_node->hasAttribute('ref_url') ? $ref_node->getAttribute('ref_url') : '';
$ref_id = $ref_node->getAttribute('ref_id');
2018-05-07 10:51:08 -04:00
2018-07-26 08:33:50 -04:00
$oval->add_Reference(new oval_ref($id, $source, $url, $ref_id));
2018-05-07 10:51:08 -04:00
2018-07-26 08:33:50 -04:00
if (is_null($oval->get_PDI_ID()) && $source == 'CCE') {
$cce = $db->get_CCE($ref_id);
2018-05-07 10:51:08 -04:00
2018-07-26 08:33:50 -04:00
if (!is_null($cce)) {
$oval->set_PDI_ID($cce->get_PDI_ID());
}
}
2018-05-07 10:51:08 -04:00
}
2018-07-26 08:33:50 -04:00
if ($db->save_Oval($oval)) {
error_log("Saved oval id: " . $oval->get_Oval_ID());
}
else {
error_log("Error saving oval id: " . $oval->get_Oval_ID());
}
2018-05-07 10:51:08 -04:00
}
}
2018-07-26 08:33:50 -04:00
function usage()
{
print <<<EOO
2018-05-07 10:51:08 -04:00
Purpose: This program was written to look at all files in the /tmp directory, determine what parser is needed, then call that parser with the appropriate flags.
Usage: background_results.php -s={ste_id} [-i=1] [-t=1] [--help]
-s={STE ID} The ID of the ST&E to know what to assign the results to
-i=1 Ignore hidden Excel worksheets (only used on Excel eChecklist files) (defaulted to false)
-t={Target Name} The name of the target to evaluate (only used on host data collection)
--help This screen
EOO;
}