sagacity/exec/background_results.php

<?php
/**
 * File: background_results.php
 * Author: Ryan Prather
 * Purpose: Background script file that will call appropriate function for files found
 * Created: Feb 26, 2014
 *
 * Portions Copyright 2016-2018: Cyber Perspectives, LLC, All rights reserved
 * Released under the Apache v2.0 License
 *
 * Portions Copyright (c) 2012-2015, Salient Federal Solutions
 * Portions Copyright (c) 2008-2011, Science Applications International Corporation (SAIC)
 * Released under Modified BSD License
 *
 * See license.txt for details
 *
 * Change Log:
 *  - Feb 26, 2014 - File created
 *  - May 05, 2014 - Converted parsing functions to classes for threading
 *  - Sep 1, 2016 - Copyright updated, added CWD parameter option,
 * 					Converted to constants, made script execution platform independent
 *  - Oct 24, 2016 - Added debug output and cleaned up script string generation
 *  - Nov 7, 2016 - If it ain't broke, don't fix it! Had to revert to a previous version because intended improvements broke it
 *  - Dec 7, 2016 - Fixed bug where Windows threading was not being started,
 *                  Changed PHP constant to PHP_BIN, and make sure that script continues running until last result file is done.
 *  - Jan 30, 2017 - Converted script to use parse_config.ini file instead of command line parameters and set script to remove config file when all files are completely parsed
 *  - Feb 15, 2017 - Converted file_types constants to defined constants and removed unnecessary parameters from parse_* scripts string creation
 *  - Feb 21, 2017 - Fixed path issues with scripts not running
 *  - Oct 23, 2017 - Conditionally delete parse_config.ini only if not in DEBUG log level
 *  - Oct 27, 2017 - Fix to remove desktop.ini files if found
 *  - May 24, 2018 - Moved a couple code blocks because of being out of order
 */
error_reporting(E_ALL);

$cmd = getopt("t::", ["help::"]);

$conf = parse_ini_file("parse_config.ini", false);

set_time_limit(0);

include_once 'config.inc';
include_once 'database.inc';
include_once 'helper.inc';
include_once 'vendor/autoload.php';

use Monolog\Logger;
use Monolog\Handler\StreamHandler;

$log_level = Logger::ERROR;
switch (LOG_LEVEL) {
    case E_WARNING:
        $log_level = Logger::WARNING;
        break;
    case E_NOTICE:
        $log_level = Logger::NOTICE;
        break;
    case E_DEBUG:
        $log_level = Logger::DEBUG;
}

$log = new Logger('result_import');
$log->pushHandler(new StreamHandler(LOG_PATH . "/result_import.log", $log_level));

$debug = (LOG_LEVEL == E_DEBUG ? true : false);

if (isset($cmd['help']) || !is_numeric($conf['ste']) || !isset($conf['doc_root'])) {
    die(usage());
}

chdir(TMP);
check_path(TMP . "/echecklist");
check_path(TMP . "/nessus");
check_path(TMP . "/nmap");
check_path(TMP . "/scc");
check_path(TMP . "/stig_viewer");
check_path(TMP . "/terminated");
check_path(TMP . "/unsupported");

$dbh = new db();

$files   = glob("*.*");
$stack   = [];
$threads = [];

foreach ($files as $file) {
    $res = FileDetection($file);
    $log->debug("File detected", $res);

    switch ($res['type']) {
        case NESSUS:
            $stack[] = [
                'exec'   => 'nessus',
                'file'   => $file,
                'ste'    => $conf['ste'],
                'source' => 'nessus'
            ];
            break;
        case SCC_XCCDF:
            $stack[] = [
                'exec'   => 'scc_xccdf',
                'file'   => $file,
                'ste'    => $conf['ste'],
                'source' => 'scc_xccdf'
            ];
            break;
        case STIG_VIEWER_CKL:
            $stack[] = [
                'exec'   => 'stig_viewer',
                'file'   => $file,
                'ste'    => $conf['ste'],
                'source' => 'stig_viewer'
            ];
            break;
        case TECH_ECHECKLIST_EXCEL:
            $ignore  = false;
            if (isset($conf['ignore'])) {
                $ignore = true;
            }
            $stack[] = [
                'exec'          => 'excel_echecklist',
                'file'          => $file,
                'ste'           => $conf['ste'],
                'ignore_hidden' => $ignore,
                'source'        => 'echecklist'
            ];
            break;
        case ECHECKLIST_CSV:
            $stack[] = [
                'exec'   => 'csv_echecklist',
                'file'   => $file,
                'ste'    => $conf['ste'],
                'source' => 'echecklist'
            ];
            break;
        case PROC_ECHECKLIST_EXCEL:
            $stack[] = [
                'exec' => 'proc_echecklist',
                'file' => $file,
                'ste'  => $conf['ste']
            ];
            break;
        case HOST_DATA_COLLECTION:
            $stack[] = [
                'exec'   => 'data_collection',
                'file'   => $file,
                'ste'    => $conf['ste'],
                'target' => $cmd['t'],
                'source' => 'data_collection'
            ];
            break;
        case NMAP_GREPABLE:
        case NMAP_TEXT:
        case NMAP_XML:
            $stack[] = [
                'exec'   => 'nmap',
                'file'   => $file,
                'ste'    => $conf['ste'],
                'source' => 'nmap'
            ];
            break;
        case MBSA_TEXT:
        case MBSA_XML:
            $stack[] = [
                'exec'   => 'mbsa',
                'file'   => $file,
                'ste'    => $conf['ste'],
                'source' => 'mbsa'
            ];
            break;
        case MSSQL_XML:
            $stack[] = [
                'exec'   => 'mssql',
                'file'   => $file,
                'ste'    => $conf['ste'],
                'source' => 'mssql'
            ];
            break;
        case DIRECTORY:
            break;
        case strpos("UNSUPPORTED", $file) !== false:
            rename($file, realpath(TMP . "/unsupported/" . basename($file)));
            break;
        default:
            error_log("Do not have a parser for " . $file);
    }
}

$log->debug("Current script stack", $stack);

foreach ($stack as $key => $s) {
    $existing = $dbh->get_Running_Script_Status($s['ste'], $s['file']);
    if (isset($existing['status']) && $existing['status'] == 'RUNNING') {
        $log->warning("Script to parse " . basename($s['file']) . " is already running");
        unset($stack[$key]);
        continue;
    }

    $ignore = '';
    if ($s['source'] == 'echecklist' && $s['ignore_hidden']) {
        $ignore = " -i=1";
    }

    $stack[$key]['script'] = realpath(defined('PHP_BIN') ? PHP_BIN : PHP) .
        " -c " . realpath(PHP_CONF) . " " .
        " -f " . realpath(DOC_ROOT . "/exec/parse_{$s['exec']}.php") . " --" .
        " -f=\"{$s['file']}\"" .
        $ignore .
        ($debug ? " --debug" : "");

    $log->debug("Adding parser for " . basename($s['file']));

    $dbh->add_Running_Script(basename($s['file']), $s['ste'], $s['source'], $conf['location']);
}

$count = 0;

chdir(realpath(DOC_ROOT . "/exec"));

foreach ($stack as $s) {
    $threads[] = new Cocur\BackgroundProcess\BackgroundProcess($s['script']);
    end($threads)->run();

    $log->info("Starting parser script {$s['script']}");

    sleep(3);
    $count++;

    while ($count >= MAX_RESULTS) {
        $log->debug("Current MAX_RESULTS met at " . MAX_RESULTS);
        sleep(1);
        $count = $dbh->get_Running_Script_Count($conf['ste']);
    }
}

do {
    sleep(1);
}
while ($dbh->get_Running_Script_Count($conf['ste']));

if (!$debug) {
    unlink(DOC_ROOT . "/exec/parse_config.ini");
}

/**
 * Function to import SCC Oval XML Result files
 *
 * @param string $file
 */
function import_SCC_OVAL($file)
{
    if (preg_match('/.*Results\_iavm\_(2009|2010)|Results\_USGCB/i', $file)) {
        return;
    }

    $target_data = array();
    $db          = new db();
    $match       = array();
    preg_match('/\_SCC-(\d\.?)+\_(\d{4}\-\d{2}\-\d{2}\_\d{6})\_OVAL/', $file, $match);
    $time_stamp  = $match[2];
    $dt          = DateTime::createFromFormat('Y-m-d_His', $time_stamp);

    $source = $db->get_Sources('SCC');
    $dom    = new DOMDocument();
    $dom->load($file);

    $csv  = fopen("scc/" . substr(basename($file), 0, -3) . "csv", 'w');
    $ste  = $db->get_STE($GLOBALS['opt']['s'])[0];
    $scan = new scan(null, $source, $ste, 1, basename($file), $dt->format('Y-m-d H:i:s'));
    $scan->set_ID($db->save_Scan($scan));

    $x = new DOMXPath($dom);

    $sysinfo = $x->query('/oval-res:oval_results/oval-res:results/oval-res:system/oval-sc:oval_system_characteristics/oval-sc:system_info')->item(0);

    $target_data['os_name']   = $x->query('oval-sc:os_name', $sysinfo)->item(0)->textContent;
    $target_data['os_ver']    = $x->query('oval-sc:os_version', $sysinfo)->item(0)->textContent;
    $target_data['host_name'] = $x->query('oval-sc:primary_host_name', $sysinfo)->item(0)->textContent;
    $interfaces               = $x->query('oval-sc:interfaces/oval-sc:interface', $sysinfo);
    $int_count                = 0;

    foreach ($interfaces as $node) {
        $target_data['interface_name' . $int_count] = $x->query('oval-sc:interface_name', $node)->item(0)->textContent;
        $target_data['ip' . $int_count]             = $x->query('oval-sc:ip_address', $node)->item(0)->textContent;
        $target_data['mac' . $int_count]            = $x->query('oval-sc:mac_address', $node)->item(0)->textContent;

        $int_count++;
    }

    $defs = $x->query('/oval-res:oval_results/oval-def:oval_definitions/oval-def:definitions/oval-def:definition');

    foreach ($defs as $node) {
        $id = $node->getAttribute('id');
        print "Checking oval id: $id" . PHP_EOL;
        //$meta = $x->query('oval-def:metadata', $node)->item(0);

        $title = $x->query('oval-def:metadata/oval-def:title', $node)->item(0)->textContent;
        $desc  = $x->query('oval-def:metadata/oval-def:description', $node)->item(0)->textContent;
        $plat  = $x->query('oval-def:metadata/oval-def:affected/oval-def:platform', $node)->item(0)->textContent;

        $ext = $x->query('oval-def:criteria/oval-def:extend_definition', $node);

        if ($ext->length > 0) {
            $ext_def    = $ext->item(0)->getAttribute('definition_ref');
            $ext_def_op = $x->query('oval-def:criteria', $node)->item(0)->getAttribute('operator');
        }
        else {
            $ext_def    = '';
            $ext_def_op = '';
        }

        $ref  = $x->query('oval-def:metadata/oval-def:reference', $node);
        $oval = $db->get_Oval($id);

        if ($oval->get_PDI_ID()) {
            print "current oval: " . print_r($oval, true);
            $oval->clear_References();
        }
        else {
            $oval = new oval(null, $id, $title, $desc, $plat, $ext_def, $ext_def_op);
        }

        foreach ($ref as $ref_node) {
            $source = $ref_node->getAttribute('source') == 'http://cce.mitre.org' ? 'CCE' : $ref_node->getAttribute('source');
            $url    = $ref_node->hasAttribute('ref_url') ? $ref_node->getAttribute('ref_url') : '';
            $ref_id = $ref_node->getAttribute('ref_id');

            $oval->add_Reference(new oval_ref($id, $source, $url, $ref_id));

            if (is_null($oval->get_PDI_ID()) && $source == 'CCE') {
                $cce = $db->get_CCE($ref_id);

                if (!is_null($cce)) {
                    $oval->set_PDI_ID($cce->get_PDI_ID());
                }
            }
        }

        if ($db->save_Oval($oval)) {
            error_log("Saved oval id: " . $oval->get_Oval_ID());
        }
        else {
            error_log("Error saving oval id: " . $oval->get_Oval_ID());
        }
    }
}

function usage()
{
    print <<<EOO
Purpose: This program was written to look at all files in the /tmp directory, determine what parser is needed, then call that parser with the appropriate flags.

Usage: background_results.php -s={ste_id} [-i=1] [-t=1] [--help]

 -s={STE ID}        The ID of the ST&E to know what to assign the results to
 -i=1               Ignore hidden Excel worksheets (only used on Excel eChecklist files) (defaulted to false)
 -t={Target Name}   The name of the target to evaluate (only used on host data collection)

 --help             This screen

EOO;
}