sagacity/exec/parse_cve.php

244 lines
7.3 KiB
PHP
Raw Normal View History

2018-05-07 10:51:08 -04:00
<?php
/**
* File: parse_cve.php
* Author: Ryan Prather
* Purpose: To parse CVE repository retrieved from http://cve.mitre.org/data/downloads/index.html
* Created: Jul 9, 2014
*
* Portions Copyright 2016: Cyber Perspectives, All rights reserved
* Released under the Apache v2.0 License
*
* Portions Copyright (c) 2012-2015, Salient Federal Solutions
* Portions Copyright (c) 2008-2011, Science Applications International Corporation (SAIC)
* Released under Modified BSD License
*
* See license.txt for details
*
* Change Log:
* - Jul 9, 2014 - File created
* - Sep 1, 2016 - Copyright Updated and converted to constants
* - Nov 21, 2016 - Added print out to display the number of new CVEs imported
* - Feb 15, 2017 - Retrieve and store all existing CVEs in memory to use to evaluate if one needs to be imported
* This will now not update existing content.
* - Nov 25, 2017 - Fixed bug #342
* - Dec 27, 2017 - Syntax updates, and added update to load date
*/
$cmd = getopt("f:", ['debug::', 'help::']);
if (!isset($cmd['f']) || isset($cmd['help'])) {
die(usage());
}
elseif (!file_exists($cmd['f'])) {
die("Could not find file {$cmd['f']}\n");
}
include_once 'config.inc';
include_once "database.inc";
include_once 'helper.inc';
chdir(TMP);
set_time_limit(0);
$sys = new db();
$err = new Sagacity_Error($cmd['f']);
$existing_cves = [];
$sys->help->select("cve_db", ['cve_id']);
$cves = $sys->help->execute();
if (!is_null($cves) && is_array($cves) && count($cves)) {
foreach ($cves as $cve) {
$existing_cves["{$cve['cve_id']}"] = 1;
}
}
print "Currently " . count($existing_cves) . " in the DB" . PHP_EOL;
$sys->help->update("settings", ['meta_value' => new DateTime()], [
[
'field' => 'meta_key',
'op' => '=',
'value' => 'cve-load-date'
]
]);
$sys->help->execute();
$doc = new DOMDocument();
$doc->load($cmd['f']);
$items = getValue($doc, "/x:cve/x:item", null, true);
$existing = 0;
$new = 0;
$count = 0;
print "Total of {$items->length} CVEs to parse" . PHP_EOL;
if ($items->length) {
$new_cves = [];
$new_cve_refs = [];
$new_cve_web = [];
$cve_fields = [
'cve_id', 'seq', 'status', 'phase', 'phase_date', 'desc'
];
$ref_fields = [
'cve_seq', 'source', 'url', 'val'
];
$web_fields = [
'cve_id', 'xml'
];
foreach ($items as $node) {
$name = getValue($doc, '@name', $node);
if (!isset($existing_cves["$name"])) {
$count++;
$cve_xml = new DOMDocument();
$cve_node = $cve_xml->importNode($node, true);
$cve_xml->appendChild($cve_node);
$pi = $cve_xml->createProcessingInstruction('xml-stylesheet', 'type="text/xsl" href="cve.xsl"');
$cve_xml->insertBefore($pi, $cve_xml->getElementsByTagName("item")->item(0));
$cve_xml->xmlStandalone = true;
$tmp = str_replace(" xmlns=\"http://cve.mitre.org/cve/downloads\"", "", $cve_xml->saveXML());
$cve_xml->loadXML($tmp);
$refs = [];
$type = getValue($doc, '@type', $node);
$seq = getValue($doc, '@seq', $node);
$desc = getValue($doc, 'x:desc', $node);
$status = getValue($doc, 'x:status', $node);
$phase = getValue($doc, 'x:phase', $node);
$phase_dt_str = getValue($doc, 'x:phase/@date', $node);
$phase_dt = new DateTime($phase_dt_str);
$new_cves[] = [
$name, $seq, $status, $phase, $phase_dt, $desc
];
$tmp_refs = getValue($doc, 'x:refs/x:ref', $node, true);
for ($x = 0; $x < $tmp_refs->length; $x++) {
$refs[$x]['val'] = getValue($doc, '.', $tmp_refs->item($x));
$refs[$x]['src'] = getValue($doc, '@source', $tmp_refs->item($x));
$refs[$x]['url'] = getValue($doc, '@url', $tmp_refs->item($x));
}
foreach ($refs as $key => $ref) {
$new_cve_refs[] = [
$name, $ref['src'], $ref['url'], $ref['val']
];
}
$new_cve_web[] = [$name, $cve_xml->saveXML()];
$err->script_log("new cve $name");
$new++;
print "*";
}
else {
/*
$sql = "REPLACE INTO `reference`.`cve_db` (`type`, `seq`, `status`, `phase`, `phase_date`, `desc`) VALUES (" .
"'$type'," .
"'$seq'," .
"'{$sys->help->real_escape_string($status)}'," .
"'{$sys->help->real_escape_string($phase)}'," .
"'{$phase_dt->format('Y-m-d')}'," .
"'{$sys->help->real_escape_string($desc)}') " .
"WHERE cve_id='{$sys->help->real_escape_string($name)}'";
$sys->help->real_query($sql);
foreach ($refs as $key => $ref) {
$sql = "SELECT `id` FROM `reference`.`cve_references` WHERE `cve_seq`='" .
$sys->help->real_escape_string($name) . "' AND `source`='" .
$sys->help->real_escape_string($ref['src']) . "'";
$res2 = $sys->help->query($sql);
$row2 = $res2->fetch_array(MYSQLI_ASSOC);
if ($row2['id']) {
$sql = "UPDATE `reference`.`cve_references` SET `url`='" . $sys->help->real_escape_string($ref['url']) .
"', val='" . $sys->help->real_escape_string($ref['val']) . "' WHERE `id`=" . $row2['id'];
$sys->help->real_query($sql);
}
else {
$sql = "INSERT INTO `reference`.`cve_references` (`cve_seq`,`source`,`url`,`val`) VALUES ('" .
$sys->help->real_escape_string($name) . "','" . $sys->help->real_escape_string($ref['src']) .
"','" . $sys->help->real_escape_string($ref['url']) . "','" .
$sys->help->real_escape_string($ref['val']) . "')";
$sys->help->real_query($sql);
}
}
$err->script_log("existing cve $name");
*/
$existing++;
print ".";
}
if (($existing + $new) % 100 == 0) {
if (count($new_cves)) {
$sys->help->extended_insert("cve_db", $cve_fields, $new_cves, true);
$sys->help->execute();
}
if (count($new_cve_refs)) {
$sys->help->extended_insert("cve_references", $ref_fields, $new_cve_refs, true);
$sys->help->execute();
}
if (count($new_cve_web)) {
$sys->help->extended_replace("cve_web", $web_fields, $new_cve_web);
$sys->help->execute();
}
$new_cves = [];
$new_cve_refs = [];
$new_cve_web = [];
print "\t" . ($existing + $new) . " completed" . PHP_EOL;
$sys->help->update("settings", ['meta_value' => number_format((($existing + $new) / $items->length) * 100, 2)], [
[
'field' => 'meta_key',
'op' => '=',
'value' => 'cve-progress'
]
]);
$sys->help->execute();
}
}
if (is_array($new_cves) && count($new_cves)) {
$sys->help->extended_insert("cve_db", $cve_fields, $new_cves, true);
$sys->help->execute();
}
if (is_array($new_cve_refs) && count($new_cve_refs)) {
$sys->help->extended_insert("cve_references", $ref_fields, $new_cve_refs, true);
$sys->help->execute();
}
if (is_array($new_cve_web) && count($new_cve_web)) {
$sys->help->extended_replace("cve_web", $web_fields, $new_cve_web);
$sys->help->execute();
}
}
function usage() {
print <<<EOO
Purpose: Parse the CVE file (allitems.xml) retrieved from http://cve.mitre.org/data/downloads/allitems.xml
Usage: php parse_cve.php -f={CVE filename} [--debug] [--help]
-f={CVE filename} The file to be parsed (allitems.xml). Can be absolute or relative path.
--debug Debugging output
--help This screen
EOO;
}