253 lines
7.5 KiB
PHP
253 lines
7.5 KiB
PHP
<?php
|
|
|
|
/**
|
|
* File: parse_cve.php
|
|
* Author: Ryan Prather
|
|
* Purpose: To parse CVE repository retrieved from http://cve.mitre.org/data/downloads/index.html
|
|
* Created: Jul 9, 2014
|
|
*
|
|
* Portions Copyright 2016: Cyber Perspectives, All rights reserved
|
|
* Released under the Apache v2.0 License
|
|
*
|
|
* Portions Copyright (c) 2012-2015, Salient Federal Solutions
|
|
* Portions Copyright (c) 2008-2011, Science Applications International Corporation (SAIC)
|
|
* Released under Modified BSD License
|
|
*
|
|
* See license.txt for details
|
|
*
|
|
* Change Log:
|
|
* - Jul 9, 2014 - File created
|
|
* - Sep 1, 2016 - Copyright Updated and converted to constants
|
|
* - Nov 21, 2016 - Added print out to display the number of new CVEs imported
|
|
* - Feb 15, 2017 - Retrieve and store all existing CVEs in memory to use to evaluate if one needs to be imported
|
|
* This will now not update existing content.
|
|
* - Nov 25, 2017 - Fixed bug #342
|
|
* - Dec 27, 2017 - Syntax updates, and added update to load date
|
|
*/
|
|
$cmd = getopt("f:", ['debug::', 'help::']);
|
|
|
|
if (!isset($cmd['f']) || isset($cmd['help'])) {
|
|
die(usage());
|
|
}
|
|
elseif (!file_exists($cmd['f'])) {
|
|
die("Could not find file {$cmd['f']}\n");
|
|
}
|
|
|
|
include_once 'config.inc';
|
|
include_once "database.inc";
|
|
include_once 'helper.inc';
|
|
|
|
chdir(TMP);
|
|
|
|
set_time_limit(0);
|
|
|
|
$sys = new db();
|
|
$err = new Sagacity_Error($cmd['f']);
|
|
|
|
$existing_cves = [];
|
|
|
|
$sys->help->select("cve_db", ['cve_id']);
|
|
$cves = $sys->help->execute();
|
|
if (!is_null($cves) && is_array($cves) && count($cves)) {
|
|
foreach ($cves as $cve) {
|
|
$existing_cves["{$cve['cve_id']}"] = 1;
|
|
}
|
|
}
|
|
|
|
print "Currently " . count($existing_cves) . " in the DB" . PHP_EOL;
|
|
|
|
$sys->help->update("settings", ['meta_value' => new DateTime()], [
|
|
[
|
|
'field' => 'meta_key',
|
|
'op' => '=',
|
|
'value' => 'cve-load-date'
|
|
]
|
|
]);
|
|
$sys->help->execute();
|
|
|
|
$doc = new DOMDocument();
|
|
$doc->load($cmd['f']);
|
|
|
|
$items = getValue($doc, "/x:cve/x:item", null, true);
|
|
$existing = 0;
|
|
$new = 0;
|
|
$count = 0;
|
|
|
|
print "Total of {$items->length} CVEs to parse" . PHP_EOL;
|
|
|
|
if ($items->length) {
|
|
$new_cves = [];
|
|
$new_cve_refs = [];
|
|
$new_cve_web = [];
|
|
|
|
$cve_fields = [
|
|
'cve_id', 'seq', 'status', 'phase', 'phase_date', 'desc'
|
|
];
|
|
$ref_fields = [
|
|
'cve_seq', 'source', 'url', 'val'
|
|
];
|
|
$web_fields = [
|
|
'cve_id', 'xml'
|
|
];
|
|
|
|
foreach ($items as $node) {
|
|
$name = getValue($doc, '@name', $node);
|
|
|
|
if (!isset($existing_cves["$name"])) {
|
|
$count++;
|
|
$cve_xml = new DOMDocument();
|
|
$cve_node = $cve_xml->importNode($node, true);
|
|
$cve_xml->appendChild($cve_node);
|
|
|
|
$pi = $cve_xml->createProcessingInstruction('xml-stylesheet', 'type="text/xsl" href="cve.xsl"');
|
|
$cve_xml->insertBefore($pi, $cve_xml->getElementsByTagName("item")->item(0));
|
|
$cve_xml->xmlStandalone = true;
|
|
|
|
$tmp = str_replace(" xmlns=\"http://cve.mitre.org/cve/downloads\"", "", $cve_xml->saveXML());
|
|
$cve_xml->loadXML($tmp);
|
|
|
|
$refs = [];
|
|
$type = getValue($doc, '@type', $node);
|
|
$seq = getValue($doc, '@seq', $node);
|
|
$desc = getValue($doc, 'x:desc', $node);
|
|
$status = getValue($doc, 'x:status', $node);
|
|
$phase = getValue($doc, 'x:phase', $node);
|
|
$phase_dt_str = getValue($doc, 'x:phase/@date', $node);
|
|
$phase_dt = new DateTime($phase_dt_str);
|
|
|
|
$new_cves[] = [
|
|
$name, $seq, $status, $phase, $phase_dt, $desc
|
|
];
|
|
|
|
$tmp_refs = getValue($doc, 'x:refs/x:ref', $node, true);
|
|
|
|
for ($x = 0; $x < $tmp_refs->length; $x++) {
|
|
$refs[$x]['val'] = getValue($doc, '.', $tmp_refs->item($x));
|
|
$refs[$x]['src'] = getValue($doc, '@source', $tmp_refs->item($x));
|
|
$refs[$x]['url'] = getValue($doc, '@url', $tmp_refs->item($x));
|
|
}
|
|
|
|
foreach ($refs as $key => $ref) {
|
|
$new_cve_refs[] = [
|
|
$name, $ref['src'], $ref['url'], $ref['val']
|
|
];
|
|
}
|
|
|
|
$new_cve_web[] = [$name, $cve_xml->saveXML()];
|
|
|
|
$err->script_log("new cve $name");
|
|
$new++;
|
|
print "*";
|
|
}
|
|
else {
|
|
/*
|
|
$sql = "REPLACE INTO `reference`.`cve_db` (`type`, `seq`, `status`, `phase`, `phase_date`, `desc`) VALUES (" .
|
|
"'$type'," .
|
|
"'$seq'," .
|
|
"'{$sys->help->real_escape_string($status)}'," .
|
|
"'{$sys->help->real_escape_string($phase)}'," .
|
|
"'{$phase_dt->format('Y-m-d')}'," .
|
|
"'{$sys->help->real_escape_string($desc)}') " .
|
|
"WHERE cve_id='{$sys->help->real_escape_string($name)}'";
|
|
$sys->help->real_query($sql);
|
|
|
|
foreach ($refs as $key => $ref) {
|
|
$sql = "SELECT `id` FROM `reference`.`cve_references` WHERE `cve_seq`='" .
|
|
$sys->help->real_escape_string($name) . "' AND `source`='" .
|
|
$sys->help->real_escape_string($ref['src']) . "'";
|
|
$res2 = $sys->help->query($sql);
|
|
$row2 = $res2->fetch_array(MYSQLI_ASSOC);
|
|
|
|
if ($row2['id']) {
|
|
$sql = "UPDATE `reference`.`cve_references` SET `url`='" . $sys->help->real_escape_string($ref['url']) .
|
|
"', val='" . $sys->help->real_escape_string($ref['val']) . "' WHERE `id`=" . $row2['id'];
|
|
$sys->help->real_query($sql);
|
|
}
|
|
else {
|
|
$sql = "INSERT INTO `reference`.`cve_references` (`cve_seq`,`source`,`url`,`val`) VALUES ('" .
|
|
$sys->help->real_escape_string($name) . "','" . $sys->help->real_escape_string($ref['src']) .
|
|
"','" . $sys->help->real_escape_string($ref['url']) . "','" .
|
|
$sys->help->real_escape_string($ref['val']) . "')";
|
|
$sys->help->real_query($sql);
|
|
}
|
|
}
|
|
|
|
$err->script_log("existing cve $name");
|
|
*/
|
|
$existing++;
|
|
print ".";
|
|
}
|
|
|
|
if (($existing + $new) % 100 == 0) {
|
|
if (count($new_cves)) {
|
|
$sys->help->extended_insert("cve_db", $cve_fields, $new_cves, true);
|
|
$sys->help->execute();
|
|
}
|
|
|
|
if (count($new_cve_refs)) {
|
|
$sys->help->extended_insert("cve_references", $ref_fields, $new_cve_refs, true);
|
|
$sys->help->execute();
|
|
}
|
|
|
|
if (count($new_cve_web)) {
|
|
$sys->help->extended_replace("cve_web", $web_fields, $new_cve_web);
|
|
$sys->help->execute();
|
|
}
|
|
|
|
$new_cves = [];
|
|
$new_cve_refs = [];
|
|
$new_cve_web = [];
|
|
|
|
print "\t" . ($existing + $new) . " completed" . PHP_EOL;
|
|
|
|
$sys->help->update("settings", ['meta_value' => number_format((($existing + $new) / $items->length) * 100, 2)], [
|
|
[
|
|
'field' => 'meta_key',
|
|
'op' => '=',
|
|
'value' => 'cve-progress'
|
|
]
|
|
]);
|
|
$sys->help->execute();
|
|
}
|
|
}
|
|
|
|
if (is_array($new_cves) && count($new_cves)) {
|
|
$sys->help->extended_insert("cve_db", $cve_fields, $new_cves, true);
|
|
$sys->help->execute();
|
|
}
|
|
|
|
if (is_array($new_cve_refs) && count($new_cve_refs)) {
|
|
$sys->help->extended_insert("cve_references", $ref_fields, $new_cve_refs, true);
|
|
$sys->help->execute();
|
|
}
|
|
|
|
if (is_array($new_cve_web) && count($new_cve_web)) {
|
|
$sys->help->extended_replace("cve_web", $web_fields, $new_cve_web);
|
|
$sys->help->execute();
|
|
}
|
|
|
|
$sys->help->update("settings", ['meta_value' => 100], [
|
|
[
|
|
'field' => 'meta_key',
|
|
'op' => IN,
|
|
'value' => ['cve-dl-progress', 'cve-progress']
|
|
]
|
|
]);
|
|
$sys->help->execute();
|
|
}
|
|
|
|
function usage() {
|
|
print <<<EOO
|
|
Purpose: Parse the CVE file (allitems.xml) retrieved from http://cve.mitre.org/data/downloads/allitems.xml
|
|
|
|
Usage: php parse_cve.php -f={CVE filename} [--debug] [--help]
|
|
|
|
-f={CVE filename} The file to be parsed (allitems.xml). Can be absolute or relative path.
|
|
|
|
--debug Debugging output
|
|
--help This screen
|
|
|
|
|
|
EOO;
|
|
}
|