sagacity/exec/parse_nvd_json_cve.php

217 lines
5.8 KiB
PHP
Raw Normal View History

2018-05-07 10:51:08 -04:00
<?php
/**
* File: parse_nvd_json_cve
* Author: Ryan Prather <ryan.prather@cyberperspectives.com>
* Purpose:
2018-07-26 08:33:50 -04:00
* Created: Apr 29, 2018
2018-05-07 10:51:08 -04:00
*
2018-07-26 08:33:50 -04:00
* Copyright 2018: Cyber Perspective, LLC, All rights reserved
2018-05-07 10:51:08 -04:00
* Released under the Apache v2.0 License
*
* See license.txt for details
*
* Change Log:
2018-07-26 08:33:50 -04:00
* - Apr 29, 2018 - File created
* - May 10, 2018 - Formatting and fixed performance issue on Windows (bug #403)
* - Jun 5, 2018 - Fix for bug #425
2018-05-07 10:51:08 -04:00
*/
include_once 'config.inc';
include_once 'helper.inc';
include_once 'database.inc';
include_once 'vendor/autoload.php';
use Monolog\Logger;
use Monolog\Handler\StreamHandler;
use Monolog\Formatter\LineFormatter;
ini_set('memory_limit', '2G');
$cmd = getopt("f:");
if (!isset($cmd['f']) || isset($cmd['h'])) {
2018-07-26 08:33:50 -04:00
die(usage());
2018-05-07 10:51:08 -04:00
}
$log_level = Logger::ERROR;
switch (LOG_LEVEL) {
case E_WARNING:
$log_level = Logger::WARNING;
break;
case E_NOTICE:
$log_level = Logger::NOTICE;
break;
case E_DEBUG:
$log_level = Logger::DEBUG;
break;
}
$log = new Logger("nvd_cve");
$log->pushHandler(new StreamHandler(LOG_PATH . "/nvd_cve.log", $log_level));
2018-07-26 08:33:50 -04:00
$db = new db();
$json = json_decode(file_get_contents($cmd['f']));
2018-05-07 10:51:08 -04:00
$existing_cves = [];
$db->help->select("cve_db", ['cve_id']);
$cves = $db->help->execute();
if (is_array($cves) && count($cves)) {
2018-07-26 08:33:50 -04:00
foreach ($cves as $cve) {
$existing_cves["{$cve['cve_id']}"] = 1;
}
2018-05-07 10:51:08 -04:00
}
print "Currently " . count($existing_cves) . " in DB" . PHP_EOL . "Parsing: " . count($json->CVE_Items) . " items" . PHP_EOL;
2018-07-26 08:33:50 -04:00
$db_cpes = [];
$new_cves = [];
2018-05-07 10:51:08 -04:00
$new_cve_refs = [];
2018-07-26 08:33:50 -04:00
$new_cve_web = [];
$sw_rows = [];
$new = 0;
$existing = 0;
$db->help->select("software", ['id', 'cpe']);
$rows = $db->help->execute();
foreach ($rows as $row) {
$db_cpes["{$row['cpe']}"] = $row['id'];
}
2018-05-07 10:51:08 -04:00
$cve_fields = [
2018-07-26 08:33:50 -04:00
'cve_id', 'seq', 'status', 'phase', 'phase_date', 'desc'
2018-05-07 10:51:08 -04:00
];
$ref_fields = [
2018-07-26 08:33:50 -04:00
'cve_seq', 'source', 'url', 'val'
2018-05-07 10:51:08 -04:00
];
$web_fields = [
2018-07-26 08:33:50 -04:00
'cve_id', 'xml'
2018-05-07 10:51:08 -04:00
];
foreach ($json->CVE_Items as $cve) {
2018-07-26 08:33:50 -04:00
if (!isset($existing_cves["{$cve->cve->CVE_data_meta->ID}"])) {
$log->debug("Adding {$cve->cve->CVE_data_meta->ID}");
$new++;
$desc = [];
$status = null;
$phase = null;
$cpes = [];
$name = $cve->cve->CVE_data_meta->ID;
$type = $cve->cve->data_type;
$seq = $cve->cve->CVE_data_meta->ID;
$pd = new DateTime($cve->publishedDate);
$lmd = new DateTime($cve->lastModifiedDate);
if (is_array($cve->cve->description->description_data) && count($cve->cve->description->description_data)) {
foreach ($cve->cve->description->description_data as $d) {
$desc[] = $d->value;
}
}
2018-05-07 10:51:08 -04:00
2018-07-26 08:33:50 -04:00
$new_cves[] = [
$name, $seq, $status, $phase, $pd, implode(PHP_EOL, $desc)
2018-05-07 10:51:08 -04:00
];
2018-07-26 08:33:50 -04:00
if (is_array($cve->cve->references->reference_data) && count($cve->cve->references->reference_data)) {
foreach ($cve->cve->references->reference_data as $ref) {
$log->debug("Adding reference {$ref->url}");
$new_cve_refs[] = [
$name, null, $ref->url, null
];
2018-05-07 10:51:08 -04:00
}
2018-07-26 08:33:50 -04:00
}
if (is_array($cve->configurations->nodes) && count($cve->configurations->nodes)) {
foreach ($cve->configurations->nodes as $n) {
if (isset($n->cpe) && is_array($n->cpe) && count($n->cpe)) {
foreach ($n->cpe as $cpe) {
if (isset($cpe->cpe22Uri)) {
$cpes[] = $cpe->cpe22Uri;
}
elseif (isset($cpe->cpeMatchString)) {
$cpes[] = $cpe->cpeMatchString;
}
}
}
2018-05-07 10:51:08 -04:00
}
}
2018-07-26 08:33:50 -04:00
if (count($cpes)) {
foreach ($cpes as $cpe) {
if (isset($db_cpes["{$cpe}"])) {
$sw_rows[] = [$name, $db_cpes["{$cpe}"]];
}
}
2018-05-07 10:51:08 -04:00
}
2018-07-26 08:33:50 -04:00
print "*";
2018-05-07 10:51:08 -04:00
}
2018-07-26 08:33:50 -04:00
else {
$existing++;
print ".";
2018-05-07 10:51:08 -04:00
}
2018-07-26 08:33:50 -04:00
if (($new + $existing) % 100 == 0) {
if (count($new_cves)) {
$db->help->extended_insert("cve_db", $cve_fields, $new_cves, true);
$db->help->execute();
}
2018-05-07 10:51:08 -04:00
2018-07-26 08:33:50 -04:00
if (count($new_cve_refs)) {
$db->help->extended_insert("cve_references", $ref_fields, $new_cve_refs, true);
$db->help->execute();
}
2018-05-07 10:51:08 -04:00
2018-07-26 08:33:50 -04:00
if (count($sw_rows)) {
$db->help->extended_insert("cve_sw_lookup", ['cve_id', 'sw_id'], $sw_rows, true);
$db->help->execute();
}
2018-05-07 10:51:08 -04:00
2018-07-26 08:33:50 -04:00
$new_cves = [];
$new_cve_refs = [];
$new_cve_web = [];
$sw_rows = [];
print "\t" . ($existing + $new) . " completed" . PHP_EOL;
$db->help->update("settings", ['meta_value' => number_format((($existing + $new) / count($json->CVE_Items)) * 100, 2)], [
[
'field' => 'meta_key',
'value' => 'nvd-cve-progress'
]
]);
$db->help->execute();
}
2018-05-07 10:51:08 -04:00
}
if (count($new_cves)) {
2018-07-26 08:33:50 -04:00
$db->help->extended_insert("cve_db", $cve_fields, $new_cves, true);
$db->help->execute();
2018-05-07 10:51:08 -04:00
}
if (count($new_cve_refs)) {
2018-07-26 08:33:50 -04:00
$db->help->extended_insert("cve_references", $ref_fields, $new_cve_refs, true);
$db->help->execute();
2018-05-07 10:51:08 -04:00
}
if (count($sw_rows)) {
2018-07-26 08:33:50 -04:00
$db->help->extended_insert("cve_sw_lookup", ['cve_id', 'sw_id'], $sw_rows, true);
$db->help->execute();
2018-05-07 10:51:08 -04:00
}
unlink($cmd['f']);
print PHP_EOL;
2018-07-26 08:33:50 -04:00
function usage()
{
print <<<EOF
2018-05-07 10:51:08 -04:00
Purpose: To import the National Vulnerability Database (NVD) CVE JSON files
Usage: php parse_nvd_json_cve.php -f={JSON file} [-h]
-f={JSON file} The CVE file to import
-h This screen
EOF;
}