sagacity/exec/parse_cve.php

<?php

/**
 * File: parse_cve.php
 * Author: Ryan Prather
 * Purpose: To parse CVE repository retrieved from http://cve.mitre.org/data/downloads/index.html
 * Created: Jul 9, 2014
 *
 * Portions Copyright 2016: Cyber Perspectives, All rights reserved
 * Released under the Apache v2.0 License
 *
 * Portions Copyright (c) 2012-2015, Salient Federal Solutions
 * Portions Copyright (c) 2008-2011, Science Applications International Corporation (SAIC)
 * Released under Modified BSD License
 *
 * See license.txt for details
 *
 * Change Log:
 *  - Jul 9, 2014 - File created
 *  - Sep 1, 2016 - Copyright Updated and converted to constants
 *  - Nov 21, 2016 - Added print out to display the number of new CVEs imported
 *  - Feb 15, 2017 - Retrieve and store all existing CVEs in memory to use to evaluate if one needs to be imported
 *                   This will now not update existing content.
 *  - Nov 25, 2017 - Fixed bug #342
 *  - Dec 27, 2017 - Syntax updates, and added update to load date
 */
$cmd = getopt("f:", ['debug::', 'help::']);

if (!isset($cmd['f']) || isset($cmd['help'])) {
  die(usage());
}
elseif (!file_exists($cmd['f'])) {
  die("Could not find file {$cmd['f']}\n");
}

include_once 'config.inc';
include_once "database.inc";
include_once 'helper.inc';

chdir(TMP);

set_time_limit(0);

$sys = new db();
$err = new Sagacity_Error($cmd['f']);

$existing_cves = [];

$sys->help->select("cve_db", ['cve_id']);
$cves = $sys->help->execute();
if (!is_null($cves) && is_array($cves) && count($cves)) {
  foreach ($cves as $cve) {
    $existing_cves["{$cve['cve_id']}"] = 1;
  }
}

print "Currently " . count($existing_cves) . " in the DB" . PHP_EOL;

$sys->help->update("settings", ['meta_value' => new DateTime()], [
  [
    'field' => 'meta_key',
    'op'    => '=',
    'value' => 'cve-load-date'
  ]
]);
$sys->help->execute();

$doc = new DOMDocument();
$doc->load($cmd['f']);

$items = getValue($doc, "/x:cve/x:item", null, true);
$existing = 0;
$new = 0;
$count = 0;

print "Total of {$items->length} CVEs to parse" . PHP_EOL;

if ($items->length) {
  $new_cves = [];
  $new_cve_refs = [];
  $new_cve_web = [];

  $cve_fields = [
    'cve_id', 'seq', 'status', 'phase', 'phase_date', 'desc'
  ];
  $ref_fields = [
    'cve_seq', 'source', 'url', 'val'
  ];
  $web_fields = [
    'cve_id', 'xml'
  ];

  foreach ($items as $node) {
    $name = getValue($doc, '@name', $node);

    if (!isset($existing_cves["$name"])) {
      $count++;
      $cve_xml = new DOMDocument();
      $cve_node = $cve_xml->importNode($node, true);
      $cve_xml->appendChild($cve_node);

      $pi = $cve_xml->createProcessingInstruction('xml-stylesheet', 'type="text/xsl" href="cve.xsl"');
      $cve_xml->insertBefore($pi, $cve_xml->getElementsByTagName("item")->item(0));
      $cve_xml->xmlStandalone = true;

      $tmp = str_replace(" xmlns=\"http://cve.mitre.org/cve/downloads\"", "", $cve_xml->saveXML());
      $cve_xml->loadXML($tmp);

      $refs = [];
      $type = getValue($doc, '@type', $node);
      $seq = getValue($doc, '@seq', $node);
      $desc = getValue($doc, 'x:desc', $node);
      $status = getValue($doc, 'x:status', $node);
      $phase = getValue($doc, 'x:phase', $node);
      $phase_dt_str = getValue($doc, 'x:phase/@date', $node);
      $phase_dt = new DateTime($phase_dt_str);

      $new_cves[] = [
        $name, $seq, $status, $phase, $phase_dt, $desc
      ];

      $tmp_refs = getValue($doc, 'x:refs/x:ref', $node, true);

      for ($x = 0; $x < $tmp_refs->length; $x++) {
        $refs[$x]['val'] = getValue($doc, '.', $tmp_refs->item($x));
        $refs[$x]['src'] = getValue($doc, '@source', $tmp_refs->item($x));
        $refs[$x]['url'] = getValue($doc, '@url', $tmp_refs->item($x));
      }

      foreach ($refs as $key => $ref) {
        $new_cve_refs[] = [
          $name, $ref['src'], $ref['url'], $ref['val']
        ];
      }

      $new_cve_web[] = [$name, $cve_xml->saveXML()];

      $err->script_log("new cve $name");
      $new++;
      print "*";
    }
    else {
      /*
        $sql = "REPLACE INTO `reference`.`cve_db` (`type`, `seq`, `status`, `phase`, `phase_date`, `desc`) VALUES (" .
        "'$type'," .
        "'$seq'," .
        "'{$sys->help->real_escape_string($status)}'," .
        "'{$sys->help->real_escape_string($phase)}'," .
        "'{$phase_dt->format('Y-m-d')}'," .
        "'{$sys->help->real_escape_string($desc)}') " .
        "WHERE cve_id='{$sys->help->real_escape_string($name)}'";
        $sys->help->real_query($sql);

        foreach ($refs as $key => $ref) {
        $sql = "SELECT `id` FROM `reference`.`cve_references` WHERE `cve_seq`='" .
        $sys->help->real_escape_string($name) . "' AND `source`='" .
        $sys->help->real_escape_string($ref['src']) . "'";
        $res2 = $sys->help->query($sql);
        $row2 = $res2->fetch_array(MYSQLI_ASSOC);

        if ($row2['id']) {
        $sql = "UPDATE `reference`.`cve_references` SET `url`='" . $sys->help->real_escape_string($ref['url']) .
        "', val='" . $sys->help->real_escape_string($ref['val']) . "' WHERE `id`=" . $row2['id'];
        $sys->help->real_query($sql);
        }
        else {
        $sql = "INSERT INTO `reference`.`cve_references` (`cve_seq`,`source`,`url`,`val`) VALUES ('" .
        $sys->help->real_escape_string($name) . "','" . $sys->help->real_escape_string($ref['src']) .
        "','" . $sys->help->real_escape_string($ref['url']) . "','" .
        $sys->help->real_escape_string($ref['val']) . "')";
        $sys->help->real_query($sql);
        }
        }

        $err->script_log("existing cve $name");
       */
      $existing++;
      print ".";
    }

    if (($existing + $new) % 100 == 0) {
      if (count($new_cves)) {
        $sys->help->extended_insert("cve_db", $cve_fields, $new_cves, true);
        $sys->help->execute();
      }

      if (count($new_cve_refs)) {
        $sys->help->extended_insert("cve_references", $ref_fields, $new_cve_refs, true);
        $sys->help->execute();
      }

      if (count($new_cve_web)) {
        $sys->help->extended_replace("cve_web", $web_fields, $new_cve_web);
        $sys->help->execute();
      }

      $new_cves = [];
      $new_cve_refs = [];
      $new_cve_web = [];

      print "\t" . ($existing + $new) . " completed" . PHP_EOL;

      $sys->help->update("settings", ['meta_value' => number_format((($existing + $new) / $items->length) * 100, 2)], [
        [
          'field' => 'meta_key',
          'op'    => '=',
          'value' => 'cve-progress'
        ]
      ]);
      $sys->help->execute();
    }
  }

  if (is_array($new_cves) && count($new_cves)) {
    $sys->help->extended_insert("cve_db", $cve_fields, $new_cves, true);
    $sys->help->execute();
  }

  if (is_array($new_cve_refs) && count($new_cve_refs)) {
    $sys->help->extended_insert("cve_references", $ref_fields, $new_cve_refs, true);
    $sys->help->execute();
  }

  if (is_array($new_cve_web) && count($new_cve_web)) {
    $sys->help->extended_replace("cve_web", $web_fields, $new_cve_web);
    $sys->help->execute();
  }
}

function usage() {
  print <<<EOO
Purpose: Parse the CVE file (allitems.xml) retrieved from http://cve.mitre.org/data/downloads/allitems.xml

Usage: php parse_cve.php -f={CVE filename} [--debug] [--help]

 -f={CVE filename}    The file to be parsed (allitems.xml).  Can be absolute or relative path.

 --debug              Debugging output
 --help               This screen


EOO;
}