checklist.inc - deleted duplicate BIND 9 checklist icon entry

finding.inc - removed ID property to prevent duplicate findings from being added to the table
host_list.inc - deleted unused constructor
import.inc - formatting
db_schema.json - removed sagacity.findings.id field (making tgt_id and pdi_id new primary keys), and updated references
Database_Baseline.zip - updated routines for above change
background_results.php - fixed bug #19
export-ckl.php - performance adjustments
parse_excel_echecklist.php - performance improvements, ensure duplicate findings are not created, make eChecklist true status, update for removing findings.id field
parse_nvd_json_cve.php - convert reading json to array instead of object for reading CPEs (which were updated to CPE 2.3 instead of 2.2)
parse_* - remove findings.id field
database.inc - formatting, and update for removing findings.id field
index.php - ensure user can't import a host list without uploading a host list file

Fixed:
#65, #51, #28, #27, #10

classes/checklist.inc

@@ -21,6 +21,7 @@
  *  - May 13, 2017 - Added WindowsFirewall.jpg image for checklist
  *  - May 19, 2017 - Fixed typo for WindowsFirewall
  *  - Aug 23, 2017 - JO, Expanded checklist icons
+ *  - Nov 6, 2018 - Deleted duplicate BIND 9 checklist icon entry
  */
 
 /**
@@ -545,9 +546,6 @@ class checklist
                 case (preg_match("/Mobile Device/i", $this->name) ? true : false):
                     $this->icon = 'mobile-device.jpg';
                     break;
-                case (preg_match("/BIND 9/i", $this->name) ? true : false):
-                    $this->icon = 'BIND DNS.jpg';
-                    break;
                 case (preg_match("/Remote Access/i", $this->name) ? true : false):
                     $this->icon = 'remote-access.gif';
                     break;

classes/finding.inc

@@ -22,6 +22,7 @@
  *  - May 25, 2017 - Fixed bug of get_Category method returning empty severity (defaults to II if empty)
  *  - Jan 10, 2018 - Formatting
  *  - May 24, 2018 - Simplified get_Finding_Status_ID method
+ *  - Nov 6, 2018 - Removed ID property to keep from duplicate findings
  */
 
 /**
@@ -32,13 +33,6 @@
  */
 class finding {
 
-  /**
-   * Finding ID
-   *
-   * @var integer
-   */
-  protected $id = null;
-
   /**
    * Target ID
    *
@@ -155,7 +149,6 @@ class finding {
   /**
    * Constructor
    *
-   * @param integer $int_ID
    * @param integer $int_Tgt_ID
    * @param integer $int_PDI_ID
    * @param integer $int_Scan_ID
@@ -165,8 +158,7 @@ class finding {
    * @param string $str_Orig_Src
    * @param integer $int_Finding_Itr
    */
-  public function __construct($int_ID, $int_Tgt_ID, $int_PDI_ID, $int_Scan_ID, $Finding_Status, $str_Notes, $int_Change_ID, $str_Orig_Src, $int_Finding_Itr) {
+  public function __construct($int_Tgt_ID, $int_PDI_ID, $int_Scan_ID, $Finding_Status, $str_Notes, $int_Change_ID, $str_Orig_Src, $int_Finding_Itr) {
-    $this->id = $int_ID;
     $this->tgt_id = $int_Tgt_ID;
     $this->pdi_id = $int_PDI_ID;
     $this->scan_id = $int_Scan_ID;
@@ -182,15 +174,6 @@ class finding {
     $this->finding_itr = $int_Finding_Itr;
   }
 
-  /**
-   * Getter function for Finding ID
-   *
-   * @return integer
-   */
-  public function get_ID() {
-    return $this->id;
-  }
-
   /**
    * Getter function for target ID
    *

classes/host_list.inc

@@ -13,6 +13,7 @@
  * Change Log:
  * - Jan 16, 2018 - File created
  * - Feb 6, 2018 - Added getter/setter methods
+ * - Nov 6, 2018 - Deleted unused constructor
  */
 
 /**
@@ -65,14 +66,6 @@ class host_list
      */
     private $_scanNotes = null;
 
-    /**
-     * Constructor
-     */
-    public function __construct()
-    {
-
-    }
-
     /**
      * Getter function for _targetId
      *

exec/background_results.php

@@ -237,7 +237,7 @@ do {
 }
 while ($dbh->get_Running_Script_Count($conf['ste']));
 
-if (!$debug) {
+if (!$debug && file_exists(DOC_ROOT . "/exec/parse_config.ini")) {
     unlink(DOC_ROOT . "/exec/parse_config.ini");
 }
 

exec/export-ckl.php

@@ -190,10 +190,11 @@ if ($tgt_count = count($tgts)) {
       $total_stigs += $pdi_count = (is_array($pdis) ? count($pdis) : 0);
       $count = 0;
 
+      $findings = $db->get_Finding($tgt);
+
       foreach ($pdis as $pdi) {
-        $find = $db->get_Finding($tgt, new stig($pdi['pdi_id'], $pdi['STIG_ID'], null));
+        if (isset($findings[$pdi['pdi_id']])) {
-        if (is_array($find) && count($find) && isset($find[0]) && is_a($find[0], 'finding')) {
+          $find = $findings[$pdi['pdi_id']];
-          $find = $find[0];
         }
 
         $sev = 'low';

exec/parse_excel_echecklist.php

@@ -25,6 +25,7 @@
  *  - Aug 28, 2017 - Fixed couple minor bugs
  *  - Jan 15, 2018 - Formatting, reorganized use statements, and cleaned up
  *  - May 24, 2018 - Attempt to fix bug #413
+ *  - Nov 6, 2018 - performance improvements, ensure duplicate findings are not created, make eChecklist true status, update for removing findings.id 
  */
 $cmd = getopt("f:", ['debug::', 'help::']);
 set_time_limit(0);
@@ -126,7 +127,7 @@ foreach ($objSS->getWorksheetIterator() as $wksht) {
         continue;
     }
 
-$scan->isTerminated();
+    $scan->isTerminated();
 
 	$log->notice("Reading from {$wksht->getTitle()}");
 
@@ -155,6 +156,7 @@ $scan->isTerminated();
     $short_title_col = Coordinate::stringFromColumnIndex($idx['short_title']);
     $row_count       = $highestRow = $wksht->getHighestDataRow() - 10;
     $highestCol      = $wksht->getHighestDataColumn(10);
+    $tgt_findings    = [];
 
     for ($col = 'F' ; $col != $highestCol ; $col++) {
         $cell = $wksht->getCell($col . '10');
@@ -219,7 +221,9 @@ $scan->isTerminated();
             $scan->add_Target_to_Host_List($hl);
         }
 
-        if (preg_match('/Overall/i', $cell->getValue())) {
+        $tgt_findings[$tgt->get_ID()] = $db->get_Finding($tgt);
+
+        if (preg_match('/overall/i', $cell->getValue())) {
             $log->debug("Found overall: {$cell->getColumn()}");
             break;
         }
@@ -294,34 +298,24 @@ $scan->isTerminated();
             $status = $wksht->getCell(Coordinate::stringFromColumnIndex($idx['target'] + $x) . $row->getRowIndex())
                 ->getValue();
 
-			$log->debug("{$tgt->get_Name()} {$stig->get_ID()} ($status)");
+			$findings = $tgt_findings[$tgt->get_ID()];
-
+			if (is_array($findings) && count($findings) && isset($findings[$stig->get_PDI_ID()]) && is_a($findings[$stig->get_PDI_ID()], 'finding')) {
-            $finding = $db->get_Finding($tgt, $stig);
-
-            if (is_array($finding) && count($finding) && isset($finding[0]) && is_a($finding[0], 'finding')) {
                 /** @var finding $tmp */
-                $tmp = $finding[0];
+                $tmp = $findings[$stig->get_PDI_ID()];
-
-                if(preg_match("/Not a Finding|Not Applicable/i", $status)) {
-                    $ds = $tmp->get_Deconflicted_Status($status);
-                    $tmp->set_Finding_Status_By_String($ds);
-                }
-                else {
-                    $tmp->set_Finding_Status_By_String($status);
-                }
 
+                $tmp->set_Finding_Status_By_String($status);
                 $tmp->set_Notes($notes);
                 $tmp->set_Category($cat_lvl);
 
                 $updated_findings[] = $tmp;
             }
             else {
-                $tmp = new finding(null, $tgt->get_ID(), $stig->get_PDI_ID(), $scan->get_ID(), $status, $notes, null, null, null);
+                $tmp = new finding($tgt->get_ID(), $stig->get_PDI_ID(), $scan->get_ID(), $status, $notes, null, null, null);
                 $tmp->set_Category($cat_lvl);
 
                 $new_findings[] = $tmp;
             }
-
+            $log->debug("{$tgt->get_Name()} {$stig->get_ID()} ({$tmp->get_Finding_Status_String()})");
             $x++;
         }
 
@@ -347,10 +341,8 @@ $scan->isTerminated();
     }
 }
 
-/**
+/** @var host_list $h */
- * @var host_list $h
+foreach($scan->get_Host_List() as $h) {
- */
-foreach($host_list as $h) {
     $db->update_Target_Counts($h->getTargetId());
 }
 

exec/parse_host_data_collection.php

@@ -112,10 +112,8 @@ foreach ($files as $file) {
 
 $scan_id = 0;
 
-foreach ($findings as $key => $find) {
+/** @var finding $find */
-  if (false) {
+foreach ($findings as $find) {
-    $find = new finding();
-  }
   $ret = array();
   if ($find->get_Scan_ID()) {
     $scan_id = $find->get_Scan_ID();

exec/parse_mssql.php

@@ -181,10 +181,8 @@ class mssql_parser extends scan_xml_parser {
     // check for finding
     $finding = $this->db->get_Finding($this->tgt, $this->stig);
     if (is_array($finding) && count($finding)) {
+        /** @var finding $finding */
       $finding = $finding[0];
-      if (false) {
-        $finding = new finding();
-      }
 
       $finding->prepend_Notes("(MSSQL) " . $this->notes);
       if ($finding->get_Finding_Status_String() != "Not Reviewed" && $finding->get_Finding_Status_String() != $this->status) {
@@ -199,7 +197,7 @@ class mssql_parser extends scan_xml_parser {
       $this->updated_findings[$finding->get_PDI_ID()] = $finding;
     }
     else {
-      $finding = new finding(null, $this->tgt->get_ID(), $this->stig->get_PDI_ID(), $this->scan->get - ID(), $this->status, $this->notes, finding::NC, "MSSQL", 1);
+      $finding = new finding($this->tgt->get_ID(), $this->stig->get_PDI_ID(), $this->scan->get - ID(), $this->status, $this->notes, finding::NC, "MSSQL", 1);
 
       $this->new_findings[$this->stig->get_PDI_ID()] = $finding;
     }

exec/parse_nessus.php

@@ -1142,7 +1142,7 @@ class nessus_parser extends scan_xml_parser
                     }
                 }
                 else {
-                    $tmp = new finding(null, $this->tgt->get_ID(), $this->plugin->result->stig->get_PDI_ID(), $this->scan->get_ID(), $this->plugin->result->status, "[{$this->tgt->get_Name()}]: {$note}", finding::NC, "Nessus", 1);
+                    $tmp = new finding($this->tgt->get_ID(), $this->plugin->result->stig->get_PDI_ID(), $this->scan->get_ID(), $this->plugin->result->status, "[{$this->tgt->get_Name()}]: {$note}", finding::NC, "Nessus", 1);
                     if (!is_null($pdi)) {
                         $tmp->set_Category($pdi->get_Category_Level());
                     }
@@ -1178,7 +1178,7 @@ class nessus_parser extends scan_xml_parser
                 $stig = new stig($pdi_id, $this->plugin->result->stig, $this->plugin->desc);
                 $this->db->add_Stig($stig);
 
-                $tmp = new finding(null, $this->tgt->get_ID(), $pdi->get_ID(), $this->scan->get_ID(), $this->plugin->result->status, "[" . $this->tgt->get_Name() . "]: " . $note, finding::NC, "Nessus", 1);
+                $tmp = new finding($this->tgt->get_ID(), $pdi->get_ID(), $this->scan->get_ID(), $this->plugin->result->status, "[" . $this->tgt->get_Name() . "]: " . $note, finding::NC, "Nessus", 1);
                 $tmp->set_Category($this->plugin->result->cat);
 
                 if (isset($this->new_findings[$tmp->get_PDI_ID()])) {
@@ -1215,10 +1215,8 @@ class nessus_parser extends scan_xml_parser
             }
 
             if (is_a($finding, 'finding')) {
+                /** @var finding $finding */
                 $this->log->script_log("Updating finding");
-                if (false) {
-                    $finding = new finding();
-                }
                 if ($this->debug) {
                     $this->log->script_log("Finding exists: " . print_r($finding, true), E_DEBUG);
                 }
@@ -1265,7 +1263,7 @@ class nessus_parser extends scan_xml_parser
             }
             else {
                 $this->log->script_log("Adding new finding");
-                $tmp = new finding(null, $this->tgt->get_ID(), $this->plugin->db_plugin->get_PDI_ID(), $this->scan->get_ID(), $this->plugin->result->status, $note, finding::NC, "Nessus", 1);
+                $tmp = new finding($this->tgt->get_ID(), $this->plugin->db_plugin->get_PDI_ID(), $this->scan->get_ID(), $this->plugin->result->status, $note, finding::NC, "Nessus", 1);
                 $tmp->set_Category($this->plugin->result->cat);
 
                 $this->new_findings[$tmp->get_PDI_ID()] = $tmp;

exec/parse_nvd_json_cve.php

@@ -49,7 +49,7 @@ $log = new Logger("nvd_cve");
 $log->pushHandler(new StreamHandler(LOG_PATH . "/nvd_cve.log", $log_level));
 
 $db            = new db();
-$json          = json_decode(file_get_contents($cmd['f']));
+$json          = json_decode(file_get_contents($cmd['f']), true);
 $existing_cves = [];
 
 $db->help->select("cve_db", ['cve_id']);
@@ -60,19 +60,21 @@ if (is_array($cves) && count($cves)) {
     }
 }
 
-print "Currently " . count($existing_cves) . " in DB" . PHP_EOL . "Parsing: " . count($json->CVE_Items) . " items" . PHP_EOL;
+print "Currently " . count($existing_cves) . " in DB" . PHP_EOL . "Parsing: " . count($json['CVE_Items']) . " items" . PHP_EOL;
 
 $db_cpes      = [];
+$db_cpes23    = [];
 $new_cves     = [];
 $new_cve_refs = [];
 $sw_rows      = [];
 $new          = 0;
 $existing     = 0;
 
-$db->help->select("software", ['id', 'cpe']);
+$db->help->select("software", ['id', 'cpe', 'cpe23']);
 $rows = $db->help->execute();
 foreach ($rows as $row) {
     $db_cpes["{$row['cpe']}"] = $row['id'];
+    $db_cpes23["{$row['cpe23']}"] = $row['id'];
 }
 
 $cve_fields = [
@@ -82,24 +84,22 @@ $ref_fields = [
     'cve_seq', 'source', 'url', 'val'
 ];
 
-foreach ($json->CVE_Items as $cve) {
+foreach ($json['CVE_Items'] as $cve) {
-    if (!isset($existing_cves["{$cve->cve->CVE_data_meta->ID}"])) {
+    if (!isset($existing_cves["{$cve['cve']['CVE_data_meta']['ID']}"])) {
-        $log->debug("Adding {$cve->cve->CVE_data_meta->ID}");
+        $log->debug("Adding {$cve['cve']['CVE_data_meta']['ID']}");
         $new++;
 
         $desc   = [];
         $status = null;
         $phase  = null;
         $cpes   = [];
-        $name   = $cve->cve->CVE_data_meta->ID;
+        $name   = $cve['cve']['CVE_data_meta']['ID'];
-        $type   = $cve->cve->data_type;
+        $seq    = $cve['cve']['CVE_data_meta']['ID'];
-        $seq    = $cve->cve->CVE_data_meta->ID;
+        $pd     = new DateTime($cve['publishedDate']);
-        $pd     = new DateTime($cve->publishedDate);
-        $lmd    = new DateTime($cve->lastModifiedDate);
 
-        if (is_array($cve->cve->description->description_data) && count($cve->cve->description->description_data)) {
+        if (is_array($cve['cve']['description']['description_data']) && count($cve['cve']['description']['description_data'])) {
-            foreach ($cve->cve->description->description_data as $d) {
+            foreach ($cve['cve']['description']['description_data'] as $d) {
-                $desc[] = $d->value;
+                $desc[] = $d['value'];
             }
         }
 
@@ -107,24 +107,21 @@ foreach ($json->CVE_Items as $cve) {
             $name, $seq, $status, $phase, $pd, implode(PHP_EOL, $desc)
         ];
 
-        if (is_array($cve->cve->references->reference_data) && count($cve->cve->references->reference_data)) {
+        if (is_array($cve['cve']['references']['reference_data']) && count($cve['cve']['references']['reference_data'])) {
-            foreach ($cve->cve->references->reference_data as $ref) {
+            foreach ($cve['cve']['references']['reference_data'] as $ref) {
-                $log->debug("Adding reference {$ref->url}");
+                $log->debug("Adding reference {$ref['url']}");
                 $new_cve_refs[] = [
-                    $name, null, $ref->url, null
+                    $name, null, $ref['url'], null
                 ];
             }
         }
 
-        if (is_array($cve->configurations->nodes) && count($cve->configurations->nodes)) {
+        if(is_array($cve['configurations']['nodes']) && count($cve['configurations']['nodes'])) {
-            foreach ($cve->configurations->nodes as $n) {
+            foreach($cve['configurations']['nodes'] as $n) {
-                if (isset($n->cpe) && is_array($n->cpe) && count($n->cpe)) {
+                if(isset($n['cpe_match']) && is_array($n['cpe_match']) && count($n['cpe_match'])) {
-                    foreach ($n->cpe as $cpe) {
+                    foreach($n['cpe_match'] as $c) {
-                        if (isset($cpe->cpe22Uri)) {
+                        if($c['vulnerable'] && $c['cpe23Uri']) {
-                            $cpes[] = $cpe->cpe22Uri;
+                            $cpes[] = $c['cpe23Uri'];
-                        }
-                        elseif (isset($cpe->cpeMatchString)) {
-                            $cpes[] = $cpe->cpeMatchString;
                         }
                     }
                 }
@@ -135,6 +132,8 @@ foreach ($json->CVE_Items as $cve) {
             foreach ($cpes as $cpe) {
                 if (isset($db_cpes["{$cpe}"])) {
                     $sw_rows[] = [$name, $db_cpes["{$cpe}"]];
+                } elseif (isset($db_cpes23["{$cpe}"])) {
+                    $sw_rows[] = [$name, $db_cpes23["{$cpe}"]];
                 }
             }
         }
@@ -185,7 +184,7 @@ if (count($sw_rows)) {
     $db->help->execute();
 }
 
-unlink($cmd['f']);
+//unlink($cmd['f']);
 
 print PHP_EOL;
 

exec/parse_scc_xccdf.php

@@ -104,8 +104,6 @@ class scc_parser extends scan_xml_parser
 
     var $found_rule = false;
 
-    var $log = null;
-
     /**
      * Constructor
      *
@@ -665,9 +663,7 @@ class scc_parser extends scan_xml_parser
             */
 
             if (is_array($existing_findings) && count($existing_findings) && isset($existing_findings[$pdi_id])) {
-                /**
+                /** @var finding $finding */
-                 * @var finding $finding
-                 */
                 $finding = $existing_findings[$pdi_id];
 
                 $finding->set_Finding_Status_By_String($finding->get_Deconflicted_Status($group['status']));
@@ -679,7 +675,7 @@ class scc_parser extends scan_xml_parser
 
                 $update_findings[$pdi_id] = $finding;
             } else {
-                $new_findings[$pdi_id] = new finding(null, $this->tgt->get_ID(), $pdi_id, $this->scan->get_ID(), $group['status'], $note, finding::NC, null, 1);
+                $new_findings[$pdi_id] = new finding($this->tgt->get_ID(), $pdi_id, $this->scan->get_ID(), $group['status'], $note, finding::NC, null, 1);
             }
         }
 
@@ -689,11 +685,11 @@ class scc_parser extends scan_xml_parser
         $hl->setTargetId($this->tgt->get_ID());
         $hl->setTargetName($this->tgt->get_Name());
         $hl->setFindingCount(count($new_findings) + count($update_findings));
-        $hl->setScanError(false);
 
         $this->db->update_Target_Counts($this->tgt->get_ID());
 
         $this->scan->add_Target_to_Host_List($hl);
+        $this->db->update_Scan_Host_List($this->scan);
     }
 }
 

inc/xml_parser.inc

@@ -178,7 +178,7 @@ class scan_xml_parser
     /**
      * The previous stack element
      *
-     * @var unknown
+     * @var string
      */
     var $previous = null;
 
@@ -192,9 +192,9 @@ class scan_xml_parser
     /**
      * Construct
      *
-     * @param unknown $obj_in
+     * @param mixed $obj_in
-     * @param unknown $ste_id_in
+     * @param int $ste_id_in
-     * @param unknown $scan_fname
+     * @param string $scan_fname
      */
     function __construct($obj_in, $ste_id_in, $scan_fname)
     {

ste/export.php

@@ -43,9 +43,12 @@ use PhpOffice\PhpSpreadsheet\Writer\Ods;
 use PhpOffice\PhpSpreadsheet\Writer\Csv;
 use PhpOffice\PhpSpreadsheet\Writer\Html;
 use PhpOffice\PhpSpreadsheet\Cell\Coordinate;
+use PhpOffice\PhpSpreadsheet\Worksheet;
 use Monolog\Logger;
 use Monolog\Handler\StreamHandler;
 
+global $conditions, $validation, $borders;
+
 set_time_limit(0);
 $db = new db();
 $emass_ccis = null;
@@ -137,7 +140,6 @@ $host_status = array(
 foreach ($findings as $worksheet_name => $data) {
   $log->debug("Looping through worksheet $worksheet_name");
   $chk_arr = [];
-  $named_range = '';
 
   // Build the "Checklist" cell string with titles of all checklists on this worksheet
   foreach ($data['checklists'] as $key => $chk_id) {
@@ -354,7 +356,7 @@ $log->debug("Writing complete");
 /**
  * Update the header on the worksheet
  *
- * @param \PhpOffice\PhpSpreadsheet\Worksheet $sheet
+ * @param Worksheet $sheet
  * @param array:integer $tgts
  * @param db $db
  */

ste/index.php

@@ -604,7 +604,7 @@ include_once 'header.inc';
     </form>
 
     <form method='post' action='#' style='margin-left: 20px;'
-          onsubmit="$('#submit').attr('disabled', true);return true;" id='host-list-form'>
+          onsubmit="if(!$('#host-list-file').val()){return false;}$('#submit').attr('disabled', true);return true;" id='host-list-form'>
         <div style='font-weight:400;color:red;'>Must keep 'host-list' as part of the filename</div>
         <input type='hidden' name='file' id='host-list-file' style='display:none;' />
         <input type='hidden' name='action' value='import_host_list' />