Fix bug #49

Revert update_db.php to download compilation STIG library instead of individual as links are inconsistent (#60, #64, #61
Add scape_webpage method to helper.inc for future efforts
Fixed typo in uninstall.bat

exec/export-ckl.php

@@ -73,8 +73,6 @@ $xml = new Array2XML();
 $xml->standalone = true;
 $xml->formatOutput = true;
 
-$chk_comp_count = 0;
-$tgt_comp_count = 0;
 $total_chk_count = 0;
 $total_stigs = 0;
 
@@ -85,7 +83,6 @@ if ($tgt_count = count($tgts)) {
     $host_ip = (is_array($tgt->interfaces) && count($tgt->interfaces) ? current($tgt->interfaces)->get_IPv4() : null);
     $host_fqdn = (is_array($tgt->interfaces) && count($tgt->interfaces) ? current($tgt->interfaces)->get_FQDN() : null);
     $host_mac = (is_array($tgt->interfaces) && count($tgt->interfaces) ? current($tgt->interfaces)->get_MAC() : null);
-    //$host_mac = (count($tgt->interfaces) ? current($tgt->interfaces)->get_Mac() : null);
 
     print "Target: {$tgt->get_Name()}" . PHP_EOL;
 
@@ -122,11 +119,11 @@ if ($tgt_count = count($tgts)) {
       $arr = [
         '@comment' => "CyberPerspectives Sagacity v" . VER,
         'ASSET' => [
+          'ROLE'            => 'None',
           'ASSET_TYPE'      => 'Computing',
           'HOST_NAME'       => $tgt->get_Name(),
           'HOST_IP'         => $host_ip,
           'HOST_MAC'        => $host_mac,
-          'HOST_GUID'       => '',
           'HOST_FQDN'       => $host_fqdn,
           'TECH_AREA'       => '',
           'TARGET_KEY'      => '',
@@ -220,7 +217,7 @@ if ($tgt_count = count($tgts)) {
         }
 
         // decoding because check contents are already encoded
-        $cc = str_replace("\\n", "<br />", htmlentities(html_entity_decode($pdi['check_contents'])));
+        //$cc = str_replace("\\n", "\n", htmlentities(html_entity_decode($pdi['check_contents'])));
 
         $stig_data = array_merge([
           [
@@ -257,11 +254,11 @@ if ($tgt_count = count($tgts)) {
           ],
           [
             'VULN_ATTRIBUTE' => 'Check_Content',
-            'ATTRIBUTE_DATA' => $cc
+            'ATTRIBUTE_DATA' => htmlentities(str_replace("\\n", "\n", html_entity_decode(html_entity_decode($pdi['check_contents']))))
           ],
           [
             'VULN_ATTRIBUTE' => 'Fix_Text',
-            'ATTRIBUTE_DATA' => htmlentities($pdi['fix_text'])
+            'ATTRIBUTE_DATA' => htmlentities(str_replace("\\n", "\n", html_entity_decode(html_entity_decode($pdi['fix_text']))))
           ],
           [
             'VULN_ATTRIBUTE' => 'False_Positives',
@@ -301,7 +298,11 @@ if ($tgt_count = count($tgts)) {
           ],
           [
             'VULN_ATTRIBUTE' => 'Check_Content_Ref',
-            'ATTRIBUTE_DATA' => ''
+            'ATTRIBUTE_DATA' => 'M'
+          ],
+          [
+            'VULN_ATTRIBUTE' => 'Weight',
+            'ATTRIBUTE_DATA' => '10.0'
           ],
           [
             'VULN_ATTRIBUTE' => 'Class',
@@ -321,6 +322,7 @@ if ($tgt_count = count($tgts)) {
         $notes = '';
 
         if (is_a($find, 'finding')) {
+            /** @var finding $find */
           $status = $status_map[$find->get_Finding_Status_String()];
           $notes = $find->get_Notes();
         }

exec/update_db.php

@@ -461,6 +461,7 @@ if (isset($cmd['nasl'])) {
         'nasl-count'       => 0
     ]);
     $count = 0;
+    check_path(TMP . "/nessus_plugins");
 
     // Capture start time for performance monitoring
     $diff->resetClock();
@@ -601,55 +602,93 @@ if (isset($cmd['stig'])) {
         'stig-progress'    => 0,
         'stig-count'       => 0
     ]);
+    $path = TMP . "/stigs";
     check_path(TMP . "/stigs");
-    $path = TMP . "/stigs/zip";
+    check_path(TMP . "/stigs/zip");
-    check_path($path);
+    $sunset_array = [];
-    $stigUrlArray = [];
-    $tmp = [];
-    $tmp1 = [];
-    $tmp2 = [];
-    $tmp3 = [];
 
     $diff->resetClock();
     print "Started STIG ingestion ({$diff->getStartClockTime()})" . PHP_EOL;
 
-    $url_1 = "https://iase.disa.mil/stigs/Pages/a-z.aspx";
+    $mon      = '01';
-    $url_2 = "https://iase.disa.mil";
+    $prev_mon = '10';
-    $sunset_url = "https://iase.disa.mil/stigs/Lists/Sunset%20Master%20List/FinalView.aspx";
+    $year     = (int) $current_date->format("Y");
-    $regex = "/a href=\"([^ ]+zip\/U_[^ ]+STIG\.zip)[^\>]+\>([^\<]+)\<\/a\>/i";
 
-    if (!isset($cmd['po']) || isset($cmd['do'])) {
+    if (between($current_date->format("n"), 4, 6)) {
-        $log->debug("Checking url: $url_1");
+        $mon      = '04';
-        $pg_contents = file_get_contents($url_1);
+        $prev_mon = '01';
-
+    }
-        if(preg_match("/RefreshPageTo\(event, ([^\}]+\})/i", $pg_contents, $tmp)) {
+    elseif (between($current_date->format("n"), 7, 9)) {
-            $url_2 .= str_replace(["\u0026", '"'], ["&", ""], html_entity_decode($tmp[1]));
+        $mon      = '07';
+        $prev_mon = '04';
+    }
+    elseif (between($current_date->format("n"), 10, 12)) {
+        $mon      = '10';
+        $prev_mon = '07';
     }
 
-        $log->debug("Checking url: $url_2");
+    $current_url    = "https://iasecontent.disa.mil/stigs/zip/Compilations/U_SRG-STIG_Library_{$year}_{$mon}.zip";
-        $pg_contents2 = file_get_contents($url_2);
+    $current_v2_url = "https://iasecontent.disa.mil/stigs/zip/Compilations/U_SRG-STIG_Library_{$year}_{$mon}_v2.zip";
-        $log->debug("Checking url: $sunset_url");
+    $sunset_url     = "https://iase.disa.mil/stigs/Lists/Sunset%20Master%20List/FinalView.aspx";
-        $sunset_contents = file_get_contents($sunset_url);
+    $stig_fname = "{$path}/stig_library-{$year}_{$mon}.zip";
 
-        $log->debug("Retrieving all matches");
+    if (!file_exists($stig_fname) && ping("disa.mil") && !isset($cmd['po'])) {
-        preg_match_all($regex, $pg_contents, $tmp1);
+        if (isset($cmd['u'])) {
-        preg_match_all($regex, $pg_contents2, $tmp2);
+            $url = $cmd['u'];
-        preg_match_all($regex, $sunset_contents, $tmp3);
+            $log->debug("Checking for $url");
 
-        $stigUrlArray = array_merge($tmp1[1], $tmp2[1], $tmp3[1]);
+            if (url_exists($url)) {
-        $log->debug("Match count: " . count($stigUrlArray));
+                download_file($url, $stig_fname, $db->help, 'stig-dl-progress');
+            }
+        }
+        else {
+            $log->debug("Checking for $current_url");
 
-        print "Downloading " . count($stigUrlArray) . PHP_EOL;
+            if ($found = url_exists($current_url)) {
+                download_file($current_url, $stig_fname, $db->help, 'stig-dl-progress');
+            }
+            if (!$found) {
+                $log->debug("Checking for $current_v2_url");
 
-        $stigUrlArray = array_unique(array_map(function($url){return str_replace("http://", "https://", $url);}, $stigUrlArray);
+                if ($found = url_exists($current_v2_url)) {
-        sort($stigUrlArray);
+                    download_file($current_v2_url, $stig_fname, $db->help, 'stig-dl-progress');
-        $log->debug("stig array", $stigUrlArray);
+                }
+            }
+            if ($mon == '01') {
+                $year--;
+            }
 
-        if(is_array($stigUrlArray) && count($stigUrlArray)) {
+            $prev_url    = "https://iasecontent.disa.mil/stigs/zip/Compilations/U_SRG-STIG_Library_{$year}_{$prev_mon}.zip";
-            foreach($stigUrlArray as $url) {
+            $prev_v2_url = "https://iasecontent.disa.mil/stigs/zip/Compilations/U_SRG-STIG_Library_{$year}_{$prev_mon}_v2.zip";
-                $stigFname = basename($url);
+
-                $log->debug("Downloading $stigFname");
+            if (!$found) {
-                download_file($url, "{$path}/$stigFname");
+                $log->debug("Checking for $prev_url");
+                if ($found = url_exists($prev_url)) {
+                    download_file($prev_url, $stig_fname, $db->help, 'stig-dl-progress');
+                }
+            }
+            if (!$found) {
+                $log->debug("Checking for $prev_v2_url");
+                if (url_exists($prev_v2_url)) {
+                    download_file($prev_v2_url, $stig_fname, $db->help, 'stig-dl-progress');
+                }
+            }
+        }
+    }
+
+    if(ping("disa.mil") && !isset($cmd['po'])) {
+        $log->debug("Checking for $sunset_url");
+
+        if(url_exists($sunset_url)) {
+            $log->debug("Downloading sunset STIGs");
+            $contents = file_get_contents($sunset_url);
+            preg_match_all("/a href=\"([^ ]+STIG\.zip)/", $contents, $sunset_array);
+
+            if(is_array($sunset_array) && isset($sunset_array[1]) && count($sunset_array[1])) {
+                foreach($sunset_array[1] as $url) {
+                    $sunset_fname = basename($url);
+                    download_file($url, TMP . "/stigs/zip/{$sunset_fname}");
+                }
             }
         }
     }

inc/helper.inc

@@ -855,3 +855,29 @@ function convert_log_level()
             return Logger::ERROR;
     }
 }
+
+/**
+ * Helper method to scrape a web page
+ *
+ * @param string $url
+ *
+ * @return string
+ */
+function scrape_webpage($url)
+{
+    $config = [
+        CURLOPT_RETURNTRANSFER => true,
+        CURLOPT_FOLLOWLOCATION => true,
+        CURLOPT_HEADER => true,
+        CURLOPT_SSL_VERIFYPEER => false,
+        CURLOPT_USERAGENT => 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.13) Gecko/20080311 Firefox/2.0.0.13',
+        CURLOPT_URL => $url
+    ];
+    $c = curl_init();
+
+    curl_setopt_array($c, $config);
+
+    $output = curl_exec($c);
+
+    return $output;
+}

uninstall.bat

@@ -43,7 +43,7 @@ if "%result%"=="1" (
 )
 
 echo.
-echo Thank you for trying Sagacity.  If you have any questions or comments, please echo contact us at https://www.cyberperspectives.com/contact_us
+echo Thank you for trying Sagacity.  If you have any questions or comments, please contact us at https://www.cyberperspectives.com/contact_us
 echo.
 
 if "%result%"=="1" (