Fix bug #49

Revert update_db.php to download compilation STIG library instead of individual as links are inconsistent (#60, #64, #61 Add scape_webpage method to helper.inc for future efforts Fixed typo in uninstall.bat
2018-10-29 13:19:31 -04:00
parent ab9005f0c3
commit 5d8711d494
4 changed files with 111 additions and 44 deletions
--- a/exec/export-ckl.php
+++ b/exec/export-ckl.php
@@ -73,8 +73,6 @@ $xml = new Array2XML();
 $xml->standalone = true;
 $xml->formatOutput = true;

-$chk_comp_count = 0;
-$tgt_comp_count = 0;
 $total_chk_count = 0;
 $total_stigs = 0;

@@ -85,7 +83,6 @@ if ($tgt_count = count($tgts)) {
    $host_ip = (is_array($tgt->interfaces) && count($tgt->interfaces) ? current($tgt->interfaces)->get_IPv4() : null);
    $host_fqdn = (is_array($tgt->interfaces) && count($tgt->interfaces) ? current($tgt->interfaces)->get_FQDN() : null);
    $host_mac = (is_array($tgt->interfaces) && count($tgt->interfaces) ? current($tgt->interfaces)->get_MAC() : null);
-    //$host_mac = (count($tgt->interfaces) ? current($tgt->interfaces)->get_Mac() : null);

    print "Target: {$tgt->get_Name()}" . PHP_EOL;

@@ -122,11 +119,11 @@ if ($tgt_count = count($tgts)) {
      $arr = [
        '@comment' => "CyberPerspectives Sagacity v" . VER,
        'ASSET' => [
+          'ROLE'            => 'None',
          'ASSET_TYPE'      => 'Computing',
          'HOST_NAME'       => $tgt->get_Name(),
          'HOST_IP'         => $host_ip,
          'HOST_MAC'        => $host_mac,
-          'HOST_GUID'       => '',
          'HOST_FQDN'       => $host_fqdn,
          'TECH_AREA'       => '',
          'TARGET_KEY'      => '',
@@ -220,7 +217,7 @@ if ($tgt_count = count($tgts)) {
        }

        // decoding because check contents are already encoded
-        $cc = str_replace("\\n", "<br />", htmlentities(html_entity_decode($pdi['check_contents'])));
+        //$cc = str_replace("\\n", "\n", htmlentities(html_entity_decode($pdi['check_contents'])));

        $stig_data = array_merge([
          [
@@ -257,11 +254,11 @@ if ($tgt_count = count($tgts)) {
          ],
          [
            'VULN_ATTRIBUTE' => 'Check_Content',
-            'ATTRIBUTE_DATA' => $cc
+            'ATTRIBUTE_DATA' => htmlentities(str_replace("\\n", "\n", html_entity_decode(html_entity_decode($pdi['check_contents']))))
          ],
          [
            'VULN_ATTRIBUTE' => 'Fix_Text',
-            'ATTRIBUTE_DATA' => htmlentities($pdi['fix_text'])
+            'ATTRIBUTE_DATA' => htmlentities(str_replace("\\n", "\n", html_entity_decode(html_entity_decode($pdi['fix_text']))))
          ],
          [
            'VULN_ATTRIBUTE' => 'False_Positives',
@@ -301,7 +298,11 @@ if ($tgt_count = count($tgts)) {
          ],
          [
            'VULN_ATTRIBUTE' => 'Check_Content_Ref',
-            'ATTRIBUTE_DATA' => ''
+            'ATTRIBUTE_DATA' => 'M'
+          ],
+          [
+            'VULN_ATTRIBUTE' => 'Weight',
+            'ATTRIBUTE_DATA' => '10.0'
          ],
          [
            'VULN_ATTRIBUTE' => 'Class',
@@ -321,6 +322,7 @@ if ($tgt_count = count($tgts)) {
        $notes = '';

        if (is_a($find, 'finding')) {
+            /** @var finding $find */
          $status = $status_map[$find->get_Finding_Status_String()];
          $notes = $find->get_Notes();
        }
--- a/exec/update_db.php
+++ b/exec/update_db.php
@@ -461,6 +461,7 @@ if (isset($cmd['nasl'])) {
        'nasl-count'       => 0
    ]);
    $count = 0;
+    check_path(TMP . "/nessus_plugins");

    // Capture start time for performance monitoring
    $diff->resetClock();
@@ -601,55 +602,93 @@ if (isset($cmd['stig'])) {
        'stig-progress'    => 0,
        'stig-count'       => 0
    ]);
+    $path = TMP . "/stigs";
    check_path(TMP . "/stigs");
-    $path = TMP . "/stigs/zip";
-    check_path($path);
-    $stigUrlArray = [];
-    $tmp = [];
-    $tmp1 = [];
-    $tmp2 = [];
-    $tmp3 = [];
+    check_path(TMP . "/stigs/zip");
+    $sunset_array = [];

    $diff->resetClock();
    print "Started STIG ingestion ({$diff->getStartClockTime()})" . PHP_EOL;

-    $url_1 = "https://iase.disa.mil/stigs/Pages/a-z.aspx";
-    $url_2 = "https://iase.disa.mil";
-    $sunset_url = "https://iase.disa.mil/stigs/Lists/Sunset%20Master%20List/FinalView.aspx";
-    $regex = "/a href=\"([^ ]+zip\/U_[^ ]+STIG\.zip)[^\>]+\>([^\<]+)\<\/a\>/i";
+    $mon      = '01';
+    $prev_mon = '10';
+    $year     = (int) $current_date->format("Y");

-    if (!isset($cmd['po']) || isset($cmd['do'])) {
-        $log->debug("Checking url: $url_1");
-        $pg_contents = file_get_contents($url_1);
+    if (between($current_date->format("n"), 4, 6)) {
+        $mon      = '04';
+        $prev_mon = '01';
+    }
+    elseif (between($current_date->format("n"), 7, 9)) {
+        $mon      = '07';
+        $prev_mon = '04';
+    }
+    elseif (between($current_date->format("n"), 10, 12)) {
+        $mon      = '10';
+        $prev_mon = '07';
+    }

-        if(preg_match("/RefreshPageTo\(event, ([^\}]+\})/i", $pg_contents, $tmp)) {
-            $url_2 .= str_replace(["\u0026", '"'], ["&", ""], html_entity_decode($tmp[1]));
+    $current_url    = "https://iasecontent.disa.mil/stigs/zip/Compilations/U_SRG-STIG_Library_{$year}_{$mon}.zip";
+    $current_v2_url = "https://iasecontent.disa.mil/stigs/zip/Compilations/U_SRG-STIG_Library_{$year}_{$mon}_v2.zip";
+    $sunset_url     = "https://iase.disa.mil/stigs/Lists/Sunset%20Master%20List/FinalView.aspx";
+    $stig_fname = "{$path}/stig_library-{$year}_{$mon}.zip";
+
+    if (!file_exists($stig_fname) && ping("disa.mil") && !isset($cmd['po'])) {
+        if (isset($cmd['u'])) {
+            $url = $cmd['u'];
+            $log->debug("Checking for $url");
+
+            if (url_exists($url)) {
+                download_file($url, $stig_fname, $db->help, 'stig-dl-progress');
+            }
        }
+        else {
+            $log->debug("Checking for $current_url");

-        $log->debug("Checking url: $url_2");
-        $pg_contents2 = file_get_contents($url_2);
-        $log->debug("Checking url: $sunset_url");
-        $sunset_contents = file_get_contents($sunset_url);
+            if ($found = url_exists($current_url)) {
+                download_file($current_url, $stig_fname, $db->help, 'stig-dl-progress');
+            }
+            if (!$found) {
+                $log->debug("Checking for $current_v2_url");

-        $log->debug("Retrieving all matches");
-        preg_match_all($regex, $pg_contents, $tmp1);
-        preg_match_all($regex, $pg_contents2, $tmp2);
-        preg_match_all($regex, $sunset_contents, $tmp3);
+                if ($found = url_exists($current_v2_url)) {
+                    download_file($current_v2_url, $stig_fname, $db->help, 'stig-dl-progress');
+                }
+            }
+            if ($mon == '01') {
+                $year--;
+            }

-        $stigUrlArray = array_merge($tmp1[1], $tmp2[1], $tmp3[1]);
-        $log->debug("Match count: " . count($stigUrlArray));
+            $prev_url    = "https://iasecontent.disa.mil/stigs/zip/Compilations/U_SRG-STIG_Library_{$year}_{$prev_mon}.zip";
+            $prev_v2_url = "https://iasecontent.disa.mil/stigs/zip/Compilations/U_SRG-STIG_Library_{$year}_{$prev_mon}_v2.zip";

-        print "Downloading " . count($stigUrlArray) . PHP_EOL;
+            if (!$found) {
+                $log->debug("Checking for $prev_url");
+                if ($found = url_exists($prev_url)) {
+                    download_file($prev_url, $stig_fname, $db->help, 'stig-dl-progress');
+                }
+            }
+            if (!$found) {
+                $log->debug("Checking for $prev_v2_url");
+                if (url_exists($prev_v2_url)) {
+                    download_file($prev_v2_url, $stig_fname, $db->help, 'stig-dl-progress');
+                }
+            }
+        }
+    }

-        $stigUrlArray = array_unique(array_map(function($url){return str_replace("http://", "https://", $url);}, $stigUrlArray);
-        sort($stigUrlArray);
-        $log->debug("stig array", $stigUrlArray);
+    if(ping("disa.mil") && !isset($cmd['po'])) {
+        $log->debug("Checking for $sunset_url");

-        if(is_array($stigUrlArray) && count($stigUrlArray)) {
-            foreach($stigUrlArray as $url) {
-                $stigFname = basename($url);
-                $log->debug("Downloading $stigFname");
-                download_file($url, "{$path}/$stigFname");
+        if(url_exists($sunset_url)) {
+            $log->debug("Downloading sunset STIGs");
+            $contents = file_get_contents($sunset_url);
+            preg_match_all("/a href=\"([^ ]+STIG\.zip)/", $contents, $sunset_array);
+
+            if(is_array($sunset_array) && isset($sunset_array[1]) && count($sunset_array[1])) {
+                foreach($sunset_array[1] as $url) {
+                    $sunset_fname = basename($url);
+                    download_file($url, TMP . "/stigs/zip/{$sunset_fname}");
+                }
            }
        }
    }
--- a/inc/helper.inc
+++ b/inc/helper.inc
@@ -855,3 +855,29 @@ function convert_log_level()
            return Logger::ERROR;
    }
 }
+
+/**
+ * Helper method to scrape a web page
+ *
+ * @param string $url
+ *
+ * @return string
+ */
+function scrape_webpage($url)
+{
+    $config = [
+        CURLOPT_RETURNTRANSFER => true,
+        CURLOPT_FOLLOWLOCATION => true,
+        CURLOPT_HEADER => true,
+        CURLOPT_SSL_VERIFYPEER => false,
+        CURLOPT_USERAGENT => 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.13) Gecko/20080311 Firefox/2.0.0.13',
+        CURLOPT_URL => $url
+    ];
+    $c = curl_init();
+
+    curl_setopt_array($c, $config);
+
+    $output = curl_exec($c);
+
+    return $output;
+}
--- a/uninstall.bat
+++ b/uninstall.bat
@@ -43,7 +43,7 @@ if "%result%"=="1" (
 )

 echo.
-echo Thank you for trying Sagacity.  If you have any questions or comments, please echo contact us at https://www.cyberperspectives.com/contact_us
+echo Thank you for trying Sagacity.  If you have any questions or comments, please contact us at https://www.cyberperspectives.com/contact_us
 echo.

 if "%result%"=="1" (