From 5d8711d494ecfdad0d60d800df5eb317f0e8af69 Mon Sep 17 00:00:00 2001 From: Ryan Prather Date: Mon, 29 Oct 2018 13:19:31 -0400 Subject: [PATCH] Fix bug #49 Revert update_db.php to download compilation STIG library instead of individual as links are inconsistent (#60, #64, #61 Add scape_webpage method to helper.inc for future efforts Fixed typo in uninstall.bat --- exec/export-ckl.php | 18 ++++---- exec/update_db.php | 109 ++++++++++++++++++++++++++++++-------------- inc/helper.inc | 26 +++++++++++ uninstall.bat | 2 +- 4 files changed, 111 insertions(+), 44 deletions(-) diff --git a/exec/export-ckl.php b/exec/export-ckl.php index 9a56d9a..01b1fac 100644 --- a/exec/export-ckl.php +++ b/exec/export-ckl.php @@ -73,8 +73,6 @@ $xml = new Array2XML(); $xml->standalone = true; $xml->formatOutput = true; -$chk_comp_count = 0; -$tgt_comp_count = 0; $total_chk_count = 0; $total_stigs = 0; @@ -85,7 +83,6 @@ if ($tgt_count = count($tgts)) { $host_ip = (is_array($tgt->interfaces) && count($tgt->interfaces) ? current($tgt->interfaces)->get_IPv4() : null); $host_fqdn = (is_array($tgt->interfaces) && count($tgt->interfaces) ? current($tgt->interfaces)->get_FQDN() : null); $host_mac = (is_array($tgt->interfaces) && count($tgt->interfaces) ? current($tgt->interfaces)->get_MAC() : null); - //$host_mac = (count($tgt->interfaces) ? current($tgt->interfaces)->get_Mac() : null); print "Target: {$tgt->get_Name()}" . PHP_EOL; @@ -122,11 +119,11 @@ if ($tgt_count = count($tgts)) { $arr = [ '@comment' => "CyberPerspectives Sagacity v" . VER, 'ASSET' => [ + 'ROLE' => 'None', 'ASSET_TYPE' => 'Computing', 'HOST_NAME' => $tgt->get_Name(), 'HOST_IP' => $host_ip, 'HOST_MAC' => $host_mac, - 'HOST_GUID' => '', 'HOST_FQDN' => $host_fqdn, 'TECH_AREA' => '', 'TARGET_KEY' => '', @@ -220,7 +217,7 @@ if ($tgt_count = count($tgts)) { } // decoding because check contents are already encoded - $cc = str_replace("\\n", "
", htmlentities(html_entity_decode($pdi['check_contents']))); + //$cc = str_replace("\\n", "\n", htmlentities(html_entity_decode($pdi['check_contents']))); $stig_data = array_merge([ [ @@ -257,11 +254,11 @@ if ($tgt_count = count($tgts)) { ], [ 'VULN_ATTRIBUTE' => 'Check_Content', - 'ATTRIBUTE_DATA' => $cc + 'ATTRIBUTE_DATA' => htmlentities(str_replace("\\n", "\n", html_entity_decode(html_entity_decode($pdi['check_contents'])))) ], [ 'VULN_ATTRIBUTE' => 'Fix_Text', - 'ATTRIBUTE_DATA' => htmlentities($pdi['fix_text']) + 'ATTRIBUTE_DATA' => htmlentities(str_replace("\\n", "\n", html_entity_decode(html_entity_decode($pdi['fix_text'])))) ], [ 'VULN_ATTRIBUTE' => 'False_Positives', @@ -301,7 +298,11 @@ if ($tgt_count = count($tgts)) { ], [ 'VULN_ATTRIBUTE' => 'Check_Content_Ref', - 'ATTRIBUTE_DATA' => '' + 'ATTRIBUTE_DATA' => 'M' + ], + [ + 'VULN_ATTRIBUTE' => 'Weight', + 'ATTRIBUTE_DATA' => '10.0' ], [ 'VULN_ATTRIBUTE' => 'Class', @@ -321,6 +322,7 @@ if ($tgt_count = count($tgts)) { $notes = ''; if (is_a($find, 'finding')) { + /** @var finding $find */ $status = $status_map[$find->get_Finding_Status_String()]; $notes = $find->get_Notes(); } diff --git a/exec/update_db.php b/exec/update_db.php index 44a8be5..0d7f329 100644 --- a/exec/update_db.php +++ b/exec/update_db.php @@ -461,6 +461,7 @@ if (isset($cmd['nasl'])) { 'nasl-count' => 0 ]); $count = 0; + check_path(TMP . "/nessus_plugins"); // Capture start time for performance monitoring $diff->resetClock(); @@ -601,55 +602,93 @@ if (isset($cmd['stig'])) { 'stig-progress' => 0, 'stig-count' => 0 ]); + $path = TMP . "/stigs"; check_path(TMP . "/stigs"); - $path = TMP . "/stigs/zip"; - check_path($path); - $stigUrlArray = []; - $tmp = []; - $tmp1 = []; - $tmp2 = []; - $tmp3 = []; + check_path(TMP . "/stigs/zip"); + $sunset_array = []; $diff->resetClock(); print "Started STIG ingestion ({$diff->getStartClockTime()})" . PHP_EOL; - $url_1 = "https://iase.disa.mil/stigs/Pages/a-z.aspx"; - $url_2 = "https://iase.disa.mil"; - $sunset_url = "https://iase.disa.mil/stigs/Lists/Sunset%20Master%20List/FinalView.aspx"; - $regex = "/a href=\"([^ ]+zip\/U_[^ ]+STIG\.zip)[^\>]+\>([^\<]+)\<\/a\>/i"; + $mon = '01'; + $prev_mon = '10'; + $year = (int) $current_date->format("Y"); - if (!isset($cmd['po']) || isset($cmd['do'])) { - $log->debug("Checking url: $url_1"); - $pg_contents = file_get_contents($url_1); + if (between($current_date->format("n"), 4, 6)) { + $mon = '04'; + $prev_mon = '01'; + } + elseif (between($current_date->format("n"), 7, 9)) { + $mon = '07'; + $prev_mon = '04'; + } + elseif (between($current_date->format("n"), 10, 12)) { + $mon = '10'; + $prev_mon = '07'; + } - if(preg_match("/RefreshPageTo\(event, ([^\}]+\})/i", $pg_contents, $tmp)) { - $url_2 .= str_replace(["\u0026", '"'], ["&", ""], html_entity_decode($tmp[1])); + $current_url = "https://iasecontent.disa.mil/stigs/zip/Compilations/U_SRG-STIG_Library_{$year}_{$mon}.zip"; + $current_v2_url = "https://iasecontent.disa.mil/stigs/zip/Compilations/U_SRG-STIG_Library_{$year}_{$mon}_v2.zip"; + $sunset_url = "https://iase.disa.mil/stigs/Lists/Sunset%20Master%20List/FinalView.aspx"; + $stig_fname = "{$path}/stig_library-{$year}_{$mon}.zip"; + + if (!file_exists($stig_fname) && ping("disa.mil") && !isset($cmd['po'])) { + if (isset($cmd['u'])) { + $url = $cmd['u']; + $log->debug("Checking for $url"); + + if (url_exists($url)) { + download_file($url, $stig_fname, $db->help, 'stig-dl-progress'); + } } + else { + $log->debug("Checking for $current_url"); - $log->debug("Checking url: $url_2"); - $pg_contents2 = file_get_contents($url_2); - $log->debug("Checking url: $sunset_url"); - $sunset_contents = file_get_contents($sunset_url); + if ($found = url_exists($current_url)) { + download_file($current_url, $stig_fname, $db->help, 'stig-dl-progress'); + } + if (!$found) { + $log->debug("Checking for $current_v2_url"); - $log->debug("Retrieving all matches"); - preg_match_all($regex, $pg_contents, $tmp1); - preg_match_all($regex, $pg_contents2, $tmp2); - preg_match_all($regex, $sunset_contents, $tmp3); + if ($found = url_exists($current_v2_url)) { + download_file($current_v2_url, $stig_fname, $db->help, 'stig-dl-progress'); + } + } + if ($mon == '01') { + $year--; + } - $stigUrlArray = array_merge($tmp1[1], $tmp2[1], $tmp3[1]); - $log->debug("Match count: " . count($stigUrlArray)); + $prev_url = "https://iasecontent.disa.mil/stigs/zip/Compilations/U_SRG-STIG_Library_{$year}_{$prev_mon}.zip"; + $prev_v2_url = "https://iasecontent.disa.mil/stigs/zip/Compilations/U_SRG-STIG_Library_{$year}_{$prev_mon}_v2.zip"; - print "Downloading " . count($stigUrlArray) . PHP_EOL; + if (!$found) { + $log->debug("Checking for $prev_url"); + if ($found = url_exists($prev_url)) { + download_file($prev_url, $stig_fname, $db->help, 'stig-dl-progress'); + } + } + if (!$found) { + $log->debug("Checking for $prev_v2_url"); + if (url_exists($prev_v2_url)) { + download_file($prev_v2_url, $stig_fname, $db->help, 'stig-dl-progress'); + } + } + } + } - $stigUrlArray = array_unique(array_map(function($url){return str_replace("http://", "https://", $url);}, $stigUrlArray); - sort($stigUrlArray); - $log->debug("stig array", $stigUrlArray); + if(ping("disa.mil") && !isset($cmd['po'])) { + $log->debug("Checking for $sunset_url"); - if(is_array($stigUrlArray) && count($stigUrlArray)) { - foreach($stigUrlArray as $url) { - $stigFname = basename($url); - $log->debug("Downloading $stigFname"); - download_file($url, "{$path}/$stigFname"); + if(url_exists($sunset_url)) { + $log->debug("Downloading sunset STIGs"); + $contents = file_get_contents($sunset_url); + preg_match_all("/a href=\"([^ ]+STIG\.zip)/", $contents, $sunset_array); + + if(is_array($sunset_array) && isset($sunset_array[1]) && count($sunset_array[1])) { + foreach($sunset_array[1] as $url) { + $sunset_fname = basename($url); + download_file($url, TMP . "/stigs/zip/{$sunset_fname}"); + } } } } diff --git a/inc/helper.inc b/inc/helper.inc index e0c7aa0..857dce2 100644 --- a/inc/helper.inc +++ b/inc/helper.inc @@ -855,3 +855,29 @@ function convert_log_level() return Logger::ERROR; } } + +/** + * Helper method to scrape a web page + * + * @param string $url + * + * @return string + */ +function scrape_webpage($url) +{ + $config = [ + CURLOPT_RETURNTRANSFER => true, + CURLOPT_FOLLOWLOCATION => true, + CURLOPT_HEADER => true, + CURLOPT_SSL_VERIFYPEER => false, + CURLOPT_USERAGENT => 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.13) Gecko/20080311 Firefox/2.0.0.13', + CURLOPT_URL => $url + ]; + $c = curl_init(); + + curl_setopt_array($c, $config); + + $output = curl_exec($c); + + return $output; +} diff --git a/uninstall.bat b/uninstall.bat index 5adc570..3b49375 100644 --- a/uninstall.bat +++ b/uninstall.bat @@ -43,7 +43,7 @@ if "%result%"=="1" ( ) echo. -echo Thank you for trying Sagacity. If you have any questions or comments, please echo contact us at https://www.cyberperspectives.com/contact_us +echo Thank you for trying Sagacity. If you have any questions or comments, please contact us at https://www.cyberperspectives.com/contact_us echo. if "%result%"=="1" (