Revert update_db.php to download compilation STIG library instead of individual as links are inconsistent (#60, #64, #61
Add scape_webpage method to helper.inc for future efforts
Fixed typo in uninstall.bat
This commit is contained in:
Ryan Prather 2018-10-29 13:19:31 -04:00
parent ab9005f0c3
commit 5d8711d494
4 changed files with 111 additions and 44 deletions

View File

@ -73,8 +73,6 @@ $xml = new Array2XML();
$xml->standalone = true; $xml->standalone = true;
$xml->formatOutput = true; $xml->formatOutput = true;
$chk_comp_count = 0;
$tgt_comp_count = 0;
$total_chk_count = 0; $total_chk_count = 0;
$total_stigs = 0; $total_stigs = 0;
@ -85,7 +83,6 @@ if ($tgt_count = count($tgts)) {
$host_ip = (is_array($tgt->interfaces) && count($tgt->interfaces) ? current($tgt->interfaces)->get_IPv4() : null); $host_ip = (is_array($tgt->interfaces) && count($tgt->interfaces) ? current($tgt->interfaces)->get_IPv4() : null);
$host_fqdn = (is_array($tgt->interfaces) && count($tgt->interfaces) ? current($tgt->interfaces)->get_FQDN() : null); $host_fqdn = (is_array($tgt->interfaces) && count($tgt->interfaces) ? current($tgt->interfaces)->get_FQDN() : null);
$host_mac = (is_array($tgt->interfaces) && count($tgt->interfaces) ? current($tgt->interfaces)->get_MAC() : null); $host_mac = (is_array($tgt->interfaces) && count($tgt->interfaces) ? current($tgt->interfaces)->get_MAC() : null);
//$host_mac = (count($tgt->interfaces) ? current($tgt->interfaces)->get_Mac() : null);
print "Target: {$tgt->get_Name()}" . PHP_EOL; print "Target: {$tgt->get_Name()}" . PHP_EOL;
@ -122,11 +119,11 @@ if ($tgt_count = count($tgts)) {
$arr = [ $arr = [
'@comment' => "CyberPerspectives Sagacity v" . VER, '@comment' => "CyberPerspectives Sagacity v" . VER,
'ASSET' => [ 'ASSET' => [
'ROLE' => 'None',
'ASSET_TYPE' => 'Computing', 'ASSET_TYPE' => 'Computing',
'HOST_NAME' => $tgt->get_Name(), 'HOST_NAME' => $tgt->get_Name(),
'HOST_IP' => $host_ip, 'HOST_IP' => $host_ip,
'HOST_MAC' => $host_mac, 'HOST_MAC' => $host_mac,
'HOST_GUID' => '',
'HOST_FQDN' => $host_fqdn, 'HOST_FQDN' => $host_fqdn,
'TECH_AREA' => '', 'TECH_AREA' => '',
'TARGET_KEY' => '', 'TARGET_KEY' => '',
@ -220,7 +217,7 @@ if ($tgt_count = count($tgts)) {
} }
// decoding because check contents are already encoded // decoding because check contents are already encoded
$cc = str_replace("\\n", "<br />", htmlentities(html_entity_decode($pdi['check_contents']))); //$cc = str_replace("\\n", "\n", htmlentities(html_entity_decode($pdi['check_contents'])));
$stig_data = array_merge([ $stig_data = array_merge([
[ [
@ -257,11 +254,11 @@ if ($tgt_count = count($tgts)) {
], ],
[ [
'VULN_ATTRIBUTE' => 'Check_Content', 'VULN_ATTRIBUTE' => 'Check_Content',
'ATTRIBUTE_DATA' => $cc 'ATTRIBUTE_DATA' => htmlentities(str_replace("\\n", "\n", html_entity_decode(html_entity_decode($pdi['check_contents']))))
], ],
[ [
'VULN_ATTRIBUTE' => 'Fix_Text', 'VULN_ATTRIBUTE' => 'Fix_Text',
'ATTRIBUTE_DATA' => htmlentities($pdi['fix_text']) 'ATTRIBUTE_DATA' => htmlentities(str_replace("\\n", "\n", html_entity_decode(html_entity_decode($pdi['fix_text']))))
], ],
[ [
'VULN_ATTRIBUTE' => 'False_Positives', 'VULN_ATTRIBUTE' => 'False_Positives',
@ -301,7 +298,11 @@ if ($tgt_count = count($tgts)) {
], ],
[ [
'VULN_ATTRIBUTE' => 'Check_Content_Ref', 'VULN_ATTRIBUTE' => 'Check_Content_Ref',
'ATTRIBUTE_DATA' => '' 'ATTRIBUTE_DATA' => 'M'
],
[
'VULN_ATTRIBUTE' => 'Weight',
'ATTRIBUTE_DATA' => '10.0'
], ],
[ [
'VULN_ATTRIBUTE' => 'Class', 'VULN_ATTRIBUTE' => 'Class',
@ -321,6 +322,7 @@ if ($tgt_count = count($tgts)) {
$notes = ''; $notes = '';
if (is_a($find, 'finding')) { if (is_a($find, 'finding')) {
/** @var finding $find */
$status = $status_map[$find->get_Finding_Status_String()]; $status = $status_map[$find->get_Finding_Status_String()];
$notes = $find->get_Notes(); $notes = $find->get_Notes();
} }

View File

@ -461,6 +461,7 @@ if (isset($cmd['nasl'])) {
'nasl-count' => 0 'nasl-count' => 0
]); ]);
$count = 0; $count = 0;
check_path(TMP . "/nessus_plugins");
// Capture start time for performance monitoring // Capture start time for performance monitoring
$diff->resetClock(); $diff->resetClock();
@ -601,55 +602,93 @@ if (isset($cmd['stig'])) {
'stig-progress' => 0, 'stig-progress' => 0,
'stig-count' => 0 'stig-count' => 0
]); ]);
$path = TMP . "/stigs";
check_path(TMP . "/stigs"); check_path(TMP . "/stigs");
$path = TMP . "/stigs/zip"; check_path(TMP . "/stigs/zip");
check_path($path); $sunset_array = [];
$stigUrlArray = [];
$tmp = [];
$tmp1 = [];
$tmp2 = [];
$tmp3 = [];
$diff->resetClock(); $diff->resetClock();
print "Started STIG ingestion ({$diff->getStartClockTime()})" . PHP_EOL; print "Started STIG ingestion ({$diff->getStartClockTime()})" . PHP_EOL;
$url_1 = "https://iase.disa.mil/stigs/Pages/a-z.aspx"; $mon = '01';
$url_2 = "https://iase.disa.mil"; $prev_mon = '10';
$sunset_url = "https://iase.disa.mil/stigs/Lists/Sunset%20Master%20List/FinalView.aspx"; $year = (int) $current_date->format("Y");
$regex = "/a href=\"([^ ]+zip\/U_[^ ]+STIG\.zip)[^\>]+\>([^\<]+)\<\/a\>/i";
if (!isset($cmd['po']) || isset($cmd['do'])) { if (between($current_date->format("n"), 4, 6)) {
$log->debug("Checking url: $url_1"); $mon = '04';
$pg_contents = file_get_contents($url_1); $prev_mon = '01';
}
elseif (between($current_date->format("n"), 7, 9)) {
$mon = '07';
$prev_mon = '04';
}
elseif (between($current_date->format("n"), 10, 12)) {
$mon = '10';
$prev_mon = '07';
}
if(preg_match("/RefreshPageTo\(event, ([^\}]+\})/i", $pg_contents, $tmp)) { $current_url = "https://iasecontent.disa.mil/stigs/zip/Compilations/U_SRG-STIG_Library_{$year}_{$mon}.zip";
$url_2 .= str_replace(["\u0026", '"'], ["&", ""], html_entity_decode($tmp[1])); $current_v2_url = "https://iasecontent.disa.mil/stigs/zip/Compilations/U_SRG-STIG_Library_{$year}_{$mon}_v2.zip";
$sunset_url = "https://iase.disa.mil/stigs/Lists/Sunset%20Master%20List/FinalView.aspx";
$stig_fname = "{$path}/stig_library-{$year}_{$mon}.zip";
if (!file_exists($stig_fname) && ping("disa.mil") && !isset($cmd['po'])) {
if (isset($cmd['u'])) {
$url = $cmd['u'];
$log->debug("Checking for $url");
if (url_exists($url)) {
download_file($url, $stig_fname, $db->help, 'stig-dl-progress');
}
} }
else {
$log->debug("Checking for $current_url");
$log->debug("Checking url: $url_2"); if ($found = url_exists($current_url)) {
$pg_contents2 = file_get_contents($url_2); download_file($current_url, $stig_fname, $db->help, 'stig-dl-progress');
$log->debug("Checking url: $sunset_url"); }
$sunset_contents = file_get_contents($sunset_url); if (!$found) {
$log->debug("Checking for $current_v2_url");
$log->debug("Retrieving all matches"); if ($found = url_exists($current_v2_url)) {
preg_match_all($regex, $pg_contents, $tmp1); download_file($current_v2_url, $stig_fname, $db->help, 'stig-dl-progress');
preg_match_all($regex, $pg_contents2, $tmp2); }
preg_match_all($regex, $sunset_contents, $tmp3); }
if ($mon == '01') {
$year--;
}
$stigUrlArray = array_merge($tmp1[1], $tmp2[1], $tmp3[1]); $prev_url = "https://iasecontent.disa.mil/stigs/zip/Compilations/U_SRG-STIG_Library_{$year}_{$prev_mon}.zip";
$log->debug("Match count: " . count($stigUrlArray)); $prev_v2_url = "https://iasecontent.disa.mil/stigs/zip/Compilations/U_SRG-STIG_Library_{$year}_{$prev_mon}_v2.zip";
print "Downloading " . count($stigUrlArray) . PHP_EOL; if (!$found) {
$log->debug("Checking for $prev_url");
if ($found = url_exists($prev_url)) {
download_file($prev_url, $stig_fname, $db->help, 'stig-dl-progress');
}
}
if (!$found) {
$log->debug("Checking for $prev_v2_url");
if (url_exists($prev_v2_url)) {
download_file($prev_v2_url, $stig_fname, $db->help, 'stig-dl-progress');
}
}
}
}
$stigUrlArray = array_unique(array_map(function($url){return str_replace("http://", "https://", $url);}, $stigUrlArray); if(ping("disa.mil") && !isset($cmd['po'])) {
sort($stigUrlArray); $log->debug("Checking for $sunset_url");
$log->debug("stig array", $stigUrlArray);
if(is_array($stigUrlArray) && count($stigUrlArray)) { if(url_exists($sunset_url)) {
foreach($stigUrlArray as $url) { $log->debug("Downloading sunset STIGs");
$stigFname = basename($url); $contents = file_get_contents($sunset_url);
$log->debug("Downloading $stigFname"); preg_match_all("/a href=\"([^ ]+STIG\.zip)/", $contents, $sunset_array);
download_file($url, "{$path}/$stigFname");
if(is_array($sunset_array) && isset($sunset_array[1]) && count($sunset_array[1])) {
foreach($sunset_array[1] as $url) {
$sunset_fname = basename($url);
download_file($url, TMP . "/stigs/zip/{$sunset_fname}");
}
} }
} }
} }

View File

@ -855,3 +855,29 @@ function convert_log_level()
return Logger::ERROR; return Logger::ERROR;
} }
} }
/**
* Helper method to scrape a web page
*
* @param string $url
*
* @return string
*/
function scrape_webpage($url)
{
$config = [
CURLOPT_RETURNTRANSFER => true,
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_HEADER => true,
CURLOPT_SSL_VERIFYPEER => false,
CURLOPT_USERAGENT => 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.13) Gecko/20080311 Firefox/2.0.0.13',
CURLOPT_URL => $url
];
$c = curl_init();
curl_setopt_array($c, $config);
$output = curl_exec($c);
return $output;
}

View File

@ -43,7 +43,7 @@ if "%result%"=="1" (
) )
echo. echo.
echo Thank you for trying Sagacity. If you have any questions or comments, please echo contact us at https://www.cyberperspectives.com/contact_us echo Thank you for trying Sagacity. If you have any questions or comments, please contact us at https://www.cyberperspectives.com/contact_us
echo. echo.
if "%result%"=="1" ( if "%result%"=="1" (