From 17dbe134ccfafc3d6e455e0e62d095c7baa0e67d Mon Sep 17 00:00:00 2001 From: Ryan Prather Date: Fri, 19 Oct 2018 18:45:08 -0400 Subject: [PATCH] Database_Baseline.zip - Revisions for creating views and routines installer.php - Fix bug when update_db.php - Converted STIG download to identify the zip files from the a-z master list and download them individually instead of downloading the compilation zip file. Also integrated the sunset list into the same process so ALL STIGs are downloaded and imported at the same time --- Database_Baseline.zip | Bin 1193012 -> 1192408 bytes exec/installer.php | 4 +- exec/update_db.php | 190 +++++++++++++++++++----------------------- 3 files changed, 86 insertions(+), 108 deletions(-) diff --git a/Database_Baseline.zip b/Database_Baseline.zip index cf21074b30604b85126230bc1564f5a97ed23731..59a6c07e6483b3b70343e875b96434fa7ac192bf 100644 GIT binary patch delta 2400 zcmV-m37__~=t$V(NPvU^gaU*Egam{Iga(8Mgb0KQgbIWUgbaiYgbsucgb;)gv=UYV zf7Yf_O#s3`ka-9I0JJs$02KfL0CQnyVPk1@d0%pGb#!TNWpgfbacu1yS#Q%Y_?b!j z4}L&uD;31J1q@NzYy`Rll9uZw328c_rAcYh!G!qle7=)73F%ll7YXL$yKH}#?RuRC zDD{p#Kg`Y^?(ST>hK|ncS7S6=FP3QUf5!E_TL_K9G+Tuq(4Fh|uJ7IJqj|DIZ^HQQ zZPfj8|E_cQb`K*Gm(EZW>TB3~h*ufLGPb(2T*6W}|gBzn`ZwHS||g4dl>#nVhVKoud*yk>nY2;pa!#H!oe;I~EyxcEB z0T2hM?8K}{Hz`#CtdcAt5Uki&#fZRegE)+jDV(fISeU+Um&3>2B5={8$DD^gFb0l+ z8+!ypeg%wDpNdSlOHh^Z2QQ9;m0&Ges`<)pwO8U&wyaV+;v@@TMRgk)CRpQiJT7xA zLRNJ0KFW`kDcFH!9!UgXe~R_TAFz0|~^()PyGcD#Q@t2zY8Nd>m>k9gqA5 z#)x7^B}~~ur(px{0#@N<-U=+7FmLiP3h)*@O^HK;zag-~laLy5e>Dm~dIog}Ow+MW z!2~$-FrGmiXc10xf6&{Cs(BVt*;QTlKb{C(CpDs(DuAEF5`@%LpzLfJNFZEOG$PHy zV{QnfqC{~YJPzOo{)q!zAhgot0F~MbNT+~ux=1-;Tvq7LLd^an1Pdn%vxo|sZzzCg z$C;QGQWNwf08?-YLTV}yUJ6EumsQCq0I8J$=9C8QXKRYrf42xEBBl(O3L8lW7fRt< zw{fKI5=vZiu=B~*WHQSkykjI8vI1-Z!K_mb;d8v$6xr<+1ZRyBp&@P_TITd6&hLkV z2?*xndGd$~tddB$3-zkZQKP&F=yPD%$YAKj$Vp? zo+%iC3(*aLe`9r+4Q_UQ)VbE_BiTaM>_Vp~FFUTaogFL^*K4#-Oq?~y`H{41(meK2 zFF+zV^*RECd3hG&^I)0JJU%X6$5>1fTofL^nvhG3nBg;frpSaCG5h+Mg9j-^aiSzS zSub5O1B@h_3;|P=tbwxd?NDU2Ko1)xx-#XP(;+Yfe>uM1kJxfEh87-Ap(G}!&WTi$ zrIDZ*n`=hWMC3S$X0tFo0XnCg&gnVmkVS*T6*%~CNsSR>F0_`EFjw^ST|Th(OQEk{ zh#nNrv__f*cnau&u8l|Ce1I+c;e{8ky{J=e4!zTb(zr~b&_7d}KLoM=apA2_I-BM_ z$!hu0e~ULreJH}VvmeUkG6DJ<5}@CidbCTv%6Zl1*_oProg?~RCP0@7kdpuW{E$<#7z1m~jNy*-fesdlIv zyF0WUJ%LpVj+50Ha0bSjF~a;L_n#DIGO*xJOHt^`)CjU+u7^W6QOaZ;XFtyL^5EMz z(N%afWqDcs)le~wQBs&6Con^EYRJ%3Hc5gcWUTY;Ij?w*Ep6U*HiN&EvB~$Gx?t?SfQ1QtEY^3`HWBck)*j!q`|84;kLB@uQ(tS4VxYDHZ z1;8z6dX9R-wH+BCxvXorxbwZduH!o2v|fm%q@S?}pB&X6xjwz$|Bhf8IJvSclx@kk*ZE0RE8Cz!{q|7~J?bj%5aW z3S}^)^=}-_e2J65J!L?pj>@Q`1|4-rn9Q@!-YRghaWl70_$KUUBFyL;HA8rj=eFrd4rQzPNsiC&O;C!bbZG!TafIJX)}2C1eg5gAM_Oshm1K%ge>|;h z%9j4+bg3j-O`;@IDGDs-@mJhlUDQ4V^`$aGrJC_(Ih)uPZ0r;W*01pkgw>^MxO5Hw ziEG$sgSG{y0E2I`e*&-tQwOSwRc~_lrrei;VI0ETHC}zbKC%Uf{Asehfx|1Yr`WE} z4YnTXCPykF3e0BgTfII212p{UfAm<63PGY>vi&b!g7MX7bDY>hXVsgWnnyhhq4Z8jGd>+Ef>l0) zs(m0Ok<`ad6szeSRMlH2iq-H6NUSk=`#{V24O1_0002Ts$O~k delta 2991 zcmV;g3sCgf<4CmVNPvU^gaU*Egam{Iga(8Mgb0KQgbIWUgbaiYgbsucgb;)gv=UYV ze>9R!O-(Nh$<+z~0FPS$02KfL0CQnyVPk1@d0%pGb#!TNWpgfbacu1zYmbva@UxfP zf7lNuZIfy|-|-b|%PCr^_TlRbAf+dfmI}0bG5&XFAFwarX!WY6CAll?&SQ6Hc6OLq zc9r-jaZg+?NKfw{9$dMCUY?F#_tA7Ue_x`bYgdnMAk+(zG!8zZTUYN~J-X9IvnWPy zgT?KesQK;gZTt4k7Dgn<&!Qx~hvuirm+)#Bd2Wc{>ba@=(M|k&IVS>pWvh#zj{JD- z#|ZwCAX;3)4ZD8rN*J9S9vmKwj4^sV==bdIv(B*79UDU%itKLRG_3K%$B&DOfA5!~ z4UN&HKORv$y<9>Z^!uH$Ik4c)vWzai1-znShGunM8Y8qyXLm(96U%%(F`ykr_XTDw zJ-3lueNDtX9t;ihsb#+~-g9)JFsC7x>4ZQ9?3aU{0qEIn8|8%!YhZUK{A4aNp;Kz0v&vs=n8e^;BaKVL?%8=sgL)-$3RuGTxb^@koTb&p_Uo#@k|utig%9_NTU&*j^a;ixlDOA$t7S z`{c&1m-?}t01mBWI7L-}!$jE~8lAC0IE(yHXM{#ZA5;Jx9-t%q=md+@e-5TjJ1>wP zFrN4rM`XIfbIRQWmIC{@M-g1c;|iLQsj}P#+XTf3$fBINwmWtq#&?^9UIcG(-g<|(Uea;2S3%qEd6s4z2UzA81 zoTzxd_t^%-6k3!B73+DTDA6Y{2zDBzp-3@3$!rKOP}K$ZMm8Y1@mY!)9d$de?BPmsV=X^PRteDl1rZ$w3pd<2(6wC@f2)p2IAZK4t1wN<5izxQ=!DPX|MD{Osk! ziPbZXP0Q#V9?+ZNL+7ji(Ru8A{{-Re9}bRMf);is0RJ8_n8SmD=pN_^TurXA>~faN zBy_H3!9FSQeh6PbXcA3Y`CWEK;3_W<1@i049 z$F^``i(v3e!cgjT=JJd4B4um7+ zoaLNIu|00$J{4Kstj!L&VP;n!|A(ZS8(y6dX#%nFG8Iys(a0$LQ$|a9A+F)%*3SOy z&DMFFEma)cZmDLAWEH6XN_hw!0J_0J$de94f3g!&u0oyMdB!47UueFy2Y!H>#wbp0 zaHKV$ump#i?4TEYOT8*3JqoiR^de~wHY_|Vq5*JQ#q%UOP!?CnV+?xBa#rL?1(;D9 zj+!x%fpR~VyRjM@P6yv8PPIA0{&SO1$~1?C7NJF&E8l?ka5VI>x$vhAR>_-#h0WrU zf33145xpwB*{8$7PDv*z(+5)^rL zy{O$*@IGmJ(y?b@hurO`&bgvji-4w2Y){8V>zaFa5< zMi0ppX9awGIPH>`FLqHYK%6mZb#f54e_%6rO$19lws_;as~C&51mEV1cQGM{Ok#%5 z<}*bm#K^QScOP&kc~k5#iaxKF4jBPPk_|=o(6hs8vR~VTwql@3<;x^8#dJ9z+KO^ z!etZ&-lzMMNnCH~Lx#3G*>A?GRn%n2aD zcY2SvE;L(cXpARAYlI*#);v0bA$(#yHLb$~T8@q;$b9Gx>2_LFX;{BqqNT8!i(4!{N0t-8B%%?e|y8e7t=bVR1`W8 zy8obu+@e|f)s1}zPj0qC6TT7K=*%X4Wz^0m0e5PBI0^nbL2CYk9d~tK8aDf4ZodGZ z_X4cQRc|odahEfDRqrtW@=J`;w362s{(?h+`8e)k z)6VY-^q@9ZMtCS>e|%9w{q>Fe*`MP6mJe(9CVm$to`_{{<@Z+pKe6%~MZo?)`_8}B z-|}<(z12Q%s|^y+4D=?u$K)M<{5XTs&3pWS1MTZ&zK0BYex#XUGqb(rNY^tY88OGK z#=g;N1290tPXi`?cJ@NOXvV*|jVqIWt5tOJL_WX`Qft+Ze=ru0m6L#?z&|^NV=I2| z-oK0}zN&~$T=GrPd;{|Y-KOl3ANqsEkLXaB(a7$aFbD3Y)GNb z-Y-Jq;(V_TnEY^H>HJ_yw&=LwlJklgf^q~hKNVR9D?H)3^z3Jjq@UwVv8s-EE;`1U zVpSaQ%+EWP6i-EFE&%G`W466$z6|};pB^6k1h@DK5|AVgG?GnCO)m_|)d~Ork6V|P zFA_Bt$vD!DLSxhclL@?#L1WYblL@?#L1UNUFA^*QR1KF8FcM_~p%9mLFcLrl!4sFb lFcL)qvKp8FFcL-rSSOc9F%m=q9x9iOF%m@vy)6;|001#CnWg{$ diff --git a/exec/installer.php b/exec/installer.php index 5075dca..1391e4d 100644 --- a/exec/installer.php +++ b/exec/installer.php @@ -41,7 +41,7 @@ $db_step = [ 'cpe' => ['filter' => FILTER_VALIDATE_BOOLEAN], 'cve' => ['filter' => FILTER_VALIDATE_BOOLEAN], 'stig' => ['filter' => FILTER_VALIDATE_BOOLEAN], - 'update-freq' => ['filter' => FILTER_VALIDATE_INT, 'flag' => FILTER_NULL_ON_FAILURE] + 'update-freq' => ['filter' => FILTER_VALIDATE_FLOAT, 'flag' => FILTER_NULL_ON_FAILURE] ]; $company_step = [ 'company' => $params, @@ -194,7 +194,6 @@ function save_Database($params) unset($db); } - $successful = true; $zip = new ZipArchive(); $db = new mysqli(DB_SERVER, $params['root-uname'], $params['root-pwd'], 'mysql'); if ($db->connect_errno && $db->connect_errno == 1045) { @@ -361,7 +360,6 @@ EOO; if (preg_grep("/Access Denied/i", $output)) { $errors[] = $output; - $successful = false; } else { unlink($file); diff --git a/exec/update_db.php b/exec/update_db.php index e722171..3acedfd 100644 --- a/exec/update_db.php +++ b/exec/update_db.php @@ -601,12 +601,96 @@ if (isset($cmd['stig'])) { 'stig-progress' => 0, 'stig-count' => 0 ]); - $path = TMP . "/stigs"; + $path = TMP . "/stigs/zip"; check_path($path); + $stigUrlArray = []; + $tmp = []; + $tmp1 = []; + $tmp2 = []; + $tmp3 = []; $diff->resetClock(); print "Started STIG ingestion ({$diff->getStartClockTime()})" . PHP_EOL; + $url_1 = "https://iase.disa.mil/stigs/Pages/a-z.aspx"; + $url_2 = "https://iase.disa.mil"; + $sunset_url = "https://iase.disa.mil/stigs/Lists/Sunset%20Master%20List/FinalView.aspx"; + $regex = "/a href=\"([^ ]+zip\/U_[^ ]+STIG\.zip)[^\>]+\>([^\<]+)\<\/a\>/i"; + + if (!isset($cmd['po']) || isset($cmd['do'])) { + $log->debug("Checking url: $url_1"); + $pg_contents = file_get_contents($url_1); + + if(preg_match("/RefreshPageTo\(event, ([^\}]+\})/i", $pg_contents, $tmp)) { + $url_2 .= str_replace(["\u0026", '"'], ["&", ""], html_entity_decode($tmp[1])); + } + + $log->debug("Checking url: $url_2"); + $pg_contents2 = file_get_contents($url_2); + $log->debug("Checking url: $sunset_url"); + $sunset_contents = file_get_contents($sunset_url); + + $log->debug("Retrieving all matches"); + preg_match_all($regex, $pg_contents, $tmp1); + preg_match_all($regex, $pg_contents2, $tmp2); + preg_match_all($regex, $sunset_contents, $tmp3); + + $stigUrlArray = array_merge($tmp1[1], $tmp2[1], $tmp3[1]); + $log->debug("Match count: " . count($stigUrlArray)); + + print "Downloading " . count($stigUrlArray) . PHP_EOL; + if(is_array($stigUrlArray) && count($stigUrlArray)) { + foreach($stigUrlArray as $url) { + $stigFname = basename($url); + $log->debug("Downloading $stigFname"); + download_file($url, "{$path}/$stigFname"); + } + } + } + + if (!isset($cmd['do']) || isset($cmd['po'])) { + $stig_files = array_merge( + glob("{$path}/*.zip"), glob("{$path}/*.xml"), glob(TMP . "/*.zip"), glob(TMP . "/*.xml"), glob(TMP . "/stigs/xml/*.xml") + ); + if (!count($stig_files)) { + die("Could not locate any XCCDF STIG libraries " . realpath(TMP)); + } + + $script = realpath(defined('PHP_BIN') ? PHP_BIN : PHP) . + " -c " . realpath(PHP_CONF) . + " -f " . realpath(DOC_ROOT . "/exec/background_stigs.php") . " --" . + (isset($cmd['exclude']) && $cmd['exclude'] ? " --exclude=\"{$cmd['exclude']}\"" : "") . + " --delete"; + + $log->debug("Script to run $script"); + passthru($script); + } + + $db->help->select_count("sagacity.stigs"); + $stig_count = $db->help->execute(); + + $db->set_Setting("stig-count", $stig_count); + + $diff->stopClock(); + + print PHP_EOL . "Finished at {$diff->getEndClockTime()}" . PHP_EOL . + "Total Time: {$diff->getDiffString()}" . PHP_EOL; + + sleep(3); +} + +if (is_a($diff->getTotalDiff(), 'DateInterval')) { + print "Total Script Time: {$diff->getTotalDiffString()}" . PHP_EOL; +} + +/** + * Function to download the latest STIG compilation library zip file for extraction and updating + */ +function getStigLibrary() +{ + global $current_date, $cmd, $log, $db; + $path = TMP; + $mon = '01'; $prev_mon = '10'; $year = (int) $current_date->format("Y"); @@ -671,109 +755,6 @@ if (isset($cmd['stig'])) { } } } - - if (!isset($cmd['do']) || isset($cmd['po'])) { - $stig_files = array_merge( - glob("{$path}/*.zip"), glob("{$path}/*.xml"), glob(TMP . "/*.zip"), glob(TMP . "/*.xml"), glob(TMP . "/stigs/xml/*.xml") - ); - if (!file_exists($stig_fname) && !count($stig_files)) { - die("Could not locate $stig_fname or find any other zip files in " . realpath(TMP)); - } - - $script = realpath(defined('PHP_BIN') ? PHP_BIN : PHP) . - " -c " . realpath(PHP_CONF) . - " -f " . realpath(DOC_ROOT . "/exec/background_stigs.php") . " --" . - (isset($cmd['exclude']) && $cmd['exclude'] ? " --exclude=\"{$cmd['exclude']}\"" : "") . - " --delete"; - - $log->debug("Script to run $script"); - passthru($script); - } - - $db->help->select_count("sagacity.stigs"); - $stig_count = $db->help->execute(); - - $db->set_Setting("stig-count", $stig_count); - - $diff->stopClock(); - - print PHP_EOL . "Finished at {$diff->getEndClockTime()}" . PHP_EOL . - "Total Time: {$diff->getDiffString()}" . PHP_EOL; - - sleep(3); -} - -/** - * Update Sunset STIG library from DISA content - */ -if (isset($cmd['sunset'])) { - $db->set_Setting_Array([ - 'stig-dl-progress' => 0, - 'stig-progress' => 0, - 'stig-count' => 0 - ]); - $path = TMP . "/stigs/zip"; - check_path($path); - $sunset_array = []; - - $diff->resetClock(); - print "Started Sunset STIG ingestion ({$diff->getStartClockTime()})" . PHP_EOL; - - $sunset_url="https://iase.disa.mil/stigs/Lists/Sunset%20Master%20List/FinalView.aspx"; - - if (ping("disa.mil") && !isset($cmd['po'])) { - $log->debug("Checking for $sunset_url"); - if ($found = url_exists($sunset_url)) { - $contents=file_get_contents($sunset_url); - } - - if (!$found) { - $log->debug("Unable to download $sunset_url, aborting Sunset"); - die("Unable to open $sunset_url, aborting Sunset"); - } - - preg_match_all("/a href=\"([^ ]+zip\/U_[^ ]+STIG\.zip)/", $contents, $sunset_array); - - foreach($sunset_array[1] as $url) { - $sunset_fname = basename($url); - download_file($url, "{$path}/$sunset_fname"); - } - } - - if (!isset($cmd['do']) || isset($cmd['po'])) { - $stig_files = array_merge( - glob("{$path}/*.zip"), glob("{$path}/*.xml"), - glob(TMP . "/*.zip"), glob(TMP . "/*.xml"), glob(TMP . "/stigs/xml/*.xml") - ); - if (!count($stig_files)) { - die("Could not find any other zip files in " . realpath(TMP)); - } - - $script = realpath(defined('PHP_BIN') ? PHP_BIN : PHP) . - " -c " . realpath(PHP_CONF) . - " -f " . realpath(DOC_ROOT . "/exec/background_stigs.php") . " --" . - (isset($cmd['exclude']) && $cmd['exclude'] ? " --exclude=\"{$cmd['exclude']}\"" : "") . - " --delete"; - - $log->debug("Script to run $script"); - passthru($script); - } - - $db->help->select_count("sagacity.stigs"); - $stig_count = $db->help->execute(); - - $db->set_Setting("stig-count", $stig_count); - - $diff->stopClock(); - - print PHP_EOL . "Finished at {$diff->getEndClockTime()}" . PHP_EOL . - "Total Time: {$diff->getDiffString()}" . PHP_EOL; - - sleep(3); -} - -if (is_a($diff->getTotalDiff(), 'DateInterval')) { - print "Total Script Time: {$diff->getTotalDiffString()}" . PHP_EOL; } /** @@ -793,7 +774,6 @@ Usage: php update_db.php [--cpe] [--cve] [--nvd] [--nasl] [--stig] [-u={URL}] [- --nasl To download OpenVAS NVT library and update NASL files You can also extract *.nasl files from the Nessus library to $tmp/nessus_plugins and it will include these in the update --stig To download and update the STIG library - --sunset To download and update the STIG library with the STIGs DISA has archived --do To download the files only...do not call the parsers will overwrite any existing files --po To parse the downloaded files only, do not download