From 58a9d004e36c1b9fec1565b3541ae87ba4432a3a Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Thu, 1 Mar 2018 10:07:14 -0600 Subject: [PATCH 001/110] Files generated by PkgTemplates --- .gitignore | 6 ++ .gitlab-ci.yml | 129 ++++++++++++++++++++++++++++++++++++ README.md | 5 ++ REQUIRE | 1 + docs/make.jl | 16 +++++ docs/src/assets/invenia.css | 75 +++++++++++++++++++++ docs/src/assets/logo.png | Bin 0 -> 7274 bytes docs/src/index.md | 5 ++ src/AWSBatch.jl | 6 ++ test/runtests.jl | 7 ++ 10 files changed, 250 insertions(+) create mode 100644 .gitignore create mode 100644 .gitlab-ci.yml create mode 100644 README.md create mode 100644 REQUIRE create mode 100644 docs/make.jl create mode 100644 docs/src/assets/invenia.css create mode 100644 docs/src/assets/logo.png create mode 100644 docs/src/index.md create mode 100644 src/AWSBatch.jl create mode 100644 test/runtests.jl diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7be7ea3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +.DS_Store +*.jl.cov +*.jl.*.cov +*.jl.mem +/docs/build/ +/docs/site/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..90c57a6 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,129 @@ +stages: + - test + - coverage + - docs + +.test_shell: &test_shell + artifacts: + name: "$CI_JOB_NAME coverage" + expire_in: 1 week + paths: + - "$CI_JOB_NAME coverage/" + before_script: + - curl -s -o julia-ci https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/julia-ci + - chmod +x julia-ci + - ./julia-ci install $JULIA_VERSION + script: + - source julia-ci export + # - export PYTHON="" # Configure PyCall to use the Conda.jl package's Python + - julia -e "Pkg.clone(pwd()); Pkg.build(\"$PKG_NAME\"); Pkg.test(\"$PKG_NAME\"; coverage=true)" + - ./julia-ci coverage + after_script: + - ./julia-ci clean + +.test_shell_06: &test_shell_06 + variables: + JULIA_VERSION: "0.6" + <<: *test_shell + +.test_shell_07-: &test_shell_07- + variables: + JULIA_VERSION: "0.7-" + allow_failure: true + <<: *test_shell + +.test_docker: &test_docker + artifacts: + name: "$CI_JOB_NAME coverage" + expire_in: 1 week + paths: + - "$CI_JOB_NAME coverage" + script: + # - export PYTHON="" # Configure PyCall to use the Conda.jl package's Python + - julia -e "Pkg.clone(pwd()); Pkg.build(\"$PKG_NAME\"); Pkg.test(\"$PKG_NAME\"; coverage=true)" + - julia-coverage "$PKG_NAME" + + +"0.6 (Mac)": + tags: + - mac + - shell-ci + <<: *test_shell_06 + +"0.6 (Linux, 64-bit, Docker)": + image: 292522074875.dkr.ecr.us-east-1.amazonaws.com/julia-gitlab-ci:0.6 + tags: + - linux + - 64-bit + - docker-ci + <<: *test_docker + +"0.6 (Linux, 32-bit, Shell)": + tags: + - linux + - 32-bit + - shell-ci + <<: *test_shell_06 + +"0.7- (Mac)": + tags: + - mac + - shell-ci + <<: *test_shell_07- + +"0.7- (Linux, 64-bit)": + tags: + - linux + - 64-bit + - shell-ci + <<: *test_shell_07- + +"0.7- (Linux, 32-bit)": + tags: + - linux + - 32-bit + - shell-ci + <<: *test_shell_07- + +"Coverage": + stage: coverage + image: 292522074875.dkr.ecr.us-east-1.amazonaws.com/julia-gitlab-ci:0.6 + coverage: /Test Coverage (\d+\.\d+%)/ + artifacts: + name: combined_coverage + expire_in: 1 week + paths: + - combined_coverage/ + tags: + - linux + - docker-ci + script: + - genhtml --version + - mkdir $CI_PROJECT_DIR/combined_coverage + - cp -r $CI_PROJECT_DIR/src $CI_PROJECT_DIR/combined_coverage/ + - ls */*.info | xargs -I{} echo '--summary "{}"' | xargs lcov --directory src + - echo "Test Coverage $(genhtml -o $CI_PROJECT_DIR/combined_coverage --no-prefix */coverage.info 2>&1 | grep lines | awk '{print $2}')" + - find $CI_PROJECT_DIR/combined_coverage -type f -name "*.jl" -delete + +"Documentation": + stage: docs + tags: + - docs + only: + - master + - develop + - docs + - tags # special keyword for all tags + variables: + JULIA_VERSION: "0.6" + before_script: + - curl -s -o julia-ci https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/julia-ci + - chmod +x julia-ci + - ./julia-ci install $JULIA_VERSION + script: + - source julia-ci export + - julia --depwarn=no -e "Pkg.clone(pwd()); Pkg.build(\"$PKG_NAME\"); Pkg.add(\"Documenter\")" + - julia --depwarn=no -e "cd(Pkg.dir(\"$PKG_NAME\", \"docs\")); include(\"make.jl\")" + - julia --depwarn=no -e "const DOCS_DIR = joinpath(\"/mnt\", \"docs\", \"$CI_PROJECT_NAMESPACE\", \"$CI_PROJECT_NAME\", \"$CI_BUILD_REF_NAME\"); mkpath(DOCS_DIR); cp(Pkg.dir(\"$PKG_NAME\", \"docs\", \"build\"), DOCS_DIR; remove_destination=true, follow_symlinks=true)" + after_script: + - ./julia-ci clean diff --git a/README.md b/README.md new file mode 100644 index 0000000..7a312c6 --- /dev/null +++ b/README.md @@ -0,0 +1,5 @@ +# AWSBatch + +[![Latest](https://img.shields.io/badge/docs-latest-blue.svg)](https://doc.invenia.ca/invenia/AWSBatch.jl/master) +[![Build Status](https://gitlab.invenia.ca/invenia/AWSBatch.jl/badges/master/build.svg)](https://gitlab.invenia.ca/invenia/AWSBatch.jl/commits/master) +[![Coverage](https://gitlab.invenia.ca/invenia/AWSBatch.jl/badges/master/coverage.svg)](https://gitlab.invenia.ca/invenia/AWSBatch.jl/commits/master) diff --git a/REQUIRE b/REQUIRE new file mode 100644 index 0000000..137767a --- /dev/null +++ b/REQUIRE @@ -0,0 +1 @@ +julia 0.6 diff --git a/docs/make.jl b/docs/make.jl new file mode 100644 index 0000000..490b6e0 --- /dev/null +++ b/docs/make.jl @@ -0,0 +1,16 @@ +using Documenter, AWSBatch + +makedocs(; + modules=[AWSBatch], + format=:html, + pages=[ + "Home" => "index.md", + ], + repo="https://gitlab.invenia.ca/invenia/AWSBatch.jl/blob/{commit}{path}#L{line}", + sitename="AWSBatch.jl", + authors="Nicole Epp", + assets=[ + "assets/invenia.css", + "assets/logo.png", + ], +) diff --git a/docs/src/assets/invenia.css b/docs/src/assets/invenia.css new file mode 100644 index 0000000..343c6f2 --- /dev/null +++ b/docs/src/assets/invenia.css @@ -0,0 +1,75 @@ +/* Links */ + +a { + color: #4595D1; +} + +a:hover, a:focus { + color: #194E82; +} + +/* Navigation */ + +nav.toc ul a:hover, +nav.toc ul.internal a:hover { + color: #FFFFFF; + background-color: #4595D1; +} + +nav.toc ul .toctext { + color: #FFFFFF; +} + +nav.toc { + box-shadow: none; + color: #FFFFFF; + background-color: #194E82; +} + +nav.toc li.current > .toctext { + color: #FFFFFF; + background-color: #4595D1; + border-top-width: 0px; + border-bottom-width: 0px; +} + +nav.toc ul.internal a { + color: #194E82; + background-color: #FFFFFF; +} + +/* Text */ + +article#docs a.nav-anchor { + color: #194E82; +} + +article#docs blockquote { + font-style: italic; +} + +/* Code */ + +code .hljs-meta { + color: #4595D1; +} + +code .hljs-keyword { + color: #194E82; +} + +pre, code { + font-family: "Liberation Mono", "Consolas", "DejaVu Sans Mono", "Ubuntu Mono", "Courier New", "andale mono", "lucida console", monospace; +} + +/* mkdocs (old) */ + +/*.navbar-default { + background-color: #194E82; +} + +.navbar-default .navbar-nav > .active > a, +.navbar-default .navbar-nav > .active > a:hover, +.navbar-default .navbar-nav > .active > a:focus { + background-color: #4595D1; +}*/ diff --git a/docs/src/assets/logo.png b/docs/src/assets/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..214b5c9a49c2bb5a0db1de74ea83b25b56d2051e GIT binary patch literal 7274 zcmYj$cRbc@^#2F9qC$8gTisR}MfNV6kd!^sZFbv)%y^JpA=x8JX2wmKN%l<0CNq0v zC48^z_WXX|??2`Bajxr}^FHr$u5*rOcQlm9&d{Gh5QI!cS^h495M#m5lO(6Wza9bd zRNy~SM`e8%1fc~#(^3&wrF{cZ2%W2fo~yQlrR!sJXAE-lp~C|Vs|wcK3Ue1@{?OB@ z6(flt47)1wH}822FON;Rjp%$kHJP+FQ#Ts$Jzy-L_<|l&0+Sw7&@WlT&)=9@F7S*6 z^bzz27=8AwRP&q`&6L+dJgru@k^{$HHpcrCaMfHiIp??E{jTTzwQ1q%O*_##?12KQ z$wMjoS0;nDHDBhPsx}2www_GxIvIBTm0BD;?p&i9^x&K?I$GVYV0nh>=G}ajHYPPg zUB^&yw6UIOWnkL8g3X-%$WRfU&$)DLRD`f?)HYg&5+R$U^&gC5jmSn8y+_9}Qs#*` zVwfI&>%vwr;IZ(+( zfK@$7%!+7l-RK+lr*-;h{Fn1B36L4;bfxgFDBV69gwRvQK2QKJ_?o+!;T3R5O7AKG zRLu0t23YQ9?~&M406$mq9M-g-L(0-LPhk*C=doW-cEg0S*Dad~kOKWH1lM13H!~#x z!J2o`r+8xtpoufK)0_Ey7MeFM-vqi-zxU%Dz%Q&@?n)DIxXvBF_6~3N_0Jdo4rNr zG(g+yxA`a^{$izlRDypF_Jue=yC*LjaCMV3CbXBt?~DxI1e5-L+=;s}LHBWf<)PJE z0Kwmna#4uCHzC^{nT|VsJ4NHK>RC450e#(k;r%@DH;q<;4XdAm-hgjwI;sfL!YS*h zGsKY>4*iw5_SuSw0AgJ(PB>X3AbE8;6a~!s(G~S1>l{3edKiao_ycUJcNc0e$HRR- zCg;CTI{!bS$`OhlCM}etz@*~)GB$3xCx;)zVZVt3Qq-qOM~(b@aylsIzi+0zaR`JO ziBSAuT8TfquFKMdy@Mwdy;U6crx!ds@t<6iG56+?l%fAl*EA#U6>sxfuGb>f=H`kT@cpL4xoH2!@D zAykb0HB{i&E_`9VBt_dk@(nq&Dc{S8{pWocE0PBMa*15c3Q(+Rvhxv&358K2yld$j zBuH`ENsJvI=={*M=>0=(3taV;ju=VDLzmoARdRU!mdV;dXpkjD;V2#-y!vN6v1WJG zqZ508f44M(D~SSyc8RyZk!z)Q39XwX*a5uhS)&$$4P}YUcn_~> z2Yx%rW6{e+8(oX

PEcbNeX(+oT6DusO4z36R_WRbHNcVceza1GeU=Q%r!)&Nt}I z$J0{OqeelR`hNj5?=1OI%W-C-hTf=Ru@mk6Aw>}gm_asq3*e$sRRLIwinUEfdWClg|yPwHaMEL zPqU`^SNBD2e>qk%##UL|SOFn3dTw@-D{6iD6-k@nCC28T}77>I^ptvWW(ZCt6IileBQwQ}6QubVptmgwFSaOAI@oqz6+7&!=FB5ifMcR!{Bk16 zHdN@l{)0nhz$RxEfP1~bOZ%aa8qX42Gry~I?V?la$5jwgWu8szptVKSkLaAF(=pJZ!u4uhrJY~P<&{-vbQT8vwQ)1@5Q3V z(pk!5PtBx8fu<@nazRe-t4jx*Uwpr}8?aMRm@~F*`P*PzG8tq`hea;R{E=#hW`$2j z%wHqp*cG&0Z$b4k;Lndse@@4&984Iv4>Wi}_Ii4A9d=yoWNmM2-fiuE*-L^Hb#t!8 z63nEaXBQ5f4wfYu6#=eurT3m**LmtoBKOqdIz54CIRm@Dr2<3bor0H^EWjP*UAkWttq{yvGb=OWzvaZK0JyAl3;XW#n+DWjXcF2<5uNE)n z035wFU%}S=!b!d%X7Zwri6tN+TdRZH_OfNSOclA8hKBfw&c~}wToF9nmr8}O6H?Vt zmv7J$fKcDw#~wbFVc6e71CUuiE|vJqJi`G()T+DJkQMR*#HgycHWbnM;;pwpfMZ67 zzC==DZ(RQTq0HHW2qczFCMHELl{IX*tU!j7_U&F1%rVKjxRt7fvd+*T-1iG+uRceA zwnhOgQcFm1bxTCY8sEQCa<=wENh39OYm@k#m6#dPLL+UEz-;bXxU2%N7GbR$T1>`H zzqJJLVk~Vta&guYL6#Sv-(i`hr(#qJw$Ss`C)gw{mTMGw_rz{Ebw(0asEskMo5QS& z9QFKvJN{Q`n(4;lL4jXKFxmDNy}NGGCFaL`ar@h)w4S}#ezkEe0pZfw*^ zlr+BB#_xQw4yGiXXWt2Ff2en*defP3hMHAtKwmSaG^I!iCWp=U$KPq&dIa;ZD+hLT zk)T4?YiHJsA6=~I7$V;l73j$hqsGI-6^(-eeWv`T=SCKJoE0*wz#e z;{s(Rsk10Sm^|7tryp4qt}sIv%et8DBdq(xTb<+|`ICRPvlk=;>Z+=_NA9oG%>7g! z5)MF%4{JZF^GZSLTMt2@B*DgJij{Bd6rq~=Q6m$>52eqVY{k8bw)Bn(E=v;&x%Ii<|0#^6oh4B>oTfz5$ynNOxr#^ri*u~=k++-dPJzecmjYwtx@T~a z{={U|pNzU!)9(DcBE3MB`zY7Vmx9>nrOT6%9o+*a!hqz<#W5k;Gs{DzLF*l%G z^6f^@;l%Ds$6R-|PxQGN1TWle_sclf$$xoDbUb#Ce2ltE3jqvd+W; z@YJVE0s3mhQ<4TACdKuVz&8Lx>6qpa=yzB=uMIMN*k!fpU3bNNHxky^rGJpGcNg#N}$1 z9Zpprr2x?(BTliXQu*c=3P5kc_;vZeW>GA7;>k9i+lk$NUfX?j-P5>Z`(I#SOOCL5 zEt^>UL|aFia@);8UDx?Y6Wu^N@N|H(eD$YwkVWk!tb7B^{ub+Bdgr;)YMiV)8_Ee< zIHQPPl(g;GBt4LCm;k87Xpvn^d&$@yQsVO{S@`1f>(SyY8`w!NddPX%H_PSG7FJ1a zlUr2@)8U|th`FERX(ys#HAvu2{wlkG;>Aa2b!79IH=zm$vSXNc=^5`RD}2KvtCQq( z5Idu18;EsOts*OSp<52OBZLMf7&nT%U3&NQ0PWXZH0@5iC#iXqGg`#nef@Vtt;V`g zvT3^2Ap5Gshzw{GB!hnhuc4{NIV%GyvDT)U0{Hs&NX*JDbFGSRWP@9)d+SZ)$Z#1` zweMXQ-+%$RmTORZ5mBYp?S|8OZwnRPm{+;^B zR*2Do+xu(Ql7z@me^RD{N)+aot1wzFYc;>@gShRK0eI*3Q7^(O+i+1_dtRAqJ`8`Y zrr&(T4-Jd9@jNVloB$VcmlmT@%^$4ISc2=f@g;bM`bW0!f?XqLsYVKRAfS8yjk*_j zP`rw-eh3*pUifnf`PSriVa)O8nx7UfdO$JT958W{beYP{^4OqO{KqaO=}YjCrPJ!` zlqR!_$1ey7F57^3SFRBu3X@4U?A+FwTrpeD`|Cq{*Js5KKosN%V*`yI6cX)p205 zfF9Iur1X4#$o2;zq|%UphTUs-rak(Ehxja#dY`9X%Pt1mQH^VIL~B1pXlW_T2gFXd zyw;4@;7qBDM;nOXWb!5lKmRN@g<}$Pf3t6ipu|M{GfNJqV)bV-m(5EkZvW7pxsOvs zBb%*Te6!TzZW}S(Szx{`id^X%3^@#2?3W<|skMNgB>&CL2hC8-?xI+QUcQ# zjO`AX19BO{*^FSei7#TOW?s4*wR5^Yz+2;>Qv~u?b}c^YU(!dae)o|uv3G3&1Tc+x0A=RzeMaFzor5>aTHrB{qz67)k1&Z9IA?K94mk^luB51cUCx4s_?FfUjacu}Ren$UeSj_~vY z;|Y|(>5=(`v*<1pubRJ$+r)@p>D7^@f}}GhE@VFEe9yTF)vpAZa=$Nc|9GX2n*h>1 zayC2Eo@i~id2+Dbc0wi}3>nde5hHbR0YpBgr04gWyxMYB&R3G_4C64lD45BDmqFw0 zHhCB|T2`Q0T61OOEoUkQRIp`olxb{$Zs__Bpr+@=yU%aYpbA~g&*7e91e2+HQp}hIa ze1Nntt%gPjc2#}C(>t2rvb@OyQ>)%u&#w~|Z8V@mh{&`|t_n$GdEPJ5FSF}-2xYh8 z*^lal4`iah@zBEUS|`S$;z9f}!NbHyWLJ@HLGYeQ%EDUe3;b*S zuwA6T+37&@>58e8krIeqahjL5>)UgTW*5AVLx_&Y6*dT%K!-5nl8s`V7KbC};rD(X zHOwT^tHk!b`8c@C-Wn7?FjvGmbj1dcN=o3H?VXCNI;D;Y%t~w*Bf&!=E37l*T(+bF zP%^R1KrwpAjrEbK8Vfnn{njfe0V!hk{x<6L@$vw;|CN692rXvv-^^&`ccnJhOxEW^ z>YKrU44|}#P&BiBrHO%2@%Wfc<>;TP_{71LQ@nCVh?WDa$gCWm6dH zsm1rz446q``@jihiYKGrtI53W;gD~oGbB)<^xwGsJtshE|NgxOo84_4u!z~A1sjk4fa8K~7uf3R zmym7z#o`nEwnYZ2T6TtGD(gl?n|s`fe*H8*zBMow#EK1V!~6U6Am(P8y}It?&+KjU zVMr$V7155524#){L6N;}m5W-wG94OpY5gNy7Wr(FcjWpmtbN_ZqxmbpdF7=*@!OWp zhN-uHg;Sf@`e`sELJ|TUzue$5q|e|g!IQAjGO2@Kik6P#a4K%JMrkO3kXe?{RZPSM zNXfH2zb;9`N0dtBZ?f4c6GuSaCgyni73LrZ7W$6KoJRX5C&}kY!7{?O=t6nm#X{#@ z>3~3Ia>kK6$4cO)99@Q+B5n1|mqoC3es`YU$AMryyun5AwiZr4X3XtQRobUgDFIaX z`MYUYUoh;WQr)z04jNx_tMiVtfSFtRz|2tT%a?wt8fH=Yn~b${uIJw*75yMS2>JlT zvW&h$CN{uI-X5$BJPYT*{cHkU1iVHeAbGAA*|AU3fy9t65QD9-Z>~idAfvPG5TZOqTrJ+JDBJf+| z@v8&{Fj1xrqDE2#iUa#`TVO{EygL6ch(+>T6A5#cO&-J-0Zup)=9T!|WAL?I(7EXtt$cwvcZqJ2~5cm)jSI zd90*IC!P{V1u7}h3;nR+jq_Ks9)8abOj%yqbvpREQ?bJGT=PA&YGCft!_UzX$_c?> ztFTEbs|-!^5}Dpnr`vmWA+xCBE+GJ)HPaNVJT+F2rkWN+vL{cM+ZDA)pv*)XT>g0o6Lo*6kUVHj5Zg}u7aDAuyjH;*gIkCor5&dJsRUhA*Z_5KJd5NGBP+D**;zP(co1 zq=d0eD~~ag0nLZ~XQ1v-sC&tBwDt%BO40ax?I|y(IstbEtmqM^f~J*l0oNtFUr&VKMhqx2fQm-$l71T4QtgZDqX1Yw%m$(# z3c!Y6%FxYw`9FN1@!n_I7j%lyREzu$$-Xe)+#ukg4mSXyGYl)D=-?+>@TtXI%zh#^ zLyf!n95fvMF<_>cu7UxxaMuA%waR=V$qOa3;QssDTf^ovP%;EcI^kVRi?bjG$hN?> ztaEOj{%UN7L`{S;D8G;daJHTSt2@3iarmV8GeQv3`YV03g890uEZ|Z~8Lmy84X)mN zToV2Njs?SGWqdlNzZV8RfCES) Date: Mon, 5 Mar 2018 14:19:36 -0600 Subject: [PATCH 002/110] Initial AWSBatch.jl implementation --- REQUIRE | 3 + docs/src/index.md | 97 ++++++++++ src/AWSBatch.jl | 479 +++++++++++++++++++++++++++++++++++++++++++++- test/mock.jl | 102 ++++++++++ test/runtests.jl | 210 +++++++++++++++++++- 5 files changed, 888 insertions(+), 3 deletions(-) create mode 100644 test/mock.jl diff --git a/REQUIRE b/REQUIRE index 137767a..de13dad 100644 --- a/REQUIRE +++ b/REQUIRE @@ -1 +1,4 @@ julia 0.6 +AWSSDK 0.2.0 +Mocking 0.3.3 +Memento 0.5.0 diff --git a/docs/src/index.md b/docs/src/index.md index 7a312c6..e21d344 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -3,3 +3,100 @@ [![Latest](https://img.shields.io/badge/docs-latest-blue.svg)](https://doc.invenia.ca/invenia/AWSBatch.jl/master) [![Build Status](https://gitlab.invenia.ca/invenia/AWSBatch.jl/badges/master/build.svg)](https://gitlab.invenia.ca/invenia/AWSBatch.jl/commits/master) [![Coverage](https://gitlab.invenia.ca/invenia/AWSBatch.jl/badges/master/coverage.svg)](https://gitlab.invenia.ca/invenia/AWSBatch.jl/commits/master) + +AWSBatch.jl provides a small set of methods for working with AWS Batch jobs from julia. + +## Installation + +AWSBatch assumes that you already have an AWS account configured with: + +1. An [ECR repository](https://aws.amazon.com/ecr/) and a docker image pushed to it [[1]](http://docs.aws.amazon.com/AmazonECR/latest/userguide/docker-push-ecr-image.html). +2. An [IAM role](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html) to apply to the batch jobs. +3. A compute environment and job queue for submitting jobs to [[2]](http://docs.aws.amazon.com/batch/latest/userguide/Batch_GetStarted.html#first-run-step-2). + +Please review the +["Getting Started with AWS Batch"](http://docs.aws.amazon.com/batch/latest/userguide/Batch_GetStarted.html) guide and example +[CloudFormation template](https://s3-us-west-2.amazonaws.com/cloudformation-templates-us-west-2/Managed_EC2_Batch_Environment.template) for more details. + +## Basic Usage + +```julia +julia> using AWSBatch + +julia> using Memento + +julia> Memento.config("info"; fmt="[{level} | {name}]: {msg}") +Logger(root) + +julia> job = BatchJob( + name="Demo", + definition="AWSBatchJobDefinition", + queue="AWSBatchJobQueue", + container=Dict( + "image" => "000000000000.dkr.ecr.us-east-1.amazonaws.com/demo:latest", + "role" => "arn:aws:iam::000000000000:role/AWSBatchJobRole", + "vcpus" => 1, + "memory" => 1024, + "cmd" => `julia -e 'println("Hello World!")'`, + ), + ) +AWSBatch.BatchJob("", "Demo", AWSBatch.BatchJobDefinition("AWSBatchJobDefinition"), "AWSBatchJobQueue", "", AWSBatch.BatchJobContainer("000000000000.dkr.ecr.us-east-1.amazonaws.com/demo:latest", 1, 1024, "arn:aws:iam::000000000000:role/AWSBatchJobRole", `julia -e 'println("Hello World!")'`)) + +julia> submit(job) +[info | AWSBatch]: Registered job definition arn:aws:batch:us-east-1:000000000000:job-definition/AWSBatchJobDefinition:1. +[info | AWSBatch]: Submitted job Demo::00000000-0000-0000-0000-000000000000. +Dict{String,Any} with 2 entries: + "jobId" => "00000000-0000-0000-0000-000000000000" + "jobName" => "Demo" + +julia> wait(job, [AWSBatch.SUCCEEDED]) +[info | AWSBatch]: Demo::00000000-0000-0000-0000-000000000000 status SUBMITTED +[info | AWSBatch]: Demo::00000000-0000-0000-0000-000000000000 status STARTING +[info | AWSBatch]: Demo::00000000-0000-0000-0000-000000000000 status SUCCEEDED +true + +julia> results = logs(job) +[info | AWSBatch]: Fetching log events from Demo/default/00000000-0000-0000-0000-000000000000 +[info | AWSBatch]: Hello World! +1-element Array{Any,1}: + Dict{String,Any}(Pair{String,Any}("ingestionTime", 1505846649863),Pair{String,Any}("message", "Hello World!"),Pair{String,Any}("timestamp", 1505846649786),Pair{String,Any}("eventId", "00000000000000000000000000000000000000000000000000000000")) +``` + +## Public API + +### BatchJob + +```@docs +AWSBatch.BatchJob +AWSBatch.BatchJob() +AWSBatch.BatchStatus +AWSBatch.isregistered(::BatchJob) +AWSBatch.register!(::BatchJob) +AWSBatch.deregister!(::BatchJob) +AWSBatch.describe(::BatchJob) +AWSBatch.submit(::BatchJob) +Base.wait(::BatchJob, ::Vector{BatchStatus}, ::Vector{BatchStatus}) +AWSBatch.logs(::BatchJob) +``` + +### BatchJobDefinition + +```@docs +AWSBatch.BatchJobDefinition +AWSBatch.describe(::BatchJobDefinition) +AWSBatch.isregistered(::BatchJobDefinition) +``` + +### BatchJobContainer + +```@docs +AWSBatch.BatchJobContainer +AWSBatch.BatchJobContainer(::Associative) +``` + +## Private API + +```@docs +AWSBatch.lookupARN(::BatchJob) +AWSBatch.update!(::BatchJobContainer, ::Associative) +``` diff --git a/src/AWSBatch.jl b/src/AWSBatch.jl index 3ceac30..6eab4d7 100644 --- a/src/AWSBatch.jl +++ b/src/AWSBatch.jl @@ -1,6 +1,483 @@ __precompile__() module AWSBatch -# Package code goes here. +using AWSSDK +using AWSSDK.Batch +using AWSSDK.CloudWatchLogs +using AWSSDK.S3 +using Memento +using Mocking + +import AWSSDK.Batch: + describe_job_definitions, describe_jobs, register_job_definition, + deregister_job_definition, submit_job + +import AWSSDK.CloudWatchLogs: get_log_events + +export + BatchJob, + BatchJobDefinition, + BatchJobContainer, + BatchStatus, + S3Results, + isregistered, + register!, + deregister!, + submit, + describe, + wait, + logs + + +const logger = getlogger(current_module()) +# Register the module level logger at runtime so that folks can access the logger via `getlogger(MyModule)` +# NOTE: If this line is not included then the precompiled `MyModule.logger` won't be registered at runtime. +__init__() = Memento.register(logger) + + +##################### +# BatchStatus +##################### + +@enum BatchStatus SUBMITTED PENDING RUNNABLE STARTING RUNNING SUCCEEDED FAILED UNKNOWN +const global _status_strs = map(s -> string(s) => s, instances(BatchStatus)) |> Dict +status(x::String) = _status_strs[x] + +@doc """ + BatchStatus + +An enum for representing different possible batch job states. + +See [docs](http://docs.aws.amazon.com/batch/latest/userguide/job_states.html) for details. +""" BatchStatus + +################################## +# BatchJobContainer +################################## + +""" + BatchJob + +Stores configuration information about a batch job's container properties. + +# Fields +- image::String: the ECR container image to use for the ECS task +- vcpus::Int: # of cpus available in the ECS task container +- memory::Int: memory allocated to the ECS task container (in MB) +- role::String: IAM role to apply to the ECS task +- cmd::Cmd: command to execute in the batch job +""" +mutable struct BatchJobContainer + image::String + vcpus::Int + memory::Int + role::String + cmd::Cmd +end + +""" + BatchJobContainer(container::T) + +Creates a BatchJobContainer from the values passed in the `container` dictionary. Uses +default values if the field is not specified. +""" +function BatchJobContainer(container::Associative) + BatchJobContainer( + get(container, "image", ""), + get(container, "vcpus", 1), + get(container, "memory", 1024), + get(container, "role", ""), + get(container, "cmd", ``), + ) +end + +""" + update!(container::BatchJobContainer, container_details::Associative) + +Updates the BatchJobContainer default values with the AWS response dict. +""" +function update!(container::BatchJobContainer, details::Associative) + # Only update fields that are still using the default values since explict arguments + # passed in have priority over aws job definition parameters + isempty(container.image) && (container.image = details["image"]) + container.vcpus == 1 && (container.vcpus = details["vcpus"]) + container.memory == 1024 && (container.memory = details["memory"]) + isempty(container.role) && (container.role = details["jobRoleArn"]) + isempty(container.cmd) && (container.cmd = Cmd(Vector{String}(details["command"]))) +end + +################################## +# BatchJobDefinition +################################## + +""" + BatchJobDefinition + +Stores the job definition name or arn including the revision. +""" +mutable struct BatchJobDefinition + name::AbstractString end + +""" + describe(definition::BatchJobDefinition) + +Describes a job given it's definition. Returns the response dictionary. +Requires permissions to access "batch:DescribeJobDefinitions". +""" +function describe(definition::BatchJobDefinition) + if isempty(definition.name) + return Dict("jobDefinitions" => []) + elseif startswith(definition.name, "arn:") + return describe_job_definitions(Dict("jobDefinitions" => [definition.name])) + else + return @mock describe_job_definitions(Dict("jobDefinitionName" => definition.name)) + end +end + +""" + isregistered(definition::BatchJobDefinition) + +Checks if a BatchJobDefinition is registered. +""" +function isregistered(definition::BatchJobDefinition) + j = describe(definition) + active_definitions = filter!(d -> d["status"] == "ACTIVE", get(j, "jobDefinitions", [])) + return !isempty(active_definitions) +end + +################################## +# BatchJob +################################## + +""" + BatchJob + +Stores configuration information about a batch job in order to: + +- `register` a new job definition +- `submit` a new job to batch +- `describe` a batch job +- check if a batch job definition `isregistered` +- `deregister` a job definition +- `wait` for a job to complete +- fetch `logs` + +# Fields +- id::String: jobId +- name:String: jobName +- definition::BatchJobDefinition: job definition +- queue::String: queue to insert the batch job into +- region::String: AWS region to use +- container::BatchJobContainer: job container properties (image, vcpus, memory, role, cmd) +""" +mutable struct BatchJob + id::String + name::String + definition::BatchJobDefinition + queue::String + region::String + container::BatchJobContainer +end + +""" + BatchJob(; + id="", + name="", + queue ="", + region="", + definition=BatchJobDefinition(""), + container=BatchJobContainer(Dict()) + ) + +Handles creating a BatchJob based on various potential defaults. +For example, default job fields can be inferred from an existing job defintion or existing +job (if currently running in a batch job) + +Order of priority from lowest to highest: + +1. Job definition parameters +2. Inferred environment (e.g., `AWS_BATCH_JOB_ID` environment variable set) +3. Explict arguments passed in via `kwargs`. +""" +function BatchJob(; + id="", + name="", + queue="", + region="", + definition=BatchJobDefinition(""), + container=BatchJobContainer(Dict()) +) + if !isa(definition, BatchJobDefinition) + definition = BatchJobDefinition(definition) + end + + if !isa(container, BatchJobContainer) + container = BatchJobContainer(container) + end + + # Determine if the job definition already exists and update it + resp = describe(definition) + if !isempty(resp["jobDefinitions"]) + details = first(resp["jobDefinitions"]) + + update!(container, details["containerProperties"]) + end + + if haskey(ENV, "AWS_BATCH_JOB_ID") + # Environmental variables set by the AWS Batch service. They were discovered by + # inspecting the running AWS Batch job in the ECS task interface. + job_id = ENV["AWS_BATCH_JOB_ID"] + job_queue = ENV["AWS_BATCH_JQ_NAME"] + + # Get the zone information from the EC2 instance metadata. + zone = @mock readstring( + pipeline( + `curl http://169.254.169.254/latest/meta-data/placement/availability-zone`; + stderr=DevNull + ) + ) + job_region = chop(zone) + + # Requires permissions to access to "batch:DescribeJobs" + resp = @mock describe_jobs(Dict("jobs" => [job_id])) + + if length(resp["jobs"]) > 0 + details = first(resp["jobs"]) + + isempty(id) && (id = job_id) + isempty(name) && (name = details["jobName"]) + isempty(queue) && (queue = job_queue) + isempty(region) && (region = job_region) + if isempty(definition.name) + definition = BatchJobDefinition(details["jobDefinition"]) + end + + update!(container, details["container"]) + else + warn(logger, "No jobs found with id: $job_id.") + end + end + + return BatchJob(id, name, definition, queue, region, container) +end + +""" + isregistered(job::BatchJob) -> Bool + +Checks if a job is registered. +""" +isregistered(job::BatchJob) = isregistered(job.definition) + +""" + register!(job::BatchJob) + +Registers a new job definition. If no job definition exists, a new job definition is created +under the current job specifications, where the new job definition will be `job.name`. +""" +function register!(job::BatchJob) + isempty(job.definition.name) && (job.definition.name = job.name) + + debug(logger, "Registering job definition $(job.definition.name).") + input = [ + "type" => "container", + "containerProperties" => [ + "image" => job.container.image, + "vcpus" => job.container.vcpus, + "memory" => job.container.memory, + "command" => job.container.cmd.exec, + "jobRoleArn" => job.container.role, + ], + "jobDefinitionName" => job.definition.name, + ] + + resp = register_job_definition(input) + job.definition.name = resp["jobDefinitionArn"] + info(logger, "Registered job definition $(job.definition.name).") +end + +""" + deregister!(job::BatchJob) + +Deregisters an AWS Batch job. +""" +function deregister!(job::BatchJob) + isempty(job.definition.name) && (job.definition.name = job.name) + debug(logger, "Deregistering job definition $(job.definition.name).") + resp = deregister_job_definition(Dict("jobDefinition" => job.definition.name)) + info(logger, "Deregistered job definition $(job.definition.name).") +end + +""" + submit(job::BatchJob) -> Dict + +Handles submitting the batch job and registering a new job definition if necessary. +If no valid job definition exists (see `AWSBatch.lookupARN`) then a new job definition will be +created. Once the job has been submitted this function will return the response dictionary. +""" +function submit(job::BatchJob) + job.definition.name = lookupARN(job) + + if isempty(job.definition.name) + register!(job) + end + + debug(logger, "Submitting job $(job.name).") + input = [ + "jobName" => job.name, + "jobDefinition" => job.definition.name, + "jobQueue" => job.queue, + "containerOverrides" => [ + "vcpus" => job.container.vcpus, + "memory" => job.container.memory, + "command" => job.container.cmd.exec, + ] + ] + debug(logger, "Input: $input") + resp = submit_job(input) + + job.id = resp["jobId"] + info(logger, "Submitted job $(job.name)::$(job.id).") + + return resp +end + +""" + describe(job::BatchJob) -> Dict + +If job.id is set then this function is simply responsible for fetch a dictionary for +describing the batch job. +""" +function describe(job::BatchJob) + isempty(job.id) && error(logger, ArgumentError("job.id is not set")) + resp = describe_jobs(; jobs=[job.id]) + isempty(resp["jobs"]) && error(logger, "Job $(job.name)::$(job.id) not found.") + debug(logger, "Job $(job.name)::$(job.id): $resp") + return first(resp["jobs"]) +end + +""" + lookupARN(job::BatchJob) -> String + +Looks up the ARN for the latest job definition that can be reused for the current `BatchJob`. +A job definition can only be reused if: + +1. status = ACTIVE +2. type = container +3. image = job.container.image +4. jobRoleArn = job.container.role +""" +function lookupARN(job::BatchJob) + resp = describe(job.definition) + + isempty(resp["jobDefinitions"]) && return "" + latest = first(resp["jobDefinitions"]) + + for definition in resp["jobDefinitions"] + if definition["status"] == "ACTIVE" && definition["revision"] > latest["revision"] + latest = definition + end + end + + if ( + latest["status"] == "ACTIVE" && + latest["type"] == "container" && + latest["containerProperties"]["image"] == job.container.image && + latest["containerProperties"]["jobRoleArn"] == job.container.role + ) + return latest["jobDefinitionArn"] + else + return "" + end +end + +""" + wait( + job::BatchJob, + cond::Vector{BatchStatus}=[RUNNING, SUCCEEDED], + failure::Vector{BatchStatus}=[FAILED]; + timeout=600, + delay=5 + ) + +Polls the batch job state until it hits one of the conditions in `cond`. +The loop will exit if it hits a `failure` condition and will not catch any excpetions. +The polling interval can be controlled with `delay` and `timeout` provides a maximum +polling time. +""" +function Base.wait( + job::BatchJob, + cond::Vector{BatchStatus}=[RUNNING, SUCCEEDED], + failure::Vector{BatchStatus}=[FAILED]; + timeout=600, + delay=5 +) + time = 0 + completed = false + last_state = UNKNOWN + + tic() + while time < timeout + j = describe(job) + + time += toq() + state = status(j["status"]) + + if state != last_state + info(logger, "$(job.name)::$(job.id) status $state") + end + + last_state = state + + if state in cond + completed = true + break + elseif state in failure + error(logger, "Job $(job.name)::$(job.id) hit failure condition $state.") + else + tic() + sleep(delay) + end + end + + if !completed + error( + logger, + "Waiting on job $(job.name)::$(job.id) timed out. Last known state $last_state." + ) + end + return completed +end + +""" + logs(job::BatchJob) -> Vector{String} + +Fetches the logStreamName, fetches the CloudWatch logs and returns a vector of messages. + +NOTES: +- The `logStreamName`` isn't available until the job is RUNNING, so you may want to use `wait(job)` or + `wait(job, [AWSBatch.SUCCEEDED])` prior to calling `logs`. +- We do not support pagination, so this function is limited to 10,000 log messages by default. +""" +function logs(job::BatchJob) + container_details = describe(job)["container"] + events = Vector{String}() + + if "logStreamName" in keys(container_details) + stream = container_details["logStreamName"] + + info(logger, "Fetching log events from $stream") + events = get_log_events(; logGroupName="/aws/batch/job", logStreamName=stream)["events"] + + for event in events + info(logger, event["message"]) + end + else + info(logger, "No log events found for job $(job.name)::$(job.id).") + end + + return events +end + +end # AWSBatch diff --git a/test/mock.jl b/test/mock.jl new file mode 100644 index 0000000..d31bf43 --- /dev/null +++ b/test/mock.jl @@ -0,0 +1,102 @@ +import Base: AbstractCmd, CmdRedirect + +const BATCH_ENVS = ( + "AWS_BATCH_JOB_ID" => "24fa2d7a-64c4-49d2-8b47-f8da4fbde8e9", + "AWS_BATCH_JQ_NAME" => "HighPriority" +) + +const DESCRIBE_JOBS_DEF_RESP = Dict( + "jobDefinitions" => [ + Dict( + "type" => "container", + "containerProperties" => Dict( + "command" => [ + "sleep", + "60" + ], + "environment" => [ + + ], + "image" => "busybox", + "memory" => 128, + "mountPoints" => [ + + ], + "ulimits" => [ + + ], + "vcpus" => 1, + "volumes" => [ + + ], + "jobRoleArn" => "arn:aws:iam::012345678910:role/sleep60", + ), + "jobDefinitionArn" => "arn:aws:batch:us-east-1:012345678910:job-definition/sleep60:1", + "jobDefinitionName" => "sleep60", + "revision" => 1, + "status" => "ACTIVE" + ) + ] +) + +const DESCRIBE_JOBS_RESP = Dict( + "jobs" => [ + Dict( + "container" => Dict( + "command" => [ + "sleep", + "60" + ], + "containerInstanceArn" => "arn:aws:ecs:us-east-1:012345678910:container-instance/5406d7cd-58bd-4b8f-9936-48d7c6b1526c", + "environment" => [ + + ], + "exitCode" => 0, + "image" => "busybox", + "memory" => 128, + "mountPoints" => [ + + ], + "ulimits" => [ + + ], + "vcpus" => 1, + "volumes" => [ + + ], + "jobRoleArn" => "arn:aws:iam::012345678910:role/sleep60", + ), + "createdAt" => 1480460782010, + "dependsOn" => [ + + ], + "jobDefinition" => "sleep60", + "jobId" => "24fa2d7a-64c4-49d2-8b47-f8da4fbde8e9", + "jobName" => "example", + "jobQueue" => "arn:aws:batch:us-east-1:012345678910:job-queue/HighPriority", + "parameters" => Dict( + + ), + "startedAt" => 1480460816500, + "status" => "SUCCEEDED", + "stoppedAt" => 1480460880699 + ) + ] +) + + +""" + Mock.readstring(cmd::CmdRedirect, pass::Bool=true) + +Mocks the CmdRedirect produced from ``pipeline(`curl http://169.254.169.254/latest/meta-data/placement/availability-zone`)`` +to just return "us-east-1". +""" +function mock_readstring(cmd::CmdRedirect) + cmd_exec = cmd.cmd.exec + + result = if cmd_exec[1] == "curl" && contains(cmd_exec[2], "availability-zone") + return "us-east-1" + else + return Base.readstring(cmd) + end +end diff --git a/test/runtests.jl b/test/runtests.jl index 8136490..7ebe4ca 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,7 +1,213 @@ +using Mocking +Mocking.enable() + using AWSBatch using Base.Test +using Memento +using AWSSDK + +import AWSSDK.Batch: describe_job_definitions + +const PKG_DIR = abspath(dirname(@__FILE__), "..") +const REV = cd(() -> readchomp(`git rev-parse HEAD`), PKG_DIR) +const IMAGE_DEFINITION = "292522074875.dkr.ecr.us-east-1.amazonaws.com/aws-tools:latest" +const JOB_ROLE = "arn:aws:iam::292522074875:role/AWSBatchClusterManagerJobRole" +const JOB_DEFINITION = "AWSBatch" +const JOB_NAME = "AWSBatchTest" +const JOB_QUEUE = "Replatforming-Manager" + +Memento.config("debug"; fmt="[{level} | {name}]: {msg}") +setlevel!(getlogger(AWSBatch), "info") + +include("mock.jl") @testset "AWSBatch.jl" begin - # Write your own tests here. - @test 1 == 2 + @testset "Job Construction" begin + @testset "Defaults" begin + job = BatchJob() + + @test isempty(job.id) + @test isempty(job.name) + @test isempty(job.queue) + @test isempty(job.region) + + @test isempty(job.definition.name) + + @test isempty(job.container.image) + @test job.container.vcpus == 1 + @test job.container.memory == 1024 + @test isempty(job.container.role) + @test isempty(job.container.cmd) + end + + @testset "From Job Definition" begin + patch = @patch describe_job_definitions(args...) = DESCRIBE_JOBS_DEF_RESP + + apply(patch; debug=true) do + job = BatchJob(name=JOB_NAME, definition="sleep60") + + @test isempty(job.id) + @test job.name == "AWSBatchTest" + @test isempty(job.queue) + @test isempty(job.region) + + @test job.definition.name == "sleep60" + + @test job.container.image == "busybox" + @test job.container.vcpus == 1 + @test job.container.memory == 128 + @test job.container.role == "arn:aws:iam::012345678910:role/sleep60" + @test job.container.cmd == `sleep 60` + end + end + + @testset "From Current Job" begin + withenv(BATCH_ENVS...) do + patches = [ + @patch readstring(cmd::AbstractCmd) = mock_readstring(cmd) + @patch describe_jobs(args...) = DESCRIBE_JOBS_RESP + ] + + apply(patches; debug=true) do + job = BatchJob() + + @test job.id == "24fa2d7a-64c4-49d2-8b47-f8da4fbde8e9" + @test job.name == "example" + @test job.queue == "HighPriority" + @test job.region == "us-east-" + + @test job.definition.name == "sleep60" + + + @test job.container.image == "busybox" + @test job.container.vcpus == 1 + @test job.container.memory == 128 + @test job.container.role == "arn:aws:iam::012345678910:role/sleep60" + @test job.container.cmd == `sleep 60` + end + end + end + + @testset "From Multiple" begin + withenv(BATCH_ENVS...) do + patches = [ + @patch readstring(cmd::AbstractCmd) = mock_readstring(cmd) + @patch describe_jobs(args...) = DESCRIBE_JOBS_RESP + ] + + apply(patches; debug=true) do + job = BatchJob() + + @test job.id == "24fa2d7a-64c4-49d2-8b47-f8da4fbde8e9" + @test job.name == "example" + @test job.definition.name == "sleep60" + @test job.queue == "HighPriority" + @test job.region == "us-east-" + + @test job.container.image == "busybox" + @test job.container.vcpus == 1 + @test job.container.memory == 128 + @test job.container.role == "arn:aws:iam::012345678910:role/sleep60" + @test job.container.cmd == `sleep 60` + end + end + end + + @testset "Reuse job definition" begin + withenv(BATCH_ENVS...) do + patches = [ + @patch readstring(cmd::AbstractCmd) = mock_readstring(cmd) + @patch describe_jobs(args...) = DESCRIBE_JOBS_RESP + ] + + apply(patches; debug=true) do + job = BatchJob() + + @test job.id == "24fa2d7a-64c4-49d2-8b47-f8da4fbde8e9" + @test job.name == "example" + @test job.definition.name == "sleep60" + @test job.queue == "HighPriority" + @test job.region == "us-east-" + + @test job.container.image == "busybox" + @test job.container.vcpus == 1 + @test job.container.memory == 128 + @test job.container.role == "arn:aws:iam::012345678910:role/sleep60" + @test job.container.cmd == `sleep 60` + end + end + end + + end + + @testset "Job Submission" begin + job = BatchJob(; + name = JOB_NAME, + definition = JOB_DEFINITION, + queue = JOB_QUEUE, + container = Dict( + "image" => IMAGE_DEFINITION, + "vcpus" => 1, + "memory" => 1024, + "role" => JOB_ROLE, + "cmd" => `julia -e 'println("Hello World!")'`, + ), + ) + + submit(job) + @test isregistered(job) == true + @test wait(job, [AWSBatch.SUCCEEDED]) == true + deregister!(job) + events = logs(job) + + @test length(events) == 1 + @test contains(first(events)["message"], "Hello World!") + end + + @testset "Job Timed Out" begin + job = BatchJob(; + name = JOB_NAME, + definition = JOB_DEFINITION, + queue = JOB_QUEUE, + container = Dict( + "image" => IMAGE_DEFINITION, + "vcpus" => 1, + "memory" => 1024, + "role" => JOB_ROLE, + "cmd" => `sleep 60`, + ), + ) + + submit(job) + @test isregistered(job) == true + @test_throws ErrorException wait(job, [AWSBatch.SUCCEEDED]; timeout=0) + deregister!(job) + events = logs(job) + + @test length(events) == 0 + end + + @testset "Failed Job" begin + job = BatchJob(; + name = JOB_NAME, + definition = JOB_DEFINITION, + queue = JOB_QUEUE, + container = Dict( + "image" => IMAGE_DEFINITION, + "vcpus" => 1, + "memory" => 1024, + "role" => JOB_ROLE, + "cmd" => `julia -e 'error("Cmd failed")'`, + ), + ) + + submit(job) + @test isregistered(job) == true + @test_throws ErrorException wait(job, [AWSBatch.SUCCEEDED]) + deregister!(job) + events = logs(job) + + @test length(events) == 3 + @test contains(first(events)["message"], "ERROR: Cmd failed") + end end From 0df45d23355a43faa204021a3e092134eaa985bf Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Tue, 6 Mar 2018 15:03:32 -0600 Subject: [PATCH 003/110] Update gitlab CI for live tests --- .gitlab-ci.yml | 79 ++++++++------------------------------------------ 1 file changed, 12 insertions(+), 67 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 90c57a6..c9fedce 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,7 +3,10 @@ stages: - coverage - docs -.test_shell: &test_shell +"Live Tests": + stage: test + variables: + LIVE: "true" # Runs the online tests against AWS artifacts: name: "$CI_JOB_NAME coverage" expire_in: 1 week @@ -12,82 +15,23 @@ stages: before_script: - curl -s -o julia-ci https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/julia-ci - chmod +x julia-ci - - ./julia-ci install $JULIA_VERSION + - ./julia-ci install 0.6 script: - source julia-ci export - # - export PYTHON="" # Configure PyCall to use the Conda.jl package's Python - - julia -e "Pkg.clone(pwd()); Pkg.build(\"$PKG_NAME\"); Pkg.test(\"$PKG_NAME\"; coverage=true)" + - export PATH="$PATH:/usr/local/bin" + - unset SSL_CERT_DIR + - julia --compilecache=no -e "Pkg.clone(pwd()); Pkg.build(\"$PKG_NAME\"); Pkg.test(\"$PKG_NAME\"; coverage=true)" - ./julia-ci coverage after_script: - ./julia-ci clean - -.test_shell_06: &test_shell_06 - variables: - JULIA_VERSION: "0.6" - <<: *test_shell - -.test_shell_07-: &test_shell_07- - variables: - JULIA_VERSION: "0.7-" - allow_failure: true - <<: *test_shell - -.test_docker: &test_docker - artifacts: - name: "$CI_JOB_NAME coverage" - expire_in: 1 week - paths: - - "$CI_JOB_NAME coverage" - script: - # - export PYTHON="" # Configure PyCall to use the Conda.jl package's Python - - julia -e "Pkg.clone(pwd()); Pkg.build(\"$PKG_NAME\"); Pkg.test(\"$PKG_NAME\"; coverage=true)" - - julia-coverage "$PKG_NAME" - - -"0.6 (Mac)": tags: - - mac - shell-ci - <<: *test_shell_06 - -"0.6 (Linux, 64-bit, Docker)": - image: 292522074875.dkr.ecr.us-east-1.amazonaws.com/julia-gitlab-ci:0.6 - tags: - linux - - 64-bit - - docker-ci - <<: *test_docker - -"0.6 (Linux, 32-bit, Shell)": - tags: - - linux - - 32-bit - - shell-ci - <<: *test_shell_06 - -"0.7- (Mac)": - tags: - - mac - - shell-ci - <<: *test_shell_07- - -"0.7- (Linux, 64-bit)": - tags: - - linux - - 64-bit - - shell-ci - <<: *test_shell_07- - -"0.7- (Linux, 32-bit)": - tags: - - linux - - 32-bit - - shell-ci - <<: *test_shell_07- + - docker + - ecr "Coverage": stage: coverage - image: 292522074875.dkr.ecr.us-east-1.amazonaws.com/julia-gitlab-ci:0.6 coverage: /Test Coverage (\d+\.\d+%)/ artifacts: name: combined_coverage @@ -95,8 +39,9 @@ stages: paths: - combined_coverage/ tags: + - shell-ci - linux - - docker-ci + - amd64 script: - genhtml --version - mkdir $CI_PROJECT_DIR/combined_coverage From c88a14c252fddac44fa7ff5140877ecb212338e6 Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Tue, 6 Mar 2018 17:07:57 -0600 Subject: [PATCH 004/110] Leave out Memento logging configuration steps --- docs/src/index.md | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/docs/src/index.md b/docs/src/index.md index e21d344..0393bdd 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -23,11 +23,6 @@ Please review the ```julia julia> using AWSBatch -julia> using Memento - -julia> Memento.config("info"; fmt="[{level} | {name}]: {msg}") -Logger(root) - julia> job = BatchJob( name="Demo", definition="AWSBatchJobDefinition", @@ -43,25 +38,20 @@ julia> job = BatchJob( AWSBatch.BatchJob("", "Demo", AWSBatch.BatchJobDefinition("AWSBatchJobDefinition"), "AWSBatchJobQueue", "", AWSBatch.BatchJobContainer("000000000000.dkr.ecr.us-east-1.amazonaws.com/demo:latest", 1, 1024, "arn:aws:iam::000000000000:role/AWSBatchJobRole", `julia -e 'println("Hello World!")'`)) julia> submit(job) -[info | AWSBatch]: Registered job definition arn:aws:batch:us-east-1:000000000000:job-definition/AWSBatchJobDefinition:1. -[info | AWSBatch]: Submitted job Demo::00000000-0000-0000-0000-000000000000. Dict{String,Any} with 2 entries: "jobId" => "00000000-0000-0000-0000-000000000000" "jobName" => "Demo" julia> wait(job, [AWSBatch.SUCCEEDED]) -[info | AWSBatch]: Demo::00000000-0000-0000-0000-000000000000 status SUBMITTED -[info | AWSBatch]: Demo::00000000-0000-0000-0000-000000000000 status STARTING -[info | AWSBatch]: Demo::00000000-0000-0000-0000-000000000000 status SUCCEEDED true julia> results = logs(job) -[info | AWSBatch]: Fetching log events from Demo/default/00000000-0000-0000-0000-000000000000 -[info | AWSBatch]: Hello World! 1-element Array{Any,1}: Dict{String,Any}(Pair{String,Any}("ingestionTime", 1505846649863),Pair{String,Any}("message", "Hello World!"),Pair{String,Any}("timestamp", 1505846649786),Pair{String,Any}("eventId", "00000000000000000000000000000000000000000000000000000000")) ``` +AWSBatch also supports Memento logging for more detailed usage information. + ## Public API ### BatchJob From 5909c7be55ecbaedd8c8da6d255a88e1c75210fe Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Wed, 7 Mar 2018 10:29:12 -0600 Subject: [PATCH 005/110] Add comment for unsetting the SSL_CERT_DIR step in gitlab CI --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c9fedce..e01602a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,7 +19,7 @@ stages: script: - source julia-ci export - export PATH="$PATH:/usr/local/bin" - - unset SSL_CERT_DIR + - unset SSL_CERT_DIR # Work around LibGit2 initialization error in julia 0.6 - julia --compilecache=no -e "Pkg.clone(pwd()); Pkg.build(\"$PKG_NAME\"); Pkg.test(\"$PKG_NAME\"; coverage=true)" - ./julia-ci coverage after_script: From 4f22a998d91724ab8b92a14a9de7f2244d147bab Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Wed, 7 Mar 2018 13:29:50 -0600 Subject: [PATCH 006/110] Address MR comments --- src/AWSBatch.jl | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/AWSBatch.jl b/src/AWSBatch.jl index 6eab4d7..e0a63d9 100644 --- a/src/AWSBatch.jl +++ b/src/AWSBatch.jl @@ -313,11 +313,12 @@ end submit(job::BatchJob) -> Dict Handles submitting the batch job and registering a new job definition if necessary. -If no valid job definition exists (see `AWSBatch.lookupARN`) then a new job definition will be -created. Once the job has been submitted this function will return the response dictionary. +If no valid job definition exists (see `AWSBatch.job_definition_arn`) then a new job +definition will be created. Once the job has been submitted this function will return the +response dictionary. """ function submit(job::BatchJob) - job.definition.name = lookupARN(job) + job.definition.name = job_definition_arn(job) if isempty(job.definition.name) register!(job) @@ -358,17 +359,17 @@ function describe(job::BatchJob) end """ - lookupARN(job::BatchJob) -> String + job_definition_arn(job::BatchJob) -> String -Looks up the ARN for the latest job definition that can be reused for the current `BatchJob`. -A job definition can only be reused if: +Looks up the ARN (Amazaon Resource Name) for the latest job definition that can be reused +for the current `BatchJob`. A job definition can only be reused if: 1. status = ACTIVE 2. type = container 3. image = job.container.image 4. jobRoleArn = job.container.role """ -function lookupARN(job::BatchJob) +function job_definition_arn(job::BatchJob) resp = describe(job.definition) isempty(resp["jobDefinitions"]) && return "" From d2da8ef79cc5df37219723eddcfb294a2dbe4d6a Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Wed, 7 Mar 2018 13:29:28 -0600 Subject: [PATCH 007/110] Force using Docker CI --- .gitlab-ci.yml | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e01602a..47222a0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,8 +3,9 @@ stages: - coverage - docs -"Live Tests": +"Online Tests": stage: test + image: 292522074875.dkr.ecr.us-east-1.amazonaws.com/julia-gitlab-ci:0.6 variables: LIVE: "true" # Runs the online tests against AWS artifacts: @@ -12,26 +13,16 @@ stages: expire_in: 1 week paths: - "$CI_JOB_NAME coverage/" - before_script: - - curl -s -o julia-ci https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/julia-ci - - chmod +x julia-ci - - ./julia-ci install 0.6 script: - - source julia-ci export - - export PATH="$PATH:/usr/local/bin" - - unset SSL_CERT_DIR # Work around LibGit2 initialization error in julia 0.6 - julia --compilecache=no -e "Pkg.clone(pwd()); Pkg.build(\"$PKG_NAME\"); Pkg.test(\"$PKG_NAME\"; coverage=true)" - - ./julia-ci coverage - after_script: - - ./julia-ci clean + - julia-coverage "$PKG_NAME" tags: - - shell-ci - linux - - docker - - ecr + - docker-ci "Coverage": stage: coverage + image: 292522074875.dkr.ecr.us-east-1.amazonaws.com/julia-gitlab-ci:0.6 coverage: /Test Coverage (\d+\.\d+%)/ artifacts: name: combined_coverage @@ -39,9 +30,8 @@ stages: paths: - combined_coverage/ tags: - - shell-ci - linux - - amd64 + - docker-ci script: - genhtml --version - mkdir $CI_PROJECT_DIR/combined_coverage From 358797a09d5068accfe7c33a9157b777484b3a9b Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Thu, 8 Mar 2018 11:35:48 -0600 Subject: [PATCH 008/110] Update documentation --- docs/src/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/index.md b/docs/src/index.md index 0393bdd..9bda85c 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -87,6 +87,6 @@ AWSBatch.BatchJobContainer(::Associative) ## Private API ```@docs -AWSBatch.lookupARN(::BatchJob) +AWSBatch.job_definition_arn(::BatchJob) AWSBatch.update!(::BatchJobContainer, ::Associative) ``` From 83be8e9d6aa50a270876e92b58143beac9eab84c Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Thu, 8 Mar 2018 14:59:06 -0600 Subject: [PATCH 009/110] Rename submit to submit! for consistency With register! and deregister! and other internal functions. --- docs/src/index.md | 4 ++-- src/AWSBatch.jl | 6 +++--- test/runtests.jl | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/src/index.md b/docs/src/index.md index 9bda85c..e8f2ced 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -37,7 +37,7 @@ julia> job = BatchJob( ) AWSBatch.BatchJob("", "Demo", AWSBatch.BatchJobDefinition("AWSBatchJobDefinition"), "AWSBatchJobQueue", "", AWSBatch.BatchJobContainer("000000000000.dkr.ecr.us-east-1.amazonaws.com/demo:latest", 1, 1024, "arn:aws:iam::000000000000:role/AWSBatchJobRole", `julia -e 'println("Hello World!")'`)) -julia> submit(job) +julia> submit!(job) Dict{String,Any} with 2 entries: "jobId" => "00000000-0000-0000-0000-000000000000" "jobName" => "Demo" @@ -64,7 +64,7 @@ AWSBatch.isregistered(::BatchJob) AWSBatch.register!(::BatchJob) AWSBatch.deregister!(::BatchJob) AWSBatch.describe(::BatchJob) -AWSBatch.submit(::BatchJob) +AWSBatch.submit!(::BatchJob) Base.wait(::BatchJob, ::Vector{BatchStatus}, ::Vector{BatchStatus}) AWSBatch.logs(::BatchJob) ``` diff --git a/src/AWSBatch.jl b/src/AWSBatch.jl index e0a63d9..1eca120 100644 --- a/src/AWSBatch.jl +++ b/src/AWSBatch.jl @@ -24,7 +24,7 @@ export isregistered, register!, deregister!, - submit, + submit!, describe, wait, logs @@ -310,14 +310,14 @@ function deregister!(job::BatchJob) end """ - submit(job::BatchJob) -> Dict + submit!(job::BatchJob) -> Dict Handles submitting the batch job and registering a new job definition if necessary. If no valid job definition exists (see `AWSBatch.job_definition_arn`) then a new job definition will be created. Once the job has been submitted this function will return the response dictionary. """ -function submit(job::BatchJob) +function submit!(job::BatchJob) job.definition.name = job_definition_arn(job) if isempty(job.definition.name) diff --git a/test/runtests.jl b/test/runtests.jl index 7ebe4ca..e0e021c 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -154,7 +154,7 @@ include("mock.jl") ), ) - submit(job) + submit!(job) @test isregistered(job) == true @test wait(job, [AWSBatch.SUCCEEDED]) == true deregister!(job) @@ -178,7 +178,7 @@ include("mock.jl") ), ) - submit(job) + submit!(job) @test isregistered(job) == true @test_throws ErrorException wait(job, [AWSBatch.SUCCEEDED]; timeout=0) deregister!(job) @@ -201,7 +201,7 @@ include("mock.jl") ), ) - submit(job) + submit!(job) @test isregistered(job) == true @test_throws ErrorException wait(job, [AWSBatch.SUCCEEDED]) deregister!(job) From 7d40287c558228a3963ddfb576fdbe758192d632 Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Mon, 2 Apr 2018 16:37:56 +0000 Subject: [PATCH 010/110] Move away from using mutable structs --- REQUIRE | 1 + docs/src/index.md | 15 ++-- src/AWSBatch.jl | 178 +++++++++++++++++++++++++--------------------- test/runtests.jl | 45 ++++++------ 4 files changed, 125 insertions(+), 114 deletions(-) diff --git a/REQUIRE b/REQUIRE index de13dad..aba931e 100644 --- a/REQUIRE +++ b/REQUIRE @@ -1,4 +1,5 @@ julia 0.6 AWSSDK 0.2.0 +Compat 0.41.0 Mocking 0.3.3 Memento 0.5.0 diff --git a/docs/src/index.md b/docs/src/index.md index e8f2ced..41df1b5 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -27,13 +27,11 @@ julia> job = BatchJob( name="Demo", definition="AWSBatchJobDefinition", queue="AWSBatchJobQueue", - container=Dict( - "image" => "000000000000.dkr.ecr.us-east-1.amazonaws.com/demo:latest", - "role" => "arn:aws:iam::000000000000:role/AWSBatchJobRole", - "vcpus" => 1, - "memory" => 1024, - "cmd" => `julia -e 'println("Hello World!")'`, - ), + image = "000000000000.dkr.ecr.us-east-1.amazonaws.com/demo:latest", + role = "arn:aws:iam::000000000000:role/AWSBatchJobRole", + vcpus = 1, + memory = 1024, + cmd = `julia -e 'println("Hello World!")'`, ) AWSBatch.BatchJob("", "Demo", AWSBatch.BatchJobDefinition("AWSBatchJobDefinition"), "AWSBatchJobQueue", "", AWSBatch.BatchJobContainer("000000000000.dkr.ecr.us-east-1.amazonaws.com/demo:latest", 1, 1024, "arn:aws:iam::000000000000:role/AWSBatchJobRole", `julia -e 'println("Hello World!")'`)) @@ -65,6 +63,7 @@ AWSBatch.register!(::BatchJob) AWSBatch.deregister!(::BatchJob) AWSBatch.describe(::BatchJob) AWSBatch.submit!(::BatchJob) +AWSBatch.status(::BatchJob) Base.wait(::BatchJob, ::Vector{BatchStatus}, ::Vector{BatchStatus}) AWSBatch.logs(::BatchJob) ``` @@ -81,12 +80,10 @@ AWSBatch.isregistered(::BatchJobDefinition) ```@docs AWSBatch.BatchJobContainer -AWSBatch.BatchJobContainer(::Associative) ``` ## Private API ```@docs AWSBatch.job_definition_arn(::BatchJob) -AWSBatch.update!(::BatchJobContainer, ::Associative) ``` diff --git a/src/AWSBatch.jl b/src/AWSBatch.jl index 1eca120..41e2180 100644 --- a/src/AWSBatch.jl +++ b/src/AWSBatch.jl @@ -6,6 +6,7 @@ using AWSSDK.Batch using AWSSDK.CloudWatchLogs using AWSSDK.S3 +using Compat using Memento using Mocking @@ -15,17 +16,19 @@ import AWSSDK.Batch: import AWSSDK.CloudWatchLogs: get_log_events +import Compat: Nothing + export BatchJob, BatchJobDefinition, BatchJobContainer, BatchStatus, - S3Results, isregistered, register!, deregister!, submit!, describe, + status, wait, logs @@ -42,7 +45,7 @@ __init__() = Memento.register(logger) @enum BatchStatus SUBMITTED PENDING RUNNABLE STARTING RUNNING SUCCEEDED FAILED UNKNOWN const global _status_strs = map(s -> string(s) => s, instances(BatchStatus)) |> Dict -status(x::String) = _status_strs[x] +Base.parse(::Type{BatchStatus}, str::AbstractString) = _status_strs[str] @doc """ BatchStatus @@ -68,7 +71,7 @@ Stores configuration information about a batch job's container properties. - role::String: IAM role to apply to the ECS task - cmd::Cmd: command to execute in the batch job """ -mutable struct BatchJobContainer +struct BatchJobContainer image::String vcpus::Int memory::Int @@ -76,37 +79,6 @@ mutable struct BatchJobContainer cmd::Cmd end -""" - BatchJobContainer(container::T) - -Creates a BatchJobContainer from the values passed in the `container` dictionary. Uses -default values if the field is not specified. -""" -function BatchJobContainer(container::Associative) - BatchJobContainer( - get(container, "image", ""), - get(container, "vcpus", 1), - get(container, "memory", 1024), - get(container, "role", ""), - get(container, "cmd", ``), - ) -end - -""" - update!(container::BatchJobContainer, container_details::Associative) - -Updates the BatchJobContainer default values with the AWS response dict. -""" -function update!(container::BatchJobContainer, details::Associative) - # Only update fields that are still using the default values since explict arguments - # passed in have priority over aws job definition parameters - isempty(container.image) && (container.image = details["image"]) - container.vcpus == 1 && (container.vcpus = details["vcpus"]) - container.memory == 1024 && (container.memory = details["memory"]) - isempty(container.role) && (container.role = details["jobRoleArn"]) - isempty(container.cmd) && (container.cmd = Cmd(Vector{String}(details["command"]))) -end - ################################## # BatchJobDefinition ################################## @@ -116,7 +88,7 @@ end Stores the job definition name or arn including the revision. """ -mutable struct BatchJobDefinition +struct BatchJobDefinition name::AbstractString end @@ -127,9 +99,7 @@ Describes a job given it's definition. Returns the response dictionary. Requires permissions to access "batch:DescribeJobDefinitions". """ function describe(definition::BatchJobDefinition) - if isempty(definition.name) - return Dict("jobDefinitions" => []) - elseif startswith(definition.name, "arn:") + if startswith(definition.name, "arn:") return describe_job_definitions(Dict("jobDefinitions" => [definition.name])) else return @mock describe_job_definitions(Dict("jobDefinitionName" => definition.name)) @@ -167,7 +137,7 @@ Stores configuration information about a batch job in order to: # Fields - id::String: jobId - name:String: jobName -- definition::BatchJobDefinition: job definition +- definition::Union{BatchJobDefinition, Nothing}: job definition - queue::String: queue to insert the batch job into - region::String: AWS region to use - container::BatchJobContainer: job container properties (image, vcpus, memory, role, cmd) @@ -175,7 +145,7 @@ Stores configuration information about a batch job in order to: mutable struct BatchJob id::String name::String - definition::BatchJobDefinition + definition::Union{BatchJobDefinition, Nothing} queue::String region::String container::BatchJobContainer @@ -183,12 +153,16 @@ end """ BatchJob(; - id="", - name="", - queue ="", - region="", - definition=BatchJobDefinition(""), - container=BatchJobContainer(Dict()) + id::String="", + name::String="", + queue::String="", + region::String="", + definition::Union{String, Nothing}=nothing, + image::String="", + vcpus::Integer=1, + memory::Integer=1024, + role::String="", + cmd::Cmd=``, ) Handles creating a BatchJob based on various potential defaults. @@ -202,27 +176,37 @@ Order of priority from lowest to highest: 3. Explict arguments passed in via `kwargs`. """ function BatchJob(; - id="", - name="", - queue="", - region="", - definition=BatchJobDefinition(""), - container=BatchJobContainer(Dict()) + id::String="", + name::String="", + queue::String="", + region::String="", + definition::Union{String, Nothing}=nothing, + image::String="", + vcpus::Integer=1, + memory::Integer=1024, + role::String="", + cmd::Cmd=``, ) - if !isa(definition, BatchJobDefinition) - definition = BatchJobDefinition(definition) - end - if !isa(container, BatchJobContainer) - container = BatchJobContainer(container) + if definition !== nothing + definition = isempty(definition) ? nothing : BatchJobDefinition(definition) end # Determine if the job definition already exists and update it - resp = describe(definition) - if !isempty(resp["jobDefinitions"]) - details = first(resp["jobDefinitions"]) - - update!(container, details["containerProperties"]) + if definition !== nothing + resp = describe(definition) + if !isempty(resp["jobDefinitions"]) + details = first(resp["jobDefinitions"]) + + # Only update fields that are using the default values since explict arguments + # passed in have priority over aws job definition parameters + container = details["containerProperties"] + isempty(image) && (image = container["image"]) + vcpus == 1 && (vcpus = container["vcpus"]) + memory == 1024 && (memory = container["memory"]) + isempty(role) && (role = container["jobRoleArn"]) + isempty(cmd) && (cmd = Cmd(Vector{String}(container["command"]))) + end end if haskey(ENV, "AWS_BATCH_JOB_ID") @@ -250,25 +234,36 @@ function BatchJob(; isempty(name) && (name = details["jobName"]) isempty(queue) && (queue = job_queue) isempty(region) && (region = job_region) - if isempty(definition.name) + if definition === nothing definition = BatchJobDefinition(details["jobDefinition"]) end - update!(container, details["container"]) + # Only update fields that are using the default values since explict arguments + # passed in have priority over aws job definition parameters + container = details["container"] + isempty(image) && (image = container["image"]) + vcpus == 1 && (vcpus = container["vcpus"]) + memory == 1024 && (memory = container["memory"]) + isempty(role) && (role = container["jobRoleArn"]) + isempty(cmd) && (cmd = Cmd(Vector{String}(container["command"]))) else warn(logger, "No jobs found with id: $job_id.") end end - return BatchJob(id, name, definition, queue, region, container) + job_container = BatchJobContainer(image, vcpus, memory, role, cmd) + return BatchJob(id, name, definition, queue, region, job_container) end """ isregistered(job::BatchJob) -> Bool -Checks if a job is registered. +Checks if a job is registered. If no job definition exists, a new job definition is created +under the current job specifications, where the new job definition will be `job.name`. """ -isregistered(job::BatchJob) = isregistered(job.definition) +function isregistered(job::BatchJob) + return job.definition !== nothing && isregistered(job.definition) +end """ register!(job::BatchJob) @@ -277,7 +272,7 @@ Registers a new job definition. If no job definition exists, a new job definitio under the current job specifications, where the new job definition will be `job.name`. """ function register!(job::BatchJob) - isempty(job.definition.name) && (job.definition.name = job.name) + job.definition === nothing && (job.definition = BatchJobDefinition(job.name)) debug(logger, "Registering job definition $(job.definition.name).") input = [ @@ -293,17 +288,18 @@ function register!(job::BatchJob) ] resp = register_job_definition(input) - job.definition.name = resp["jobDefinitionArn"] + job.definition = BatchJobDefinition(resp["jobDefinitionArn"]) info(logger, "Registered job definition $(job.definition.name).") end """ deregister!(job::BatchJob) -Deregisters an AWS Batch job. +Deregisters an AWS Batch job. If no job definition exists, a new job definition is created +under the current job specifications, where the new job definition will be `job.name`. """ function deregister!(job::BatchJob) - isempty(job.definition.name) && (job.definition.name = job.name) + job.definition === nothing && (job.definition = BatchJobDefinition(job.name)) debug(logger, "Deregistering job definition $(job.definition.name).") resp = deregister_job_definition(Dict("jobDefinition" => job.definition.name)) info(logger, "Deregistered job definition $(job.definition.name).") @@ -318,10 +314,12 @@ definition will be created. Once the job has been submitted this function will r response dictionary. """ function submit!(job::BatchJob) - job.definition.name = job_definition_arn(job) + definition = job_definition_arn(job) - if isempty(job.definition.name) + if definition === nothing register!(job) + else + job.definition = BatchJobDefinition(definition) end debug(logger, "Submitting job $(job.name).") @@ -351,7 +349,15 @@ If job.id is set then this function is simply responsible for fetch a dictionary describing the batch job. """ function describe(job::BatchJob) - isempty(job.id) && error(logger, ArgumentError("job.id is not set")) + # Make sure the job.id has been set + if isempty(job.id) + error( + logger, + ArgumentError("job.id has not been set, call `submit!` to set it first.") + ) + end + + # Get AWS job description resp = describe_jobs(; jobs=[job.id]) isempty(resp["jobs"]) && error(logger, "Job $(job.name)::$(job.id) not found.") debug(logger, "Job $(job.name)::$(job.id): $resp") @@ -359,7 +365,7 @@ function describe(job::BatchJob) end """ - job_definition_arn(job::BatchJob) -> String + job_definition_arn(job::BatchJob) -> Union{String, Nothing} Looks up the ARN (Amazaon Resource Name) for the latest job definition that can be reused for the current `BatchJob`. A job definition can only be reused if: @@ -369,12 +375,13 @@ for the current `BatchJob`. A job definition can only be reused if: 3. image = job.container.image 4. jobRoleArn = job.container.role """ -function job_definition_arn(job::BatchJob) +function job_definition_arn(job::BatchJob)::Union{String, Nothing} + job.definition === nothing && return nothing + resp = describe(job.definition) + isempty(resp["jobDefinitions"]) && return nothing - isempty(resp["jobDefinitions"]) && return "" latest = first(resp["jobDefinitions"]) - for definition in resp["jobDefinitions"] if definition["status"] == "ACTIVE" && definition["revision"] > latest["revision"] latest = definition @@ -389,10 +396,19 @@ function job_definition_arn(job::BatchJob) ) return latest["jobDefinitionArn"] else - return "" + return nothing end end +""" status(job::BatchJob) -> BatchStatus + +Returns the current status of a BatchJob. # TODO: add to autodocs +""" +function status(job::BatchJob)::BatchStatus + details = describe(job) + return parse(BatchStatus, details["status"]) +end + """ wait( job::BatchJob, @@ -420,10 +436,8 @@ function Base.wait( tic() while time < timeout - j = describe(job) - time += toq() - state = status(j["status"]) + state = status(job) if state != last_state info(logger, "$(job.name)::$(job.id) status $state") diff --git a/test/runtests.jl b/test/runtests.jl index e0e021c..0e3c545 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,11 +2,13 @@ using Mocking Mocking.enable() using AWSBatch +using AWSSDK using Base.Test +using Compat using Memento -using AWSSDK import AWSSDK.Batch: describe_job_definitions +import Compat: Nothing const PKG_DIR = abspath(dirname(@__FILE__), "..") const REV = cd(() -> readchomp(`git rev-parse HEAD`), PKG_DIR) @@ -31,13 +33,15 @@ include("mock.jl") @test isempty(job.queue) @test isempty(job.region) - @test isempty(job.definition.name) + @test job.definition === nothing @test isempty(job.container.image) @test job.container.vcpus == 1 @test job.container.memory == 1024 @test isempty(job.container.role) @test isempty(job.container.cmd) + + @test_throws ArgumentError status(job) end @testset "From Job Definition" begin @@ -145,18 +149,17 @@ include("mock.jl") name = JOB_NAME, definition = JOB_DEFINITION, queue = JOB_QUEUE, - container = Dict( - "image" => IMAGE_DEFINITION, - "vcpus" => 1, - "memory" => 1024, - "role" => JOB_ROLE, - "cmd" => `julia -e 'println("Hello World!")'`, - ), + image = IMAGE_DEFINITION, + vcpus = 1, + memory = 1024, + role = JOB_ROLE, + cmd = `julia -e 'println("Hello World!")'`, ) submit!(job) @test isregistered(job) == true @test wait(job, [AWSBatch.SUCCEEDED]) == true + @test status(job) == AWSBatch.SUCCEEDED deregister!(job) events = logs(job) @@ -169,13 +172,11 @@ include("mock.jl") name = JOB_NAME, definition = JOB_DEFINITION, queue = JOB_QUEUE, - container = Dict( - "image" => IMAGE_DEFINITION, - "vcpus" => 1, - "memory" => 1024, - "role" => JOB_ROLE, - "cmd" => `sleep 60`, - ), + image = IMAGE_DEFINITION, + vcpus = 1, + memory = 1024, + role = JOB_ROLE, + cmd = `sleep 60`, ) submit!(job) @@ -192,13 +193,11 @@ include("mock.jl") name = JOB_NAME, definition = JOB_DEFINITION, queue = JOB_QUEUE, - container = Dict( - "image" => IMAGE_DEFINITION, - "vcpus" => 1, - "memory" => 1024, - "role" => JOB_ROLE, - "cmd" => `julia -e 'error("Cmd failed")'`, - ), + image = IMAGE_DEFINITION, + vcpus = 1, + memory = 1024, + role = JOB_ROLE, + cmd = `julia -e 'error("Cmd failed")'`, ) submit!(job) From f4557d49922295a342396a419931284c5d1db2c0 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 10 Apr 2018 09:59:09 -0500 Subject: [PATCH 011/110] Create LogEvent type Note: Compat AbstractDict was introduced in 0.41 --- src/AWSBatch.jl | 30 ++++++++++++++++-------------- src/deprecated.jl | 3 +++ src/log_event.jl | 27 +++++++++++++++++++++++++++ test/log_event.jl | 36 ++++++++++++++++++++++++++++++++++++ test/runtests.jl | 12 +++++++----- 5 files changed, 89 insertions(+), 19 deletions(-) create mode 100644 src/deprecated.jl create mode 100644 src/log_event.jl create mode 100644 test/log_event.jl diff --git a/src/AWSBatch.jl b/src/AWSBatch.jl index 41e2180..f776d4b 100644 --- a/src/AWSBatch.jl +++ b/src/AWSBatch.jl @@ -30,7 +30,8 @@ export describe, status, wait, - logs + logs, + log_events const logger = getlogger(current_module()) @@ -39,6 +40,9 @@ const logger = getlogger(current_module()) __init__() = Memento.register(logger) +include("log_event.jl") + + ##################### # BatchStatus ##################### @@ -466,33 +470,31 @@ function Base.wait( end """ - logs(job::BatchJob) -> Vector{String} + log_events(job::BatchJob) -> Vector{LogEvent} Fetches the logStreamName, fetches the CloudWatch logs and returns a vector of messages. NOTES: -- The `logStreamName`` isn't available until the job is RUNNING, so you may want to use `wait(job)` or - `wait(job, [AWSBatch.SUCCEEDED])` prior to calling `logs`. -- We do not support pagination, so this function is limited to 10,000 log messages by default. +- The `logStreamName` isn't available until the job is RUNNING, so you may want to use + `wait(job)` or `wait(job, [AWSBatch.SUCCEEDED])` prior to calling this function. +- We do not support pagination, so this function is limited to 10,000 log messages by + default. """ -function logs(job::BatchJob) +function log_events(job::BatchJob) container_details = describe(job)["container"] - events = Vector{String}() if "logStreamName" in keys(container_details) stream = container_details["logStreamName"] info(logger, "Fetching log events from $stream") - events = get_log_events(; logGroupName="/aws/batch/job", logStreamName=stream)["events"] - - for event in events - info(logger, event["message"]) - end + output = get_log_events(logGroupName="/aws/batch/job", logStreamName=stream) + return convert.(LogEvent, output["events"]) else info(logger, "No log events found for job $(job.name)::$(job.id).") + return LogEvent[] end - - return events end +include("deprecated.jl") + end # AWSBatch diff --git a/src/deprecated.jl b/src/deprecated.jl new file mode 100644 index 0000000..9f8e1d1 --- /dev/null +++ b/src/deprecated.jl @@ -0,0 +1,3 @@ +using Base: @deprecate + +@deprecate logs(job::BatchJob) [Dict("eventId" => e.event_id, "ingestionTime" => e.ingestion_time, "timestamp" => e.timestamp, "message" => e.message) for e in log_events(job)] diff --git a/src/log_event.jl b/src/log_event.jl new file mode 100644 index 0000000..ab4480b --- /dev/null +++ b/src/log_event.jl @@ -0,0 +1,27 @@ +using Compat: AbstractDict + +struct LogEvent + id::String + ingestion_time::DateTime # in UTC + timestamp::DateTime # in UTC + message::String +end + +function Base.convert(::Type{LogEvent}, d::AbstractDict) + LogEvent( + d["eventId"], + Dates.unix2datetime(d["ingestionTime"] / 1000), + Dates.unix2datetime(d["timestamp"] / 1000), + d["message"], + ) +end + +function Base.print(io::IO, event::LogEvent) + print(io, rpad(event.timestamp, 23), " ", event.message) +end + +function Base.print(io::IO, log_events::Vector{LogEvent}) + for event in log_events + println(io, event) + end +end diff --git a/test/log_event.jl b/test/log_event.jl new file mode 100644 index 0000000..c9ad6df --- /dev/null +++ b/test/log_event.jl @@ -0,0 +1,36 @@ +@testset "LogEvent" begin + @testset "constructor" begin + event = AWSBatch.LogEvent("123", DateTime(2018, 1, 2), DateTime(2018, 1, 1), "hello world!") + + @test event.id == "123" + @test event.ingestion_time == DateTime(2018, 1, 2) + @test event.timestamp == DateTime(2018, 1, 1) + @test event.message == "hello world!" + end + + @testset "convert" begin + d = Dict( + "eventId" => "456", + "ingestionTime" => 1, + "timestamp" => 2, + "message" => "from a dict", + ) + + event = convert(AWSBatch.LogEvent, d) + + @test event.id == "456" + @test event.ingestion_time == DateTime(1970, 1, 1, 0, 0, 0, 1) + @test event.timestamp == DateTime(1970, 1, 1, 0, 0, 0, 2) + @test event.message == "from a dict" + end + + @testset "print" begin + event = AWSBatch.LogEvent("123", DateTime(2018, 1, 2), DateTime(2018, 1, 1), "hello world!") + + @test sprint(print, event) == "2018-01-01T00:00:00 hello world!" + @test sprint(print, [event, event]) == """ + 2018-01-01T00:00:00 hello world! + 2018-01-01T00:00:00 hello world! + """ + end +end diff --git a/test/runtests.jl b/test/runtests.jl index 0e3c545..e539a93 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -24,6 +24,8 @@ setlevel!(getlogger(AWSBatch), "info") include("mock.jl") @testset "AWSBatch.jl" begin + include("log_event.jl") + @testset "Job Construction" begin @testset "Defaults" begin job = BatchJob() @@ -161,10 +163,10 @@ include("mock.jl") @test wait(job, [AWSBatch.SUCCEEDED]) == true @test status(job) == AWSBatch.SUCCEEDED deregister!(job) - events = logs(job) + events = log_events(job) @test length(events) == 1 - @test contains(first(events)["message"], "Hello World!") + @test contains(first(events).message, "Hello World!") end @testset "Job Timed Out" begin @@ -183,7 +185,7 @@ include("mock.jl") @test isregistered(job) == true @test_throws ErrorException wait(job, [AWSBatch.SUCCEEDED]; timeout=0) deregister!(job) - events = logs(job) + events = log_events(job) @test length(events) == 0 end @@ -204,9 +206,9 @@ include("mock.jl") @test isregistered(job) == true @test_throws ErrorException wait(job, [AWSBatch.SUCCEEDED]) deregister!(job) - events = logs(job) + events = log_events(job) @test length(events) == 3 - @test contains(first(events)["message"], "ERROR: Cmd failed") + @test contains(first(events).message, "ERROR: Cmd failed") end end From 6c3900c03c37cf5ce72bf6b8c663af0bde677da4 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 11 Apr 2018 15:20:50 -0500 Subject: [PATCH 012/110] Fix broken deprecation --- src/deprecated.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/deprecated.jl b/src/deprecated.jl index 9f8e1d1..c351113 100644 --- a/src/deprecated.jl +++ b/src/deprecated.jl @@ -1,3 +1,3 @@ using Base: @deprecate -@deprecate logs(job::BatchJob) [Dict("eventId" => e.event_id, "ingestionTime" => e.ingestion_time, "timestamp" => e.timestamp, "message" => e.message) for e in log_events(job)] +@deprecate logs(job::BatchJob) [Dict("eventId" => e.id, "ingestionTime" => e.ingestion_time, "timestamp" => e.timestamp, "message" => e.message) for e in log_events(job)] From 6588aa067e3b99ee35c8e169c5b313c3588eef58 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 10 Apr 2018 16:09:21 -0500 Subject: [PATCH 013/110] Rename BatchStatus to JobState --- src/AWSBatch.jl | 36 ++++++++++-------------------------- src/deprecated.jl | 1 + src/job_state.jl | 13 +++++++++++++ test/job_state.jl | 18 ++++++++++++++++++ test/runtests.jl | 1 + 5 files changed, 43 insertions(+), 26 deletions(-) create mode 100644 src/job_state.jl create mode 100644 test/job_state.jl diff --git a/src/AWSBatch.jl b/src/AWSBatch.jl index f776d4b..cfa648f 100644 --- a/src/AWSBatch.jl +++ b/src/AWSBatch.jl @@ -22,7 +22,7 @@ export BatchJob, BatchJobDefinition, BatchJobContainer, - BatchStatus, + JobState, isregistered, register!, deregister!, @@ -41,23 +41,7 @@ __init__() = Memento.register(logger) include("log_event.jl") - - -##################### -# BatchStatus -##################### - -@enum BatchStatus SUBMITTED PENDING RUNNABLE STARTING RUNNING SUCCEEDED FAILED UNKNOWN -const global _status_strs = map(s -> string(s) => s, instances(BatchStatus)) |> Dict -Base.parse(::Type{BatchStatus}, str::AbstractString) = _status_strs[str] - -@doc """ - BatchStatus - -An enum for representing different possible batch job states. - -See [docs](http://docs.aws.amazon.com/batch/latest/userguide/job_states.html) for details. -""" BatchStatus +include("job_state.jl") ################################## # BatchJobContainer @@ -122,7 +106,7 @@ function isregistered(definition::BatchJobDefinition) end ################################## -# BatchJob +# BatchJob ################################## """ @@ -404,20 +388,20 @@ function job_definition_arn(job::BatchJob)::Union{String, Nothing} end end -""" status(job::BatchJob) -> BatchStatus +""" status(job::BatchJob) -> JobState Returns the current status of a BatchJob. # TODO: add to autodocs """ -function status(job::BatchJob)::BatchStatus +function status(job::BatchJob)::JobState details = describe(job) - return parse(BatchStatus, details["status"]) + return parse(JobState, details["status"]) end """ wait( job::BatchJob, - cond::Vector{BatchStatus}=[RUNNING, SUCCEEDED], - failure::Vector{BatchStatus}=[FAILED]; + cond::Vector{JobState}=[RUNNING, SUCCEEDED], + failure::Vector{JobState}=[FAILED]; timeout=600, delay=5 ) @@ -429,8 +413,8 @@ polling time. """ function Base.wait( job::BatchJob, - cond::Vector{BatchStatus}=[RUNNING, SUCCEEDED], - failure::Vector{BatchStatus}=[FAILED]; + cond::Vector{JobState}=[RUNNING, SUCCEEDED], + failure::Vector{JobState}=[FAILED]; timeout=600, delay=5 ) diff --git a/src/deprecated.jl b/src/deprecated.jl index c351113..170f883 100644 --- a/src/deprecated.jl +++ b/src/deprecated.jl @@ -1,3 +1,4 @@ using Base: @deprecate @deprecate logs(job::BatchJob) [Dict("eventId" => e.id, "ingestionTime" => e.ingestion_time, "timestamp" => e.timestamp, "message" => e.message) for e in log_events(job)] +@deprecate BatchStatus JobState diff --git a/src/job_state.jl b/src/job_state.jl new file mode 100644 index 0000000..d96bc81 --- /dev/null +++ b/src/job_state.jl @@ -0,0 +1,13 @@ +# https://docs.aws.amazon.com/batch/latest/userguide/job_states.html +@doc """ + BatchStatus + +An enum for representing different possible AWS Batch job states. + +See [docs](http://docs.aws.amazon.com/batch/latest/userguide/job_states.html) for details. +""" JobState + +@enum JobState SUBMITTED PENDING RUNNABLE STARTING RUNNING SUCCEEDED FAILED UNKNOWN + +const STATE_MAP = Dict(string(s) => s for s in instances(JobState)) +Base.parse(::Type{JobState}, str::AbstractString) = STATE_MAP[str] diff --git a/test/job_state.jl b/test/job_state.jl new file mode 100644 index 0000000..9e01ae6 --- /dev/null +++ b/test/job_state.jl @@ -0,0 +1,18 @@ +using AWSBatch: JobState, SUBMITTED, PENDING, RUNNABLE, STARTING, RUNNING, SUCCEEDED, FAILED, UNKNOWN + +@testset "JobState" begin + @testset "parse" begin + @test parse(JobState, "SUBMITTED") == SUBMITTED + @test parse(JobState, "PENDING") == PENDING + @test parse(JobState, "RUNNABLE") == RUNNABLE + @test parse(JobState, "STARTING") == STARTING + @test parse(JobState, "RUNNING") == RUNNING + @test parse(JobState, "SUCCEEDED") == SUCCEEDED + @test parse(JobState, "FAILED") == FAILED + @test parse(JobState, "UNKNOWN") == UNKNOWN + end + + @testset "order" begin + @test SUBMITTED < PENDING < RUNNABLE < STARTING < RUNNING < SUCCEEDED < FAILED + end +end diff --git a/test/runtests.jl b/test/runtests.jl index e539a93..5907873 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -25,6 +25,7 @@ include("mock.jl") @testset "AWSBatch.jl" begin include("log_event.jl") + include("job_state.jl") @testset "Job Construction" begin @testset "Defaults" begin From 4880bced1e6e7c6a6484df3ecdaba6c07d2b3a40 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 10 Apr 2018 16:58:39 -0500 Subject: [PATCH 014/110] Move away from tic/toc for timeout --- src/AWSBatch.jl | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/AWSBatch.jl b/src/AWSBatch.jl index cfa648f..c3b1169 100644 --- a/src/AWSBatch.jl +++ b/src/AWSBatch.jl @@ -418,13 +418,11 @@ function Base.wait( timeout=600, delay=5 ) - time = 0 completed = false last_state = UNKNOWN - tic() - while time < timeout - time += toq() + start_time = time() # System time in seconds since epoch + while time() - start_time < timeout state = status(job) if state != last_state @@ -439,7 +437,6 @@ function Base.wait( elseif state in failure error(logger, "Job $(job.name)::$(job.id) hit failure condition $state.") else - tic() sleep(delay) end end From 434de32fd001b20f197f7b56e7654a587a30f7c1 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 10 Apr 2018 17:10:24 -0500 Subject: [PATCH 015/110] Refactor wait --- src/AWSBatch.jl | 80 ++++++++++++++++++++++++++++++++++++------------- 1 file changed, 60 insertions(+), 20 deletions(-) diff --git a/src/AWSBatch.jl b/src/AWSBatch.jl index c3b1169..ba8d7e8 100644 --- a/src/AWSBatch.jl +++ b/src/AWSBatch.jl @@ -399,9 +399,8 @@ end """ wait( - job::BatchJob, - cond::Vector{JobState}=[RUNNING, SUCCEEDED], - failure::Vector{JobState}=[FAILED]; + cond::Function, + job::BatchJob; timeout=600, delay=5 ) @@ -410,44 +409,85 @@ Polls the batch job state until it hits one of the conditions in `cond`. The loop will exit if it hits a `failure` condition and will not catch any excpetions. The polling interval can be controlled with `delay` and `timeout` provides a maximum polling time. + +# Examples +```julia +julia> wait(state -> state < SUCCEEDED, job) +true +``` """ function Base.wait( - job::BatchJob, - cond::Vector{JobState}=[RUNNING, SUCCEEDED], - failure::Vector{JobState}=[FAILED]; + cond::Function, + job::BatchJob; timeout=600, delay=5 ) completed = false - last_state = UNKNOWN + last_state = PENDING + initial = true start_time = time() # System time in seconds since epoch while time() - start_time < timeout state = status(job) - if state != last_state + if state != last_state || initial info(logger, "$(job.name)::$(job.id) status $state") + + if !cond(state) + completed = true + break + end + + last_state = state + end + + initial = false + sleep(delay) + end + + if !completed + message = "Waiting on job $(job.name)::$(job.id) timed out." + + if !initial + message *= " Last known state $last_state." end - last_state = state + error(logger, message) + end + + return completed +end + +""" + wait( + job::BatchJob, + cond::Vector{JobState}=[RUNNING, SUCCEEDED], + failure::Vector{JobState}=[FAILED]; + timeout=600, + delay=5 + ) +Polls the batch job state until it hits one of the conditions in `cond`. +The loop will exit if it hits a `failure` condition and will not catch any excpetions. +The polling interval can be controlled with `delay` and `timeout` provides a maximum +polling time. +""" +function Base.wait( + job::BatchJob, + cond::Vector{JobState}=[RUNNING, SUCCEEDED], + failure::Vector{JobState}=[FAILED]; + kwargs..., +) + wait(job; kwargs...) do state if state in cond - completed = true - break + false elseif state in failure error(logger, "Job $(job.name)::$(job.id) hit failure condition $state.") + false else - sleep(delay) + true end end - - if !completed - error( - logger, - "Waiting on job $(job.name)::$(job.id) timed out. Last known state $last_state." - ) - end - return completed end """ From a20e42b84cc6e38ba83a7da240837a5e675610ca Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 11 Apr 2018 16:12:58 -0500 Subject: [PATCH 016/110] Remove JobState UNKNOWN --- src/job_state.jl | 2 +- test/job_state.jl | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/job_state.jl b/src/job_state.jl index d96bc81..fdc0157 100644 --- a/src/job_state.jl +++ b/src/job_state.jl @@ -7,7 +7,7 @@ An enum for representing different possible AWS Batch job states. See [docs](http://docs.aws.amazon.com/batch/latest/userguide/job_states.html) for details. """ JobState -@enum JobState SUBMITTED PENDING RUNNABLE STARTING RUNNING SUCCEEDED FAILED UNKNOWN +@enum JobState SUBMITTED PENDING RUNNABLE STARTING RUNNING SUCCEEDED FAILED const STATE_MAP = Dict(string(s) => s for s in instances(JobState)) Base.parse(::Type{JobState}, str::AbstractString) = STATE_MAP[str] diff --git a/test/job_state.jl b/test/job_state.jl index 9e01ae6..a08329f 100644 --- a/test/job_state.jl +++ b/test/job_state.jl @@ -1,7 +1,8 @@ -using AWSBatch: JobState, SUBMITTED, PENDING, RUNNABLE, STARTING, RUNNING, SUCCEEDED, FAILED, UNKNOWN +using AWSBatch: JobState, SUBMITTED, PENDING, RUNNABLE, STARTING, RUNNING, SUCCEEDED, FAILED @testset "JobState" begin @testset "parse" begin + @test length(instances(JobState)) == 7 @test parse(JobState, "SUBMITTED") == SUBMITTED @test parse(JobState, "PENDING") == PENDING @test parse(JobState, "RUNNABLE") == RUNNABLE @@ -9,7 +10,6 @@ using AWSBatch: JobState, SUBMITTED, PENDING, RUNNABLE, STARTING, RUNNING, SUCCE @test parse(JobState, "RUNNING") == RUNNING @test parse(JobState, "SUCCEEDED") == SUCCEEDED @test parse(JobState, "FAILED") == FAILED - @test parse(JobState, "UNKNOWN") == UNKNOWN end @testset "order" begin From 6a759d2b118a6f934407f18acb11e2399a8a8116 Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Tue, 24 Apr 2018 10:12:48 -0500 Subject: [PATCH 017/110] Refactor AWSBatch interface --- docs/src/index.md | 52 ++--- src/AWSBatch.jl | 513 +++++++----------------------------------- src/batch_job.jl | 194 ++++++++++++++++ src/deprecated.jl | 12 + src/job_definition.jl | 129 +++++++++++ src/job_state.jl | 2 +- src/log_event.jl | 5 + test/mock.jl | 17 -- test/runtests.jl | 439 ++++++++++++++++++++++-------------- 9 files changed, 722 insertions(+), 641 deletions(-) create mode 100644 src/batch_job.jl create mode 100644 src/job_definition.jl diff --git a/docs/src/index.md b/docs/src/index.md index 41df1b5..691b839 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -23,7 +23,7 @@ Please review the ```julia julia> using AWSBatch -julia> job = BatchJob( +julia> job = run_batch( name="Demo", definition="AWSBatchJobDefinition", queue="AWSBatchJobQueue", @@ -33,57 +33,55 @@ julia> job = BatchJob( memory = 1024, cmd = `julia -e 'println("Hello World!")'`, ) -AWSBatch.BatchJob("", "Demo", AWSBatch.BatchJobDefinition("AWSBatchJobDefinition"), "AWSBatchJobQueue", "", AWSBatch.BatchJobContainer("000000000000.dkr.ecr.us-east-1.amazonaws.com/demo:latest", 1, 1024, "arn:aws:iam::000000000000:role/AWSBatchJobRole", `julia -e 'println("Hello World!")'`)) - -julia> submit!(job) -Dict{String,Any} with 2 entries: - "jobId" => "00000000-0000-0000-0000-000000000000" - "jobName" => "Demo" +AWSBatch.BatchJob("00000000-0000-0000-0000-000000000000") julia> wait(job, [AWSBatch.SUCCEEDED]) true -julia> results = logs(job) -1-element Array{Any,1}: - Dict{String,Any}(Pair{String,Any}("ingestionTime", 1505846649863),Pair{String,Any}("message", "Hello World!"),Pair{String,Any}("timestamp", 1505846649786),Pair{String,Any}("eventId", "00000000000000000000000000000000000000000000000000000000")) +julia> results = log_events(job) +1-element Array{AWSBatch.LogEvent,1}: + AWSBatch.LogEvent("00000000000000000000000000000000000000000000000000000000", 2018-04-23T19:41:18.765, 2018-04-23T19:41:18.677, "Hello World!") ``` AWSBatch also supports Memento logging for more detailed usage information. -## Public API +## API + +```@docs +run_batch() +``` ### BatchJob ```@docs AWSBatch.BatchJob -AWSBatch.BatchJob() -AWSBatch.BatchStatus -AWSBatch.isregistered(::BatchJob) -AWSBatch.register!(::BatchJob) -AWSBatch.deregister!(::BatchJob) +AWSBatch.submit(::AbstractString, ::JobDefinition, ::AbstractString) AWSBatch.describe(::BatchJob) -AWSBatch.submit!(::BatchJob) AWSBatch.status(::BatchJob) -Base.wait(::BatchJob, ::Vector{BatchStatus}, ::Vector{BatchStatus}) -AWSBatch.logs(::BatchJob) +Base.wait(::Function, ::BatchJob) +Base.wait(::BatchJob, ::Vector{JobState}, ::Vector{JobState}) +AWSBatch.log_events(::BatchJob) ``` -### BatchJobDefinition +### JobDefinition ```@docs -AWSBatch.BatchJobDefinition -AWSBatch.describe(::BatchJobDefinition) -AWSBatch.isregistered(::BatchJobDefinition) +AWSBatch.JobDefinition +AWSBatch.register(::AbstractString) # Does this display kwargs correctly +AWSBatch.job_definition_arn(::JobDefinition) +AWSBatch.deregister(::JobDefinition) +AWSBatch.isregistered(::JobDefinition) +AWSBatch.describe(::JobDefinition) ``` -### BatchJobContainer +### JobState ```@docs -AWSBatch.BatchJobContainer +AWSBatch.JobState ``` -## Private API +### LogEvent ```@docs -AWSBatch.job_definition_arn(::BatchJob) +AWSBatch.LogEvent ``` diff --git a/src/AWSBatch.jl b/src/AWSBatch.jl index ba8d7e8..c0b5ced 100644 --- a/src/AWSBatch.jl +++ b/src/AWSBatch.jl @@ -10,28 +10,20 @@ using Compat using Memento using Mocking -import AWSSDK.Batch: - describe_job_definitions, describe_jobs, register_job_definition, - deregister_job_definition, submit_job - -import AWSSDK.CloudWatchLogs: get_log_events - import Compat: Nothing export BatchJob, - BatchJobDefinition, - BatchJobContainer, + JobDefinition, JobState, - isregistered, - register!, - deregister!, - submit!, + run_batch, describe, status, wait, - logs, - log_events + log_events, + isregistered, + register, + deregister const logger = getlogger(current_module()) @@ -42,478 +34,141 @@ __init__() = Memento.register(logger) include("log_event.jl") include("job_state.jl") +include("job_definition.jl") +include("batch_job.jl") -################################## -# BatchJobContainer -################################## - -""" - BatchJob - -Stores configuration information about a batch job's container properties. - -# Fields -- image::String: the ECR container image to use for the ECS task -- vcpus::Int: # of cpus available in the ECS task container -- memory::Int: memory allocated to the ECS task container (in MB) -- role::String: IAM role to apply to the ECS task -- cmd::Cmd: command to execute in the batch job -""" -struct BatchJobContainer - image::String - vcpus::Int - memory::Int - role::String - cmd::Cmd -end - -################################## -# BatchJobDefinition -################################## - -""" - BatchJobDefinition - -Stores the job definition name or arn including the revision. -""" -struct BatchJobDefinition - name::AbstractString -end - -""" - describe(definition::BatchJobDefinition) - -Describes a job given it's definition. Returns the response dictionary. -Requires permissions to access "batch:DescribeJobDefinitions". -""" -function describe(definition::BatchJobDefinition) - if startswith(definition.name, "arn:") - return describe_job_definitions(Dict("jobDefinitions" => [definition.name])) - else - return @mock describe_job_definitions(Dict("jobDefinitionName" => definition.name)) - end -end - -""" - isregistered(definition::BatchJobDefinition) - -Checks if a BatchJobDefinition is registered. -""" -function isregistered(definition::BatchJobDefinition) - j = describe(definition) - active_definitions = filter!(d -> d["status"] == "ACTIVE", get(j, "jobDefinitions", [])) - return !isempty(active_definitions) -end - -################################## -# BatchJob -################################## - -""" - BatchJob - -Stores configuration information about a batch job in order to: - -- `register` a new job definition -- `submit` a new job to batch -- `describe` a batch job -- check if a batch job definition `isregistered` -- `deregister` a job definition -- `wait` for a job to complete -- fetch `logs` - -# Fields -- id::String: jobId -- name:String: jobName -- definition::Union{BatchJobDefinition, Nothing}: job definition -- queue::String: queue to insert the batch job into -- region::String: AWS region to use -- container::BatchJobContainer: job container properties (image, vcpus, memory, role, cmd) -""" -mutable struct BatchJob - id::String - name::String - definition::Union{BatchJobDefinition, Nothing} - queue::String - region::String - container::BatchJobContainer -end """ - BatchJob(; - id::String="", - name::String="", - queue::String="", - region::String="", - definition::Union{String, Nothing}=nothing, - image::String="", + run_batch(; + name::AbstractString="", + queue::AbstractString="", + region::AbstractString="", + definition::Union{AbstractString, Nothing}=nothing, + image::AbstractString="", vcpus::Integer=1, memory::Integer=1024, - role::String="", + role::AbstractString="", cmd::Cmd=``, - ) + ) -> BatchJob -Handles creating a BatchJob based on various potential defaults. -For example, default job fields can be inferred from an existing job defintion or existing -job (if currently running in a batch job) +Handles submitting a BatchJob based on various potential defaults. +For example, default job fields can be inferred from an existing job definition or an +existing job (if currently running in a batch job). -Order of priority from lowest to highest: +Order of priority from highest to lowest: -1. Job definition parameters +1. Explict arguments passed in via `kwargs`. 2. Inferred environment (e.g., `AWS_BATCH_JOB_ID` environment variable set) -3. Explict arguments passed in via `kwargs`. +3. Job definition parameters + +If no valid job definition exists (see [`AWSBatch.job_definition_arn`](@ref) then a new job +definition will be created and registered based on the job parameters. """ -function BatchJob(; - id::String="", - name::String="", - queue::String="", - region::String="", - definition::Union{String, Nothing}=nothing, - image::String="", +function run_batch(; + name::AbstractString="", + queue::AbstractString="", + region::AbstractString="", + definition::Union{AbstractString, Nothing}=nothing, + image::AbstractString="", vcpus::Integer=1, memory::Integer=1024, - role::String="", + role::AbstractString="", cmd::Cmd=``, ) - if definition !== nothing - definition = isempty(definition) ? nothing : BatchJobDefinition(definition) + definition = isempty(definition) ? nothing : JobDefinition(definition) end - # Determine if the job definition already exists and update it + # Determine if the job definition already exists and update the default job parameters if definition !== nothing - resp = describe(definition) - if !isempty(resp["jobDefinitions"]) - details = first(resp["jobDefinitions"]) + response = describe(definition) + if !isempty(response["jobDefinitions"]) + details = first(response["jobDefinitions"]) - # Only update fields that are using the default values since explict arguments - # passed in have priority over aws job definition parameters container = details["containerProperties"] isempty(image) && (image = container["image"]) + isempty(role) && (role = container["jobRoleArn"]) + + # Update container override parameters vcpus == 1 && (vcpus = container["vcpus"]) memory == 1024 && (memory = container["memory"]) - isempty(role) && (role = container["jobRoleArn"]) isempty(cmd) && (cmd = Cmd(Vector{String}(container["command"]))) end end + # Get inferred environment parameters if haskey(ENV, "AWS_BATCH_JOB_ID") # Environmental variables set by the AWS Batch service. They were discovered by # inspecting the running AWS Batch job in the ECS task interface. job_id = ENV["AWS_BATCH_JOB_ID"] job_queue = ENV["AWS_BATCH_JQ_NAME"] - # Get the zone information from the EC2 instance metadata. - zone = @mock readstring( - pipeline( - `curl http://169.254.169.254/latest/meta-data/placement/availability-zone`; - stderr=DevNull - ) - ) - job_region = chop(zone) - # Requires permissions to access to "batch:DescribeJobs" - resp = @mock describe_jobs(Dict("jobs" => [job_id])) + response = @mock describe_jobs(Dict("jobs" => [job_id])) - if length(resp["jobs"]) > 0 - details = first(resp["jobs"]) + # Use the job's description to only update fields that are using the default + # values since explict arguments passed in via `kwargs` have higher priority + if length(response["jobs"]) > 0 + details = first(response["jobs"]) - isempty(id) && (id = job_id) + # Update the job's required parameters isempty(name) && (name = details["jobName"]) + definition === nothing && (definition = JobDefinition(details["jobDefinition"])) isempty(queue) && (queue = job_queue) - isempty(region) && (region = job_region) - if definition === nothing - definition = BatchJobDefinition(details["jobDefinition"]) - end - # Only update fields that are using the default values since explict arguments - # passed in have priority over aws job definition parameters + # Update the container parameters container = details["container"] isempty(image) && (image = container["image"]) + isempty(role) && (role = container["jobRoleArn"]) + + # Update container overrides vcpus == 1 && (vcpus = container["vcpus"]) memory == 1024 && (memory = container["memory"]) - isempty(role) && (role = container["jobRoleArn"]) isempty(cmd) && (cmd = Cmd(Vector{String}(container["command"]))) else warn(logger, "No jobs found with id: $job_id.") end end - job_container = BatchJobContainer(image, vcpus, memory, role, cmd) - return BatchJob(id, name, definition, queue, region, job_container) -end - -""" - isregistered(job::BatchJob) -> Bool - -Checks if a job is registered. If no job definition exists, a new job definition is created -under the current job specifications, where the new job definition will be `job.name`. -""" -function isregistered(job::BatchJob) - return job.definition !== nothing && isregistered(job.definition) -end - -""" - register!(job::BatchJob) - -Registers a new job definition. If no job definition exists, a new job definition is created -under the current job specifications, where the new job definition will be `job.name`. -""" -function register!(job::BatchJob) - job.definition === nothing && (job.definition = BatchJobDefinition(job.name)) - - debug(logger, "Registering job definition $(job.definition.name).") - input = [ - "type" => "container", - "containerProperties" => [ - "image" => job.container.image, - "vcpus" => job.container.vcpus, - "memory" => job.container.memory, - "command" => job.container.cmd.exec, - "jobRoleArn" => job.container.role, - ], - "jobDefinitionName" => job.definition.name, - ] - - resp = register_job_definition(input) - job.definition = BatchJobDefinition(resp["jobDefinitionArn"]) - info(logger, "Registered job definition $(job.definition.name).") -end - -""" - deregister!(job::BatchJob) - -Deregisters an AWS Batch job. If no job definition exists, a new job definition is created -under the current job specifications, where the new job definition will be `job.name`. -""" -function deregister!(job::BatchJob) - job.definition === nothing && (job.definition = BatchJobDefinition(job.name)) - debug(logger, "Deregistering job definition $(job.definition.name).") - resp = deregister_job_definition(Dict("jobDefinition" => job.definition.name)) - info(logger, "Deregistered job definition $(job.definition.name).") -end - -""" - submit!(job::BatchJob) -> Dict - -Handles submitting the batch job and registering a new job definition if necessary. -If no valid job definition exists (see `AWSBatch.job_definition_arn`) then a new job -definition will be created. Once the job has been submitted this function will return the -response dictionary. -""" -function submit!(job::BatchJob) - definition = job_definition_arn(job) - - if definition === nothing - register!(job) - else - job.definition = BatchJobDefinition(definition) - end - - debug(logger, "Submitting job $(job.name).") - input = [ - "jobName" => job.name, - "jobDefinition" => job.definition.name, - "jobQueue" => job.queue, - "containerOverrides" => [ - "vcpus" => job.container.vcpus, - "memory" => job.container.memory, - "command" => job.container.cmd.exec, - ] - ] - debug(logger, "Input: $input") - resp = submit_job(input) - - job.id = resp["jobId"] - info(logger, "Submitted job $(job.name)::$(job.id).") - - return resp -end - -""" - describe(job::BatchJob) -> Dict - -If job.id is set then this function is simply responsible for fetch a dictionary for -describing the batch job. -""" -function describe(job::BatchJob) - # Make sure the job.id has been set - if isempty(job.id) - error( - logger, - ArgumentError("job.id has not been set, call `submit!` to set it first.") - ) - end - - # Get AWS job description - resp = describe_jobs(; jobs=[job.id]) - isempty(resp["jobs"]) && error(logger, "Job $(job.name)::$(job.id) not found.") - debug(logger, "Job $(job.name)::$(job.id): $resp") - return first(resp["jobs"]) -end - -""" - job_definition_arn(job::BatchJob) -> Union{String, Nothing} - -Looks up the ARN (Amazaon Resource Name) for the latest job definition that can be reused -for the current `BatchJob`. A job definition can only be reused if: - -1. status = ACTIVE -2. type = container -3. image = job.container.image -4. jobRoleArn = job.container.role -""" -function job_definition_arn(job::BatchJob)::Union{String, Nothing} - job.definition === nothing && return nothing - - resp = describe(job.definition) - isempty(resp["jobDefinitions"]) && return nothing + # Reuse a previously registered job definition if available. + # If no job definition exists that can be reused, a new job definition is created + # under the current job specifications. + if definition !== nothing + reusable_def = @mock job_definition_arn(definition; image=image, role=role) - latest = first(resp["jobDefinitions"]) - for definition in resp["jobDefinitions"] - if definition["status"] == "ACTIVE" && definition["revision"] > latest["revision"] - latest = definition + if reusable_def !== nothing + definition = reusable_def + else + definition = @mock register( + definition.name; + image=image, + role=role, + vcpus=vcpus, + memory=memory, + cmd=cmd, + ) end - end - - if ( - latest["status"] == "ACTIVE" && - latest["type"] == "container" && - latest["containerProperties"]["image"] == job.container.image && - latest["containerProperties"]["jobRoleArn"] == job.container.role - ) - return latest["jobDefinitionArn"] else - return nothing - end -end - -""" status(job::BatchJob) -> JobState - -Returns the current status of a BatchJob. # TODO: add to autodocs -""" -function status(job::BatchJob)::JobState - details = describe(job) - return parse(JobState, details["status"]) -end - -""" - wait( - cond::Function, - job::BatchJob; - timeout=600, - delay=5 - ) - -Polls the batch job state until it hits one of the conditions in `cond`. -The loop will exit if it hits a `failure` condition and will not catch any excpetions. -The polling interval can be controlled with `delay` and `timeout` provides a maximum -polling time. - -# Examples -```julia -julia> wait(state -> state < SUCCEEDED, job) -true -``` -""" -function Base.wait( - cond::Function, - job::BatchJob; - timeout=600, - delay=5 -) - completed = false - last_state = PENDING - initial = true - - start_time = time() # System time in seconds since epoch - while time() - start_time < timeout - state = status(job) - - if state != last_state || initial - info(logger, "$(job.name)::$(job.id) status $state") - - if !cond(state) - completed = true - break - end - - last_state = state - end - - initial = false - sleep(delay) - end - - if !completed - message = "Waiting on job $(job.name)::$(job.id) timed out." - - if !initial - message *= " Last known state $last_state." - end - - error(logger, message) + # Use the job name as the definiton name since the definition name was not specified + definition = @mock register( + name; + image=image, + role=role, + vcpus=vcpus, + memory=memory, + cmd=cmd, + ) end - return completed -end - -""" - wait( - job::BatchJob, - cond::Vector{JobState}=[RUNNING, SUCCEEDED], - failure::Vector{JobState}=[FAILED]; - timeout=600, - delay=5 + # The parameters that can be overridden are `memory`, `vcpus`, `cmd`, and `environment` + # See https://docs.aws.amazon.com/batch/latest/APIReference/API_ContainerOverrides.html + container_overrides = Dict( + "vcpus" => vcpus, + "memory" => memory, + "cmd" => cmd, ) -Polls the batch job state until it hits one of the conditions in `cond`. -The loop will exit if it hits a `failure` condition and will not catch any excpetions. -The polling interval can be controlled with `delay` and `timeout` provides a maximum -polling time. -""" -function Base.wait( - job::BatchJob, - cond::Vector{JobState}=[RUNNING, SUCCEEDED], - failure::Vector{JobState}=[FAILED]; - kwargs..., -) - wait(job; kwargs...) do state - if state in cond - false - elseif state in failure - error(logger, "Job $(job.name)::$(job.id) hit failure condition $state.") - false - else - true - end - end -end - -""" - log_events(job::BatchJob) -> Vector{LogEvent} - -Fetches the logStreamName, fetches the CloudWatch logs and returns a vector of messages. - -NOTES: -- The `logStreamName` isn't available until the job is RUNNING, so you may want to use - `wait(job)` or `wait(job, [AWSBatch.SUCCEEDED])` prior to calling this function. -- We do not support pagination, so this function is limited to 10,000 log messages by - default. -""" -function log_events(job::BatchJob) - container_details = describe(job)["container"] - - if "logStreamName" in keys(container_details) - stream = container_details["logStreamName"] - - info(logger, "Fetching log events from $stream") - output = get_log_events(logGroupName="/aws/batch/job", logStreamName=stream) - return convert.(LogEvent, output["events"]) - else - info(logger, "No log events found for job $(job.name)::$(job.id).") - return LogEvent[] - end + return @mock submit(name, definition, queue; container=container_overrides) end include("deprecated.jl") diff --git a/src/batch_job.jl b/src/batch_job.jl new file mode 100644 index 0000000..052bd48 --- /dev/null +++ b/src/batch_job.jl @@ -0,0 +1,194 @@ +import AWSSDK.Batch: describe_jobs, submit_job +import AWSSDK.CloudWatchLogs: get_log_events + + +""" + BatchJob + +Stores a batch job id in order to: + +- `describe` a job and its parameters +- check on the `status` of a job +- `wait` for a job to complete +- fetch `log_events` + +# Fields +- `id::AbstractString`: jobId +""" +struct BatchJob + id::AbstractString +end + +""" + submit( + name::AbstractString, + definition::JobDefinition, + queue::AbstractString; + container::AbstractDict=Dict() +) -> BatchJob + +Handles submitting the batch job. Returns a `BatchJob` wrapper for the id. +""" +function submit( + name::AbstractString, + definition::JobDefinition, + queue::AbstractString; + container::AbstractDict=Dict() +) + debug(logger, "Submitting job $name.") + input = [ + "jobName" => name, + "jobDefinition" => definition.name, + "jobQueue" => queue, + "containerOverrides" => container, + ] + debug(logger, "Input: $input") + + response = submit_job(input) + job = BatchJob(response["jobId"]) + + info(logger, "Submitted job $(name)::$(job.id).") + + return job +end + +""" + describe(job::BatchJob) -> Dict + +If job.id is set then this function is simply responsible for fetch a dictionary for +describing the batch job. +""" +function describe(job::BatchJob) + response = @mock describe_jobs(; jobs=[job.id]) + isempty(response["jobs"]) && error(logger, "Job $(job.id) not found.") + debug(logger, "Job $(job.id): $response") + return first(response["jobs"]) +end + + +""" status(job::BatchJob) -> JobState + +Returns the current status of a job. +""" +function status(job::BatchJob)::JobState + details = describe(job) + return parse(JobState, details["status"]) +end + +""" + wait( + cond::Function, + job::BatchJob; + timeout=600, + delay=5 + ) + +Polls the batch job state until it hits one of the conditions in `cond`. +The loop will exit if it hits a `failure` condition and will not catch any excpetions. +The polling interval can be controlled with `delay` and `timeout` provides a maximum +polling time. + +# Examples +```julia +julia> wait(state -> state < SUCCEEDED, job) +true +``` +""" +function Base.wait( + cond::Function, + job::BatchJob; + timeout=600, + delay=5 +) + completed = false + last_state = PENDING + initial = true + + start_time = time() # System time in seconds since epoch + while time() - start_time < timeout + state = status(job) + + if state != last_state || initial + info(logger, "$(job.id) status $state") + + if !cond(state) + completed = true + break + end + + last_state = state + end + + initial && (initial = false) + sleep(delay) + end + + if !completed + message = "Waiting on job $(job.id) timed out." + + if !initial + message *= " Last known state $last_state." + end + + error(logger, message) + end + + return completed +end + +""" + wait( + job::BatchJob, + cond::Vector{JobState}=[RUNNING, SUCCEEDED], + failure::Vector{JobState}=[FAILED]; + kwargs..., + ) + +Polls the batch job state until it hits one of the conditions in `cond`. +The loop will exit if it hits a `failure` condition and will not catch any excpetions. +The polling interval can be controlled with `delay` and `timeout` provides a maximum +polling time. +""" +function Base.wait( + job::BatchJob, + cond::Vector{JobState}=[RUNNING, SUCCEEDED], + failure::Vector{JobState}=[FAILED]; + kwargs..., +) + wait(job; kwargs...) do state + if state in cond + false + elseif state in failure + error(logger, "Job $(job.id) hit failure condition $state.") + false + else + true + end + end +end + +""" + log_events(job::BatchJob) -> Vector{LogEvent} + +Fetches the logStreamName, fetches the CloudWatch logs and returns a vector of messages. + +NOTES: +- The `logStreamName` isn't available until the job is RUNNING, so you may want to use + `wait(job)` or `wait(job, [AWSBatch.SUCCEEDED])` prior to calling this function. +- We do not support pagination, so this function is limited to 10,000 log messages by + default. +""" +function log_events(job::BatchJob) + container_details = describe(job)["container"] + + if "logStreamName" in keys(container_details) + stream = container_details["logStreamName"] + + info(logger, "Fetching log events from $stream") + output = get_log_events(logGroupName="/aws/batch/job", logStreamName=stream) + return convert.(LogEvent, output["events"]) + else + info(logger, "No log events found for job $(job.id).") + return LogEvent[] + end +end diff --git a/src/deprecated.jl b/src/deprecated.jl index 170f883..0bb5333 100644 --- a/src/deprecated.jl +++ b/src/deprecated.jl @@ -2,3 +2,15 @@ using Base: @deprecate @deprecate logs(job::BatchJob) [Dict("eventId" => e.id, "ingestionTime" => e.ingestion_time, "timestamp" => e.timestamp, "message" => e.message) for e in log_events(job)] @deprecate BatchStatus JobState + +@deprecate BatchJob(; kwargs...) run_batch(; filter((x)->(x[1] != :id), kwargs)...) # id is no longer a valid kwarg +@deprecate submit!(job::BatchJob) job # Creating BatchJob's now automatically submits the job and submit doesn't need to be explicitly called +@deprecate job_definition_arn(job::BatchJob) job_definition_arn(JobDefinition(describe(job)["jobDefinition"])) + +# Deprecate methods that now are called explicitly on JobDefinition's and not on BatchJob's +@deprecate isregistered(job::BatchJob) isregistered(JobDefinition(describe(job)["jobDefinition"])) +@deprecate register!(job::BatchJob) JobDefinition(describe(job)["jobDefinition"]) # Job definition should already be registered for a created BatchJob, so just look it up +@deprecate deregister!(job::BatchJob) deregister(JobDefinition(describe(job)["jobDefinition"])) + +@deprecate register(job_definition::JobDefinition) register(job_definition.name) + diff --git a/src/job_definition.jl b/src/job_definition.jl new file mode 100644 index 0000000..7ce86c8 --- /dev/null +++ b/src/job_definition.jl @@ -0,0 +1,129 @@ +import AWSSDK.Batch: + describe_job_definitions, register_job_definition, deregister_job_definition + +""" + JobDefinition + +Stores the job definition name or arn including the revision. +""" +struct JobDefinition + name::AbstractString +end + +""" + register( + definition::AbstractString; + role::AbstractString="", + image::AbstractString="", + vcpus::Integer=1, + memory::Integer=1024, + cmd::Cmd=`` + ) -> JobDefinition + +Registers a new job definition. +""" +function register( + definition::AbstractString; + image::AbstractString="", + role::AbstractString="", + vcpus::Integer=1, + memory::Integer=1024, + cmd::Cmd=`` +) + debug(logger, "Registering job definition $definition.") + input = [ + "type" => "container", + "containerProperties" => [ + "image" => image, + "vcpus" => vcpus, + "memory" => memory, + "command" => cmd.exec, + "jobRoleArn" => role, + ], + "jobDefinitionName" => definition, + ] + + response = register_job_definition(input) + definition = JobDefinition(response["jobDefinitionArn"]) + info(logger, "Registered job definition $(definition.name).") + return definition +end + +""" + job_definition_arn( + definition::JobDefinition; + image::AbstractString="", + role::AbstractString="" + ) -> Union{JobDefinition, Nothing} + +Looks up the ARN (Amazon Resource Name) for the latest job definition that can be reused. +Returns a JobDefinition with the ARN that can be reused or `nothing`. + +A job definition can only be reused if: + +1. status = ACTIVE +2. type = container +3. image = the current job's image +4. jobRoleArn = the current job's role +""" +function job_definition_arn( + definition::JobDefinition; + image::AbstractString="", + role::AbstractString="" +) + response = describe(definition) + isempty(response["jobDefinitions"]) && return nothing + + latest = first(response["jobDefinitions"]) + for definition in response["jobDefinitions"] + if definition["status"] == "ACTIVE" && definition["revision"] > latest["revision"] + latest = definition + end + end + if ( + latest["status"] == "ACTIVE" && + latest["type"] == "container" && + (latest["containerProperties"]["image"] == image || isempty(image)) && + (latest["containerProperties"]["jobRoleArn"] == role || isempty(role)) + ) + return JobDefinition(latest["jobDefinitionArn"]) + else + return nothing + end +end + +""" + deregister(job::JobDefinition) + +Deregisters an AWS Batch job. +""" +function deregister(definition::JobDefinition) + debug(logger, "Deregistering job definition $(definition.name).") + resp = deregister_job_definition(Dict("jobDefinition" => definition.name)) + info(logger, "Deregistered job definition $(definition.name).") +end + +""" + isregistered(definition::JobDefinition) -> Bool + +Checks if a JobDefinition is registered. +""" +function isregistered(definition::JobDefinition) + j = describe(definition) + active_definitions = filter!(d -> d["status"] == "ACTIVE", get(j, "jobDefinitions", [])) + return !isempty(active_definitions) +end + +""" + describe(definition::JobDefinition) -> Dict + +Describes a job given it's definition. Returns the response dictionary. +Requires permissions to access "batch:DescribeJobDefinitions". +""" +function describe(definition::JobDefinition) + if startswith(definition.name, "arn:") + return @mock describe_job_definitions(Dict("jobDefinitions" => [definition.name])) + else + return @mock describe_job_definitions(Dict("jobDefinitionName" => definition.name)) + end +end diff --git a/src/job_state.jl b/src/job_state.jl index fdc0157..78f46df 100644 --- a/src/job_state.jl +++ b/src/job_state.jl @@ -1,6 +1,6 @@ # https://docs.aws.amazon.com/batch/latest/userguide/job_states.html @doc """ - BatchStatus + JobState An enum for representing different possible AWS Batch job states. diff --git a/src/log_event.jl b/src/log_event.jl index ab4480b..e8518c9 100644 --- a/src/log_event.jl +++ b/src/log_event.jl @@ -1,5 +1,10 @@ using Compat: AbstractDict +""" + LogEvent + +A struct for representing an event in an AWS Batch job log. +""" struct LogEvent id::String ingestion_time::DateTime # in UTC diff --git a/test/mock.jl b/test/mock.jl index d31bf43..b63aa4d 100644 --- a/test/mock.jl +++ b/test/mock.jl @@ -83,20 +83,3 @@ const DESCRIBE_JOBS_RESP = Dict( ) ] ) - - -""" - Mock.readstring(cmd::CmdRedirect, pass::Bool=true) - -Mocks the CmdRedirect produced from ``pipeline(`curl http://169.254.169.254/latest/meta-data/placement/availability-zone`)`` -to just return "us-east-1". -""" -function mock_readstring(cmd::CmdRedirect) - cmd_exec = cmd.cmd.exec - - result = if cmd_exec[1] == "curl" && contains(cmd_exec[2], "availability-zone") - return "us-east-1" - else - return Base.readstring(cmd) - end -end diff --git a/test/runtests.jl b/test/runtests.jl index 5907873..4377419 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,16 +2,9 @@ using Mocking Mocking.enable() using AWSBatch -using AWSSDK using Base.Test -using Compat using Memento -import AWSSDK.Batch: describe_job_definitions -import Compat: Nothing - -const PKG_DIR = abspath(dirname(@__FILE__), "..") -const REV = cd(() -> readchomp(`git rev-parse HEAD`), PKG_DIR) const IMAGE_DEFINITION = "292522074875.dkr.ecr.us-east-1.amazonaws.com/aws-tools:latest" const JOB_ROLE = "arn:aws:iam::292522074875:role/AWSBatchClusterManagerJobRole" const JOB_DEFINITION = "AWSBatch" @@ -23,193 +16,305 @@ setlevel!(getlogger(AWSBatch), "info") include("mock.jl") -@testset "AWSBatch.jl" begin - include("log_event.jl") - include("job_state.jl") - - @testset "Job Construction" begin - @testset "Defaults" begin - job = BatchJob() - - @test isempty(job.id) - @test isempty(job.name) - @test isempty(job.queue) - @test isempty(job.region) - - @test job.definition === nothing - @test isempty(job.container.image) - @test job.container.vcpus == 1 - @test job.container.memory == 1024 - @test isempty(job.container.role) - @test isempty(job.container.cmd) +function verify_job_submission(name, definition, queue, container, expected) + @test name == expected["name"] + @test definition.name == expected["definition"] + @test queue == expected["queue"] + @test container == expected["container"] - @test_throws ArgumentError status(job) - end + return BatchJob(expected["id"]) +end - @testset "From Job Definition" begin - patch = @patch describe_job_definitions(args...) = DESCRIBE_JOBS_DEF_RESP +function verify_job_definition(definition, image, role, vcpus, memory, cmd, expected) + @test definition == expected["definition"] + @test image == expected["image"] + @test role == expected["role"] + @test vcpus == expected["vcpus"] + @test memory == expected["memory"] + @test cmd == expected["cmd"] - apply(patch; debug=true) do - job = BatchJob(name=JOB_NAME, definition="sleep60") + return JobDefinition(definition) +end - @test isempty(job.id) - @test job.name == "AWSBatchTest" - @test isempty(job.queue) - @test isempty(job.region) - @test job.definition.name == "sleep60" +@testset "AWSBatch.jl" begin + include("log_event.jl") + include("job_state.jl") - @test job.container.image == "busybox" - @test job.container.vcpus == 1 - @test job.container.memory == 128 - @test job.container.role == "arn:aws:iam::012345678910:role/sleep60" - @test job.container.cmd == `sleep 60` + @testset "`run_batch` preprocessing" begin + @testset "Defaults" begin + expected_job_parameters = Dict( + "name" => "", + "definition" => "", + "queue" => "", + "container" => Dict("cmd" => ``, "memory" => 1024, "vcpus" => 1), + "id" => "", + ) + + expected_job_definition = Dict( + "definition" => "", + "image" => "", + "role" => "", + "vcpus" => 1, + "memory" => 1024, + "cmd" => ``, + ) + + patches = [ + @patch register( + definition; + image="", + role="", + vcpus="", + memory=1024, + cmd="" + ) = verify_job_definition( + definition, + image, + role, + vcpus, + memory, + cmd, + expected_job_definition + ) + @patch submit( + name, + definition, + queue; + container=Dict() + ) = verify_job_submission( + name, + definition, + queue, + container, + expected_job_parameters + ) + ] + + apply(patches) do + job = run_batch() + @test job.id == "" end end - @testset "From Current Job" begin - withenv(BATCH_ENVS...) do - patches = [ - @patch readstring(cmd::AbstractCmd) = mock_readstring(cmd) - @patch describe_jobs(args...) = DESCRIBE_JOBS_RESP - ] - - apply(patches; debug=true) do - job = BatchJob() - - @test job.id == "24fa2d7a-64c4-49d2-8b47-f8da4fbde8e9" - @test job.name == "example" - @test job.queue == "HighPriority" - @test job.region == "us-east-" - - @test job.definition.name == "sleep60" - - - @test job.container.image == "busybox" - @test job.container.vcpus == 1 - @test job.container.memory == 128 - @test job.container.role == "arn:aws:iam::012345678910:role/sleep60" - @test job.container.cmd == `sleep 60` - end + @testset "From Job Definition" begin + expected_job_parameters = Dict( + "name" => "AWSBatchTest", + "definition" => "sleep60", + "queue" => "", + "container" => Dict("cmd" => `sleep 60`, "memory" => 128, "vcpus" => 1), + "id" => "24fa2d7a-64c4-49d2-8b47-f8da4fbde8e9", + ) + + expected_job_definition = Dict( + "definition" => "sleep60", + "image" => "busybox", + "role" => "arn:aws:iam::012345678910:role/sleep60", + "vcpus" => 1, + "memory" => 128, + "cmd" => `sleep 60`, + ) + + patches = [ + @patch describe_job_definitions(args...) = DESCRIBE_JOBS_DEF_RESP + @patch job_definition_arn(definition; image="", role="") = nothing + @patch register( + definition; + image="", + role="", + vcpus="", + memory=1024, + cmd="" + ) = verify_job_definition( + definition, + image, + role, + vcpus, + memory, + cmd, + expected_job_definition + ) + @patch submit( + name, + definition, + queue; + container=Dict() + ) = verify_job_submission( + name, + definition, + queue, + container, + expected_job_parameters + ) + ] + + apply(patches) do + job = run_batch(; name=JOB_NAME, definition="sleep60") + @test job.id == "24fa2d7a-64c4-49d2-8b47-f8da4fbde8e9" end end - @testset "From Multiple" begin + @testset "From Current Job" begin withenv(BATCH_ENVS...) do + expected_job_parameters = Dict( + "name" => "example", + "definition" => "sleep60", + "queue" => "HighPriority", + "container" => Dict("cmd" => `sleep 60`, "memory" => 128, "vcpus" => 1), + "id" => "24fa2d7a-64c4-49d2-8b47-f8da4fbde8e9", + ) + + expected_job_definition = Dict( + "definition" => "sleep60", + "image" => "busybox", + "role" => "arn:aws:iam::012345678910:role/sleep60", + "vcpus" => 1, + "memory" => 128, + "cmd" => `sleep 60`, + ) + patches = [ - @patch readstring(cmd::AbstractCmd) = mock_readstring(cmd) @patch describe_jobs(args...) = DESCRIBE_JOBS_RESP + @patch job_definition_arn(definition; image="", role="") = nothing + @patch register( + definition; + image="", + role="", + vcpus="", + memory=1024, + cmd="" + ) = verify_job_definition( + definition, + image, + role, + vcpus, + memory, + cmd, + expected_job_definition + ) + @patch submit( + name, + definition, + queue; + container=Dict() + ) = verify_job_submission( + name, + definition, + queue, + container, + expected_job_parameters + ) ] - apply(patches; debug=true) do - job = BatchJob() - - @test job.id == "24fa2d7a-64c4-49d2-8b47-f8da4fbde8e9" - @test job.name == "example" - @test job.definition.name == "sleep60" - @test job.queue == "HighPriority" - @test job.region == "us-east-" - - @test job.container.image == "busybox" - @test job.container.vcpus == 1 - @test job.container.memory == 128 - @test job.container.role == "arn:aws:iam::012345678910:role/sleep60" - @test job.container.cmd == `sleep 60` + apply(patches) do + job = run_batch() + @test job.id == "24fa2d7a-64c4-49d2-8b47-f8da4fbde8e9" end end end + end - @testset "Reuse job definition" begin - withenv(BATCH_ENVS...) do - patches = [ - @patch readstring(cmd::AbstractCmd) = mock_readstring(cmd) - @patch describe_jobs(args...) = DESCRIBE_JOBS_RESP - ] - - apply(patches; debug=true) do - job = BatchJob() + @testset "Online" begin + info("Running ONLINE tests") + + @testset "Job Submission" begin + job = run_batch(; + name = JOB_NAME, + definition = JOB_DEFINITION, + queue = JOB_QUEUE, + image = IMAGE_DEFINITION, + vcpus = 1, + memory = 1024, + role = JOB_ROLE, + cmd = `julia -e 'println("Hello World!")'`, + ) + + @test wait(job, [AWSBatch.SUCCEEDED]) == true + @test status(job) == AWSBatch.SUCCEEDED + + # Test job details were set correctly + job_details = describe(job) + @test job_details["jobName"] == JOB_NAME + @test job_details["jobQueue"] == ( + "arn:aws:batch:us-east-1:292522074875:job-queue/Replatforming-Manager" + ) + + # Test job definition and container parameters were set correctly + job_definition = JobDefinition(job_details["jobDefinition"]) + @test isregistered(job_definition) == true + + job_definition_details = first(describe(job_definition)["jobDefinitions"]) + + @test job_definition_details["jobDefinitionName"] == JOB_DEFINITION + @test job_definition_details["status"] == "ACTIVE" + @test job_definition_details["type"] == "container" + + container_properties = job_definition_details["containerProperties"] + @test container_properties["image"] == IMAGE_DEFINITION + @test container_properties["vcpus"] == 1 + @test container_properties["memory"] == 1024 + @test container_properties["command"] == [ + "julia", + "-e", + "println(\"Hello World!\")" + ] + @test container_properties["jobRoleArn"] == JOB_ROLE + + deregister(job_definition) + + events = log_events(job) + @test length(events) == 1 + @test contains(first(events).message, "Hello World!") + end - @test job.id == "24fa2d7a-64c4-49d2-8b47-f8da4fbde8e9" - @test job.name == "example" - @test job.definition.name == "sleep60" - @test job.queue == "HighPriority" - @test job.region == "us-east-" + @testset "Job Timed Out" begin + job = run_batch(; + name = JOB_NAME, + definition = JOB_DEFINITION, + queue = JOB_QUEUE, + image = IMAGE_DEFINITION, + vcpus = 1, + memory = 1024, + role = JOB_ROLE, + cmd = `sleep 60`, + ) - @test job.container.image == "busybox" - @test job.container.vcpus == 1 - @test job.container.memory == 128 - @test job.container.role == "arn:aws:iam::012345678910:role/sleep60" - @test job.container.cmd == `sleep 60` - end - end - end + job_definition = JobDefinition(describe(job)["jobDefinition"]) + @test isregistered(job_definition) == true - end + info("Testing job timeout") + @test_throws ErrorException wait(job, [AWSBatch.SUCCEEDED]; timeout=0) - @testset "Job Submission" begin - job = BatchJob(; - name = JOB_NAME, - definition = JOB_DEFINITION, - queue = JOB_QUEUE, - image = IMAGE_DEFINITION, - vcpus = 1, - memory = 1024, - role = JOB_ROLE, - cmd = `julia -e 'println("Hello World!")'`, - ) - - submit!(job) - @test isregistered(job) == true - @test wait(job, [AWSBatch.SUCCEEDED]) == true - @test status(job) == AWSBatch.SUCCEEDED - deregister!(job) - events = log_events(job) - - @test length(events) == 1 - @test contains(first(events).message, "Hello World!") - end + deregister(job_definition) - @testset "Job Timed Out" begin - job = BatchJob(; - name = JOB_NAME, - definition = JOB_DEFINITION, - queue = JOB_QUEUE, - image = IMAGE_DEFINITION, - vcpus = 1, - memory = 1024, - role = JOB_ROLE, - cmd = `sleep 60`, - ) - - submit!(job) - @test isregistered(job) == true - @test_throws ErrorException wait(job, [AWSBatch.SUCCEEDED]; timeout=0) - deregister!(job) - events = log_events(job) - - @test length(events) == 0 - end + events = log_events(job) + @test length(events) == 0 + end - @testset "Failed Job" begin - job = BatchJob(; - name = JOB_NAME, - definition = JOB_DEFINITION, - queue = JOB_QUEUE, - image = IMAGE_DEFINITION, - vcpus = 1, - memory = 1024, - role = JOB_ROLE, - cmd = `julia -e 'error("Cmd failed")'`, - ) - - submit!(job) - @test isregistered(job) == true - @test_throws ErrorException wait(job, [AWSBatch.SUCCEEDED]) - deregister!(job) - events = log_events(job) - - @test length(events) == 3 - @test contains(first(events).message, "ERROR: Cmd failed") + @testset "Failed Job" begin + job = run_batch(; + name = JOB_NAME, + definition = JOB_DEFINITION, + queue = JOB_QUEUE, + image = IMAGE_DEFINITION, + vcpus = 1, + memory = 1024, + role = JOB_ROLE, + cmd = `julia -e 'error("Cmd failed")'`, + ) + + job_definition = JobDefinition(describe(job)["jobDefinition"]) + @test isregistered(job_definition) == true + + info("Testing job failure") + @test_throws ErrorException wait(job, [AWSBatch.SUCCEEDED]) + + deregister(job_definition) + + events = log_events(job) + @test length(events) == 3 + @test contains(first(events).message, "ERROR: Cmd failed") + end end end From 37b9cebcd7459341b1e2ceebe3a3c1a7e7fd7e2e Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Thu, 26 Apr 2018 10:59:34 -0500 Subject: [PATCH 018/110] Clean up interface refactoring --- docs/src/index.md | 3 +- src/AWSBatch.jl | 54 +++++++++-- src/batch_job.jl | 31 +++++-- src/deprecated.jl | 12 +-- src/job_definition.jl | 11 ++- test/mock.jl | 28 ++++++ test/runtests.jl | 211 +++++++++++------------------------------- 7 files changed, 167 insertions(+), 183 deletions(-) diff --git a/docs/src/index.md b/docs/src/index.md index 691b839..4ac5927 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -57,6 +57,7 @@ run_batch() AWSBatch.BatchJob AWSBatch.submit(::AbstractString, ::JobDefinition, ::AbstractString) AWSBatch.describe(::BatchJob) +AWSBatch.JobDefinition(::BatchJob) AWSBatch.status(::BatchJob) Base.wait(::Function, ::BatchJob) Base.wait(::BatchJob, ::Vector{JobState}, ::Vector{JobState}) @@ -67,7 +68,7 @@ AWSBatch.log_events(::BatchJob) ```@docs AWSBatch.JobDefinition -AWSBatch.register(::AbstractString) # Does this display kwargs correctly +AWSBatch.register(::AbstractString) AWSBatch.job_definition_arn(::JobDefinition) AWSBatch.deregister(::JobDefinition) AWSBatch.isregistered(::JobDefinition) diff --git a/src/AWSBatch.jl b/src/AWSBatch.jl index c0b5ced..55646fb 100644 --- a/src/AWSBatch.jl +++ b/src/AWSBatch.jl @@ -1,7 +1,8 @@ __precompile__() module AWSBatch -using AWSSDK +using AWSCore: AWSConfig, AWSCredentials + using AWSSDK.Batch using AWSSDK.CloudWatchLogs using AWSSDK.S3 @@ -10,6 +11,7 @@ using Compat using Memento using Mocking +import Base: showerror import Compat: Nothing export @@ -38,6 +40,16 @@ include("job_definition.jl") include("batch_job.jl") +struct BatchEnvironmentError <: Exception + message::String +end + +function showerror(io::IO, e::BatchEnvironmentError) + print(io, "BatchEnvironmentError: ") + print(io, e.message) +end + + """ run_batch(; name::AbstractString="", @@ -91,7 +103,7 @@ function run_batch(; # Update container override parameters vcpus == 1 && (vcpus = container["vcpus"]) - memory == 1024 && (memory = container["memory"]) + (memory == 1024 || memory < 1) && (memory = container["memory"]) isempty(cmd) && (cmd = Cmd(Vector{String}(container["command"]))) end end @@ -103,6 +115,15 @@ function run_batch(; job_id = ENV["AWS_BATCH_JOB_ID"] job_queue = ENV["AWS_BATCH_JQ_NAME"] + # Get the zone information from the EC2 instance metadata. + zone = @mock readstring( + pipeline( + `curl http://169.254.169.254/latest/meta-data/placement/availability-zone`; + stderr=DevNull + ) + ) + isempty(region) && (region = chop(zone)) + # Requires permissions to access to "batch:DescribeJobs" response = @mock describe_jobs(Dict("jobs" => [job_id])) @@ -123,40 +144,53 @@ function run_batch(; # Update container overrides vcpus == 1 && (vcpus = container["vcpus"]) - memory == 1024 && (memory = container["memory"]) + (memory == 1024 || memory < 1) && (memory = container["memory"]) isempty(cmd) && (cmd = Cmd(Vector{String}(container["command"]))) else warn(logger, "No jobs found with id: $job_id.") end end + # Error if required parameters were not explicitly set and cannot be inferred + if isempty(name) || isempty(queue) || memory < 1 + throw(BatchEnvironmentError( + "Unable to perform AWS Batch introspection when not running within " * + "an AWS Batch job. Current job parameters are: " * + "name: $name\n" * + "queue: $queue\n" * + "memory: $memory\n" + )) + end + # Reuse a previously registered job definition if available. # If no job definition exists that can be reused, a new job definition is created # under the current job specifications. if definition !== nothing - reusable_def = @mock job_definition_arn(definition; image=image, role=role) + reusable_def = job_definition_arn(definition; image=image, role=role) if reusable_def !== nothing definition = reusable_def else - definition = @mock register( + definition = register( definition.name; image=image, role=role, vcpus=vcpus, memory=memory, cmd=cmd, + region=region, ) end else # Use the job name as the definiton name since the definition name was not specified - definition = @mock register( + definition = register( name; image=image, role=role, vcpus=vcpus, memory=memory, cmd=cmd, + region=region, ) end @@ -168,7 +202,13 @@ function run_batch(; "cmd" => cmd, ) - return @mock submit(name, definition, queue; container=container_overrides) + return submit( + name, + definition, + queue; + container=container_overrides, + region=region, + ) end include("deprecated.jl") diff --git a/src/batch_job.jl b/src/batch_job.jl index 052bd48..e05c500 100644 --- a/src/batch_job.jl +++ b/src/batch_job.jl @@ -21,11 +21,12 @@ end """ submit( - name::AbstractString, - definition::JobDefinition, - queue::AbstractString; - container::AbstractDict=Dict() -) -> BatchJob + name::AbstractString, + definition::JobDefinition, + queue::AbstractString; + container::AbstractDict=Dict(), + region::AbstractString="", + ) -> BatchJob Handles submitting the batch job. Returns a `BatchJob` wrapper for the id. """ @@ -33,8 +34,12 @@ function submit( name::AbstractString, definition::JobDefinition, queue::AbstractString; - container::AbstractDict=Dict() + container::AbstractDict=Dict(), + region::AbstractString="", ) + region = isempty(region) ? "us-east-1" : region + config = AWSConfig(:creds => AWSCredentials(), :region => region) + debug(logger, "Submitting job $name.") input = [ "jobName" => name, @@ -44,7 +49,7 @@ function submit( ] debug(logger, "Input: $input") - response = submit_job(input) + response = @mock submit_job(config, input) job = BatchJob(response["jobId"]) info(logger, "Submitted job $(name)::$(job.id).") @@ -55,8 +60,7 @@ end """ describe(job::BatchJob) -> Dict -If job.id is set then this function is simply responsible for fetch a dictionary for -describing the batch job. +Provides details about the AWS batch job. """ function describe(job::BatchJob) response = @mock describe_jobs(; jobs=[job.id]) @@ -65,8 +69,15 @@ function describe(job::BatchJob) return first(response["jobs"]) end +""" + JobDefinition + +Returns the job definition corresponding to a batch job. +""" +JobDefinition(job::BatchJob) = JobDefinition(describe(job)["jobDefinition"]) -""" status(job::BatchJob) -> JobState +""" + status(job::BatchJob) -> JobState Returns the current status of a job. """ diff --git a/src/deprecated.jl b/src/deprecated.jl index 0bb5333..1e24519 100644 --- a/src/deprecated.jl +++ b/src/deprecated.jl @@ -3,14 +3,14 @@ using Base: @deprecate @deprecate logs(job::BatchJob) [Dict("eventId" => e.id, "ingestionTime" => e.ingestion_time, "timestamp" => e.timestamp, "message" => e.message) for e in log_events(job)] @deprecate BatchStatus JobState -@deprecate BatchJob(; kwargs...) run_batch(; filter((x)->(x[1] != :id), kwargs)...) # id is no longer a valid kwarg -@deprecate submit!(job::BatchJob) job # Creating BatchJob's now automatically submits the job and submit doesn't need to be explicitly called -@deprecate job_definition_arn(job::BatchJob) job_definition_arn(JobDefinition(describe(job)["jobDefinition"])) +@deprecate BatchJob(; id=id, kwargs...) BatchJob(id) +@deprecate submit!(job::BatchJob) submit +@deprecate job_definition_arn(job::BatchJob) job_definition_arn(JobDefinition(job)) # Deprecate methods that now are called explicitly on JobDefinition's and not on BatchJob's -@deprecate isregistered(job::BatchJob) isregistered(JobDefinition(describe(job)["jobDefinition"])) -@deprecate register!(job::BatchJob) JobDefinition(describe(job)["jobDefinition"]) # Job definition should already be registered for a created BatchJob, so just look it up -@deprecate deregister!(job::BatchJob) deregister(JobDefinition(describe(job)["jobDefinition"])) +@deprecate isregistered(job::BatchJob) isregistered(JobDefinition(job)) +@deprecate register!(job::BatchJob) register +@deprecate deregister!(job::BatchJob) deregister(JobDefinition(job)) @deprecate register(job_definition::JobDefinition) register(job_definition.name) diff --git a/src/job_definition.jl b/src/job_definition.jl index 7ce86c8..b49dea3 100644 --- a/src/job_definition.jl +++ b/src/job_definition.jl @@ -17,7 +17,8 @@ end image::AbstractString="", vcpus::Integer=1, memory::Integer=1024, - cmd::Cmd=`` + cmd::Cmd=``, + region::AbstractString="", ) -> JobDefinition Registers a new job definition. @@ -28,8 +29,12 @@ function register( role::AbstractString="", vcpus::Integer=1, memory::Integer=1024, - cmd::Cmd=`` + cmd::Cmd=``, + region::AbstractString="", ) + region = isempty(region) ? "us-east-1" : region + config = AWSConfig(:creds => AWSCredentials(), :region => region) + debug(logger, "Registering job definition $definition.") input = [ "type" => "container", @@ -43,7 +48,7 @@ function register( "jobDefinitionName" => definition, ] - response = register_job_definition(input) + response = @mock register_job_definition(config, input) definition = JobDefinition(response["jobDefinitionArn"]) info(logger, "Registered job definition $(definition.name).") return definition diff --git a/test/mock.jl b/test/mock.jl index b63aa4d..3d2e657 100644 --- a/test/mock.jl +++ b/test/mock.jl @@ -5,6 +5,17 @@ const BATCH_ENVS = ( "AWS_BATCH_JQ_NAME" => "HighPriority" ) +const SUBMIT_JOB_RESP = Dict( + "jobName" => "example", + "jobId" => "24fa2d7a-64c4-49d2-8b47-f8da4fbde8e9", +) + +const REGISTER_JOB_DEF_RESP = Dict( + "jobDefinitionName" => "sleep60", + "jobDefinitionArn" => "arn:aws:batch:us-east-1:012345678910:job-definition/sleep60:1", + "revision"=>1, +) + const DESCRIBE_JOBS_DEF_RESP = Dict( "jobDefinitions" => [ Dict( @@ -83,3 +94,20 @@ const DESCRIBE_JOBS_RESP = Dict( ) ] ) + + +""" + Mock.readstring(cmd::CmdRedirect, pass::Bool=true) + +Mocks the CmdRedirect produced from ``pipeline(`curl http://169.254.169.254/latest/meta-data/placement/availability-zone`)`` +to just return "us-east-1". +""" +function mock_readstring(cmd::CmdRedirect) + cmd_exec = cmd.cmd.exec + + result = if cmd_exec[1] == "curl" && contains(cmd_exec[2], "availability-zone") + return "us-east-1" + else + return Base.readstring(cmd) + end +end diff --git a/test/runtests.jl b/test/runtests.jl index 4377419..7b906bf 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,6 +2,8 @@ using Mocking Mocking.enable() using AWSBatch +using AWSCore: AWSConfig + using Base.Test using Memento @@ -17,24 +19,18 @@ setlevel!(getlogger(AWSBatch), "info") include("mock.jl") -function verify_job_submission(name, definition, queue, container, expected) - @test name == expected["name"] - @test definition.name == expected["definition"] - @test queue == expected["queue"] - @test container == expected["container"] - - return BatchJob(expected["id"]) +function register_job_def(config::AWSConfig, input::AbstractArray, expected::AbstractArray) + @test input == expected + return REGISTER_JOB_DEF_RESP end -function verify_job_definition(definition, image, role, vcpus, memory, cmd, expected) - @test definition == expected["definition"] - @test image == expected["image"] - @test role == expected["role"] - @test vcpus == expected["vcpus"] - @test memory == expected["memory"] - @test cmd == expected["cmd"] +function submit_job(config::AWSConfig, input::AbstractArray, expected::AbstractArray) + @test input == expected + + cmd = Dict(input)["containerOverrides"]["cmd"] + @spawn run(cmd) - return JobDefinition(definition) + return SUBMIT_JOB_RESP end @@ -43,168 +39,71 @@ end include("job_state.jl") @testset "`run_batch` preprocessing" begin - @testset "Defaults" begin - expected_job_parameters = Dict( - "name" => "", - "definition" => "", - "queue" => "", - "container" => Dict("cmd" => ``, "memory" => 1024, "vcpus" => 1), - "id" => "", - ) - - expected_job_definition = Dict( - "definition" => "", - "image" => "", - "role" => "", - "vcpus" => 1, - "memory" => 1024, - "cmd" => ``, - ) - - patches = [ - @patch register( - definition; - image="", - role="", - vcpus="", - memory=1024, - cmd="" - ) = verify_job_definition( - definition, - image, - role, - vcpus, - memory, - cmd, - expected_job_definition - ) - @patch submit( - name, - definition, - queue; - container=Dict() - ) = verify_job_submission( - name, - definition, - queue, - container, - expected_job_parameters - ) - ] - apply(patches) do - job = run_batch() - @test job.id == "" + @testset "Defaults" begin + withenv("AWS_BATCH_JOB_ID" => nothing) do + @test_throws AWSBatch.BatchEnvironmentError run_batch() end end @testset "From Job Definition" begin - expected_job_parameters = Dict( - "name" => "AWSBatchTest", - "definition" => "sleep60", - "queue" => "", - "container" => Dict("cmd" => `sleep 60`, "memory" => 128, "vcpus" => 1), - "id" => "24fa2d7a-64c4-49d2-8b47-f8da4fbde8e9", - ) - - expected_job_definition = Dict( - "definition" => "sleep60", - "image" => "busybox", - "role" => "arn:aws:iam::012345678910:role/sleep60", - "vcpus" => 1, - "memory" => 128, - "cmd" => `sleep 60`, - ) + expected_job = [ + "jobName" => "example", + "jobDefinition" => "arn:aws:batch:us-east-1:012345678910:job-definition/sleep60:1", + "jobQueue" => "HighPriority", + "containerOverrides" => Dict( + "cmd" => `sleep 60`, + "memory" => 128, + "vcpus" => 1, + ), + ] patches = [ @patch describe_job_definitions(args...) = DESCRIBE_JOBS_DEF_RESP - @patch job_definition_arn(definition; image="", role="") = nothing - @patch register( - definition; - image="", - role="", - vcpus="", - memory=1024, - cmd="" - ) = verify_job_definition( - definition, - image, - role, - vcpus, - memory, - cmd, - expected_job_definition - ) - @patch submit( - name, - definition, - queue; - container=Dict() - ) = verify_job_submission( - name, - definition, - queue, - container, - expected_job_parameters - ) + @patch submit_job(config, input) = submit_job(config, input, expected_job) ] apply(patches) do - job = run_batch(; name=JOB_NAME, definition="sleep60") + job = run_batch(; name="example", definition="sleep60", queue="HighPriority") @test job.id == "24fa2d7a-64c4-49d2-8b47-f8da4fbde8e9" end end @testset "From Current Job" begin withenv(BATCH_ENVS...) do - expected_job_parameters = Dict( - "name" => "example", - "definition" => "sleep60", - "queue" => "HighPriority", - "container" => Dict("cmd" => `sleep 60`, "memory" => 128, "vcpus" => 1), - "id" => "24fa2d7a-64c4-49d2-8b47-f8da4fbde8e9", - ) - - expected_job_definition = Dict( - "definition" => "sleep60", - "image" => "busybox", - "role" => "arn:aws:iam::012345678910:role/sleep60", - "vcpus" => 1, - "memory" => 128, - "cmd" => `sleep 60`, - ) + expected_job = [ + "jobName" => "example", + "jobDefinition" => "arn:aws:batch:us-east-1:012345678910:job-definition/sleep60:1", + "jobQueue" => "HighPriority", + "containerOverrides" => Dict( + "cmd" => `sleep 60`, + "memory" => 128, + "vcpus" => 1, + ), + ] + + expected_job_def = [ + "type" => "container", + "containerProperties" => [ + "image" => "busybox", + "vcpus" => 1, + "memory" => 128, + "command" => ["sleep", "60"], + "jobRoleArn" => "arn:aws:iam::012345678910:role/sleep60", + ], + "jobDefinitionName" => "sleep60", + ] patches = [ + @patch readstring(cmd::AbstractCmd) = mock_readstring(cmd) @patch describe_jobs(args...) = DESCRIBE_JOBS_RESP - @patch job_definition_arn(definition; image="", role="") = nothing - @patch register( - definition; - image="", - role="", - vcpus="", - memory=1024, - cmd="" - ) = verify_job_definition( - definition, - image, - role, - vcpus, - memory, - cmd, - expected_job_definition - ) - @patch submit( - name, - definition, - queue; - container=Dict() - ) = verify_job_submission( - name, - definition, - queue, - container, - expected_job_parameters + @patch describe_job_definitions(args...) = Dict("jobDefinitions" => Dict()) + @patch register_job_definition(config, input) = register_job_def( + config, + input, + expected_job_def, ) + @patch submit_job(config, input) = submit_job(config, input, expected_job) ] apply(patches) do From de319a480f4f5d7430ce4efc09a255333a81a53a Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Thu, 26 Apr 2018 13:46:29 -0500 Subject: [PATCH 019/110] Update REQUIRE file --- REQUIRE | 1 + 1 file changed, 1 insertion(+) diff --git a/REQUIRE b/REQUIRE index aba931e..3b8377d 100644 --- a/REQUIRE +++ b/REQUIRE @@ -1,4 +1,5 @@ julia 0.6 +AWSCore 0.3.0 AWSSDK 0.2.0 Compat 0.41.0 Mocking 0.3.3 From 2d1b61b3a8ecabee4251bc158b75a45e76d8f525 Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Fri, 27 Apr 2018 15:03:07 -0500 Subject: [PATCH 020/110] Support running tests without using the legacy AWS account --- .gitlab-ci.yml | 2 +- test/REQUIRE | 1 + test/run_batch.jl | 93 +++++++++++++++ test/runtests.jl | 297 +++++++++++++++++----------------------------- 4 files changed, 204 insertions(+), 189 deletions(-) create mode 100644 test/REQUIRE create mode 100644 test/run_batch.jl diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 47222a0..63c5e63 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,7 +7,7 @@ stages: stage: test image: 292522074875.dkr.ecr.us-east-1.amazonaws.com/julia-gitlab-ci:0.6 variables: - LIVE: "true" # Runs the online tests against AWS + ONLINE: "batch" # Runs the online tests against AWS artifacts: name: "$CI_JOB_NAME coverage" expire_in: 1 week diff --git a/test/REQUIRE b/test/REQUIRE new file mode 100644 index 0000000..6693cd1 --- /dev/null +++ b/test/REQUIRE @@ -0,0 +1 @@ +AWSTools 0.1.0 diff --git a/test/run_batch.jl b/test/run_batch.jl new file mode 100644 index 0000000..484c83d --- /dev/null +++ b/test/run_batch.jl @@ -0,0 +1,93 @@ +include("mock.jl") + + +function register_job_def(config::AWSConfig, input::AbstractArray, expected::AbstractArray) + @test input == expected + return REGISTER_JOB_DEF_RESP +end + +function submit_job(config::AWSConfig, input::AbstractArray, expected::AbstractArray) + @test input == expected + + cmd = Dict(input)["containerOverrides"]["cmd"] + @spawn run(cmd) + + return SUBMIT_JOB_RESP +end + + +@testset "run_batch" begin + + @testset "Defaults" begin + withenv("AWS_BATCH_JOB_ID" => nothing) do + @test_throws AWSBatch.BatchEnvironmentError run_batch() + end + end + + @testset "From Job Definition" begin + expected_job = [ + "jobName" => "example", + "jobDefinition" => "arn:aws:batch:us-east-1:012345678910:job-definition/sleep60:1", + "jobQueue" => "HighPriority", + "containerOverrides" => Dict( + "cmd" => `sleep 60`, + "memory" => 128, + "vcpus" => 1, + ), + ] + + patches = [ + @patch describe_job_definitions(args...) = DESCRIBE_JOBS_DEF_RESP + @patch submit_job(config, input) = submit_job(config, input, expected_job) + ] + + apply(patches) do + job = run_batch(; name="example", definition="sleep60", queue="HighPriority") + @test job.id == "24fa2d7a-64c4-49d2-8b47-f8da4fbde8e9" + end + end + + @testset "From Current Job" begin + withenv(BATCH_ENVS...) do + expected_job = [ + "jobName" => "example", + "jobDefinition" => "arn:aws:batch:us-east-1:012345678910:job-definition/sleep60:1", + "jobQueue" => "HighPriority", + "containerOverrides" => Dict( + "cmd" => `sleep 60`, + "memory" => 128, + "vcpus" => 1, + ), + ] + + expected_job_def = [ + "type" => "container", + "containerProperties" => [ + "image" => "busybox", + "vcpus" => 1, + "memory" => 128, + "command" => ["sleep", "60"], + "jobRoleArn" => "arn:aws:iam::012345678910:role/sleep60", + ], + "jobDefinitionName" => "sleep60", + ] + + patches = [ + @patch readstring(cmd::AbstractCmd) = mock_readstring(cmd) + @patch describe_jobs(args...) = DESCRIBE_JOBS_RESP + @patch describe_job_definitions(args...) = Dict("jobDefinitions" => Dict()) + @patch register_job_definition(config, input) = register_job_def( + config, + input, + expected_job_def, + ) + @patch submit_job(config, input) = submit_job(config, input, expected_job) + ] + + apply(patches) do + job = run_batch() + @test job.id == "24fa2d7a-64c4-49d2-8b47-f8da4fbde8e9" + end + end + end +end diff --git a/test/runtests.jl b/test/runtests.jl index 7b906bf..d3ee07f 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -3,217 +3,138 @@ Mocking.enable() using AWSBatch using AWSCore: AWSConfig +using AWSTools.CloudFormation: stack_output using Base.Test using Memento -const IMAGE_DEFINITION = "292522074875.dkr.ecr.us-east-1.amazonaws.com/aws-tools:latest" -const JOB_ROLE = "arn:aws:iam::292522074875:role/AWSBatchClusterManagerJobRole" -const JOB_DEFINITION = "AWSBatch" -const JOB_NAME = "AWSBatchTest" -const JOB_QUEUE = "Replatforming-Manager" - -Memento.config("debug"; fmt="[{level} | {name}]: {msg}") -setlevel!(getlogger(AWSBatch), "info") - -include("mock.jl") +# Enables the running of the "batch" online tests. e.g ONLINE=batch +const ONLINE = strip.(split(get(ENV, "ONLINE", ""), r"\s*,\s*")) -function register_job_def(config::AWSConfig, input::AbstractArray, expected::AbstractArray) - @test input == expected - return REGISTER_JOB_DEF_RESP -end - -function submit_job(config::AWSConfig, input::AbstractArray, expected::AbstractArray) - @test input == expected +# Partially emulates the output from the AWS batch manager test stack +const LEGACY_STACK = Dict( + "JobQueueArn" => "arn:aws:batch:us-east-1:292522074875:job-queue/Replatforming-Manager", + "JobName" => "AWSBatchTest", + "JobDefinitionName" => "AWSBatch", + "JobRoleArn" => "arn:aws:iam::292522074875:role/AWSBatchClusterManagerJobRole", + "EcrUri" => "292522074875.dkr.ecr.us-east-1.amazonaws.com/aws-tools:latest", +) - cmd = Dict(input)["containerOverrides"]["cmd"] - @spawn run(cmd) +const AWS_STACKNAME = get(ENV, "AWS_STACKNAME", "") +const STACK = isempty(AWS_STACKNAME) ? LEGACY_STACK : stack_output(AWS_STACKNAME) - return SUBMIT_JOB_RESP -end +Memento.config("debug"; fmt="[{level} | {name}]: {msg}") +setlevel!(getlogger(AWSBatch), "info") @testset "AWSBatch.jl" begin include("log_event.jl") include("job_state.jl") - - @testset "`run_batch` preprocessing" begin - - @testset "Defaults" begin - withenv("AWS_BATCH_JOB_ID" => nothing) do - @test_throws AWSBatch.BatchEnvironmentError run_batch() - end - end - - @testset "From Job Definition" begin - expected_job = [ - "jobName" => "example", - "jobDefinition" => "arn:aws:batch:us-east-1:012345678910:job-definition/sleep60:1", - "jobQueue" => "HighPriority", - "containerOverrides" => Dict( - "cmd" => `sleep 60`, - "memory" => 128, - "vcpus" => 1, - ), - ] - - patches = [ - @patch describe_job_definitions(args...) = DESCRIBE_JOBS_DEF_RESP - @patch submit_job(config, input) = submit_job(config, input, expected_job) - ] - - apply(patches) do - job = run_batch(; name="example", definition="sleep60", queue="HighPriority") - @test job.id == "24fa2d7a-64c4-49d2-8b47-f8da4fbde8e9" - end - end - - @testset "From Current Job" begin - withenv(BATCH_ENVS...) do - expected_job = [ - "jobName" => "example", - "jobDefinition" => "arn:aws:batch:us-east-1:012345678910:job-definition/sleep60:1", - "jobQueue" => "HighPriority", - "containerOverrides" => Dict( - "cmd" => `sleep 60`, - "memory" => 128, - "vcpus" => 1, - ), - ] - - expected_job_def = [ - "type" => "container", - "containerProperties" => [ - "image" => "busybox", - "vcpus" => 1, - "memory" => 128, - "command" => ["sleep", "60"], - "jobRoleArn" => "arn:aws:iam::012345678910:role/sleep60", - ], - "jobDefinitionName" => "sleep60", + include("run_batch.jl") + + if "batch" in ONLINE + @testset "Online" begin + info("Running ONLINE tests") + + @testset "Job Submission" begin + job = run_batch(; + name = STACK["JobName"], + definition = STACK["JobDefinitionName"], + queue = STACK["JobQueueArn"], + image = STACK["EcrUri"], + vcpus = 1, + memory = 1024, + role = STACK["JobRoleArn"], + cmd = `julia -e 'println("Hello World!")'`, + ) + + @test wait(job, [AWSBatch.SUCCEEDED]) == true + @test status(job) == AWSBatch.SUCCEEDED + + # Test job details were set correctly + job_details = describe(job) + @test job_details["jobName"] == STACK["JobName"] + @test job_details["jobQueue"] == STACK["JobQueueArn"] + + # Test job definition and container parameters were set correctly + job_definition = JobDefinition(job_details["jobDefinition"]) + @test isregistered(job_definition) == true + + job_definition_details = first(describe(job_definition)["jobDefinitions"]) + + @test job_definition_details["jobDefinitionName"] == STACK["JobDefinitionName"] + @test job_definition_details["status"] == "ACTIVE" + @test job_definition_details["type"] == "container" + + container_properties = job_definition_details["containerProperties"] + @test container_properties["image"] == STACK["EcrUri"] + @test container_properties["vcpus"] == 1 + @test container_properties["memory"] == 1024 + @test container_properties["command"] == [ + "julia", + "-e", + "println(\"Hello World!\")" ] + @test container_properties["jobRoleArn"] == STACK["JobRoleArn"] - patches = [ - @patch readstring(cmd::AbstractCmd) = mock_readstring(cmd) - @patch describe_jobs(args...) = DESCRIBE_JOBS_RESP - @patch describe_job_definitions(args...) = Dict("jobDefinitions" => Dict()) - @patch register_job_definition(config, input) = register_job_def( - config, - input, - expected_job_def, - ) - @patch submit_job(config, input) = submit_job(config, input, expected_job) - ] + deregister(job_definition) - apply(patches) do - job = run_batch() - @test job.id == "24fa2d7a-64c4-49d2-8b47-f8da4fbde8e9" - end + events = log_events(job) + @test length(events) == 1 + @test contains(first(events).message, "Hello World!") end - end - end - - @testset "Online" begin - info("Running ONLINE tests") - - @testset "Job Submission" begin - job = run_batch(; - name = JOB_NAME, - definition = JOB_DEFINITION, - queue = JOB_QUEUE, - image = IMAGE_DEFINITION, - vcpus = 1, - memory = 1024, - role = JOB_ROLE, - cmd = `julia -e 'println("Hello World!")'`, - ) - - @test wait(job, [AWSBatch.SUCCEEDED]) == true - @test status(job) == AWSBatch.SUCCEEDED - - # Test job details were set correctly - job_details = describe(job) - @test job_details["jobName"] == JOB_NAME - @test job_details["jobQueue"] == ( - "arn:aws:batch:us-east-1:292522074875:job-queue/Replatforming-Manager" - ) - - # Test job definition and container parameters were set correctly - job_definition = JobDefinition(job_details["jobDefinition"]) - @test isregistered(job_definition) == true - - job_definition_details = first(describe(job_definition)["jobDefinitions"]) - - @test job_definition_details["jobDefinitionName"] == JOB_DEFINITION - @test job_definition_details["status"] == "ACTIVE" - @test job_definition_details["type"] == "container" - - container_properties = job_definition_details["containerProperties"] - @test container_properties["image"] == IMAGE_DEFINITION - @test container_properties["vcpus"] == 1 - @test container_properties["memory"] == 1024 - @test container_properties["command"] == [ - "julia", - "-e", - "println(\"Hello World!\")" - ] - @test container_properties["jobRoleArn"] == JOB_ROLE - - deregister(job_definition) - - events = log_events(job) - @test length(events) == 1 - @test contains(first(events).message, "Hello World!") - end - @testset "Job Timed Out" begin - job = run_batch(; - name = JOB_NAME, - definition = JOB_DEFINITION, - queue = JOB_QUEUE, - image = IMAGE_DEFINITION, - vcpus = 1, - memory = 1024, - role = JOB_ROLE, - cmd = `sleep 60`, - ) + @testset "Job Timed Out" begin + job = run_batch(; + name = STACK["JobName"], + definition = STACK["JobDefinitionName"], + queue = STACK["JobQueueArn"], + image = STACK["EcrUri"], + vcpus = 1, + memory = 1024, + role = STACK["JobRoleArn"], + cmd = `sleep 60`, + ) - job_definition = JobDefinition(describe(job)["jobDefinition"]) - @test isregistered(job_definition) == true + job_definition = JobDefinition(describe(job)["jobDefinition"]) + @test isregistered(job_definition) == true - info("Testing job timeout") - @test_throws ErrorException wait(job, [AWSBatch.SUCCEEDED]; timeout=0) + info("Testing job timeout") + @test_throws ErrorException wait(job, [AWSBatch.SUCCEEDED]; timeout=0) - deregister(job_definition) + deregister(job_definition) - events = log_events(job) - @test length(events) == 0 - end + events = log_events(job) + @test length(events) == 0 + end - @testset "Failed Job" begin - job = run_batch(; - name = JOB_NAME, - definition = JOB_DEFINITION, - queue = JOB_QUEUE, - image = IMAGE_DEFINITION, - vcpus = 1, - memory = 1024, - role = JOB_ROLE, - cmd = `julia -e 'error("Cmd failed")'`, - ) - - job_definition = JobDefinition(describe(job)["jobDefinition"]) - @test isregistered(job_definition) == true - - info("Testing job failure") - @test_throws ErrorException wait(job, [AWSBatch.SUCCEEDED]) - - deregister(job_definition) - - events = log_events(job) - @test length(events) == 3 - @test contains(first(events).message, "ERROR: Cmd failed") + @testset "Failed Job" begin + job = run_batch(; + name = STACK["JobName"], + definition = STACK["JobDefinitionName"], + queue = STACK["JobQueueArn"], + image = STACK["EcrUri"], + vcpus = 1, + memory = 1024, + role = STACK["JobRoleArn"], + cmd = `julia -e 'error("Cmd failed")'`, + ) + + job_definition = JobDefinition(describe(job)["jobDefinition"]) + @test isregistered(job_definition) == true + + info("Testing job failure") + @test_throws ErrorException wait(job, [AWSBatch.SUCCEEDED]) + + deregister(job_definition) + + events = log_events(job) + @test length(events) == 3 + @test contains(first(events).message, "ERROR: Cmd failed") + end end + else + warn("Skipping ONLINE tests") end end From 3198427ff9c9223e49cd04deebe56869cc589fd9 Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Fri, 27 Apr 2018 15:22:43 -0500 Subject: [PATCH 021/110] Fix deprecations --- src/AWSBatch.jl | 6 +++--- src/batch_job.jl | 2 +- src/deprecated.jl | 19 +++++++++++++++---- test/runtests.jl | 6 ++++-- 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/src/AWSBatch.jl b/src/AWSBatch.jl index 55646fb..0178bc1 100644 --- a/src/AWSBatch.jl +++ b/src/AWSBatch.jl @@ -55,7 +55,7 @@ end name::AbstractString="", queue::AbstractString="", region::AbstractString="", - definition::Union{AbstractString, Nothing}=nothing, + definition::Union{AbstractString, JobDefinition, Nothing}=nothing, image::AbstractString="", vcpus::Integer=1, memory::Integer=1024, @@ -80,14 +80,14 @@ function run_batch(; name::AbstractString="", queue::AbstractString="", region::AbstractString="", - definition::Union{AbstractString, Nothing}=nothing, + definition::Union{AbstractString, JobDefinition, Nothing}=nothing, image::AbstractString="", vcpus::Integer=1, memory::Integer=1024, role::AbstractString="", cmd::Cmd=``, ) - if definition !== nothing + if isa(definition, AbstractString) definition = isempty(definition) ? nothing : JobDefinition(definition) end diff --git a/src/batch_job.jl b/src/batch_job.jl index e05c500..97ec1c9 100644 --- a/src/batch_job.jl +++ b/src/batch_job.jl @@ -40,7 +40,7 @@ function submit( region = isempty(region) ? "us-east-1" : region config = AWSConfig(:creds => AWSCredentials(), :region => region) - debug(logger, "Submitting job $name.") + debug(logger, "Submitting job $name") input = [ "jobName" => name, "jobDefinition" => definition.name, diff --git a/src/deprecated.jl b/src/deprecated.jl index 1e24519..c0a9f61 100644 --- a/src/deprecated.jl +++ b/src/deprecated.jl @@ -3,14 +3,25 @@ using Base: @deprecate @deprecate logs(job::BatchJob) [Dict("eventId" => e.id, "ingestionTime" => e.ingestion_time, "timestamp" => e.timestamp, "message" => e.message) for e in log_events(job)] @deprecate BatchStatus JobState -@deprecate BatchJob(; id=id, kwargs...) BatchJob(id) -@deprecate submit!(job::BatchJob) submit +@deprecate submit!(job::BatchJob) error("`submit!(job::BatchJob)` is no longer supported. Look at `submit` or `run_batch` to submit batch jobs") @deprecate job_definition_arn(job::BatchJob) job_definition_arn(JobDefinition(job)) # Deprecate methods that now are called explicitly on JobDefinition's and not on BatchJob's @deprecate isregistered(job::BatchJob) isregistered(JobDefinition(job)) -@deprecate register!(job::BatchJob) register -@deprecate deregister!(job::BatchJob) deregister(JobDefinition(job)) +@deprecate register!(job::BatchJob) error("`register`!(job::BatchJob)` is no longer supported. Look at `register` to register job definitions") +@deprecate deregister!(job::BatchJob) error("`deregister`!(job::BatchJob)` is no longer supported. Look at `deregister` to de-register job definitions") @deprecate register(job_definition::JobDefinition) register(job_definition.name) +function BatchJob(; id="", kwargs...) + if !isempty(id) && isempty(kwargs) + Base.depwarn("BatchJob(; id=id)` is deprecated, use `BatchJob(id)` instead", :BatchJob) + BatchJob(id) + elseif isempty(id) && !isempty(kwargs) + Base.depwarn("`BatchJob(; kwargs...)` is deprecated, use `run_batch(; kwargs...)` instead", :BatchJob) + run_batch(; kwargs...) + else + Base.depwarn("`BatchJob(; id=id, kwargs...)` is deprecated, ignoring `id` and using `run_batch(; kwargs...)` instead", :BatchJob) + run_batch(; kwargs...) + end +end diff --git a/test/runtests.jl b/test/runtests.jl index d3ee07f..1b33d80 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -86,6 +86,8 @@ setlevel!(getlogger(AWSBatch), "info") end @testset "Job Timed Out" begin + info("Testing job timeout") + job = run_batch(; name = STACK["JobName"], definition = STACK["JobDefinitionName"], @@ -100,7 +102,6 @@ setlevel!(getlogger(AWSBatch), "info") job_definition = JobDefinition(describe(job)["jobDefinition"]) @test isregistered(job_definition) == true - info("Testing job timeout") @test_throws ErrorException wait(job, [AWSBatch.SUCCEEDED]; timeout=0) deregister(job_definition) @@ -110,6 +111,8 @@ setlevel!(getlogger(AWSBatch), "info") end @testset "Failed Job" begin + info("Testing job failure") + job = run_batch(; name = STACK["JobName"], definition = STACK["JobDefinitionName"], @@ -124,7 +127,6 @@ setlevel!(getlogger(AWSBatch), "info") job_definition = JobDefinition(describe(job)["jobDefinition"]) @test isregistered(job_definition) == true - info("Testing job failure") @test_throws ErrorException wait(job, [AWSBatch.SUCCEEDED]) deregister(job_definition) From e0b528b0f8be0d6d204206704c94f90973f2cac5 Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Fri, 27 Apr 2018 16:00:36 -0500 Subject: [PATCH 022/110] Force mocking being enabled when running tests --- test/runtests.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index 1b33d80..7ef2e0d 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,5 +1,5 @@ using Mocking -Mocking.enable() +Mocking.enable(force=true) using AWSBatch using AWSCore: AWSConfig From 3d72895ff86a7d52890fcb8029c5552e9c32d579 Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Fri, 27 Apr 2018 17:06:17 -0500 Subject: [PATCH 023/110] Fix override command not getting set properly --- src/AWSBatch.jl | 4 ++-- test/run_batch.jl | 8 ++------ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/AWSBatch.jl b/src/AWSBatch.jl index 0178bc1..9998697 100644 --- a/src/AWSBatch.jl +++ b/src/AWSBatch.jl @@ -194,12 +194,12 @@ function run_batch(; ) end - # The parameters that can be overridden are `memory`, `vcpus`, `cmd`, and `environment` + # Parameters that can be overridden are `memory`, `vcpus`, `command`, and `environment` # See https://docs.aws.amazon.com/batch/latest/APIReference/API_ContainerOverrides.html container_overrides = Dict( "vcpus" => vcpus, "memory" => memory, - "cmd" => cmd, + "command" => cmd.exec, ) return submit( diff --git a/test/run_batch.jl b/test/run_batch.jl index 484c83d..2b37d28 100644 --- a/test/run_batch.jl +++ b/test/run_batch.jl @@ -8,10 +8,6 @@ end function submit_job(config::AWSConfig, input::AbstractArray, expected::AbstractArray) @test input == expected - - cmd = Dict(input)["containerOverrides"]["cmd"] - @spawn run(cmd) - return SUBMIT_JOB_RESP end @@ -30,7 +26,7 @@ end "jobDefinition" => "arn:aws:batch:us-east-1:012345678910:job-definition/sleep60:1", "jobQueue" => "HighPriority", "containerOverrides" => Dict( - "cmd" => `sleep 60`, + "command" => ["sleep", "60"], "memory" => 128, "vcpus" => 1, ), @@ -54,7 +50,7 @@ end "jobDefinition" => "arn:aws:batch:us-east-1:012345678910:job-definition/sleep60:1", "jobQueue" => "HighPriority", "containerOverrides" => Dict( - "cmd" => `sleep 60`, + "command" => ["sleep", "60"], "memory" => 128, "vcpus" => 1, ), From acbceb5d78aa938c6f6ecbe98ec17e6128128a35 Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Mon, 30 Apr 2018 10:31:36 -0500 Subject: [PATCH 024/110] Updates tests --- test/runtests.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 7ef2e0d..d45ea60 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -58,7 +58,7 @@ setlevel!(getlogger(AWSBatch), "info") @test job_details["jobQueue"] == STACK["JobQueueArn"] # Test job definition and container parameters were set correctly - job_definition = JobDefinition(job_details["jobDefinition"]) + job_definition = JobDefinition(job) @test isregistered(job_definition) == true job_definition_details = first(describe(job_definition)["jobDefinitions"]) @@ -99,7 +99,7 @@ setlevel!(getlogger(AWSBatch), "info") cmd = `sleep 60`, ) - job_definition = JobDefinition(describe(job)["jobDefinition"]) + job_definition = JobDefinition(job) @test isregistered(job_definition) == true @test_throws ErrorException wait(job, [AWSBatch.SUCCEEDED]; timeout=0) @@ -124,7 +124,7 @@ setlevel!(getlogger(AWSBatch), "info") cmd = `julia -e 'error("Cmd failed")'`, ) - job_definition = JobDefinition(describe(job)["jobDefinition"]) + job_definition = JobDefinition(job) @test isregistered(job_definition) == true @test_throws ErrorException wait(job, [AWSBatch.SUCCEEDED]) From 3e076fdb146d6169f60163c5e6eb8702606e1fec Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Mon, 30 Apr 2018 16:56:53 -0500 Subject: [PATCH 025/110] Store only the ARN in a JobDefinition --- docs/src/index.md | 4 +- src/AWSBatch.jl | 21 +++---- src/batch_job.jl | 2 +- src/deprecated.jl | 2 +- src/job_definition.jl | 132 +++++++++++++++++++++++------------------- src/log_event.jl | 2 - test/run_batch.jl | 32 +++++++++- 7 files changed, 118 insertions(+), 77 deletions(-) diff --git a/docs/src/index.md b/docs/src/index.md index 4ac5927..cd8ade4 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -68,11 +68,11 @@ AWSBatch.log_events(::BatchJob) ```@docs AWSBatch.JobDefinition +AWSBatch.job_definition_arn(::AbstractString) AWSBatch.register(::AbstractString) -AWSBatch.job_definition_arn(::JobDefinition) AWSBatch.deregister(::JobDefinition) AWSBatch.isregistered(::JobDefinition) -AWSBatch.describe(::JobDefinition) +AWSBatch.describe(::Union{AbstractString, JobDefinition}) ``` ### JobState diff --git a/src/AWSBatch.jl b/src/AWSBatch.jl index 9998697..06bb2e8 100644 --- a/src/AWSBatch.jl +++ b/src/AWSBatch.jl @@ -7,12 +7,13 @@ using AWSSDK.Batch using AWSSDK.CloudWatchLogs using AWSSDK.S3 -using Compat using Memento using Mocking +using Compat: Nothing +using Compat: AbstractDict + import Base: showerror -import Compat: Nothing export BatchJob, @@ -88,7 +89,7 @@ function run_batch(; cmd::Cmd=``, ) if isa(definition, AbstractString) - definition = isempty(definition) ? nothing : JobDefinition(definition) + definition = isempty(definition) ? nothing : definition end # Determine if the job definition already exists and update the default job parameters @@ -134,7 +135,7 @@ function run_batch(; # Update the job's required parameters isempty(name) && (name = details["jobName"]) - definition === nothing && (definition = JobDefinition(details["jobDefinition"])) + definition === nothing && (definition = details["jobDefinition"]) isempty(queue) && (queue = job_queue) # Update the container parameters @@ -165,14 +166,14 @@ function run_batch(; # Reuse a previously registered job definition if available. # If no job definition exists that can be reused, a new job definition is created # under the current job specifications. - if definition !== nothing - reusable_def = job_definition_arn(definition; image=image, role=role) + if isa(definition, AbstractString) + reusable_job_definition_arn = job_definition_arn(definition; image=image, role=role) - if reusable_def !== nothing - definition = reusable_def + if reusable_job_definition_arn !== nothing + definition = JobDefinition(reusable_job_definition_arn) else definition = register( - definition.name; + definition; image=image, role=role, vcpus=vcpus, @@ -181,7 +182,7 @@ function run_batch(; region=region, ) end - else + elseif definition === nothing # Use the job name as the definiton name since the definition name was not specified definition = register( name; diff --git a/src/batch_job.jl b/src/batch_job.jl index 97ec1c9..cac1153 100644 --- a/src/batch_job.jl +++ b/src/batch_job.jl @@ -43,7 +43,7 @@ function submit( debug(logger, "Submitting job $name") input = [ "jobName" => name, - "jobDefinition" => definition.name, + "jobDefinition" => definition.arn, "jobQueue" => queue, "containerOverrides" => container, ] diff --git a/src/deprecated.jl b/src/deprecated.jl index c0a9f61..cb13a33 100644 --- a/src/deprecated.jl +++ b/src/deprecated.jl @@ -11,7 +11,7 @@ using Base: @deprecate @deprecate register!(job::BatchJob) error("`register`!(job::BatchJob)` is no longer supported. Look at `register` to register job definitions") @deprecate deregister!(job::BatchJob) error("`deregister`!(job::BatchJob)` is no longer supported. Look at `deregister` to de-register job definitions") -@deprecate register(job_definition::JobDefinition) register(job_definition.name) +@deprecate register(job_definition::JobDefinition) register(job_definition.arn) function BatchJob(; id="", kwargs...) if !isempty(id) && isempty(kwargs) diff --git a/src/job_definition.jl b/src/job_definition.jl index b49dea3..b7ddc09 100644 --- a/src/job_definition.jl +++ b/src/job_definition.jl @@ -4,15 +4,68 @@ import AWSSDK.Batch: """ JobDefinition -Stores the job definition name or arn including the revision. +Stores the job definition arn including the revision. """ struct JobDefinition - name::AbstractString + arn::AbstractString + + function JobDefinition(name::AbstractString) + if startswith(name, "arn:") + new(name) + else + arn = job_definition_arn(name) + arn === nothing && error("No job definition ARN found for $name") + new(arn) + end + end +end + +""" + job_definition_arn( + definition_name::AbstractString; + image::AbstractString="", + role::AbstractString="" + ) -> Union{AbstractString, Nothing} + +Looks up the ARN (Amazon Resource Name) for the latest job definition that can be reused. +Returns a JobDefinition with the ARN that can be reused or `nothing`. + +A job definition can only be reused if: + +1. status = ACTIVE +2. type = container +3. image = the current job's image +4. jobRoleArn = the current job's role +""" +function job_definition_arn( + definition_name::AbstractString; + image::AbstractString="", + role::AbstractString="" +) + response = describe(definition_name) + isempty(response["jobDefinitions"]) && return nothing + + latest = first(response["jobDefinitions"]) + for definition in response["jobDefinitions"] + if definition["status"] == "ACTIVE" && definition["revision"] > latest["revision"] + latest = definition + end + end + if ( + latest["status"] == "ACTIVE" && + latest["type"] == "container" && + (latest["containerProperties"]["image"] == image || isempty(image)) && + (latest["containerProperties"]["jobRoleArn"] == role || isempty(role)) + ) + return latest["jobDefinitionArn"] + else + return nothing + end end """ register( - definition::AbstractString; + definition_name::AbstractString; role::AbstractString="", image::AbstractString="", vcpus::Integer=1, @@ -24,7 +77,7 @@ end Registers a new job definition. """ function register( - definition::AbstractString; + definition_name::AbstractString; image::AbstractString="", role::AbstractString="", vcpus::Integer=1, @@ -35,7 +88,7 @@ function register( region = isempty(region) ? "us-east-1" : region config = AWSConfig(:creds => AWSCredentials(), :region => region) - debug(logger, "Registering job definition $definition.") + debug(logger, "Registering job definition $definition_name.") input = [ "type" => "container", "containerProperties" => [ @@ -45,67 +98,24 @@ function register( "command" => cmd.exec, "jobRoleArn" => role, ], - "jobDefinitionName" => definition, + "jobDefinitionName" => definition_name, ] response = @mock register_job_definition(config, input) definition = JobDefinition(response["jobDefinitionArn"]) - info(logger, "Registered job definition $(definition.name).") + info(logger, "Registered job definition $(definition.arn).") return definition end -""" - job_definition_arn( - definition::JobDefinition; - image::AbstractString="", - role::AbstractString="" - ) -> Union{JobDefinition, Nothing} - -Looks up the ARN (Amazon Resource Name) for the latest job definition that can be reused. -Returns a JobDefinition with the ARN that can be reused or `nothing`. - -A job definition can only be reused if: - -1. status = ACTIVE -2. type = container -3. image = the current job's image -4. jobRoleArn = the current job's role -""" -function job_definition_arn( - definition::JobDefinition; - image::AbstractString="", - role::AbstractString="" -) - response = describe(definition) - isempty(response["jobDefinitions"]) && return nothing - - latest = first(response["jobDefinitions"]) - for definition in response["jobDefinitions"] - if definition["status"] == "ACTIVE" && definition["revision"] > latest["revision"] - latest = definition - end - end - if ( - latest["status"] == "ACTIVE" && - latest["type"] == "container" && - (latest["containerProperties"]["image"] == image || isempty(image)) && - (latest["containerProperties"]["jobRoleArn"] == role || isempty(role)) - ) - return JobDefinition(latest["jobDefinitionArn"]) - else - return nothing - end -end - """ deregister(job::JobDefinition) Deregisters an AWS Batch job. """ function deregister(definition::JobDefinition) - debug(logger, "Deregistering job definition $(definition.name).") - resp = deregister_job_definition(Dict("jobDefinition" => definition.name)) - info(logger, "Deregistered job definition $(definition.name).") + debug(logger, "Deregistering job definition $(definition.arn).") + resp = deregister_job_definition(Dict("jobDefinition" => definition.arn)) + info(logger, "Deregistered job definition $(definition.arn).") end """ @@ -120,15 +130,19 @@ function isregistered(definition::JobDefinition) end """ - describe(definition::JobDefinition) -> Dict + describe(definition::Union{AbstractString, JobDefinition}) -> Dict -Describes a job given it's definition. Returns the response dictionary. +Describes a job given it's definition name or arn. Returns the response dictionary. Requires permissions to access "batch:DescribeJobDefinitions". """ -function describe(definition::JobDefinition) - if startswith(definition.name, "arn:") - return @mock describe_job_definitions(Dict("jobDefinitions" => [definition.name])) +function describe(definition::Union{AbstractString, JobDefinition}) + if isa(definition, AbstractString) + if startswith(definition, "arn:") + return @mock describe_job_definitions(Dict("jobDefinitions" => [definition])) + else + return @mock describe_job_definitions(Dict("jobDefinitionName" => definition)) + end else - return @mock describe_job_definitions(Dict("jobDefinitionName" => definition.name)) + return @mock describe_job_definitions(Dict("jobDefinitions" => [definition.arn])) end end diff --git a/src/log_event.jl b/src/log_event.jl index e8518c9..6f7e55c 100644 --- a/src/log_event.jl +++ b/src/log_event.jl @@ -1,5 +1,3 @@ -using Compat: AbstractDict - """ LogEvent diff --git a/test/run_batch.jl b/test/run_batch.jl index 2b37d28..18b4590 100644 --- a/test/run_batch.jl +++ b/test/run_batch.jl @@ -81,8 +81,36 @@ end ] apply(patches) do - job = run_batch() - @test job.id == "24fa2d7a-64c4-49d2-8b47-f8da4fbde8e9" + job = run_batch() + @test job.id == "24fa2d7a-64c4-49d2-8b47-f8da4fbde8e9" + end + end + end + + @testset "Using a Job Definition" begin + withenv(BATCH_ENVS...) do + expected_job = [ + "jobName" => "example", + "jobDefinition" => "arn:aws:batch:us-east-1:012345678910:job-definition/sleep60:1", + "jobQueue" => "HighPriority", + "containerOverrides" => Dict( + "command" => ["sleep", "60"], + "memory" => 128, + "vcpus" => 1, + ), + ] + + patches = [ + @patch readstring(cmd::AbstractCmd) = mock_readstring(cmd) + @patch describe_jobs(args...) = DESCRIBE_JOBS_RESP + @patch describe_job_definitions(args...) = Dict("jobDefinitions" => Dict()) + @patch submit_job(config, input) = submit_job(config, input, expected_job) + ] + + apply(patches) do + definition = JobDefinition("arn:aws:batch:us-east-1:012345678910:job-definition/sleep60:1") + job = run_batch(definition=definition) + @test job.id == "24fa2d7a-64c4-49d2-8b47-f8da4fbde8e9" end end end From 9af5ffc8f1ae237c1d8ce1f7d16c0cf2fb5c77bd Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 2 May 2018 14:23:38 -0500 Subject: [PATCH 026/110] Refactor job definition further --- src/AWSBatch.jl | 2 +- src/job_definition.jl | 27 +++++++++++++-------------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/src/AWSBatch.jl b/src/AWSBatch.jl index 06bb2e8..3793391 100644 --- a/src/AWSBatch.jl +++ b/src/AWSBatch.jl @@ -94,7 +94,7 @@ function run_batch(; # Determine if the job definition already exists and update the default job parameters if definition !== nothing - response = describe(definition) + response = describe_job_definition(definition) if !isempty(response["jobDefinitions"]) details = first(response["jobDefinitions"]) diff --git a/src/job_definition.jl b/src/job_definition.jl index b7ddc09..5dc214b 100644 --- a/src/job_definition.jl +++ b/src/job_definition.jl @@ -42,7 +42,7 @@ function job_definition_arn( image::AbstractString="", role::AbstractString="" ) - response = describe(definition_name) + response = describe_job_definition(definition_name) isempty(response["jobDefinitions"]) && return nothing latest = first(response["jobDefinitions"]) @@ -125,24 +125,23 @@ Checks if a JobDefinition is registered. """ function isregistered(definition::JobDefinition) j = describe(definition) - active_definitions = filter!(d -> d["status"] == "ACTIVE", get(j, "jobDefinitions", [])) - return !isempty(active_definitions) + return any(d -> d["status"] == "ACTIVE", get(j, "jobDefinitions", [])) end """ - describe(definition::Union{AbstractString, JobDefinition}) -> Dict + describe(definition::JobDefinition) -> Dict -Describes a job given it's definition name or arn. Returns the response dictionary. -Requires permissions to access "batch:DescribeJobDefinitions". +Describes a job definition as a dictionary. Requires the IAM permissions +"batch:DescribeJobDefinitions". """ -function describe(definition::Union{AbstractString, JobDefinition}) - if isa(definition, AbstractString) - if startswith(definition, "arn:") - return @mock describe_job_definitions(Dict("jobDefinitions" => [definition])) - else - return @mock describe_job_definitions(Dict("jobDefinitionName" => definition)) - end +describe(definition::JobDefinition) = describe_job_definition(definition) + +describe_job_definition(definition::JobDefinition) = describe_job_definition(definition.arn) +function describe_job_definition(definition::AbstractString) + query = if startswith(definition, "arn:") + Dict("jobDefinitions" => [definition]) else - return @mock describe_job_definitions(Dict("jobDefinitions" => [definition.arn])) + Dict("jobDefinitionName" => definition) end + return @mock describe_job_definitions(query) end From 53d061f33a97343779a2fe200c8cfe189b641290 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 2 May 2018 14:52:44 -0500 Subject: [PATCH 027/110] Add CI for non-online tests --- .gitlab-ci.yml | 102 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 96 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 63c5e63..303c617 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,24 +1,114 @@ stages: - test + - online-test - coverage - docs +.test_shell: &test_shell + artifacts: + name: "$CI_JOB_NAME coverage" + expire_in: 1 week + paths: + - "$CI_JOB_NAME coverage/" + before_script: + - curl -s -o julia-ci https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/julia-ci + - chmod +x julia-ci + - ./julia-ci install $JULIA_VERSION + script: + - source julia-ci export + - julia -e "Pkg.clone(pwd()); Pkg.build(\"$PKG_NAME\"); Pkg.test(\"$PKG_NAME\"; coverage=true)" + - ./julia-ci coverage + after_script: + - ./julia-ci clean + +.test_shell_06: &test_shell_06 + variables: + JULIA_VERSION: "0.6" + ONLINE: "" + <<: *test_shell + +.test_shell_07-: &test_shell_07- + variables: + JULIA_VERSION: "0.7-" + ONLINE: "" + allow_failure: true + <<: *test_shell + + +.test_docker: &test_docker + artifacts: + name: coverage + expire_in: 1 week + paths: + - "$CI_JOB_NAME coverage/" + variables: + ONLINE: "" + script: + - julia -e "Pkg.clone(pwd()); Pkg.build(\"$PKG_NAME\"); Pkg.test(\"$PKG_NAME\"; coverage=true)" + - julia-coverage $PKG_NAME + +.test_docker_06: &test_docker_06 + image: 292522074875.dkr.ecr.us-east-1.amazonaws.com/julia-gitlab-ci:0.6 + <<: *test_docker + + +"0.6 (Mac)": + tags: + - mac + - shell-ci + <<: *test_shell_06 + +"0.6 (Linux, 64-bit)": + tags: + - linux + - 64-bit + - docker-ci + <<: *test_docker_06 + +"0.6 (Linux, 32-bit)": + tags: + - linux + - 32-bit + - shell-ci + <<: *test_shell_06 + +"0.7- (Mac)": + tags: + - mac + - shell-ci + <<: *test_shell_07- + +"0.7- (Linux, 64-bit)": + tags: + - linux + - 64-bit + - shell-ci + <<: *test_shell_07- + +"0.7- (Linux, 32-bit)": + tags: + - linux + - 32-bit + - shell-ci + <<: *test_shell_07- + + "Online Tests": - stage: test + stage: online-test image: 292522074875.dkr.ecr.us-east-1.amazonaws.com/julia-gitlab-ci:0.6 - variables: - ONLINE: "batch" # Runs the online tests against AWS artifacts: name: "$CI_JOB_NAME coverage" expire_in: 1 week paths: - "$CI_JOB_NAME coverage/" - script: - - julia --compilecache=no -e "Pkg.clone(pwd()); Pkg.build(\"$PKG_NAME\"); Pkg.test(\"$PKG_NAME\"; coverage=true)" - - julia-coverage "$PKG_NAME" tags: - linux - docker-ci + variables: + ONLINE: "batch" # Runs the online tests against AWS + script: + - julia -e "Pkg.clone(pwd()); Pkg.build(\"$PKG_NAME\"); Pkg.test(\"$PKG_NAME\"; coverage=true)" + - julia-coverage "$PKG_NAME" "Coverage": stage: coverage From b551b9cf07298c63a291891e6080967d4f042d84 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Thu, 3 May 2018 17:26:53 -0500 Subject: [PATCH 028/110] Add ComputeEnvironment and JobQueue --- REQUIRE | 1 + src/AWSBatch.jl | 5 +++ src/compute_environment.jl | 32 ++++++++++++++++ src/job_queue.jl | 44 ++++++++++++++++++++++ test/compute_environment.jl | 37 ++++++++++++++++++ test/job_queue.jl | 75 +++++++++++++++++++++++++++++++++++++ test/mock.jl | 34 +++++++++++++++++ test/run_batch.jl | 3 -- test/runtests.jl | 4 ++ 9 files changed, 232 insertions(+), 3 deletions(-) create mode 100644 src/compute_environment.jl create mode 100644 src/job_queue.jl create mode 100644 test/compute_environment.jl create mode 100644 test/job_queue.jl diff --git a/REQUIRE b/REQUIRE index 3b8377d..5d62464 100644 --- a/REQUIRE +++ b/REQUIRE @@ -4,3 +4,4 @@ AWSSDK 0.2.0 Compat 0.41.0 Mocking 0.3.3 Memento 0.5.0 +DataStructures 0.2.9 \ No newline at end of file diff --git a/src/AWSBatch.jl b/src/AWSBatch.jl index 3793391..e844665 100644 --- a/src/AWSBatch.jl +++ b/src/AWSBatch.jl @@ -12,11 +12,14 @@ using Mocking using Compat: Nothing using Compat: AbstractDict +using DataStructures: OrderedDict import Base: showerror export BatchJob, + ComputeEnvironment, + JobQueue, JobDefinition, JobState, run_batch, @@ -36,6 +39,8 @@ __init__() = Memento.register(logger) include("log_event.jl") +include("compute_environment.jl") +include("job_queue.jl") include("job_state.jl") include("job_definition.jl") include("batch_job.jl") diff --git a/src/compute_environment.jl b/src/compute_environment.jl new file mode 100644 index 0000000..27427fe --- /dev/null +++ b/src/compute_environment.jl @@ -0,0 +1,32 @@ +using AWSSDK.Batch: describe_compute_environments + +struct ComputeEnvironment + arn::String + + function ComputeEnvironment(ce::AbstractString) + arn = compute_environment_arn(ce) + arn === nothing && error("No compute environment ARN found for $ce") + new(arn) + end +end + +Base.:(==)(a::ComputeEnvironment, b::ComputeEnvironment) = a.arn == b.arn + +describe(ce::ComputeEnvironment) = describe_compute_environment(ce) +max_vcpus(ce::ComputeEnvironment) = describe(ce)["computeResources"]["maxvCpus"] + +function compute_environment_arn(ce::AbstractString) + startswith(ce, "arn:") && return ce + json = describe_compute_environment(ce) + isempty(json) ? nothing : json["computeEnvironmentArn"] +end + +describe_compute_environment(ce::ComputeEnvironment) = describe_compute_environment(ce.arn) + +function describe_compute_environment(ce::AbstractString)::OrderedDict + json = @mock describe_compute_environments(Dict("computeEnvironments" => [ce])) + envs = json["computeEnvironments"] + len = length(envs)::Int + @assert len <= 1 + return len == 1 ? first(envs) : OrderedDict() +end diff --git a/src/job_queue.jl b/src/job_queue.jl new file mode 100644 index 0000000..1b5679d --- /dev/null +++ b/src/job_queue.jl @@ -0,0 +1,44 @@ +using AWSSDK.Batch: describe_job_queues + +struct JobQueue + arn::String + + function JobQueue(queue::AbstractString) + arn = job_queue_arn(queue) + arn === nothing && error("No job queue ARN found for: $queue") + new(arn) + end +end + +Base.:(==)(a::JobQueue, b::JobQueue) = a.arn == b.arn + +describe(queue::JobQueue) = describe_job_queue(queue) +max_vcpus(queue::JobQueue) = sum(max_vcpus(ce) for ce in compute_environments(queue)) + +function compute_environments(queue::JobQueue) + ce_order = describe(queue)["computeEnvironmentOrder"] + + compute_envs = Vector{ComputeEnvironment}(length(ce_order)) + for ce in ce_order + i, arn = ce["order"], ce["computeEnvironment"] + compute_envs[i] = ComputeEnvironment(arn) + end + + return compute_envs +end + +function job_queue_arn(queue::AbstractString) + startswith(queue, "arn:") && return queue + json = describe_job_queue(queue) + isempty(json) ? nothing : json["jobQueueArn"] +end + +describe_job_queue(queue::JobQueue) = describe_job_queue(queue.arn) + +function describe_job_queue(queue::AbstractString)::OrderedDict + json = @mock describe_job_queues(Dict("jobQueues" => [queue])) + queues = json["jobQueues"] + len = length(queues)::Int + @assert len <= 1 + return len == 1 ? first(queues) : OrderedDict() +end diff --git a/test/compute_environment.jl b/test/compute_environment.jl new file mode 100644 index 0000000..293a026 --- /dev/null +++ b/test/compute_environment.jl @@ -0,0 +1,37 @@ +using DataStructures: OrderedDict + +@testset "ComputeEnvironment" begin + @testset "constructor" begin + arn = "arn:aws:batch:us-east-1:000000000000:compute-environment/ce" + @test ComputeEnvironment(arn).arn == arn + + patch = describe_compute_environments_patch( + OrderedDict( + "computeEnvironmentName" => "ce-name", + "computeEnvironmentArn" => arn, + ) + ) + apply(patch) do + @test ComputeEnvironment("ce-name").arn == arn + end + + patch = describe_compute_environments_patch() + apply(patch) do + @test_throws ErrorException ComputeEnvironment("ce-name") + end + end + + @testset "max_vcpus" begin + ce = ComputeEnvironment("arn:aws:batch:us-east-1:000000000000:compute-environment/ce") + patch = describe_compute_environments_patch( + OrderedDict( + "computeEnvironmentArn" => ce.arn, + "computeResources" => OrderedDict("maxvCpus" => 5), + ), + ) + + apply(patch) do + @test AWSBatch.max_vcpus(ce) == 5 + end + end +end diff --git a/test/job_queue.jl b/test/job_queue.jl new file mode 100644 index 0000000..2120712 --- /dev/null +++ b/test/job_queue.jl @@ -0,0 +1,75 @@ +using DataStructures: OrderedDict + +@testset "JobQueue" begin + @testset "constructor" begin + arn = "arn:aws:batch:us-east-1:000000000000:job-queue/queue" + @test JobQueue(arn).arn == arn + + patch = describe_job_queues_patch( + OrderedDict( + "jobQueueName" => "queue-name", + "jobQueueArn" => arn, + ) + ) + apply(patch) do + @test JobQueue("queue-name").arn == arn + end + + patch = describe_job_queues_patch() + apply(patch) do + @test_throws ErrorException JobQueue("queue-name") + end + end + + @testset "compute_environments" begin + # Note: to date we've only used queues with a single compute environment + queue = JobQueue("arn:aws:batch:us-east-1:000000000000:job-queue/queue") + patch = describe_job_queues_patch( + OrderedDict( + "jobQueueArn" => queue.arn, + "computeEnvironmentOrder" => [ + OrderedDict("order" => 2, "computeEnvironment" => "arn:aws:batch:us-east-1:000000000000:compute-environment/two"), + OrderedDict("order" => 1, "computeEnvironment" => "arn:aws:batch:us-east-1:000000000000:compute-environment/one"), + ], + ) + ) + + expected = [ + ComputeEnvironment("arn:aws:batch:us-east-1:000000000000:compute-environment/one"), + ComputeEnvironment("arn:aws:batch:us-east-1:000000000000:compute-environment/two"), + ] + + apply(patch) do + @test AWSBatch.compute_environments(queue) == expected + end + end + + @testset "max_vcpus" begin + queue = JobQueue("arn:aws:batch:us-east-1:000000000000:job-queue/queue") + patches = [ + describe_job_queues_patch( + OrderedDict( + "jobQueueArn" => queue.arn, + "computeEnvironmentOrder" => [ + OrderedDict("order" => 1, "computeEnvironment" => "arn:aws:batch:us-east-1:000000000000:compute-environment/one"), + OrderedDict("order" => 2, "computeEnvironment" => "arn:aws:batch:us-east-1:000000000000:compute-environment/two"), + ], + ) + ) + describe_compute_environments_patch([ + OrderedDict( + "computeEnvironmentArn" => "arn:aws:batch:us-east-1:000000000000:compute-environment/one", + "computeResources" => OrderedDict("maxvCpus" => 7), + ), + OrderedDict( + "computeEnvironmentArn" => "arn:aws:batch:us-east-1:000000000000:compute-environment/two", + "computeResources" => OrderedDict("maxvCpus" => 8), + ) + ]) + ] + + apply(patches) do + @test AWSBatch.max_vcpus(queue) == 15 + end + end +end diff --git a/test/mock.jl b/test/mock.jl index 3d2e657..3a9ffed 100644 --- a/test/mock.jl +++ b/test/mock.jl @@ -1,4 +1,5 @@ import Base: AbstractCmd, CmdRedirect +using DataStructures: OrderedDict const BATCH_ENVS = ( "AWS_BATCH_JOB_ID" => "24fa2d7a-64c4-49d2-8b47-f8da4fbde8e9", @@ -111,3 +112,36 @@ function mock_readstring(cmd::CmdRedirect) return Base.readstring(cmd) end end + + +function describe_compute_environments_patch(output::Vector=[]) + @patch function describe_compute_environments(d::Dict) + compute_envs = d["computeEnvironments"] + @assert length(compute_envs) == 1 + ce = first(compute_envs) + + key = startswith(ce, "arn:") ? "computeEnvironmentArn" : "computeEnvironmentName" + results = filter(d -> d[key] == ce, output) + OrderedDict("computeEnvironments" => results) + end +end + +function describe_compute_environments_patch(output::OrderedDict) + describe_compute_environments_patch([output]) +end + +function describe_job_queues_patch(output::Vector=[]) + @patch function describe_job_queues(d::Dict) + queues = d["jobQueues"] + @assert length(queues) == 1 + queue = first(queues) + + key = startswith(queue, "arn:") ? "jobQueueArn" : "jobQueueName" + results = filter(d -> d[key] == queue, output) + OrderedDict("jobQueues" => output) + end +end + +function describe_job_queues_patch(output::OrderedDict) + describe_job_queues_patch([output]) +end diff --git a/test/run_batch.jl b/test/run_batch.jl index 18b4590..3df8eb3 100644 --- a/test/run_batch.jl +++ b/test/run_batch.jl @@ -1,6 +1,3 @@ -include("mock.jl") - - function register_job_def(config::AWSConfig, input::AbstractArray, expected::AbstractArray) @test input == expected return REGISTER_JOB_DEF_RESP diff --git a/test/runtests.jl b/test/runtests.jl index d45ea60..dd1862a 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -27,8 +27,12 @@ const STACK = isempty(AWS_STACKNAME) ? LEGACY_STACK : stack_output(AWS_STACKNAME Memento.config("debug"; fmt="[{level} | {name}]: {msg}") setlevel!(getlogger(AWSBatch), "info") +include("mock.jl") + @testset "AWSBatch.jl" begin + include("compute_environment.jl") + include("job_queue.jl") include("log_event.jl") include("job_state.jl") include("run_batch.jl") From 9f1e24d27d6dd19f55d35b2825bdb8db1f1606e2 Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Fri, 4 May 2018 15:30:37 +0000 Subject: [PATCH 029/110] Support array jobs --- src/AWSBatch.jl | 3 +++ src/batch_job.jl | 31 +++++++++++++++++------- src/job_definition.jl | 8 +++---- test/runtests.jl | 55 ++++++++++++++++++++++++++++++++++++++----- 4 files changed, 78 insertions(+), 19 deletions(-) diff --git a/src/AWSBatch.jl b/src/AWSBatch.jl index 3793391..2275d13 100644 --- a/src/AWSBatch.jl +++ b/src/AWSBatch.jl @@ -62,6 +62,7 @@ end memory::Integer=1024, role::AbstractString="", cmd::Cmd=``, + num_jobs::Integer=1, ) -> BatchJob Handles submitting a BatchJob based on various potential defaults. @@ -87,6 +88,7 @@ function run_batch(; memory::Integer=1024, role::AbstractString="", cmd::Cmd=``, + num_jobs::Integer=1, ) if isa(definition, AbstractString) definition = isempty(definition) ? nothing : definition @@ -209,6 +211,7 @@ function run_batch(; queue; container=container_overrides, region=region, + num_jobs=num_jobs, ) end diff --git a/src/batch_job.jl b/src/batch_job.jl index cac1153..f2c1bb0 100644 --- a/src/batch_job.jl +++ b/src/batch_job.jl @@ -26,6 +26,7 @@ end queue::AbstractString; container::AbstractDict=Dict(), region::AbstractString="", + num_jobs::Integer=1, ) -> BatchJob Handles submitting the batch job. Returns a `BatchJob` wrapper for the id. @@ -36,23 +37,35 @@ function submit( queue::AbstractString; container::AbstractDict=Dict(), region::AbstractString="", + num_jobs::Integer=1, ) region = isempty(region) ? "us-east-1" : region config = AWSConfig(:creds => AWSCredentials(), :region => region) - debug(logger, "Submitting job $name") + debug(logger, "Submitting job \"$name\"") input = [ "jobName" => name, "jobDefinition" => definition.arn, "jobQueue" => queue, "containerOverrides" => container, ] + + if num_jobs > 1 + # https://docs.aws.amazon.com/batch/latest/userguide/array_jobs.html + @assert 2 <= num_jobs <= 10_000 + push!(input, "arrayProperties" => ["size" => num_jobs]) + end + debug(logger, "Input: $input") response = @mock submit_job(config, input) job = BatchJob(response["jobId"]) - info(logger, "Submitted job $(name)::$(job.id).") + if num_jobs > 1 + info(logger, "Submitted array job \"$(name)\" ($(job.id), n=$(num_jobs))") + else + info(logger, "Submitted job \"$(name)\" ($(job.id))") + end return job end @@ -135,10 +148,10 @@ function Base.wait( end if !completed - message = "Waiting on job $(job.id) timed out." + message = "Waiting on job $(job.id) timed out" if !initial - message *= " Last known state $last_state." + message *= " Last known state $last_state" end error(logger, message) @@ -170,7 +183,7 @@ function Base.wait( if state in cond false elseif state in failure - error(logger, "Job $(job.id) hit failure condition $state.") + error(logger, "Job $(job.id) hit failure condition $state") false else true @@ -190,16 +203,16 @@ NOTES: default. """ function log_events(job::BatchJob) - container_details = describe(job)["container"] + job_details = describe(job) - if "logStreamName" in keys(container_details) - stream = container_details["logStreamName"] + if "logStreamName" in keys(job_details["container"]) + stream = job_details["container"]["logStreamName"] info(logger, "Fetching log events from $stream") output = get_log_events(logGroupName="/aws/batch/job", logStreamName=stream) return convert.(LogEvent, output["events"]) else - info(logger, "No log events found for job $(job.id).") + info(logger, "No log events found for job $(job.id)") return LogEvent[] end end diff --git a/src/job_definition.jl b/src/job_definition.jl index 5dc214b..91e5075 100644 --- a/src/job_definition.jl +++ b/src/job_definition.jl @@ -88,7 +88,7 @@ function register( region = isempty(region) ? "us-east-1" : region config = AWSConfig(:creds => AWSCredentials(), :region => region) - debug(logger, "Registering job definition $definition_name.") + debug(logger, "Registering job definition \"$definition_name\"") input = [ "type" => "container", "containerProperties" => [ @@ -103,7 +103,7 @@ function register( response = @mock register_job_definition(config, input) definition = JobDefinition(response["jobDefinitionArn"]) - info(logger, "Registered job definition $(definition.arn).") + info(logger, "Registered job definition \"$(definition.arn)\"") return definition end @@ -113,9 +113,9 @@ end Deregisters an AWS Batch job. """ function deregister(definition::JobDefinition) - debug(logger, "Deregistering job definition $(definition.arn).") + debug(logger, "Deregistering job definition \"$(definition.arn)\"") resp = deregister_job_definition(Dict("jobDefinition" => definition.arn)) - info(logger, "Deregistered job definition $(definition.arn).") + info(logger, "Deregistered job definition \"$(definition.arn)\"") end """ diff --git a/test/runtests.jl b/test/runtests.jl index d45ea60..b2ce03a 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -52,6 +52,10 @@ setlevel!(getlogger(AWSBatch), "info") @test wait(job, [AWSBatch.SUCCEEDED]) == true @test status(job) == AWSBatch.SUCCEEDED + events = log_events(job) + @test length(events) == 1 + @test first(events).message == "Hello World!" + # Test job details were set correctly job_details = describe(job) @test job_details["jobName"] == STACK["JobName"] @@ -79,17 +83,56 @@ setlevel!(getlogger(AWSBatch), "info") @test container_properties["jobRoleArn"] == STACK["JobRoleArn"] deregister(job_definition) + end + @testset "Array job" begin + job = run_batch(; + name = "AWSBatchArrayJobTest", + definition = STACK["JobDefinitionName"], + queue = STACK["JobQueueArn"], + image = STACK["EcrUri"], + vcpus = 1, + memory = 1024, + role = STACK["JobRoleArn"], + cmd = `julia -e 'println("Hello World!")'`, + num_jobs = 3, + ) + + @test wait(job, [AWSBatch.SUCCEEDED]) == true + @test status(job) == AWSBatch.SUCCEEDED + + # Test array job was submitted properly + status_summary = Dict( + "STARTING" => 0, "FAILED" => 0, "RUNNING" => 0, "SUCCEEDED" => 3, + "RUNNABLE" => 0, "SUBMITTED" => 0, "PENDING" => 0, + ) + job_details = describe(job) + @test job_details["arrayProperties"]["statusSummary"] == status_summary + @test job_details["arrayProperties"]["size"] == 3 + + # Test no log events for the job submitted events = log_events(job) - @test length(events) == 1 - @test contains(first(events).message, "Hello World!") + @test length(events) == 0 + + # Test logs for each individual job that is part of the job array + for i in 0:2 + job_id = "$(job.id):$i" + events = log_events(BatchJob(job_id)) + + @test length(events) == 1 + @test first(events).message == "Hello World!" + end + + # Deregister the job definition + job_definition = JobDefinition(job) + deregister(job_definition) end @testset "Job Timed Out" begin info("Testing job timeout") job = run_batch(; - name = STACK["JobName"], + name = "AWSBatchTimeOutJobTest", definition = STACK["JobDefinitionName"], queue = STACK["JobQueueArn"], image = STACK["EcrUri"], @@ -114,14 +157,14 @@ setlevel!(getlogger(AWSBatch), "info") info("Testing job failure") job = run_batch(; - name = STACK["JobName"], + name = "AWSBatchFailedJobTest", definition = STACK["JobDefinitionName"], queue = STACK["JobQueueArn"], image = STACK["EcrUri"], vcpus = 1, memory = 1024, role = STACK["JobRoleArn"], - cmd = `julia -e 'error("Cmd failed")'`, + cmd = `julia -e 'error("Testing job failure")'`, ) job_definition = JobDefinition(job) @@ -133,7 +176,7 @@ setlevel!(getlogger(AWSBatch), "info") events = log_events(job) @test length(events) == 3 - @test contains(first(events).message, "ERROR: Cmd failed") + @test first(events).message == "ERROR: Testing job failure" end end else From 128c6f5951b6f7ac7b40d77a58d8ac9c3b88dc4f Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Thu, 24 May 2018 15:32:00 -0500 Subject: [PATCH 030/110] Remove unused dependency --- src/AWSBatch.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/AWSBatch.jl b/src/AWSBatch.jl index 8789b32..344fa35 100644 --- a/src/AWSBatch.jl +++ b/src/AWSBatch.jl @@ -5,7 +5,6 @@ using AWSCore: AWSConfig, AWSCredentials using AWSSDK.Batch using AWSSDK.CloudWatchLogs -using AWSSDK.S3 using Memento using Mocking From 603ea64172323f5c15f00aa47ba308a2a88f06df Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Wed, 23 May 2018 14:16:09 -0500 Subject: [PATCH 031/110] Switch online tests to new CI --- .gitlab-ci.yml | 23 +++++++++++++++++------ test/runtests.jl | 39 +++++++++++++++++++-------------------- 2 files changed, 36 insertions(+), 26 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 303c617..c77160e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -92,23 +92,34 @@ stages: - shell-ci <<: *test_shell_07- - "Online Tests": stage: online-test - image: 292522074875.dkr.ecr.us-east-1.amazonaws.com/julia-gitlab-ci:0.6 + tags: + - batch + - ecr + - linux + - shell-ci artifacts: name: "$CI_JOB_NAME coverage" expire_in: 1 week paths: - "$CI_JOB_NAME coverage/" - tags: - - linux - - docker-ci variables: + AWS_STACKNAME: aws-batch-manager-test ONLINE: "batch" # Runs the online tests against AWS + JULIA_VERSION: "0.6" + before_script: + - curl -s -o julia-ci https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/julia-ci + - chmod +x julia-ci + - ./julia-ci install $JULIA_VERSION script: + - source julia-ci export + - export PATH="$PATH:/usr/local/bin" + - unset SSL_CERT_DIR # https://github.com/JuliaLang/julia/issues/20439 - julia -e "Pkg.clone(pwd()); Pkg.build(\"$PKG_NAME\"); Pkg.test(\"$PKG_NAME\"; coverage=true)" - - julia-coverage "$PKG_NAME" + - ./julia-ci coverage + after_script: + - ./julia-ci clean "Coverage": stage: coverage diff --git a/test/runtests.jl b/test/runtests.jl index af6058e..8adfd2e 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -14,11 +14,11 @@ const ONLINE = strip.(split(get(ENV, "ONLINE", ""), r"\s*,\s*")) # Partially emulates the output from the AWS batch manager test stack const LEGACY_STACK = Dict( - "JobQueueArn" => "arn:aws:batch:us-east-1:292522074875:job-queue/Replatforming-Manager", - "JobName" => "AWSBatchTest", - "JobDefinitionName" => "AWSBatch", + "ManagerJobQueueArn" => "arn:aws:batch:us-east-1:292522074875:job-queue/Replatforming-Manager", + "JobName" => "aws-batch-test", + "JobDefinitionName" => "aws-batch-test", "JobRoleArn" => "arn:aws:iam::292522074875:role/AWSBatchClusterManagerJobRole", - "EcrUri" => "292522074875.dkr.ecr.us-east-1.amazonaws.com/aws-tools:latest", + "EcrUri" => "292522074875.dkr.ecr.us-east-1.amazonaws.com/aws-cluster-managers-test:latest", ) const AWS_STACKNAME = get(ENV, "AWS_STACKNAME", "") @@ -43,9 +43,9 @@ include("mock.jl") @testset "Job Submission" begin job = run_batch(; - name = STACK["JobName"], - definition = STACK["JobDefinitionName"], - queue = STACK["JobQueueArn"], + name = "aws-batch-test", + definition = "aws-batch-test", + queue = STACK["ManagerJobQueueArn"], image = STACK["EcrUri"], vcpus = 1, memory = 1024, @@ -62,8 +62,8 @@ include("mock.jl") # Test job details were set correctly job_details = describe(job) - @test job_details["jobName"] == STACK["JobName"] - @test job_details["jobQueue"] == STACK["JobQueueArn"] + @test job_details["jobName"] == "aws-batch-test" + @test job_details["jobQueue"] == STACK["ManagerJobQueueArn"] # Test job definition and container parameters were set correctly job_definition = JobDefinition(job) @@ -71,7 +71,7 @@ include("mock.jl") job_definition_details = first(describe(job_definition)["jobDefinitions"]) - @test job_definition_details["jobDefinitionName"] == STACK["JobDefinitionName"] + @test job_definition_details["jobDefinitionName"] == "aws-batch-test" @test job_definition_details["status"] == "ACTIVE" @test job_definition_details["type"] == "container" @@ -91,9 +91,9 @@ include("mock.jl") @testset "Array job" begin job = run_batch(; - name = "AWSBatchArrayJobTest", - definition = STACK["JobDefinitionName"], - queue = STACK["JobQueueArn"], + name = "aws-batch-array-job-test", + definition = "aws-batch-test", + queue = STACK["ManagerJobQueueArn"], image = STACK["EcrUri"], vcpus = 1, memory = 1024, @@ -136,9 +136,9 @@ include("mock.jl") info("Testing job timeout") job = run_batch(; - name = "AWSBatchTimeOutJobTest", - definition = STACK["JobDefinitionName"], - queue = STACK["JobQueueArn"], + name = "aws-bath-timeout-job-test", + definition = "aws-batch-test", + queue = STACK["ManagerJobQueueArn"], image = STACK["EcrUri"], vcpus = 1, memory = 1024, @@ -161,9 +161,9 @@ include("mock.jl") info("Testing job failure") job = run_batch(; - name = "AWSBatchFailedJobTest", - definition = STACK["JobDefinitionName"], - queue = STACK["JobQueueArn"], + name = "aws-batch-failed-job-test", + definition = "aws-batch-test", + queue = STACK["ManagerJobQueueArn"], image = STACK["EcrUri"], vcpus = 1, memory = 1024, @@ -179,7 +179,6 @@ include("mock.jl") deregister(job_definition) events = log_events(job) - @test length(events) == 3 @test first(events).message == "ERROR: Testing job failure" end end From a2d51cca883138fa8f82de0752c0972a025067ec Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Mon, 28 May 2018 11:17:05 -0500 Subject: [PATCH 032/110] Update test ECR uri --- test/runtests.jl | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 8adfd2e..35966f3 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -23,6 +23,7 @@ const LEGACY_STACK = Dict( const AWS_STACKNAME = get(ENV, "AWS_STACKNAME", "") const STACK = isempty(AWS_STACKNAME) ? LEGACY_STACK : stack_output(AWS_STACKNAME) +const JULIA_BAKED_IMAGE = "292522074875.dkr.ecr.us-east-1.amazonaws.com/julia-baked:0.6" Memento.config("debug"; fmt="[{level} | {name}]: {msg}") setlevel!(getlogger(AWSBatch), "info") @@ -46,7 +47,7 @@ include("mock.jl") name = "aws-batch-test", definition = "aws-batch-test", queue = STACK["ManagerJobQueueArn"], - image = STACK["EcrUri"], + image = JULIA_BAKED_IMAGE, vcpus = 1, memory = 1024, role = STACK["JobRoleArn"], @@ -76,7 +77,7 @@ include("mock.jl") @test job_definition_details["type"] == "container" container_properties = job_definition_details["containerProperties"] - @test container_properties["image"] == STACK["EcrUri"] + @test container_properties["image"] == JULIA_BAKED_IMAGE @test container_properties["vcpus"] == 1 @test container_properties["memory"] == 1024 @test container_properties["command"] == [ @@ -94,7 +95,7 @@ include("mock.jl") name = "aws-batch-array-job-test", definition = "aws-batch-test", queue = STACK["ManagerJobQueueArn"], - image = STACK["EcrUri"], + image = JULIA_BAKED_IMAGE, vcpus = 1, memory = 1024, role = STACK["JobRoleArn"], @@ -139,7 +140,7 @@ include("mock.jl") name = "aws-bath-timeout-job-test", definition = "aws-batch-test", queue = STACK["ManagerJobQueueArn"], - image = STACK["EcrUri"], + image = JULIA_BAKED_IMAGE, vcpus = 1, memory = 1024, role = STACK["JobRoleArn"], @@ -164,7 +165,7 @@ include("mock.jl") name = "aws-batch-failed-job-test", definition = "aws-batch-test", queue = STACK["ManagerJobQueueArn"], - image = STACK["EcrUri"], + image = JULIA_BAKED_IMAGE, vcpus = 1, memory = 1024, role = STACK["JobRoleArn"], From 50f569263b7fe46a3721c83697e49ef00223296b Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Wed, 23 May 2018 10:31:52 -0500 Subject: [PATCH 033/110] Support job parameters --- src/AWSBatch.jl | 5 +++++ src/batch_job.jl | 5 ++++- src/job_definition.jl | 3 +++ test/run_batch.jl | 10 +++++++--- test/runtests.jl | 2 ++ 5 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/AWSBatch.jl b/src/AWSBatch.jl index 344fa35..469afae 100644 --- a/src/AWSBatch.jl +++ b/src/AWSBatch.jl @@ -67,6 +67,7 @@ end role::AbstractString="", cmd::Cmd=``, num_jobs::Integer=1, + parameters::Dict{String, String} = Dict{String, String}(), ) -> BatchJob Handles submitting a BatchJob based on various potential defaults. @@ -93,6 +94,7 @@ function run_batch(; role::AbstractString="", cmd::Cmd=``, num_jobs::Integer=1, + parameters::Dict{String, String} = Dict{String, String}(), ) if isa(definition, AbstractString) definition = isempty(definition) ? nothing : definition @@ -186,6 +188,7 @@ function run_batch(; memory=memory, cmd=cmd, region=region, + parameters=parameters, ) end elseif definition === nothing @@ -198,6 +201,7 @@ function run_batch(; memory=memory, cmd=cmd, region=region, + parameters=parameters, ) end @@ -214,6 +218,7 @@ function run_batch(; definition, queue; container=container_overrides, + parameters=parameters, region=region, num_jobs=num_jobs, ) diff --git a/src/batch_job.jl b/src/batch_job.jl index f2c1bb0..5ac1fd2 100644 --- a/src/batch_job.jl +++ b/src/batch_job.jl @@ -25,6 +25,7 @@ end definition::JobDefinition, queue::AbstractString; container::AbstractDict=Dict(), + parameters::Dict{String,String} = Dict{String, String}(), region::AbstractString="", num_jobs::Integer=1, ) -> BatchJob @@ -36,6 +37,7 @@ function submit( definition::JobDefinition, queue::AbstractString; container::AbstractDict=Dict(), + parameters::Dict{String,String} = Dict{String, String}(), region::AbstractString="", num_jobs::Integer=1, ) @@ -45,8 +47,9 @@ function submit( debug(logger, "Submitting job \"$name\"") input = [ "jobName" => name, - "jobDefinition" => definition.arn, "jobQueue" => queue, + "jobDefinition" => definition.arn, + "parameters" => parameters, "containerOverrides" => container, ] diff --git a/src/job_definition.jl b/src/job_definition.jl index 91e5075..5e0a5be 100644 --- a/src/job_definition.jl +++ b/src/job_definition.jl @@ -72,6 +72,7 @@ end memory::Integer=1024, cmd::Cmd=``, region::AbstractString="", + parameters::Dict{String,String} = Dict{String, String}(), ) -> JobDefinition Registers a new job definition. @@ -84,6 +85,7 @@ function register( memory::Integer=1024, cmd::Cmd=``, region::AbstractString="", + parameters::Dict{String, String} = Dict{String, String}(), ) region = isempty(region) ? "us-east-1" : region config = AWSConfig(:creds => AWSCredentials(), :region => region) @@ -91,6 +93,7 @@ function register( debug(logger, "Registering job definition \"$definition_name\"") input = [ "type" => "container", + "parameters" => parameters, "containerProperties" => [ "image" => image, "vcpus" => vcpus, diff --git a/test/run_batch.jl b/test/run_batch.jl index 3df8eb3..8235681 100644 --- a/test/run_batch.jl +++ b/test/run_batch.jl @@ -20,8 +20,9 @@ end @testset "From Job Definition" begin expected_job = [ "jobName" => "example", - "jobDefinition" => "arn:aws:batch:us-east-1:012345678910:job-definition/sleep60:1", "jobQueue" => "HighPriority", + "jobDefinition" => "arn:aws:batch:us-east-1:012345678910:job-definition/sleep60:1", + "parameters" => Dict{String,String}(), "containerOverrides" => Dict( "command" => ["sleep", "60"], "memory" => 128, @@ -44,8 +45,9 @@ end withenv(BATCH_ENVS...) do expected_job = [ "jobName" => "example", - "jobDefinition" => "arn:aws:batch:us-east-1:012345678910:job-definition/sleep60:1", "jobQueue" => "HighPriority", + "jobDefinition" => "arn:aws:batch:us-east-1:012345678910:job-definition/sleep60:1", + "parameters" => Dict{String,String}(), "containerOverrides" => Dict( "command" => ["sleep", "60"], "memory" => 128, @@ -55,6 +57,7 @@ end expected_job_def = [ "type" => "container", + "parameters" => Dict{String,String}(), "containerProperties" => [ "image" => "busybox", "vcpus" => 1, @@ -88,8 +91,9 @@ end withenv(BATCH_ENVS...) do expected_job = [ "jobName" => "example", - "jobDefinition" => "arn:aws:batch:us-east-1:012345678910:job-definition/sleep60:1", "jobQueue" => "HighPriority", + "jobDefinition" => "arn:aws:batch:us-east-1:012345678910:job-definition/sleep60:1", + "parameters" => Dict{String,String}(), "containerOverrides" => Dict( "command" => ["sleep", "60"], "memory" => 128, diff --git a/test/runtests.jl b/test/runtests.jl index 35966f3..f10f554 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -52,6 +52,7 @@ include("mock.jl") memory = 1024, role = STACK["JobRoleArn"], cmd = `julia -e 'println("Hello World!")'`, + parameters = Dict{String, String}("region" => "us-east-1"), ) @test wait(job, [AWSBatch.SUCCEEDED]) == true @@ -65,6 +66,7 @@ include("mock.jl") job_details = describe(job) @test job_details["jobName"] == "aws-batch-test" @test job_details["jobQueue"] == STACK["ManagerJobQueueArn"] + @test job_details["parameters"] == Dict("region" => "us-east-1") # Test job definition and container parameters were set correctly job_definition = JobDefinition(job) From db6d86ddad42e438eed13f644b70020b6ad3ec74 Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Wed, 23 May 2018 11:14:39 -0500 Subject: [PATCH 034/110] Fix spacing --- src/AWSBatch.jl | 4 ++-- src/batch_job.jl | 4 ++-- src/job_definition.jl | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/AWSBatch.jl b/src/AWSBatch.jl index 469afae..8d48f46 100644 --- a/src/AWSBatch.jl +++ b/src/AWSBatch.jl @@ -67,7 +67,7 @@ end role::AbstractString="", cmd::Cmd=``, num_jobs::Integer=1, - parameters::Dict{String, String} = Dict{String, String}(), + parameters::Dict{String, String}=Dict{String, String}(), ) -> BatchJob Handles submitting a BatchJob based on various potential defaults. @@ -94,7 +94,7 @@ function run_batch(; role::AbstractString="", cmd::Cmd=``, num_jobs::Integer=1, - parameters::Dict{String, String} = Dict{String, String}(), + parameters::Dict{String, String}=Dict{String, String}(), ) if isa(definition, AbstractString) definition = isempty(definition) ? nothing : definition diff --git a/src/batch_job.jl b/src/batch_job.jl index 5ac1fd2..af55516 100644 --- a/src/batch_job.jl +++ b/src/batch_job.jl @@ -25,7 +25,7 @@ end definition::JobDefinition, queue::AbstractString; container::AbstractDict=Dict(), - parameters::Dict{String,String} = Dict{String, String}(), + parameters::Dict{String,String}=Dict{String, String}(), region::AbstractString="", num_jobs::Integer=1, ) -> BatchJob @@ -37,7 +37,7 @@ function submit( definition::JobDefinition, queue::AbstractString; container::AbstractDict=Dict(), - parameters::Dict{String,String} = Dict{String, String}(), + parameters::Dict{String,String}=Dict{String, String}(), region::AbstractString="", num_jobs::Integer=1, ) diff --git a/src/job_definition.jl b/src/job_definition.jl index 5e0a5be..832face 100644 --- a/src/job_definition.jl +++ b/src/job_definition.jl @@ -72,7 +72,7 @@ end memory::Integer=1024, cmd::Cmd=``, region::AbstractString="", - parameters::Dict{String,String} = Dict{String, String}(), + parameters::Dict{String,String}=Dict{String, String}(), ) -> JobDefinition Registers a new job definition. @@ -85,7 +85,7 @@ function register( memory::Integer=1024, cmd::Cmd=``, region::AbstractString="", - parameters::Dict{String, String} = Dict{String, String}(), + parameters::Dict{String, String}=Dict{String, String}(), ) region = isempty(region) ? "us-east-1" : region config = AWSConfig(:creds => AWSCredentials(), :region => region) From e51cdaf4f72960cbd28997b1a59dbf8dadc25e99 Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Wed, 23 May 2018 13:30:28 -0500 Subject: [PATCH 035/110] Add job parameter test example --- test/runtests.jl | 64 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 2 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index f10f554..877ec74 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -92,6 +92,62 @@ include("mock.jl") deregister(job_definition) end + @testset "Job parameters" begin + # Use parameter substitution placeholders in the command field + command = Cmd(["julia", "-e", "Ref::juliacmd"]) + + # Set a default output string when registering the job definition + job_definition = register( + "aws-batch-parameters-test"; + image=STACK["EcrUri"], + role=STACK["JobRoleArn"], + vcpus=1, + memory=1024, + cmd=command, + region="us-east-1", + parameters=Dict("juliacmd" => "println(\"Default String\")"), + ) + + # Override the default output string + job = run_batch(; + name = "aws-batch-parameters-test", + definition = job_definition, + queue = STACK["JobQueueArn"], + image = STACK["EcrUri"], + vcpus = 1, + memory = 1024, + role = STACK["JobRoleArn"], + cmd = command, + parameters=Dict("juliacmd" => "println(\"Hello World!\")"), + ) + + @test wait(state -> state < AWSBatch.SUCCEEDED, job) + @test status(job) == AWSBatch.SUCCEEDED + + # Test the default string was overrriden succesfully + events = log_events(job) + @test length(events) == 1 + @test first(events).message == "Hello World!" + + # Test job details were set correctly + job_details = describe(job) + @test job_details["parameters"] == Dict( + "juliacmd" => "println(\"Hello World!\")" + ) + + job_definition = JobDefinition(job) + job_definition_details = first(describe(job_definition)["jobDefinitions"]) + job_definition_details["parameters"] = Dict( + "juliacmd" => "println(\"Default String\")" + ) + + container_properties = job_definition_details["containerProperties"] + @test container_properties["command"] == ["julia", "-e", "Ref::juliacmd"] + + # Deregister job definition + deregister(job_definition) + end + @testset "Array job" begin job = run_batch(; name = "aws-batch-array-job-test", @@ -105,7 +161,7 @@ include("mock.jl") num_jobs = 3, ) - @test wait(job, [AWSBatch.SUCCEEDED]) == true + @test wait(state -> state < AWSBatch.SUCCEEDED, job) @test status(job) == AWSBatch.SUCCEEDED # Test array job was submitted properly @@ -152,7 +208,11 @@ include("mock.jl") job_definition = JobDefinition(job) @test isregistered(job_definition) == true - @test_throws ErrorException wait(job, [AWSBatch.SUCCEEDED]; timeout=0) + @test_throws ErrorException wait( + state -> state < AWSBatch.SUCCEEDED, + job; + timeout=0 + ) deregister(job_definition) From 03e4df2f823606f6fe2c838c14ea9ac218dabbae Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Mon, 28 May 2018 14:26:45 -0500 Subject: [PATCH 036/110] Fix parameter tests --- test/runtests.jl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 877ec74..25af2ed 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -14,7 +14,7 @@ const ONLINE = strip.(split(get(ENV, "ONLINE", ""), r"\s*,\s*")) # Partially emulates the output from the AWS batch manager test stack const LEGACY_STACK = Dict( - "ManagerJobQueueArn" => "arn:aws:batch:us-east-1:292522074875:job-queue/Replatforming-Manager", + "ManagerJobQueueArn" => "Replatforming-Manager", "JobName" => "aws-batch-test", "JobDefinitionName" => "aws-batch-test", "JobRoleArn" => "arn:aws:iam::292522074875:role/AWSBatchClusterManagerJobRole", @@ -65,7 +65,7 @@ include("mock.jl") # Test job details were set correctly job_details = describe(job) @test job_details["jobName"] == "aws-batch-test" - @test job_details["jobQueue"] == STACK["ManagerJobQueueArn"] + @test contains(job_details["jobQueue"], STACK["ManagerJobQueueArn"]) @test job_details["parameters"] == Dict("region" => "us-east-1") # Test job definition and container parameters were set correctly @@ -99,7 +99,7 @@ include("mock.jl") # Set a default output string when registering the job definition job_definition = register( "aws-batch-parameters-test"; - image=STACK["EcrUri"], + image=JULIA_BAKED_IMAGE, role=STACK["JobRoleArn"], vcpus=1, memory=1024, @@ -112,8 +112,8 @@ include("mock.jl") job = run_batch(; name = "aws-batch-parameters-test", definition = job_definition, - queue = STACK["JobQueueArn"], - image = STACK["EcrUri"], + queue = STACK["ManagerJobQueueArn"], + image = JULIA_BAKED_IMAGE, vcpus = 1, memory = 1024, role = STACK["JobRoleArn"], From 1c8ae70b49ddcf214059f5ecb4d5950c35cbfa67 Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Wed, 22 Aug 2018 14:49:55 +0000 Subject: [PATCH 037/110] Add flag that disallows job registration when running batch jobs --- REQUIRE | 3 ++- src/AWSBatch.jl | 54 ++++++++++++++++++++++++------------------- src/batch_job.jl | 2 +- src/job_definition.jl | 48 +++++++++++++++++++++++++------------- test/runtests.jl | 34 ++++++++++++++++++++++++++- 5 files changed, 98 insertions(+), 43 deletions(-) diff --git a/REQUIRE b/REQUIRE index 5d62464..759fde9 100644 --- a/REQUIRE +++ b/REQUIRE @@ -1,7 +1,8 @@ julia 0.6 +AutoHashEquals 0.2 AWSCore 0.3.0 AWSSDK 0.2.0 Compat 0.41.0 Mocking 0.3.3 -Memento 0.5.0 +Memento 0.7 DataStructures 0.2.9 \ No newline at end of file diff --git a/src/AWSBatch.jl b/src/AWSBatch.jl index 8d48f46..7a5fddd 100644 --- a/src/AWSBatch.jl +++ b/src/AWSBatch.jl @@ -1,16 +1,14 @@ __precompile__() module AWSBatch -using AWSCore: AWSConfig, AWSCredentials - +using AutoHashEquals using AWSSDK.Batch using AWSSDK.CloudWatchLogs - using Memento using Mocking -using Compat: Nothing -using Compat: AbstractDict +using AWSCore: AWSConfig, AWSCredentials +using Compat: Nothing, AbstractDict using DataStructures: OrderedDict import Base: showerror @@ -28,7 +26,8 @@ export log_events, isregistered, register, - deregister + deregister, + BatchEnvironmentError const logger = getlogger(current_module()) @@ -95,6 +94,7 @@ function run_batch(; cmd::Cmd=``, num_jobs::Integer=1, parameters::Dict{String, String}=Dict{String, String}(), + allow_job_registration::Bool=true, ) if isa(definition, AbstractString) definition = isempty(definition) ? nothing : definition @@ -165,21 +165,28 @@ function run_batch(; throw(BatchEnvironmentError( "Unable to perform AWS Batch introspection when not running within " * "an AWS Batch job. Current job parameters are: " * - "name: $name\n" * - "queue: $queue\n" * - "memory: $memory\n" + "\nname=$name" * + "\nqueue=$queue" * + "\nmemory=$memory" )) end # Reuse a previously registered job definition if available. - # If no job definition exists that can be reused, a new job definition is created - # under the current job specifications. if isa(definition, AbstractString) reusable_job_definition_arn = job_definition_arn(definition; image=image, role=role) if reusable_job_definition_arn !== nothing definition = JobDefinition(reusable_job_definition_arn) - else + end + elseif definition === nothing + # Use the job name as the definiton name since the definition name was not specified + definition = name + end + + # If no job definition exists that can be reused, a new job definition is created + # under the current job specifications. + if isa(definition, AbstractString) + if allow_job_registration definition = register( definition; image=image, @@ -190,19 +197,18 @@ function run_batch(; region=region, parameters=parameters, ) + else + throw(BatchEnvironmentError(string( + "Attempting to register job definition \"$definition\" but registering ", + "job definitions is disallowed. Current job definition parameters are: ", + "\nimage=$image", + "\nrole=$role", + "\nvcpus=$vcpus", + "\nmemory=$memory", + "\ncmd=$cmd", + "\nparameters=$parameters", + ))) end - elseif definition === nothing - # Use the job name as the definiton name since the definition name was not specified - definition = register( - name; - image=image, - role=role, - vcpus=vcpus, - memory=memory, - cmd=cmd, - region=region, - parameters=parameters, - ) end # Parameters that can be overridden are `memory`, `vcpus`, `command`, and `environment` diff --git a/src/batch_job.jl b/src/batch_job.jl index af55516..8d67826 100644 --- a/src/batch_job.jl +++ b/src/batch_job.jl @@ -15,7 +15,7 @@ Stores a batch job id in order to: # Fields - `id::AbstractString`: jobId """ -struct BatchJob +@auto_hash_equals struct BatchJob id::AbstractString end diff --git a/src/job_definition.jl b/src/job_definition.jl index 832face..ee9c75b 100644 --- a/src/job_definition.jl +++ b/src/job_definition.jl @@ -6,7 +6,7 @@ import AWSSDK.Batch: Stores the job definition arn including the revision. """ -struct JobDefinition +@auto_hash_equals struct JobDefinition arn::AbstractString function JobDefinition(name::AbstractString) @@ -40,27 +40,43 @@ A job definition can only be reused if: function job_definition_arn( definition_name::AbstractString; image::AbstractString="", - role::AbstractString="" + role::AbstractString="", ) response = describe_job_definition(definition_name) - isempty(response["jobDefinitions"]) && return nothing + if !isempty(response["jobDefinitions"]) + + latest = first(response["jobDefinitions"]) + for definition in response["jobDefinitions"] + if definition["status"] == "ACTIVE" && definition["revision"] > latest["revision"] + latest = definition + end + end - latest = first(response["jobDefinitions"]) - for definition in response["jobDefinitions"] - if definition["status"] == "ACTIVE" && definition["revision"] > latest["revision"] - latest = definition + if ( + latest["status"] == "ACTIVE" && + latest["type"] == "container" && + (latest["containerProperties"]["image"] == image || isempty(image)) && + (latest["containerProperties"]["jobRoleArn"] == role || isempty(role)) + ) + info( + logger, + string( + "Found previously registered job definition: ", + "\"$(latest["jobDefinitionArn"])\"", + ) + ) + return latest["jobDefinitionArn"] end end - if ( - latest["status"] == "ACTIVE" && - latest["type"] == "container" && - (latest["containerProperties"]["image"] == image || isempty(image)) && - (latest["containerProperties"]["jobRoleArn"] == role || isempty(role)) + + notice( + logger, + string( + "Did not find a previously registered ACTIVE job definition for ", + "\"$definition_name\".", + ) ) - return latest["jobDefinitionArn"] - else - return nothing - end + return nothing end """ diff --git a/test/runtests.jl b/test/runtests.jl index 25af2ed..83cb8f9 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -25,7 +25,7 @@ const AWS_STACKNAME = get(ENV, "AWS_STACKNAME", "") const STACK = isempty(AWS_STACKNAME) ? LEGACY_STACK : stack_output(AWS_STACKNAME) const JULIA_BAKED_IMAGE = "292522074875.dkr.ecr.us-east-1.amazonaws.com/julia-baked:0.6" -Memento.config("debug"; fmt="[{level} | {name}]: {msg}") +Memento.config!("debug"; fmt="[{level} | {name}]: {msg}") setlevel!(getlogger(AWSBatch), "info") include("mock.jl") @@ -89,9 +89,41 @@ include("mock.jl") ] @test container_properties["jobRoleArn"] == STACK["JobRoleArn"] + # Reuse job definition + job = run_batch(; + name = "aws-batch-test", + definition = "aws-batch-test", + queue = STACK["ManagerJobQueueArn"], + image = JULIA_BAKED_IMAGE, + vcpus = 1, + memory = 1024, + role = STACK["JobRoleArn"], + cmd = `julia -e 'println("Hello World!")'`, + parameters = Dict{String, String}("region" => "us-east-1"), + ) + + @test wait(job, [AWSBatch.SUCCEEDED]) == true + @test status(job) == AWSBatch.SUCCEEDED + + # Test job definition and container parameters were set correctly + job_definition_2 = JobDefinition(job) + @test job_definition_2 == job_definition + deregister(job_definition) end + @testset "Job registration disallowed" begin + @test_throws BatchEnvironmentError run_batch(; + name = "aws-batch-no-job-registration-test", + queue = STACK["ManagerJobQueueArn"], + image = JULIA_BAKED_IMAGE, + role = STACK["JobRoleArn"], + cmd = `julia -e 'println("Hello World!")'`, + parameters = Dict{String, String}("region" => "us-east-1"), + allow_job_registration = false, + ) + end + @testset "Job parameters" begin # Use parameter substitution placeholders in the command field command = Cmd(["julia", "-e", "Ref::juliacmd"]) From 948a4a6e23f43fa98f74536904d6f670239f991d Mon Sep 17 00:00:00 2001 From: Alex Arslan Date: Wed, 22 Aug 2018 18:13:41 +0000 Subject: [PATCH 038/110] Updates for Julia 0.7 --- .gitlab-ci.yml | 46 ++++++++++++++++++++++++++++++++++++---------- REQUIRE | 4 ++-- src/AWSBatch.jl | 8 +++----- src/job_queue.jl | 2 +- test/REQUIRE | 1 + test/mock.jl | 4 ++-- test/runtests.jl | 6 ++++-- 7 files changed, 49 insertions(+), 22 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c77160e..f8855fc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,7 +16,7 @@ stages: - ./julia-ci install $JULIA_VERSION script: - source julia-ci export - - julia -e "Pkg.clone(pwd()); Pkg.build(\"$PKG_NAME\"); Pkg.test(\"$PKG_NAME\"; coverage=true)" + - ./julia-ci test - ./julia-ci coverage after_script: - ./julia-ci clean @@ -27,13 +27,19 @@ stages: ONLINE: "" <<: *test_shell -.test_shell_07-: &test_shell_07- +.test_shell_07: &test_shell_07 variables: - JULIA_VERSION: "0.7-" + JULIA_VERSION: "0.7" ONLINE: "" allow_failure: true <<: *test_shell +.test_shell_nightly: &test_shell_nightly + variables: + JULIA_VERSION: "nightly" + ONLINE: "" + allow_failure: true + <<: *test_shell .test_docker: &test_docker artifacts: @@ -72,25 +78,45 @@ stages: - shell-ci <<: *test_shell_06 -"0.7- (Mac)": +"0.7 (Mac)": + tags: + - mac + - shell-ci + <<: *test_shell_07 + +"0.7 (Linux, 64-bit)": + tags: + - linux + - 64-bit + - shell-ci + <<: *test_shell_07 + +"0.7 (Linux, 32-bit)": + tags: + - linux + - 32-bit + - shell-ci + <<: *test_shell_07 + +"Nightly (Mac)": tags: - mac - shell-ci - <<: *test_shell_07- + <<: *test_shell_nightly -"0.7- (Linux, 64-bit)": +"Nightly (Linux, 64-bit)": tags: - linux - 64-bit - shell-ci - <<: *test_shell_07- + <<: *test_shell_nightly -"0.7- (Linux, 32-bit)": +"Nightly (Linux, 32-bit)": tags: - linux - 32-bit - shell-ci - <<: *test_shell_07- + <<: *test_shell_nightly "Online Tests": stage: online-test @@ -116,7 +142,7 @@ stages: - source julia-ci export - export PATH="$PATH:/usr/local/bin" - unset SSL_CERT_DIR # https://github.com/JuliaLang/julia/issues/20439 - - julia -e "Pkg.clone(pwd()); Pkg.build(\"$PKG_NAME\"); Pkg.test(\"$PKG_NAME\"; coverage=true)" + - ./julia-ci test - ./julia-ci coverage after_script: - ./julia-ci clean diff --git a/REQUIRE b/REQUIRE index 759fde9..bb70ca8 100644 --- a/REQUIRE +++ b/REQUIRE @@ -2,7 +2,7 @@ julia 0.6 AutoHashEquals 0.2 AWSCore 0.3.0 AWSSDK 0.2.0 -Compat 0.41.0 +Compat 0.69.0 Mocking 0.3.3 Memento 0.7 -DataStructures 0.2.9 \ No newline at end of file +DataStructures 0.2.9 diff --git a/src/AWSBatch.jl b/src/AWSBatch.jl index 7a5fddd..75019cf 100644 --- a/src/AWSBatch.jl +++ b/src/AWSBatch.jl @@ -8,11 +8,9 @@ using Memento using Mocking using AWSCore: AWSConfig, AWSCredentials -using Compat: Nothing, AbstractDict +using Compat: Nothing, AbstractDict, @__MODULE__, undef using DataStructures: OrderedDict -import Base: showerror - export BatchJob, ComputeEnvironment, @@ -30,7 +28,7 @@ export BatchEnvironmentError -const logger = getlogger(current_module()) +const logger = getlogger(@__MODULE__) # Register the module level logger at runtime so that folks can access the logger via `getlogger(MyModule)` # NOTE: If this line is not included then the precompiled `MyModule.logger` won't be registered at runtime. __init__() = Memento.register(logger) @@ -48,7 +46,7 @@ struct BatchEnvironmentError <: Exception message::String end -function showerror(io::IO, e::BatchEnvironmentError) +function Base.showerror(io::IO, e::BatchEnvironmentError) print(io, "BatchEnvironmentError: ") print(io, e.message) end diff --git a/src/job_queue.jl b/src/job_queue.jl index 1b5679d..322295b 100644 --- a/src/job_queue.jl +++ b/src/job_queue.jl @@ -18,7 +18,7 @@ max_vcpus(queue::JobQueue) = sum(max_vcpus(ce) for ce in compute_environments(qu function compute_environments(queue::JobQueue) ce_order = describe(queue)["computeEnvironmentOrder"] - compute_envs = Vector{ComputeEnvironment}(length(ce_order)) + compute_envs = Vector{ComputeEnvironment}(undef, length(ce_order)) for ce in ce_order i, arn = ce["order"], ce["computeEnvironment"] compute_envs[i] = ComputeEnvironment(arn) diff --git a/test/REQUIRE b/test/REQUIRE index 6693cd1..4b62649 100644 --- a/test/REQUIRE +++ b/test/REQUIRE @@ -1 +1,2 @@ AWSTools 0.1.0 +Memento 0.7.0 diff --git a/test/mock.jl b/test/mock.jl index 3a9ffed..cc8bd63 100644 --- a/test/mock.jl +++ b/test/mock.jl @@ -106,10 +106,10 @@ to just return "us-east-1". function mock_readstring(cmd::CmdRedirect) cmd_exec = cmd.cmd.exec - result = if cmd_exec[1] == "curl" && contains(cmd_exec[2], "availability-zone") + result = if cmd_exec[1] == "curl" && occursin("availability-zone", cmd_exec[2]) return "us-east-1" else - return Base.readstring(cmd) + return read(cmd, String) end end diff --git a/test/runtests.jl b/test/runtests.jl index 83cb8f9..d37583d 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -5,7 +5,9 @@ using AWSBatch using AWSCore: AWSConfig using AWSTools.CloudFormation: stack_output -using Base.Test +using Compat: occursin +using Compat.Test +using Compat.Dates using Memento @@ -65,7 +67,7 @@ include("mock.jl") # Test job details were set correctly job_details = describe(job) @test job_details["jobName"] == "aws-batch-test" - @test contains(job_details["jobQueue"], STACK["ManagerJobQueueArn"]) + @test occursin(STACK["ManagerJobQueueArn"], job_details["jobQueue"]) @test job_details["parameters"] == Dict("region" => "us-east-1") # Test job definition and container parameters were set correctly From b4fd8fa91fe241d0850ece7d78e0e5eec5035828 Mon Sep 17 00:00:00 2001 From: Alex Arslan Date: Wed, 5 Sep 2018 14:04:56 -0700 Subject: [PATCH 039/110] Fix a couple more deprecations in Julia 0.7 In particular, the use of `DateTime` inside the module without having imported the Dates stdlib module, a call to `warn` in the tests, and a use of the old, camel-cased `devnull`. --- src/AWSBatch.jl | 5 +++-- test/runtests.jl | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/AWSBatch.jl b/src/AWSBatch.jl index 75019cf..7093726 100644 --- a/src/AWSBatch.jl +++ b/src/AWSBatch.jl @@ -6,9 +6,10 @@ using AWSSDK.Batch using AWSSDK.CloudWatchLogs using Memento using Mocking +using Compat.Dates using AWSCore: AWSConfig, AWSCredentials -using Compat: Nothing, AbstractDict, @__MODULE__, undef +using Compat: Nothing, AbstractDict, @__MODULE__, undef, devnull using DataStructures: OrderedDict export @@ -126,7 +127,7 @@ function run_batch(; zone = @mock readstring( pipeline( `curl http://169.254.169.254/latest/meta-data/placement/availability-zone`; - stderr=DevNull + stderr=devnull ) ) isempty(region) && (region = chop(zone)) diff --git a/test/runtests.jl b/test/runtests.jl index d37583d..31237d5 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -28,7 +28,8 @@ const STACK = isempty(AWS_STACKNAME) ? LEGACY_STACK : stack_output(AWS_STACKNAME const JULIA_BAKED_IMAGE = "292522074875.dkr.ecr.us-east-1.amazonaws.com/julia-baked:0.6" Memento.config!("debug"; fmt="[{level} | {name}]: {msg}") -setlevel!(getlogger(AWSBatch), "info") +const logger = getlogger(AWSBatch) +setlevel!(logger, "info") include("mock.jl") @@ -280,6 +281,6 @@ include("mock.jl") end end else - warn("Skipping ONLINE tests") + warn(logger, "Skipping ONLINE tests") end end From 2f6f023befd698aa256a9985dc3e5d68dd8ba126 Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Wed, 12 Sep 2018 13:08:55 -0500 Subject: [PATCH 040/110] Update read for 0.7 --- src/AWSBatch.jl | 5 +++-- test/mock.jl | 14 +++++++------- test/run_batch.jl | 4 ++-- test/runtests.jl | 6 +++--- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/AWSBatch.jl b/src/AWSBatch.jl index 7093726..76b72e9 100644 --- a/src/AWSBatch.jl +++ b/src/AWSBatch.jl @@ -124,11 +124,12 @@ function run_batch(; job_queue = ENV["AWS_BATCH_JQ_NAME"] # Get the zone information from the EC2 instance metadata. - zone = @mock readstring( + zone = @mock read( pipeline( `curl http://169.254.169.254/latest/meta-data/placement/availability-zone`; stderr=devnull - ) + ), + String ) isempty(region) && (region = chop(zone)) diff --git a/test/mock.jl b/test/mock.jl index cc8bd63..e08bf01 100644 --- a/test/mock.jl +++ b/test/mock.jl @@ -98,18 +98,18 @@ const DESCRIBE_JOBS_RESP = Dict( """ - Mock.readstring(cmd::CmdRedirect, pass::Bool=true) + Mock.read(cmd::CmdRedirect, ::Type{String}) -Mocks the CmdRedirect produced from ``pipeline(`curl http://169.254.169.254/latest/meta-data/placement/availability-zone`)`` -to just return "us-east-1". +Mocks the CmdRedirect produced from +``pipeline(`curl http://169.254.169.254/latest/meta-data/placement/availability-zone`)`` +to just return "us-east-1a". """ -function mock_readstring(cmd::CmdRedirect) +function mock_read(cmd::CmdRedirect, ::Type{String}) cmd_exec = cmd.cmd.exec - result = if cmd_exec[1] == "curl" && occursin("availability-zone", cmd_exec[2]) - return "us-east-1" + return "us-east-1a" else - return read(cmd, String) + return Base.read(cmd, String) end end diff --git a/test/run_batch.jl b/test/run_batch.jl index 8235681..07af9e1 100644 --- a/test/run_batch.jl +++ b/test/run_batch.jl @@ -69,7 +69,7 @@ end ] patches = [ - @patch readstring(cmd::AbstractCmd) = mock_readstring(cmd) + @patch read(cmd::AbstractCmd, ::Type{String}) = mock_read(cmd, String) @patch describe_jobs(args...) = DESCRIBE_JOBS_RESP @patch describe_job_definitions(args...) = Dict("jobDefinitions" => Dict()) @patch register_job_definition(config, input) = register_job_def( @@ -102,7 +102,7 @@ end ] patches = [ - @patch readstring(cmd::AbstractCmd) = mock_readstring(cmd) + @patch read(cmd::AbstractCmd, ::Type{String}) = mock_read(cmd, String) @patch describe_jobs(args...) = DESCRIBE_JOBS_RESP @patch describe_job_definitions(args...) = Dict("jobDefinitions" => Dict()) @patch submit_job(config, input) = submit_job(config, input, expected_job) diff --git a/test/runtests.jl b/test/runtests.jl index 31237d5..c498e8e 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -186,7 +186,7 @@ include("mock.jl") @testset "Array job" begin job = run_batch(; name = "aws-batch-array-job-test", - definition = "aws-batch-test", + definition = "aws-batch-array-job-test", queue = STACK["ManagerJobQueueArn"], image = JULIA_BAKED_IMAGE, vcpus = 1, @@ -231,7 +231,7 @@ include("mock.jl") job = run_batch(; name = "aws-bath-timeout-job-test", - definition = "aws-batch-test", + definition = "aws-bath-timeout-job-test", queue = STACK["ManagerJobQueueArn"], image = JULIA_BAKED_IMAGE, vcpus = 1, @@ -260,7 +260,7 @@ include("mock.jl") job = run_batch(; name = "aws-batch-failed-job-test", - definition = "aws-batch-test", + definition = "aws-batch-failed-job-test", queue = STACK["ManagerJobQueueArn"], image = JULIA_BAKED_IMAGE, vcpus = 1, From 432dd7d1f18971eaf7859436195fcee6330b2906 Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Wed, 12 Sep 2018 16:10:39 -0500 Subject: [PATCH 041/110] More 0.7 updates --- .gitlab-ci.yml | 2 +- test/REQUIRE | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f8855fc..b9fd61d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,6 +13,7 @@ stages: before_script: - curl -s -o julia-ci https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/julia-ci - chmod +x julia-ci + - ./julia-ci install-cred-helper - ./julia-ci install $JULIA_VERSION script: - source julia-ci export @@ -31,7 +32,6 @@ stages: variables: JULIA_VERSION: "0.7" ONLINE: "" - allow_failure: true <<: *test_shell .test_shell_nightly: &test_shell_nightly diff --git a/test/REQUIRE b/test/REQUIRE index 4b62649..3a8022f 100644 --- a/test/REQUIRE +++ b/test/REQUIRE @@ -1,2 +1 @@ -AWSTools 0.1.0 -Memento 0.7.0 +AWSTools 0.3.1 From 9969908c710c925ada3bd125fede17eec88dda76 Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Wed, 19 Sep 2018 11:30:02 -0500 Subject: [PATCH 042/110] Update tests and increase job timeouts --- src/AWSBatch.jl | 3 ++- src/batch_job.jl | 12 ++++++++++-- test/runtests.jl | 31 ++++++++++++++++++++----------- 3 files changed, 32 insertions(+), 14 deletions(-) diff --git a/src/AWSBatch.jl b/src/AWSBatch.jl index 76b72e9..886337c 100644 --- a/src/AWSBatch.jl +++ b/src/AWSBatch.jl @@ -26,7 +26,8 @@ export isregistered, register, deregister, - BatchEnvironmentError + BatchEnvironmentError, + BatchJobError const logger = getlogger(@__MODULE__) diff --git a/src/batch_job.jl b/src/batch_job.jl index 8d67826..dcf708e 100644 --- a/src/batch_job.jl +++ b/src/batch_job.jl @@ -1,6 +1,14 @@ import AWSSDK.Batch: describe_jobs, submit_job import AWSSDK.CloudWatchLogs: get_log_events +struct BatchJobError <: Exception + message::String +end + +function Base.showerror(io::IO, e::BatchJobError) + print(io, "BatchJobError: ") + print(io, e.message) +end """ BatchJob @@ -157,7 +165,7 @@ function Base.wait( message *= " Last known state $last_state" end - error(logger, message) + throw(BatchJobError(message)) end return completed @@ -186,7 +194,7 @@ function Base.wait( if state in cond false elseif state in failure - error(logger, "Job $(job.id) hit failure condition $state") + throw(BatchJobError("Job $(job.id) hit failure condition $state")) false else true diff --git a/test/runtests.jl b/test/runtests.jl index c498e8e..3a9d46e 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -26,6 +26,7 @@ const LEGACY_STACK = Dict( const AWS_STACKNAME = get(ENV, "AWS_STACKNAME", "") const STACK = isempty(AWS_STACKNAME) ? LEGACY_STACK : stack_output(AWS_STACKNAME) const JULIA_BAKED_IMAGE = "292522074875.dkr.ecr.us-east-1.amazonaws.com/julia-baked:0.6" +const JOB_TIMEOUT = 900 Memento.config!("debug"; fmt="[{level} | {name}]: {msg}") const logger = getlogger(AWSBatch) @@ -43,7 +44,7 @@ include("mock.jl") if "batch" in ONLINE @testset "Online" begin - info("Running ONLINE tests") + info(logger, "Running ONLINE tests") @testset "Job Submission" begin job = run_batch(; @@ -58,7 +59,7 @@ include("mock.jl") parameters = Dict{String, String}("region" => "us-east-1"), ) - @test wait(job, [AWSBatch.SUCCEEDED]) == true + @test wait(job, [AWSBatch.SUCCEEDED]; timeout=JOB_TIMEOUT) == true @test status(job) == AWSBatch.SUCCEEDED events = log_events(job) @@ -105,7 +106,7 @@ include("mock.jl") parameters = Dict{String, String}("region" => "us-east-1"), ) - @test wait(job, [AWSBatch.SUCCEEDED]) == true + @test wait(job, [AWSBatch.SUCCEEDED]; timeout=JOB_TIMEOUT) == true @test status(job) == AWSBatch.SUCCEEDED # Test job definition and container parameters were set correctly @@ -156,7 +157,7 @@ include("mock.jl") parameters=Dict("juliacmd" => "println(\"Hello World!\")"), ) - @test wait(state -> state < AWSBatch.SUCCEEDED, job) + @test wait(state -> state < AWSBatch.SUCCEEDED, job; timeout=JOB_TIMEOUT) @test status(job) == AWSBatch.SUCCEEDED # Test the default string was overrriden succesfully @@ -196,7 +197,7 @@ include("mock.jl") num_jobs = 3, ) - @test wait(state -> state < AWSBatch.SUCCEEDED, job) + @test wait(state -> state < AWSBatch.SUCCEEDED, job; timeout=JOB_TIMEOUT) @test status(job) == AWSBatch.SUCCEEDED # Test array job was submitted properly @@ -227,10 +228,10 @@ include("mock.jl") end @testset "Job Timed Out" begin - info("Testing job timeout") + info(logger, "Testing job timeout") job = run_batch(; - name = "aws-bath-timeout-job-test", + name = "aws-batch-timeout-job-test", definition = "aws-bath-timeout-job-test", queue = STACK["ManagerJobQueueArn"], image = JULIA_BAKED_IMAGE, @@ -243,7 +244,7 @@ include("mock.jl") job_definition = JobDefinition(job) @test isregistered(job_definition) == true - @test_throws ErrorException wait( + @test_throws BatchJobError wait( state -> state < AWSBatch.SUCCEEDED, job; timeout=0 @@ -256,7 +257,7 @@ include("mock.jl") end @testset "Failed Job" begin - info("Testing job failure") + info(logger, "Testing job failure") job = run_batch(; name = "aws-batch-failed-job-test", @@ -272,12 +273,20 @@ include("mock.jl") job_definition = JobDefinition(job) @test isregistered(job_definition) == true - @test_throws ErrorException wait(job, [AWSBatch.SUCCEEDED]) + @test_throws BatchJobError wait( + job, + [AWSBatch.SUCCEEDED]; + timeout=JOB_TIMEOUT + ) deregister(job_definition) events = log_events(job) - @test first(events).message == "ERROR: Testing job failure" + + # Cannot guarantee this job failure will always have logs + if length(events) > 0 + @test first(events).message == "ERROR: Testing job failure" + end end end else From 56e26b839a1516e0d794729364f55ecad3c18634 Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Wed, 19 Sep 2018 13:26:46 -0500 Subject: [PATCH 043/110] Add job id to BatchJobError --- src/batch_job.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/batch_job.jl b/src/batch_job.jl index dcf708e..8a3ee73 100644 --- a/src/batch_job.jl +++ b/src/batch_job.jl @@ -2,6 +2,7 @@ import AWSSDK.Batch: describe_jobs, submit_job import AWSSDK.CloudWatchLogs: get_log_events struct BatchJobError <: Exception + job_id::AbstractString message::String end @@ -165,7 +166,7 @@ function Base.wait( message *= " Last known state $last_state" end - throw(BatchJobError(message)) + throw(BatchJobError(job.id, message)) end return completed @@ -194,7 +195,7 @@ function Base.wait( if state in cond false elseif state in failure - throw(BatchJobError("Job $(job.id) hit failure condition $state")) + throw(BatchJobError(job.id, "Job $(job.id) hit failure condition $state")) false else true From 16400f6d563324748996f44a8e1b54beebc476e1 Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Fri, 21 Sep 2018 10:03:56 -0500 Subject: [PATCH 044/110] Update CI testing to 1.0 --- .gitlab-ci.yml | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b9fd61d..84b79b9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -22,15 +22,16 @@ stages: after_script: - ./julia-ci clean -.test_shell_06: &test_shell_06 + +.test_shell_0_6: &test_shell_0_6 variables: JULIA_VERSION: "0.6" ONLINE: "" <<: *test_shell -.test_shell_07: &test_shell_07 +.test_shell_1_0: &test_shell_1_0 variables: - JULIA_VERSION: "0.7" + JULIA_VERSION: "1.0" ONLINE: "" <<: *test_shell @@ -41,6 +42,7 @@ stages: allow_failure: true <<: *test_shell + .test_docker: &test_docker artifacts: name: coverage @@ -53,7 +55,7 @@ stages: - julia -e "Pkg.clone(pwd()); Pkg.build(\"$PKG_NAME\"); Pkg.test(\"$PKG_NAME\"; coverage=true)" - julia-coverage $PKG_NAME -.test_docker_06: &test_docker_06 +.test_docker_0_6: &test_docker_0_6 image: 292522074875.dkr.ecr.us-east-1.amazonaws.com/julia-gitlab-ci:0.6 <<: *test_docker @@ -62,41 +64,41 @@ stages: tags: - mac - shell-ci - <<: *test_shell_06 + <<: *test_shell_0_6 "0.6 (Linux, 64-bit)": tags: - linux - 64-bit - docker-ci - <<: *test_docker_06 + <<: *test_docker_0_6 "0.6 (Linux, 32-bit)": tags: - linux - 32-bit - shell-ci - <<: *test_shell_06 + <<: *test_shell_0_6 -"0.7 (Mac)": +"1.0 (Mac)": tags: - mac - shell-ci - <<: *test_shell_07 + <<: *test_shell_1_0 -"0.7 (Linux, 64-bit)": +"1.0 (Linux, 64-bit)": tags: - linux - 64-bit - shell-ci - <<: *test_shell_07 + <<: *test_shell_1_0 -"0.7 (Linux, 32-bit)": +"1.0 (Linux, 32-bit)": tags: - linux - 32-bit - shell-ci - <<: *test_shell_07 + <<: *test_shell_1_0 "Nightly (Mac)": tags: @@ -118,6 +120,7 @@ stages: - shell-ci <<: *test_shell_nightly + "Online Tests": stage: online-test tags: @@ -147,6 +150,7 @@ stages: after_script: - ./julia-ci clean + "Coverage": stage: coverage image: 292522074875.dkr.ecr.us-east-1.amazonaws.com/julia-gitlab-ci:0.6 @@ -167,6 +171,7 @@ stages: - echo "Test Coverage $(genhtml -o $CI_PROJECT_DIR/combined_coverage --no-prefix */coverage.info 2>&1 | grep lines | awk '{print $2}')" - find $CI_PROJECT_DIR/combined_coverage -type f -name "*.jl" -delete + "Documentation": stage: docs tags: From 5f8fe6e02e2cd04c606f53c69d62eb819d3b41ce Mon Sep 17 00:00:00 2001 From: Sam Morrison Date: Tue, 30 Oct 2018 17:06:48 +0000 Subject: [PATCH 045/110] Updates gitlab-ci and makedoc kwargs --- .gitlab-ci.yml | 21 +++++++++------------ docs/make.jl | 2 ++ test/runtests.jl | 2 +- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 84b79b9..0100e84 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,7 +13,6 @@ stages: before_script: - curl -s -o julia-ci https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/julia-ci - chmod +x julia-ci - - ./julia-ci install-cred-helper - ./julia-ci install $JULIA_VERSION script: - source julia-ci export @@ -56,7 +55,7 @@ stages: - julia-coverage $PKG_NAME .test_docker_0_6: &test_docker_0_6 - image: 292522074875.dkr.ecr.us-east-1.amazonaws.com/julia-gitlab-ci:0.6 + image: 468665244580.dkr.ecr.us-east-1.amazonaws.com/julia-gitlab-ci:0.6 <<: *test_docker @@ -153,7 +152,7 @@ stages: "Coverage": stage: coverage - image: 292522074875.dkr.ecr.us-east-1.amazonaws.com/julia-gitlab-ci:0.6 + image: 468665244580.dkr.ecr.us-east-1.amazonaws.com/julia-gitlab-ci:0.6 coverage: /Test Coverage (\d+\.\d+%)/ artifacts: name: combined_coverage @@ -163,13 +162,13 @@ stages: tags: - linux - docker-ci + before_script: + - curl -s -o julia-ci https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/julia-ci + - chmod +x julia-ci script: - - genhtml --version - - mkdir $CI_PROJECT_DIR/combined_coverage - - cp -r $CI_PROJECT_DIR/src $CI_PROJECT_DIR/combined_coverage/ - - ls */*.info | xargs -I{} echo '--summary "{}"' | xargs lcov --directory src - - echo "Test Coverage $(genhtml -o $CI_PROJECT_DIR/combined_coverage --no-prefix */coverage.info 2>&1 | grep lines | awk '{print $2}')" - - find $CI_PROJECT_DIR/combined_coverage -type f -name "*.jl" -delete + - ./julia-ci publish-coverage combined_coverage + after_script: + - ./julia-ci clean "Documentation": @@ -189,8 +188,6 @@ stages: - ./julia-ci install $JULIA_VERSION script: - source julia-ci export - - julia --depwarn=no -e "Pkg.clone(pwd()); Pkg.build(\"$PKG_NAME\"); Pkg.add(\"Documenter\")" - - julia --depwarn=no -e "cd(Pkg.dir(\"$PKG_NAME\", \"docs\")); include(\"make.jl\")" - - julia --depwarn=no -e "const DOCS_DIR = joinpath(\"/mnt\", \"docs\", \"$CI_PROJECT_NAMESPACE\", \"$CI_PROJECT_NAME\", \"$CI_BUILD_REF_NAME\"); mkpath(DOCS_DIR); cp(Pkg.dir(\"$PKG_NAME\", \"docs\", \"build\"), DOCS_DIR; remove_destination=true, follow_symlinks=true)" + - ./julia-ci docs after_script: - ./julia-ci clean diff --git a/docs/make.jl b/docs/make.jl index 490b6e0..111ce4b 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -13,4 +13,6 @@ makedocs(; "assets/invenia.css", "assets/logo.png", ], + strict = true, + checkdocs = :none, ) diff --git a/test/runtests.jl b/test/runtests.jl index 3a9d46e..413403f 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -25,7 +25,7 @@ const LEGACY_STACK = Dict( const AWS_STACKNAME = get(ENV, "AWS_STACKNAME", "") const STACK = isempty(AWS_STACKNAME) ? LEGACY_STACK : stack_output(AWS_STACKNAME) -const JULIA_BAKED_IMAGE = "292522074875.dkr.ecr.us-east-1.amazonaws.com/julia-baked:0.6" +const JULIA_BAKED_IMAGE = "468665244580.dkr.ecr.us-east-1.amazonaws.com/julia-baked:0.6" const JOB_TIMEOUT = 900 Memento.config!("debug"; fmt="[{level} | {name}]: {msg}") From a6ac7b03b13ff1190e28bf32f93e6f9b4660e3cc Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Tue, 30 Oct 2018 13:10:52 -0500 Subject: [PATCH 046/110] Fix documentation for describing job definitions --- docs/src/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/index.md b/docs/src/index.md index cd8ade4..84f746a 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -72,7 +72,7 @@ AWSBatch.job_definition_arn(::AbstractString) AWSBatch.register(::AbstractString) AWSBatch.deregister(::JobDefinition) AWSBatch.isregistered(::JobDefinition) -AWSBatch.describe(::Union{AbstractString, JobDefinition}) +AWSBatch.describe(::JobDefinition) ``` ### JobState From 8eb3057cca7af4dad855398307a7c41fe38b673b Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Tue, 30 Oct 2018 13:13:46 -0500 Subject: [PATCH 047/110] Remove legacy stack default from tests --- README.md | 14 ++++++++++++++ test/runtests.jl | 21 +++++++++------------ 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 7a312c6..36d783d 100644 --- a/README.md +++ b/README.md @@ -3,3 +3,17 @@ [![Latest](https://img.shields.io/badge/docs-latest-blue.svg)](https://doc.invenia.ca/invenia/AWSBatch.jl/master) [![Build Status](https://gitlab.invenia.ca/invenia/AWSBatch.jl/badges/master/build.svg)](https://gitlab.invenia.ca/invenia/AWSBatch.jl/commits/master) [![Coverage](https://gitlab.invenia.ca/invenia/AWSBatch.jl/badges/master/coverage.svg)](https://gitlab.invenia.ca/invenia/AWSBatch.jl/commits/master) + +# Running the tests + +To run the ONLINE batch tests you must first set the environmental variables `ONLINE` and +`AWS_STACKNAME`. + +```julia +ENV["ONLINE"] = "batch" + +ENV["AWS_STACKNAME"] = "aws-batch-manager-test" +``` + +To make an `aws-batch-manager-test` compatible stack you can use the AWSClusterManagers.jl +CloudFormation template [test/batch.yml](https://gitlab.invenia.ca/invenia/AWSClusterManagers.jl/blob/master/test/batch.yml). diff --git a/test/runtests.jl b/test/runtests.jl index 413403f..00b5d79 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -14,17 +14,10 @@ using Memento # Enables the running of the "batch" online tests. e.g ONLINE=batch const ONLINE = strip.(split(get(ENV, "ONLINE", ""), r"\s*,\s*")) -# Partially emulates the output from the AWS batch manager test stack -const LEGACY_STACK = Dict( - "ManagerJobQueueArn" => "Replatforming-Manager", - "JobName" => "aws-batch-test", - "JobDefinitionName" => "aws-batch-test", - "JobRoleArn" => "arn:aws:iam::292522074875:role/AWSBatchClusterManagerJobRole", - "EcrUri" => "292522074875.dkr.ecr.us-east-1.amazonaws.com/aws-cluster-managers-test:latest", -) - +# Run the tests on a stack created with the "test/batch.yml" CloudFormation template +# found in AWSClusterMangers.jl const AWS_STACKNAME = get(ENV, "AWS_STACKNAME", "") -const STACK = isempty(AWS_STACKNAME) ? LEGACY_STACK : stack_output(AWS_STACKNAME) +const STACK = !isempty(AWS_STACKNAME) ? stack_output(AWS_STACKNAME) : Dict() const JULIA_BAKED_IMAGE = "468665244580.dkr.ecr.us-east-1.amazonaws.com/julia-baked:0.6" const JOB_TIMEOUT = 900 @@ -42,7 +35,7 @@ include("mock.jl") include("job_state.jl") include("run_batch.jl") - if "batch" in ONLINE + if "batch" in ONLINE && !isempty(AWS_STACKNAME) @testset "Online" begin info(logger, "Running ONLINE tests") @@ -290,6 +283,10 @@ include("mock.jl") end end else - warn(logger, "Skipping ONLINE tests") + warn( + logger, + "Skipping ONLINE tests. Set `ENV[\"ONLINE\"] = \"batch\"` and " * + "`ENV[\"AWS_STACKNAME\"]` to run." + ) end end From 5c4fb3720c5163cffe9ed2591d8b14c3c384d3da Mon Sep 17 00:00:00 2001 From: morris25 Date: Mon, 3 Dec 2018 08:55:56 -0600 Subject: [PATCH 048/110] Use Pages for docs --- .gitlab-ci.yml | 12 +++++------- README.md | 2 +- docs/src/index.md | 2 +- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0100e84..96cdc6e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -171,14 +171,9 @@ stages: - ./julia-ci clean -"Documentation": +pages: stage: docs - tags: - - docs only: - - master - - develop - - docs - tags # special keyword for all tags variables: JULIA_VERSION: "0.6" @@ -188,6 +183,9 @@ stages: - ./julia-ci install $JULIA_VERSION script: - source julia-ci export - - ./julia-ci docs + - ./julia-ci publish-docs public after_script: - ./julia-ci clean + artifacts: + paths: + - public/ diff --git a/README.md b/README.md index 36d783d..cbff91d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # AWSBatch -[![Latest](https://img.shields.io/badge/docs-latest-blue.svg)](https://doc.invenia.ca/invenia/AWSBatch.jl/master) +[![Latest](https://img.shields.io/badge/docs-latest-blue.svg)](https://invenia.pages.invenia.ca/AWSBatch.jl/) [![Build Status](https://gitlab.invenia.ca/invenia/AWSBatch.jl/badges/master/build.svg)](https://gitlab.invenia.ca/invenia/AWSBatch.jl/commits/master) [![Coverage](https://gitlab.invenia.ca/invenia/AWSBatch.jl/badges/master/coverage.svg)](https://gitlab.invenia.ca/invenia/AWSBatch.jl/commits/master) diff --git a/docs/src/index.md b/docs/src/index.md index 84f746a..0e7b440 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -1,6 +1,6 @@ # AWSBatch -[![Latest](https://img.shields.io/badge/docs-latest-blue.svg)](https://doc.invenia.ca/invenia/AWSBatch.jl/master) +[![Latest](https://img.shields.io/badge/docs-latest-blue.svg)](https://invenia.pages.invenia.ca/AWSBatch.jl/) [![Build Status](https://gitlab.invenia.ca/invenia/AWSBatch.jl/badges/master/build.svg)](https://gitlab.invenia.ca/invenia/AWSBatch.jl/commits/master) [![Coverage](https://gitlab.invenia.ca/invenia/AWSBatch.jl/badges/master/coverage.svg)](https://gitlab.invenia.ca/invenia/AWSBatch.jl/commits/master) From bebf549b0b399f72588e0bf9f263c15620fbbb9e Mon Sep 17 00:00:00 2001 From: morris25 Date: Mon, 3 Dec 2018 15:34:15 -0600 Subject: [PATCH 049/110] Removes docs dir arg --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 96cdc6e..a769a41 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -183,7 +183,7 @@ pages: - ./julia-ci install $JULIA_VERSION script: - source julia-ci export - - ./julia-ci publish-docs public + - ./julia-ci publish-docs after_script: - ./julia-ci clean artifacts: From dc5aaf0b4f8f4a0f1faa5ee27cd2a7266acc552a Mon Sep 17 00:00:00 2001 From: morris25 Date: Tue, 4 Dec 2018 13:48:08 -0600 Subject: [PATCH 050/110] Deploy docs on master --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a769a41..0a3e217 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -174,7 +174,7 @@ stages: pages: stage: docs only: - - tags # special keyword for all tags + - master variables: JULIA_VERSION: "0.6" before_script: From bb169d2c0e2074f6c75b8f68fbc549306d69661d Mon Sep 17 00:00:00 2001 From: morris25 Date: Thu, 13 Dec 2018 13:11:33 -0600 Subject: [PATCH 051/110] Use gitlab-ci templates --- .gitlab-ci.yml | 133 +++++++------------------------------------------ 1 file changed, 18 insertions(+), 115 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0a3e217..499a563 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,122 +2,77 @@ stages: - test - online-test - coverage - - docs + - documentation -.test_shell: &test_shell - artifacts: - name: "$CI_JOB_NAME coverage" - expire_in: 1 week - paths: - - "$CI_JOB_NAME coverage/" - before_script: - - curl -s -o julia-ci https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/julia-ci - - chmod +x julia-ci - - ./julia-ci install $JULIA_VERSION - script: - - source julia-ci export - - ./julia-ci test - - ./julia-ci coverage - after_script: - - ./julia-ci clean - - -.test_shell_0_6: &test_shell_0_6 - variables: - JULIA_VERSION: "0.6" - ONLINE: "" - <<: *test_shell - -.test_shell_1_0: &test_shell_1_0 - variables: - JULIA_VERSION: "1.0" - ONLINE: "" - <<: *test_shell - -.test_shell_nightly: &test_shell_nightly - variables: - JULIA_VERSION: "nightly" - ONLINE: "" - allow_failure: true - <<: *test_shell - - -.test_docker: &test_docker - artifacts: - name: coverage - expire_in: 1 week - paths: - - "$CI_JOB_NAME coverage/" - variables: - ONLINE: "" - script: - - julia -e "Pkg.clone(pwd()); Pkg.build(\"$PKG_NAME\"); Pkg.test(\"$PKG_NAME\"; coverage=true)" - - julia-coverage $PKG_NAME +include: + - https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/gitlab-templates/test_templates.yml + - https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/gitlab-templates/coverage.yml + - https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/gitlab-templates/documentation.yml -.test_docker_0_6: &test_docker_0_6 - image: 468665244580.dkr.ecr.us-east-1.amazonaws.com/julia-gitlab-ci:0.6 - <<: *test_docker +variables: + JULIA_IMAGE: 468665244580.dkr.ecr.us-east-1.amazonaws.com/julia-gitlab-ci:0.6 + ONLINE: "" "0.6 (Mac)": tags: - mac - shell-ci - <<: *test_shell_0_6 + extends: .test_shell_0_6 "0.6 (Linux, 64-bit)": tags: - linux - 64-bit - docker-ci - <<: *test_docker_0_6 + extends: .test_docker "0.6 (Linux, 32-bit)": tags: - linux - 32-bit - shell-ci - <<: *test_shell_0_6 + extends: .test_shell_0_6 "1.0 (Mac)": tags: - mac - shell-ci - <<: *test_shell_1_0 + extends: .test_shell_1_0 "1.0 (Linux, 64-bit)": tags: - linux - 64-bit - shell-ci - <<: *test_shell_1_0 + extends: .test_shell_1_0 "1.0 (Linux, 32-bit)": tags: - linux - 32-bit - shell-ci - <<: *test_shell_1_0 + extends: .test_shell_1_0 "Nightly (Mac)": tags: - mac - shell-ci - <<: *test_shell_nightly + extends: .test_shell_nightly "Nightly (Linux, 64-bit)": tags: - linux - 64-bit - shell-ci - <<: *test_shell_nightly + extends: .test_shell_nightly "Nightly (Linux, 32-bit)": tags: - linux - 32-bit - shell-ci - <<: *test_shell_nightly + extends: .test_shell_nightly "Online Tests": @@ -127,65 +82,13 @@ stages: - ecr - linux - shell-ci - artifacts: - name: "$CI_JOB_NAME coverage" - expire_in: 1 week - paths: - - "$CI_JOB_NAME coverage/" variables: AWS_STACKNAME: aws-batch-manager-test ONLINE: "batch" # Runs the online tests against AWS - JULIA_VERSION: "0.6" - before_script: - - curl -s -o julia-ci https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/julia-ci - - chmod +x julia-ci - - ./julia-ci install $JULIA_VERSION script: - source julia-ci export - export PATH="$PATH:/usr/local/bin" - unset SSL_CERT_DIR # https://github.com/JuliaLang/julia/issues/20439 - ./julia-ci test - ./julia-ci coverage - after_script: - - ./julia-ci clean - - -"Coverage": - stage: coverage - image: 468665244580.dkr.ecr.us-east-1.amazonaws.com/julia-gitlab-ci:0.6 - coverage: /Test Coverage (\d+\.\d+%)/ - artifacts: - name: combined_coverage - expire_in: 1 week - paths: - - combined_coverage/ - tags: - - linux - - docker-ci - before_script: - - curl -s -o julia-ci https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/julia-ci - - chmod +x julia-ci - script: - - ./julia-ci publish-coverage combined_coverage - after_script: - - ./julia-ci clean - - -pages: - stage: docs - only: - - master - variables: - JULIA_VERSION: "0.6" - before_script: - - curl -s -o julia-ci https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/julia-ci - - chmod +x julia-ci - - ./julia-ci install $JULIA_VERSION - script: - - source julia-ci export - - ./julia-ci publish-docs - after_script: - - ./julia-ci clean - artifacts: - paths: - - public/ + extends: .test_shell_0_6 From 97f83ae3a1d3e36f200aa123d1f15300f94618aa Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Thu, 20 Dec 2018 13:15:00 -0600 Subject: [PATCH 052/110] Setup with TestRole --- .gitlab-ci.yml | 161 +++++++++++++++++++++++++++++-- REQUIRE | 2 +- test/batch.yml | 246 +++++++++++++++++++++++++++++++++++++++++++++++ test/runtests.jl | 16 +-- 4 files changed, 406 insertions(+), 19 deletions(-) create mode 100644 test/batch.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 499a563..28f7ff9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,8 +1,9 @@ stages: + - setup - test - - online-test - coverage - documentation + - teardown include: - https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/gitlab-templates/test_templates.yml @@ -10,9 +11,106 @@ include: - https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/gitlab-templates/documentation.yml variables: + STACKNAME_PREFIX: sandbox-awsbatch + AMAZON_LINUX_IMAGE: amazonlinux:2 JULIA_IMAGE: 468665244580.dkr.ecr.us-east-1.amazonaws.com/julia-gitlab-ci:0.6 ONLINE: "" +# Variables that require shell execution or depend on a variable that does +.runtime_variables: &runtime_variables + | + # Declare runtime variables + # + # Make unique stacks for each executed pipeline for `master`. This will allow fast successive merges + # to work correctly. + if [[ ${CI_COMMIT_REF_SLUG} == "master" ]]; then + STACKNAME="${STACKNAME_PREFIX}-master-${CI_PIPELINE_ID}" + else + STACKNAME="${STACKNAME_PREFIX}-${CI_COMMIT_REF_SLUG/\//}" # Replace an forward slashes with a hyphen + fi + ACCOUNT_ID=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | grep -oP '(?<="accountId" : ")[^"]*(?=")') + export AWS_DEFAULT_REGION=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | grep -oP '(?<="region" : ")[^"]*(?=")') + +.awscli_install: &awscli_install + | + # Build cloudspy and its dependencies + NEED_SUDO="sudo" + + # docker-ci has slightly different install parameters + if [[ ${CI_RUNNER_TAGS} == *"docker-ci"* ]]; then + # Install which command + yum -y install which + # Install epel to get python36 rather than python3 + amazon-linux-extras install epel + # We don't need to use sudo on the docker-ci tagged instances + NEED_SUDO="" + fi + + # Install python36, virtualenv, and necessary packages + ${NEED_SUDO} yum -y install python36 python36-devel python-virtualenv gcc git + + # Install a virtualenv with python 3 and activate it + virtualenv --python=$(which python36) venv + source venv/bin/activate + + # Install AWSCli + pip install --upgrade awscli + + # Print aws version + aws --version + +.cloudspy_install: &cloudspy_install + | + # Install Cloudspy in the env + pip install git+https://gitlab-ci-token:${CI_BUILD_TOKEN}@gitlab.invenia.ca/infrastructure/cloudspy.git#egg=cloudspy + # Export temp credentials. Make sure this path is absolute as Julia can change directories during testing + export AWS_SHARED_CREDENTIALS_FILE="$(pwd)/tmp-creds" + +.assume_test_profile: &assume_test_profile + | + aws-credentials \ + --credentials-file=$AWS_SHARED_CREDENTIALS_FILE \ + --role-arn arn:aws:iam::${ACCOUNT_ID}:role/${STACKNAME}-TestRole \ + --role-session-name test + export AWS_PROFILE=test + + +"Setup Environment": + stage: setup + except: + - tags + - master + - /^.+\/.*master$/ # e.g. jh/validate-master + when: always + environment: + name: branch/$CI_COMMIT_REF_SLUG + on_stop: "Delete Environment" + script: + - echo "Setting up environment" + +"Create Stack": + stage: setup + except: + - tags + image: $AMAZON_LINUX_IMAGE + tags: + - docker-ci + - ci-account + before_script: + - *runtime_variables + - yum -y update + - *awscli_install + - *cloudspy_install + script: + - aws cloudformation validate-template --template-body file://test/batch.yml + - | + aws-create-stack \ + --role-arn arn:aws:iam::${ACCOUNT_ID}:role/CloudFormationAdmin \ + --stackname $STACKNAME \ + --template-body ./test/batch.yml \ + --wait \ + --params CIRoleArn=arn:aws:iam::${ACCOUNT_ID}:role/GitLabCIRunnerRole + "0.6 (Mac)": tags: @@ -76,19 +174,62 @@ variables: "Online Tests": - stage: online-test + stage: test tags: - - batch + - amzn2 - ecr - - linux - - shell-ci + - docker + - ci-account variables: - AWS_STACKNAME: aws-batch-manager-test ONLINE: "batch" # Runs the online tests against AWS script: + - *runtime_variables + - *awscli_install + - *cloudspy_install + - *assume_test_profile + # Execute online tests - source julia-ci export - - export PATH="$PATH:/usr/local/bin" - - unset SSL_CERT_DIR # https://github.com/JuliaLang/julia/issues/20439 - - ./julia-ci test + - AWS_STACKNAME=$STACKNAME ./julia-ci test - ./julia-ci coverage - extends: .test_shell_0_6 + extends: .test_shell_1_0 + + +.delete: &delete + tags: + - docker-ci + - ci-account + image: $AMAZON_LINUX_IMAGE + before_script: + - *runtime_variables + - *awscli_install + script: + - eval $(aws-stack-outputs $STACKNAME) + - | + aws cloudformation delete-stack \ + --role-arn arn:aws:iam::${ACCOUNT_ID}:role/CloudFormationAdmin \ + --stack-name $STACKNAME + - aws cloudformation wait stack-delete-complete --stack-name $STACKNAME + +"Delete Environment": + stage: teardown + except: + - tags + - master + - /^.+\/.*master$/ + when: manual + environment: + name: branch/$CI_COMMIT_REF_SLUG + action: stop + dependencies: + - "Create Stack" + variables: + GIT_STRATEGY: none # Avoid checking out a branch after deletion + <<: *delete + +"Delete Stack": + stage: teardown + only: + - master + - /^.+\/.*master$/ + when: always + <<: *delete diff --git a/REQUIRE b/REQUIRE index bb70ca8..02dd36b 100644 --- a/REQUIRE +++ b/REQUIRE @@ -1,6 +1,6 @@ julia 0.6 AutoHashEquals 0.2 -AWSCore 0.3.0 +AWSCore 0.3.0 # Online tests require 0.5.2 but that version is Julia 1.0+ AWSSDK 0.2.0 Compat 0.69.0 Mocking 0.3.3 diff --git a/test/batch.yml b/test/batch.yml new file mode 100644 index 0000000..f012887 --- /dev/null +++ b/test/batch.yml @@ -0,0 +1,246 @@ +# https://gitlab.invenia.ca/invenia/AWSBatch.jl/blob/master/test/batch.yml +# +# Creates a bare bones AWS Batch environment used to test the AWSBatch.jl. +# +# ``` +# aws cloudformation create-stack \ +# --stack-name aws-batch-test \ +# --template-body file://test/batch.yml \ +# --capabilities CAPABILITY_NAMED_IAM +# ``` +# +# If you are planning on destroying your stack after you have finished running tests you +# can use the following parameters to make the tests avoid delays with scaling the compute +# environments by being slightly more expensive: +# +# ``` +# aws cloudformation create-stack \ +# --stack-name aws-batch-test-fast \ +# --template-body file://test/batch.yml \ +# --capabilities CAPABILITY_NAMED_IAM \ +# --parameters \ +# ParameterKey=MinVCPUs,ParameterValue=16 +# ``` + +AWSTemplateFormatVersion: 2010-09-09 +Description: >- + A bare bones AWS Batch environment used to test the AWSBatch.jl. + +# https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/parameters-section-structure.html#aws-specific-parameter-types +Parameters: + VPCCidrBlock: + # https://docs.aws.amazon.com/AmazonVPC/latest/UserGuide/VPC_Subnets.html#VPC_Sizing + Description: >- + The IP address range used for batch instances in the new VPC. + Type: String + Default: 10.0.0.0/16 + AllowedPattern: "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})" + MinVCPUs: + Description: >- + The minimum number of VCPUs to be available. Setting to 1 or higher makes jobs start + faster but will cost us more when instances are idle. Note you cannot decrease the + minimum number of VCPUs with a stack update. + Type: Number + Default: 0 + MaxVCPUs: + Description: >- + The maximum number of VCPUs. Typically this number does not need to be touched + Type: Number + Default: 16 + ProvisioningModel: + Description: Spot instances are cheaper than on-demand but can be abruptly terminated + Type: String + Default: spot + AllowedValues: + - on-demand + - spot + CIRoleArn: + Description: The role ARN used when executing GitLab CI test stage jobs. + Type: String + Default: "" + AllowedPattern: "|arn:aws:iam::\\d{12}:role/[^/]+" + +Conditions: + OnDemandComputeEnvironment: !Equals [!Ref ProvisioningModel, on-demand] + Testing: !Not [!Equals [!Ref CIRoleArn, ""]] + +Resources: + ComputeEnvironment: + Type: AWS::Batch::ComputeEnvironment + DependsOn: BatchServiceRole # Removing the ServiceRole before deleting the ComputeEnvironment will cause issues + Properties: + Type: MANAGED + ComputeEnvironmentName: !Ref AWS::StackName + ComputeResources: + Type: !If [OnDemandComputeEnvironment, EC2, SPOT] + BidPercentage: 100 + MinvCpus: !Ref MinVCPUs + MaxvCpus: !Ref MaxVCPUs + InstanceTypes: + - optimal + Subnets: + - !Ref Subnet + SecurityGroupIds: + - !Ref SecurityGroup + InstanceRole: !Ref IamInstanceProfile + SpotIamFleetRole: !Ref BatchSpotFleetRole + Tags: + Name: !Sub "AWS Batch (${AWS::StackName})" + ServiceRole: !Ref BatchServiceRole + JobQueue: + Type: AWS::Batch::JobQueue + Properties: + JobQueueName: !Sub ${AWS::StackName} + Priority: 1 + ComputeEnvironmentOrder: + - Order: 1 + ComputeEnvironment: !Ref ComputeEnvironment + + IamInstanceProfile: + Type: AWS::IAM::InstanceProfile + Properties: + Roles: + - !Ref EcsInstanceRole + EcsInstanceRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: 2008-10-17 + Statement: + - Sid: '' + Effect: Allow + Principal: + Service: ec2.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role + BatchServiceRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: batch.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AWSBatchServiceRole + + # http://docs.aws.amazon.com/batch/latest/userguide/spot_fleet_IAM_role.html + BatchSpotFleetRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: spotfleet.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AmazonEC2SpotFleetTaggingRole + + SecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: EC2 Security Group for instances launched in the VPC by Batch + VpcId: !Ref VPC + VPC: + Type: AWS::EC2::VPC + Properties: + CidrBlock: !Ref VPCCidrBlock + Subnet: + Type: AWS::EC2::Subnet + Properties: + CidrBlock: !Ref VPCCidrBlock + VpcId: !Ref VPC + MapPublicIpOnLaunch: True + RouteTable: + Type: AWS::EC2::RouteTable + Properties: + VpcId: !Ref VPC + SubnetRouteTableAssociation: + Type: AWS::EC2::SubnetRouteTableAssociation + Properties: + RouteTableId: !Ref RouteTable + SubnetId: !Ref Subnet + Route: + Type: AWS::EC2::Route + DependsOn: VPCGatewayAttachment + Properties: + RouteTableId: !Ref RouteTable + DestinationCidrBlock: 0.0.0.0/0 + GatewayId: !Ref InternetGateway + InternetGateway: + Type: AWS::EC2::InternetGateway + VPCGatewayAttachment: + Type: AWS::EC2::VPCGatewayAttachment + Properties: + VpcId: !Ref VPC + InternetGatewayId: !Ref InternetGateway + + #============================================# + + JobRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com # Note: Shouldn't be batch.amazonaws.com + Action: sts:AssumeRole + + TestRole: + Type: AWS::IAM::Role + Condition: Testing + Properties: + RoleName: !Sub ${AWS::StackName}-TestRole + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + AWS: !Ref CIRoleArn + Action: sts:AssumeRole + + # Necessary permissions for running the AWSBatch online tests. + TestPolicy: + Type: AWS::IAM::Policy + Condition: Testing + Properties: + PolicyName: TestPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + # Permissions used by AWSBatch.jl + - Effect: Allow + Action: + - batch:RegisterJobDefinition + - batch:DescribeJobDefinitions + - batch:DeregisterJobDefinition + - batch:SubmitJob + - batch:DescribeJobs + - batch:DescribeJobQueues + - batch:DescribeComputeEnvironments + Resource: "*" + - Effect: Allow + Action: logs:GetLogEvents + Resource: "*" + - Effect: Allow + Action: iam:PassRole + Resource: !GetAtt JobRole.Arn + + # Permissions for AWSTools.stack_output + - Effect: Allow + Action: cloudformation:DescribeStacks + Resource: !Sub arn:aws:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/${AWS::StackName}/* + Roles: [!Ref TestRole] + +Outputs: + JobQueueArn: + Value: !Ref JobQueue + JobRoleArn: + Value: !GetAtt JobRole.Arn diff --git a/test/runtests.jl b/test/runtests.jl index 00b5d79..1c56649 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -43,7 +43,7 @@ include("mock.jl") job = run_batch(; name = "aws-batch-test", definition = "aws-batch-test", - queue = STACK["ManagerJobQueueArn"], + queue = STACK["JobQueueArn"], image = JULIA_BAKED_IMAGE, vcpus = 1, memory = 1024, @@ -62,7 +62,7 @@ include("mock.jl") # Test job details were set correctly job_details = describe(job) @test job_details["jobName"] == "aws-batch-test" - @test occursin(STACK["ManagerJobQueueArn"], job_details["jobQueue"]) + @test occursin(STACK["JobQueueArn"], job_details["jobQueue"]) @test job_details["parameters"] == Dict("region" => "us-east-1") # Test job definition and container parameters were set correctly @@ -90,7 +90,7 @@ include("mock.jl") job = run_batch(; name = "aws-batch-test", definition = "aws-batch-test", - queue = STACK["ManagerJobQueueArn"], + queue = STACK["JobQueueArn"], image = JULIA_BAKED_IMAGE, vcpus = 1, memory = 1024, @@ -112,7 +112,7 @@ include("mock.jl") @testset "Job registration disallowed" begin @test_throws BatchEnvironmentError run_batch(; name = "aws-batch-no-job-registration-test", - queue = STACK["ManagerJobQueueArn"], + queue = STACK["JobQueueArn"], image = JULIA_BAKED_IMAGE, role = STACK["JobRoleArn"], cmd = `julia -e 'println("Hello World!")'`, @@ -141,7 +141,7 @@ include("mock.jl") job = run_batch(; name = "aws-batch-parameters-test", definition = job_definition, - queue = STACK["ManagerJobQueueArn"], + queue = STACK["JobQueueArn"], image = JULIA_BAKED_IMAGE, vcpus = 1, memory = 1024, @@ -181,7 +181,7 @@ include("mock.jl") job = run_batch(; name = "aws-batch-array-job-test", definition = "aws-batch-array-job-test", - queue = STACK["ManagerJobQueueArn"], + queue = STACK["JobQueueArn"], image = JULIA_BAKED_IMAGE, vcpus = 1, memory = 1024, @@ -226,7 +226,7 @@ include("mock.jl") job = run_batch(; name = "aws-batch-timeout-job-test", definition = "aws-bath-timeout-job-test", - queue = STACK["ManagerJobQueueArn"], + queue = STACK["JobQueueArn"], image = JULIA_BAKED_IMAGE, vcpus = 1, memory = 1024, @@ -255,7 +255,7 @@ include("mock.jl") job = run_batch(; name = "aws-batch-failed-job-test", definition = "aws-batch-failed-job-test", - queue = STACK["ManagerJobQueueArn"], + queue = STACK["JobQueueArn"], image = JULIA_BAKED_IMAGE, vcpus = 1, memory = 1024, From d0418f62b5f9f605a94cc386c8958c37721c1d3a Mon Sep 17 00:00:00 2001 From: morris25 Date: Wed, 16 Jan 2019 12:39:30 -0600 Subject: [PATCH 053/110] Uses aws_config to get credentials --- src/AWSBatch.jl | 2 +- src/batch_job.jl | 2 +- src/job_definition.jl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/AWSBatch.jl b/src/AWSBatch.jl index 886337c..b0ad4a5 100644 --- a/src/AWSBatch.jl +++ b/src/AWSBatch.jl @@ -8,7 +8,7 @@ using Memento using Mocking using Compat.Dates -using AWSCore: AWSConfig, AWSCredentials +using AWSCore: aws_config using Compat: Nothing, AbstractDict, @__MODULE__, undef, devnull using DataStructures: OrderedDict diff --git a/src/batch_job.jl b/src/batch_job.jl index 8a3ee73..dad0ebb 100644 --- a/src/batch_job.jl +++ b/src/batch_job.jl @@ -51,7 +51,7 @@ function submit( num_jobs::Integer=1, ) region = isempty(region) ? "us-east-1" : region - config = AWSConfig(:creds => AWSCredentials(), :region => region) + config = aws_config(region = region) debug(logger, "Submitting job \"$name\"") input = [ diff --git a/src/job_definition.jl b/src/job_definition.jl index ee9c75b..616c29d 100644 --- a/src/job_definition.jl +++ b/src/job_definition.jl @@ -104,7 +104,7 @@ function register( parameters::Dict{String, String}=Dict{String, String}(), ) region = isempty(region) ? "us-east-1" : region - config = AWSConfig(:creds => AWSCredentials(), :region => region) + config = aws_config(region = region) debug(logger, "Registering job definition \"$definition_name\"") input = [ From 02ff4460f509fd1b4df1c5c85d00f999e47442ca Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 23 Jan 2019 11:40:31 -0600 Subject: [PATCH 054/110] Drop Julia 0.6 and add Julia 1.1 --- .gitlab-ci.yml | 36 ++++++++++++++++-------------------- REQUIRE | 2 +- 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 28f7ff9..17fd182 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,19 +1,16 @@ stages: - setup - test - - coverage - - documentation - teardown include: - - https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/gitlab-templates/test_templates.yml - - https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/gitlab-templates/coverage.yml - - https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/gitlab-templates/documentation.yml + - https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/templates/hidden-jobs.yml + - https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/templates/teardown.yml variables: STACKNAME_PREFIX: sandbox-awsbatch AMAZON_LINUX_IMAGE: amazonlinux:2 - JULIA_IMAGE: 468665244580.dkr.ecr.us-east-1.amazonaws.com/julia-gitlab-ci:0.6 + JULIA_IMAGE: 468665244580.dkr.ecr.us-east-1.amazonaws.com/julia-gitlab-ci:1.0 ONLINE: "" # Variables that require shell execution or depend on a variable that does @@ -111,46 +108,45 @@ variables: --wait \ --params CIRoleArn=arn:aws:iam::${ACCOUNT_ID}:role/GitLabCIRunnerRole - -"0.6 (Mac)": +"1.0 (Mac)": tags: - mac - shell-ci - extends: .test_shell_0_6 + extends: .test_shell_1_0 -"0.6 (Linux, 64-bit)": +"1.0 (Linux, 64-bit)": tags: - linux - 64-bit - - docker-ci - extends: .test_docker + - shell-ci + extends: .test_shell_1_0 -"0.6 (Linux, 32-bit)": +"1.0 (Linux, 32-bit)": tags: - linux - 32-bit - shell-ci - extends: .test_shell_0_6 + extends: .test_shell_1_0 -"1.0 (Mac)": +"1.1 (Mac)": tags: - mac - shell-ci - extends: .test_shell_1_0 + extends: .test_shell_1_1 -"1.0 (Linux, 64-bit)": +"1.1 (Linux, 64-bit)": tags: - linux - 64-bit - shell-ci - extends: .test_shell_1_0 + extends: .test_shell_1_1 -"1.0 (Linux, 32-bit)": +"1.1 (Linux, 32-bit)": tags: - linux - 32-bit - shell-ci - extends: .test_shell_1_0 + extends: .test_shell_1_1 "Nightly (Mac)": tags: diff --git a/REQUIRE b/REQUIRE index 02dd36b..f714ed3 100644 --- a/REQUIRE +++ b/REQUIRE @@ -1,4 +1,4 @@ -julia 0.6 +julia 0.7 AutoHashEquals 0.2 AWSCore 0.3.0 # Online tests require 0.5.2 but that version is Julia 1.0+ AWSSDK 0.2.0 From 9d5c65eec199489b376c6669d6a307ced321fc8b Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 23 Jan 2019 11:42:01 -0600 Subject: [PATCH 055/110] Drop deprecations --- src/deprecated.jl | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/src/deprecated.jl b/src/deprecated.jl index cb13a33..6d5b39d 100644 --- a/src/deprecated.jl +++ b/src/deprecated.jl @@ -1,27 +1 @@ using Base: @deprecate - -@deprecate logs(job::BatchJob) [Dict("eventId" => e.id, "ingestionTime" => e.ingestion_time, "timestamp" => e.timestamp, "message" => e.message) for e in log_events(job)] -@deprecate BatchStatus JobState - -@deprecate submit!(job::BatchJob) error("`submit!(job::BatchJob)` is no longer supported. Look at `submit` or `run_batch` to submit batch jobs") -@deprecate job_definition_arn(job::BatchJob) job_definition_arn(JobDefinition(job)) - -# Deprecate methods that now are called explicitly on JobDefinition's and not on BatchJob's -@deprecate isregistered(job::BatchJob) isregistered(JobDefinition(job)) -@deprecate register!(job::BatchJob) error("`register`!(job::BatchJob)` is no longer supported. Look at `register` to register job definitions") -@deprecate deregister!(job::BatchJob) error("`deregister`!(job::BatchJob)` is no longer supported. Look at `deregister` to de-register job definitions") - -@deprecate register(job_definition::JobDefinition) register(job_definition.arn) - -function BatchJob(; id="", kwargs...) - if !isempty(id) && isempty(kwargs) - Base.depwarn("BatchJob(; id=id)` is deprecated, use `BatchJob(id)` instead", :BatchJob) - BatchJob(id) - elseif isempty(id) && !isempty(kwargs) - Base.depwarn("`BatchJob(; kwargs...)` is deprecated, use `run_batch(; kwargs...)` instead", :BatchJob) - run_batch(; kwargs...) - else - Base.depwarn("`BatchJob(; id=id, kwargs...)` is deprecated, ignoring `id` and using `run_batch(; kwargs...)` instead", :BatchJob) - run_batch(; kwargs...) - end -end From 76c046cbb0bd5fe31252400701acaa5cb13f6106 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 23 Jan 2019 11:43:04 -0600 Subject: [PATCH 056/110] Drop Compat --- REQUIRE | 1 - src/AWSBatch.jl | 8 +++----- src/batch_job.jl | 4 ++-- test/runtests.jl | 6 ++---- 4 files changed, 7 insertions(+), 12 deletions(-) diff --git a/REQUIRE b/REQUIRE index f714ed3..3d9a7cb 100644 --- a/REQUIRE +++ b/REQUIRE @@ -2,7 +2,6 @@ julia 0.7 AutoHashEquals 0.2 AWSCore 0.3.0 # Online tests require 0.5.2 but that version is Julia 1.0+ AWSSDK 0.2.0 -Compat 0.69.0 Mocking 0.3.3 Memento 0.7 DataStructures 0.2.9 diff --git a/src/AWSBatch.jl b/src/AWSBatch.jl index b0ad4a5..bfa0fd4 100644 --- a/src/AWSBatch.jl +++ b/src/AWSBatch.jl @@ -2,15 +2,13 @@ __precompile__() module AWSBatch using AutoHashEquals +using AWSCore: aws_config using AWSSDK.Batch using AWSSDK.CloudWatchLogs +using DataStructures: OrderedDict +using Dates using Memento using Mocking -using Compat.Dates - -using AWSCore: aws_config -using Compat: Nothing, AbstractDict, @__MODULE__, undef, devnull -using DataStructures: OrderedDict export BatchJob, diff --git a/src/batch_job.jl b/src/batch_job.jl index dad0ebb..ea3fee9 100644 --- a/src/batch_job.jl +++ b/src/batch_job.jl @@ -1,5 +1,5 @@ -import AWSSDK.Batch: describe_jobs, submit_job -import AWSSDK.CloudWatchLogs: get_log_events +using AWSSDK.Batch: describe_jobs, submit_job +using AWSSDK.CloudWatchLogs: get_log_events struct BatchJobError <: Exception job_id::AbstractString diff --git a/test/runtests.jl b/test/runtests.jl index 1c56649..9f185f2 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -4,11 +4,9 @@ Mocking.enable(force=true) using AWSBatch using AWSCore: AWSConfig using AWSTools.CloudFormation: stack_output - -using Compat: occursin -using Compat.Test -using Compat.Dates +using Dates using Memento +using Test # Enables the running of the "batch" online tests. e.g ONLINE=batch From bd45401311c412af43a29884b972b2af71a5d7bb Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 23 Jan 2019 12:20:48 -0600 Subject: [PATCH 057/110] Add Project.toml and drop Julia 0.7 --- Project.toml | 29 +++++++++++++++++++++++++++++ REQUIRE | 4 ++-- test/REQUIRE | 1 - 3 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 Project.toml delete mode 100644 test/REQUIRE diff --git a/Project.toml b/Project.toml new file mode 100644 index 0000000..e3a0dd6 --- /dev/null +++ b/Project.toml @@ -0,0 +1,29 @@ +name = "AWSBatch" +uuid = "dcae83d4-2881-5875-9d49-e5534165e9c0" +authors = ["Nicole Epp ", "Curtis Vogt Date: Wed, 23 Jan 2019 12:59:05 -0600 Subject: [PATCH 058/110] Replace DataStructures with OrderedCollections --- Project.toml | 3 +-- REQUIRE | 2 +- src/AWSBatch.jl | 2 +- test/compute_environment.jl | 2 +- test/job_queue.jl | 2 +- test/mock.jl | 2 +- 6 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Project.toml b/Project.toml index e3a0dd6..6e0dd50 100644 --- a/Project.toml +++ b/Project.toml @@ -6,17 +6,16 @@ authors = ["Nicole Epp ", "Curtis Vogt "24fa2d7a-64c4-49d2-8b47-f8da4fbde8e9", From 5bbe02053321e802bb23447695a5bba4dd21e7ea Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Thu, 24 Jan 2019 09:03:15 -0600 Subject: [PATCH 059/110] Allow running online tests on tags --- .gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 17fd182..585bd42 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -87,8 +87,6 @@ variables: "Create Stack": stage: setup - except: - - tags image: $AMAZON_LINUX_IMAGE tags: - docker-ci @@ -225,6 +223,7 @@ variables: "Delete Stack": stage: teardown only: + - tags - master - /^.+\/.*master$/ when: always From bc8f5ce79ea73771a7675f158b4d4036ddd38b91 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Thu, 24 Jan 2019 09:15:14 -0600 Subject: [PATCH 060/110] Use cloudspy Docker image for faster CI --- .gitlab-ci.yml | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 585bd42..e2dafed 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,7 +9,7 @@ include: variables: STACKNAME_PREFIX: sandbox-awsbatch - AMAZON_LINUX_IMAGE: amazonlinux:2 + CLOUDSPY_IMAGE: 468665244580.dkr.ecr.us-east-1.amazonaws.com/cloudspy JULIA_IMAGE: 468665244580.dkr.ecr.us-east-1.amazonaws.com/julia-gitlab-ci:1.0 ONLINE: "" @@ -28,6 +28,9 @@ variables: ACCOUNT_ID=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | grep -oP '(?<="accountId" : ")[^"]*(?=")') export AWS_DEFAULT_REGION=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | grep -oP '(?<="region" : ")[^"]*(?=")') + # Export temp credentials. Make sure this path is absolute as Julia can change directories during testing + export AWS_SHARED_CREDENTIALS_FILE="$(pwd)/tmp-creds" + .awscli_install: &awscli_install | # Build cloudspy and its dependencies @@ -60,8 +63,6 @@ variables: | # Install Cloudspy in the env pip install git+https://gitlab-ci-token:${CI_BUILD_TOKEN}@gitlab.invenia.ca/infrastructure/cloudspy.git#egg=cloudspy - # Export temp credentials. Make sure this path is absolute as Julia can change directories during testing - export AWS_SHARED_CREDENTIALS_FILE="$(pwd)/tmp-creds" .assume_test_profile: &assume_test_profile | @@ -87,15 +88,12 @@ variables: "Create Stack": stage: setup - image: $AMAZON_LINUX_IMAGE + image: $CLOUDSPY_IMAGE tags: - docker-ci - ci-account before_script: - *runtime_variables - - yum -y update - - *awscli_install - - *cloudspy_install script: - aws cloudformation validate-template --template-body file://test/batch.yml - | @@ -192,10 +190,9 @@ variables: tags: - docker-ci - ci-account - image: $AMAZON_LINUX_IMAGE + image: $CLOUDSPY_IMAGE before_script: - *runtime_variables - - *awscli_install script: - eval $(aws-stack-outputs $STACKNAME) - | From 2e8e29156085bb79df35a21d110bf90079bb986f Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Thu, 24 Jan 2019 11:33:20 -0600 Subject: [PATCH 061/110] Run online tests on different versions of Julia --- .gitlab-ci.yml | 20 ++++++++++++++++---- README.md | 8 +++----- test/runtests.jl | 28 ++++++++++++++++------------ 3 files changed, 35 insertions(+), 21 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e2dafed..d99c6c3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,7 +11,6 @@ variables: STACKNAME_PREFIX: sandbox-awsbatch CLOUDSPY_IMAGE: 468665244580.dkr.ecr.us-east-1.amazonaws.com/cloudspy JULIA_IMAGE: 468665244580.dkr.ecr.us-east-1.amazonaws.com/julia-gitlab-ci:1.0 - ONLINE: "" # Variables that require shell execution or depend on a variable that does .runtime_variables: &runtime_variables @@ -165,7 +164,7 @@ variables: extends: .test_shell_nightly -"Online Tests": +.test_batch: stage: test tags: - amzn2 @@ -173,7 +172,7 @@ variables: - docker - ci-account variables: - ONLINE: "batch" # Runs the online tests against AWS + TESTS: "batch" # Runs the online tests against AWS script: - *runtime_variables - *awscli_install @@ -183,7 +182,20 @@ variables: - source julia-ci export - AWS_STACKNAME=$STACKNAME ./julia-ci test - ./julia-ci coverage - extends: .test_shell_1_0 + extends: .test_shell + +"1.0 (AWS Batch)": + variables: + JULIA_VERSION: "1.0" + extends: .test_batch + +"1.1 (AWS Batch)": + variables: + JULIA_VERSION: "1.1" + extends: .test_batch + +# Note: We cannot test AWS Batch on nightly as we don't build "julia-baked" images with +# the nightly Julia revision. .delete: &delete diff --git a/README.md b/README.md index cbff91d..5cd0d1c 100644 --- a/README.md +++ b/README.md @@ -6,14 +6,12 @@ # Running the tests -To run the ONLINE batch tests you must first set the environmental variables `ONLINE` and +To run the online AWS Batch tests you must first set the environmental variables `TESTS` and `AWS_STACKNAME`. ```julia -ENV["ONLINE"] = "batch" - +ENV["TESTS"] = "batch" ENV["AWS_STACKNAME"] = "aws-batch-manager-test" ``` -To make an `aws-batch-manager-test` compatible stack you can use the AWSClusterManagers.jl -CloudFormation template [test/batch.yml](https://gitlab.invenia.ca/invenia/AWSClusterManagers.jl/blob/master/test/batch.yml). +To make an `aws-batch-manager-test` compatible stack you can use the CloudFormation template [test/batch.yml](./test/batch.yml). diff --git a/test/runtests.jl b/test/runtests.jl index 9f185f2..0f1e5c6 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -9,14 +9,14 @@ using Memento using Test -# Enables the running of the "batch" online tests. e.g ONLINE=batch -const ONLINE = strip.(split(get(ENV, "ONLINE", ""), r"\s*,\s*")) +# Controls the running of various tests: "local", "batch" +const TESTS = strip.(split(get(ENV, "TESTS", "local"), r"\s*,\s*")) # Run the tests on a stack created with the "test/batch.yml" CloudFormation template # found in AWSClusterMangers.jl const AWS_STACKNAME = get(ENV, "AWS_STACKNAME", "") const STACK = !isempty(AWS_STACKNAME) ? stack_output(AWS_STACKNAME) : Dict() -const JULIA_BAKED_IMAGE = "468665244580.dkr.ecr.us-east-1.amazonaws.com/julia-baked:0.6" +const JULIA_BAKED_IMAGE = "468665244580.dkr.ecr.us-east-1.amazonaws.com/julia-baked:$VERSION" const JOB_TIMEOUT = 900 Memento.config!("debug"; fmt="[{level} | {name}]: {msg}") @@ -27,15 +27,19 @@ include("mock.jl") @testset "AWSBatch.jl" begin - include("compute_environment.jl") - include("job_queue.jl") - include("log_event.jl") - include("job_state.jl") - include("run_batch.jl") + if "local" in TESTS + include("compute_environment.jl") + include("job_queue.jl") + include("log_event.jl") + include("job_state.jl") + include("run_batch.jl") + else + warn(logger, "Skipping \"local\" tests. Set `ENV[\"TESTS\"] = \"local\"` to run.") + end - if "batch" in ONLINE && !isempty(AWS_STACKNAME) - @testset "Online" begin - info(logger, "Running ONLINE tests") + if "batch" in TESTS && !isempty(AWS_STACKNAME) + @testset "AWS Batch" begin + info(logger, "Running AWS Batch tests") @testset "Job Submission" begin job = run_batch(; @@ -283,7 +287,7 @@ include("mock.jl") else warn( logger, - "Skipping ONLINE tests. Set `ENV[\"ONLINE\"] = \"batch\"` and " * + "Skipping \"batch\" tests. Set `ENV[\"TESTS\"] = \"batch\"` and " * "`ENV[\"AWS_STACKNAME\"]` to run." ) end From fdcc8ba858e83c45330268ef253f72178c18d873 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Fri, 25 Jan 2019 10:42:31 -0600 Subject: [PATCH 062/110] Address concurrency issue with job submission test --- test/runtests.jl | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 0f1e5c6..f67ffa8 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -42,9 +42,18 @@ include("mock.jl") info(logger, "Running AWS Batch tests") @testset "Job Submission" begin + definition = "aws-batch-test" + + # Append the job ID to the definition when running on the CI. Doing this + # will allow this test to successfully reuse the job definition later when + # concurrent CI jobs are running AWS Batch tests at the same time. + if haskey(ENV, "CI_JOB_ID") + definition *= "-" * ENV["CI_JOB_ID"] + end + job = run_batch(; name = "aws-batch-test", - definition = "aws-batch-test", + definition = definition, queue = STACK["JobQueueArn"], image = JULIA_BAKED_IMAGE, vcpus = 1, @@ -73,7 +82,7 @@ include("mock.jl") job_definition_details = first(describe(job_definition)["jobDefinitions"]) - @test job_definition_details["jobDefinitionName"] == "aws-batch-test" + @test job_definition_details["jobDefinitionName"] == definition @test job_definition_details["status"] == "ACTIVE" @test job_definition_details["type"] == "container" @@ -91,7 +100,7 @@ include("mock.jl") # Reuse job definition job = run_batch(; name = "aws-batch-test", - definition = "aws-batch-test", + definition = definition, queue = STACK["JobQueueArn"], image = JULIA_BAKED_IMAGE, vcpus = 1, From 4a5e430e97ad53868b6b9ec04adb3c57d365deca Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Fri, 8 Feb 2019 10:33:52 -0600 Subject: [PATCH 063/110] Add dependency for VPCGatewayAttachment https://gitlab.invenia.ca/invenia/AWSBatch.jl/issues/15 --- test/batch.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/test/batch.yml b/test/batch.yml index f012887..d475979 100644 --- a/test/batch.yml +++ b/test/batch.yml @@ -176,6 +176,7 @@ Resources: Type: AWS::EC2::InternetGateway VPCGatewayAttachment: Type: AWS::EC2::VPCGatewayAttachment + DependsOn: Subnet # Detaching the gateway can fail if the VPC still contains public addresses Properties: VpcId: !Ref VPC InternetGatewayId: !Ref InternetGateway From 2a15dff551cff9807f85776bb9aeeb52764bdd33 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Thu, 14 Feb 2019 10:50:03 -0600 Subject: [PATCH 064/110] Trial new Docker CI setup --- .gitlab-ci.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d99c6c3..f2f43df 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,7 +10,6 @@ include: variables: STACKNAME_PREFIX: sandbox-awsbatch CLOUDSPY_IMAGE: 468665244580.dkr.ecr.us-east-1.amazonaws.com/cloudspy - JULIA_IMAGE: 468665244580.dkr.ecr.us-east-1.amazonaws.com/julia-gitlab-ci:1.0 # Variables that require shell execution or depend on a variable that does .runtime_variables: &runtime_variables @@ -113,8 +112,8 @@ variables: tags: - linux - 64-bit - - shell-ci - extends: .test_shell_1_0 + - docker-ci + extends: .test_docker_1_0 "1.0 (Linux, 32-bit)": tags: @@ -133,8 +132,8 @@ variables: tags: - linux - 64-bit - - shell-ci - extends: .test_shell_1_1 + - docker-ci + extends: .test_docker_1_1 "1.1 (Linux, 32-bit)": tags: @@ -153,8 +152,8 @@ variables: tags: - linux - 64-bit - - shell-ci - extends: .test_shell_nightly + - docker-ci + extends: .test_docker_nightly "Nightly (Linux, 32-bit)": tags: From f3615b81b63723b743d178d44389f31d173a0777 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Thu, 14 Feb 2019 10:58:44 -0600 Subject: [PATCH 065/110] Avoid using deprecated runner tags --- .gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f2f43df..8f5fe3e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -167,8 +167,7 @@ variables: stage: test tags: - amzn2 - - ecr - - docker + - docker-build - ci-account variables: TESTS: "batch" # Runs the online tests against AWS From 4030805b7a6050b3273698534461a374662ea57f Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Wed, 6 Mar 2019 21:40:42 +0000 Subject: [PATCH 066/110] Change default memory for 'run_batch' to an obvious default value --- src/AWSBatch.jl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/AWSBatch.jl b/src/AWSBatch.jl index bf4e425..21f95b5 100644 --- a/src/AWSBatch.jl +++ b/src/AWSBatch.jl @@ -60,7 +60,7 @@ end definition::Union{AbstractString, JobDefinition, Nothing}=nothing, image::AbstractString="", vcpus::Integer=1, - memory::Integer=1024, + memory::Integer=-1, role::AbstractString="", cmd::Cmd=``, num_jobs::Integer=1, @@ -87,7 +87,7 @@ function run_batch(; definition::Union{AbstractString, JobDefinition, Nothing}=nothing, image::AbstractString="", vcpus::Integer=1, - memory::Integer=1024, + memory::Integer=-1, role::AbstractString="", cmd::Cmd=``, num_jobs::Integer=1, @@ -110,7 +110,7 @@ function run_batch(; # Update container override parameters vcpus == 1 && (vcpus = container["vcpus"]) - (memory == 1024 || memory < 1) && (memory = container["memory"]) + memory < 0 && (memory = container["memory"]) isempty(cmd) && (cmd = Cmd(Vector{String}(container["command"]))) end end @@ -152,7 +152,7 @@ function run_batch(; # Update container overrides vcpus == 1 && (vcpus = container["vcpus"]) - (memory == 1024 || memory < 1) && (memory = container["memory"]) + memory < 0 && (memory = container["memory"]) isempty(cmd) && (cmd = Cmd(Vector{String}(container["command"]))) else warn(logger, "No jobs found with id: $job_id.") @@ -160,7 +160,7 @@ function run_batch(; end # Error if required parameters were not explicitly set and cannot be inferred - if isempty(name) || isempty(queue) || memory < 1 + if isempty(name) || isempty(queue) || memory < 0 throw(BatchEnvironmentError( "Unable to perform AWS Batch introspection when not running within " * "an AWS Batch job. Current job parameters are: " * From 8f706958351886a0838f29e0dcf9e8f8f7904d0c Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Wed, 19 Jun 2019 16:38:40 -0500 Subject: [PATCH 067/110] Add Project.toml for docs --- .gitignore | 1 + docs/Manifest.toml | 229 +++++++++++++++++++++++++++++++++++++++++++++ docs/Project.toml | 6 ++ docs/make.jl | 14 +-- 4 files changed, 244 insertions(+), 6 deletions(-) create mode 100644 docs/Manifest.toml create mode 100644 docs/Project.toml diff --git a/.gitignore b/.gitignore index 7be7ea3..07c5a08 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ *.jl.mem /docs/build/ /docs/site/ +/Manifest.toml diff --git a/docs/Manifest.toml b/docs/Manifest.toml new file mode 100644 index 0000000..2251b0e --- /dev/null +++ b/docs/Manifest.toml @@ -0,0 +1,229 @@ +[[AWSBatch]] +deps = ["AWSCore", "AWSSDK", "AutoHashEquals", "Dates", "Memento", "Mocking", "OrderedCollections", "Test"] +git-tree-sha1 = "46b75dea75b19d0025cfed7ff859a906bede2eb6" +uuid = "dcae83d4-2881-5875-9d49-e5534165e9c0" +version = "1.0.0" + +[[AWSCore]] +deps = ["Base64", "DataStructures", "Dates", "HTTP", "IniFile", "JSON", "LazyJSON", "MbedTLS", "Retry", "Sockets", "SymDict", "Test", "XMLDict"] +git-tree-sha1 = "d9b2ada3bf289c504f765fec863cacea5aeb0541" +uuid = "4f1ea46c-232b-54a6-9b17-cc2d0f3e6598" +version = "0.6.0" + +[[AWSSDK]] +deps = ["AWSCore"] +git-tree-sha1 = "89c95eab2bc1128fe247c31e9774d47eb7ef0d40" +uuid = "0d499d91-6ae5-5d63-9313-12987b87d5ad" +version = "0.4.0" + +[[AutoHashEquals]] +git-tree-sha1 = "45bb6705d93be619b81451bb2006b7ee5d4e4453" +uuid = "15f4f7f2-30c1-5605-9d31-71845cf9641f" +version = "0.2.0" + +[[Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" + +[[BinaryProvider]] +deps = ["Libdl", "SHA"] +git-tree-sha1 = "c7361ce8a2129f20b0e05a89f7070820cfed6648" +uuid = "b99e7846-7c00-51b0-8f62-c81ae34c0232" +version = "0.5.4" + +[[Compat]] +deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"] +git-tree-sha1 = "84aa74986c5b9b898b0d1acaf3258741ee64754f" +uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" +version = "2.1.0" + +[[DataStructures]] +deps = ["InteractiveUtils", "OrderedCollections", "Random", "Serialization", "Test"] +git-tree-sha1 = "ca971f03e146cf144a9e2f2ce59674f5bf0e8038" +uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +version = "0.15.0" + +[[Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[[DelimitedFiles]] +deps = ["Mmap"] +uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" + +[[Distributed]] +deps = ["LinearAlgebra", "Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" + +[[DocStringExtensions]] +deps = ["LibGit2", "Markdown", "Pkg", "Test"] +git-tree-sha1 = "4d30e889c9f106a51ffa4791a88ffd4765bf20c3" +uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" +version = "0.7.0" + +[[Documenter]] +deps = ["Base64", "DocStringExtensions", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "REPL", "Unicode"] +git-tree-sha1 = "38509269fc99a9bc450fdb9e17e805021f3e5b1b" +uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +version = "0.22.4" + +[[EzXML]] +deps = ["BinaryProvider", "Libdl", "Pkg", "Printf", "Test"] +git-tree-sha1 = "ad00b79cca4bb3eabb4209217859c553af4401f5" +uuid = "8f5d6c58-4d21-5cfd-889c-e3ad7ee6a615" +version = "0.9.1" + +[[HTTP]] +deps = ["Base64", "Dates", "IniFile", "MbedTLS", "Sockets"] +git-tree-sha1 = "854fad2a2b9cc6678f70ed15a65fc7e572056d39" +uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" +version = "0.8.2" + +[[IniFile]] +deps = ["Test"] +git-tree-sha1 = "098e4d2c533924c921f9f9847274f2ad89e018b8" +uuid = "83e8ac13-25f8-5344-8a64-a9f2b223428f" +version = "0.5.0" + +[[InteractiveUtils]] +deps = ["LinearAlgebra", "Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" + +[[IterTools]] +deps = ["SparseArrays", "Test"] +git-tree-sha1 = "79246285c43602384e6f1943b3554042a3712056" +uuid = "c8e1da08-722c-5040-9ed9-7db0dc04731e" +version = "1.1.1" + +[[JSON]] +deps = ["Dates", "Distributed", "Mmap", "Sockets", "Test", "Unicode"] +git-tree-sha1 = "1f7a25b53ec67f5e9422f1f551ee216503f4a0fa" +uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +version = "0.20.0" + +[[LazyJSON]] +deps = ["DataStructures", "JSON", "Mmap", "Test"] +git-tree-sha1 = "c02f54c69bf8d7ef39906608d1fc5011d1670e9c" +uuid = "fc18253b-5e1b-504c-a4a2-9ece4944c004" +version = "0.1.1" + +[[LibGit2]] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" + +[[Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[[LinearAlgebra]] +deps = ["Libdl"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + +[[Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" + +[[Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" + +[[MbedTLS]] +deps = ["BinaryProvider", "Dates", "Distributed", "Libdl", "Random", "Sockets", "Test"] +git-tree-sha1 = "2d94286a9c2f52c63a16146bb86fd6cdfbf677c6" +uuid = "739be429-bea8-5141-9913-cc70e7f3736d" +version = "0.6.8" + +[[Memento]] +deps = ["Dates", "Distributed", "JSON", "Serialization", "Sockets", "Syslogs", "Test", "TimeZones", "UUIDs"] +git-tree-sha1 = "090463b13da88689e5eae6468a6f531a21392175" +uuid = "f28f55f0-a522-5efc-85c2-fe41dfb9b2d9" +version = "0.12.1" + +[[Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" + +[[Mocking]] +deps = ["Compat", "Dates"] +git-tree-sha1 = "4bf69aaf823b119b034e091e16b18311aa191663" +uuid = "78c3b35d-d492-501b-9361-3d52fe80e533" +version = "0.5.7" + +[[OrderedCollections]] +deps = ["Random", "Serialization", "Test"] +git-tree-sha1 = "c4c13474d23c60d20a67b217f1d7f22a40edf8f1" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.1.0" + +[[Pkg]] +deps = ["Dates", "LibGit2", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" + +[[Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" + +[[REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + +[[Random]] +deps = ["Serialization"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[Retry]] +deps = ["Test"] +git-tree-sha1 = "56bfdfca33e70883e96fd398548ebd4d405b41fe" +uuid = "20febd7b-183b-5ae2-ac4a-720e7ce64774" +version = "0.4.0" + +[[SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" + +[[Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[SharedArrays]] +deps = ["Distributed", "Mmap", "Random", "Serialization"] +uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383" + +[[Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" + +[[SparseArrays]] +deps = ["LinearAlgebra", "Random"] +uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + +[[Statistics]] +deps = ["LinearAlgebra", "SparseArrays"] +uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" + +[[SymDict]] +deps = ["Test"] +git-tree-sha1 = "0108ccdaea3ef69d9680eeafc8d5ad198b896ec8" +uuid = "2da68c74-98d7-5633-99d6-8493888d7b1e" +version = "0.3.0" + +[[Syslogs]] +deps = ["Printf", "Sockets"] +git-tree-sha1 = "46badfcc7c6e74535cc7d833a91f4ac4f805f86d" +uuid = "cea106d9-e007-5e6c-ad93-58fe2094e9c4" +version = "0.3.0" + +[[Test]] +deps = ["Distributed", "InteractiveUtils", "Logging", "Random"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[[TimeZones]] +deps = ["Dates", "EzXML", "Mocking", "Printf", "Serialization", "Unicode"] +git-tree-sha1 = "859bfc1832ea52e413c96fa5c92130516db62bdb" +uuid = "f269a46b-ccf7-5d73-abea-4c690281aa53" +version = "0.9.1" + +[[UUIDs]] +deps = ["Random"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[[Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" + +[[XMLDict]] +deps = ["DataStructures", "EzXML", "IterTools", "Test"] +git-tree-sha1 = "77a40486f4e5c81c57867d056933022bc4c5fe02" +uuid = "228000da-037f-5747-90a9-8195ccbf91a5" +version = "0.3.0" diff --git a/docs/Project.toml b/docs/Project.toml new file mode 100644 index 0000000..308d7fa --- /dev/null +++ b/docs/Project.toml @@ -0,0 +1,6 @@ +[deps] +AWSBatch = "dcae83d4-2881-5875-9d49-e5534165e9c0" +Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" + +[compat] +Documenter = "~0.22" diff --git a/docs/make.jl b/docs/make.jl index 111ce4b..5889699 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -2,17 +2,19 @@ using Documenter, AWSBatch makedocs(; modules=[AWSBatch], - format=:html, + format=Documenter.HTML( + prettyurls=false, + assets=[ + "assets/invenia.css", + "assets/logo.png", + ], + ), pages=[ "Home" => "index.md", ], repo="https://gitlab.invenia.ca/invenia/AWSBatch.jl/blob/{commit}{path}#L{line}", sitename="AWSBatch.jl", - authors="Nicole Epp", - assets=[ - "assets/invenia.css", - "assets/logo.png", - ], + authors="Invenia Technical Computing", strict = true, checkdocs = :none, ) From 670f60c50e4da93421689fd096c5648575b16420 Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Thu, 20 Jun 2019 12:27:54 -0500 Subject: [PATCH 068/110] Address merge request comments --- docs/Manifest.toml | 6 +++--- docs/Project.toml | 2 +- docs/make.jl | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/Manifest.toml b/docs/Manifest.toml index 2251b0e..45448c8 100644 --- a/docs/Manifest.toml +++ b/docs/Manifest.toml @@ -1,8 +1,8 @@ [[AWSBatch]] -deps = ["AWSCore", "AWSSDK", "AutoHashEquals", "Dates", "Memento", "Mocking", "OrderedCollections", "Test"] -git-tree-sha1 = "46b75dea75b19d0025cfed7ff859a906bede2eb6" +deps = ["AWSCore", "AWSSDK", "AutoHashEquals", "Dates", "Memento", "Mocking", "OrderedCollections"] +path = ".." uuid = "dcae83d4-2881-5875-9d49-e5534165e9c0" -version = "1.0.0" +version = "1.0.0+" [[AWSCore]] deps = ["Base64", "DataStructures", "Dates", "HTTP", "IniFile", "JSON", "LazyJSON", "MbedTLS", "Retry", "Sockets", "SymDict", "Test", "XMLDict"] diff --git a/docs/Project.toml b/docs/Project.toml index 308d7fa..c64f9d2 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -3,4 +3,4 @@ AWSBatch = "dcae83d4-2881-5875-9d49-e5534165e9c0" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" [compat] -Documenter = "~0.22" +Documenter = "~0.22.4" diff --git a/docs/make.jl b/docs/make.jl index 5889699..e5edf99 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -3,7 +3,7 @@ using Documenter, AWSBatch makedocs(; modules=[AWSBatch], format=Documenter.HTML( - prettyurls=false, + prettyurls=get(ENV, "CI", nothing) == "true", assets=[ "assets/invenia.css", "assets/logo.png", From 794e38f12e588dc0719061fc69e56d95429d2620 Mon Sep 17 00:00:00 2001 From: Sam Morrison Date: Tue, 9 Jul 2019 19:24:34 +0000 Subject: [PATCH 069/110] Fix CI python version --- .gitlab-ci.yml | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8f5fe3e..348c699 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,20 +35,13 @@ variables: NEED_SUDO="sudo" # docker-ci has slightly different install parameters - if [[ ${CI_RUNNER_TAGS} == *"docker-ci"* ]]; then - # Install which command - yum -y install which - # Install epel to get python36 rather than python3 - amazon-linux-extras install epel - # We don't need to use sudo on the docker-ci tagged instances - NEED_SUDO="" - fi + [[ $CI_DISPOSABLE_ENVIRONMENT == "true" ]] && SUDO="" || SUDO="sudo" - # Install python36, virtualenv, and necessary packages - ${NEED_SUDO} yum -y install python36 python36-devel python-virtualenv gcc git + # Install python3 and necessary packages + ${SUDO} yum -y install python3 python3-devel gcc git # Install a virtualenv with python 3 and activate it - virtualenv --python=$(which python36) venv + python3 -m venv venv source venv/bin/activate # Install AWSCli From 87d6b42088c859aaae19f491636a1526f026a3ed Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Fri, 19 Jul 2019 14:05:43 -0500 Subject: [PATCH 070/110] Automatically delete unused CI stacks --- test/batch.yml | 88 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/test/batch.yml b/test/batch.yml index d475979..d19f2b6 100644 --- a/test/batch.yml +++ b/test/batch.yml @@ -63,6 +63,7 @@ Parameters: Conditions: OnDemandComputeEnvironment: !Equals [!Ref ProvisioningModel, on-demand] Testing: !Not [!Equals [!Ref CIRoleArn, ""]] + IsCI: !Equals [!Ref "AWS::AccountId", 813030647716] Resources: ComputeEnvironment: @@ -148,6 +149,7 @@ Resources: VpcId: !Ref VPC VPC: Type: AWS::EC2::VPC + DependsOn: DeleteStackRole Properties: CidrBlock: !Ref VPCCidrBlock Subnet: @@ -182,6 +184,92 @@ Resources: InternetGatewayId: !Ref InternetGateway #============================================# +# Automatically delete stacks in the CI account that haven't been updated recently. + + DeleteStackRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Statement: + - Effect: Allow + Principal: + Service: lambda.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole + Policies: + - PolicyName: DeleteStackPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - cloudformation:DescribeStacks + - cloudformation:DeleteStack + Resource: !Ref AWS::StackId + - Effect: Allow + Action: + - logs:* + - ec2:* + - iam:* + - lambda:* + - batch:* + - events:* + Resource: "*" + + DeleteStackLambda: + Type: AWS::Lambda::Function + Properties: + Handler: index.lambda_handler + Runtime: python3.6 + Timeout: 5 + Role: !GetAtt DeleteStackRole.Arn + Environment: + Variables: + STACK_NAME: !Ref AWS::StackName + WAIT_PERIOD: 3 # hours + Code: + ZipFile: | + from datetime import datetime, timedelta + import boto3 + import os + STACK_NAME = os.environ["STACK_NAME"] + WAIT_PERIOD = os.environ["WAIT_PERIOD"] + + def lambda_handler(event, context): + cfn = boto3.resource('cloudformation') + stack = cfn.Stack(STACK_NAME) + wait_period = timedelta(hours=int(WAIT_PERIOD)) + if stack.last_updated_time is None: + last_update = stack.creation_time + else: + last_update = stack.last_updated_time + tzaware_now = datetime.now(last_update.tzinfo) + if tzaware_now >= last_update + wait_period: + print(f"Deleting stack after {WAIT_PERIOD} hours without updates.") + stack.delete() + + DeleteStackPermission: + Condition: IsCI + Type: AWS::Lambda::Permission + Properties: + FunctionName: !Ref DeleteStackLambda + Action: lambda:InvokeFunction + Principal: events.amazonaws.com + SourceArn: !GetAtt DeleteStackRule.Arn + + DeleteStackRule: + Condition: IsCI + Type: AWS::Events::Rule + Properties: + Description: Trigger deletion if stack has not been updated recently. + ScheduleExpression: cron(0 * * * ? *) # Run every hour + State: ENABLED + Targets: + - Id: DeleteStackLambda + Arn: !GetAtt DeleteStackLambda.Arn + + #============================================# JobRole: Type: AWS::IAM::Role From 68cd18ab76182a9b893ed817c7ba335af74a797b Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Fri, 19 Jul 2019 14:29:30 -0500 Subject: [PATCH 071/110] Increase lambda timeout to one minute --- test/batch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/batch.yml b/test/batch.yml index d19f2b6..250ad6a 100644 --- a/test/batch.yml +++ b/test/batch.yml @@ -222,7 +222,7 @@ Resources: Properties: Handler: index.lambda_handler Runtime: python3.6 - Timeout: 5 + Timeout: 60 Role: !GetAtt DeleteStackRole.Arn Environment: Variables: From c70e409fdcec8f0f5f91b5c1be1b0dd0956b2894 Mon Sep 17 00:00:00 2001 From: Mary Jo Ramos Date: Tue, 3 Sep 2019 18:38:04 +0000 Subject: [PATCH 072/110] Update .gitlab-ci.yml to use job matrix --- .gitlab-ci.yml | 159 ++++++++--------------------------------------- Project.toml | 17 ++--- REQUIRE | 7 --- src/AWSBatch.jl | 1 - test/runtests.jl | 19 +++++- 5 files changed, 52 insertions(+), 151 deletions(-) delete mode 100644 REQUIRE diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 348c699..304155a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,68 +1,18 @@ -stages: - - setup - - test - - teardown - +--- include: - - https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/templates/hidden-jobs.yml - - https://gitlab.invenia.ca/infrastructure/gitlab-ci-helper/raw/master/templates/teardown.yml - + - project: infrastructure/gitlab-ci-helper + file: /templates/julia.yml + variables: - STACKNAME_PREFIX: sandbox-awsbatch + STACK_NAME_PREFIX: sandbox-awsbatch CLOUDSPY_IMAGE: 468665244580.dkr.ecr.us-east-1.amazonaws.com/cloudspy # Variables that require shell execution or depend on a variable that does -.runtime_variables: &runtime_variables - | - # Declare runtime variables - # - # Make unique stacks for each executed pipeline for `master`. This will allow fast successive merges - # to work correctly. - if [[ ${CI_COMMIT_REF_SLUG} == "master" ]]; then - STACKNAME="${STACKNAME_PREFIX}-master-${CI_PIPELINE_ID}" - else - STACKNAME="${STACKNAME_PREFIX}-${CI_COMMIT_REF_SLUG/\//}" # Replace an forward slashes with a hyphen - fi - ACCOUNT_ID=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | grep -oP '(?<="accountId" : ")[^"]*(?=")') - export AWS_DEFAULT_REGION=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document | grep -oP '(?<="region" : ")[^"]*(?=")') - - # Export temp credentials. Make sure this path is absolute as Julia can change directories during testing - export AWS_SHARED_CREDENTIALS_FILE="$(pwd)/tmp-creds" - -.awscli_install: &awscli_install +.setup: &setup | - # Build cloudspy and its dependencies - NEED_SUDO="sudo" - - # docker-ci has slightly different install parameters - [[ $CI_DISPOSABLE_ENVIRONMENT == "true" ]] && SUDO="" || SUDO="sudo" - - # Install python3 and necessary packages - ${SUDO} yum -y install python3 python3-devel gcc git - - # Install a virtualenv with python 3 and activate it - python3 -m venv venv - source venv/bin/activate - - # Install AWSCli - pip install --upgrade awscli - - # Print aws version - aws --version - -.cloudspy_install: &cloudspy_install - | - # Install Cloudspy in the env - pip install git+https://gitlab-ci-token:${CI_BUILD_TOKEN}@gitlab.invenia.ca/infrastructure/cloudspy.git#egg=cloudspy - -.assume_test_profile: &assume_test_profile - | - aws-credentials \ - --credentials-file=$AWS_SHARED_CREDENTIALS_FILE \ - --role-arn arn:aws:iam::${ACCOUNT_ID}:role/${STACKNAME}-TestRole \ - --role-session-name test - export AWS_PROFILE=test - + echo "$common_functions" > common && source common + STACK_NAME=$(stack_name $STACK_NAME_PREFIX) + ACCOUNT_ID=$(aws_account_id) "Setup Environment": stage: setup @@ -84,78 +34,17 @@ variables: - docker-ci - ci-account before_script: - - *runtime_variables + - *setup script: - aws cloudformation validate-template --template-body file://test/batch.yml - | aws-create-stack \ --role-arn arn:aws:iam::${ACCOUNT_ID}:role/CloudFormationAdmin \ - --stackname $STACKNAME \ + --stackname $STACK_NAME \ --template-body ./test/batch.yml \ --wait \ --params CIRoleArn=arn:aws:iam::${ACCOUNT_ID}:role/GitLabCIRunnerRole -"1.0 (Mac)": - tags: - - mac - - shell-ci - extends: .test_shell_1_0 - -"1.0 (Linux, 64-bit)": - tags: - - linux - - 64-bit - - docker-ci - extends: .test_docker_1_0 - -"1.0 (Linux, 32-bit)": - tags: - - linux - - 32-bit - - shell-ci - extends: .test_shell_1_0 - -"1.1 (Mac)": - tags: - - mac - - shell-ci - extends: .test_shell_1_1 - -"1.1 (Linux, 64-bit)": - tags: - - linux - - 64-bit - - docker-ci - extends: .test_docker_1_1 - -"1.1 (Linux, 32-bit)": - tags: - - linux - - 32-bit - - shell-ci - extends: .test_shell_1_1 - -"Nightly (Mac)": - tags: - - mac - - shell-ci - extends: .test_shell_nightly - -"Nightly (Linux, 64-bit)": - tags: - - linux - - 64-bit - - docker-ci - extends: .test_docker_nightly - -"Nightly (Linux, 32-bit)": - tags: - - linux - - 32-bit - - shell-ci - extends: .test_shell_nightly - - .test_batch: stage: test tags: @@ -165,13 +54,11 @@ variables: variables: TESTS: "batch" # Runs the online tests against AWS script: - - *runtime_variables - - *awscli_install - - *cloudspy_install - - *assume_test_profile + - *setup + - assume_test_role # Execute online tests - source julia-ci export - - AWS_STACKNAME=$STACKNAME ./julia-ci test + - AWS_STACKNAME=$STACK_NAME ./julia-ci test - ./julia-ci coverage extends: .test_shell @@ -185,9 +72,15 @@ variables: JULIA_VERSION: "1.1" extends: .test_batch -# Note: We cannot test AWS Batch on nightly as we don't build "julia-baked" images with -# the nightly Julia revision. +"1.2 (AWS Batch)": + variables: + JULIA_VERSION: "1.2" + extends: .test_batch +"Nightly (AWS Batch)": + variables: + JULIA_VERSION: "nightly" + extends: .test_batch .delete: &delete tags: @@ -195,14 +88,14 @@ variables: - ci-account image: $CLOUDSPY_IMAGE before_script: - - *runtime_variables + - *setup script: - - eval $(aws-stack-outputs $STACKNAME) + - eval $(aws-stack-outputs $STACK_NAME) - | aws cloudformation delete-stack \ --role-arn arn:aws:iam::${ACCOUNT_ID}:role/CloudFormationAdmin \ - --stack-name $STACKNAME - - aws cloudformation wait stack-delete-complete --stack-name $STACKNAME + --stack-name $STACK_NAME + - aws cloudformation wait stack-delete-complete --stack-name $STACK_NAME "Delete Environment": stage: teardown diff --git a/Project.toml b/Project.toml index 6e0dd50..b1df4e3 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,7 @@ name = "AWSBatch" uuid = "dcae83d4-2881-5875-9d49-e5534165e9c0" -authors = ["Nicole Epp ", "Curtis Vogt "" + ) + ) + version_numbers = VersionNumber.(filter(v -> !endswith(v, "^{}"), julia_tags)) + latest_version = maximum(version_numbers) + is_nightly = VERSION > latest_version + + "468665244580.dkr.ecr.us-east-1.amazonaws.com/julia-baked:" * (is_nightly ? "nightly" : "$VERSION") +end + Memento.config!("debug"; fmt="[{level} | {name}]: {msg}") const logger = getlogger(AWSBatch) setlevel!(logger, "info") include("mock.jl") - @testset "AWSBatch.jl" begin if "local" in TESTS include("compute_environment.jl") From 4ab44831ebf65fba339158d791df510f505e7765 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Mon, 9 Sep 2019 10:18:41 -0500 Subject: [PATCH 073/110] Cleanup JULIA_BAKED_IMAGE logic --- test/runtests.jl | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index c1d415a..ce42d60 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -19,20 +19,13 @@ const STACK = !isempty(AWS_STACKNAME) ? stack_output(AWS_STACKNAME) : Dict() const JOB_TIMEOUT = 900 const JULIA_BAKED_IMAGE = let - julia_tags = split( - replace( - read( - `git ls-remote --tags https://github.com/JuliaLang/julia`, - String - ), - r".*\/" => "" - ) - ) - version_numbers = VersionNumber.(filter(v -> !endswith(v, "^{}"), julia_tags)) - latest_version = maximum(version_numbers) - is_nightly = VERSION > latest_version + output = read(`git ls-remote --tags https://github.com/JuliaLang/julia`, String) + tags = split(replace(output, r".*\/" => "")) + versions = VersionNumber.(filter(v -> !endswith(v, "^{}"), tags)) + latest_version = maximum(versions) + docker_tag = VERSION > latest_version ? "nightly" : "$VERSION" - "468665244580.dkr.ecr.us-east-1.amazonaws.com/julia-baked:" * (is_nightly ? "nightly" : "$VERSION") + "468665244580.dkr.ecr.us-east-1.amazonaws.com/julia-baked:$docker_tag" end Memento.config!("debug"; fmt="[{level} | {name}]: {msg}") From 2e86f1451f396b65349de6991f4a775ab35af54a Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Mon, 9 Sep 2019 10:18:47 -0500 Subject: [PATCH 074/110] Support Mocking 0.7 --- Project.toml | 2 +- test/run_batch.jl | 17 +++++++---------- test/runtests.jl | 4 +++- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/Project.toml b/Project.toml index b1df4e3..5a9d0e9 100644 --- a/Project.toml +++ b/Project.toml @@ -18,7 +18,7 @@ AWSSDK = "0.2, 0.3, 0.4" AWSTools = "0.3.1, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1" AutoHashEquals = "0.2.0" Memento = "0.7, 0.8, 0.9, 0.10, 0.11, 0.12" -Mocking = "0.3.3, 0.4, 0.5, 0.6" +Mocking = "0.3.3, 0.4, 0.5, 0.6, 0.7" julia = "1" [extras] diff --git a/test/run_batch.jl b/test/run_batch.jl index 07af9e1..dd7cfd6 100644 --- a/test/run_batch.jl +++ b/test/run_batch.jl @@ -1,9 +1,9 @@ -function register_job_def(config::AWSConfig, input::AbstractArray, expected::AbstractArray) +function _register_job_def(config::AWSConfig, input::AbstractArray, expected::AbstractArray) @test input == expected return REGISTER_JOB_DEF_RESP end -function submit_job(config::AWSConfig, input::AbstractArray, expected::AbstractArray) +function _submit_job(config::AWSConfig, input::AbstractArray, expected::AbstractArray) @test input == expected return SUBMIT_JOB_RESP end @@ -32,7 +32,7 @@ end patches = [ @patch describe_job_definitions(args...) = DESCRIBE_JOBS_DEF_RESP - @patch submit_job(config, input) = submit_job(config, input, expected_job) + @patch submit_job(config, input) = _submit_job(config, input, expected_job) ] apply(patches) do @@ -72,12 +72,9 @@ end @patch read(cmd::AbstractCmd, ::Type{String}) = mock_read(cmd, String) @patch describe_jobs(args...) = DESCRIBE_JOBS_RESP @patch describe_job_definitions(args...) = Dict("jobDefinitions" => Dict()) - @patch register_job_definition(config, input) = register_job_def( - config, - input, - expected_job_def, - ) - @patch submit_job(config, input) = submit_job(config, input, expected_job) + @patch register_job_definition(config, input) = + _register_job_def(config, input, expected_job_def) + @patch submit_job(config, input) = _submit_job(config, input, expected_job) ] apply(patches) do @@ -105,7 +102,7 @@ end @patch read(cmd::AbstractCmd, ::Type{String}) = mock_read(cmd, String) @patch describe_jobs(args...) = DESCRIBE_JOBS_RESP @patch describe_job_definitions(args...) = Dict("jobDefinitions" => Dict()) - @patch submit_job(config, input) = submit_job(config, input, expected_job) + @patch submit_job(config, input) = _submit_job(config, input, expected_job) ] apply(patches) do diff --git a/test/runtests.jl b/test/runtests.jl index ce42d60..976e8c5 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,7 +1,9 @@ using Mocking -Mocking.enable(force=true) +Mocking.enable(; force=true) using AWSBatch +using AWSBatch: describe_compute_environments, describe_jobs, describe_job_definitions, + describe_job_queues, register_job_definition, submit_job using AWSCore: AWSConfig using AWSTools.CloudFormation: stack_output using Dates From 4418745f8846aef54bd1229896bcecfaf7b2bbe3 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Mon, 9 Sep 2019 13:17:16 -0500 Subject: [PATCH 075/110] Set project version to 1.1.0 --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 5a9d0e9..51c42c0 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "AWSBatch" uuid = "dcae83d4-2881-5875-9d49-e5534165e9c0" authors = ["Invenia Technical Computing"] -version = "1.0.1" +version = "1.1.0" [deps] AWSCore = "4f1ea46c-232b-54a6-9b17-cc2d0f3e6598" From e59d0c45ba2ed6a6f2f24ab433d30298fd16bfe3 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Mon, 9 Sep 2019 10:18:47 -0500 Subject: [PATCH 076/110] Upgrade to Mocking 0.7 --- Project.toml | 2 +- test/mock.jl | 4 ++-- test/runtests.jl | 8 +++----- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/Project.toml b/Project.toml index 51c42c0..5e7909b 100644 --- a/Project.toml +++ b/Project.toml @@ -18,7 +18,7 @@ AWSSDK = "0.2, 0.3, 0.4" AWSTools = "0.3.1, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1" AutoHashEquals = "0.2.0" Memento = "0.7, 0.8, 0.9, 0.10, 0.11, 0.12" -Mocking = "0.3.3, 0.4, 0.5, 0.6, 0.7" +Mocking = "0.7" julia = "1" [extras] diff --git a/test/mock.jl b/test/mock.jl index d80431b..fc7685e 100644 --- a/test/mock.jl +++ b/test/mock.jl @@ -115,7 +115,7 @@ end function describe_compute_environments_patch(output::Vector=[]) - @patch function describe_compute_environments(d::Dict) + @patch function AWSBatch.describe_compute_environments(d::Dict) compute_envs = d["computeEnvironments"] @assert length(compute_envs) == 1 ce = first(compute_envs) @@ -131,7 +131,7 @@ function describe_compute_environments_patch(output::OrderedDict) end function describe_job_queues_patch(output::Vector=[]) - @patch function describe_job_queues(d::Dict) + @patch function AWSBatch.describe_job_queues(d::Dict) queues = d["jobQueues"] @assert length(queues) == 1 queue = first(queues) diff --git a/test/runtests.jl b/test/runtests.jl index 976e8c5..14e0a04 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,15 +1,13 @@ -using Mocking -Mocking.enable(; force=true) - using AWSBatch -using AWSBatch: describe_compute_environments, describe_jobs, describe_job_definitions, - describe_job_queues, register_job_definition, submit_job +using AWSBatch: describe_jobs, describe_job_definitions, register_job_definition, submit_job using AWSCore: AWSConfig using AWSTools.CloudFormation: stack_output using Dates using Memento +using Mocking using Test +Mocking.activate() # Controls the running of various tests: "local", "batch" const TESTS = strip.(split(get(ENV, "TESTS", "local"), r"\s*,\s*")) From 249ab471a21b87eccb20a8063cb065b78e287d4c Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Mon, 16 Sep 2019 08:58:04 -0500 Subject: [PATCH 077/110] Set project version to 1.2.0 --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 5e7909b..d05b78e 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "AWSBatch" uuid = "dcae83d4-2881-5875-9d49-e5534165e9c0" authors = ["Invenia Technical Computing"] -version = "1.1.0" +version = "1.2.0" [deps] AWSCore = "4f1ea46c-232b-54a6-9b17-cc2d0f3e6598" From 11b50d0f3169a4f9f47df8e8a1c713533b277ddf Mon Sep 17 00:00:00 2001 From: morris25 Date: Mon, 30 Sep 2019 14:22:28 -0500 Subject: [PATCH 078/110] Replaces curl pipeline with AWSTools region function --- Project.toml | 4 ++-- src/AWSBatch.jl | 10 ++-------- test/mock.jl | 18 ------------------ test/run_batch.jl | 4 ++-- test/runtests.jl | 1 + 5 files changed, 7 insertions(+), 30 deletions(-) diff --git a/Project.toml b/Project.toml index d05b78e..c8bf782 100644 --- a/Project.toml +++ b/Project.toml @@ -6,6 +6,7 @@ version = "1.2.0" [deps] AWSCore = "4f1ea46c-232b-54a6-9b17-cc2d0f3e6598" AWSSDK = "0d499d91-6ae5-5d63-9313-12987b87d5ad" +AWSTools = "83bcdc74-1232-581c-948a-f29122bf8259" AutoHashEquals = "15f4f7f2-30c1-5605-9d31-71845cf9641f" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Memento = "f28f55f0-a522-5efc-85c2-fe41dfb9b2d9" @@ -22,8 +23,7 @@ Mocking = "0.7" julia = "1" [extras] -AWSTools = "83bcdc74-1232-581c-948a-f29122bf8259" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["AWSTools", "Test"] +test = ["Test"] diff --git a/src/AWSBatch.jl b/src/AWSBatch.jl index 6409557..f1f9abf 100644 --- a/src/AWSBatch.jl +++ b/src/AWSBatch.jl @@ -4,6 +4,7 @@ using AutoHashEquals using AWSCore: aws_config using AWSSDK.Batch using AWSSDK.CloudWatchLogs +using AWSTools.EC2: instance_region using OrderedCollections: OrderedDict using Dates using Memento @@ -122,14 +123,7 @@ function run_batch(; job_queue = ENV["AWS_BATCH_JQ_NAME"] # Get the zone information from the EC2 instance metadata. - zone = @mock read( - pipeline( - `curl http://169.254.169.254/latest/meta-data/placement/availability-zone`; - stderr=devnull - ), - String - ) - isempty(region) && (region = chop(zone)) + isempty(region) && (region = @mock instance_region()) # Requires permissions to access to "batch:DescribeJobs" response = @mock describe_jobs(Dict("jobs" => [job_id])) diff --git a/test/mock.jl b/test/mock.jl index fc7685e..51dc978 100644 --- a/test/mock.jl +++ b/test/mock.jl @@ -96,24 +96,6 @@ const DESCRIBE_JOBS_RESP = Dict( ] ) - -""" - Mock.read(cmd::CmdRedirect, ::Type{String}) - -Mocks the CmdRedirect produced from -``pipeline(`curl http://169.254.169.254/latest/meta-data/placement/availability-zone`)`` -to just return "us-east-1a". -""" -function mock_read(cmd::CmdRedirect, ::Type{String}) - cmd_exec = cmd.cmd.exec - result = if cmd_exec[1] == "curl" && occursin("availability-zone", cmd_exec[2]) - return "us-east-1a" - else - return Base.read(cmd, String) - end -end - - function describe_compute_environments_patch(output::Vector=[]) @patch function AWSBatch.describe_compute_environments(d::Dict) compute_envs = d["computeEnvironments"] diff --git a/test/run_batch.jl b/test/run_batch.jl index dd7cfd6..a0eaf3d 100644 --- a/test/run_batch.jl +++ b/test/run_batch.jl @@ -69,7 +69,7 @@ end ] patches = [ - @patch read(cmd::AbstractCmd, ::Type{String}) = mock_read(cmd, String) + @patch instance_region() = "us-east-1" @patch describe_jobs(args...) = DESCRIBE_JOBS_RESP @patch describe_job_definitions(args...) = Dict("jobDefinitions" => Dict()) @patch register_job_definition(config, input) = @@ -99,7 +99,7 @@ end ] patches = [ - @patch read(cmd::AbstractCmd, ::Type{String}) = mock_read(cmd, String) + @patch instance_region() = "us-east-1" @patch describe_jobs(args...) = DESCRIBE_JOBS_RESP @patch describe_job_definitions(args...) = Dict("jobDefinitions" => Dict()) @patch submit_job(config, input) = _submit_job(config, input, expected_job) diff --git a/test/runtests.jl b/test/runtests.jl index 14e0a04..00d1833 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,6 +2,7 @@ using AWSBatch using AWSBatch: describe_jobs, describe_job_definitions, register_job_definition, submit_job using AWSCore: AWSConfig using AWSTools.CloudFormation: stack_output +using AWSTools.EC2: instance_region using Dates using Memento using Mocking From 8b195c9da46afc139befeb21caef5e936beee81a Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Tue, 15 Oct 2019 14:34:23 -0500 Subject: [PATCH 079/110] Allow failure on Nightly AWS Batch job --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 304155a..0662f67 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,7 @@ include: - project: infrastructure/gitlab-ci-helper file: /templates/julia.yml - + variables: STACK_NAME_PREFIX: sandbox-awsbatch CLOUDSPY_IMAGE: 468665244580.dkr.ecr.us-east-1.amazonaws.com/cloudspy @@ -80,6 +80,7 @@ variables: "Nightly (AWS Batch)": variables: JULIA_VERSION: "nightly" + allow_failure: true extends: .test_batch .delete: &delete From a8c463a163561a2a825906a1524e9336cd99dd1b Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 13 Nov 2019 09:44:48 -0600 Subject: [PATCH 080/110] Handle non-existent log streams --- Project.toml | 6 ++++-- src/AWSBatch.jl | 2 +- src/batch_job.jl | 49 +++++++++++++++++++++++++++++++++++++---------- test/batch_job.jl | 43 +++++++++++++++++++++++++++++++++++++++++ test/runtests.jl | 5 ++++- 5 files changed, 91 insertions(+), 14 deletions(-) create mode 100644 test/batch_job.jl diff --git a/Project.toml b/Project.toml index c8bf782..eee2faf 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "AWSBatch" uuid = "dcae83d4-2881-5875-9d49-e5534165e9c0" authors = ["Invenia Technical Computing"] -version = "1.2.0" +version = "1.3.0" [deps] AWSCore = "4f1ea46c-232b-54a6-9b17-cc2d0f3e6598" @@ -18,12 +18,14 @@ AWSCore = "0.5.2, 0.6" AWSSDK = "0.2, 0.3, 0.4" AWSTools = "0.3.1, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1" AutoHashEquals = "0.2.0" +HTTP = "0.8.1" Memento = "0.7, 0.8, 0.9, 0.10, 0.11, 0.12" Mocking = "0.7" julia = "1" [extras] +HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Test"] +test = ["HTTP", "Test"] diff --git a/src/AWSBatch.jl b/src/AWSBatch.jl index f1f9abf..4da4f60 100644 --- a/src/AWSBatch.jl +++ b/src/AWSBatch.jl @@ -1,7 +1,7 @@ module AWSBatch using AutoHashEquals -using AWSCore: aws_config +using AWSCore: AWSException, aws_config using AWSSDK.Batch using AWSSDK.CloudWatchLogs using AWSTools.EC2: instance_region diff --git a/src/batch_job.jl b/src/batch_job.jl index ea3fee9..2a85217 100644 --- a/src/batch_job.jl +++ b/src/batch_job.jl @@ -204,9 +204,10 @@ function Base.wait( end """ - log_events(job::BatchJob) -> Vector{LogEvent} + log_events(job::BatchJob, Nothing) -> Union{Vector{LogEvent}, Nothing} -Fetches the logStreamName, fetches the CloudWatch logs and returns a vector of messages. +Fetches the logStreamName, fetches the CloudWatch logs and returns a vector of log events. +If the log stream does not currently exist then `nothing` is returned. NOTES: - The `logStreamName` isn't available until the job is RUNNING, so you may want to use @@ -214,17 +215,45 @@ NOTES: - We do not support pagination, so this function is limited to 10,000 log messages by default. """ -function log_events(job::BatchJob) +function log_events( + job::BatchJob, + stream_missing_return_type::Union{Type{Vector{LogEvent}},Type{Nothing}}=Vector{LogEvent}, +) + if stream_missing_return_type !== Nothing + Base.depwarn( + "Non-existent log streams will return `nothing` instead of `LogEvent[]` " * + "in the future. Use `log_events(job, Nothing)` to adopt the new behaviour.", + :log_events, + ) + end + job_details = describe(job) - if "logStreamName" in keys(job_details["container"]) + if haskey(job_details["container"], "logStreamName") stream = job_details["container"]["logStreamName"] - - info(logger, "Fetching log events from $stream") - output = get_log_events(logGroupName="/aws/batch/job", logStreamName=stream) - return convert.(LogEvent, output["events"]) else - info(logger, "No log events found for job $(job.id)") - return LogEvent[] + return stream_missing_return_type() end + + info(logger, "Fetching log events from $stream") + output = try + @mock get_log_events(logGroupName="/aws/batch/job", logStreamName=stream) + catch e + # The batch job has a reference to a log stream but the stream has not yet been + # created. + if ( + e isa AWSException && + e.cause.status == 400 && + e.info["message"] == "The specified log stream does not exist." + ) + return stream_missing_return_type() + end + + rethrow() + end + + return convert.(LogEvent, output["events"]) end + +# TODO: Once we default to returning `nothing` +# @deprecate log_events(job::BatchJob, ::Union{Type{Vector{LogEvent}},Type{Nothing}}) log_events(job) diff --git a/test/batch_job.jl b/test/batch_job.jl new file mode 100644 index 0000000..68b75ec --- /dev/null +++ b/test/batch_job.jl @@ -0,0 +1,43 @@ +@testset "BatchJob" begin + job = BatchJob("00000000-0000-0000-0000-000000000000") + + @testset "log_events" begin + @testset "Log stream not yet created" begin + # When a AWS Batch job is first submitted the description of the job will not + # contain a reference to a log stream + patch = @patch describe_jobs(args...; kwargs...) = Dict( + "jobs" => [Dict("container" => Dict())] + ) + apply(patch) do + @test log_events(job, Nothing) === nothing + end + end + + @testset "Log stream does not exist" begin + # The AWS Batch job references a log stream but the stream has not yet been + # created. + dne_exception = AWSException( + HTTP.StatusError( + 400, + "", + "", + HTTP.Messages.Response( + 400, + Dict("Content-Type" => "application/x-amz-json-1.1"); + body="""{"__type":"ResourceNotFoundException","message":"The specified log stream does not exist."}""" + ) + ) + ) + + patches = [ + @patch describe_jobs(args...; kwargs...) = Dict( + "jobs" => [Dict("container" => Dict("logStreamName" => ""))] + ) + @patch get_log_events(; kwargs...) = throw(dne_exception) + ] + apply(patches) do + @test log_events(job, Nothing) === nothing # TODO: Suppress "Fetching log events from" + end + end + end +end diff --git a/test/runtests.jl b/test/runtests.jl index 00d1833..9f59a01 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,9 +1,11 @@ using AWSBatch using AWSBatch: describe_jobs, describe_job_definitions, register_job_definition, submit_job -using AWSCore: AWSConfig +using AWSCore: AWSConfig, AWSException +using AWSSDK.CloudWatchLogs: get_log_events using AWSTools.CloudFormation: stack_output using AWSTools.EC2: instance_region using Dates +using HTTP: HTTP using Memento using Mocking using Test @@ -41,6 +43,7 @@ include("mock.jl") include("job_queue.jl") include("log_event.jl") include("job_state.jl") + include("batch_job.jl") include("run_batch.jl") else warn(logger, "Skipping \"local\" tests. Set `ENV[\"TESTS\"] = \"local\"` to run.") From 5808dd8a30e96ae5817f25a07af26a4eeff7ba96 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 13 Nov 2019 12:57:56 -0600 Subject: [PATCH 081/110] Add `@__VERSION__` macro --- Project.toml | 1 + src/AWSBatch.jl | 1 + src/version.jl | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+) create mode 100644 src/version.jl diff --git a/Project.toml b/Project.toml index eee2faf..fc440a2 100644 --- a/Project.toml +++ b/Project.toml @@ -12,6 +12,7 @@ Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Memento = "f28f55f0-a522-5efc-85c2-fe41dfb9b2d9" Mocking = "78c3b35d-d492-501b-9361-3d52fe80e533" OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" [compat] AWSCore = "0.5.2, 0.6" diff --git a/src/AWSBatch.jl b/src/AWSBatch.jl index 4da4f60..7c7cc3a 100644 --- a/src/AWSBatch.jl +++ b/src/AWSBatch.jl @@ -34,6 +34,7 @@ const logger = getlogger(@__MODULE__) __init__() = Memento.register(logger) +include("version.jl") include("log_event.jl") include("compute_environment.jl") include("job_queue.jl") diff --git a/src/version.jl b/src/version.jl new file mode 100644 index 0000000..1ebc0a6 --- /dev/null +++ b/src/version.jl @@ -0,0 +1,38 @@ +# TODO: Temporary until this is turned into a package +using Pkg + +""" + @__VERSION__ -> Union{VersionNumber, Nothing} + +Get the `VersionNumber` of the package which expands this macro. If executed outside of a +package `nothing` will be returned. +""" +macro __VERSION__() + ctxt = Pkg.Types.Context() + pkg_id = Base.PkgId(__module__) + pkg_id.uuid === nothing && return nothing + + project = ctxt.env.project + project_name = @static if v"1.0-" <= VERSION < v"1.1-" + project["name"] + else + project.name + end + + pkg_info = if project_name == pkg_id.name + project + else + _ctxt = @static VERSION < v"1.4-" ? ctxt.env : ctxt + Pkg.Types.manifest_info(_ctxt, pkg_id.uuid) + end + + version = @static if v"1.0-" <= VERSION < v"1.1-" + VersionNumber(pkg_info["version"]) + elseif v"1.1-" <= VERSION < v"1.2-" + VersionNumber(pkg_info.version) + else + pkg_info.version + end + + return version +end From 1791c1ae46cac741ecb135da95842108108fff6d Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 13 Nov 2019 13:03:01 -0600 Subject: [PATCH 082/110] Drop deprecation once minor version increases --- src/batch_job.jl | 130 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 88 insertions(+), 42 deletions(-) diff --git a/src/batch_job.jl b/src/batch_job.jl index 2a85217..94b3ef0 100644 --- a/src/batch_job.jl +++ b/src/batch_job.jl @@ -203,57 +203,103 @@ function Base.wait( end end -""" - log_events(job::BatchJob, Nothing) -> Union{Vector{LogEvent}, Nothing} -Fetches the logStreamName, fetches the CloudWatch logs and returns a vector of log events. -If the log stream does not currently exist then `nothing` is returned. +if @__VERSION__() < v"1.4" + @doc """ + log_events(job::BatchJob, Nothing) -> Union{Vector{LogEvent}, Nothing} -NOTES: -- The `logStreamName` isn't available until the job is RUNNING, so you may want to use - `wait(job)` or `wait(job, [AWSBatch.SUCCEEDED])` prior to calling this function. -- We do not support pagination, so this function is limited to 10,000 log messages by - default. -""" -function log_events( - job::BatchJob, - stream_missing_return_type::Union{Type{Vector{LogEvent}},Type{Nothing}}=Vector{LogEvent}, -) - if stream_missing_return_type !== Nothing - Base.depwarn( - "Non-existent log streams will return `nothing` instead of `LogEvent[]` " * - "in the future. Use `log_events(job, Nothing)` to adopt the new behaviour.", - :log_events, - ) - end + Fetches the logStreamName, fetches the CloudWatch logs and returns a vector of log events. + If the log stream does not currently exist then `nothing` is returned. - job_details = describe(job) + NOTES: + - The `logStreamName` isn't available until the job is RUNNING, so you may want to use + `wait(job)` or `wait(job, [AWSBatch.SUCCEEDED])` prior to calling this function. + - We do not support pagination, so this function is limited to 10,000 log messages by + default. + """ + function log_events( + job::BatchJob, + stream_missing_return_type::Union{Type{Vector{LogEvent}},Type{Nothing}}=Vector{LogEvent}, + ) + if stream_missing_return_type !== Nothing + Base.depwarn( + "Non-existent log streams will return `nothing` instead of `LogEvent[]` " * + "in the future. Use `log_events(job, Nothing)` to adopt the new behaviour.", + :log_events, + ) + end - if haskey(job_details["container"], "logStreamName") - stream = job_details["container"]["logStreamName"] - else - return stream_missing_return_type() - end + job_details = describe(job) - info(logger, "Fetching log events from $stream") - output = try - @mock get_log_events(logGroupName="/aws/batch/job", logStreamName=stream) - catch e - # The batch job has a reference to a log stream but the stream has not yet been - # created. - if ( - e isa AWSException && - e.cause.status == 400 && - e.info["message"] == "The specified log stream does not exist." - ) + if haskey(job_details["container"], "logStreamName") + stream = job_details["container"]["logStreamName"] + else return stream_missing_return_type() end - rethrow() + info(logger, "Fetching log events from $stream") + output = try + @mock get_log_events(logGroupName="/aws/batch/job", logStreamName=stream) + catch e + # The batch job has a reference to a log stream but the stream has not yet been + # created. + if ( + e isa AWSException && + e.cause.status == 400 && + e.info["message"] == "The specified log stream does not exist." + ) + return stream_missing_return_type() + end + + rethrow() + end + + return convert.(LogEvent, output["events"]) end - return convert.(LogEvent, output["events"]) +else + @doc """ + log_events(job::BatchJob) -> Union{Vector{LogEvent}, Nothing} + + Fetches the logStreamName, fetches the CloudWatch logs and returns a vector of log events. + If the log stream does not currently exist then `nothing` is returned. + + NOTES: + - The `logStreamName` isn't available until the job is RUNNING, so you may want to use + `wait(job)` or `wait(job, [AWSBatch.SUCCEEDED])` prior to calling this function. + - We do not support pagination, so this function is limited to 10,000 log messages by + default. + """ + function log_events(job::BatchJob) + job_details = describe(job) + + if haskey(job_details["container"], "logStreamName") + stream = job_details["container"]["logStreamName"] + else + return nothing + end + + info(logger, "Fetching log events from $stream") + output = try + @mock get_log_events(logGroupName="/aws/batch/job", logStreamName=stream) + catch e + # The batch job has a reference to a log stream but the stream has not yet been + # created. + if ( + e isa AWSException && + e.cause.status == 400 && + e.info["message"] == "The specified log stream does not exist." + ) + return nothing + end + + rethrow() + end + + return convert.(LogEvent, output["events"]) + end end -# TODO: Once we default to returning `nothing` -# @deprecate log_events(job::BatchJob, ::Union{Type{Vector{LogEvent}},Type{Nothing}}) log_events(job) +if v"1.4" <= @__VERSION__() < v"1.5" + @deprecate log_events(job::BatchJob, ::Union{Type{Vector{LogEvent}},Type{Nothing}}) log_events(job) +end From 0cd859537eb8da33646e91c16ab2bc55aa487cc2 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 13 Nov 2019 13:40:13 -0600 Subject: [PATCH 083/110] Add additional tests --- test/batch_job.jl | 45 ++++++++++++++++++++++++++++++++++----------- test/mock.jl | 19 +++++++++++++++++++ test/runtests.jl | 3 ++- 3 files changed, 55 insertions(+), 12 deletions(-) diff --git a/test/batch_job.jl b/test/batch_job.jl index 68b75ec..f11f85b 100644 --- a/test/batch_job.jl +++ b/test/batch_job.jl @@ -2,18 +2,16 @@ job = BatchJob("00000000-0000-0000-0000-000000000000") @testset "log_events" begin - @testset "Log stream not yet created" begin + @testset "Stream not yet created" begin # When a AWS Batch job is first submitted the description of the job will not # contain a reference to a log stream - patch = @patch describe_jobs(args...; kwargs...) = Dict( - "jobs" => [Dict("container" => Dict())] - ) - apply(patch) do + patches = log_events_patches(log_stream_name=nothing) + apply(patches) do @test log_events(job, Nothing) === nothing end end - @testset "Log stream does not exist" begin + @testset "Stream DNE" begin # The AWS Batch job references a log stream but the stream has not yet been # created. dne_exception = AWSException( @@ -29,14 +27,39 @@ ) ) - patches = [ - @patch describe_jobs(args...; kwargs...) = Dict( - "jobs" => [Dict("container" => Dict("logStreamName" => ""))] + patches = log_events_patches(events=() -> throw(dne_exception)) + apply(patches) do + @test log_events(job, Nothing) === nothing # TODO: Suppress "Fetching log events from" + end + end + + @testset "Stream with no events" begin + patches = log_events_patches(events=[]) + apply(patches) do + @test log_events(job, Nothing) == LogEvent[] + end + end + + @testset "Stream with events" begin + events = [ + Dict( + "eventId" => "0" ^ 56, + "ingestionTime" => 1573672813145, + "timestamp" => 1573672813145, + "message" => "hello world!", ) - @patch get_log_events(; kwargs...) = throw(dne_exception) ] + patches = log_events_patches(events=events) + apply(patches) do - @test log_events(job, Nothing) === nothing # TODO: Suppress "Fetching log events from" + @test log_events(job, Nothing) == [ + LogEvent( + "0" ^ 56, + DateTime(2019, 11, 13, 19, 20, 13, 145), + DateTime(2019, 11, 13, 19, 20, 13, 145), + "hello world!", + ) + ] end end end diff --git a/test/mock.jl b/test/mock.jl index 51dc978..75871c6 100644 --- a/test/mock.jl +++ b/test/mock.jl @@ -127,3 +127,22 @@ end function describe_job_queues_patch(output::OrderedDict) describe_job_queues_patch([output]) end + +function log_events_patches(; log_stream_name="mock_stream", events=[]) + job_descriptions = if log_stream_name === nothing + Dict("jobs" => [Dict("container" => Dict())]) + else + Dict("jobs" => [Dict("container" => Dict("logStreamName" => log_stream_name))]) + end + + log_events_function = if events isa AbstractVector + () -> Dict("events" => events) + else + events + end + + return [ + @patch describe_jobs(args...; kwargs...) = job_descriptions + @patch get_log_events(; kwargs...) = log_events_function() + ] +end diff --git a/test/runtests.jl b/test/runtests.jl index 9f59a01..a348d4a 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,5 +1,6 @@ using AWSBatch -using AWSBatch: describe_jobs, describe_job_definitions, register_job_definition, submit_job +using AWSBatch: LogEvent, describe_jobs, describe_job_definitions, register_job_definition, + submit_job using AWSCore: AWSConfig, AWSException using AWSSDK.CloudWatchLogs: get_log_events using AWSTools.CloudFormation: stack_output From 1b8493e80663d2055f65e8af72f7333ae6cba80e Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Thu, 14 Nov 2019 00:01:12 -0600 Subject: [PATCH 084/110] Refactor internals of `@__VERSION__` Based upon suggestions here: https://github.com/JuliaLang/Pkg.jl/pull/1501 --- src/version.jl | 41 ++++++++++++++++------------------------- 1 file changed, 16 insertions(+), 25 deletions(-) diff --git a/src/version.jl b/src/version.jl index 1ebc0a6..ee69479 100644 --- a/src/version.jl +++ b/src/version.jl @@ -1,5 +1,15 @@ # TODO: Temporary until this is turned into a package -using Pkg +using Pkg: Pkg + +# https://github.com/JuliaLang/julia/pull/33128 +if VERSION < v"1.4.0-DEV.397" + function pkgdir(m::Module) + rootmodule = Base.moduleroot(m) + path = pathof(rootmodule) + path === nothing && return nothing + return dirname(dirname(path)) + end +end """ @__VERSION__ -> Union{VersionNumber, Nothing} @@ -8,31 +18,12 @@ Get the `VersionNumber` of the package which expands this macro. If executed out package `nothing` will be returned. """ macro __VERSION__() - ctxt = Pkg.Types.Context() - pkg_id = Base.PkgId(__module__) - pkg_id.uuid === nothing && return nothing - - project = ctxt.env.project - project_name = @static if v"1.0-" <= VERSION < v"1.1-" - project["name"] - else - project.name - end + pkg_dir = pkgdir(__module__) - pkg_info = if project_name == pkg_id.name - project + if pkg_dir !== nothing + project_data = Pkg.TOML.parsefile(Pkg.Types.projectfile_path(pkg_dir)) + return VersionNumber(project_data["version"]) else - _ctxt = @static VERSION < v"1.4-" ? ctxt.env : ctxt - Pkg.Types.manifest_info(_ctxt, pkg_id.uuid) + return nothing end - - version = @static if v"1.0-" <= VERSION < v"1.1-" - VersionNumber(pkg_info["version"]) - elseif v"1.1-" <= VERSION < v"1.2-" - VersionNumber(pkg_info.version) - else - pkg_info.version - end - - return version end From 9c7fbefe1e0758990e034432f3694a714e766ebb Mon Sep 17 00:00:00 2001 From: Rory Finnegan Date: Mon, 25 Nov 2019 21:14:39 +0000 Subject: [PATCH 085/110] Don't list .png files in assets --- docs/make.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/make.jl b/docs/make.jl index e5edf99..6ce9c71 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -6,7 +6,6 @@ makedocs(; prettyurls=get(ENV, "CI", nothing) == "true", assets=[ "assets/invenia.css", - "assets/logo.png", ], ), pages=[ From 97592e95709327dfa30a176b7c9bb4dc2de8150e Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Fri, 6 Dec 2019 10:34:35 -0600 Subject: [PATCH 086/110] Support pagination with log events --- src/batch_job.jl | 40 +++--------------------------------- src/log_event.jl | 45 ++++++++++++++++++++++++++++++++++++++++ test/batch_job.jl | 52 ----------------------------------------------- test/log_event.jl | 52 +++++++++++++++++++++++++++++++++++++++++++++++ test/mock.jl | 16 ++++++++++----- 5 files changed, 111 insertions(+), 94 deletions(-) diff --git a/src/batch_job.jl b/src/batch_job.jl index 94b3ef0..4a87762 100644 --- a/src/batch_job.jl +++ b/src/batch_job.jl @@ -214,8 +214,6 @@ if @__VERSION__() < v"1.4" NOTES: - The `logStreamName` isn't available until the job is RUNNING, so you may want to use `wait(job)` or `wait(job, [AWSBatch.SUCCEEDED])` prior to calling this function. - - We do not support pagination, so this function is limited to 10,000 log messages by - default. """ function log_events( job::BatchJob, @@ -238,23 +236,9 @@ if @__VERSION__() < v"1.4" end info(logger, "Fetching log events from $stream") - output = try - @mock get_log_events(logGroupName="/aws/batch/job", logStreamName=stream) - catch e - # The batch job has a reference to a log stream but the stream has not yet been - # created. - if ( - e isa AWSException && - e.cause.status == 400 && - e.info["message"] == "The specified log stream does not exist." - ) - return stream_missing_return_type() - end + events = log_events("/aws/batch/job", stream) - rethrow() - end - - return convert.(LogEvent, output["events"]) + return events !== nothing ? events : stream_missing_return_type() end else @@ -267,8 +251,6 @@ else NOTES: - The `logStreamName` isn't available until the job is RUNNING, so you may want to use `wait(job)` or `wait(job, [AWSBatch.SUCCEEDED])` prior to calling this function. - - We do not support pagination, so this function is limited to 10,000 log messages by - default. """ function log_events(job::BatchJob) job_details = describe(job) @@ -280,23 +262,7 @@ else end info(logger, "Fetching log events from $stream") - output = try - @mock get_log_events(logGroupName="/aws/batch/job", logStreamName=stream) - catch e - # The batch job has a reference to a log stream but the stream has not yet been - # created. - if ( - e isa AWSException && - e.cause.status == 400 && - e.info["message"] == "The specified log stream does not exist." - ) - return nothing - end - - rethrow() - end - - return convert.(LogEvent, output["events"]) + return log_events("/aws/batch/job", stream) end end diff --git a/src/log_event.jl b/src/log_event.jl index 6f7e55c..cba99cf 100644 --- a/src/log_event.jl +++ b/src/log_event.jl @@ -28,3 +28,48 @@ function Base.print(io::IO, log_events::Vector{LogEvent}) println(io, event) end end + + +""" + log_events(log_group, log_stream) -> Union{Vector{LogEvent}, Nothing} + +Fetches the CloudWatch log from the specified log group and stream as a `Vector` of +`LogEvent`s. If the log stream does not exist then `nothing` will be returned. +""" +function log_events(log_group::AbstractString, log_stream::AbstractString) + events = LogEvent[] + + curr_token = nothing + next_token = nothing + + # We've hit the end of the stream if the next token matches the current one. + while next_token != curr_token || next_token === nothing + response = try + @mock get_log_events( + logGroupName=log_group, + logStreamName=log_stream, + nextToken=next_token, + ) + catch e + # The specified log stream does not exist. Specifically, this can occur when + # a batch job has a reference to a log stream but the stream has not yet been + # created. + if ( + e isa AWSException && + e.cause.status == 400 && + e.info["message"] == "The specified log stream does not exist." + ) + return nothing + end + + rethrow() + end + + append!(events, convert.(LogEvent, response["events"])) + + curr_token = next_token + next_token = response["nextForwardToken"] + end + + return events +end diff --git a/test/batch_job.jl b/test/batch_job.jl index f11f85b..aec8490 100644 --- a/test/batch_job.jl +++ b/test/batch_job.jl @@ -10,57 +10,5 @@ @test log_events(job, Nothing) === nothing end end - - @testset "Stream DNE" begin - # The AWS Batch job references a log stream but the stream has not yet been - # created. - dne_exception = AWSException( - HTTP.StatusError( - 400, - "", - "", - HTTP.Messages.Response( - 400, - Dict("Content-Type" => "application/x-amz-json-1.1"); - body="""{"__type":"ResourceNotFoundException","message":"The specified log stream does not exist."}""" - ) - ) - ) - - patches = log_events_patches(events=() -> throw(dne_exception)) - apply(patches) do - @test log_events(job, Nothing) === nothing # TODO: Suppress "Fetching log events from" - end - end - - @testset "Stream with no events" begin - patches = log_events_patches(events=[]) - apply(patches) do - @test log_events(job, Nothing) == LogEvent[] - end - end - - @testset "Stream with events" begin - events = [ - Dict( - "eventId" => "0" ^ 56, - "ingestionTime" => 1573672813145, - "timestamp" => 1573672813145, - "message" => "hello world!", - ) - ] - patches = log_events_patches(events=events) - - apply(patches) do - @test log_events(job, Nothing) == [ - LogEvent( - "0" ^ 56, - DateTime(2019, 11, 13, 19, 20, 13, 145), - DateTime(2019, 11, 13, 19, 20, 13, 145), - "hello world!", - ) - ] - end - end end end diff --git a/test/log_event.jl b/test/log_event.jl index c9ad6df..2f2e218 100644 --- a/test/log_event.jl +++ b/test/log_event.jl @@ -34,3 +34,55 @@ """ end end + +@testset "log_events" begin + @testset "Stream DNE" begin + dne_exception = AWSException( + HTTP.StatusError( + 400, + "", + "", + HTTP.Messages.Response( + 400, + Dict("Content-Type" => "application/x-amz-json-1.1"); + body="""{"__type":"ResourceNotFoundException","message":"The specified log stream does not exist."}""" + ) + ) + ) + + patches = log_events_patches(exception=dne_exception) + apply(patches) do + @test log_events("group", "dne-stream") === nothing # TODO: Suppress "Fetching log events from" + end + end + + @testset "Stream with no events" begin + patches = log_events_patches(events=[]) + apply(patches) do + @test log_events("group", "stream") == LogEvent[] + end + end + + @testset "Stream with events" begin + events = [ + Dict( + "eventId" => "0" ^ 56, + "ingestionTime" => 1573672813145, + "timestamp" => 1573672813145, + "message" => "hello world!", + ) + ] + patches = log_events_patches(events=events) + + apply(patches) do + @test log_events("group", "stream") == [ + LogEvent( + "0" ^ 56, + DateTime(2019, 11, 13, 19, 20, 13, 145), + DateTime(2019, 11, 13, 19, 20, 13, 145), + "hello world!", + ) + ] + end + end +end diff --git a/test/mock.jl b/test/mock.jl index 75871c6..d7a06ed 100644 --- a/test/mock.jl +++ b/test/mock.jl @@ -128,21 +128,27 @@ function describe_job_queues_patch(output::OrderedDict) describe_job_queues_patch([output]) end -function log_events_patches(; log_stream_name="mock_stream", events=[]) +function log_events_patches(; log_stream_name="mock_stream", events=[], exception=nothing) job_descriptions = if log_stream_name === nothing Dict("jobs" => [Dict("container" => Dict())]) else Dict("jobs" => [Dict("container" => Dict("logStreamName" => log_stream_name))]) end - log_events_function = if events isa AbstractVector - () -> Dict("events" => events) + get_log_events_patch = if exception !== nothing + @patch get_log_events(; kwargs...) = throw(exception) else - events + @patch function get_log_events(; kwargs...) + if get(kwargs, :nextToken, nothing) === nothing + Dict("events" => events, "nextForwardToken" => "0") + else + Dict("events" => [], "nextForwardToken" => "0") + end + end end return [ @patch describe_jobs(args...; kwargs...) = job_descriptions - @patch get_log_events(; kwargs...) = log_events_function() + get_log_events_patch ] end From a28d3e0cd1c42263feff1a6d3c49974be776a44c Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Fri, 6 Dec 2019 10:54:35 -0600 Subject: [PATCH 087/110] Remove deprecated log_events method --- src/batch_job.jl | 73 +++++++++++------------------------------------ test/batch_job.jl | 2 +- test/runtests.jl | 6 ++-- 3 files changed, 21 insertions(+), 60 deletions(-) diff --git a/src/batch_job.jl b/src/batch_job.jl index 4a87762..20c6028 100644 --- a/src/batch_job.jl +++ b/src/batch_job.jl @@ -204,68 +204,29 @@ function Base.wait( end -if @__VERSION__() < v"1.4" - @doc """ - log_events(job::BatchJob, Nothing) -> Union{Vector{LogEvent}, Nothing} - - Fetches the logStreamName, fetches the CloudWatch logs and returns a vector of log events. - If the log stream does not currently exist then `nothing` is returned. - - NOTES: - - The `logStreamName` isn't available until the job is RUNNING, so you may want to use - `wait(job)` or `wait(job, [AWSBatch.SUCCEEDED])` prior to calling this function. - """ - function log_events( - job::BatchJob, - stream_missing_return_type::Union{Type{Vector{LogEvent}},Type{Nothing}}=Vector{LogEvent}, - ) - if stream_missing_return_type !== Nothing - Base.depwarn( - "Non-existent log streams will return `nothing` instead of `LogEvent[]` " * - "in the future. Use `log_events(job, Nothing)` to adopt the new behaviour.", - :log_events, - ) - end - - job_details = describe(job) +""" + log_events(job::BatchJob) -> Union{Vector{LogEvent}, Nothing} - if haskey(job_details["container"], "logStreamName") - stream = job_details["container"]["logStreamName"] - else - return stream_missing_return_type() - end +Fetches the logStreamName, fetches the CloudWatch logs and returns a vector of log events. +If the log stream does not currently exist then `nothing` is returned. - info(logger, "Fetching log events from $stream") - events = log_events("/aws/batch/job", stream) +NOTES: +- The `logStreamName` isn't available until the job is RUNNING, so you may want to use + `wait(job)` or `wait(job, [AWSBatch.SUCCEEDED])` prior to calling this function. +""" +function log_events(job::BatchJob) + job_details = describe(job) - return events !== nothing ? events : stream_missing_return_type() + if haskey(job_details["container"], "logStreamName") + stream = job_details["container"]["logStreamName"] + else + return nothing end -else - @doc """ - log_events(job::BatchJob) -> Union{Vector{LogEvent}, Nothing} - - Fetches the logStreamName, fetches the CloudWatch logs and returns a vector of log events. - If the log stream does not currently exist then `nothing` is returned. - - NOTES: - - The `logStreamName` isn't available until the job is RUNNING, so you may want to use - `wait(job)` or `wait(job, [AWSBatch.SUCCEEDED])` prior to calling this function. - """ - function log_events(job::BatchJob) - job_details = describe(job) - - if haskey(job_details["container"], "logStreamName") - stream = job_details["container"]["logStreamName"] - else - return nothing - end - - info(logger, "Fetching log events from $stream") - return log_events("/aws/batch/job", stream) - end + info(logger, "Fetching log events from $stream") + return log_events("/aws/batch/job", stream) end -if v"1.4" <= @__VERSION__() < v"1.5" +if @__VERSION__() < v"1.5" @deprecate log_events(job::BatchJob, ::Union{Type{Vector{LogEvent}},Type{Nothing}}) log_events(job) end diff --git a/test/batch_job.jl b/test/batch_job.jl index aec8490..268457d 100644 --- a/test/batch_job.jl +++ b/test/batch_job.jl @@ -7,7 +7,7 @@ # contain a reference to a log stream patches = log_events_patches(log_stream_name=nothing) apply(patches) do - @test log_events(job, Nothing) === nothing + @test log_events(job) === nothing end end end diff --git a/test/runtests.jl b/test/runtests.jl index a348d4a..32709ea 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -228,7 +228,7 @@ include("mock.jl") # Test no log events for the job submitted events = log_events(job) - @test length(events) == 0 + @test events === nothing # Test logs for each individual job that is part of the job array for i in 0:2 @@ -270,7 +270,7 @@ include("mock.jl") deregister(job_definition) events = log_events(job) - @test length(events) == 0 + @test events === nothing end @testset "Failed Job" begin @@ -301,7 +301,7 @@ include("mock.jl") events = log_events(job) # Cannot guarantee this job failure will always have logs - if length(events) > 0 + if events !== nothing @test first(events).message == "ERROR: Testing job failure" end end From f3e1570152eb37466269ce51b800ec3d764e761d Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 10 Dec 2019 17:00:42 +0000 Subject: [PATCH 088/110] Grammar change to log_events docstring --- src/batch_job.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/batch_job.jl b/src/batch_job.jl index 20c6028..7f7da42 100644 --- a/src/batch_job.jl +++ b/src/batch_job.jl @@ -207,7 +207,7 @@ end """ log_events(job::BatchJob) -> Union{Vector{LogEvent}, Nothing} -Fetches the logStreamName, fetches the CloudWatch logs and returns a vector of log events. +Fetches the logStreamName, fetches the CloudWatch logs, and returns a vector of log events. If the log stream does not currently exist then `nothing` is returned. NOTES: From ad6d82a9a93ea662ae4157f8a5bb47e7dcf9175e Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Fri, 6 Dec 2019 11:08:24 -0600 Subject: [PATCH 089/110] Add status_reason accessor --- src/AWSBatch.jl | 1 + src/batch_job.jl | 11 +++++++++++ test/batch_job.jl | 31 +++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+) diff --git a/src/AWSBatch.jl b/src/AWSBatch.jl index 7c7cc3a..c18c075 100644 --- a/src/AWSBatch.jl +++ b/src/AWSBatch.jl @@ -19,6 +19,7 @@ export run_batch, describe, status, + status_reason, wait, log_events, isregistered, diff --git a/src/batch_job.jl b/src/batch_job.jl index 7f7da42..c4c7260 100644 --- a/src/batch_job.jl +++ b/src/batch_job.jl @@ -111,6 +111,17 @@ function status(job::BatchJob)::JobState return parse(JobState, details["status"]) end +""" + status_reason(job::BatchJob) -> Union{String, Nothing} + +A short, human-readable string to provide additional details about the current status of the +job. +""" +function status_reason(job::BatchJob) + details = describe(job) + return get(details, "statusReason", nothing) +end + """ wait( cond::Function, diff --git a/test/batch_job.jl b/test/batch_job.jl index 268457d..e229976 100644 --- a/test/batch_job.jl +++ b/test/batch_job.jl @@ -1,6 +1,37 @@ @testset "BatchJob" begin job = BatchJob("00000000-0000-0000-0000-000000000000") + @testset "status_reason" begin + @testset "not provided" begin + patch = @patch function describe_jobs(; kwargs...) + Dict( + "jobs" => [ + Dict() + ] + ) + end + + apply(patch) do + @test status_reason(job) === nothing + end + end + + @testset "provided" begin + reason = "Essential container in task exited" + patch = @patch function describe_jobs(; kwargs...) + Dict( + "jobs" => [ + Dict("statusReason" => reason) + ] + ) + end + + apply(patch) do + @test status_reason(job) == reason + end + end + end + @testset "log_events" begin @testset "Stream not yet created" begin # When a AWS Batch job is first submitted the description of the job will not From 79391505fc80bf406fdcb468cfd89fe6fa62a698 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Fri, 6 Dec 2019 11:08:32 -0600 Subject: [PATCH 090/110] Set project version to 1.4.0 --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index fc440a2..1d5c137 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "AWSBatch" uuid = "dcae83d4-2881-5875-9d49-e5534165e9c0" authors = ["Invenia Technical Computing"] -version = "1.3.0" +version = "1.4.0" [deps] AWSCore = "4f1ea46c-232b-54a6-9b17-cc2d0f3e6598" From 36dcf38ee4ec8f96d2d94a11e138c2790b52c37e Mon Sep 17 00:00:00 2001 From: Nicole Epp Date: Mon, 24 Feb 2020 12:58:59 -0600 Subject: [PATCH 091/110] Sleep before calling log_events --- test/runtests.jl | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/runtests.jl b/test/runtests.jl index 00d1833..c74b1c7 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -75,6 +75,9 @@ include("mock.jl") @test wait(job, [AWSBatch.SUCCEEDED]; timeout=JOB_TIMEOUT) == true @test status(job) == AWSBatch.SUCCEEDED + # Sleep for 5 seconds because sometimes cloudwatch logs aren't available + # right away + sleep(5) events = log_events(job) @test length(events) == 1 @test first(events).message == "Hello World!" @@ -174,6 +177,9 @@ include("mock.jl") @test status(job) == AWSBatch.SUCCEEDED # Test the default string was overrriden succesfully + # Sleep for 5 seconds because sometimes cloudwatch logs aren't available + # right away + sleep(5) events = log_events(job) @test length(events) == 1 @test first(events).message == "Hello World!" @@ -223,6 +229,9 @@ include("mock.jl") @test job_details["arrayProperties"]["size"] == 3 # Test no log events for the job submitted + # Sleep for 5 seconds because sometimes cloudwatch logs aren't available + # right away + sleep(5) events = log_events(job) @test length(events) == 0 @@ -265,6 +274,9 @@ include("mock.jl") deregister(job_definition) + # Sleep for 5 seconds because sometimes cloudwatch logs aren't available + # right away + sleep(5) events = log_events(job) @test length(events) == 0 end @@ -294,6 +306,9 @@ include("mock.jl") deregister(job_definition) + # Sleep for 5 seconds because sometimes cloudwatch logs aren't available + # right away + sleep(5) events = log_events(job) # Cannot guarantee this job failure will always have logs From 0f736c26e1ca97e5df07cfe4f63a95af48405978 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Thu, 12 Mar 2020 08:21:19 -0500 Subject: [PATCH 092/110] Address cloudspy deprecations --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0662f67..39f744c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -38,10 +38,10 @@ variables: script: - aws cloudformation validate-template --template-body file://test/batch.yml - | - aws-create-stack \ + aws-deploy-stack \ --role-arn arn:aws:iam::${ACCOUNT_ID}:role/CloudFormationAdmin \ - --stackname $STACK_NAME \ - --template-body ./test/batch.yml \ + --stack-name $STACK_NAME \ + --template-file ./test/batch.yml \ --wait \ --params CIRoleArn=arn:aws:iam::${ACCOUNT_ID}:role/GitLabCIRunnerRole From 8ecb507d335275cc4b0b7486f9a65c632e461022 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Thu, 12 Mar 2020 08:46:11 -0500 Subject: [PATCH 093/110] GitLab CI before_script available for extension --- .gitlab-ci.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 39f744c..9354214 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -53,13 +53,10 @@ variables: - ci-account variables: TESTS: "batch" # Runs the online tests against AWS - script: + before_script: - *setup - assume_test_role - # Execute online tests - - source julia-ci export - - AWS_STACKNAME=$STACK_NAME ./julia-ci test - - ./julia-ci coverage + - export AWS_STACKNAME=$STACK_NAME # Needed for tests extends: .test_shell "1.0 (AWS Batch)": From c56b3f73f6727cbc5618c1cf62525c33b3f81f8d Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 17 Mar 2020 13:32:05 -0500 Subject: [PATCH 094/110] Refactor "Job Timed Out" test The online "Job Timed Out" test was testing the timeout behaviour of our `wait` method and not actually the AWS Batch job timeout (https://docs.aws.amazon.com/batch/latest/userguide/job_timeouts.html). This meant it could be implemented with Mocking instead of an online test. The reason this refactoring occurred is that since we queried for log events almost immediately after submitting the job. This resulted in a chance that the log event stream could be created which would change `events` from `nothing` to `LogEvent[]` and fail the test. --- test/batch_job.jl | 60 +++++++++++++++++++++++++++++++++++++++++++++++ test/runtests.jl | 33 +------------------------- 2 files changed, 61 insertions(+), 32 deletions(-) diff --git a/test/batch_job.jl b/test/batch_job.jl index e229976..d29da52 100644 --- a/test/batch_job.jl +++ b/test/batch_job.jl @@ -42,4 +42,64 @@ end end end + + @testset "wait" begin + # Generate a patch which returns the next status each time it is requested + function status_patch(states) + index = 1 + + return @patch function describe_jobs(; kwargs...) + json = Dict( + "jobs" => [ + Dict("status" => states[index]) + ] + ) + + if index < length(states) + index += 1 + end + + return json + end + end + + @testset "success" begin + # Encounter all states possible for a successful job + states = ["SUBMITTED", "PENDING", "RUNNABLE", "STARTING", "RUNNING", "SUCCEEDED"] + apply(status_patch(states)) do + @test_log logger "info" r"^[\d-]+ status \w+" begin + @test wait(state -> state < AWSBatch.SUCCEEDED, job; delay=0.1) == true + end + @test status(job) == AWSBatch.SUCCEEDED + end + end + + @testset "failed" begin + # Encounter all states possible for a failed job + states = ["SUBMITTED", "PENDING", "RUNNABLE", "STARTING", "RUNNING", "FAILED"] + apply(status_patch(states)) do + @test_log logger "info" r"^[\d-]+ status \w+" begin + @test wait(state -> state < AWSBatch.SUCCEEDED, job; delay=0.1) == true + end + @test status(job) == AWSBatch.FAILED + end + end + + @testset "timeout" begin + apply(status_patch(["SUBMITTED", "RUNNING", "SUCCEEDED"])) do + started = time() + @test_nolog logger "info" r".*" begin + @test_throws BatchJobError wait( + state -> state < AWSBatch.SUCCEEDED, + job; + delay=0.1, + timeout=0 + ) + end + duration = time() - started + @test status(job) != AWSBatch.SUCCEEDED # Requires a minimum of 3 states + @test duration < 1 # Less than 1 second + end + end + end end diff --git a/test/runtests.jl b/test/runtests.jl index e053153..9983456 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -8,6 +8,7 @@ using AWSTools.EC2: instance_region using Dates using HTTP: HTTP using Memento +using Memento.TestUtils: @test_log, @test_nolog using Mocking using Test @@ -253,38 +254,6 @@ include("mock.jl") deregister(job_definition) end - @testset "Job Timed Out" begin - info(logger, "Testing job timeout") - - job = run_batch(; - name = "aws-batch-timeout-job-test", - definition = "aws-bath-timeout-job-test", - queue = STACK["JobQueueArn"], - image = JULIA_BAKED_IMAGE, - vcpus = 1, - memory = 1024, - role = STACK["JobRoleArn"], - cmd = `sleep 60`, - ) - - job_definition = JobDefinition(job) - @test isregistered(job_definition) == true - - @test_throws BatchJobError wait( - state -> state < AWSBatch.SUCCEEDED, - job; - timeout=0 - ) - - deregister(job_definition) - - # Sleep for 5 seconds because sometimes cloudwatch logs aren't available - # right away - sleep(5) - events = log_events(job) - @test events === nothing - end - @testset "Failed Job" begin info(logger, "Testing job failure") From e9ca8d26b4453c7ecc5bf1b2ce8f027e2f5b4d43 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 18 Mar 2020 08:36:42 -0500 Subject: [PATCH 095/110] Run online tests on Julia 1.3 and 1.4 --- .gitlab-ci.yml | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9354214..f6edb32 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -59,19 +59,14 @@ variables: - export AWS_STACKNAME=$STACK_NAME # Needed for tests extends: .test_shell -"1.0 (AWS Batch)": +"1.3 (AWS Batch)": variables: - JULIA_VERSION: "1.0" + JULIA_VERSION: "1.3" extends: .test_batch -"1.1 (AWS Batch)": +"1.4 (AWS Batch)": variables: - JULIA_VERSION: "1.1" - extends: .test_batch - -"1.2 (AWS Batch)": - variables: - JULIA_VERSION: "1.2" + JULIA_VERSION: "1.4" extends: .test_batch "Nightly (AWS Batch)": From 8dee2dfec22151ddb158bbb93b91545d850eca1c Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 17 Mar 2020 14:52:19 -0500 Subject: [PATCH 096/110] Wait for log events for certain tests It appears that even when a job status becomes SUCCEEDED or FAILED the output from the job may not have been written to the log stream yet. As such we'll wait a duration for the logs to become available. Since we don't precisely know what is going on here we'll also emit some warnings to assist in further investigations. --- test/runtests.jl | 65 ++++++++++++++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 24 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 9983456..d1c84e7 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -22,6 +22,7 @@ const TESTS = strip.(split(get(ENV, "TESTS", "local"), r"\s*,\s*")) const AWS_STACKNAME = get(ENV, "AWS_STACKNAME", "") const STACK = !isempty(AWS_STACKNAME) ? stack_output(AWS_STACKNAME) : Dict() const JOB_TIMEOUT = 900 +const LOG_TIMEOUT = 30 const JULIA_BAKED_IMAGE = let output = read(`git ls-remote --tags https://github.com/JuliaLang/julia`, String) @@ -37,6 +38,39 @@ Memento.config!("debug"; fmt="[{level} | {name}]: {msg}") const logger = getlogger(AWSBatch) setlevel!(logger, "info") + +# We've been having issues with the log stream being created but no log events are present. +# - https://gitlab.invenia.ca/invenia/AWSBatch.jl/issues/28 +# - https://gitlab.invenia.ca/invenia/AWSBatch.jl/issues/30 +# +# This function allows for us to wait if logs are not present but avoids blocking when +# logs are ready. +# +# Note: The timeout duration expects the job has reached the SUCCEEDED or FAILED state and +# is not expected to last long enough for a job to complete running. +function wait_for_log_events(job::BatchJob) + events = nothing + + # Convert to Float64 until this is merged and we no longer use versions of Julia + # without PR (probably Julia 1.5+): https://github.com/JuliaLang/julia/pull/35103 + timedwait(Float64(LOG_TIMEOUT); pollint=Float64(5)) do + events = log_events(job) + + # Note: These warnings should assist in determining the special circumstances + # the log events not being present. Eventually warnings should be removed. + if events === nothing + notice(logger, "Log stream for $(job.id) does not exist") + elseif isempty(events) + notice(logger, "No log events for $(job.id)") + end + + # Wait for log stream to exist and contain at least one event + events !== nothing && !isempty(events) + end + + return events +end + include("mock.jl") @testset "AWSBatch.jl" begin @@ -80,10 +114,7 @@ include("mock.jl") @test wait(job, [AWSBatch.SUCCEEDED]; timeout=JOB_TIMEOUT) == true @test status(job) == AWSBatch.SUCCEEDED - # Sleep for 5 seconds because sometimes cloudwatch logs aren't available - # right away - sleep(5) - events = log_events(job) + events = wait_for_log_events(job) @test length(events) == 1 @test first(events).message == "Hello World!" @@ -181,11 +212,7 @@ include("mock.jl") @test wait(state -> state < AWSBatch.SUCCEEDED, job; timeout=JOB_TIMEOUT) @test status(job) == AWSBatch.SUCCEEDED - # Test the default string was overrriden succesfully - # Sleep for 5 seconds because sometimes cloudwatch logs aren't available - # right away - sleep(5) - events = log_events(job) + events = wait_for_log_events(job) @test length(events) == 1 @test first(events).message == "Hello World!" @@ -233,17 +260,14 @@ include("mock.jl") @test job_details["arrayProperties"]["statusSummary"] == status_summary @test job_details["arrayProperties"]["size"] == 3 - # Test no log events for the job submitted - # Sleep for 5 seconds because sometimes cloudwatch logs aren't available - # right away - sleep(5) + # No log stream will exist for the parent job events = log_events(job) @test events === nothing # Test logs for each individual job that is part of the job array for i in 0:2 job_id = "$(job.id):$i" - events = log_events(BatchJob(job_id)) + events = wait_for_log_events(BatchJob(job_id)) @test length(events) == 1 @test first(events).message == "Hello World!" @@ -277,17 +301,10 @@ include("mock.jl") timeout=JOB_TIMEOUT ) - deregister(job_definition) + events = wait_for_log_events(job) + @test first(events).message == "ERROR: Testing job failure" - # Sleep for 5 seconds because sometimes cloudwatch logs aren't available - # right away - sleep(5) - events = log_events(job) - - # Cannot guarantee this job failure will always have logs - if events !== nothing - @test first(events).message == "ERROR: Testing job failure" - end + deregister(job_definition) end end else From 531c6bedda61ec8d65666239d1c857e7f7b5c24b Mon Sep 17 00:00:00 2001 From: morris25 Date: Fri, 27 Mar 2020 11:07:48 -0500 Subject: [PATCH 097/110] Update AWSSDK version --- Project.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index 1d5c137..22ee29b 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "AWSBatch" uuid = "dcae83d4-2881-5875-9d49-e5534165e9c0" authors = ["Invenia Technical Computing"] -version = "1.4.0" +version = "1.4.1" [deps] AWSCore = "4f1ea46c-232b-54a6-9b17-cc2d0f3e6598" @@ -16,7 +16,7 @@ Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" [compat] AWSCore = "0.5.2, 0.6" -AWSSDK = "0.2, 0.3, 0.4" +AWSSDK = "0.2, 0.3, 0.4, 0.5" AWSTools = "0.3.1, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1" AutoHashEquals = "0.2.0" HTTP = "0.8.1" From 03257d7cecc7572f28caef29200231358b9e0051 Mon Sep 17 00:00:00 2001 From: Mary Jo Ramos Date: Mon, 6 Apr 2020 23:07:16 +0000 Subject: [PATCH 098/110] Add Memento v1 --- Project.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index 22ee29b..5fb255c 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "AWSBatch" uuid = "dcae83d4-2881-5875-9d49-e5534165e9c0" authors = ["Invenia Technical Computing"] -version = "1.4.1" +version = "1.4.2" [deps] AWSCore = "4f1ea46c-232b-54a6-9b17-cc2d0f3e6598" @@ -20,7 +20,7 @@ AWSSDK = "0.2, 0.3, 0.4, 0.5" AWSTools = "0.3.1, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1" AutoHashEquals = "0.2.0" HTTP = "0.8.1" -Memento = "0.7, 0.8, 0.9, 0.10, 0.11, 0.12" +Memento = "0.7, 0.8, 0.9, 0.10, 0.11, 0.12, 0.13, 1" Mocking = "0.7" julia = "1" From f43db04823e2960619541e56cd2fb988ce81d111 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 2 Jun 2020 08:18:58 -0500 Subject: [PATCH 099/110] Tests require command-line git --- .gitlab-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f6edb32..61dae7a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -45,6 +45,12 @@ variables: --wait \ --params CIRoleArn=arn:aws:iam::${ACCOUNT_ID}:role/GitLabCIRunnerRole +.test: + before_script: + - echo "$common_functions" > common && source common + # `git` needed to determine latest release of Julia + - package_install git + .test_batch: stage: test tags: From fe48071be31d32858f3a20cf9532201beda6591b Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 21 Jul 2020 15:59:53 -0500 Subject: [PATCH 100/110] Moved gitlab-ci-helper --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 61dae7a..99f0182 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,6 @@ --- include: - - project: infrastructure/gitlab-ci-helper + - project: invenia/gitlab-ci-helper file: /templates/julia.yml variables: From 5d115411a0ce84e756b6db05bacde35bb5c07591 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Fri, 24 Jul 2020 09:30:56 -0500 Subject: [PATCH 101/110] Update to use ci-init --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 99f0182..62379d9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,7 +10,7 @@ variables: # Variables that require shell execution or depend on a variable that does .setup: &setup | - echo "$common_functions" > common && source common + echo "$ci_init" > ci_init && source ci_init && rm ci_init STACK_NAME=$(stack_name $STACK_NAME_PREFIX) ACCOUNT_ID=$(aws_account_id) @@ -47,7 +47,7 @@ variables: .test: before_script: - - echo "$common_functions" > common && source common + - echo "$ci_init" > ci_init && source ci_init && rm ci_init # `git` needed to determine latest release of Julia - package_install git From 0104ea9333f880794d319871ebbdda5446d02def Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Fri, 9 Oct 2020 10:23:18 -0500 Subject: [PATCH 102/110] Workaround cloudspy image being x86 specific --- .gitlab-ci.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 62379d9..b93f0c8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -33,6 +33,9 @@ variables: tags: - docker-ci - ci-account + # TODO: Needed until cloudspy supports multi-arch images: + # https://gitlab.invenia.ca/invenia/cloudspy/-/issues/85 + - x86 before_script: - *setup script: @@ -82,10 +85,13 @@ variables: extends: .test_batch .delete: &delete + image: $CLOUDSPY_IMAGE tags: - docker-ci - ci-account - image: $CLOUDSPY_IMAGE + # TODO: Needed until cloudspy supports multi-arch images: + # https://gitlab.invenia.ca/invenia/cloudspy/-/issues/85 + - x86 before_script: - *setup script: From 6a2e833d22be0a2cca084a5f8f5d1952c49f8308 Mon Sep 17 00:00:00 2001 From: Matt Brzezinski Date: Mon, 7 Dec 2020 12:26:00 -0600 Subject: [PATCH 103/110] Converted CI to use GHA --- .github/workflows/CI.yml | 68 ++++++++++++ bors.toml | 7 ++ src/AWSBatch.jl | 33 +----- src/batch_job.jl | 9 -- src/deprecated.jl | 1 - src/exceptions.jl | 10 ++ src/job_queue.jl | 5 +- test/{ => resources}/batch.yml | 186 +++++++-------------------------- 8 files changed, 127 insertions(+), 192 deletions(-) create mode 100644 .github/workflows/CI.yml create mode 100644 bors.toml delete mode 100644 src/deprecated.jl create mode 100644 src/exceptions.jl rename test/{ => resources}/batch.yml (61%) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml new file mode 100644 index 0000000..409c523 --- /dev/null +++ b/.github/workflows/CI.yml @@ -0,0 +1,68 @@ +name: CI +# Run on master, any tag or any pull request +on: + push: + branches: + - master + - staging + - trying + tags: '*' + schedule: + - cron: '0 2 * * *' # Daily at 2 a.m. UTC +jobs: + test: + name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} + runs-on: ${{ matrix.os }} + continue-on-error: ${{ matrix.version == 'nightly' }} + strategy: + fail-fast: false + matrix: + version: + - 1 + os: + - ubuntu-latest + - macOS-latest + arch: + - x64 + include: + # Add a 1.0 job just to make sure we still support it + - os: ubuntu-latest + version: 1.0.5 + arch: x64 + # Add a 1.3 job because that's what Invenia actually uses + - os: ubuntu-latest + version: 1.3 + arch: x64 + steps: + - uses: actions/checkout@v2 + - uses: julia-actions/setup-julia@v1 + with: + version: ${{ matrix.version }} + arch: ${{ matrix.arch }} + - uses: actions/cache@v1 + env: + cache-name: cache-artifacts + with: + path: ~/.julia/artifacts + key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} + restore-keys: | + ${{ runner.os }}-test-${{ env.cache-name }}- + ${{ runner.os }}-test- + ${{ runner.os }}- + - uses: julia-actions/julia-buildpkg@latest + - run: | + git config --global user.name Tester + git config --global user.email te@st.er + - env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + uses: julia-actions/julia-runtest@latest + - name: Notify slack fail + if: failure() + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + uses: voxmedia/github-action-slack-notify-build@v1 + with: + channel: nightly-dev + status: FAILED + color: danger diff --git a/bors.toml b/bors.toml new file mode 100644 index 0000000..d0da08f --- /dev/null +++ b/bors.toml @@ -0,0 +1,7 @@ +status = [ + "Julia 1.0.5 - ubuntu-latest - x64", + "Julia 1.3 - ubuntu-latest - x64", + "Julia 1 - macOS-latest - x64", + "Julia 1 - ubuntu-latest - x64", +] +delete-merged-branches = true diff --git a/src/AWSBatch.jl b/src/AWSBatch.jl index c18c075..9369f7f 100644 --- a/src/AWSBatch.jl +++ b/src/AWSBatch.jl @@ -10,23 +10,9 @@ using Dates using Memento using Mocking -export - BatchJob, - ComputeEnvironment, - JobQueue, - JobDefinition, - JobState, - run_batch, - describe, - status, - status_reason, - wait, - log_events, - isregistered, - register, - deregister, - BatchEnvironmentError, - BatchJobError +export BatchJob, ComputeEnvironment, BatchEnvironmentError, BatchJobError +export JobQueue, JobDefinition, JobState +export run_batch, describe, status, status_reason, wait, log_events, isregistered, register, deregister const logger = getlogger(@__MODULE__) @@ -35,6 +21,7 @@ const logger = getlogger(@__MODULE__) __init__() = Memento.register(logger) +include("exceptions.jl") include("version.jl") include("log_event.jl") include("compute_environment.jl") @@ -44,16 +31,6 @@ include("job_definition.jl") include("batch_job.jl") -struct BatchEnvironmentError <: Exception - message::String -end - -function Base.showerror(io::IO, e::BatchEnvironmentError) - print(io, "BatchEnvironmentError: ") - print(io, e.message) -end - - """ run_batch(; name::AbstractString="", @@ -224,6 +201,4 @@ function run_batch(; ) end -include("deprecated.jl") - end # AWSBatch diff --git a/src/batch_job.jl b/src/batch_job.jl index c4c7260..a0ae16c 100644 --- a/src/batch_job.jl +++ b/src/batch_job.jl @@ -1,15 +1,6 @@ using AWSSDK.Batch: describe_jobs, submit_job using AWSSDK.CloudWatchLogs: get_log_events -struct BatchJobError <: Exception - job_id::AbstractString - message::String -end - -function Base.showerror(io::IO, e::BatchJobError) - print(io, "BatchJobError: ") - print(io, e.message) -end """ BatchJob diff --git a/src/deprecated.jl b/src/deprecated.jl deleted file mode 100644 index 6d5b39d..0000000 --- a/src/deprecated.jl +++ /dev/null @@ -1 +0,0 @@ -using Base: @deprecate diff --git a/src/exceptions.jl b/src/exceptions.jl new file mode 100644 index 0000000..651a5cf --- /dev/null +++ b/src/exceptions.jl @@ -0,0 +1,10 @@ +struct BatchEnvironmentError <: Exception + message::String +end +Base.showerror(io::IO, e::BatchEnvironmentError) = print(io, "BatchEnvironmentError: ", e.message) + +struct BatchJobError <: Exception + job_id::AbstractString + message::String +end +Base.showerror(io::IO, e::BatchJobError) = print(io, "BatchJobError: $(e.job_id)", e.message) diff --git a/src/job_queue.jl b/src/job_queue.jl index 322295b..dc6e8bf 100644 --- a/src/job_queue.jl +++ b/src/job_queue.jl @@ -11,10 +11,11 @@ struct JobQueue end Base.:(==)(a::JobQueue, b::JobQueue) = a.arn == b.arn - describe(queue::JobQueue) = describe_job_queue(queue) +describe_job_queue(queue::JobQueue) = describe_job_queue(queue.arn) max_vcpus(queue::JobQueue) = sum(max_vcpus(ce) for ce in compute_environments(queue)) + function compute_environments(queue::JobQueue) ce_order = describe(queue)["computeEnvironmentOrder"] @@ -27,13 +28,13 @@ function compute_environments(queue::JobQueue) return compute_envs end + function job_queue_arn(queue::AbstractString) startswith(queue, "arn:") && return queue json = describe_job_queue(queue) isempty(json) ? nothing : json["jobQueueArn"] end -describe_job_queue(queue::JobQueue) = describe_job_queue(queue.arn) function describe_job_queue(queue::AbstractString)::OrderedDict json = @mock describe_job_queues(Dict("jobQueues" => [queue])) diff --git a/test/batch.yml b/test/resources/batch.yml similarity index 61% rename from test/batch.yml rename to test/resources/batch.yml index 250ad6a..a6b8525 100644 --- a/test/batch.yml +++ b/test/resources/batch.yml @@ -28,6 +28,9 @@ Description: >- # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/parameters-section-structure.html#aws-specific-parameter-types Parameters: + PublicCIUser: + Description: User which can assume the testing role + Type: String VPCCidrBlock: # https://docs.aws.amazon.com/AmazonVPC/latest/UserGuide/VPC_Subnets.html#VPC_Sizing Description: >- @@ -54,16 +57,9 @@ Parameters: AllowedValues: - on-demand - spot - CIRoleArn: - Description: The role ARN used when executing GitLab CI test stage jobs. - Type: String - Default: "" - AllowedPattern: "|arn:aws:iam::\\d{12}:role/[^/]+" Conditions: OnDemandComputeEnvironment: !Equals [!Ref ProvisioningModel, on-demand] - Testing: !Not [!Equals [!Ref CIRoleArn, ""]] - IsCI: !Equals [!Ref "AWS::AccountId", 813030647716] Resources: ComputeEnvironment: @@ -96,52 +92,6 @@ Resources: ComputeEnvironmentOrder: - Order: 1 ComputeEnvironment: !Ref ComputeEnvironment - - IamInstanceProfile: - Type: AWS::IAM::InstanceProfile - Properties: - Roles: - - !Ref EcsInstanceRole - EcsInstanceRole: - Type: AWS::IAM::Role - Properties: - AssumeRolePolicyDocument: - Version: 2008-10-17 - Statement: - - Sid: '' - Effect: Allow - Principal: - Service: ec2.amazonaws.com - Action: sts:AssumeRole - ManagedPolicyArns: - - arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role - BatchServiceRole: - Type: AWS::IAM::Role - Properties: - AssumeRolePolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Principal: - Service: batch.amazonaws.com - Action: sts:AssumeRole - ManagedPolicyArns: - - arn:aws:iam::aws:policy/service-role/AWSBatchServiceRole - - # http://docs.aws.amazon.com/batch/latest/userguide/spot_fleet_IAM_role.html - BatchSpotFleetRole: - Type: AWS::IAM::Role - Properties: - AssumeRolePolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Principal: - Service: spotfleet.amazonaws.com - Action: sts:AssumeRole - ManagedPolicyArns: - - arn:aws:iam::aws:policy/service-role/AmazonEC2SpotFleetTaggingRole - SecurityGroup: Type: AWS::EC2::SecurityGroup Properties: @@ -149,7 +99,6 @@ Resources: VpcId: !Ref VPC VPC: Type: AWS::EC2::VPC - DependsOn: DeleteStackRole Properties: CidrBlock: !Ref VPCCidrBlock Subnet: @@ -182,96 +131,26 @@ Resources: Properties: VpcId: !Ref VPC InternetGatewayId: !Ref InternetGateway - - #============================================# -# Automatically delete stacks in the CI account that haven't been updated recently. - - DeleteStackRole: + IamInstanceProfile: + Type: AWS::IAM::InstanceProfile + Properties: + Roles: + - !Ref EcsInstanceRole + EcsInstanceRole: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: + Version: 2008-10-17 Statement: - - Effect: Allow + - Sid: '' + Effect: Allow Principal: - Service: lambda.amazonaws.com + Service: ec2.amazonaws.com Action: sts:AssumeRole ManagedPolicyArns: - - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole - Policies: - - PolicyName: DeleteStackPolicy - PolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Action: - - cloudformation:DescribeStacks - - cloudformation:DeleteStack - Resource: !Ref AWS::StackId - - Effect: Allow - Action: - - logs:* - - ec2:* - - iam:* - - lambda:* - - batch:* - - events:* - Resource: "*" - - DeleteStackLambda: - Type: AWS::Lambda::Function - Properties: - Handler: index.lambda_handler - Runtime: python3.6 - Timeout: 60 - Role: !GetAtt DeleteStackRole.Arn - Environment: - Variables: - STACK_NAME: !Ref AWS::StackName - WAIT_PERIOD: 3 # hours - Code: - ZipFile: | - from datetime import datetime, timedelta - import boto3 - import os - STACK_NAME = os.environ["STACK_NAME"] - WAIT_PERIOD = os.environ["WAIT_PERIOD"] - - def lambda_handler(event, context): - cfn = boto3.resource('cloudformation') - stack = cfn.Stack(STACK_NAME) - wait_period = timedelta(hours=int(WAIT_PERIOD)) - if stack.last_updated_time is None: - last_update = stack.creation_time - else: - last_update = stack.last_updated_time - tzaware_now = datetime.now(last_update.tzinfo) - if tzaware_now >= last_update + wait_period: - print(f"Deleting stack after {WAIT_PERIOD} hours without updates.") - stack.delete() - - DeleteStackPermission: - Condition: IsCI - Type: AWS::Lambda::Permission - Properties: - FunctionName: !Ref DeleteStackLambda - Action: lambda:InvokeFunction - Principal: events.amazonaws.com - SourceArn: !GetAtt DeleteStackRule.Arn - - DeleteStackRule: - Condition: IsCI - Type: AWS::Events::Rule - Properties: - Description: Trigger deletion if stack has not been updated recently. - ScheduleExpression: cron(0 * * * ? *) # Run every hour - State: ENABLED - Targets: - - Id: DeleteStackLambda - Arn: !GetAtt DeleteStackLambda.Arn - - #============================================# - - JobRole: + - arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role + BatchServiceRole: + # http://docs.aws.amazon.com/batch/latest/userguide/spot_fleet_IAM_role.html Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: @@ -279,32 +158,42 @@ Resources: Statement: - Effect: Allow Principal: - Service: ecs-tasks.amazonaws.com # Note: Shouldn't be batch.amazonaws.com + Service: batch.amazonaws.com Action: sts:AssumeRole - - TestRole: + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AWSBatchServiceRole + BatchSpotFleetRole: Type: AWS::IAM::Role - Condition: Testing Properties: - RoleName: !Sub ${AWS::StackName}-TestRole AssumeRolePolicyDocument: Version: 2012-10-17 Statement: - Effect: Allow Principal: - AWS: !Ref CIRoleArn + Service: spotfleet.amazonaws.com Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AmazonEC2SpotFleetTaggingRole + JobRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com # Note: Shouldn't be batch.amazonaws.com + Action: sts:AssumeRole - # Necessary permissions for running the AWSBatch online tests. TestPolicy: Type: AWS::IAM::Policy - Condition: Testing Properties: PolicyName: TestPolicy + Users: + - !Ref PublicCIUser PolicyDocument: Version: 2012-10-17 Statement: - # Permissions used by AWSBatch.jl - Effect: Allow Action: - batch:RegisterJobDefinition @@ -321,15 +210,10 @@ Resources: - Effect: Allow Action: iam:PassRole Resource: !GetAtt JobRole.Arn - - # Permissions for AWSTools.stack_output - Effect: Allow Action: cloudformation:DescribeStacks Resource: !Sub arn:aws:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/${AWS::StackName}/* - Roles: [!Ref TestRole] Outputs: JobQueueArn: Value: !Ref JobQueue - JobRoleArn: - Value: !GetAtt JobRole.Arn From 767de477de83b53dfcf31f3cb9dd191f574aa5f8 Mon Sep 17 00:00:00 2001 From: Matt Brzezinski Date: Mon, 7 Dec 2020 12:31:55 -0600 Subject: [PATCH 104/110] Moved slack notifications to its own job --- .github/workflows/CI.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 409c523..afad829 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -57,12 +57,18 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} uses: julia-actions/julia-runtest@latest - - name: Notify slack fail - if: failure() - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} - uses: voxmedia/github-action-slack-notify-build@v1 + slack: + name: Notify Slack Failure + needs: test + runs-on: ubuntu-latest + if: github.event == 'schedule' + steps: + - uses: technote-space/workflow-conclusion-action@v2 + - uses: voxmedia/github-action-slack-notify-build@v1 + if: env.WORKFLOW_CONCLUSION == 'failure' with: channel: nightly-dev status: FAILED color: danger + env: + SLACK_BOT_TOKEN: ${{ secrets.DEV_SLACK_BOT_TOKEN }} From fe9364997936d16ef6ed95ccb6bbff8f72873824 Mon Sep 17 00:00:00 2001 From: Matt Brzezinski Date: Wed, 17 Feb 2021 13:53:19 -0600 Subject: [PATCH 105/110] Removed GitLab CI file --- .gitlab-ci.yml | 128 ------------------------------------------------- 1 file changed, 128 deletions(-) delete mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index b93f0c8..0000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,128 +0,0 @@ ---- -include: - - project: invenia/gitlab-ci-helper - file: /templates/julia.yml - -variables: - STACK_NAME_PREFIX: sandbox-awsbatch - CLOUDSPY_IMAGE: 468665244580.dkr.ecr.us-east-1.amazonaws.com/cloudspy - -# Variables that require shell execution or depend on a variable that does -.setup: &setup - | - echo "$ci_init" > ci_init && source ci_init && rm ci_init - STACK_NAME=$(stack_name $STACK_NAME_PREFIX) - ACCOUNT_ID=$(aws_account_id) - -"Setup Environment": - stage: setup - except: - - tags - - master - - /^.+\/.*master$/ # e.g. jh/validate-master - when: always - environment: - name: branch/$CI_COMMIT_REF_SLUG - on_stop: "Delete Environment" - script: - - echo "Setting up environment" - -"Create Stack": - stage: setup - image: $CLOUDSPY_IMAGE - tags: - - docker-ci - - ci-account - # TODO: Needed until cloudspy supports multi-arch images: - # https://gitlab.invenia.ca/invenia/cloudspy/-/issues/85 - - x86 - before_script: - - *setup - script: - - aws cloudformation validate-template --template-body file://test/batch.yml - - | - aws-deploy-stack \ - --role-arn arn:aws:iam::${ACCOUNT_ID}:role/CloudFormationAdmin \ - --stack-name $STACK_NAME \ - --template-file ./test/batch.yml \ - --wait \ - --params CIRoleArn=arn:aws:iam::${ACCOUNT_ID}:role/GitLabCIRunnerRole - -.test: - before_script: - - echo "$ci_init" > ci_init && source ci_init && rm ci_init - # `git` needed to determine latest release of Julia - - package_install git - -.test_batch: - stage: test - tags: - - amzn2 - - docker-build - - ci-account - variables: - TESTS: "batch" # Runs the online tests against AWS - before_script: - - *setup - - assume_test_role - - export AWS_STACKNAME=$STACK_NAME # Needed for tests - extends: .test_shell - -"1.3 (AWS Batch)": - variables: - JULIA_VERSION: "1.3" - extends: .test_batch - -"1.4 (AWS Batch)": - variables: - JULIA_VERSION: "1.4" - extends: .test_batch - -"Nightly (AWS Batch)": - variables: - JULIA_VERSION: "nightly" - allow_failure: true - extends: .test_batch - -.delete: &delete - image: $CLOUDSPY_IMAGE - tags: - - docker-ci - - ci-account - # TODO: Needed until cloudspy supports multi-arch images: - # https://gitlab.invenia.ca/invenia/cloudspy/-/issues/85 - - x86 - before_script: - - *setup - script: - - eval $(aws-stack-outputs $STACK_NAME) - - | - aws cloudformation delete-stack \ - --role-arn arn:aws:iam::${ACCOUNT_ID}:role/CloudFormationAdmin \ - --stack-name $STACK_NAME - - aws cloudformation wait stack-delete-complete --stack-name $STACK_NAME - -"Delete Environment": - stage: teardown - except: - - tags - - master - - /^.+\/.*master$/ - when: manual - environment: - name: branch/$CI_COMMIT_REF_SLUG - action: stop - dependencies: - - "Create Stack" - variables: - GIT_STRATEGY: none # Avoid checking out a branch after deletion - <<: *delete - -"Delete Stack": - stage: teardown - only: - - tags - - master - - /^.+\/.*master$/ - when: always - <<: *delete From 0c8888dd6a788b8b719e51db9aa9653688128c18 Mon Sep 17 00:00:00 2001 From: Matt Brzezinski Date: Wed, 17 Feb 2021 13:57:54 -0600 Subject: [PATCH 106/110] Added suggestions from @fchorney and @nicoleepp --- .github/workflows/CI.yml | 35 +++++++++++++++++++++-------------- bors.toml | 2 +- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index afad829..262022e 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -7,31 +7,30 @@ on: - staging - trying tags: '*' - schedule: + schedule: - cron: '0 2 * * *' # Daily at 2 a.m. UTC jobs: test: name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} runs-on: ${{ matrix.os }} - continue-on-error: ${{ matrix.version == 'nightly' }} strategy: fail-fast: false matrix: version: - - 1 + - "1.0" # LTS + - "1" # Latest release os: - ubuntu-latest - macOS-latest arch: - x64 + - x86 + exclude: + - os: macOS-latest + arch: x86 include: - # Add a 1.0 job just to make sure we still support it - os: ubuntu-latest - version: 1.0.5 - arch: x64 - # Add a 1.3 job because that's what Invenia actually uses - - os: ubuntu-latest - version: 1.3 + version: "1.0" arch: x64 steps: - uses: actions/checkout@v2 @@ -39,7 +38,7 @@ jobs: with: version: ${{ matrix.version }} arch: ${{ matrix.arch }} - - uses: actions/cache@v1 + - uses: actions/cache@v1 env: cache-name: cache-artifacts with: @@ -49,10 +48,18 @@ jobs: ${{ runner.os }}-test-${{ env.cache-name }}- ${{ runner.os }}-test- ${{ runner.os }}- + - uses: actions/cache@v2 + env: + cache-name: cache-artifacts + with: + path: ~/.julia/artifacts + key: ${{ runner.os }}-${{ matrix.arch }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} + restore-keys: | + ${{ runner.os }}-${{ matrix.arch }}-test-${{ env.cache-name }}- + ${{ runner.os }}-${{ matrix.arch }}-test- + ${{ runner.os }}-${{ matrix.arch }}- + ${{ runner.os }}- - uses: julia-actions/julia-buildpkg@latest - - run: | - git config --global user.name Tester - git config --global user.email te@st.er - env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -61,7 +68,7 @@ jobs: name: Notify Slack Failure needs: test runs-on: ubuntu-latest - if: github.event == 'schedule' + if: always() && github.event_name == 'schedule' steps: - uses: technote-space/workflow-conclusion-action@v2 - uses: voxmedia/github-action-slack-notify-build@v1 diff --git a/bors.toml b/bors.toml index d0da08f..39d0db2 100644 --- a/bors.toml +++ b/bors.toml @@ -1,6 +1,6 @@ status = [ "Julia 1.0.5 - ubuntu-latest - x64", - "Julia 1.3 - ubuntu-latest - x64", + "Julia 1.5 - ubuntu-latest - x64", "Julia 1 - macOS-latest - x64", "Julia 1 - ubuntu-latest - x64", ] From 1c81e5edabb46b9b65060ccbb2d9edb5e7a1a884 Mon Sep 17 00:00:00 2001 From: Matt Brzezinski Date: Wed, 17 Feb 2021 14:24:24 -0600 Subject: [PATCH 107/110] Added CompatHelper, JuliaNightly, and TagBot GHA --- .github/workflows/CompatHelper.yml | 16 +++++++++++++++ .github/workflows/JuliaNightly.yml | 32 ++++++++++++++++++++++++++++++ .github/workflows/TagBot.yml | 15 ++++++++++++++ 3 files changed, 63 insertions(+) create mode 100644 .github/workflows/CompatHelper.yml create mode 100644 .github/workflows/JuliaNightly.yml create mode 100644 .github/workflows/TagBot.yml diff --git a/.github/workflows/CompatHelper.yml b/.github/workflows/CompatHelper.yml new file mode 100644 index 0000000..6c96707 --- /dev/null +++ b/.github/workflows/CompatHelper.yml @@ -0,0 +1,16 @@ +name: CompatHelper +on: + schedule: + - cron: '0 0 * * *' # Everyday at midnight + workflow_dispatch: +jobs: + CompatHelper: + runs-on: ubuntu-latest + steps: + - name: Pkg.add("CompatHelper") + run: julia -e 'using Pkg; Pkg.add("CompatHelper")' + - name: CompatHelper.main() + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COMPATHELPER_PRIV: ${{ secrets.DOCUMENTER_KEY }} + run: julia -e 'using CompatHelper; CompatHelper.main()' diff --git a/.github/workflows/JuliaNightly.yml b/.github/workflows/JuliaNightly.yml new file mode 100644 index 0000000..4128452 --- /dev/null +++ b/.github/workflows/JuliaNightly.yml @@ -0,0 +1,32 @@ +name: JuliaNightly +# Nightly Scheduled Julia Nightly Run +on: + schedule: + - cron: '0 2 * * *' # Daily at 2 AM UTC (8 PM CST) +jobs: + test: + name: Julia Nightly - Ubuntu - x64 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: julia-actions/setup-julia@v1 + with: + version: nightly + arch: x64 + - uses: actions/cache@v2 + env: + cache-name: julia-nightly-cache-artifacts + with: + path: ~/.julia/artifacts + key: ${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} + restore-keys: | + ${{ env.cache-name }}- + - uses: julia-actions/julia-buildpkg@latest + - run: | + git config --global user.name Tester + git config --global user.email te@st.er + - uses: julia-actions/julia-runtest@latest + - uses: julia-actions/julia-processcoverage@v1 + - uses: codecov/codecov-action@v1 + with: + file: lcov.info diff --git a/.github/workflows/TagBot.yml b/.github/workflows/TagBot.yml new file mode 100644 index 0000000..f49313b --- /dev/null +++ b/.github/workflows/TagBot.yml @@ -0,0 +1,15 @@ +name: TagBot +on: + issue_comment: + types: + - created + workflow_dispatch: +jobs: + TagBot: + if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot' + runs-on: ubuntu-latest + steps: + - uses: JuliaRegistries/TagBot@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + ssh: ${{ secrets.DOCUMENTER_KEY }} From 593149fe252812fc9a2697c484540bd4c38c9904 Mon Sep 17 00:00:00 2001 From: Matt Brzezinski Date: Wed, 17 Feb 2021 15:03:55 -0600 Subject: [PATCH 108/110] Fixed CI.yml --- .github/workflows/CI.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 262022e..1f365b0 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -38,16 +38,6 @@ jobs: with: version: ${{ matrix.version }} arch: ${{ matrix.arch }} - - uses: actions/cache@v1 - env: - cache-name: cache-artifacts - with: - path: ~/.julia/artifacts - key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} - restore-keys: | - ${{ runner.os }}-test-${{ env.cache-name }}- - ${{ runner.os }}-test- - ${{ runner.os }}- - uses: actions/cache@v2 env: cache-name: cache-artifacts From 31114e946a70cf6efd5a7b1077383450af530848 Mon Sep 17 00:00:00 2001 From: Matt Brzezinski Date: Wed, 17 Feb 2021 15:32:02 -0600 Subject: [PATCH 109/110] Addressed small comments --- .github/workflows/CI.yml | 2 +- .github/workflows/JuliaNightly.yml | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 1f365b0..a33c905 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -30,7 +30,7 @@ jobs: arch: x86 include: - os: ubuntu-latest - version: "1.0" + version: "1.5" arch: x64 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/JuliaNightly.yml b/.github/workflows/JuliaNightly.yml index 4128452..f32912c 100644 --- a/.github/workflows/JuliaNightly.yml +++ b/.github/workflows/JuliaNightly.yml @@ -22,11 +22,4 @@ jobs: restore-keys: | ${{ env.cache-name }}- - uses: julia-actions/julia-buildpkg@latest - - run: | - git config --global user.name Tester - git config --global user.email te@st.er - uses: julia-actions/julia-runtest@latest - - uses: julia-actions/julia-processcoverage@v1 - - uses: codecov/codecov-action@v1 - with: - file: lcov.info From 782b4f30108a81aa6cb9244e067718fc9e0f845f Mon Sep 17 00:00:00 2001 From: Matt Brzezinski Date: Wed, 24 Feb 2021 09:30:13 -0600 Subject: [PATCH 110/110] Updated bors.toml --- bors.toml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/bors.toml b/bors.toml index 39d0db2..0dd65ed 100644 --- a/bors.toml +++ b/bors.toml @@ -1,7 +1,10 @@ status = [ - "Julia 1.0.5 - ubuntu-latest - x64", - "Julia 1.5 - ubuntu-latest - x64", - "Julia 1 - macOS-latest - x64", + "Julia 1.0 - ubuntu-latest - x64", + "Julia 1.0 - ubuntu-latest - x86", + "Julia 1.0 - macOS-latest - x64", "Julia 1 - ubuntu-latest - x64", + "Julia 1 - ubuntu-latest - x86", + "Julia 1 - macOS-latest - x64", + "Julia 1.5 - ubuntu-latest - x64", ] delete-merged-branches = true