From 337c2bd4884fcffaa8f5d972aa7445dbbce1de16 Mon Sep 17 00:00:00 2001 From: Danang Massandy Date: Tue, 31 Dec 2024 00:03:59 +0000 Subject: [PATCH 1/4] add fiona utilities --- deployment/docker/requirements.txt | 3 + .../core/tests/data/country.geojson | 8 + django_project/core/tests/data/gpkg.gpkg | Bin 0 -> 106496 bytes django_project/core/tests/data/shp.zip | Bin 0 -> 1029 bytes django_project/core/tests/data/shp_3857.zip | Bin 0 -> 2586 bytes .../core/tests/data/shp_no_dbf_shx.zip | Bin 0 -> 272 bytes django_project/core/tests/data/shp_no_shp.zip | Bin 0 -> 550 bytes django_project/core/tests/test_fiona.py | 233 ++++++++++++++++++ django_project/core/utils/__init__.py | 0 django_project/core/utils/fiona.py | 190 ++++++++++++++ 10 files changed, 434 insertions(+) create mode 100644 django_project/core/tests/data/country.geojson create mode 100644 django_project/core/tests/data/gpkg.gpkg create mode 100644 django_project/core/tests/data/shp.zip create mode 100644 django_project/core/tests/data/shp_3857.zip create mode 100644 django_project/core/tests/data/shp_no_dbf_shx.zip create mode 100644 django_project/core/tests/data/shp_no_shp.zip create mode 100644 django_project/core/tests/test_fiona.py create mode 100644 django_project/core/utils/__init__.py create mode 100644 django_project/core/utils/fiona.py diff --git a/deployment/docker/requirements.txt b/deployment/docker/requirements.txt index 8141082b..838df1ed 100644 --- a/deployment/docker/requirements.txt +++ b/deployment/docker/requirements.txt @@ -58,3 +58,6 @@ pyjwt cryptography Pillow + +# fiona +fiona==1.10.1 \ No newline at end of file diff --git a/django_project/core/tests/data/country.geojson b/django_project/core/tests/data/country.geojson new file mode 100644 index 00000000..ce80df85 --- /dev/null +++ b/django_project/core/tests/data/country.geojson @@ -0,0 +1,8 @@ +{ +"type": "FeatureCollection", +"name": "level_0", +"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } }, +"features": [ +{ "type": "Feature", "properties": { "name_0": "Pakistan", "code_0": "PAK" }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 70.659620789430747, 32.624727824714725 ], [ 70.825122822992526, 32.283808843198763 ], [ 70.051157430747779, 32.048938248883729 ], [ 69.968406413966889, 32.353740618315406 ], [ 70.143643861267577, 32.645223872614423 ], [ 70.42110315282703, 32.821295871012296 ], [ 70.659620789430747, 32.624727824714725 ] ] ] ] } } +] +} diff --git a/django_project/core/tests/data/gpkg.gpkg b/django_project/core/tests/data/gpkg.gpkg new file mode 100644 index 0000000000000000000000000000000000000000..b93b3836ac690cdc5f680b999e04f5f657df2df9 GIT binary patch literal 106496 zcmeI5e{37qeZY^RElSo8+DRNXkzGHtQAo}wi9aOkhb5VmbfVZuq(oAY71bp!40@xF(eyM4A?KE%dj*-J2V4Qv}hBw zK$~IN_kNMbACf8CsjcLDrgeAkecyZE_j%v<^T>ONg;`$aNVXv6S(yZ=qm;!$1qq=j zsuTX-5C4sWz{Mfs2F@0Wa=oV9hvy#(o7~NpomB7ACl7;5ll|w4`^W6>X%}W8yhZ{@ z00|%gB!C2v01`j~NB{{S0VIF~J|F@|PeR1@qaEo5G-19=V^$c~q=^DR%aw(i3L;Z?+8BXTY%zPqz^sO)x3r{9ioSLVTo(d%w=hWMj zZ+NKRNhjuKn0PEQRTeoL7##8i25JBBP_KVr(C-fnoDI;S#pFyZ9!V~*IF*>4v~$oK z@H#JEtmAVw;6JPQ%*P{h45StoIZoK%(`!!JTSt6o*y~fo7o(BniZjElf;dR_dVPca z0|EcAf5109+#jf?!?4fqS9Q1$flSV_GB2-ZAaFDmPtJg9nsY%|)!ms;B3V!4eoflW&UrmzevFc4KL=Z(Il-EQ@)fh%dzsh$VvLG-|P4H_(<@9zx^`X31{WS{%~D~Fy6o_{s;<7X4!3NC!R?eZ(9#)84`z5GJ^>+I3D zjzGRF_McJk!2<~(0VIF~kN^@u0!RP}AOR$R1dsp{_%IPTV(n_xR}OIg|6wXv%nK4g z0!RP}AOR$R1dsp{Kmter2_OLz0n_|n`~LsCl>J?kKVBjMB!C2v01`j~NB{{S0VIF~ zkN^@u0tcNyo5k8?(I5X?@%jHjFHY1S2_OL^fCP{L5`P=#Tqh$A$dd;%qW9 zADdkc$D+pj|9(Q*e{zRs1=T|WNB{{S0VIF~kN^@u0!RP}AOR$R1R5q_Kh$;Hc=w;O z8i3#bZx|g4A^{|T1dsp{Kmter2_OL^fCP{L5;#}{s=ohE9X(j;qqax@2_OL^fCP{L z5}wYHbqF4=yg_0-|NI=pdcsOg6w zFxNotI^SvfN&{jjvOfY>r;ggX7JFK*9g{_Vb(IrWi@s4G51BFhW% z2+?z?W{;#D@41DHQUl9Cw+^rPhlXP@@{sbs^ToP(rvkKc(<5WJy`(CV^@?-Uv{W3iWV$D{-2n zM~PR5IohegJlp`x8??xBy1Gr5VBThXEGt&m8XLfOzN6WZ=&?||kl{9^$8)^QrPy`3 zpx&pd%`WAucIn)iY_~ajdMuwCRx4ArfAxKbVY#7+S%y^m61N&jN?`LGNir9bBpQSN zi?g$|DXWoal2MwDl3ztN+s${4<5$;YXhEt1tId~IqwD+a7O==~LjBxKN&FR#OvM(J z43$*UJh$q5PR62%B(zD$#R~|$0Kv0_%eBjSvE7EO?A{mn}TH_SU#UJ(Ndbn zNH|&@gNMYKX=sn4lT1Rx4egoC39@8D?eXk<{D{plFkpF7i?X_0QW96wjw^0$dzPjm zVa`ouiJJ3FOQMwGGqr`2=Y`Z}4P3=_tJam>tS1YuwX&5cSLM8P%UD(wlNvWocoOLw~O@P!*}Da_bN}ZMKrdXk=j#I{I=YtdNH)s>lnK1#c2= zsAg7MQ#y6?wiIb?c3d3X)0RqwqFY4DU)Ls%L|XPAmDYz;r_j26?_ryxug|i*QZ~%O zstBc}wO6VFW{2ilgEV`a%4Ls+C(E(&y2wE)>q5D6m|~_wusfOB^LRqd&5p&sJ#m@S z)$5(v$+mN%$>wmmEKhEf^Hz~BC&SBZXnSM`MvH7psY;bHG@pS+pU?7~sCj7C zOJ_JKE%HT~htaUw;TlIe_gd&)V_DeQ#OvOen7STDz~Ou zQ>l{BdXG9|sa;xBbDKyP#8p<{uRwQXSVpOpHG4tDDTo>9s_J&%87|8Uyc$K7Iab@9 zr=#VcsSfHjs-3r8Yjz)c-TK+4e(E*5t=-Y)IO1qM+49DbA0ElJzPg_oD@1v1Kk^@h z?A2w5HfKJ!+x)~lRJ_sYUY$5fa}<5T7n>$#soOT?t571pE~zu9M1i=;%e+@e;9-L@vRQj2;EY4z77^_7kC!sN_d8e2zWY;Vv|E7ZNiTGR^N>Mp7^ zhO*;Zl-)F(7(3bQSU9ui2$S1{Wf=)ZTarG5C_CJY_kICI9b(_J1Smc7-{Vfps% z+JRx&vt|}iGc%`4!u2NOrV(A``ka(5)XX_dbMI9yC8s%3J7=z#aMPqRagS&Fi7se5 zyIOXR>n5q6QW-AC!CGa-{4J&LF0G#o4*HV@XCpzKBs86bwZ$oTZ~&|1(D^v^*qr(! z1-H%5J&7~*j(w4WkpSZt%{Vy&W;fv3vKStv=tbfsR?#eYAL*(xtwGqAX!kgNK(lMqvB$$83(S zF3ZlOX7OvqlrQDe+k|5k7GAEHZsN@H8E0wv-25~lw3v)Vq7ZP7F|U*or?LR&By+IL z6`7CCE{9`LnmA3dz+JkK;Zk0wI(2ssc|G8yRHG^d1 zHf5jbd~9^rY)`WEc__R>Yq zzklYd3&Fn7!~byjFBgJ;pLq7ip;ylbzqIq|$KDACgO+B=td|*$_Rbs zA3xjrKPQ916F>OIQuI&Du~o}ky{Q0veCkKL-X3iN^=7KndwT6_7j}MsAvkvGmCJ44 zUI_lpFTeiaNFo?yUvGcmh`+xq23#C-5<7@J!{P6LqAMIj$zyj*KTkk#z|L!*E{??&a zzW-cyAvpZM|8#odOAEoD@vr>XOiwWQpD+FP=b0z!sec&MfBe#$-S=RDqnng`sw?k!C-ZGHsxhg#X@dNIs5cWpEu=$B@Si&Z?V5i z!3Pf{fCP{L5l1e6=L2f=qAYS8{E83WAGNP6sNq!AdkR8sOL_ml)&4wX|6BX>_BRhs z5u>I^00|%gB!C2v01`j~NB{{S0VIF~4mg2dI}D2#mWnk5<;Aph)-J5%Ff2Y)idKYc zB8Oq|pmmS61NHg8{il@uJ^Rn$ga;Bp0!RP}AOR$R1dsp{Kmter2_OL^@L?cewOXlG zYm@Q^Ug^-P{{FwMhk_3tNB{{S0VIF~kN^@u0!RP}mJZ#hIg{{T2CFGEuf%f!?D zU!Hj8>vv699-E%9zHN-6nAc0>w0cHMA!FwR_Kmter2_OL^fCP{L5vya+%P%^;M$Rqx+W-T_fj_q>Ik?*9wmDob z%g$nXZ)4;278SoCxvygP%L*^z<};<;FHO4+=TpT?q+&PB#3YHvk|eq~J4@#?sXDtB z!`8pkA=nj@xRjT5NSy3a1!Y#quwo{)F7oPrn5JlW*b-CTs^0y$9FGj|?daRg=O7~e z6(E%{lqFrB?OB(t>wH(s)rh`3t@e5i{f9Ft<*gg~*2lV&0mazlcvpF2XWx1|8!yc; zQBufddi7mF4Mh5u$9j-7*(AE`#?VA^N z&yKdkZhzzX(E}J6GInD(Z`G`3sH6%xJw~F;lBO34MW={cz16lrjU*InAh9?}BvTQI z$rt4UnYTD#RLpMg`%)hjNw+1sjv@Y zkLfsxGxM{dNk*x{vHBMFaAAE4CT0vfW>;=~fhwQi1cI(2TY9dN_6 zsp1EZbeXDWr(iQAcsU_qxg`QhTbRjAuOC&8KXRu`$c`soj(i3$V3&9W7M% zTsS;7jo-)!Q=o{AO*YJUuJXboKX{K8D$J26+ySFd%34O-_xB*cF&A-&y93vA92wxp|Iidc$8^Lme0Yv_sVl0 zW8kj%X)TY*mp09e(qdjr<+FS$dx=cP;!GqQB@Z#n#9fZm1H-auCeB1BVc=E42&$B) zHnL*f$Vyu59i6AczC>ACTd7t!KAIMdxkds=00|%gB!C1yvIL%asL4ikjh+0Ru)hAR zzg0OerqjLYLcZ_uRbJ}N$QffErR@dQe)(}JKQgK%n&Bygzj`pc{sxvR=6j@n4D=zG#Fd}BtGB}nl)rfiFlGLZ_ zndLYMyw#K91@G4`WO!i}+)9^tfdAO2_tjx1H801S^5)t6$GB;fi^Q=7PT)HHy z*;fTOF3ICzCKjHICsv$G;RG4#chXEW!OTs}GT>4Wa~TrmU|K8lX_DBIWG+v9!$bXq z6C^JGXpVEzb0Eyb_Y`&~MOKh_+2g#}ka&17;B8oTXfV*Qtj{;lc*MT5gZ+)iJb2dM zu(0137*>Rt+1bc^A{Ln{!#^7s9P$MQY5(w0uYX|B?+*-|4bZbpG@P7Sj7E|xPI%HH z0(Rd;2(On^XmWBf9-3TU@%MVcSy`lE=2o0xczwCB!Kc@pw3iM=!?TPQLxx)gA8_pT z`Ud+40{&tDfNyxXKM;VxlZh2Bm*b04fzPlx+JBJ_U5J2nz!L?{3UXjh@_L3N?pYQ} zlokr2yhg|3Q(y!>kZv%Y>R1cHsu68LfM_{ZlJ!^>9U6>vF%b$gD~ZY2JecoHLFBI# z1ewhdXkCg00qQ#`WJzF(K)nD{?u9Z4#hH+9yvpswMC`&!&#;&7@eO%tIKv63C`H7~ zC&Es;Kj1%m@#1KoQgzhY2KB8d79Q&#Kck$+a*z=~M9x6%U(b(0rGnaWMkv6ut*ivC zh$?CXl%RRaj0gG${G)x^b=e0zl(ZBNxHR0^b$JbL6cynJ`i1LhyC9ui~U8) z{v!Or0|_7jB!C2v01`j~NB{{S0VIF~kN^_6T?BfXS{z*zHD;mIuRiEsK=XV3{vMyN b$2XYt_K)-rjQIV%{odh6nn2Co##Q})S5y>l literal 0 HcmV?d00001 diff --git a/django_project/core/tests/data/shp.zip b/django_project/core/tests/data/shp.zip new file mode 100644 index 0000000000000000000000000000000000000000..78b02747dc34abd9c41f79ed0096b786cf9c0ac1 GIT binary patch literal 1029 zcmWIWW@Zs#-~hs-ECrzqP;i)yfq|PrfuT5~Al@+EP%kAZEi{CefxYQmaWV{-R&X;g zvbmEHzPfJKhNU1q{qJcGK*@2mL3QgR;c9!<4ruP`ml&}dhW_+sb+*$GT z;4faz7sr+7&tx){-M&ZBt}>3*!oa7;JYa#EN00uA^DCFkTiE%r?TQgsmfVfn5aTNi zr`{Pa*;L7qkRp)p63VK%WxyVM4(3qyv2a_9g(m$!fGKqR(#D6tNct8auwveH>cN z9{qE1>Ye3GvO@1a=-7O^aFDURF698n5xx^?wP$UrZaZH;c<|GCN!R6>jeO?YeKX3} zmKHE*rnF{ho!M8fyFlT>!++0TYl&|%xccgzF0a5^w;8O))u#lv=Xu(*vzPpJP@dUv zgE^(H{MeX3&R{VbTc z_Ps<|Pigjo!&{#1SmXarpnRjlUDjak$;~U*THQPFGe#_t|BOz_S8ym_dlJ{q3k*k4 zSYi+58ekkCr=#MG3L-=KO`L(T!3HMDSRHqHUoY+w3z0j&&wX6Aw|VWKi$?E)8~HlR zE5zg~Up#1Vwl^|hN}90CNZ}j9BWH&kMzGs1T%6A94745Of&gzuCJ|;tDo0KoAlJdb zmPQbZgfx$CFmiyP8vF>!U?P$ly1~f4K{eQe3E^TQLN~yh6__O$7?^>O8yIa*fZ7=t E03_^Qq5uE@ literal 0 HcmV?d00001 diff --git a/django_project/core/tests/data/shp_3857.zip b/django_project/core/tests/data/shp_3857.zip new file mode 100644 index 0000000000000000000000000000000000000000..9df5eb3a114f499b50312cea7e6fb940fdc3c1a9 GIT binary patch literal 2586 zcmb7`dpOkF8pnTQjKMT+xg@u-u~Q+DToTE(7-1Xtn2F4s!7zz9Az~Az@YvaIjZ5N? z7!q=cNhx8bDV(;9%aFS+wwaBnJ=1f}bCBoJS?gK9-}>vb-uGSa`+k>$Etp>t5adoF zUlI!V&wz0MA^-vi#fA7B_R%%a)7SD1!J?eu02JJu<;z<#+!Yo913>)KU;y~`fx(Hd z9{_-ZEmY9KI8Fr0h44TQ+aA#&@ZAx9=lpj%!ncYwhVe(E(SA|A&toBerR_N}h^sK9 ziJ_^Q-Z0Q}Q9vAoTZ~M881as}3KyDwrc#i@r>D@+qHGfB4KdKRBk4O7{%k`!^qgK^ zLM}@Fl-Q>uj8}XU?X24u`{KQBNo}Mc+D|Ol0tGG)B99^TgDIb7KqNTpdO05PpjTIZ zViL~zn929e$w>pL&}6QIJXf}NvK;iClOY%IJ2@Ga-c5+t7PI{NGysN?ODPs7f0WXJ7F-63Lg&=_t6zBBQ_lT=Wk!q2Uj!nQREm;l@^SdjoWBrwpEe&daToA z@Eh&XrY;ykK?hE0`Bz5*Wv&rCd@~XoJ%(dnO6DiiaTd`MmWrSE+^aX9S*)06e5Gp2 z=o6{OhL@(zXfk46-XxD31$izDizU5FSkt(tIKz?Q5-YXe^%#2}lAO|tv!wGe(tdCz zR+7oy)%ISiJ=QAQ?R*57mp1C`8Z{0#qP)0mA5n$x(}o!2w^Kx~JD&|^X^TaDsQ&ow z&T&1p2{0e(8vhYw2F@&@;+~k%ip1_HcsT9o0>kIZ9H6;yY9G1fnC@`peGdOfmDT#p zYlD+*p$8btkM!l27&A`gi?3>3enJN9_i@jyJb&PrWm(rRI@bwjIx-6fKb=e{$q(Dr zFj8~mtLzE?^@eD)Ee!6-$=(4CDIVB~@Cd7z~=Qnhb2Ma6&an#G#!`cwJ|W1@|9zr5YfiOtMbQcH_&h^ec8*K8dS)J%-R zG|yaqmbzXVTKA@gSubtfF^0}33Aiemw#}?+zGWlJm=%Wb5#o^5YrQ1KS>x7+ZE|+l z&J=Y}8H5sLW$FN9+kY=%)%5X!l@p zS`+8$ZY44am#biTpSwts5KXl*SuHy(AzTR>a%^$kQMl0_e@p!0(I?uDnZ%;3N!$g< zv-&CPD^md|+G4ajtx$vNv`}I6%(eAQ6Y+@c>g~f`BXU~Pa#&#*!NB5)U1Tj5Y{FaF zz4x|&?zDaJdxDqI0^2I>erhd&CGg7Wp!w7eRX3X`=aPw1i5 z7Yq7SQfOSa8d3&AGgli>r96uD9H}*tbv=(Cp!M@DT2q(X#p7Rm=*Vp2mCYwkxaM0v z0N@qJ_GPp7eIw%b_MOVc7ws^0u3h9yE1KJKAv(ocDSs;(VhS_tPRxUJOqX)1S^pNT zxzW;)g3t4fKD@?%0V>z0knch2yyDK1rw;}+4%IBh-+gEQh~mU)t{%sizM5)~VvYJ! zd+ft0!DGv_bLzVNn5%;$ZjFCTPlE0NiAUlg&(D^*QgU+ItP>tQ9=MDw`*mC%`~0g% z^b@#*iH5@^YTx4tG`_9(68X;8E+|2?N8=* z0g>Oz-lz(j**4ru*kJb$XzNz?#?sl$&gB-=2KEoA%vScsRN2gCaNk86*xc<->9Li) i@t!xck8|H$o7g+1o`Wrf+c*F~fIHQ>UJvnJZouE#nZR}c literal 0 HcmV?d00001 diff --git a/django_project/core/tests/data/shp_no_dbf_shx.zip b/django_project/core/tests/data/shp_no_dbf_shx.zip new file mode 100644 index 0000000000000000000000000000000000000000..91c20f5be1d1825127f32defbab98aa1a21bc876 GIT binary patch literal 272 zcmWIWW@Zs#U|`^25D&`>T~#%CaT<{K28g*BWEe_Pi%a5-^olbILPIzinCI>)Oa|f7 z3T_5QmKV$n3}7NTA>et^0lrgl2F3;(m?ZP}X}cUNVY50?b#DE?6;B={EY&JKUF)@g z`FY^UEdFYCc9~xj8I=XB1(L-)6tzPpr#@TAy47K8JJ)x=Uun-COvsp4R+DD$r0etk z`Q-R{FBqR*JnMFypCQ1Tkx7mjms=%(E{1?DjUXD?<*X2wqq#c3o0Scuk`V|4fb=R5 GhXDYK1yS$- literal 0 HcmV?d00001 diff --git a/django_project/core/tests/data/shp_no_shp.zip b/django_project/core/tests/data/shp_no_shp.zip new file mode 100644 index 0000000000000000000000000000000000000000..2232cfcabcf2d25e915180a04eafd8fd277d4a41 GIT binary patch literal 550 zcmWIWW@Zs#U|`^2ILcBGDiFAEb0?5@n2mvfn?Zr0IHMrmFy2rvB`Ga5gqMN6>0EI# z43}1LGcdBeU}j(d6UplyJxfnZNJvPjIeVgkHD%d>nRNA0{&~ z1fcln!o}&l&Ojg405P(UiZdz*`Y0jbdC~#CH*p5W1{;_pV|Co+eZ9C#EJW`7KKF6e z-sZJ`E*iZHZshAMuMm@~eDR>c+1|*2DQUtkBZY4akDMKH7=dnKWD;S<9lk(Efx(tW q5Jf_0qZ^FwO^C4!3|ks6AsI}BrvtoMfpO2kzyySwfOH%qhz9^RLad4a literal 0 HcmV?d00001 diff --git a/django_project/core/tests/test_fiona.py b/django_project/core/tests/test_fiona.py new file mode 100644 index 00000000..7244c047 --- /dev/null +++ b/django_project/core/tests/test_fiona.py @@ -0,0 +1,233 @@ +# coding=utf-8 +""" +Africa Rangeland Watch (ARW). + +.. note:: Unit tests for Fiona. +""" + +import os +from django.test import TestCase +from django.core.files.uploadedfile import ( + InMemoryUploadedFile, + TemporaryUploadedFile +) + +from core.settings.utils import absolute_path +from core.utils.fiona import ( + FileType, + validate_shapefile_zip, + open_fiona_collection, + validate_collection_crs, + delete_tmp_shapefile +) + + +class TestUtilsFiona(TestCase): + """Test class for Fiona utility functions.""" + + def test_validate_shapefile_zip(self): + """Test validate shapefile.""" + # test incomplete zip + shape_file_path = absolute_path( + 'core', + 'tests', + 'data', + 'shp_no_shp.zip' + ) + is_valid, error = validate_shapefile_zip(shape_file_path) + self.assertFalse(is_valid) + self.assertEqual(len(error), 1) + self.assertEqual(error[0], 'shp_1_1.shp') + shape_file_path = absolute_path( + 'core', + 'tests', + 'data', + 'shp_no_dbf_shx.zip' + ) + is_valid, error = validate_shapefile_zip(shape_file_path) + self.assertFalse(is_valid) + self.assertEqual(len(error), 2) + self.assertEqual(error[0], 'test_2.shx') + self.assertEqual(error[1], 'test_2.dbf') + # test complete zip + shape_file_path = absolute_path( + 'core', + 'tests', + 'data', + 'shp.zip' + ) + is_valid, error = validate_shapefile_zip(shape_file_path) + self.assertTrue(is_valid) + # test using in memory file + file_stats = os.stat(shape_file_path) + with open(shape_file_path, 'rb') as file: + mem_file = InMemoryUploadedFile( + file, None, 'shp.zip', 'application/zip', + file_stats.st_size, None) + is_valid, error = validate_shapefile_zip(mem_file) + self.assertTrue(is_valid) + # test using temporary uploaded file + with open(shape_file_path, 'rb') as file: + tmp_file = TemporaryUploadedFile( + 'shp.zip', 'application/zip', file_stats.st_size, 'utf-8') + with open(tmp_file.temporary_file_path(), 'wb+') as wfile: + wfile.write(file.read()) + is_valid, error = validate_shapefile_zip(tmp_file) + self.assertTrue(is_valid) + + def test_open_fiona_collection_shp(self): + """Test open fiona collection for shapefile.""" + shape_file_path = absolute_path( + 'core', + 'tests', + 'data', + 'shp.zip' + ) + file_stats = os.stat(shape_file_path) + + # test using filepath + collection = open_fiona_collection( + shape_file_path, FileType.SHAPEFILE) + self.assertEqual(len(collection), 3) + collection.close() + + # test using InMemoryUploadedFile + with open(shape_file_path, 'rb') as file: + mem_file = InMemoryUploadedFile( + file, None, 'shp.zip', 'application/zip', + file_stats.st_size, None) + collection = open_fiona_collection( + mem_file, FileType.SHAPEFILE) + self.assertEqual(len(collection), 3) + collection.close() + delete_tmp_shapefile(collection.path) + + # test using TemporaryUploadedFile + with open(shape_file_path, 'rb') as file: + tmp_file = TemporaryUploadedFile( + 'shp.zip', 'application/zip', file_stats.st_size, 'utf-8') + with open(tmp_file.temporary_file_path(), 'wb+') as wfile: + wfile.write(file.read()) + collection = open_fiona_collection( + tmp_file, FileType.SHAPEFILE) + self.assertEqual(len(collection), 3) + collection.close() + delete_tmp_shapefile(collection.path) + + def test_open_fiona_collection_gpkg(self): + """Test open fiona collection for gpkg.""" + gpkg_file_path = absolute_path( + 'core', + 'tests', + 'data', + 'gpkg.gpkg' + ) + file_stats = os.stat(gpkg_file_path) + + # test using filepath + collection = open_fiona_collection( + gpkg_file_path, FileType.GEOPACKAGE) + self.assertEqual(len(collection), 3) + collection.close() + + # test using InMemoryUploadedFile + with open(gpkg_file_path, 'rb') as file: + mem_file = InMemoryUploadedFile( + file, None, 'gpkg.gpkg', 'application/geopackage+sqlite3', + file_stats.st_size, None) + collection = open_fiona_collection( + mem_file, FileType.GEOPACKAGE) + self.assertEqual(len(collection), 3) + collection.close() + + # test using TemporaryUploadedFile + with open(gpkg_file_path, 'rb') as file: + tmp_file = TemporaryUploadedFile( + 'gpkg.gpkg', 'application/geopackage+sqlite3', + file_stats.st_size, 'utf-8') + with open(tmp_file.temporary_file_path(), 'wb+') as wfile: + wfile.write(file.read()) + collection = open_fiona_collection( + tmp_file, FileType.GEOPACKAGE) + self.assertEqual(len(collection), 3) + collection.close() + + def test_open_fiona_collection_geojson(self): + """Test open fiona collection for geojson.""" + geojson_file_path = absolute_path( + 'core', + 'tests', + 'data', + 'country.geojson' + ) + file_stats = os.stat(geojson_file_path) + + # test using filepath + collection = open_fiona_collection( + geojson_file_path, FileType.GEOJSON) + self.assertEqual(len(collection), 1) + collection.close() + + # test using InMemoryUploadedFile + with open(geojson_file_path, 'rb') as file: + mem_file = InMemoryUploadedFile( + file, None, 'country.geojson', + 'application/geo+json', + file_stats.st_size, None) + collection = open_fiona_collection( + mem_file, FileType.GEOJSON) + self.assertEqual(len(collection), 1) + collection.close() + + # test using TemporaryUploadedFile + with open(geojson_file_path, 'rb') as file: + tmp_file = TemporaryUploadedFile( + 'country.geojson', 'application/geo+json', + file_stats.st_size, 'utf-8') + with open(tmp_file.temporary_file_path(), 'wb+') as wfile: + wfile.write(file.read()) + collection = open_fiona_collection( + tmp_file, FileType.GEOJSON) + self.assertEqual(len(collection), 1) + collection.close() + + def test_validate_collection_crs(self): + """Test validate crs.""" + # test invalid crs shp + shape_file_path = absolute_path( + 'core', + 'tests', + 'data', + 'shp_3857.zip' + ) + collection = open_fiona_collection( + shape_file_path, FileType.SHAPEFILE) + is_valid, crs = validate_collection_crs(collection) + collection.close() + self.assertFalse(is_valid) + self.assertEqual(crs, 'epsg:3857') + + # test valid crs + shape_file_path = absolute_path( + 'core', + 'tests', + 'data', + 'gpkg.gpkg' + ) + collection = open_fiona_collection( + shape_file_path, FileType.GEOPACKAGE) + is_valid, _ = validate_collection_crs(collection) + collection.close() + self.assertTrue(is_valid) + + shape_file_path = absolute_path( + 'core', + 'tests', + 'data', + 'country.geojson' + ) + collection = open_fiona_collection( + shape_file_path, FileType.GEOJSON) + is_valid, _ = validate_collection_crs(collection) + collection.close() + self.assertTrue(is_valid) diff --git a/django_project/core/utils/__init__.py b/django_project/core/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/django_project/core/utils/fiona.py b/django_project/core/utils/fiona.py new file mode 100644 index 00000000..d1f73a24 --- /dev/null +++ b/django_project/core/utils/fiona.py @@ -0,0 +1,190 @@ +# coding=utf-8 +""" +Africa Rangeland Watch (ARW). + +.. note:: Fiona shapefile helper class. +""" +import os +import zipfile +import fiona +from fiona.crs import from_epsg +from fiona.collection import Collection +from django.core.files.temp import NamedTemporaryFile +from django.core.files.uploadedfile import ( + InMemoryUploadedFile, + TemporaryUploadedFile +) + + +class FileType: + """File types.""" + + GEOJSON = 'geojson' + SHAPEFILE = 'shapefile' + GEOPACKAGE = 'geopackage' + + +def _open_collection(fp: str, type: str) -> Collection: + """Open collection from file path.""" + if type == FileType.SHAPEFILE: + file_path = f'zip://{fp}' + result = fiona.open(file_path, encoding='utf-8') + else: + result = fiona.open(fp, encoding='utf-8') + return result + + +def delete_tmp_shapefile(file_path: str): + """Delete temporary shapefile.""" + if file_path.endswith('.zip'): + cleaned_fp = file_path + if '/vsizip/' in file_path: + cleaned_fp = file_path.replace('/vsizip/', '') + if os.path.exists(cleaned_fp): + os.remove(cleaned_fp) + + +def _store_zip_memory_to_temp_file(file_obj: InMemoryUploadedFile): + """Store in-memory shapefile to temporary file.""" + with NamedTemporaryFile(delete=False, suffix='.zip') as temp_file: + for chunk in file_obj.chunks(): + temp_file.write(chunk) + path = f'zip://{temp_file.name}' + return path + + +def _read_layers_from_memory_file(fp: InMemoryUploadedFile): + """Read layers from memory file of shapefile.""" + layers = [] + file_path = None + + try: + with NamedTemporaryFile( + mode='wb+', delete=False, suffix='.zip' + ) as destination: + file_path = destination.name + for chunk in fp.chunks(): + destination.write(chunk) + + layers = fiona.listlayers(f'zip://{file_path}') + except Exception: + pass + finally: + if file_path and os.path.exists(file_path): + os.remove(file_path) + return layers + + +def _list_layers_shapefile(fp: str): + """Get layer list from shapefile.""" + layers = [] + try: + if isinstance(fp, InMemoryUploadedFile): + layers = _read_layers_from_memory_file(fp) + elif isinstance(fp, TemporaryUploadedFile): + layers = fiona.listlayers( + f'zip://{fp.temporary_file_path()}' + ) + else: + layers = fiona.listlayers(f'zip://{fp}') + except Exception: + pass + return layers + + +def validate_shapefile_zip(layer_file_path: any): + """ + Validate if shapefile zip has correct necessary files. + + Note: fiona will throw exception only if dbf or shx is missing + if there are 2 layers inside the zip, and 1 of them is invalid, + then fiona will only return 1 layer. + """ + layers = _list_layers_shapefile(layer_file_path) + is_valid = len(layers) > 0 + error = [] + names = [] + with zipfile.ZipFile(layer_file_path, 'r') as zipFile: + names = zipFile.namelist() + shp_files = [n for n in names if n.endswith('.shp')] + shx_files = [n for n in names if n.endswith('.shx')] + dbf_files = [n for n in names if n.endswith('.dbf')] + + if is_valid: + for filename in layers: + if f'{filename}.shp' not in shp_files: + error.append(f'{filename}.shp') + if f'{filename}.shx' not in shx_files: + error.append(f'{filename}.shx') + if f'{filename}.dbf' not in dbf_files: + error.append(f'{filename}.dbf') + else: + distinct_files = ( + [ + os.path.splitext(shp)[0] for shp in shp_files + ] + + [ + os.path.splitext(shx)[0] for shx in shx_files + ] + + [ + os.path.splitext(dbf)[0] for dbf in dbf_files + ] + ) + distinct_files = list(set(distinct_files)) + if len(distinct_files) == 0: + error.append('No required .shp file') + else: + for filename in distinct_files: + if f'{filename}.shp' not in shp_files: + error.append(f'{filename}.shp') + if f'{filename}.shx' not in shx_files: + error.append(f'{filename}.shx') + if f'{filename}.dbf' not in dbf_files: + error.append(f'{filename}.dbf') + is_valid = is_valid and len(error) == 0 + return is_valid, error + + +def _get_crs_epsg(crs): + """Get crs from crs dict.""" + return crs['init'] if 'init' in crs else None + + +def open_fiona_collection(file_obj, type: str) -> Collection: + """Open file_obj using fiona. + + :param file_obj: file + :type file_obj: file object + :param type: file type from FileType + :type type: str + :return: fiona collection object + :rtype: Collection + """ + # if less than <2MB, it will be InMemoryUploadedFile + if isinstance(file_obj, InMemoryUploadedFile): + if type == FileType.SHAPEFILE: + # fiona having issues with reading ZipMemoryFile + # need to store to temp file + tmp_file = _store_zip_memory_to_temp_file(file_obj) + return fiona.open(tmp_file) + else: + return fiona.open(file_obj.file) + else: + # TemporaryUploadedFile or just string to file path + if isinstance(file_obj, TemporaryUploadedFile): + file_path = ( + f'zip://{file_obj.temporary_file_path()}' if + type == FileType.SHAPEFILE else + f'{file_obj.temporary_file_path()}' + ) + return fiona.open(file_path) + else: + return _open_collection(file_obj, type) + + +def validate_collection_crs(collection: Collection): + """Validate crs to be EPSG:4326.""" + epsg_mapping = from_epsg(4326) + valid = _get_crs_epsg(collection.crs) == epsg_mapping['init'] + crs = _get_crs_epsg(collection.crs) + return valid, crs From b4c4991f2e4762fc309a6a096325b1d782a912cb Mon Sep 17 00:00:00 2001 From: Danang Massandy Date: Tue, 31 Dec 2024 00:32:31 +0000 Subject: [PATCH 2/4] add validation to upload layer --- django_project/frontend/api_views/layers.py | 103 +++++++++++++++++- .../frontend/tests/api_views/test_layers.py | 70 ++++++++++-- 2 files changed, 161 insertions(+), 12 deletions(-) diff --git a/django_project/frontend/api_views/layers.py b/django_project/frontend/api_views/layers.py index ab6f259f..0a95448a 100644 --- a/django_project/frontend/api_views/layers.py +++ b/django_project/frontend/api_views/layers.py @@ -10,9 +10,11 @@ from django.http import FileResponse from django.conf import settings from django.urls import reverse +from django.core.files.uploadedfile import TemporaryUploadedFile from rest_framework.permissions import AllowAny, IsAuthenticated from rest_framework.response import Response from rest_framework.views import APIView +from rest_framework.exceptions import ValidationError from cloud_native_gis.models import Layer, LayerUpload from cloud_native_gis.utils.main import id_generator from django.shortcuts import get_object_or_404 @@ -23,6 +25,13 @@ import_layer, detect_file_type_by_extension ) +from core.utils.fiona import ( + FileType, + validate_shapefile_zip, + validate_collection_crs, + delete_tmp_shapefile, + open_fiona_collection +) class LayerAPI(APIView): @@ -58,8 +67,97 @@ class UploadLayerAPI(APIView): permission_classes = [IsAuthenticated] + def _check_file_type(self, filename: str) -> str: + """Check file type from upload filename. + + :param filename: filename of uploaded file + :type filename: str + :return: file type + :rtype: str + """ + if filename.lower().endswith('.zip'): + return FileType.SHAPEFILE + return '' + + def _check_shapefile_zip(self, file_obj: any) -> str: + """Validate if zip shapefile has complete files. + + :param file_obj: file object + :type file_obj: file + :return: list of error + :rtype: str + """ + _, error = validate_shapefile_zip(file_obj) + if error: + return ( + 'Missing required file(s) inside zip file: \n- ' + + '\n- '.join(error) + ) + return '' + + def _remove_temp_files(self, file_obj_list: list) -> None: + """Remove temporary files. + + :param file_obj: list temporary files + :type file_obj: list + """ + for file_obj in file_obj_list: + if isinstance(file_obj, TemporaryUploadedFile): + if os.path.exists(file_obj.temporary_file_path()): + os.remove(file_obj.temporary_file_path()) + elif isinstance(file_obj, str): + delete_tmp_shapefile(file_obj) + + def _on_validation_error(self, error: str, file_obj_list: list): + """Handle when there is error on validation.""" + self._remove_temp_files(file_obj_list) + raise ValidationError({ + 'Invalid uploaded file': error + }) + def post(self, request): """Post file.""" + file = None + file_url = request.data.get('file_url', None) + if request.FILES: + file = request.FILES['file'] + elif file_url is None: + raise ValidationError({ + 'Invalid uploaded file': 'Missing required file!' + }) + + tmp_file_obj_list = [file] + + # validate uploaded file + file_type = self._check_file_type(file.name) + if file_type == '': + self._on_validation_error( + 'Unrecognized file type! Please upload the zip of shapefile!', + tmp_file_obj_list + ) + + if file_type == FileType.SHAPEFILE: + validate_shp_file = self._check_shapefile_zip(file) + if validate_shp_file != '': + self._on_validation_error( + validate_shp_file, tmp_file_obj_list) + + # open fiona collection + collection = open_fiona_collection(file, file_type) + tmp_file_obj_list.append(collection.path) + + is_valid_crs, crs = validate_collection_crs(collection) + if not is_valid_crs: + collection.close() + self._on_validation_error( + f'Incorrect CRS type: {crs}!', tmp_file_obj_list) + + # close collection + collection.close() + + # remove temporary uploaded file if any + self._remove_temp_files(tmp_file_obj_list) + # create layer layer = Layer.objects.create( created_by=request.user @@ -74,16 +172,13 @@ def post(self, request): updated_by=request.user ) - file_url = request.data.get('file_url', None) - instance = LayerUpload( created_by=request.user, layer=layer ) instance.emptying_folder() # Save files - if request.FILES: - file = request.FILES['file'] + if file: FileSystemStorage( location=instance.folder ).save(file.name, file) diff --git a/django_project/frontend/tests/api_views/test_layers.py b/django_project/frontend/tests/api_views/test_layers.py index 4f0267b7..2767ba76 100644 --- a/django_project/frontend/tests/api_views/test_layers.py +++ b/django_project/frontend/tests/api_views/test_layers.py @@ -86,17 +86,12 @@ def test_upload_layer_no_auth(self): response = view(request) self.assertEqual(response.status_code, 401) - @mock.patch('layers.tasks.import_layer.import_layer.delay') - def test_upload_layer(self, mock_import_layer): - """Test upload layer.""" - view = UploadLayerAPI.as_view() - file_path = absolute_path( - 'frontend', 'tests', 'data', 'polygons.zip' - ) + def _get_request(self, file_path, file_name=None): + """Get request for test upload.""" with open(file_path, 'rb') as data: file = SimpleUploadedFile( content=data.read(), - name=data.name, + name=file_name if file_name else data.name, content_type='multipart/form-data' ) request = self.factory.post( @@ -106,6 +101,25 @@ def test_upload_layer(self, mock_import_layer): } ) request.user = self.superuser + return request + + def _check_error( + self, response, error_detail, status_code = 400, + error_key='Invalid uploaded file' + ): + """Check for error in the response.""" + self.assertEqual(response.status_code, status_code) + self.assertIn(error_key, response.data) + self.assertEqual(str(response.data[error_key]), error_detail) + + @mock.patch('layers.tasks.import_layer.import_layer.delay') + def test_upload_layer(self, mock_import_layer): + """Test upload layer.""" + view = UploadLayerAPI.as_view() + file_path = absolute_path( + 'frontend', 'tests', 'data', 'polygons.zip' + ) + request = self._get_request(file_path) response = view(request) self.assertEqual(response.status_code, 200) self.assertIn('id', response.data) @@ -173,3 +187,43 @@ def test_upload_pmtile(self): self.assertIsNotNone(self.input_layer.url) self.layer.refresh_from_db() self.assertIsNotNone(self.layer.pmtile) + + def test_upload_invalid_type(self): + """Test upload with invalid file type.""" + view = UploadLayerAPI.as_view() + file_path = absolute_path( + 'frontend', 'tests', 'data', 'polygons.zip' + ) + request = self._get_request(file_path, 'test.txt') + response = view(request) + self._check_error( + response, + 'Unrecognized file type! Please upload the zip of shapefile!', + 400 + ) + + def test_upload_invalid_shapefile(self): + """Test upload with invalid shapefile.""" + view = UploadLayerAPI.as_view() + file_path = absolute_path( + 'core', 'tests', 'data', 'shp_no_shp.zip') + request = self._get_request(file_path) + response = view(request) + self._check_error( + response, + 'Missing required file(s) inside zip file: \n- shp_1_1.shp', + 400 + ) + + def test_upload_invalid_crs(self): + """Test upload with invalid crs.""" + view = UploadLayerAPI.as_view() + file_path = absolute_path( + 'core', 'tests', 'data', 'shp_3857.zip') + request = self._get_request(file_path) + response = view(request) + self._check_error( + response, + 'Incorrect CRS type: epsg:3857!', + 400 + ) From a4342e1173cd8baf63cfac709f081ec02402d8ea Mon Sep 17 00:00:00 2001 From: Danang Massandy Date: Tue, 31 Dec 2024 09:53:10 +0000 Subject: [PATCH 3/4] fix display validation message --- django_project/frontend/api_views/layers.py | 4 +++- django_project/frontend/src/store/uploadSlice.ts | 6 +++++- django_project/frontend/tests/api_views/test_layers.py | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/django_project/frontend/api_views/layers.py b/django_project/frontend/api_views/layers.py index 0a95448a..b49512f9 100644 --- a/django_project/frontend/api_views/layers.py +++ b/django_project/frontend/api_views/layers.py @@ -150,7 +150,9 @@ def post(self, request): if not is_valid_crs: collection.close() self._on_validation_error( - f'Incorrect CRS type: {crs}!', tmp_file_obj_list) + f'Incorrect CRS type: {crs}! Please use epsg:4326 (WGS84)!', + tmp_file_obj_list + ) # close collection collection.close() diff --git a/django_project/frontend/src/store/uploadSlice.ts b/django_project/frontend/src/store/uploadSlice.ts index 70c5ca87..60e00c03 100644 --- a/django_project/frontend/src/store/uploadSlice.ts +++ b/django_project/frontend/src/store/uploadSlice.ts @@ -48,7 +48,11 @@ export const uploadFile = createAsyncThunk( }); return response.data; } catch (error: any) { - return rejectWithValue(error.response?.data || 'Upload failed'); + let error_msg = error.response?.data || 'Upload failed'; + if (typeof error_msg === 'object') { + error_msg = Object.values(error_msg)[0]; + } + return rejectWithValue(error_msg); } } ); diff --git a/django_project/frontend/tests/api_views/test_layers.py b/django_project/frontend/tests/api_views/test_layers.py index 2767ba76..29fcbdc8 100644 --- a/django_project/frontend/tests/api_views/test_layers.py +++ b/django_project/frontend/tests/api_views/test_layers.py @@ -224,6 +224,6 @@ def test_upload_invalid_crs(self): response = view(request) self._check_error( response, - 'Incorrect CRS type: epsg:3857!', + 'Incorrect CRS type: epsg:3857! Please use epsg:4326 (WGS84)!', 400 ) From 1b1618518b7f73fa6fd30f8a5e7807356265bd89 Mon Sep 17 00:00:00 2001 From: Danang Massandy Date: Fri, 24 Jan 2025 05:57:27 +0000 Subject: [PATCH 4/4] use fiona utils from cloudnativeGIS --- .../core/tests/data/country.geojson | 8 - django_project/core/tests/data/gpkg.gpkg | Bin 106496 -> 0 bytes django_project/core/tests/data/shp.zip | Bin 1029 -> 0 bytes .../core/tests/data/shp_no_dbf_shx.zip | Bin 272 -> 0 bytes django_project/core/tests/test_fiona.py | 233 ------------------ django_project/core/utils/__init__.py | 0 django_project/core/utils/fiona.py | 190 -------------- django_project/frontend/api_views/layers.py | 14 +- .../frontend/tests/api_views/test_layers.py | 5 +- .../tests/data/shp_3857.zip | Bin .../tests/data/shp_no_shp.zip | Bin 11 files changed, 9 insertions(+), 441 deletions(-) delete mode 100644 django_project/core/tests/data/country.geojson delete mode 100644 django_project/core/tests/data/gpkg.gpkg delete mode 100644 django_project/core/tests/data/shp.zip delete mode 100644 django_project/core/tests/data/shp_no_dbf_shx.zip delete mode 100644 django_project/core/tests/test_fiona.py delete mode 100644 django_project/core/utils/__init__.py delete mode 100644 django_project/core/utils/fiona.py rename django_project/{core => frontend}/tests/data/shp_3857.zip (100%) rename django_project/{core => frontend}/tests/data/shp_no_shp.zip (100%) diff --git a/django_project/core/tests/data/country.geojson b/django_project/core/tests/data/country.geojson deleted file mode 100644 index ce80df85..00000000 --- a/django_project/core/tests/data/country.geojson +++ /dev/null @@ -1,8 +0,0 @@ -{ -"type": "FeatureCollection", -"name": "level_0", -"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } }, -"features": [ -{ "type": "Feature", "properties": { "name_0": "Pakistan", "code_0": "PAK" }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 70.659620789430747, 32.624727824714725 ], [ 70.825122822992526, 32.283808843198763 ], [ 70.051157430747779, 32.048938248883729 ], [ 69.968406413966889, 32.353740618315406 ], [ 70.143643861267577, 32.645223872614423 ], [ 70.42110315282703, 32.821295871012296 ], [ 70.659620789430747, 32.624727824714725 ] ] ] ] } } -] -} diff --git a/django_project/core/tests/data/gpkg.gpkg b/django_project/core/tests/data/gpkg.gpkg deleted file mode 100644 index b93b3836ac690cdc5f680b999e04f5f657df2df9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 106496 zcmeI5e{37qeZY^RElSo8+DRNXkzGHtQAo}wi9aOkhb5VmbfVZuq(oAY71bp!40@xF(eyM4A?KE%dj*-J2V4Qv}hBw zK$~IN_kNMbACf8CsjcLDrgeAkecyZE_j%v<^T>ONg;`$aNVXv6S(yZ=qm;!$1qq=j zsuTX-5C4sWz{Mfs2F@0Wa=oV9hvy#(o7~NpomB7ACl7;5ll|w4`^W6>X%}W8yhZ{@ z00|%gB!C2v01`j~NB{{S0VIF~J|F@|PeR1@qaEo5G-19=V^$c~q=^DR%aw(i3L;Z?+8BXTY%zPqz^sO)x3r{9ioSLVTo(d%w=hWMj zZ+NKRNhjuKn0PEQRTeoL7##8i25JBBP_KVr(C-fnoDI;S#pFyZ9!V~*IF*>4v~$oK z@H#JEtmAVw;6JPQ%*P{h45StoIZoK%(`!!JTSt6o*y~fo7o(BniZjElf;dR_dVPca z0|EcAf5109+#jf?!?4fqS9Q1$flSV_GB2-ZAaFDmPtJg9nsY%|)!ms;B3V!4eoflW&UrmzevFc4KL=Z(Il-EQ@)fh%dzsh$VvLG-|P4H_(<@9zx^`X31{WS{%~D~Fy6o_{s;<7X4!3NC!R?eZ(9#)84`z5GJ^>+I3D zjzGRF_McJk!2<~(0VIF~kN^@u0!RP}AOR$R1dsp{_%IPTV(n_xR}OIg|6wXv%nK4g z0!RP}AOR$R1dsp{Kmter2_OLz0n_|n`~LsCl>J?kKVBjMB!C2v01`j~NB{{S0VIF~ zkN^@u0tcNyo5k8?(I5X?@%jHjFHY1S2_OL^fCP{L5`P=#Tqh$A$dd;%qW9 zADdkc$D+pj|9(Q*e{zRs1=T|WNB{{S0VIF~kN^@u0!RP}AOR$R1R5q_Kh$;Hc=w;O z8i3#bZx|g4A^{|T1dsp{Kmter2_OL^fCP{L5;#}{s=ohE9X(j;qqax@2_OL^fCP{L z5}wYHbqF4=yg_0-|NI=pdcsOg6w zFxNotI^SvfN&{jjvOfY>r;ggX7JFK*9g{_Vb(IrWi@s4G51BFhW% z2+?z?W{;#D@41DHQUl9Cw+^rPhlXP@@{sbs^ToP(rvkKc(<5WJy`(CV^@?-Uv{W3iWV$D{-2n zM~PR5IohegJlp`x8??xBy1Gr5VBThXEGt&m8XLfOzN6WZ=&?||kl{9^$8)^QrPy`3 zpx&pd%`WAucIn)iY_~ajdMuwCRx4ArfAxKbVY#7+S%y^m61N&jN?`LGNir9bBpQSN zi?g$|DXWoal2MwDl3ztN+s${4<5$;YXhEt1tId~IqwD+a7O==~LjBxKN&FR#OvM(J z43$*UJh$q5PR62%B(zD$#R~|$0Kv0_%eBjSvE7EO?A{mn}TH_SU#UJ(Ndbn zNH|&@gNMYKX=sn4lT1Rx4egoC39@8D?eXk<{D{plFkpF7i?X_0QW96wjw^0$dzPjm zVa`ouiJJ3FOQMwGGqr`2=Y`Z}4P3=_tJam>tS1YuwX&5cSLM8P%UD(wlNvWocoOLw~O@P!*}Da_bN}ZMKrdXk=j#I{I=YtdNH)s>lnK1#c2= zsAg7MQ#y6?wiIb?c3d3X)0RqwqFY4DU)Ls%L|XPAmDYz;r_j26?_ryxug|i*QZ~%O zstBc}wO6VFW{2ilgEV`a%4Ls+C(E(&y2wE)>q5D6m|~_wusfOB^LRqd&5p&sJ#m@S z)$5(v$+mN%$>wmmEKhEf^Hz~BC&SBZXnSM`MvH7psY;bHG@pS+pU?7~sCj7C zOJ_JKE%HT~htaUw;TlIe_gd&)V_DeQ#OvOen7STDz~Ou zQ>l{BdXG9|sa;xBbDKyP#8p<{uRwQXSVpOpHG4tDDTo>9s_J&%87|8Uyc$K7Iab@9 zr=#VcsSfHjs-3r8Yjz)c-TK+4e(E*5t=-Y)IO1qM+49DbA0ElJzPg_oD@1v1Kk^@h z?A2w5HfKJ!+x)~lRJ_sYUY$5fa}<5T7n>$#soOT?t571pE~zu9M1i=;%e+@e;9-L@vRQj2;EY4z77^_7kC!sN_d8e2zWY;Vv|E7ZNiTGR^N>Mp7^ zhO*;Zl-)F(7(3bQSU9ui2$S1{Wf=)ZTarG5C_CJY_kICI9b(_J1Smc7-{Vfps% z+JRx&vt|}iGc%`4!u2NOrV(A``ka(5)XX_dbMI9yC8s%3J7=z#aMPqRagS&Fi7se5 zyIOXR>n5q6QW-AC!CGa-{4J&LF0G#o4*HV@XCpzKBs86bwZ$oTZ~&|1(D^v^*qr(! z1-H%5J&7~*j(w4WkpSZt%{Vy&W;fv3vKStv=tbfsR?#eYAL*(xtwGqAX!kgNK(lMqvB$$83(S zF3ZlOX7OvqlrQDe+k|5k7GAEHZsN@H8E0wv-25~lw3v)Vq7ZP7F|U*or?LR&By+IL z6`7CCE{9`LnmA3dz+JkK;Zk0wI(2ssc|G8yRHG^d1 zHf5jbd~9^rY)`WEc__R>Yq zzklYd3&Fn7!~byjFBgJ;pLq7ip;ylbzqIq|$KDACgO+B=td|*$_Rbs zA3xjrKPQ916F>OIQuI&Du~o}ky{Q0veCkKL-X3iN^=7KndwT6_7j}MsAvkvGmCJ44 zUI_lpFTeiaNFo?yUvGcmh`+xq23#C-5<7@J!{P6LqAMIj$zyj*KTkk#z|L!*E{??&a zzW-cyAvpZM|8#odOAEoD@vr>XOiwWQpD+FP=b0z!sec&MfBe#$-S=RDqnng`sw?k!C-ZGHsxhg#X@dNIs5cWpEu=$B@Si&Z?V5i z!3Pf{fCP{L5l1e6=L2f=qAYS8{E83WAGNP6sNq!AdkR8sOL_ml)&4wX|6BX>_BRhs z5u>I^00|%gB!C2v01`j~NB{{S0VIF~4mg2dI}D2#mWnk5<;Aph)-J5%Ff2Y)idKYc zB8Oq|pmmS61NHg8{il@uJ^Rn$ga;Bp0!RP}AOR$R1dsp{Kmter2_OL^@L?cewOXlG zYm@Q^Ug^-P{{FwMhk_3tNB{{S0VIF~kN^@u0!RP}mJZ#hIg{{T2CFGEuf%f!?D zU!Hj8>vv699-E%9zHN-6nAc0>w0cHMA!FwR_Kmter2_OL^fCP{L5vya+%P%^;M$Rqx+W-T_fj_q>Ik?*9wmDob z%g$nXZ)4;278SoCxvygP%L*^z<};<;FHO4+=TpT?q+&PB#3YHvk|eq~J4@#?sXDtB z!`8pkA=nj@xRjT5NSy3a1!Y#quwo{)F7oPrn5JlW*b-CTs^0y$9FGj|?daRg=O7~e z6(E%{lqFrB?OB(t>wH(s)rh`3t@e5i{f9Ft<*gg~*2lV&0mazlcvpF2XWx1|8!yc; zQBufddi7mF4Mh5u$9j-7*(AE`#?VA^N z&yKdkZhzzX(E}J6GInD(Z`G`3sH6%xJw~F;lBO34MW={cz16lrjU*InAh9?}BvTQI z$rt4UnYTD#RLpMg`%)hjNw+1sjv@Y zkLfsxGxM{dNk*x{vHBMFaAAE4CT0vfW>;=~fhwQi1cI(2TY9dN_6 zsp1EZbeXDWr(iQAcsU_qxg`QhTbRjAuOC&8KXRu`$c`soj(i3$V3&9W7M% zTsS;7jo-)!Q=o{AO*YJUuJXboKX{K8D$J26+ySFd%34O-_xB*cF&A-&y93vA92wxp|Iidc$8^Lme0Yv_sVl0 zW8kj%X)TY*mp09e(qdjr<+FS$dx=cP;!GqQB@Z#n#9fZm1H-auCeB1BVc=E42&$B) zHnL*f$Vyu59i6AczC>ACTd7t!KAIMdxkds=00|%gB!C1yvIL%asL4ikjh+0Ru)hAR zzg0OerqjLYLcZ_uRbJ}N$QffErR@dQe)(}JKQgK%n&Bygzj`pc{sxvR=6j@n4D=zG#Fd}BtGB}nl)rfiFlGLZ_ zndLYMyw#K91@G4`WO!i}+)9^tfdAO2_tjx1H801S^5)t6$GB;fi^Q=7PT)HHy z*;fTOF3ICzCKjHICsv$G;RG4#chXEW!OTs}GT>4Wa~TrmU|K8lX_DBIWG+v9!$bXq z6C^JGXpVEzb0Eyb_Y`&~MOKh_+2g#}ka&17;B8oTXfV*Qtj{;lc*MT5gZ+)iJb2dM zu(0137*>Rt+1bc^A{Ln{!#^7s9P$MQY5(w0uYX|B?+*-|4bZbpG@P7Sj7E|xPI%HH z0(Rd;2(On^XmWBf9-3TU@%MVcSy`lE=2o0xczwCB!Kc@pw3iM=!?TPQLxx)gA8_pT z`Ud+40{&tDfNyxXKM;VxlZh2Bm*b04fzPlx+JBJ_U5J2nz!L?{3UXjh@_L3N?pYQ} zlokr2yhg|3Q(y!>kZv%Y>R1cHsu68LfM_{ZlJ!^>9U6>vF%b$gD~ZY2JecoHLFBI# z1ewhdXkCg00qQ#`WJzF(K)nD{?u9Z4#hH+9yvpswMC`&!&#;&7@eO%tIKv63C`H7~ zC&Es;Kj1%m@#1KoQgzhY2KB8d79Q&#Kck$+a*z=~M9x6%U(b(0rGnaWMkv6ut*ivC zh$?CXl%RRaj0gG${G)x^b=e0zl(ZBNxHR0^b$JbL6cynJ`i1LhyC9ui~U8) z{v!Or0|_7jB!C2v01`j~NB{{S0VIF~kN^_6T?BfXS{z*zHD;mIuRiEsK=XV3{vMyN b$2XYt_K)-rjQIV%{odh6nn2Co##Q})S5y>l diff --git a/django_project/core/tests/data/shp.zip b/django_project/core/tests/data/shp.zip deleted file mode 100644 index 78b02747dc34abd9c41f79ed0096b786cf9c0ac1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1029 zcmWIWW@Zs#-~hs-ECrzqP;i)yfq|PrfuT5~Al@+EP%kAZEi{CefxYQmaWV{-R&X;g zvbmEHzPfJKhNU1q{qJcGK*@2mL3QgR;c9!<4ruP`ml&}dhW_+sb+*$GT z;4faz7sr+7&tx){-M&ZBt}>3*!oa7;JYa#EN00uA^DCFkTiE%r?TQgsmfVfn5aTNi zr`{Pa*;L7qkRp)p63VK%WxyVM4(3qyv2a_9g(m$!fGKqR(#D6tNct8auwveH>cN z9{qE1>Ye3GvO@1a=-7O^aFDURF698n5xx^?wP$UrZaZH;c<|GCN!R6>jeO?YeKX3} zmKHE*rnF{ho!M8fyFlT>!++0TYl&|%xccgzF0a5^w;8O))u#lv=Xu(*vzPpJP@dUv zgE^(H{MeX3&R{VbTc z_Ps<|Pigjo!&{#1SmXarpnRjlUDjak$;~U*THQPFGe#_t|BOz_S8ym_dlJ{q3k*k4 zSYi+58ekkCr=#MG3L-=KO`L(T!3HMDSRHqHUoY+w3z0j&&wX6Aw|VWKi$?E)8~HlR zE5zg~Up#1Vwl^|hN}90CNZ}j9BWH&kMzGs1T%6A94745Of&gzuCJ|;tDo0KoAlJdb zmPQbZgfx$CFmiyP8vF>!U?P$ly1~f4K{eQe3E^TQLN~yh6__O$7?^>O8yIa*fZ7=t E03_^Qq5uE@ diff --git a/django_project/core/tests/data/shp_no_dbf_shx.zip b/django_project/core/tests/data/shp_no_dbf_shx.zip deleted file mode 100644 index 91c20f5be1d1825127f32defbab98aa1a21bc876..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 272 zcmWIWW@Zs#U|`^25D&`>T~#%CaT<{K28g*BWEe_Pi%a5-^olbILPIzinCI>)Oa|f7 z3T_5QmKV$n3}7NTA>et^0lrgl2F3;(m?ZP}X}cUNVY50?b#DE?6;B={EY&JKUF)@g z`FY^UEdFYCc9~xj8I=XB1(L-)6tzPpr#@TAy47K8JJ)x=Uun-COvsp4R+DD$r0etk z`Q-R{FBqR*JnMFypCQ1Tkx7mjms=%(E{1?DjUXD?<*X2wqq#c3o0Scuk`V|4fb=R5 GhXDYK1yS$- diff --git a/django_project/core/tests/test_fiona.py b/django_project/core/tests/test_fiona.py deleted file mode 100644 index 7244c047..00000000 --- a/django_project/core/tests/test_fiona.py +++ /dev/null @@ -1,233 +0,0 @@ -# coding=utf-8 -""" -Africa Rangeland Watch (ARW). - -.. note:: Unit tests for Fiona. -""" - -import os -from django.test import TestCase -from django.core.files.uploadedfile import ( - InMemoryUploadedFile, - TemporaryUploadedFile -) - -from core.settings.utils import absolute_path -from core.utils.fiona import ( - FileType, - validate_shapefile_zip, - open_fiona_collection, - validate_collection_crs, - delete_tmp_shapefile -) - - -class TestUtilsFiona(TestCase): - """Test class for Fiona utility functions.""" - - def test_validate_shapefile_zip(self): - """Test validate shapefile.""" - # test incomplete zip - shape_file_path = absolute_path( - 'core', - 'tests', - 'data', - 'shp_no_shp.zip' - ) - is_valid, error = validate_shapefile_zip(shape_file_path) - self.assertFalse(is_valid) - self.assertEqual(len(error), 1) - self.assertEqual(error[0], 'shp_1_1.shp') - shape_file_path = absolute_path( - 'core', - 'tests', - 'data', - 'shp_no_dbf_shx.zip' - ) - is_valid, error = validate_shapefile_zip(shape_file_path) - self.assertFalse(is_valid) - self.assertEqual(len(error), 2) - self.assertEqual(error[0], 'test_2.shx') - self.assertEqual(error[1], 'test_2.dbf') - # test complete zip - shape_file_path = absolute_path( - 'core', - 'tests', - 'data', - 'shp.zip' - ) - is_valid, error = validate_shapefile_zip(shape_file_path) - self.assertTrue(is_valid) - # test using in memory file - file_stats = os.stat(shape_file_path) - with open(shape_file_path, 'rb') as file: - mem_file = InMemoryUploadedFile( - file, None, 'shp.zip', 'application/zip', - file_stats.st_size, None) - is_valid, error = validate_shapefile_zip(mem_file) - self.assertTrue(is_valid) - # test using temporary uploaded file - with open(shape_file_path, 'rb') as file: - tmp_file = TemporaryUploadedFile( - 'shp.zip', 'application/zip', file_stats.st_size, 'utf-8') - with open(tmp_file.temporary_file_path(), 'wb+') as wfile: - wfile.write(file.read()) - is_valid, error = validate_shapefile_zip(tmp_file) - self.assertTrue(is_valid) - - def test_open_fiona_collection_shp(self): - """Test open fiona collection for shapefile.""" - shape_file_path = absolute_path( - 'core', - 'tests', - 'data', - 'shp.zip' - ) - file_stats = os.stat(shape_file_path) - - # test using filepath - collection = open_fiona_collection( - shape_file_path, FileType.SHAPEFILE) - self.assertEqual(len(collection), 3) - collection.close() - - # test using InMemoryUploadedFile - with open(shape_file_path, 'rb') as file: - mem_file = InMemoryUploadedFile( - file, None, 'shp.zip', 'application/zip', - file_stats.st_size, None) - collection = open_fiona_collection( - mem_file, FileType.SHAPEFILE) - self.assertEqual(len(collection), 3) - collection.close() - delete_tmp_shapefile(collection.path) - - # test using TemporaryUploadedFile - with open(shape_file_path, 'rb') as file: - tmp_file = TemporaryUploadedFile( - 'shp.zip', 'application/zip', file_stats.st_size, 'utf-8') - with open(tmp_file.temporary_file_path(), 'wb+') as wfile: - wfile.write(file.read()) - collection = open_fiona_collection( - tmp_file, FileType.SHAPEFILE) - self.assertEqual(len(collection), 3) - collection.close() - delete_tmp_shapefile(collection.path) - - def test_open_fiona_collection_gpkg(self): - """Test open fiona collection for gpkg.""" - gpkg_file_path = absolute_path( - 'core', - 'tests', - 'data', - 'gpkg.gpkg' - ) - file_stats = os.stat(gpkg_file_path) - - # test using filepath - collection = open_fiona_collection( - gpkg_file_path, FileType.GEOPACKAGE) - self.assertEqual(len(collection), 3) - collection.close() - - # test using InMemoryUploadedFile - with open(gpkg_file_path, 'rb') as file: - mem_file = InMemoryUploadedFile( - file, None, 'gpkg.gpkg', 'application/geopackage+sqlite3', - file_stats.st_size, None) - collection = open_fiona_collection( - mem_file, FileType.GEOPACKAGE) - self.assertEqual(len(collection), 3) - collection.close() - - # test using TemporaryUploadedFile - with open(gpkg_file_path, 'rb') as file: - tmp_file = TemporaryUploadedFile( - 'gpkg.gpkg', 'application/geopackage+sqlite3', - file_stats.st_size, 'utf-8') - with open(tmp_file.temporary_file_path(), 'wb+') as wfile: - wfile.write(file.read()) - collection = open_fiona_collection( - tmp_file, FileType.GEOPACKAGE) - self.assertEqual(len(collection), 3) - collection.close() - - def test_open_fiona_collection_geojson(self): - """Test open fiona collection for geojson.""" - geojson_file_path = absolute_path( - 'core', - 'tests', - 'data', - 'country.geojson' - ) - file_stats = os.stat(geojson_file_path) - - # test using filepath - collection = open_fiona_collection( - geojson_file_path, FileType.GEOJSON) - self.assertEqual(len(collection), 1) - collection.close() - - # test using InMemoryUploadedFile - with open(geojson_file_path, 'rb') as file: - mem_file = InMemoryUploadedFile( - file, None, 'country.geojson', - 'application/geo+json', - file_stats.st_size, None) - collection = open_fiona_collection( - mem_file, FileType.GEOJSON) - self.assertEqual(len(collection), 1) - collection.close() - - # test using TemporaryUploadedFile - with open(geojson_file_path, 'rb') as file: - tmp_file = TemporaryUploadedFile( - 'country.geojson', 'application/geo+json', - file_stats.st_size, 'utf-8') - with open(tmp_file.temporary_file_path(), 'wb+') as wfile: - wfile.write(file.read()) - collection = open_fiona_collection( - tmp_file, FileType.GEOJSON) - self.assertEqual(len(collection), 1) - collection.close() - - def test_validate_collection_crs(self): - """Test validate crs.""" - # test invalid crs shp - shape_file_path = absolute_path( - 'core', - 'tests', - 'data', - 'shp_3857.zip' - ) - collection = open_fiona_collection( - shape_file_path, FileType.SHAPEFILE) - is_valid, crs = validate_collection_crs(collection) - collection.close() - self.assertFalse(is_valid) - self.assertEqual(crs, 'epsg:3857') - - # test valid crs - shape_file_path = absolute_path( - 'core', - 'tests', - 'data', - 'gpkg.gpkg' - ) - collection = open_fiona_collection( - shape_file_path, FileType.GEOPACKAGE) - is_valid, _ = validate_collection_crs(collection) - collection.close() - self.assertTrue(is_valid) - - shape_file_path = absolute_path( - 'core', - 'tests', - 'data', - 'country.geojson' - ) - collection = open_fiona_collection( - shape_file_path, FileType.GEOJSON) - is_valid, _ = validate_collection_crs(collection) - collection.close() - self.assertTrue(is_valid) diff --git a/django_project/core/utils/__init__.py b/django_project/core/utils/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/django_project/core/utils/fiona.py b/django_project/core/utils/fiona.py deleted file mode 100644 index d1f73a24..00000000 --- a/django_project/core/utils/fiona.py +++ /dev/null @@ -1,190 +0,0 @@ -# coding=utf-8 -""" -Africa Rangeland Watch (ARW). - -.. note:: Fiona shapefile helper class. -""" -import os -import zipfile -import fiona -from fiona.crs import from_epsg -from fiona.collection import Collection -from django.core.files.temp import NamedTemporaryFile -from django.core.files.uploadedfile import ( - InMemoryUploadedFile, - TemporaryUploadedFile -) - - -class FileType: - """File types.""" - - GEOJSON = 'geojson' - SHAPEFILE = 'shapefile' - GEOPACKAGE = 'geopackage' - - -def _open_collection(fp: str, type: str) -> Collection: - """Open collection from file path.""" - if type == FileType.SHAPEFILE: - file_path = f'zip://{fp}' - result = fiona.open(file_path, encoding='utf-8') - else: - result = fiona.open(fp, encoding='utf-8') - return result - - -def delete_tmp_shapefile(file_path: str): - """Delete temporary shapefile.""" - if file_path.endswith('.zip'): - cleaned_fp = file_path - if '/vsizip/' in file_path: - cleaned_fp = file_path.replace('/vsizip/', '') - if os.path.exists(cleaned_fp): - os.remove(cleaned_fp) - - -def _store_zip_memory_to_temp_file(file_obj: InMemoryUploadedFile): - """Store in-memory shapefile to temporary file.""" - with NamedTemporaryFile(delete=False, suffix='.zip') as temp_file: - for chunk in file_obj.chunks(): - temp_file.write(chunk) - path = f'zip://{temp_file.name}' - return path - - -def _read_layers_from_memory_file(fp: InMemoryUploadedFile): - """Read layers from memory file of shapefile.""" - layers = [] - file_path = None - - try: - with NamedTemporaryFile( - mode='wb+', delete=False, suffix='.zip' - ) as destination: - file_path = destination.name - for chunk in fp.chunks(): - destination.write(chunk) - - layers = fiona.listlayers(f'zip://{file_path}') - except Exception: - pass - finally: - if file_path and os.path.exists(file_path): - os.remove(file_path) - return layers - - -def _list_layers_shapefile(fp: str): - """Get layer list from shapefile.""" - layers = [] - try: - if isinstance(fp, InMemoryUploadedFile): - layers = _read_layers_from_memory_file(fp) - elif isinstance(fp, TemporaryUploadedFile): - layers = fiona.listlayers( - f'zip://{fp.temporary_file_path()}' - ) - else: - layers = fiona.listlayers(f'zip://{fp}') - except Exception: - pass - return layers - - -def validate_shapefile_zip(layer_file_path: any): - """ - Validate if shapefile zip has correct necessary files. - - Note: fiona will throw exception only if dbf or shx is missing - if there are 2 layers inside the zip, and 1 of them is invalid, - then fiona will only return 1 layer. - """ - layers = _list_layers_shapefile(layer_file_path) - is_valid = len(layers) > 0 - error = [] - names = [] - with zipfile.ZipFile(layer_file_path, 'r') as zipFile: - names = zipFile.namelist() - shp_files = [n for n in names if n.endswith('.shp')] - shx_files = [n for n in names if n.endswith('.shx')] - dbf_files = [n for n in names if n.endswith('.dbf')] - - if is_valid: - for filename in layers: - if f'{filename}.shp' not in shp_files: - error.append(f'{filename}.shp') - if f'{filename}.shx' not in shx_files: - error.append(f'{filename}.shx') - if f'{filename}.dbf' not in dbf_files: - error.append(f'{filename}.dbf') - else: - distinct_files = ( - [ - os.path.splitext(shp)[0] for shp in shp_files - ] + - [ - os.path.splitext(shx)[0] for shx in shx_files - ] + - [ - os.path.splitext(dbf)[0] for dbf in dbf_files - ] - ) - distinct_files = list(set(distinct_files)) - if len(distinct_files) == 0: - error.append('No required .shp file') - else: - for filename in distinct_files: - if f'{filename}.shp' not in shp_files: - error.append(f'{filename}.shp') - if f'{filename}.shx' not in shx_files: - error.append(f'{filename}.shx') - if f'{filename}.dbf' not in dbf_files: - error.append(f'{filename}.dbf') - is_valid = is_valid and len(error) == 0 - return is_valid, error - - -def _get_crs_epsg(crs): - """Get crs from crs dict.""" - return crs['init'] if 'init' in crs else None - - -def open_fiona_collection(file_obj, type: str) -> Collection: - """Open file_obj using fiona. - - :param file_obj: file - :type file_obj: file object - :param type: file type from FileType - :type type: str - :return: fiona collection object - :rtype: Collection - """ - # if less than <2MB, it will be InMemoryUploadedFile - if isinstance(file_obj, InMemoryUploadedFile): - if type == FileType.SHAPEFILE: - # fiona having issues with reading ZipMemoryFile - # need to store to temp file - tmp_file = _store_zip_memory_to_temp_file(file_obj) - return fiona.open(tmp_file) - else: - return fiona.open(file_obj.file) - else: - # TemporaryUploadedFile or just string to file path - if isinstance(file_obj, TemporaryUploadedFile): - file_path = ( - f'zip://{file_obj.temporary_file_path()}' if - type == FileType.SHAPEFILE else - f'{file_obj.temporary_file_path()}' - ) - return fiona.open(file_path) - else: - return _open_collection(file_obj, type) - - -def validate_collection_crs(collection: Collection): - """Validate crs to be EPSG:4326.""" - epsg_mapping = from_epsg(4326) - valid = _get_crs_epsg(collection.crs) == epsg_mapping['init'] - crs = _get_crs_epsg(collection.crs) - return valid, crs diff --git a/django_project/frontend/api_views/layers.py b/django_project/frontend/api_views/layers.py index b49512f9..0baf0b3e 100644 --- a/django_project/frontend/api_views/layers.py +++ b/django_project/frontend/api_views/layers.py @@ -18,6 +18,13 @@ from cloud_native_gis.models import Layer, LayerUpload from cloud_native_gis.utils.main import id_generator from django.shortcuts import get_object_or_404 +from cloud_native_gis.utils.fiona import ( + FileType, + validate_shapefile_zip, + validate_collection_crs, + delete_tmp_shapefile, + open_fiona_collection +) from layers.models import InputLayer, DataProvider, LayerGroupType from frontend.serializers.layers import LayerSerializer @@ -25,13 +32,6 @@ import_layer, detect_file_type_by_extension ) -from core.utils.fiona import ( - FileType, - validate_shapefile_zip, - validate_collection_crs, - delete_tmp_shapefile, - open_fiona_collection -) class LayerAPI(APIView): diff --git a/django_project/frontend/tests/api_views/test_layers.py b/django_project/frontend/tests/api_views/test_layers.py index 29fcbdc8..9ee9628e 100644 --- a/django_project/frontend/tests/api_views/test_layers.py +++ b/django_project/frontend/tests/api_views/test_layers.py @@ -13,7 +13,6 @@ from cloud_native_gis.models.layer_upload import LayerUpload from core.settings.utils import absolute_path -from core.factories import UserF from core.tests.common import BaseAPIViewTest from layers.models import ( InputLayer, InputLayerType, @@ -206,7 +205,7 @@ def test_upload_invalid_shapefile(self): """Test upload with invalid shapefile.""" view = UploadLayerAPI.as_view() file_path = absolute_path( - 'core', 'tests', 'data', 'shp_no_shp.zip') + 'frontend', 'tests', 'data', 'shp_no_shp.zip') request = self._get_request(file_path) response = view(request) self._check_error( @@ -219,7 +218,7 @@ def test_upload_invalid_crs(self): """Test upload with invalid crs.""" view = UploadLayerAPI.as_view() file_path = absolute_path( - 'core', 'tests', 'data', 'shp_3857.zip') + 'frontend', 'tests', 'data', 'shp_3857.zip') request = self._get_request(file_path) response = view(request) self._check_error( diff --git a/django_project/core/tests/data/shp_3857.zip b/django_project/frontend/tests/data/shp_3857.zip similarity index 100% rename from django_project/core/tests/data/shp_3857.zip rename to django_project/frontend/tests/data/shp_3857.zip diff --git a/django_project/core/tests/data/shp_no_shp.zip b/django_project/frontend/tests/data/shp_no_shp.zip similarity index 100% rename from django_project/core/tests/data/shp_no_shp.zip rename to django_project/frontend/tests/data/shp_no_shp.zip