From 49d555cb73d4e7e7bdaa2ea1c789066335a68d56 Mon Sep 17 00:00:00 2001 From: mungitoperrito Date: Mon, 9 Sep 2024 15:33:02 -0400 Subject: [PATCH 01/67] framework --- .../configuration/indexing/_category_.json | 4 +++ .../weaviate/configuration/indexing/index.md | 27 +++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 developers/weaviate/configuration/indexing/_category_.json create mode 100644 developers/weaviate/configuration/indexing/index.md diff --git a/developers/weaviate/configuration/indexing/_category_.json b/developers/weaviate/configuration/indexing/_category_.json new file mode 100644 index 0000000000..7225cca315 --- /dev/null +++ b/developers/weaviate/configuration/indexing/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "Indexing", + "position": 20 +} \ No newline at end of file diff --git a/developers/weaviate/configuration/indexing/index.md b/developers/weaviate/configuration/indexing/index.md new file mode 100644 index 0000000000..f8c9545389 --- /dev/null +++ b/developers/weaviate/configuration/indexing/index.md @@ -0,0 +1,27 @@ +--- +title: Indexing +sidebar_position: 0 +image: og/docs/configuration.jpg +# tags: ['configuration'] +--- + +This section discusses indexing Weaviate collections. + +## Vector indexes + +- [Overview](add) +- [Hierarchical Navigable Small World (HNSW) indexes](add) +- [Flat indexes](add) +- [Dynamic indexes](add) + +## Keyword indexes + +- [Overview](add) +- [indexSearchable]](add) +- [indexFilterable]](add) +- [indexRangeFilters]](add) + +## Additional considerations + +- [Async indexing](add) +- [Index level](add) From cf3c4e73ec84849ea93b1df80b0c5779c58468ed Mon Sep 17 00:00:00 2001 From: mungitoperrito Date: Wed, 11 Sep 2024 09:57:43 -0400 Subject: [PATCH 02/67] index slide --- .../weaviate/configuration/indexing/index.md | 2 +- static/og/docs/indexes.jpg | Bin 0 -> 31632 bytes 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 static/og/docs/indexes.jpg diff --git a/developers/weaviate/configuration/indexing/index.md b/developers/weaviate/configuration/indexing/index.md index f8c9545389..aebcbd5a11 100644 --- a/developers/weaviate/configuration/indexing/index.md +++ b/developers/weaviate/configuration/indexing/index.md @@ -1,7 +1,7 @@ --- title: Indexing sidebar_position: 0 -image: og/docs/configuration.jpg +image: og/docs/indexes.jpg # tags: ['configuration'] --- diff --git a/static/og/docs/indexes.jpg b/static/og/docs/indexes.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ff6608ebe7195ee6a04f98b7bf168fbd384995f4 GIT binary patch literal 31632 zcmbrm1z1#F*FQXzAQFOrfRvOp(jkpQN_PuLD@aLqDIyID(%lGxNGsBcG$#{c3H~))0&-GH3UX31GAdfOTU696G-PB9e2gr&Ik>sGDQ^h~ z^K%NZadC4(lOSEbd>QKs7BMz9G3QOPo1Fjaf2SW{co@jqsBch^sA0%>NGNznr_C@* zuue22#6Q^iKO|%nRJ2Rz7?&}xfD0@kk(9lp(!PQ>ia~LWf8vacV@k`fK zjM1s>2{`?t(lKZx%03dR_O8=%nK<}g#w5BUiJOO)k6%Dg@-AFTT1Hk* zONX*F0%FfBn%P%N@ zTT%J0>V0)h)2HT^*0%PJ&c6PE!J*-A-$$mVXJ+T-7Z#V6H#UE7ZSU;v?H@q0a60!>zD0rx7H#sihi>sg;+h3#R^ur*Kh)OT}c$tPv zb)C?}p%;^gmV1hB1KJv5XaBQ}`Tw7G_Ge@N>}v#ug@Oc_hk^$ag>5V|WHf~NW`r`p zGKxN$Z*KlzIvm_9o(Wkk70E3o;z)LFQ}~`tp48`9Ww~u#p6HI-Fni2jSfQak7f)^X z2t#uSZHZXa(0YuXrzuutQ7kVntx4hY6T<@)4SY={`$cQ#1ijKr$zR>nETcn_y5$(C zbS34*(@+r)gdQ5o2vdO*K*N_IBNTQ%#0XU2FfgcyLK)J;<#I1Yhl(k{d2Y}wDHdzy zC}`_4WcVRvWoKm}!3k8sBn-so6G#w6_Vrl7VpZTWFn%5?fxIR07Ve+exEbLU8u(yB zd2lh4HX4D7@(1v!cyPPsL*!6~y!!+du^H)4W+ zT;QvAzfDbIg-ZE1T?PUTn6hPBAB;d5afvcwA8EaC7*%#wcK&}N3q%sC^QXyN;#R4e zc%d|)Ve_2Xlte@|;G~<-Av5(@r7%~3k(o=*F*^D!C#90)E?Gn=kFLBx=7R0ia%Cah zouic7r45g#A?Larvx%PT93|)PWx>JAKU3Hfyv>enG6-i_DMHklTXlso2UOs;55YqL zALNisU?lXuu2PuP@cIIVMS#$MwFuf|@1Ua}l4jgENT9IiKR;{+(2MkS0hZ8^> zLTn02l5kNbpXdqsV{0bHXDz~?k^=f@4U zT}ntw>se;V@RvBF-Wh2!5Lo$Fyg(oTq?fs$i_UXfU_LE3(=?#3faMu;bT6FH+*np# zR_2@oh&dpL1hNufG9u7{#*me!iX{Fq7bWUmCVx({UxpkbQWq;HcxwKAuxrVG0ZK5E z93unaYZ$k(C4h5C#27Ds08f-s;Mm=Dctrrz?wrGooqx--gpYN&cuy#JfjfgQGNF z5WnHp&$P4b9rp$pBRMOwZ;A2xd0<)#3_x3Gp??o(DKG}qfJis+yNvAnRL#M_pj1w% zzGrcNS&pfomx>A4L_6Xturg?S=L6cIF9h`~V0TmiQCM=|4La#=^kR|1fxP|6?LwvW zq|5;?Kn~8oseoWHa(tv}9}<{05)+L86NXJd$_T4Sn+6(0h^+Ts_89y+VTg(hF{y4m znED@G5R4?25gLq~D5oZ?^9GY4Cw?n`?jZ{!m6~kV&5VjLd`3tz2`89~r!e1{5Thka2nC`rwt_dPg(ThipkB*P@2m40_vtDSi zGgTN{5nF*^BbfO@;Kd-EpYIgXcZ@J~$5U7V51q8=L8c-lK9eXu6L|iJ^Z9jYm@e-Q zUv{bd#%Y*-Tj)}3I@U7#MXO51;FbGJ{i0u=)NAIxHfm!_Qm@7)n-F4?TXN`qEKA`pp4DM#2TA&$$T*buKZCu$<`F zK2`9!e|#hw9qY|}Sat_NtSZvSd>2eJR(c}JC^%$8biJ%i%3{#xSv`l5rwwu zgFY4cIGm3FPROK;3Uo4O(rsWV_1U|yaH>2riVz3K!z#=iZd3YPMU|l7`;U> zg=1lyZJ#weAK5Ej(p+Rs2e#4l|C^gyl*^|u()jf1!+;eHrD?e2G~AjYqZTLa!a!;W zO9=O(ujZ7fJC^K`d%+C^96C6cKF7;Os05*m#78<2RIWe<4jGagh_rtV2sscV&PHyP zZQ59=o(g*JM1ux{p5TXgo;;tZ##D|oZ zJkZfVZq6(z^bT_GITK)3j8;It2$X^Bo{2#=Bw%1caC)!`^^3b83x570t&i&wLD6VA zDc&dwN- zrrM|4cOT3QEe*UA0H^=M2_(QHgiIh={)@Pry?|~&a0X%JfY?ZUgfko6y={)kMD5^U zNn6&DRq{$Q&MiL9AJ|6*h$3N%p1F%87M|HUT0=VE#=lO z0Ysey(E}1=iQ{t7PeOvX*L-0e zJY%jdQ-i;L(|yfR$Zk}jJgTr;FIf{`4}Y1);A--_#Qw`>Sk=^$UUuQ{q8~3T?(}_N zW}o*sda0B;>AvS##%g0TYjO%}B2T+wL~|%2WO`%M<=AnkuC}>`eTSlquAt2F>G%zb z_ac?Q;Dw*PsgqWEPGQ4%M=K&YU&@@h(4~r{89Mk1?KFl}9rcA7dgjAat&+S5RE+Xc z`Oi5C(ZN@64b|1;hhC!I>#@oTTK0GxKEB3Em+KRlJ*BK+vgYqr(Ld7);hl)SF!ki( z?<@6Ld2^yMN>tkJ8o}H8i6_Zw&|cZP;|}Xf$z_KlRubcwSha?Q2%83+0VmbIphJbc zQ&@aic)-|`k|zy*w0HsSC29ui@Zj7Un|r!rDc&lZ9J}O=m^+){+^Ur{F=8Yi%nKox-GFf3EY*>+YP1vr8`^+^r~7D@IdIicu;MVr)|OEY~!K z6KF&e0N`T%B~(<9K)@KWS#PGHp@zq9+$xraU%w_Bvz?uXM3bIdlv1%O`Rry^CRI2? zS|oruhOV3n@ChMh4GBI#KJdY;7!eu;q4LEf;fz@bB?M{0NKBx!5I#NdL;jcqNHJWP zck*Q@lz@EySKZ<0x&(T?*vz3{uCJ&>%XZ|%q*PS)5;G*y-3QMJzF-NYx#>uT_|}-6*7VqUFf`jNB@PZ2hbmrR7>gpUJ;2$sw$((yaM4>ybk)y@S>8V zu~IEE+I&C9``=jqJKjnP;A9ZV2u9D{yYpe^nFUT}!Cm@9G4h*sH39FW9Szn;3JAqu zk72)*B|hl2qNChDQJ)^%6ES;b!m&N4Sy5W@Ji{PrYs*R^R^8;;6*YNvv;6Erzq=!| zL4{wuZ<4J<`9PTMyP}o2ceJ4Q{G=Lp!AR$0!S{?)*cG9%!Bg1VMs)`#6}b0%&-Qam z0O?HFR%vh>slmeW6Uxq7iNR)10sek=Q3#vgiAr1e<9wQm?KJjjR?2rzx{xIX`UXj? zt*kc)6eoYON$&|4bfA>M3O=d%?+oy(;qCpHzajrOl;{)_Ru`)xN@=Z%<(f)`Bo<+|9BESX!BS2gMtjqabkb(JIHK%HL*%^s>YPhS)?X(c^Qn&u1G8N z+VSvGs6KjL7ts{4QT*bLSXBVNAs+K7?6Ftx2~Pu?1cQ20-Xv0w*2v87RlQG`C5QD5 z2HkP41y+li_Uur!WKQ&5{q5_7f;t!A%>q zszr1psHT<9W-GtFskOS8dB(kE4#`j(zo*FSk8=QqXg$yy{8AJmn#R3p4VC$r3@ z=E=XFUG>b_?ZpAt_tBhr9=G1kH;H*8ZEFTM3SvGQe z)|mv{!+P}X%oW__-G*A^PwYh_BesuEVfdbe*{-%%Y`SnNbRf$GE+?4iBMyT zC6_B!)um}?v>D#E3*h}>Zqes_Iq<^+X-EGXqf}Vlm9x8=E<{@8-^cED2-%fIZeMzr zt4ktw z=UXHn^IQ{UN9KxH%}Y%?7E7fz*qdguI&yPTJuUbgpK@>pjVMV*jg+i3UN-Ib4PlOaNLeb!~TDn@Tyv344+tfBR8 z(oNuCP(7cRxjob=S7n4&6oaMWuwHd<{1a9`typt;-b1WSG0`CTjo>!LirRkNAq?Z> z!K06)LG8pJsl0g-rW!-~mqlm?oJL>&e)#Y%Zx)*wCK8>?5{|wa%HR`B1Bx47_9frvYu?SXzS6+< z7dT=Tn5JnT(Tj{x8i?vTQiQlLxKyrZFTFLH>vs8?9h?D6|_HE|t z$g=l)((QBi%JSatj3Lu3tAN zz+^~^0v6*exd}2ZRKTb(T8jZyACDwsWrYpJV^DyA`7f4=3m5{mD}McnRwh5;KIIiI0v|H97a!5iu96Pc=~vG9oA-_nSopC3!EbMnNO~j z?T|~gXFB}W(dW;?bT^*1y{6g~!rWw3bI40Opcu3+;7*1QY%`3E8uci@!M1aQjiINL z^|e3RV#SII^6$gpTFj`%4IF74tz#Qf{2tx4<9E_A>!T~eVnca<{e8Jqb5+FKh}Ccf z_6NtMZ+}x(8h#$xCo#e_Ax+6GT6z1TzwWhq>J)a&DQu{uzlC$}#lt>P9PulP3k&g@ zDED{J^Go^mM2kOJ<&9gZ@vAwZ+*Nu)-pB+@_EoEq%HwjM2i9#$$+&97&i0EA{Kd?L zn!_8MpBk)5?dIP9xN-`!!)B(S-c*=$3?LFP^0ZH!v8#B_Dm}w}Gfu~(>Y7|b30K%j zT&cxWH}M<0D%ZMFG7Q$p-`mu6qtXKPr3Q1Y63jUC<=SY2PuSZ76qDFyrw4!a zB)%>8kABjq_RUjgTIPs7z5Y%ry!fIvvOG zV{E6ylDOQx7+D*=>}z`WP?%}F?~-OtrJ`GT6Y6+>pmU>}%1-cm=>8a;Fx2VzwSq_6 z-8dubMDL~UBc1B_Zt*wOex34!(=~5?it+UA85x8$Pf+-8+twZ6SvoP z3Zt$pJ@&#qMjggp?{A^}rn^}A-c!f$YD7#^m3eWdd`i&J;zFFA#k05H6W@9YIc_lM z6g{vC_a1zYgKQCZw08t|k9b;H=!5O*?J*3)h6PZElE-lp%tU3g9ki^s=A_H#y7y*ha8rn6rS z@QC7kkMR?l=h};8=9{cK@?36Qz4e>NV|uRt+qk8c%JH6=1!XX{p3yH-A6*joDXiS! z6sDUJ!W~~j5w?l?-Nxeb9d#!S!jG;N)0sBj)Deo+{e8qGHy*Nu%c8$q(_Q5rmuPEF zq}FfN)U$cIzUbB}&17>OzUPu|soS^{SD?6=PtOy#%i^{*vp@glxxcEKZnByN?etRp zl@qR0SQMAmDXdiMM5ER7_D0-JA&Wa<^X+}jcg;nG$<4?x^du*q?|a=Rce{*E>W%ib zB9G{6Lq}U}$ifJ^ue4TgPoBc$`eN1d+B+-8?61BZ+h!-<%UEb~7q0(Rfp@P@A0xCQ z$<&78hum!4jp8xah1Qik>#GvabM-j<`9FRzr?N;ng-tTQ->?Y!b)=P@{i1nO3}1z* z2Id9}KWQFLa&;hXpgfX$#5TVuYv+vd!rZv(M!1&f)3zD7h?YY32AZw22H*Okdw3&T zDN+0>>|6YXf_pR5ei3ro>I7KY*Y&3x%HeaK_wJiU8e8rxG&oj;ITah2f89A$vaC23 zFJg1Ryq8D0PPJ2F(*3D{m&+$-uUhlwDNN{5Y}rv8VeG@1N1|$0;|kob4~ra9Nu!bS ztk`P3drTum1b?ADS=eAT)G^?T;69dC*toa8tX1l9Q+ea-u@0`{9T(yf^LkRbfj%M6 zyVV6%HaIw+>KS4N)EJ7fSe^y?$KDoRCS|%9v7m?U%?R_0O!pl+lSFQ~t3J~v;aNNJse=nel=W2Fl6mA?2=e7P&D}fa_p)Bnk zDAsBQk=#Ei5)hcmKK3lL6%XO6iC#OXIan!nqd5Mj7^2m}4-}+Xm1x?OesA1Ue z-vqhRaW1rpM_%9FQ%)3^>nFt|7S5}>#qD^1*q9NQHtCIy?9lM+w0fn%!WG|MdAH(0 zP6AWa>qM*QkP!RcjQS{hHVS&IF!czzfp>$MF_%RM8|%Lf3xTVc1&m{>s9m+YtX+og z5`PPbO03lH4zW#sG)vMMyyzX*+rlKr&o2_(aE&WxDm3xro$wb-E&|C%e>W|#JW=~5$qu*Gy=!Fv=xXlJ#3dKedRKng{&|a^@7k|)GhvVYr z$ao(&?<3*!{ACy2)^yhGpD7PElgds4Mha9^(+T#Y2XY1YH>=XirhZ(rHYBdMNfz#B zx{0dY>dwlB>t0`B2+(We*Rs0H=Ci}q6p)IV2$`&51a)w z)%>wtYK&UCp0uI`FEe}eWFpO?8Ew#%&Kdf^8T-RdX!$jgg-w3CD-5?9m}rNNYU@fg zd(Bk0AKP@7+S&5l_-?zJ#i_(>>muBxl@ZG*L5WGW{57|E)tMAGqH+ya@41alVEmZZ zw_Jmd-pOqXT4sCM(^huVHHUW@$`-B``OsJlIEyCRJE=OQjgO$|W@xHfNKYHxSUzF& zXJD@j45lB$C!o5v?4aOyKT;eWRRykYl2P<5cJ7uN*E^+9k(&{BNi8ieU!HsJpZn!X z|1imJi~Zer6It36S?*ZXFdE7r)B?FD^DcZsJk!O}O~a~6x%&$9CBw2>Og_|%6Z9Ll z+0*$`(WkIZ6^xtsbT_5kQ)Gn;F+)>iemm+Z@5Ic_HUxhSm`>LFLUo4(XzOatc&tH5ZSk>I+qurO?pL@s& zXZjVD85V3j=3FAd_`v@GAbu2v8E~x*LX9($jENT`vsKkxVyef#AL(we7F{6-?wj>{ z@%yfe&rFsUbMcBx;z=QWrhBJ_Cl=KQWkDpJ0rdj==QVJ-qS@kGj#a%Pj5Z_i8a+9a z`k+hKyU`z0y!JbBE4FIjwhjxF%+aNBf6L{gVrT}J&s3x3U$=+qBaGO}-9 z_Usr#`T%By-N8(U6J|$iO0m9MZ350aNN_P?Tx5+9`UT`%FCqQ_Ey$kkt(3QvbvP=Y{_?6OddBBnXa=?7JvkoM|wq0R?su!$6V7+1vlr z4P=nh-V0yf$Y0LNbqd)m%qpdkv;JvzGdqiCz!qG7Gw83R*8eok8VY0DNi8&3Dt+~! zlOk^{K;5!vMI~1kCmHT^z>#ZO-+v^Dy%~h0YPi_8?>;!3B2By~T-fEuB#!DYh^IGV z_jG~`?uzxShyU=?kLfbYI zU!1x{p3jYN@}Sl`5XOE=d$a3F;HVYu3ttO+H#M+3N)bF|T% zCr?N7qdK$huiI?tJ`TE@ecj1HP;*fsMix#ukdaaJ6w?MJoN+)kmLTp`=Uh8KcR!sH zt%>-em(@-UYu8Fpu#DH{9?g0Q>2fm0?|~-@OO+>cXdKr>0a}TU9P$9%)e?E?fGQ!|`B(Q_| zLNZh`Y$~&>bSvqAACDQ0lPe9qEnMQ9xE@-0&*B8-#AhAL;;F~o#Q9Z^>U%Nm2K9&P z+$u&!7*cpD{^iolkCt0NGBpt5A5;tkCJ$i<5vk7#fPsrjB@Dbkq(D)*K_Ci+oEKIN zQC31g@~=d-7zD7M+ZP#paI;JxsGp%g`B$h`hp26VRwZsk_sZ(%Pek`k$yizP z_L8u+-{XB+i-PUNTvV}8XnbK+Q8N77Q@Fu{{U3@xi;at??rK|9D}Yr145{m{tUwt=K~`75hM zN7jZo-A5+H!CB9dx|nhTk9b-Gn5+uME|D`ohed>KlR-dj~y|fiETG7*QA( zKlJrMy7^y}C&VbIbbm%0P}@2yc)%{|)y98I-NnZjvWRdvp;jM6+3Sp%h`I(E|8GHsQ1|zMPy=;q_yKcrVZ0y)rXk1P!3mR|k%d#y zUVFcmS}debv9hmDpe!F~B+;IXs~`F}3U>oDAC6ihQ->jpO(@NudzsA1ROY4pTQ(Jq zz*WJ?{-Cyk6ZL0Kv)Tme5>9JTO87SyoBZ{4$z|n}`Vf+YsQ91d-9Qik@et*2xkA)P z|IqZDX~bar2<3HVjS*%xR#lnd!n#sH_0m7K8vqyxWZGY%0=f;LP98)b#{ZTx$c&?x z!{-atW*@C8*$#9r2R;mc#SFO16bhE@i)bc+X1-9v0cqJ;p#JGW1vUn44jPE1Lsy>< z1j?XQ{)d6^HU7*7vHpDc{s$()U{O8&&EcX?`x&x`?!_w3lwkAgyBCYYQQzM!za=?L zokAyhB$3}Mn_)m}h)NNtI!OBX2|00J8P3ZDhv>q~NiJOs?}Z0cj6DR181jOec2U&= z^L(DFma(zoW`xayPMQnX1a$vFrU{|?08$WYgafQd46#Mf*AWPnDt#HiC6Jv0+JylU zJ?=l%XE0MFEIW&!lKRh_0DM5tFc5>lum=znbVpVIiw2$oVG{8KWKtCwt20CP_uEJ+ zUk=sCyJR(Jy#P-Joo1dG@Q;UG^k5*Udl`NSu{H=>5ZNRDy&>a5^e-AR&h*A#^uD0` z#cxQv|BciO=KZSxoi7|{!3!IY0I39~q`ZtJL=_})n)DW7bNe5ny5XVX4|Z8tn^NmI zRQ-t`kPR{^w0qyTfE zh#Fkt_Z&PLVc%6rb+I9b<6QcrF2I|UzGvF$AIeYwb=n{%zX_ZX#EOE>ar5sE z>whFUSX+q|?;9iQo%ol0)FYR3)T2!}uU_-FwC$YHja@9YZro$elIGppUMe;gV)`1| zCcNKy3S-Lof%@_=nR`?xaB+6Ib%n7CpySxYWifxg-27J$sE*I~q?Y?|d^e%A9 z_C8&VNXUCp75y$JDV69tov@7QpG~0*IRJzgaQKG?|0Z^j9tGP|3MR3om%0CJF77KK zo;(-jeFmghwX7FYano@mv{Fo+pHT4zNFTw@gg^gLu`RnT{>0GbTM1u@_y^unrV$_! z5@cPKp+VI$SU-h*jB@>~RvkS3t9bHV*q#0R*G&{*Bum?%FYlQw+>tZ$`mF~J1X}PJ zJ1pGGd(XB8st@l<@x=4dU{@>ztbHtblJp>}B#8EG6E{JtqYQEY{E{D}@gQ@I&?ag# zk?GqMv7HQ;V#aEU$R4N_L<>kpY4(09flEpYO?3(a+Hnri!dsnnW)^9X4|4f7@}Qm2xjRU?cy5*Rz!4 zf`r-mid=GA8#%rK&{2|I+y$6~XrqJH0pkV=2NKS+H3BVr)rp`2sNd2D$Y&<`Klvo8 zuxz6`k&4*D-H;();o6yqBR}ragMGW>OLdBj*;R37!P4Py0WB%2h;+hVsboI0m$Q9_ zKl?25OcQ<%IKdR;5N)=o^yJxiy_d)wfwkxR{GF@POWL{wwY{i%8M1)kfF(oY>0fAd zJkuUsTyz;PswCR>qNw`C%2=hQ1{*SJ!QJf-X*t>XMm)mI%54Bj?TYawae2?cZQYG8Kjg@Ij7wFP@ zmo%zA^!$B&lF7?EW51Wu<0s66T~ip9r=kW5QC6`z>ql09DDIU`x(pasYCAu_-}(OV zGZo4Al;sZ_OR1)YMw3dQF3_YkZCRAf@~$jDak3QsK8@e1EXpZtP?50zxkGPpM;v{k zyxvcfr30z#f@%ekrun51BWik2ugCm_d=n(YPVM)GYv4ceaLR?S2)q=(@R)Af3WGxaY-m$0s3bH>;o2~n;UM^Yew3by9y+`ln zxP9-M-)~cL>tYYv{8JeHHs!I$$5U8ld}yTrndi8Fl0fNf1W84C%>WzG#Mcpqx=Cf4POR7}c*f29ce>S`(O<Qcom;1QsmKxX! zQ6F$Rd@b?A%+#k6)S~lxmZ2T+PL{-m^(@sD@z-9BnF%myeHTN zsfuU=AGHg7!dk$>IQFiLnr?jhd(TwW4%gLuXlSDE)9$^WuggE>p$ON;L@QboTEEz2 zIQ-!|&Oq1Yh08v`{X$+gN=@ZWb%pt1hGnSjfkm-usn7a+UMhhQUY^eKyY=FMho&q6 zzTY}!xOx^uU#7RXD#V5D9=xazWKLL!GZFKtQ$(|pD~ub6OQg=`xxv5uD*nm!#IWfr z%sQ)PLO+8!f9+%TR?`!Sz zcEOndnwglG4Lk&Y>$fzkG@Fk$iadyMcVyPKyCe!#Z4&W%suc11l+?a5 zwPXj)9hU2ibUp6Oeg!9ah%`2nR;sj@Jk=%m8(z5)9vT`s_J|L&(D&oX%1l`|`dEZ} z**fpMB->j{MUzniKPA)ODEk}xYR;!Hr)65X%)_rz-Hc`ZrOpCJ&u_@CX0MbJe@^m3 z(;E=tD_&CGA3ji+t26rDC!@{25T=#A7t{J~0y&{hU`n3Z$yL$Tme!!TdV##$ZQ0!k zCA4qPYXg6!MK^4OzRJ#yM*bSN?K9s=uQ@XT^so~Rwn{}qXL3>eI0mD*dL^9F`+Lke z<0H$YNxPbA5~Zl#>e0>Ww`q_2q=jp=QeB6oB0inm9&lQeq6!z=K>1F531$7OC?+r~ z!G^p8M_R($^;^7A(l%t~9JXQgvcv+WOY+y~7bos;^n_}#>LKOx`eaX(Q6_lI}eh;gB5MN~6kH#@`Cpeh%bXzU@w1k9fLz|JtYYWn4TGEY{Je zUu`_%PNTbZHr(aaHd%)~PhY)1DB&_V$=!Hulydw-F?3|4hp*#CTRmvkvRo+bCmi3y zZ6TuIsXlZbOj5Ra#}R8Dkrh>C|AT}a=ds6c6{bpK3M*;x)?@(@?Y9g2lZiv-!PAV? zao(FU+1g&>rB-{aIpg0x74gi)4z2#WD#;DMOv!5>wN-*LO`D?BT&!vkdPx7`-TQWa z_~fBhG0^<++0%PG*9T1BNj~fi+`L-ia_OssQnn(!s*`PeSOK0~aV^C)m-OWxlAM?F z_2G1M2@~T@e#?psJ*$<^6(qZQCVtsAB^@Woa~STUp$?l^F_EVUlgAY%!epY_9ED1wP7sK0UgY*MzJdD|cx>KdvC+lMt z-uA7I?K6Zb8p-~7EX|WJPT7*&X{DoBVbhZu_7mUAMGExZt?o;%H|vZFnrLgIMR|q7 zVkMiH4v5+peyrRr$onWax<#5Z$6H!9e|d5ZzXKzVoBVD%3Hr~jm_1XTDB-%Hx7>VJ zJFp5KcTmR}Z(pjvL%RjXTl*e7Sorc+r6q$H`s59pkE&zO9|UN>>trNX z72~|*k?FCu@IKGk0^6GPQht-J__C|I5jX$ zJzIr5uccWsJAVRiidy2#ImtpKE1-iWzKn1>906K6?bnG9f}GXr++1!Gfk z|02}ouJf(?W~21Zld-zuFN%Vk#1lf@J&kxI#XV^dZ&5d6u-EB&!BFX z>9!RA2cCtYAE=ePu~(iS3o;y*b5$Q*>OEQsxMG;ZNJsBHoGqaKi00bVlh^c_GozO@ z=ygc zjvMbLMW%6o#~9x9@vzS~>h^G&TS$?leX%TC%s5eOWYcT;G__hOCjS>D(WVIb5;g8N zrRDS7+}LlWxTSXMIqA8cV(Uoj^;^@d5)(H`7uK$(I>5=-Vdxv%C4xo)6U_2&G2u3} zZXaGLBF_w?r}pLIs&dEV@|h=S+@BwqDG>DWlTx~+k1k@k$b0;F|0H%TPy%;P_DE%J z<{2!4avT(zkF9&6etKWq!22MR_b%)OL5wJ%!x zoNterMiWH3c%!0hby;*S7eq%GG{7C1Z1KEA)tH;-C8xVgySfv0zE@ccyA>7el#+V# zox%b$R!;D7vB&jfchV?iJ|C4+yp9VY5&n3zlF;vsy%)W_q`5b$Bc7W4#c!12qkC#T zVK0_~2;9Y#Q6-i4>F=7IwHHR4ELNi18=%2}bkhgla$`~OcqkeBr&d(cHysm!_AQY> zCz_%9#MM5^P*EIKwB#>VK9^c=c3)x2*VJsIB8$fD%cy(F&E#|nHri~oJJ zwX9VuHJ*fk@)L%lmq1>rCTA>x!8Tkqm!VZ*ho92Far2J8&v01k0jVZ#v#Q?or8lzO zYrepFhTC}(f{ulenUlH)r?8&BHzL$N9b!X1D~%_e%%%|?f(fhp6dby}IyY`%47+j;C_I8tumF=U=yQ*+c)&c4O zHlg~ih#HKQn(EXyM^vC5v+pBt3foKkwt*93yrFluVu)Msea66jg$S~c9JgK6)DOjTIZ;XyKC$zoVh2E`Ur0(SHlbVcjb4?p%VuTXBciXLMOGtc;35}&$M{>)jsWYtWXKQHs~u^YTV+|YrJA1BSLKzECKW3KoZH}ZCh!v6P;|7 zB=#Y6kMYz!g+&K>oWk@$FwukN`+1t)IQ ze5CR&`o}ayj)9DhXZ6(s|L4czltIpoD0bi=yJsvhzg+F7)m} zU)>$s3mY%_%)0p;{Epc#HqdER&+h!P)Y6(!!*N1;3hR&@5?uj@G8V5GzSv-=nUabI zKvt_68^Xz?=AVJw9Ps!%f=A z;5{tD&i$*q#I?mDD{M3VT_MGKpGl*H1sh+B&!afrjUuAOj=zs6Hq>3r88R#5Stjq% zBiXM6bT1ivkvCqC`FM@Cile)5+$>J(L{~Np3#az_z~RAkvU^mL0M+WZy#83aE>?V@ zVa0DEtRJY`ULIF&&h&qst}zP7YBTD;CdyDri|f#Kgk~>-Ywkm~SkfGCv-j*)^#pU} z-VZ<}A^;)75C9|YkxI+bo44Fmw}ooz-_GD3Jrm?Rc2PPe8k~0i)ksM24jzQw$OhUXz4rE3@_+F(z$UOG_T)Wf0c~-B3h^uVdBJIPZO`BJi=$0V(o^@s z@pkc}E;ALi4vH2>k?CmdXj$4#*{qDv4Cxk@NV)P)G}6>$h3OyB2k`V(X1WP!MU;n2 zpUjYOJ~**RT(HcVA+Zh48u}7HzdXG&Rl=4V@0J=Lt;N!|tN@1-VD$F${L+)Ub;zeu zU5PqdqEOmgNbs}j$`-5kvdh7P7xbxKj+U#uZ}mERqlGV{s$WxJ zZSGUxj-V~)aB(rwd9^8fD?dj??(^en>o&E9e%fgHE7ip*cR6aox>dznrF%5Y&7ZLR z)OprHI}0lIL|5V;R$l4iom4V)5w7_3V!d7Y#N?gX#*fgPkWQ~ByH>>e4@P`Ax*RK1 zA3q;V&*yndP8-4Uf;Rl?PlqpUw5g7o51kdHerjyNGX0Y=K^!J6`Wo9Ckm zCG@uCa&xav=re<@9_7fUfh;Y~tdmJm?f_mNI5*`)(kS}kNXhZxqq$*rU25J)TB@i; znb{!>I&IEq3<1<9Ss6hD@5Dwul2JtjHtt=;Fdepgg5;OLhqsqv!(i<)qiP`iE9Uj! zRPuMQGBD3%F3^muIn{%4}RY!rbwH~I~dZ4JDz_nnhK~{^lfxr z{H9b}j`)(sV!n#}UB8ZcJ-X_e_x0~Rp6Ab{xCpq>Ieht{>c-tZ>zbuQ^F&%L4(h}S zP{?={9x6)VTEOp_4Z)j|J3%yI{>^GBn_UU%^y;+Uv)5~~%<-msiNT?A9D3AAug9as zx!?7-3E|hNwtkGhYf_ZuL=_&K+c2=UEkL9oaWMa@=aJ_7xGXAnWZ8n)3LRU8rsD2` za+jTI>*1%WTCpk#kiRMpU4vI8H9t-KFNL)2EG zrJBVpJ>y_)3JTvREOb zZ!%=Bq}nzm=t&R_CSMJz1Vt%%T~er~byjtR%n`H=L}3eIZlI&L&|1zn#|Q(N{~{xf zwvDOQeVwZ^_@l%oxNI?<(U&C**6G00tJsAk6&2~H-I(%sW^%GU&Uc#s`T3EU1^{Pr2_>H24+;L?s9!9|Sg#XUG!201`09 zh;?xwzTv_jl#e1|peyBkKRL2g;%iIrl8^LT619OOJSwwn4({%%T-OEYqpQMLoSMke z!6|7Xs89rh7IWrQfG@5g2nOB)LhvD8fkQi%-2c+bfF;O~QSUf>K{fjDdF>irY2eQo zw%x+sLS6)A|r=LwKdIW)wVL5ZDrmmog6XpVvAhvtS6a&!K=qog#a+$(uawvh9f(2&G_ zvue`1#T@Vn-trBc*|oxugIp^>5KPGNLU8@;Ao=+u|8E-usHLkvg_wo70@TM5w|`It zhqn>6`K!=hVg#y0n4F4en$ad>?8r$b-&LMR(Z&Q#xrtP2M=L7nG8D(ED?V+JzJK1d z0C{(;FamHY7a`hc$e@k~z~Be13=9M$|3#-714yl;Ao@_wNexqr8sWvtMOCv*BdspB zd;GRE6DK35=q(YZ9Kilwj8sP(==+3!Qi?-wE6 z%hfCoU&ZF9#m}=GvC_DL(leQK68E}vPg;MyJAR}%Dp&w#AWmD5W2;?=($q+;AV|3Y zrhFUrOJBO>eeVgy1Q!FnfQgw%h%04#;%}>&TTIV*GVfRfh20-bWvfv}FyfGrt+ zsci_#7ede!%6J*`@-3Ju3NY!V`rA!s2JXUar6Fhet6}+tD9#LJX83U@zU3VZV7Tvq z&8kxNy6B$QP^p?wXXIn(%RSP3i?LBoq=dOn8lc#}#bPMf4@6uP381iN@3Ux#9WDlnxuLp2&Xy4>UJALp`ZhLgIw zDGIgk`-kpyQBHOx+GxcrN7p5~--8#|G?4U{Js~^1e}5zKXZNR5*yRclm7ZeW84^tK zqu+@MGx;swt@)?ScYC0_?T9?ds(3dI_8ukEsl-??uVVi!azp>Nu*}>#3t9|jqDW{Z zG~$9Wqd>&cSBzP(!I`h9{3LROq2xCZ4s3QI<>y|mzbtYr<{pcuC2Ya13Pvg4<) z!wsLsmQulx4VvZf&PJ3i5r>S>ZAvsxy4M>1!qSFrti}45!oTxJwG~oMQH*=X{Zh{- zwcZqJmr#51Y;HhnkklujP9VD#zhCODzR!(j@>AGaPRFtJ>xof_yam1{>k!h|idg;jv;nJW_LgFE0YRmZf`Iipyg_;r;& zbox3YHJ>uvsjn2XaQt?@occr-fmYp-=$@5n&Qi~^?qG!r-){!zPz|g|l%D(RdIKrR z_4mAm_Y7?EbsI9%^rY=P1K90|F=KFzd-xW8bU&{+?eF{GpA?H2nl|O-%HC)J(Ilr} zbWzc5adL6OQBTo5AJNd&pz&|6POo0$%mi?pfUjFp8(lHDmINwO#I0n_OxEVOv^xYS0ueAX8Wthe1qGDQi!c5d#O^6{%x;Uma{0xcZF{pc!?KT zPSa?7x3$s?%wPUu&0MuE{ptA{k;$eDZ=yE;@3Q#F@O1R;rR`GlK5xoBQ|T9)*z}jP z$jz!%n6It;ek=-O)M$}-{qCpJ`etH7`+jKX!S{+695JR=LqVc7_w{|4=6OJ+kmANr zx+8Xm)h?~)Tiuc#+#0VwQJOU!X>y`%P40MnS$*g1w8Olr-^ICTrnE`E- z9vxNcZ1dW-QsFz#l~r@Or?F%4=x}*&F4VTrQd2k?aZmU9Eh*c=<|olRGv5?Ho03f@ zAhH^J&iR8dt8t-ptaBS9^_rqey=tD63hGr-pA>A{;U(O5 zH9Ir;gMc_Ypwz~~|8sU(rR?&_gI*s$282;KdKPe9v(WnBcmweu71R0228Eo4A~+DA zg7TvAHNX4($Kn?w|AT=O=oG1y2YvE|Bavz*YMkkOKpR8lvISD)tmVPSn`H6_i#pDV zh5`NI_|ME4kf(x=@!%gp2#pG~t=}ynE}(44sQ$|A{gtkmGpeiMIt@P@nS!3|M$0^p zIn92j782|S1uUanmF4YFktNsCp8?U*3fVWWp@thk8KnLxP%-%R4EUX%&Fw+z_fHywbYezD zks@k;gT7|3ro1>I2Uoi35vdZ^6e@C?5jSD*yc-01e=_9aIk`)K_l92qp52O?H34m& z$CH)Vq+8~vd_T!X@4T|yx&C7C4|eTo;5*UWi?#PXmbh4p!HO?#b^6Y$U~wxH_swVB z$2L-b5Ddo7VLfXmRdLhmuuU7Q=go)I&ULdnW7w1Y4lau@md+Q<#+6nExig%gzVEl4 zS0}|g7k~Q||5H_41H%vSLl0pwMVgHdvEPa|^*P7i+AUrhaB!Mux3Y?G()4uefP2f( zl<@QFe8ZpcqcK{k5%SEm9KtSry(WacP<5gg*&(igh$slw)Sq9AU=JwelVpRFM)a zy`sMOgJ3KeL^*go`d)7v0+;)3-<@r9oPd1-#O*_Kw<$VT76YZ|g3XtOOidM-y6Z0I zXIKeEuN<7OlMeKzFtb>aB{Dy1caq@VqL*fhwLI$&0$Gt6Y$30suF+QodM<5^%4p1H zyvows$CZ7&4`eDXA9Q`=O-)|Y`fLsMXq0i8I3wZSF;^$&Gq0~r@TYa8GDoHq!n2V( zbcz$z4p{D_YZK>C8Mht=oU=PD)8QJu*(Z7O<3dV?{rQAoyZ94N4fS-x+sKY^y=j>W z=?<}RvBMJ2uG*AHa=bL3^a!TbH}WYx9(bqt(CgdG)*1tJFh7h9hI4b=mux1%EUAKS z^m-!ZR+Er==JGZOdMZ#oKjudYU&?_cSn($aG)LiG^lp?MMqzyZmdZMdbqFf zrsSB;=r4^Kyz3mUABr3(-we0I>DTVSrFfQbqb)a=G)~`f3C*_TXVlza8mZCF;->gI zChWly=;-UdaD6jnJUM#8)NTxM!~Vv|PD~L6c2+vtjc%>5sk?i$shiu_0C9>%6HR}V z=s;H1i-(gp&OFpqCKJ@?Brk2B;$o*=zo3b9$k6X~KdlC@x1XEoKW(G$`oiM}!HIdZ zgU+s;jTDb=omD_~y$MxZW+RPb+e-GOZPZzw3SE@YM++NUmG`8&u#`KJvW9l-43s-2 zXDn<}-8XN%&_fj$)*iUthS}7=ceYA+M6t42Jh|WQh~=Q~#BJd}X@V__^gQ*>n5alZ{ ziS1yPO^w99^-6ll<={X~HD#ql<>FdIYm3-@y;iuAi1`m8B?^YJsz z6@rxdmzePj563&VHTwr#Nait}mrcVCREQzGD_NHd6YLNT%BOmrL)|*k>|5MBf^l)- z1ttb9FZm-)|1##1T^`g&YGDsA*B`)uP@>(u2*j_Lu+ z4q;5lAkpxlS^i;zE9n@Ibi2Xst{*@8{!0!DIWM}WhZ6w_MgrVrdsfVTDa!9Q^{$--l>anjVn6_>95~m^ z?|3=@WSU2^TTSMyW;zv z;TLh1Em>XiUYwbb*sa}_r+YBLe{#t%`->x0&K@))z1&5-*!kVgsHH5ify4I!mL`|v zDv20zz`FvCC^OlAF;t3YcY^99WU`7G2;HeTcZj)EuUnN%l@uNUQKE+k-+^}D_JO-5 z6iqCL{f1BUf8m!l`ps?;Tl_;50v+hU<2zonQ;D_CKL~ue-@})Eo8qyZiPB45tNO1& z`1TIONDt8_QadPc@Zubnxt3g2e{~tutaxK+7F~}i{e<{p_gm4 zztT_oz~>PxVP}`H{}Id`owlm88H8Jg>fV*dSRf@^{fL9A8Y~wvT4NNNxQ~RCV{-Z}mP={`J8^IjQ z@LB932dB0p-v$cxTVK&GAKf{%OjMRlaE@|5PGYiMZXOx&1a~cIRVwP0(*&pbb`R z?pZEh(CiugGSuMFLTU0B?LtAguHL=!kH_wCpwVNqxRCOMm=z7Tsi_;Yl11++RB)rO zoHq+B9V{n@iX2KvyEr2CH%T5VpWqgPu3PZ1`cwcHP;?NH$`qJ`Ev!iVU9e*9ndAh6Jw}W1ogkd@=eJ&Dx8J zQ)42IINX5R^MsWZtDsSu;T5ruWS=ocy4Fs$)0yhR69%YPp3C~L=k@gmEGoOF;@-XM zR_mXp;PrC!H3=Ib5XEAQm!!jZB+Rpa5F8Z`flF-N`#~_-Hk-XRGp($BOjKU^0e6$j zXGO_n*75o|QjX&E%fzU3x6ZS*iC*FBAW32XL$l1kNX@V3@u=9HNU=wZ9w9_r_N8qN+M_h^Tgw||EzXvG_ z19Bz_^=2#YXErC(CFov>Y(6o_Gq9DpUq?(Q3Val?rJ0)>6TD@CB|iwr4V51MATYQZ zd2;D`#6hzPi;$Ktr#fpk%0hPBa3A`!9hgI6xv)=G#)93eef9fBBrbKcH&()VK6son zm*1F7-ged978|ctY!rX|tVc^VkazJ+Jqs@StruVO>C-j~B~RA|%LVfzMHvnC55XFo zx96Nb=AY*pzl*l8vLC2maZY)nLs;T1#hRnxQ66dbH`I^yneIDr>E&ZamDx-l;Qs9QUT$ zx5OPg2IvZ5_N4cm2j0xByM{OLNVF9xMPH6;atTX|!3A``+?wRcclBhyarHu-TS#^d zv%j=?cQyIT_(`*x1MiJ&1eEA+N0?k8+(HtPQS>J6FkwaAMszI(Y_)Go7y*Zp)e7Vq z0v6HFN^g9xPUtV__|3G#JFE6;e18A*gX<*42fWG2K|f2-aG7Wg@BE0Pch}0!FRZ3u z!tzRnB=I&@!D%XBuLai7G<{Y4IkxE+Xv<+h_^?4j0X&d6;x&+=yMrFO-wzzG3fA(A zDVYib(j(^Y!t|QbI1I9ibFnPBC50o#D_Od5@3%C(pMxrPrnl8(-*xo%*yL#`b;U|dt3*wl|`1{(_mnBDuScp6@!p7DU7B*d(aT79FzV-qvNP5GdOAgh$!^ zdf(IYFNG$OH0l!q%xp1CrXI=+d6f+ApKke5+7~~abgWv64c6$Hj0kaRD3c0szY|M6 zrI0xCsg$HQQ8$|N<%{H={S4NMsrGR3hfYkW_KMCstgnw5wSWm9yzWo=@Z-1WeBJ4t&HmJ>U*kp z6aF?R{Z6JC`$%AW*#_x`Rp>LH#<>oSH6K|TRG_{L4akFpp&U-OMF9)0V++;0D zKB8K8JkZekD8v08HU?OmNbm#(HYddsbn$Kn%V~ol&ma)1m?mY5@=CG$jLtF`$jjNb z(XmClvge4$vQ;{Ym#)E1%$&aKPgUjT+a|NAeQb?2YjTYy z9T` z&DL5A#}?jxkbz?)osiCBs>{9S*vHFa)#KK7SI~2n&6S(Nxfb?Vfym;WJ^4qQw z1INpzBHyXfzYz~um(RGc@`J!#8s1H9p)r2BHOY2c(f1=L3^B@nASXj#FcQqzG^oI8 zi`5>&(!heZ@8o*El2@)D`k>$v*A=DU4bsG@eh`QjQp~UV7Jox{S2?Zw!hM<;hSz(2 zTFd!()`VwApQ6od=^yC#pC~ygz>tnvkxCzY<}-2uRIL8|_cBa6>uev@+{PHadtMUf z*G@y@A=BsccQdXzFXVc`&>~-+E3LfgR{FZ|saS|1N&}&0{r#1>{5D!4jnmEOAtFSI zT4zQ#{meTl z3?qHO?GPWg@f<$Sjwuy43+YE>`~?&bHc~dtjC{tv9T;9%TaAac7GqwCnY^^VDXfI@ z>5yn_{zS0jJ}JAy+RMMqHL-GV+Y^UuvEbqja4oz!m#~vLA$e(Z_BOGtbEyJ<&MV|a zGEDZYewEtbi0uYqNWGegqYektn%B9!d_no-YIe}!w8JyTD;Wh2sp(=O>U> z6Q4HB`;*pV7qap|Cw{806GmIIce|#tYd-7RWVQx;cwIb|UClXova`?Y>?gT(XU+>u z!MGl^1`h%q9j(^CT9_Kz3wKNGC7ipVmO{jMkJE(|#o-(6OR&$FN^Ga}>K z4TG67zCwj|-$K?AHYUF0b~_HK9&EUGfA*NZjS|SI&$51?*q>dgrCE@$$ri zX{4v^S{vZC-;eQF_$#V-CdhOvY)=b#nzesG7439g*j{sb7x9*Bj@8HQ$n5iln^TOo zp~9;D@~}|LI8HQ2lz$wJ`}K6Q`kL1oojPhcd`nFly?tb7g$-<2ddyqBh-glmUKz1s zooR$%5JUUM*~2kk1Bfw{fBjFe;pHy?-`_t&CO~!UoN#cr0xVRiJQ}k+H47)>IzE1~ zil;D*c{bNhzW3y2ItbVYQX5iEYkMCPdG!Fx<}I&OPFb#ea#d`#9Wszyuu zTvyGAyL2eG5SOf~n-oCtrCkehABhl2f@3d~|NA5$diXu?hbK$O-75E+zZ8$@e;EKu z%KIs{{Q+3;O(=gS9()iD>d%ai=i#3YWk=vw2Xq5~Wd@-R_TW}5Q-MA_EPcZgm(U&F z6q>^~$`VMeW%g)h_oeZ7{Qo$oFtpOB`YIKaWjqs)K8B?+Lh7;C3yz!|(1bP9Z6CW* zeyeUVbn@f>w66c}--}s#Y~?ghpC$f!9{Jcv-PbQ1ogGXUZkoQF zA(uzgTOe;je|N(@bLXApCs^#EEidm@9o@yRi#S`H*7u|z1ZvVMynId?l?uog-xGbO zvS*asHs)D&bWACRdKR=%1*b{x3Z^tA+{}j6Q*?H@iRe+F&5$ueSHzpf0{PQ1x~#Kj zd6Xp|4}xCx_5ZZ+!8Ia1YcW0ME&3$B>*^W&8@X$UBEbjNRl+AU3er_H#Zn6C^41O9=nHAx!N%7@cTKZW@eI14u^HKqMhU^m79)bNod}bmC zm0wNQVueXt*gfRcl}6?6X$)ryveb)G8WJ9Yf~-bI zgdlf_U;vi-+gm*@tO7et&bnr~JolQ&!I`N|2G})tOjo`$r8OL`U-m>itvTllDpyU& znr$6`_wQ7d9IXr+agjvFAV9VI3#QbDN0cyUJ5JTAvz~nJ%3EDzxI3&d#;` zJnj(mp}TtQVx|$F4Fcvh$KOvS)ANm=&&l6heDBP%z?5EK95%t_Gv_+()$ho|(6gI1 z7|o2rYp44&egMD$=?*_pxU15Fx54WV&?>@P?kFS?UJ^kqRD%8uWt_=O-R*e<@>~BU zl1mTv^2*^&T}4eUlFMtzi?%e5(9ns#NTipY#;AYkp=I+jQ`Un*Ya@tIR zn0phCmce-NvL68bL9O}8oJ^jWGOZ8}A4t>w4E8BPc$ol+V;%*ujn z4zp#EW~G$8sQU)fyUaZ3t*G+0E1BjAaPe{8U`_IT6SJ$PgeoO2W$AFaMf@PGx=e*iE_ER^>h)ZLcXLS;5e zzDW}8aC2aL#N5iyl4O8v{|l}p)#t*!A{ujJSPzla-^_kRIIgI-MqY%?@Z{NAq`TF- z>jf3R0dJ4!Apl$gGuuZRRMZPylm)oegzA4G(C=v?|6Mv=?#2ZcqT0_LGBfG|E*|m< zAf+e_H);1x<&LCx;0**Qcu`l*z{n<2(}b%hQnOwKX`pT3u|Syx#l8oCXAvy3UwrXD zo$H@tZ4a5+^rTTQz1d@3g-ftdBB?x$LKeTq9aAB)KMNMu!ynknO>tU`YU+-X@{8SwFeF!SM z5~C5k9;BH5949BUB8+PFf>c1kjgGr)l}A=En})Qmnj)0*yTGWtffoZN+D;U6T=jEoJ5qB82I{?nG(RuJO+{#g6Q84+SF>R z`j%BbjI$BR9LoZN1chwMK=e+&)<&AVai`WYV7UK_>i^(b1c1N6E2BsymL79S~(Nk^7nDpqHF3~a- Y$Rt@4PgkJ&05DCHma~f{!XF?02PEz8%>V!Z literal 0 HcmV?d00001 From 5bc5e84e9849c041d26a1dee38e8f14ebab7341e Mon Sep 17 00:00:00 2001 From: mungitoperrito Date: Wed, 11 Sep 2024 17:39:46 -0400 Subject: [PATCH 03/67] asynch --- _includes/indexes/dynamic-intro.mdx | 1 + _includes/indexes/enable-async.mdx | 6 + _includes/indexes/flat-intro.mdx | 1 + _includes/indexes/hnsw-intro.mdx | 1 + _includes/indexes/vector-intro.mdx | 5 + .../config-refs/schema/vector-index.md | 4 +- .../weaviate/configuration/indexing/index.md | 52 +- .../managing-resources/indexing.mdx | 18 +- values.yaml | 1857 +++++++++++++++++ 9 files changed, 1928 insertions(+), 17 deletions(-) create mode 100644 _includes/indexes/dynamic-intro.mdx create mode 100644 _includes/indexes/enable-async.mdx create mode 100644 _includes/indexes/flat-intro.mdx create mode 100644 _includes/indexes/hnsw-intro.mdx create mode 100644 _includes/indexes/vector-intro.mdx create mode 100644 values.yaml diff --git a/_includes/indexes/dynamic-intro.mdx b/_includes/indexes/dynamic-intro.mdx new file mode 100644 index 0000000000..6ff64b3ceb --- /dev/null +++ b/_includes/indexes/dynamic-intro.mdx @@ -0,0 +1 @@ +A [dynamic index](/developers/weaviate/concepts/vector-index#dynamic-index) offers a flexible approach to indexing. A dynamic index begins as a flat index, and converts automatically to an HNSW index upon reaching a threshold size. \ No newline at end of file diff --git a/_includes/indexes/enable-async.mdx b/_includes/indexes/enable-async.mdx new file mode 100644 index 0000000000..444a02b224 --- /dev/null +++ b/_includes/indexes/enable-async.mdx @@ -0,0 +1,6 @@ +To enable asynchronous indexing, set the `ASYNC_INDEXING` environment variable to `true` in your Weaviate configuration. + +- [Docker Compose](developers/weaviate/installation/docker-compose): Add the variable to `docker-compose.yml`. +- [Kubernetes](/developers/weaviate/installation/kubernetes): Add the variable to `values.yaml`. + +When enabled, asynchronous indexing is available for all of the collections in your Weaviate instance. \ No newline at end of file diff --git a/_includes/indexes/flat-intro.mdx b/_includes/indexes/flat-intro.mdx new file mode 100644 index 0000000000..538b194c3c --- /dev/null +++ b/_includes/indexes/flat-intro.mdx @@ -0,0 +1 @@ +[Flat indexes](/developers/weaviate/concepts/vector-index#flat-index) are memory-efficient. They are disk based indexes that perform brute-force vector searches. Vector search speeds with flat indexes scale linearly with the number of objects. \ No newline at end of file diff --git a/_includes/indexes/hnsw-intro.mdx b/_includes/indexes/hnsw-intro.mdx new file mode 100644 index 0000000000..624a618889 --- /dev/null +++ b/_includes/indexes/hnsw-intro.mdx @@ -0,0 +1 @@ +[HNSW](/developers/weaviate/concepts/vector-index#hierarchical-navigable-small-world-hnsw-index) are high-performance, in-memory indexes. HNSW indexes scale well, meaning that vector searches remain fast even for very large data sets. \ No newline at end of file diff --git a/_includes/indexes/vector-intro.mdx b/_includes/indexes/vector-intro.mdx new file mode 100644 index 0000000000..c66ea4b859 --- /dev/null +++ b/_includes/indexes/vector-intro.mdx @@ -0,0 +1,5 @@ +Weaviate offers three types of vector indexes, [Hierarchical Navigable Small World (HNSW)](#hnsw-indexes), [flat](#flat-indexes) and [dynamic](#dynamic-indexes). + +- HNSW indexes enable fast, scalable vector searching that works well even with very large data sets. +- Flat indexes are memory-efficient indexes that work best with small data sets. +- Dynamic indexes switch from flat indexes to HNSW indexes when a [collection](/developers/weaviate/concepts/data#collections) or [tenant](/developers/weaviate/concepts/data#multi-tenancy) reaches a threshold size. \ No newline at end of file diff --git a/developers/weaviate/config-refs/schema/vector-index.md b/developers/weaviate/config-refs/schema/vector-index.md index 4580d1f4b3..5866b3cc4f 100644 --- a/developers/weaviate/config-refs/schema/vector-index.md +++ b/developers/weaviate/config-refs/schema/vector-index.md @@ -208,9 +208,9 @@ If faster import speeds are desired, [asynchronous indexing](#asynchronous-index Available starting in `v1.22`. This is an experimental feature. Use with caution. ::: -Starting in Weaviate `1.22`, you can use asynchronous indexing by opting in. +import EnableAsynch from '/_includes/indexes/enable-async.mdx'; -To enable asynchronous indexing, set the `ASYNC_INDEXING` environment variable to `true` in your Weaviate configuration (the `docker-compose.yml` file if you use Docker Compose). This setting enables asynchronous indexing for all collections. +
Example Docker Compose configuration diff --git a/developers/weaviate/configuration/indexing/index.md b/developers/weaviate/configuration/indexing/index.md index aebcbd5a11..e526ca430a 100644 --- a/developers/weaviate/configuration/indexing/index.md +++ b/developers/weaviate/configuration/indexing/index.md @@ -5,16 +5,47 @@ image: og/docs/indexes.jpg # tags: ['configuration'] --- -This section discusses indexing Weaviate collections. +This section provides examples of how to index Weaviate collections. + +Collection objects contain vectors and properties. Vector indexes let you search the collection's vectors quickly and efficiently. Inverted indexes make filtering and searching object properties faster. ## Vector indexes -- [Overview](add) -- [Hierarchical Navigable Small World (HNSW) indexes](add) -- [Flat indexes](add) -- [Dynamic indexes](add) +import VectorIntro from '/_includes/indexes/vector-intro.mdx'; + + + +### HNSW indexes + +import HNSWIntro from '/_includes/indexes/hnsw-intro.mdx'; + + + +See also: + + - [HNSW index parameters](/developers/weaviate/config-refs/schema/vector-index#hnsw-index-parameters) + +### Flat indexes + +import FlatIntro from '/_includes/indexes/flat-intro.mdx'; + + -## Keyword indexes +See also: + +- [Flat index parameters](/developers/weaviate/config-refs/schema/vector-index#flat-indexes) + +### Dynamic indexes + +import DynamicIntro from '/_includes/indexes/dynamic-intro.mdx'; + + + +See also: + +- [Dynamic index parameters](/developers/weaviate/config-refs/schema/vector-index#dynamic-index-parameters) + +## Inverted indexes - [Overview](add) - [indexSearchable]](add) @@ -23,5 +54,10 @@ This section discusses indexing Weaviate collections. ## Additional considerations -- [Async indexing](add) -- [Index level](add) +- [Asynchronous indexing](add) + +import EnableAsynch from '/_includes/indexes/enable-async.mdx'; + + + +- [Availability levels](add) diff --git a/developers/weaviate/starter-guides/managing-resources/indexing.mdx b/developers/weaviate/starter-guides/managing-resources/indexing.mdx index 049cec4eb5..07e8a68c9c 100644 --- a/developers/weaviate/starter-guides/managing-resources/indexing.mdx +++ b/developers/weaviate/starter-guides/managing-resources/indexing.mdx @@ -30,15 +30,15 @@ Vector indexes can use *hot* or *warm* resources, depending on the index type. I ## Vector indexes -Weaviate offers three types of vector indexes, [Hierarchical Navigable Small World (HNSW)](#hnsw-indexes), [flat](#flat-indexes) and [dynamic](#dynamic-indexes). +import VectorIntro from '/_includes/indexes/vector-intro.mdx'; -- HNSW indexes enable fast, scalable vector searching that works well even with very large data sets. -- Flat indexes are memory-efficient indexes that work best with small data sets. -- Dynamic indexes provide a best-of-both-worlds approach by switching from flat to HNSW indexes when a [collection](../../concepts/data.md#collections) (or [tenant](../../concepts/data.md#multi-tenancy)) reaches a threshold size. + #### HNSW indexes -[HNSW](/developers/weaviate/concepts/vector-index#hierarchical-navigable-small-world-hnsw-index) are high-performance, in-memory indexes. HNSW indexes scale well, meaning that vector searches remain fast even for very large data sets. +import HNSWIntro from '/_includes/indexes/hnsw-intro.mdx'; + + HNSW indexes achieve this by building a multi-layered graph of objects, allowing for fast, approximate nearest neighbor searches. @@ -52,7 +52,9 @@ import CompressionAlgorithms from '/_includes/starter-guides/compression-types.m #### Flat indexes -[Flat indexes](/developers/weaviate/concepts/vector-index#flat-index) are memory-efficient. They are disk based indexes that perform brute-force vector searches. Vector search speeds with flat indexes scale linearly with the number of objects. +import FlatIntro from '/_includes/indexes/flat-intro.mdx'; + + As a result, flat indexes are best suited for cases where the number of objects is low and will not grow significantly. @@ -67,7 +69,9 @@ import DynamicAsyncRequirements from '/_includes/dynamic-index-async-req.mdx'; -A [dynamic index](/developers/weaviate/concepts/vector-index#dynamic-index) offers a flexible approach to indexing. A dynamic index begins as a flat index, and converts automatically to an HNSW index upon reaching a threshold size. +import DynamicIntro from '/_includes/indexes/dynamic-intro.mdx'; + + This can be particularly useful in multi-tenant configurations, where different tenants may have different numbers of objects. With a dynamic index, you can avoid the overhead of an HNSW index when it's not needed. diff --git a/values.yaml b/values.yaml new file mode 100644 index 0000000000..2b1ea547d9 --- /dev/null +++ b/values.yaml @@ -0,0 +1,1857 @@ +image: + # registry where weaviate image is stored + registry: cr.weaviate.io + # Tag of weaviate image to deploy + # Note: We strongly recommend you overwrite this value in your own values.yaml. + # Otherwise a mere upgrade of the chart could lead to an unexpected upgrade + # of weaviate. In accordance with Infra-as-code, you should pin this value + # down and only change it if you explicitly want to upgrade the Weaviate + # version. + tag: 1.26.3 + repo: semitechnologies/weaviate + # Image pull policy: https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy + pullPolicy: IfNotPresent + pullSecrets: [] + +# overwrite command and args if you want to run specific startup scripts, for +# example setting the nofile limit +command: ["/bin/weaviate"] +args: + - '--host' + - '0.0.0.0' + - '--port' + - '8080' + - '--scheme' + - 'http' + - '--config-file' + - '/weaviate-config/conf.yaml' + - --read-timeout=60s + - --write-timeout=60s + +# below is an example that can be used to set an arbitrary nofile limit at +# startup: +# +# command: +# - "/bin/sh" +# args: +# - "-c" +# - "ulimit -n 65535 && /bin/weaviate --host 0.0.0.0 --port 8080 --scheme http --config-file /weaviate-config/conf.yaml" + + +# it is possible to change the sysctl's 'vm.max_map_count' using initContainer for Weaviate, +# the init Container runs before Weaviate Container and sets the value for the WHOLE node +# to the one provided below. +# it is possible to run additional initContainer before Weaviate is up and running. You can specify the +# containers as a list in `extraInitContainers`, exactly how they are defined in a kubernetes manifest: +# https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ +initContainers: + sysctlInitContainer: + enabled: true + sysctlVmMaxMapCount: 524288 + image: + registry: docker.io + repo: alpine + tag: latest + pullPolicy: IfNotPresent + ensureFileOwnershipContainer: + # This init container sets the file ownerships of /var/lib/weaviate directory to the ones set in + # containerSecurityContext.runAsUser and containerSecurityContext.fsGroup settings to ensure that Weaviate is able + # to start in unprivileged configuration. + # Enable this init container only if Weaviate was configured previously without security context + # and now containerSecurityContext is provided to run Weaviate container with non-root user. + # Please be sure to set at least containerSecurityContext.runAsUser and containerSecurityContext.fsGroup. + enabled: false + + extraInitContainers: {} + # - image: some-image + # name: some-name + +# Scale replicas of Weaviate. Note that as of v1.8.0 dynamic scaling is limited +# to cases where no data is imported yet. Scaling down after importing data may +# break usability. Full dynamic scalability will be added in a future release. +replicas: 1 + +# Define how pods will be created. Possible values: OrderedReady | Parallel +# OrderedReady - pods will be created one after another +# Parallel - all pods will be created at once +podManagementPolicy: Parallel +updateStrategy: + type: RollingUpdate + # This setting is only available in K8s v1.24 and higher. + # Setting maxUnavailable to 100% results in removing all of the pods + # and re-creating them in parallel all at once. + # rollingUpdate: + # maxUnavailable: 100% +resources: {} + # requests: + # cpu: '500m' + # memory: '300Mi' + # limits: + # cpu: '1000m' + # memory: '1Gi' + +# security Context for the Weaviate Pods. The configurations are the same as setting them +# as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ +securityContext: {} + +# Security context for the Weaviate container. Override overlapping settings made at the Pod level. +containerSecurityContext: {} + # runAsUser: 1000 + # runAsGroup: 1000 + # fsGroup: 1000 + # fsGroupChangePolicy: "OnRootMismatch" + # runAsNonRoot: true + # allowPrivilegeEscalation: false + # privileged: false + # readOnlyRootFilesystem: true + +# Add a service account to the Weaviate pods if you need Weaviate to have permissions to +# access kubernetes resources or cloud provider resources. For example for it to have +# access to a backup up bucket, or if you want to restrict Weaviate pod in any way. +# By default, use the default ServiceAccount +serviceAccountName: + +# Kubernetes Cluster domain name, used for resolving intra-cluster requests, i.e +# between instances of weaviate. +# Note: The final '.' on the end of the hostname makes it a FQDN, and is required for +# DNS to resolve in all kubernetes environments. +# See https://github.com/weaviate/weaviate-helm/issues/175 for details. +clusterDomain: cluster.local. + +# The Persistent Volume Claim settings for Weaviate. If there's a +# storage.fullnameOverride field set, then the default pvc will not be +# created, instead the one defined in fullnameOverride will be used +storage: + size: 32Gi + storageClassName: "" + +# The service controls how weaviate is exposed to the outside world. If you +# don't want a public load balancer, you can also choose 'ClusterIP' to make +# weaviate only accessible within your cluster. +service: + name: weaviate + ports: + - name: http + protocol: TCP + port: 80 + # Target port is going to be the same for every port + type: LoadBalancer + loadBalancerSourceRanges: [] + # optionally set cluster IP if you want to set a static IP + clusterIP: + annotations: {} + +# The service controls how weaviate gRPC endpoint is exposed to the outside world. +# If you don't want a public load balancer, you can also choose 'ClusterIP' or `NodePort` +# to make weaviate gRPC port be only accessible within your cluster. +# This service is by default enabled but if you don't want it to be deployed in your +# environment then it can be disabled by setting enabled: false option. +grpcService: + enabled: true + name: weaviate-grpc + ports: + - name: grpc + protocol: TCP + port: 50051 + # Target port is going to be the same for every port + type: LoadBalancer + loadBalancerSourceRanges: [] + # optionally set cluster IP if you want to set a static IP + clusterIP: + annotations: {} + +# The service monitor defines prometheus monitoring for a set of services +# https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.ServiceMonitor +# Make sure to set the following prometheus values if deploying observability with the kube-prometheus-stack helm chart: +# - prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues: false +serviceMonitor: + enabled: false + interval: 30s + scrapeTimeout: 10s + +# Adjust liveness, readiness and startup probes configuration +# below is an example that can be used to switch the probeType to exec command +# readinessProbe: # (Compatible with liveness, readiness and startup probe configurations) +# probeType: exec +# probe: +# exec: +# command: ["/bin/sh", "-c", "wget --spider --server-response --tries=1 --timeout=30 -o /dev/null localhost:8080/v1/.well-known/ready"] + +startupProbe: + # For kubernetes versions prior to 1.18 startupProbe is not supported thus can be disabled. + enabled: false + probeType: httpGet + probe: + httpGet: + path: /v1/.well-known/ready + port: 8080 + initialDelaySeconds: 300 + periodSeconds: 60 + failureThreshold: 50 + successThreshold: 1 + timeoutSeconds: 3 + +livenessProbe: + probeType: httpGet + probe: + httpGet: + path: /v1/.well-known/live + port: 8080 + initialDelaySeconds: 900 + periodSeconds: 10 + failureThreshold: 30 + successThreshold: 1 + timeoutSeconds: 3 + +readinessProbe: + probeType: httpGet + probe: + httpGet: + path: /v1/.well-known/ready + port: 8080 + initialDelaySeconds: 3 + periodSeconds: 10 + failureThreshold: 3 + successThreshold: 1 + timeoutSeconds: 3 + + +terminationGracePeriodSeconds: 600 + +# Weaviate Config +# +# The following settings allow you to customize Weaviate to your needs, for +# example set authentication and authorization options. See weaviate docs +# (https://www.weaviate.io/developers/weaviate/) for all +# configuration. +authentication: + anonymous_access: + enabled: true + # This configuration allows to add API keys to Weaviate. This configuration allows only + # plain text API Keys, if you want to store the API Keys in a Kubernetes secret you can + # configure the same configuration with ENV Vars. Read the `env` section below on what + # needs to be configured. If using ENV Vars over this make sure to comment out the whole + # `apikey` section (as it is by default). ENV Vars has priority over this config. + # apikey: + # enabled: false + # # Any number of allowed API Keys as plain text + # allowed_keys: + # - readOnly-plainText-API-Key + # - admin-plainText-API-Key + # # You can either set a single user for all the listed Allowed API keys OR + # # one user per API Key, i.e. length(apikey.allowed_keys) == length(apikey.users) OR + # # length(apikey.users) == 1 + # # NOTE: Make sure the lister Users are added to the Authorization as well. + # users: + # - api-key-user-readOnly + # - api-key-user-admin + oidc: + enabled: false + # issuer: '' + # username_claim: '' + # groups_claim: '' + # client_id: '' + +authorization: + admin_list: + enabled: false + # users: + # - admin_user1 + # - admin_user2 + # - api-key-user-admin + # read_only_users: + # - readonly_user1 + # - readonly_user2 + # - api-key-user-readOnly + +query_defaults: + limit: 100 +debug: false + + +# Insert any custom environment variables or envSecrets by putting the exact name +# and desired value into the settings below. Any env name passed will be automatically +# set for the statefulSet. +env: + CLUSTER_GOSSIP_BIND_PORT: 7000 + CLUSTER_DATA_BIND_PORT: 7001 + + # Set RAFT cluster expected number of voter nodes at bootstrap. + # By default helm automatically sets this value based on the cluster size. + # RAFT_BOOTSTRAP_EXPECT: 1 + + # Set RAFT cluster bootstrap timeout (in seconds), default is 90 (seconds) + # RAFT_BOOTSTRAP_TIMEOUT: 90 + + # Set manually RAFT voter nodes. + # RAFT_JOIN value is automatically generated by "raft_configuration" + # template, but if someone wants to set this value manually then it can be done + # by setting RAFT_JOIN environment variable, example: RAFT_JOIN: "weaviate-0,weaviate-1" + # Please notice that in this case RAFT_BOOTSTRAP_EXPECT setting needs to be also adjusted manually + # to match the number of RAFT voters, so if there are 2 nodes set using RAFT_JOIN variable + # then RAFT_BOOTSTRAP_EXPECT needs to be equal 2 also. + # RAFT_JOIN: "weaviate-0" + + # Set to true if voters nodes should handle only schema. With this setting enabled + # voter nodes will not accept any data, one needs to resize the cluster using replicas + # setting so that replicas > voters. + # RAFT_METADATA_ONLY_VOTERS: false + + # The aggressiveness of the Go Garbage Collector. 100 is the default value. + GOGC: 100 + + # Expose metrics on port 2112 for Prometheus to scrape + PROMETHEUS_MONITORING_ENABLED: false + PROMETHEUS_MONITORING_GROUP: false + + # Set a MEM limit for the Weaviate Pod so it can help you both increase GC-related + # performance as well as avoid GC-related out-of-memory (“OOM”) situations + # GOMEMLIMIT: 6GiB + + # Maximum results Weaviate can query with/without pagination + # NOTE: Affects performance, do NOT set to a very high value. + # The default is 100K + QUERY_MAXIMUM_RESULTS: 100000 + + # whether to enable vector dimensions tracking metric + TRACK_VECTOR_DIMENSIONS: false + + # whether to re-index/-compute the vector dimensions metric (needed if upgrading from weaviate < v1.16.0) + REINDEX_VECTOR_DIMENSIONS_AT_STARTUP: false + + ########################## + # API Keys with ENV Vars # + ########################## + # If using ENV Vars to set up API Keys make sure to have `authentication.apikey` block commented out + # to avoid any future changes. ENV Vars has priority over the config above `authentication.apikey`. + # If using `authentication.apikey `the below ENV Vars will be used because they have priority, + # so comment them out to avoid any future changes. + # Enables API key authentication. If it is set to 'false' the AUTHENTICATION_APIKEY_ALLOWED_KEYS + # and AUTHENTICATION_APIKEY_USERS will not have any effect. + # AUTHENTICATION_APIKEY_ENABLED: 'true' + + # List one or more keys, separated by commas. Each key corresponds to a specific user identity below. + # If you want to use a kubernetes secret for the API Keys comment out this Variable and use the one in `envSecrets` below + # AUTHENTICATION_APIKEY_ALLOWED_KEYS: 'jane-secret-key,ian-secret-key' (plain text) + + # List one or more user identities, separated by commas. You can have only one User for all the keys or one user per key. + # The User/s can be a simple name or an email, no matter if it exists or not. + # NOTE: Make sure to add the users to the authorization above overwise they will not be allowed to interact with Weaviate. + # AUTHENTICATION_APIKEY_USERS: 'jane@doe.com,ian-smith' + +envSecrets: + # create a Kubernetes secret with AUTHENTICATION_APIKEY_ALLOWED_KEYS key and its respective value + # AUTHENTICATION_APIKEY_ALLOWED_KEYS: name-of-the-k8s-secret-containing-the-comma-separated-api-keys + +# Configure offload providers +offload: + s3: + enabled: false + # If one is using AWS EKS and has already configured K8s Service Account + # that holds the AWS credentials one can pass a name of that service account + # here using this setting. + # NOTE: the root `serviceAccountName` config has priority over this one, and + # if the root one is set this one will NOT overwrite it. This one is here for + # backwards compatibility. + serviceAccountName: + + envconfig: + # Configure bucket where data should be saved, this setting is mandatory + OFFLOAD_S3_BUCKET: weaviate-offload + + # Optional setting. Defaults to AWS S3 (s3.amazonaws.com). + # Set this option if you have a MinIO storage configured in your environment + # and want to use it instead of the AWS S3. + # OFFLOAD_S3_ENDPOINT: custom.minio.endpoint.address + + # Optional setting. Defaults to true. + # Set this option if you don't want to use SSL. + # OFFLOAD_S3_USE_SSL: true + + # Optional setting. Defaults to false. + # Set this option if you wan't Weaviate to create + # the bucket used for offloading tenants. Otherwise, + # if set to false Weaviate expects the bucket to be + # already created with the OFFLOAD_S3_BUCKET name + # OFFLOAD_S3_BUCKET_AUTO_CREATE: true + + # You can pass environment AWS settings here: + # Define the region + # AWS_REGION: eu-west-1 + + # For Weaviate to be able to create bucket objects it needs a user credentials to authenticate to AWS. + # The User must have permissions to read/create/delete bucket objects. + # You can pass the User credentials (access-key id and access-secret-key) in 2 ways: + # 1. by setting the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY plain values in the `secrets` section below + # this chart will create a kubernetes secret for you with these key-values pairs + # 2. create Kubernetes secret/s with AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY keys and their respective values + # Set the Key and the secret where it is set in `envSecrets` section below + secrets: {} + # AWS_ACCESS_KEY_ID: access-key-id (plain text) + # AWS_SECRET_ACCESS_KEY: secret-access-key (plain text) + + # If one has already defined secrets with AWS credentials one can pass them using + # this setting: + envSecrets: {} + # AWS_ACCESS_KEY_ID: name-of-the-k8s-secret-containing-the-key-id + # AWS_SECRET_ACCESS_KEY: name-of-the-k8s-secret-containing-the-key + +# Configure backup providers +backups: + # The backup-filesystem module enables creation of the DB backups in + # the local filesystem + filesystem: + enabled: false + envconfig: + # Configure folder where backups should be saved + BACKUP_FILESYSTEM_PATH: /tmp/backups + + s3: + enabled: false + # If one is using AWS EKS and has already configured K8s Service Account + # that holds the AWS credentials one can pass a name of that service account + # here using this setting. + # NOTE: the root `serviceAccountName` config has priority over this one, and + # if the root one is set this one will NOT overwrite it. This one is here for + # backwards compatibility. + serviceAccountName: + + envconfig: + # Configure bucket where backups should be saved, this setting is mandatory + BACKUP_S3_BUCKET: weaviate-backups + + # Optional setting. Defaults to empty string. + # Set this option if you want to save backups to a given location + # inside the bucket + # BACKUP_S3_PATH: path/inside/bucket + + # Optional setting. Defaults to AWS S3 (s3.amazonaws.com). + # Set this option if you have a MinIO storage configured in your environment + # and want to use it instead of the AWS S3. + # BACKUP_S3_ENDPOINT: custom.minio.endpoint.address + + # Optional setting. Defaults to true. + # Set this option if you don't want to use SSL. + # BACKUP_S3_USE_SSL: true + + # You can pass environment AWS settings here: + # Define the region + # AWS_REGION: eu-west-1 + + # For Weaviate to be able to create bucket objects it needs a user credentials to authenticate to AWS. + # The User must have permissions to read/create/delete bucket objects. + # You can pass the User credentials (access-key id and access-secret-key) in 2 ways: + # 1. by setting the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY plain values in the `secrets` section below + # this chart will create a kubernetes secret for you with these key-values pairs + # 2. create Kubernetes secret/s with AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY keys and their respective values + # Set the Key and the secret where it is set in `envSecrets` section below + secrets: {} + # AWS_ACCESS_KEY_ID: access-key-id (plain text) + # AWS_SECRET_ACCESS_KEY: secret-access-key (plain text) + + # If one has already defined secrets with AWS credentials one can pass them using + # this setting: + envSecrets: {} + # AWS_ACCESS_KEY_ID: name-of-the-k8s-secret-containing-the-key-id + # AWS_SECRET_ACCESS_KEY: name-of-the-k8s-secret-containing-the-key + + gcs: + enabled: false + envconfig: + # Configure bucket where backups should be saved, this setting is mandatory + BACKUP_GCS_BUCKET: weaviate-backups + + # Optional setting. Defaults to empty string. + # Set this option if you want to save backups to a given location + # inside the bucket + # BACKUP_GCS_PATH: path/inside/bucket + + # You can pass environment Google settings here: + # Define the project + # GOOGLE_CLOUD_PROJECT: project-id + + # For Weaviate to be able to create bucket objects it needs a ServiceAccount credentials to authenticate to GCP. + # The ServiceAccount must have permissions to read/create/delete bucket objects. + # You can pass the ServiceAccount credentials (as JSON) in 2 ways: + # 1. by setting the GOOGLE_APPLICATION_CREDENTIALS json as plain text in the `secrets` section below + # this chart will create a kubernetes secret for you with this key-values pairs + # 2. create a Kubernetes secret with GOOGLE_APPLICATION_CREDENTIALS key and its respective value + # Set the Key and the secret where it is set in `envSecrets` section below + secrets: {} + # GOOGLE_APPLICATION_CREDENTIALS: credentials-json-string (plain text) + + # If one has already defined a secret with GOOGLE_APPLICATION_CREDENTIALS one can pass them using + # this setting: + envSecrets: {} + # GOOGLE_APPLICATION_CREDENTIALS: name-of-the-k8s-secret-containing-the-key + + azure: + enabled: false + envconfig: + # Configure container where backups should be saved, this setting is mandatory + BACKUP_AZURE_CONTAINER: weaviate-backups + + # Optional setting. Defaults to empty string. + # Set this option if you want to save backups to a given location + # inside the container + # BACKUP_AZURE_PATH: path/inside/container + + # For Weaviate to be able to create container objects it needs a user credentials to authenticate to Azure Storage. + # The User must have permissions to read/create/delete container objects. + # You can pass the User credentials (account-name id and account-key or connection-string) in 2 ways: + # 1. by setting the AZURE_STORAGE_ACCOUNT and AZURE_STORAGE_KEY + # or AZURE_STORAGE_CONNECTION_STRING plain values in the `secrets` section below + # this chart will create a kubernetes secret for you with these key-values pairs + # 2. create Kubernetes secret/s with AZURE_STORAGE_ACCOUNT and AZURE_STORAGE_KEY + # or AZURE_STORAGE_CONNECTION_STRING and their respective values + # Set the Key and the secret where it is set in `envSecrets` section below + secrets: {} + # AZURE_STORAGE_ACCOUNT: account-name (plain text) + # AZURE_STORAGE_KEY: account-key (plain text) + # AZURE_STORAGE_CONNECTION_STRING: connection-string (plain text) + + # If one has already defined secrets with Azure Storage credentials one can pass them using + # this setting: + envSecrets: {} + # AZURE_STORAGE_ACCOUNT: name-of-the-k8s-secret-containing-the-account-name + # AZURE_STORAGE_KEY: name-of-the-k8s-secret-containing-account-key + # AZURE_STORAGE_CONNECTION_STRING: name-of-the-k8s-secret-containing-connection-string + + +# modules are extensions to Weaviate, they can be used to support various +# ML-models, but also other features unrelated to model inference. +# An inference/vectorizer module is not required, you can also run without any +# modules and import your own vectors. +modules: + + # The text2vec-contextionary module uses a fastText-based vector-space to + # derive vector embeddings for your objects. It is very efficient on CPUs, + # but in some situations it cannot reach the same level of accuracy as + # transformers-based models. + text2vec-contextionary: + + # Enable deployment of this module + enabled: false + + # The configuration below is ignored if enabled==false + fullnameOverride: contextionary + tag: en0.16.0-v1.2.1 + repo: semitechnologies/contextionary + registry: cr.weaviate.io + replicas: 1 + strategy: + type: RollingUpdate + imagePullPolicy: IfNotPresent + imagePullSecrets: [] + priorityClassName: "" + livenessProbe: + initialDelaySeconds: 120 + periodSeconds: 3 + timeoutSeconds: 3 + readinessProbe: + initialDelaySeconds: 120 + periodSeconds: 3 + envconfig: + occurrence_weight_linear_factor: 0.75 + neighbor_occurrence_ignore_percentile: 5 + enable_compound_splitting: false + extensions_storage_mode: weaviate + resources: {} + # requests: + # cpu: '500m' + # memory: '500Mi' + # limits: + # cpu: '1000m' + # memory: '5000Mi' + + # security Context for the Contextionary Pods. The configurations are the same as setting them + # as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + securityContext: {} + + # It is possible to add a ServiceAccount to this module's Pods, it can be + # used in cases where the module is in a private registry and you want to + # give access to the registry only to this pod. + # NOTE: if not set the root `serviceAccountName` config will be used. + serviceAccountName: + + # You can guide where the pods are scheduled on a per-module basis, + # as well as for Weaviate overall. Each module accepts nodeSelector, + # tolerations, and affinity configuration. If it is set on a per- + # module basis, this configuration overrides the global config. + + nodeSelector: + tolerations: + affinity: + + # The text2vec-transformers modules uses neural networks, such as BERT, + # DistilBERT, etc. to dynamically compute vector embeddings based on the + # sentence's context. It is very slow on CPUs and should run with + # CUDA-enabled GPUs for optimal performance. + text2vec-transformers: + + # Enable deployment of this module + enabled: false + + # You can set directly an inference URL of this module without deploying it with this release. + # You can do so by setting a value for the `inferenceUrl` here AND by setting the `enable` to `false` + inferenceUrl: {} + + # The configuration below is ignored if enabled==false + + # replace with model of choice, see + # https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-transformers + # for all supported models or build your own container. + tag: sentence-transformers-paraphrase-multilingual-MiniLM-L12-v2 + repo: semitechnologies/transformers-inference + registry: cr.weaviate.io + replicas: 1 + strategy: + type: RollingUpdate + imagePullPolicy: IfNotPresent + imagePullSecrets: [] + priorityClassName: "" + fullnameOverride: transformers-inference + # Deprecated setting use initialDelaySeconds instead in each probe instead + # probeInitialDelaySeconds: 120 + livenessProbe: + initialDelaySeconds: 120 + periodSeconds: 3 + timeoutSeconds: 3 + readinessProbe: + initialDelaySeconds: 120 + periodSeconds: 3 + envconfig: + # enable for CUDA support. Your K8s cluster needs to be configured + # accordingly and you need to explicitly set GPU requests & limits below + enable_cuda: false + + # only used when CUDA is enabled + nvidia_visible_devices: all + nvidia_driver_capabilities: compute,utility + + # only used when CUDA is enabled + ld_library_path: /usr/local/nvidia/lib64 + + resources: {} + # requests: + # cpu: '1000m' + # memory: '3000Mi' + # enable if running with CUDA support + # nvidia.com/gpu: 1 + # limits: + # cpu: '1000m' + # memory: '5000Mi' + # enable if running with CUDA support + # nvidia.com/gpu: 1 + + # security Context for the Transformers Pods. The configurations are the same as setting them + # as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + # applies to passageQueryService below + securityContext: {} + + # It is possible to add a ServiceAccount to this module's Pods, it can be + # used in cases where the module is in a private registry and you want to + # give access to the registry only to this pod. + # NOTE: if not set the root `serviceAccountName` config will be used. + serviceAccountName: + + # You can guide where the pods are scheduled on a per-module basis, + # as well as for Weaviate overall. Each module accepts nodeSelector, + # tolerations, and affinity configuration. If it is set on a per- + # module basis, this configuration overrides the global config. + nodeSelector: + tolerations: + affinity: + + passageQueryServices: + passage: + enabled: false + # You can set directly an inference URL of this module without deploying it with this release. + # You can do so by setting a value for the `inferenceUrl` here AND by setting the `enable` to `false` + inferenceUrl: {} + + # The configuration below is ignored if enabled==false + tag: facebook-dpr-ctx_encoder-single-nq-base + repo: semitechnologies/transformers-inference + registry: cr.weaviate.io + imagePullPolicy: IfNotPresent + imagePullSecrets: [] + priorityClassName: "" + replicas: 1 + strategy: + type: RollingUpdate + fullnameOverride: transformers-inference-passage + livenessProbe: + initialDelaySeconds: 120 + periodSeconds: 3 + timeoutSeconds: 3 + readinessProbe: + initialDelaySeconds: 120 + periodSeconds: 3 + envconfig: + # enable for CUDA support. Your K8s cluster needs to be configured + # accordingly and you need to explicitly set GPU requests & limits below + enable_cuda: false + + # only used when CUDA is enabled + nvidia_visible_devices: all + nvidia_driver_capabilities: compute,utility + + # only used when CUDA is enabled + ld_library_path: /usr/local/nvidia/lib64 + + resources: {} + # requests: + # cpu: '1000m' + # memory: '3000Mi' + # enable if running with CUDA support + # nvidia.com/gpu: 1 + # limits: + # cpu: '1000m' + # memory: '5000Mi' + # enable if running with CUDA support + # nvidia.com/gpu: 1 + + # security Context for the Transformers Pods. The configurations are the same as setting them + # as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + # applies to passageQueryService below + securityContext: {} + + # It is possible to add a ServiceAccount to this module's Pods, it can be + # used in cases where the module is in a private registry and you want to + # give access to the registry only to this pod. + # NOTE: if not set the root `serviceAccountName` config will be used. + serviceAccountName: + + # You can guide where the pods are scheduled on a per-module basis, + # as well as for Weaviate overall. Each module accepts nodeSelector, + # tolerations, and affinity configuration. If it is set on a per- + # module basis, this configuration overrides the global config. + + nodeSelector: + tolerations: + affinity: + + query: + enabled: false + # You can set directly an inference URL of this module without deploying it with this release. + # You can do so by setting a value for the `inferenceUrl` here AND by setting the `enable` to `false` + inferenceUrl: {} + + # The configuration below is ignored if enabled==false + tag: facebook-dpr-question_encoder-single-nq-base + repo: semitechnologies/transformers-inference + registry: cr.weaviate.io + imagePullPolicy: IfNotPresent + imagePullSecrets: [] + priorityClassName: "" + replicas: 1 + strategy: + type: RollingUpdate + fullnameOverride: transformers-inference-query + livenessProbe: + initialDelaySeconds: 120 + periodSeconds: 3 + timeoutSeconds: 3 + readinessProbe: + initialDelaySeconds: 120 + periodSeconds: 3 + envconfig: + # enable for CUDA support. Your K8s cluster needs to be configured + # accordingly and you need to explicitly set GPU requests & limits below + enable_cuda: false + + # only used when CUDA is enabled + nvidia_visible_devices: all + nvidia_driver_capabilities: compute,utility + + # only used when CUDA is enabled + ld_library_path: /usr/local/nvidia/lib64 + + resources: {} + # requests: + # cpu: '1000m' + # memory: '3000Mi' + # enable if running with CUDA support + # nvidia.com/gpu: 1 + + # limits: + # cpu: '1000m' + # memory: '5000Mi' + # enable if running with CUDA support + # nvidia.com/gpu: 1 + + # security Context for the Transformers Pods. The configurations are the same as setting them + # as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + # applies to passageQueryService below + securityContext: {} + + # It is possible to add a ServiceAccount to this module's Pods, it can be + # used in cases where the module is in a private registry and you want to + # give access to the registry only to this pod. + # NOTE: if not set the root `serviceAccountName` config will be used. + serviceAccountName: + + # You can guide where the pods are scheduled on a per-module basis, + # as well as for Weaviate overall. Each module accepts nodeSelector, + # tolerations, and affinity configuration. If it is set on a per- + # module basis, this configuration overrides the global config. + + nodeSelector: + tolerations: + affinity: + + # The text2vec-gpt4all is a vectorizer module that allows to run ML models from + # nomic-ai/gpt4all: https://docs.gpt4all.io/gpt4all_python_embedding.html + # These models run only on CPU only and on x86_64 arch + # The ML model is containerized in a Weaviate compatible way. + # If you want to run a different model that published ones you can follow the + # tutorial from here on how to create such a container: https://github.com/weaviate/t2v-gpt4all-models + text2vec-gpt4all: + + # Enable deployment of this module + enabled: false + + # You can set directly an inference URL of this module without deploying it with this release. + # You can do so by setting a value for the `inferenceUrl` here AND by setting the `enable` to `false` + inferenceUrl: {} + + # The configuration below is ignored if enabled==false + tag: all-MiniLM-L6-v2 + repo: semitechnologies/gpt4all-inference + registry: cr.weaviate.io + replicas: 1 + strategy: + type: RollingUpdate + imagePullPolicy: IfNotPresent + imagePullSecrets: [] + priorityClassName: "" + fullnameOverride: gpt4all-inference + livenessProbe: + initialDelaySeconds: 120 + periodSeconds: 3 + timeoutSeconds: 3 + readinessProbe: + initialDelaySeconds: 120 + periodSeconds: 3 + + resources: {} + # requests: + # cpu: '1000m' + # memory: '3000Mi' + + # limits: + # cpu: '1000m' + # memory: '5000Mi' + + # security Context for the Contextionary Pods. The configurations are the same as setting them + # as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + securityContext: {} + + # It is possible to add a ServiceAccount to this module's Pods, it can be + # used in cases where the module is in a private registry and you want to + # give access to the registry only to this pod. + # NOTE: if not set the root `serviceAccountName` config will be used. + serviceAccountName: + + # You can guide where the pods are scheduled on a per-module basis, + # as well as for Weaviate overall. Each module accepts nodeSelector, + # tolerations, and affinity configuration. If it is set on a per- + # module basis, this configuration overrides the global config. + + nodeSelector: + tolerations: + affinity: + + # The text2vec-openai module uses OpenAI Embeddings API + # to dynamically compute vector embeddings based on the + # sentence's context. + # More information about OpenAI Embeddings API can be found here: + # https://beta.openai.com/docs/guides/embeddings/what-are-embeddings + text2vec-openai: + + # enable if you want to use OpenAI module + enabled: false + + # Set your OpenAI API Key to be passed to Weaviate pod as + # an environment variable. You can set either the OpenAI's Service API key + # OR the Azure OpenAI's Service Key. Note that you cannot set both of the at the same time. + apiKey: '' + azureApiKey: '' + + # The text2vec-huggingface module uses HuggingFace API + # to dynamically compute vector embeddings based on the + # sentence's context. + # More information about HuggingFace API can be found here: + # https://huggingface.co/docs/api-inference/detailed_parameters#feature-extraction-task + text2vec-huggingface: + + # enable if you want to use HuggingFace module + enabled: false + + # Set your HuggingFace API Key to be passed to Weaviate pod as + # an environment variable + apiKey: '' + + # The text2vec-cohere module uses Cohere API + # to dynamically compute vector embeddings based on the + # sentence's context. + # More information about Cohere API can be found here: https://docs.cohere.ai/ + text2vec-cohere: + + # enable if you want to use Cohere module + enabled: false + + # Set your Cohere API Key to be passed to Weaviate pod as + # an environment variable + apiKey: '' + + # The text2vec-palm module uses Google PaLM Embeddings API + # to dynamically compute vector embeddings based on the + # sentence's context. + # More information about Google PaLM Embeddings API can be found here: + # https://developers.generativeai.google/ + text2vec-palm: + + # enable if you want to use Google PaLM module + enabled: false + + # Set your Google PaLM API Key to be passed to Weaviate pod as + # an environment variable + apiKey: '' + + # The text2vec-jinaai module uses JinaAI Embeddings API + # to dynamically compute vector embeddings based on the + # sentence's context. + # More information about JinaAI Embeddings API can be found here: + # https://jina.ai/embeddings + text2vec-jinaai: + + # enable if you want to use JinaAI module + enabled: false + + # Set your JinaAI API Key to be passed to Weaviate pod as + # an environment variable + apiKey: '' + + # The ref2vec-centroid module + ref2vec-centroid: + + # enable if you want to use Centroid module + enabled: false + + # The text2vec-aws module uses AWS Bedrock service + # to dynamically compute vector embeddings based on the + # sentence's context. + # More information about AWS Bedrock service can be found here: https://docs.aws.amazon.com/bedrock/ + text2vec-aws: + + # enable if you want to use AWS module + enabled: false + + # For Weaviate to be able to use AWS Bedrock service it needs a user credentials to authenticate to AWS. + # The User must have permissions to use AWS Bedrock service AI. + # You can pass the User credentials (access-key id and access-secret-key) in 2 ways: + # 1. by setting the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY plain values in the `secrets` section below + # this chart will create a kubernetes secret for you with these key-values pairs + # 2. create Kubernetes secret/s with AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY keys and their respective values + # Set the Key and the secret where it is set in `envSecrets` section below + secrets: {} + # AWS_ACCESS_KEY_ID: access-key-id (plain text) + # AWS_SECRET_ACCESS_KEY: secret-access-key (plain text) + + # If one has already defined secrets with AWS credentials one can pass them using + # this setting: + envSecrets: {} + # AWS_ACCESS_KEY_ID: name-of-the-k8s-secret-containing-the-key-id + # AWS_SECRET_ACCESS_KEY: name-of-the-k8s-secret-containing-the-key + + # The text2vec-voyageai module uses VoaygeAI Embeddings API + # to dynamically compute vector embeddings based on the + # sentence's context. + # More information about VoaygeAI Embeddings API can be found here: + # https://www.voyageai.com/ + text2vec-voyageai: + + # enable if you want to use VoaygeAI module + enabled: false + + # Set your VoaygeAI API Key to be passed to Weaviate pod as + # an environment variable + apiKey: '' + + # The text2vec-ollama module uses Ollama Embeddings API + # to dynamically compute vector embeddings based on the + # sentence's context. + # More information about Ollama Embeddings API can be found here: + # https://github.com/ollama/ollama/blob/main/docs/api.md#generate-embeddings + # Please note that you have to deploy Ollama container by yourself + # and then properly point the Ollama endpoint using module settings. + # More about this configuration can be found here: + # https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-ollama#ollama-endpoint + text2vec-ollama: + + # enable if you want to use Ollama module + enabled: false + + # The text2vec-octoai module uses OctoAI Embeddings API + # to dynamically compute vector embeddings based on the + # sentence's context. + # More information about OctoAI Embeddings API can be found here: + # https://octo.ai/docs/text-gen-solution/getting-started + text2vec-octoai: + + # enable if you want to use OctoAI module + enabled: false + + # Set your OctoAI API Key to be passed to Weaviate pod as + # an environment variable + apiKey: '' + + # The text2vec-mistral module uses MISTRAL AI Embeddings API + # More information about MISTRAL AI's Embeddings API can be found here: + # https://docs.mistral.ai/api/#operation/createEmbedding + text2vec-mistral: + + # enable if you want to use MISTRAL AI module + enabled: false + + # Set your MISTRAL API Key to be passed to Weaviate pod as + # an environment variable + apiKey: '' + + # The text2vec-databricks module uses Databricks Embeddings API + # to dynamically compute vector embeddings based on the + # sentence's context. + # More information about Databricks Embeddings API can be found here: + # https://docs.databricks.com/en/machine-learning/foundation-models/api-reference.html#embedding-task + text2vec-databricks: + + # enable if you want to use Databricks module + enabled: false + + # Set your Databricks Token to be passed to Weaviate pod as + # an environment variable + token: '' + + # The multi2vec-clip modules uses CLIP transformers to vectorize both images + # and text in the same vector space. It is typically slow(er) on CPUs and should + # run with CUDA-enabled GPUs for optimal performance. + multi2vec-clip: + + # Enable deployment of this module + enabled: false + + # You can set directly an inference URL of this module without deploying it with this release. + # You can do so by setting a value for the `inferenceUrl` here AND by setting the `enable` to `false` + inferenceUrl: {} + + # The configuration below is ignored if enabled==false + + # replace with model of choice, see + # https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/multi2vec-clip + # for all supported models or build your own container. + tag: sentence-transformers-clip-ViT-B-32-multilingual-v1 + repo: semitechnologies/multi2vec-clip + registry: cr.weaviate.io + replicas: 1 + strategy: + type: RollingUpdate + imagePullPolicy: IfNotPresent + imagePullSecrets: [] + priorityClassName: "" + fullnameOverride: clip-inference + livenessProbe: + initialDelaySeconds: 120 + periodSeconds: 3 + timeoutSeconds: 3 + readinessProbe: + initialDelaySeconds: 120 + periodSeconds: 3 + envconfig: + # enable for CUDA support. Your K8s cluster needs to be configured + # accordingly and you need to explicitly set GPU requests & limits below + enable_cuda: false + + # only used when CUDA is enabled + nvidia_visible_devices: all + nvidia_driver_capabilities: compute,utility + + # only used when CUDA is enabled + ld_library_path: /usr/local/nvidia/lib64 + + resources: {} + # requests: + # cpu: '1000m' + # memory: '3000Mi' + # enable if running with CUDA support + # nvidia.com/gpu: 1 + # limits: + # cpu: '1000m' + # memory: '5000Mi' + # enable if running with CUDA support + # nvidia.com/gpu: 1 + + # security Context for the Contextionary Pods. The configurations are the same as setting them + # as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + securityContext: {} + + # It is possible to add a ServiceAccount to this module's Pods, it can be + # used in cases where the module is in a private registry and you want to + # give access to the registry only to this pod. + # NOTE: if not set the root `serviceAccountName` config will be used. + serviceAccountName: + + annotations: + nodeSelector: + tolerations: + + # The multi2vec-bind is a multi modal inference container it uses Meta's open + # source ImageBind implementation as base for this module: + # https://github.com/facebookresearch/ImageBind + # It is possible to create your own version of this module by following the instructions here: + # https://github.com/weaviate/multi2vec-bind-inference + multi2vec-bind: + + # Enable deployment of this module + enabled: false + + # You can set directly an inference URL of this module without deploying it with this release. + # You can do so by setting a value for the `inferenceUrl` here AND by setting the `enable` to `false` + inferenceUrl: {} + + # The configuration below is ignored if enabled==false + tag: imagebind + repo: semitechnologies/multi2vec-bind + registry: cr.weaviate.io + replicas: 1 + strategy: + type: RollingUpdate + imagePullPolicy: IfNotPresent + imagePullSecrets: [] + priorityClassName: "" + fullnameOverride: bind-inference + livenessProbe: + initialDelaySeconds: 120 + periodSeconds: 3 + timeoutSeconds: 3 + readinessProbe: + initialDelaySeconds: 120 + periodSeconds: 3 + envconfig: + # enable for CUDA support. Your K8s cluster needs to be configured + # accordingly and you need to explicitly set GPU requests & limits below + enable_cuda: false + + # only used when CUDA is enabled + nvidia_visible_devices: all + nvidia_driver_capabilities: compute,utility + + # only used when CUDA is enabled + ld_library_path: /usr/local/nvidia/lib64 + + resources: {} + # requests: + # cpu: '1000m' + # memory: '7000Mi' + # enable if running with CUDA support + # nvidia.com/gpu: 1 + # limits: + # cpu: '1000m' + # memory: '7000Mi' + # enable if running with CUDA support + # nvidia.com/gpu: 1 + + # security Context for the Contextionary Pods. The configurations are the same as setting them + # as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + securityContext: {} + + # It is possible to add a ServiceAccount to this module's Pods, it can be + # used in cases where the module is in a private registry and you want to + # give access to the registry only to this pod. + # NOTE: if not set the root `serviceAccountName` config will be used. + serviceAccountName: + + annotations: + nodeSelector: + tolerations: + + # The multi2vec-palm module uses Google PaLM Embeddings API + # to dynamically compute vector embeddings. + # More information about Google PaLM Embeddings API can be found here: + # https://cloud.google.com/vertex-ai/generative-ai/docs/embeddings/get-multimodal-embeddings + multi2vec-palm: + + # enable if you want to use Google Multimodal PaLM module + enabled: false + + # Set your Google PaLM API Key to be passed to Weaviate pod as + # an environment variable + apiKey: '' + + # The qna-transformers module uses neural networks, such as BERT, + # DistilBERT, to find an answer in text to a given question + qna-transformers: + + # Enable deployment of this module + enabled: false + + # You can set directly an inference URL of this module without deploying it with this release. + # You can do so by setting a value for the `inferenceUrl` here AND by setting the `enable` to `false` + inferenceUrl: {} + + # The configuration below is ignored if enabled==false + tag: bert-large-uncased-whole-word-masking-finetuned-squad + repo: semitechnologies/qna-transformers + registry: cr.weaviate.io + replicas: 1 + strategy: + type: RollingUpdate + imagePullPolicy: IfNotPresent + imagePullSecrets: [] + priorityClassName: "" + fullnameOverride: qna-transformers + livenessProbe: + initialDelaySeconds: 120 + periodSeconds: 3 + timeoutSeconds: 3 + readinessProbe: + initialDelaySeconds: 120 + periodSeconds: 3 + envconfig: + # enable for CUDA support. Your K8s cluster needs to be configured + # accordingly and you need to explicitly set GPU requests & limits below + enable_cuda: false + + # only used when CUDA is enabled + nvidia_visible_devices: all + nvidia_driver_capabilities: compute,utility + + # only used when CUDA is enabled + ld_library_path: /usr/local/nvidia/lib64 + + resources: {} + # requests: + # cpu: '1000m' + # memory: '3000Mi' + # enable if running with CUDA support + # nvidia.com/gpu: 1 + # limits: + # cpu: '1000m' + # memory: '5000Mi' + # enable if running with CUDA support + # nvidia.com/gpu: 1 + + # security Context for the Contextionary Pods. The configurations are the same as setting them + # as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + securityContext: {} + + # It is possible to add a ServiceAccount to this module's Pods, it can be + # used in cases where the module is in a private registry and you want to + # give access to the registry only to this pod. + # NOTE: if not set the root `serviceAccountName` config will be used. + serviceAccountName: + + # You can guide where the pods are scheduled on a per-module basis, + # as well as for Weaviate overall. Each module accepts nodeSelector, + # tolerations, and affinity configuration. If it is set on a per- + # module basis, this configuration overrides the global config. + + nodeSelector: + tolerations: + affinity: + + # The qna-openai module uses OpenAI Completions API + # to dynamically answer given questions. + # More information about OpenAI Completions API can be found here: + # https://beta.openai.com/docs/api-reference/completions + qna-openai: + + # enable if you want to use OpenAI module + enabled: false + + # Set your OpenAI API Key to be passed to Weaviate pod as + # an environment variable. You can set either the OpenAI's Service API key + # OR the Azure OpenAI's Service Key. Note that you cannot set both of the at the same time. + apiKey: '' + azureApiKey: '' + + # The generative-openai module uses OpenAI Completions API + # along with text-davinci-003 model to behave as ChatGPT. + # More information about OpenAI Completions API can be found here: + # https://beta.openai.com/docs/api-reference/completions + generative-openai: + + # enable if you want to use OpenAI module + enabled: false + + # Set your OpenAI API Key to be passed to Weaviate pod as + # an environment variable. You can set either the OpenAI's Service API key + # OR the Azure OpenAI's Service Key. Note that you cannot set both of the at the same time. + apiKey: '' + azureApiKey: '' + + # The generative-cohere module uses Cohere Generate API + # More information about Cohere's Generate API can be found here: + # https://docs.cohere.com/reference/generate + generative-cohere: + + # enable if you want to use Cohere generative module + enabled: false + + # Set your Cohere API Key to be passed to Weaviate pod as + # an environment variable + apiKey: '' + + # The generative-palm module uses Google PaLM API. + # More information about Google PaLM API can be found here: + # https://developers.generativeai.google/ + generative-palm: + + # enable if you want to use Google PaLM module + enabled: false + + # Set your Google PaLM API Key to be passed to Weaviate pod as + # an environment variable + apiKey: '' + + # The generative-aws module uses AWS Bedrock service. + # More information about AWS Bedrock service can be found here: + # https://docs.aws.amazon.com/bedrock/ + generative-aws: + + # enable if you want to use AWS module + enabled: false + + # For Weaviate to be able to use AWS Bedrock service it needs a user credentials to authenticate to AWS. + # The User must have permissions to use AWS Bedrock service AI. + # You can pass the User credentials (access-key id and access-secret-key) in 2 ways: + # 1. by setting the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY plain values in the `secrets` section below + # this chart will create a kubernetes secret for you with these key-values pairs + # 2. create Kubernetes secret/s with AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY keys and their respective values + # Set the Key and the secret where it is set in `envSecrets` section below + secrets: {} + # AWS_ACCESS_KEY_ID: access-key-id (plain text) + # AWS_SECRET_ACCESS_KEY: secret-access-key (plain text) + + # If one has already defined secrets with AWS credentials one can pass them using + # this setting: + envSecrets: {} + # AWS_ACCESS_KEY_ID: name-of-the-k8s-secret-containing-the-key-id + # AWS_SECRET_ACCESS_KEY: name-of-the-k8s-secret-containing-the-key + + # The generative-anyscale module uses Anyscale API + # More information about Anyscale API can be found here: + # https://docs.anyscale.com/endpoints/overview + generative-anyscale: + + # enable if you want to use Anyscale module + enabled: false + + # Set your Anyscale API Key to be passed to Weaviate pod as + # an environment variable. + apiKey: '' + + # The generative-mistral module uses MISTRAL AI Generative API + # More information about MISTRAL AI's Generative API can be found here: + # https://docs.mistral.ai/ + generative-mistral: + + # enable if you want to use MISTRAL AI generative module + enabled: false + + # Set your MISTRAL API Key to be passed to Weaviate pod as + # an environment variable + apiKey: '' + + # The generative-ollama module uses Ollama Generate API + # More information about Ollama's Generate API can be found here: + # https://github.com/ollama/ollama/blob/main/docs/api.md#generate-a-completion + # Please note that you have to deploy Ollama container by yourself + # and then properly point the Ollama endpoint using module settings. + # More about this configuration can be found here: + # https://weaviate.io/developers/weaviate/modules/reader-generator-modules/generative-ollama#ollama-endpoint + generative-ollama: + + # enable if you want to use Ollama module + enabled: false + + # The generative-octoai module uses OctoAI API. + # More information about OctoAI API can be found here: + # https://octo.ai/docs/text-gen-solution/getting-started + generative-octoai: + + # enable if you want to use OctoAI module + enabled: false + + # Set your OctoAI API Key to be passed to Weaviate pod as + # an environment variable + apiKey: '' + + # The generative-anthropic module uses Anthropic API. + # More information about Anthropic API can be found here: + # https://docs.anthropic.com/en/api/getting-started + generative-anthropic: + + # enable if you want to use Anthropic module + enabled: false + + # Set your Anthropic API Key to be passed to Weaviate pod as + # an environment variable + apiKey: '' + + # The generative-friendliai module uses FriendliAI API. + # More information about FriendliAI API can be found here: + # https://docs.friendli.ai/openapi/create-chat-completions + generative-friendliai: + + # enable if you want to use FriendliAI module + enabled: false + + # Set your FriendliAI Token to be passed to Weaviate pod as + # an environment variable + token: '' + + # The generative-databricks module uses Databricks API. + # More information about Databricks API can be found here: + # https://docs.databricks.com/en/machine-learning/foundation-models/api-reference.html#completion-task + generative-databricks: + + # enable if you want to use Databricks module + enabled: false + + # Set your Databricks Token to be passed to Weaviate pod as + # an environment variable + token: '' + + # The img2vec-neural module uses neural networks, to generate + # a vector representation of the image + img2vec-neural: + + # Enable deployment of this module + enabled: false + + # You can set directly an inference URL of this module without deploying it with this release. + # You can do so by setting a value for the `inferenceUrl` here AND by setting the `enable` to `false` + inferenceUrl: {} + + tag: resnet50 + repo: semitechnologies/img2vec-pytorch + registry: cr.weaviate.io + replicas: 1 + strategy: + type: RollingUpdate + imagePullPolicy: IfNotPresent + imagePullSecrets: [] + priorityClassName: "" + fullnameOverride: img2vec-neural + livenessProbe: + initialDelaySeconds: 120 + periodSeconds: 3 + timeoutSeconds: 3 + readinessProbe: + initialDelaySeconds: 120 + periodSeconds: 3 + envconfig: + # enable for CUDA support. Your K8s cluster needs to be configured + # accordingly and you need to explicitly set GPU requests & limits below + enable_cuda: false + + # only used when CUDA is enabled + nvidia_visible_devices: all + nvidia_driver_capabilities: compute,utility + + # only used when CUDA is enabled + ld_library_path: /usr/local/nvidia/lib64 + + resources: {} + # requests: + # cpu: '1000m' + # memory: '3000Mi' + # enable if running with CUDA support + # nvidia.com/gpu: 1 + # limits: + # cpu: '1000m' + # memory: '5000Mi' + # enable if running with CUDA support + # nvidia.com/gpu: 1 + + # security Context for the Contextionary Pods. The configurations are the same as setting them + # as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + securityContext: {} + + # It is possible to add a ServiceAccount to this module's Pods, it can be + # used in cases where the module is in a private registry and you want to + # give access to the registry only to this pod. + # NOTE: if not set the root `serviceAccountName` config will be used. + serviceAccountName: + + # You can guide where the pods are scheduled on a per-module basis, + # as well as for Weaviate overall. Each module accepts nodeSelector, + # tolerations, and affinity configuration. If it is set on a per- + # module basis, this configuration overrides the global config. + + nodeSelector: + tolerations: + affinity: + + # The reranker-cohere module uses Cohere API + # to dynamically compute a score for the relevance + # of the query with each of the initial search results. + # More information about Cohere API can be found here: https://docs.cohere.com/docs/rerank-guide + reranker-cohere: + + # enable if you want to use Cohere rerank module + enabled: false + + # Set your Cohere API Key to be passed to Weaviate pod as + # an environment variable + apiKey: '' + + # The reranker-voyageai module uses VoaygeAI API + # to dynamically compute a score for the relevance + # of the query with each of the initial search results. + # More information about Cohere API can be found here: https://www.voyageai.com/ + reranker-voyageai: + + # enable if you want to use VoaygeAI module + enabled: false + + # Set your VoaygeAI API Key to be passed to Weaviate pod as + # an environment variable + apiKey: '' + + # The reranker-jinaai module uses JinaAI API + # to dynamically compute a score for the relevance + # of the query with each of the initial search results. + # More information about Cohere API can be found here: https://jina.ai/reranker + reranker-jinaai: + + # enable if you want to use JinaAI module + enabled: false + + # Set your JinaAI API Key to be passed to Weaviate pod as + # an environment variable + apiKey: '' + + # The reranker-transformers module uses Cross-Encoders for + # sentence pair scoring and sentence pair classification tasks. + # More information about Cross-Encoders can be found here: + # https://www.sbert.net/examples/applications/cross-encoder/README.html + reranker-transformers: + + # Enable deployment of this module + enabled: false + + # You can set directly an inference URL of this module without deploying it with this release. + # You can do so by setting a value for the `inferenceUrl` here AND by setting the `enable` to `false` + inferenceUrl: {} + + # The configuration below is ignored if enabled==false + tag: cross-encoder-ms-marco-MiniLM-L-6-v2 + repo: semitechnologies/reranker-transformers + registry: cr.weaviate.io + replicas: 1 + strategy: + type: RollingUpdate + imagePullPolicy: IfNotPresent + imagePullSecrets: [] + priorityClassName: "" + fullnameOverride: reranker-transformers + livenessProbe: + initialDelaySeconds: 120 + periodSeconds: 3 + timeoutSeconds: 3 + readinessProbe: + initialDelaySeconds: 120 + periodSeconds: 3 + envconfig: + # enable for CUDA support. Your K8s cluster needs to be configured + # accordingly and you need to explicitly set GPU requests & limits below + enable_cuda: false + + # only used when CUDA is enabled + nvidia_visible_devices: all + nvidia_driver_capabilities: compute,utility + + # only used when CUDA is enabled + ld_library_path: /usr/local/nvidia/lib64 + + resources: {} + # requests: + # cpu: '1000m' + # memory: '3000Mi' + # enable if running with CUDA support + # nvidia.com/gpu: 1 + # limits: + # cpu: '1000m' + # memory: '5000Mi' + # enable if running with CUDA support + # nvidia.com/gpu: 1 + + # security Context for the Contextionary Pods. The configurations are the same as setting them + # as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + securityContext: {} + + # It is possible to add a ServiceAccount to this module's Pods, it can be + # used in cases where the module is in a private registry and you want to + # give access to the registry only to this pod. + # NOTE: if not set the root `serviceAccountName` config will be used. + serviceAccountName: + + # You can guide where the pods are scheduled on a per-module basis, + # as well as for Weaviate overall. Each module accepts nodeSelector, + # tolerations, and affinity configuration. If it is set on a per- + # module basis, this configuration overrides the global config. + + nodeSelector: + tolerations: + affinity: + + # The text-spellcheck module uses spellchecker library to check + # misspellings in a given text + text-spellcheck: + + # Enable deployment of this module + enabled: false + + # You can set directly an inference URL of this module without deploying it with this release. + # You can do so by setting a value for the `inferenceUrl` here AND by setting the `enable` to `false` + inferenceUrl: {} + + # The configuration below is ignored if enabled==false + tag: pyspellchecker-en + repo: semitechnologies/text-spellcheck-model + registry: cr.weaviate.io + replicas: 1 + strategy: + type: RollingUpdate + imagePullPolicy: IfNotPresent + imagePullSecrets: [] + priorityClassName: "" + fullnameOverride: text-spellcheck + livenessProbe: + initialDelaySeconds: 120 + periodSeconds: 3 + timeoutSeconds: 3 + readinessProbe: + initialDelaySeconds: 120 + periodSeconds: 3 + + resources: {} + # requests: + # cpu: '400m' + # memory: '400Mi' + # limits: + # cpu: '500m' + # memory: '500Mi' + + # security Context for the Contextionary Pods. The configurations are the same as setting them + # as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + securityContext: {} + + # It is possible to add a ServiceAccount to this module's Pods, it can be + # used in cases where the module is in a private registry and you want to + # give access to the registry only to this pod. + # NOTE: if not set the root `serviceAccountName` config will be used. + serviceAccountName: + + # You can guide where the pods are scheduled on a per-module basis, + # as well as for Weaviate overall. Each module accepts nodeSelector, + # tolerations, and affinity configuration. If it is set on a per- + # module basis, this configuration overrides the global config. + + nodeSelector: + tolerations: + affinity: + + # The ner-transformers module uses spellchecker library to check + # misspellings in a given text + ner-transformers: + + # Enable deployment of this module + enabled: false + + # You can set directly an inference URL of this module without deploying it with this release. + # You can do so by setting a value for the `inferenceUrl` here AND by setting the `enable` to `false` + inferenceUrl: {} + + # The configuration below is ignored if enabled==false + tag: dbmdz-bert-large-cased-finetuned-conll03-english + repo: semitechnologies/ner-transformers + registry: cr.weaviate.io + replicas: 1 + strategy: + type: RollingUpdate + imagePullPolicy: IfNotPresent + imagePullSecrets: [] + priorityClassName: "" + fullnameOverride: ner-transformers + livenessProbe: + initialDelaySeconds: 120 + periodSeconds: 3 + timeoutSeconds: 3 + readinessProbe: + initialDelaySeconds: 120 + periodSeconds: 3 + envconfig: + # enable for CUDA support. Your K8s cluster needs to be configured + # accordingly and you need to explicitly set GPU requests & limits below + enable_cuda: false + + # only used when CUDA is enabled + nvidia_visible_devices: all + nvidia_driver_capabilities: compute,utility + + # only used when CUDA is enabled + ld_library_path: /usr/local/nvidia/lib64 + + resources: {} + # requests: + # cpu: '1000m' + # memory: '3000Mi' + + # enable if running with CUDA support + # nvidia.com/gpu: 1 + # limits: + # cpu: '1000m' + # memory: '5000Mi' + + # enable if running with CUDA support + # nvidia.com/gpu: 1 + + # security Context for the Contextionary Pods. The configurations are the same as setting them + # as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + securityContext: {} + + # It is possible to add a ServiceAccount to this module's Pods, it can be + # used in cases where the module is in a private registry and you want to + # give access to the registry only to this pod. + # NOTE: if not set the root `serviceAccountName` config will be used. + serviceAccountName: + + # You can guide where the pods are scheduled on a per-module basis, + # as well as for Weaviate overall. Each module accepts nodeSelector, + # tolerations, and affinity configuration. If it is set on a per- + # module basis, this configuration overrides the global config. + + nodeSelector: + tolerations: + affinity: + + # The sum-transformers module makes result texts summarizations + sum-transformers: + + # Enable deployment of this module + enabled: false + + # You can set directly an inference URL of this module without deploying it with this release. + # You can do so by setting a value for the `inferenceUrl` here AND by setting the `enable` to `false` + inferenceUrl: {} + + # The configuration below is ignored if enabled==false + tag: facebook-bart-large-cnn + repo: semitechnologies/sum-transformers + registry: cr.weaviate.io + replicas: 1 + strategy: + type: RollingUpdate + imagePullPolicy: IfNotPresent + imagePullSecrets: [] + priorityClassName: "" + fullnameOverride: sum-transformers + livenessProbe: + initialDelaySeconds: 120 + periodSeconds: 3 + timeoutSeconds: 3 + readinessProbe: + initialDelaySeconds: 120 + periodSeconds: 3 + envconfig: + # enable for CUDA support. Your K8s cluster needs to be configured + # accordingly and you need to explicitly set GPU requests & limits below + enable_cuda: false + + # only used when CUDA is enabled + nvidia_visible_devices: all + nvidia_driver_capabilities: compute,utility + + # only used when CUDA is enabled + ld_library_path: /usr/local/nvidia/lib64 + + resources: {} + # requests: + # cpu: '1000m' + # memory: '3000Mi' + # enable if running with CUDA support + # nvidia.com/gpu: 1 + # limits: + # cpu: '1000m' + # memory: '5000Mi' + # enable if running with CUDA support + # nvidia.com/gpu: 1 + + # security Context for the Contextionary Pods. The configurations are the same as setting them + # as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + securityContext: {} + + # It is possible to add a ServiceAccount to this module's Pods, it can be + # used in cases where the module is in a private registry and you want to + # give access to the registry only to this pod. + # NOTE: if not set the root `serviceAccountName` config will be used. + serviceAccountName: + + # You can guide where the pods are scheduled on a per-module basis, + # as well as for Weaviate overall. Each module accepts nodeSelector, + # tolerations, and affinity configuration. If it is set on a per- + # module basis, this configuration overrides the global config. + + nodeSelector: + tolerations: + affinity: + + # by choosing the default vectorizer module, you can tell Weaviate to always + # use this module as the vectorizer if nothing else is specified. Can be + # overwritten on a per-class basis. + # set to text2vec-transformers if running with transformers instead + default_vectorizer_module: none + +# It is also possible to configure authentication and authorization through a +# custom configmap The authorization and authentication values defined in +# values.yaml will be ignored when defining a custom config map. +custom_config_map: + enabled: false + name: 'custom-config' + +# Pass any annotations to Weaviate pods +annotations: + +extraVolumeMounts: + +extraVolumes: + +nodeSelector: + +tolerations: + +hostAliases: + +affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + podAffinityTerm: + topologyKey: "kubernetes.io/hostname" + labelSelector: + matchExpressions: + - key: "app" + operator: In + values: + - weaviate + +## Optionally specify priorityClass name for the pod +## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#pod-priority +## +priorityClassName: "" +globalPriorityClassName: "" + From 5bcd4860d3c158a91e3e99f95ce8ec2dc1b43c93 Mon Sep 17 00:00:00 2001 From: mungitoperrito Date: Wed, 11 Sep 2024 17:40:04 -0400 Subject: [PATCH 04/67] files --- values.yaml | 1857 --------------------------------------------------- 1 file changed, 1857 deletions(-) delete mode 100644 values.yaml diff --git a/values.yaml b/values.yaml deleted file mode 100644 index 2b1ea547d9..0000000000 --- a/values.yaml +++ /dev/null @@ -1,1857 +0,0 @@ -image: - # registry where weaviate image is stored - registry: cr.weaviate.io - # Tag of weaviate image to deploy - # Note: We strongly recommend you overwrite this value in your own values.yaml. - # Otherwise a mere upgrade of the chart could lead to an unexpected upgrade - # of weaviate. In accordance with Infra-as-code, you should pin this value - # down and only change it if you explicitly want to upgrade the Weaviate - # version. - tag: 1.26.3 - repo: semitechnologies/weaviate - # Image pull policy: https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy - pullPolicy: IfNotPresent - pullSecrets: [] - -# overwrite command and args if you want to run specific startup scripts, for -# example setting the nofile limit -command: ["/bin/weaviate"] -args: - - '--host' - - '0.0.0.0' - - '--port' - - '8080' - - '--scheme' - - 'http' - - '--config-file' - - '/weaviate-config/conf.yaml' - - --read-timeout=60s - - --write-timeout=60s - -# below is an example that can be used to set an arbitrary nofile limit at -# startup: -# -# command: -# - "/bin/sh" -# args: -# - "-c" -# - "ulimit -n 65535 && /bin/weaviate --host 0.0.0.0 --port 8080 --scheme http --config-file /weaviate-config/conf.yaml" - - -# it is possible to change the sysctl's 'vm.max_map_count' using initContainer for Weaviate, -# the init Container runs before Weaviate Container and sets the value for the WHOLE node -# to the one provided below. -# it is possible to run additional initContainer before Weaviate is up and running. You can specify the -# containers as a list in `extraInitContainers`, exactly how they are defined in a kubernetes manifest: -# https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ -initContainers: - sysctlInitContainer: - enabled: true - sysctlVmMaxMapCount: 524288 - image: - registry: docker.io - repo: alpine - tag: latest - pullPolicy: IfNotPresent - ensureFileOwnershipContainer: - # This init container sets the file ownerships of /var/lib/weaviate directory to the ones set in - # containerSecurityContext.runAsUser and containerSecurityContext.fsGroup settings to ensure that Weaviate is able - # to start in unprivileged configuration. - # Enable this init container only if Weaviate was configured previously without security context - # and now containerSecurityContext is provided to run Weaviate container with non-root user. - # Please be sure to set at least containerSecurityContext.runAsUser and containerSecurityContext.fsGroup. - enabled: false - - extraInitContainers: {} - # - image: some-image - # name: some-name - -# Scale replicas of Weaviate. Note that as of v1.8.0 dynamic scaling is limited -# to cases where no data is imported yet. Scaling down after importing data may -# break usability. Full dynamic scalability will be added in a future release. -replicas: 1 - -# Define how pods will be created. Possible values: OrderedReady | Parallel -# OrderedReady - pods will be created one after another -# Parallel - all pods will be created at once -podManagementPolicy: Parallel -updateStrategy: - type: RollingUpdate - # This setting is only available in K8s v1.24 and higher. - # Setting maxUnavailable to 100% results in removing all of the pods - # and re-creating them in parallel all at once. - # rollingUpdate: - # maxUnavailable: 100% -resources: {} - # requests: - # cpu: '500m' - # memory: '300Mi' - # limits: - # cpu: '1000m' - # memory: '1Gi' - -# security Context for the Weaviate Pods. The configurations are the same as setting them -# as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ -securityContext: {} - -# Security context for the Weaviate container. Override overlapping settings made at the Pod level. -containerSecurityContext: {} - # runAsUser: 1000 - # runAsGroup: 1000 - # fsGroup: 1000 - # fsGroupChangePolicy: "OnRootMismatch" - # runAsNonRoot: true - # allowPrivilegeEscalation: false - # privileged: false - # readOnlyRootFilesystem: true - -# Add a service account to the Weaviate pods if you need Weaviate to have permissions to -# access kubernetes resources or cloud provider resources. For example for it to have -# access to a backup up bucket, or if you want to restrict Weaviate pod in any way. -# By default, use the default ServiceAccount -serviceAccountName: - -# Kubernetes Cluster domain name, used for resolving intra-cluster requests, i.e -# between instances of weaviate. -# Note: The final '.' on the end of the hostname makes it a FQDN, and is required for -# DNS to resolve in all kubernetes environments. -# See https://github.com/weaviate/weaviate-helm/issues/175 for details. -clusterDomain: cluster.local. - -# The Persistent Volume Claim settings for Weaviate. If there's a -# storage.fullnameOverride field set, then the default pvc will not be -# created, instead the one defined in fullnameOverride will be used -storage: - size: 32Gi - storageClassName: "" - -# The service controls how weaviate is exposed to the outside world. If you -# don't want a public load balancer, you can also choose 'ClusterIP' to make -# weaviate only accessible within your cluster. -service: - name: weaviate - ports: - - name: http - protocol: TCP - port: 80 - # Target port is going to be the same for every port - type: LoadBalancer - loadBalancerSourceRanges: [] - # optionally set cluster IP if you want to set a static IP - clusterIP: - annotations: {} - -# The service controls how weaviate gRPC endpoint is exposed to the outside world. -# If you don't want a public load balancer, you can also choose 'ClusterIP' or `NodePort` -# to make weaviate gRPC port be only accessible within your cluster. -# This service is by default enabled but if you don't want it to be deployed in your -# environment then it can be disabled by setting enabled: false option. -grpcService: - enabled: true - name: weaviate-grpc - ports: - - name: grpc - protocol: TCP - port: 50051 - # Target port is going to be the same for every port - type: LoadBalancer - loadBalancerSourceRanges: [] - # optionally set cluster IP if you want to set a static IP - clusterIP: - annotations: {} - -# The service monitor defines prometheus monitoring for a set of services -# https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.ServiceMonitor -# Make sure to set the following prometheus values if deploying observability with the kube-prometheus-stack helm chart: -# - prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues: false -serviceMonitor: - enabled: false - interval: 30s - scrapeTimeout: 10s - -# Adjust liveness, readiness and startup probes configuration -# below is an example that can be used to switch the probeType to exec command -# readinessProbe: # (Compatible with liveness, readiness and startup probe configurations) -# probeType: exec -# probe: -# exec: -# command: ["/bin/sh", "-c", "wget --spider --server-response --tries=1 --timeout=30 -o /dev/null localhost:8080/v1/.well-known/ready"] - -startupProbe: - # For kubernetes versions prior to 1.18 startupProbe is not supported thus can be disabled. - enabled: false - probeType: httpGet - probe: - httpGet: - path: /v1/.well-known/ready - port: 8080 - initialDelaySeconds: 300 - periodSeconds: 60 - failureThreshold: 50 - successThreshold: 1 - timeoutSeconds: 3 - -livenessProbe: - probeType: httpGet - probe: - httpGet: - path: /v1/.well-known/live - port: 8080 - initialDelaySeconds: 900 - periodSeconds: 10 - failureThreshold: 30 - successThreshold: 1 - timeoutSeconds: 3 - -readinessProbe: - probeType: httpGet - probe: - httpGet: - path: /v1/.well-known/ready - port: 8080 - initialDelaySeconds: 3 - periodSeconds: 10 - failureThreshold: 3 - successThreshold: 1 - timeoutSeconds: 3 - - -terminationGracePeriodSeconds: 600 - -# Weaviate Config -# -# The following settings allow you to customize Weaviate to your needs, for -# example set authentication and authorization options. See weaviate docs -# (https://www.weaviate.io/developers/weaviate/) for all -# configuration. -authentication: - anonymous_access: - enabled: true - # This configuration allows to add API keys to Weaviate. This configuration allows only - # plain text API Keys, if you want to store the API Keys in a Kubernetes secret you can - # configure the same configuration with ENV Vars. Read the `env` section below on what - # needs to be configured. If using ENV Vars over this make sure to comment out the whole - # `apikey` section (as it is by default). ENV Vars has priority over this config. - # apikey: - # enabled: false - # # Any number of allowed API Keys as plain text - # allowed_keys: - # - readOnly-plainText-API-Key - # - admin-plainText-API-Key - # # You can either set a single user for all the listed Allowed API keys OR - # # one user per API Key, i.e. length(apikey.allowed_keys) == length(apikey.users) OR - # # length(apikey.users) == 1 - # # NOTE: Make sure the lister Users are added to the Authorization as well. - # users: - # - api-key-user-readOnly - # - api-key-user-admin - oidc: - enabled: false - # issuer: '' - # username_claim: '' - # groups_claim: '' - # client_id: '' - -authorization: - admin_list: - enabled: false - # users: - # - admin_user1 - # - admin_user2 - # - api-key-user-admin - # read_only_users: - # - readonly_user1 - # - readonly_user2 - # - api-key-user-readOnly - -query_defaults: - limit: 100 -debug: false - - -# Insert any custom environment variables or envSecrets by putting the exact name -# and desired value into the settings below. Any env name passed will be automatically -# set for the statefulSet. -env: - CLUSTER_GOSSIP_BIND_PORT: 7000 - CLUSTER_DATA_BIND_PORT: 7001 - - # Set RAFT cluster expected number of voter nodes at bootstrap. - # By default helm automatically sets this value based on the cluster size. - # RAFT_BOOTSTRAP_EXPECT: 1 - - # Set RAFT cluster bootstrap timeout (in seconds), default is 90 (seconds) - # RAFT_BOOTSTRAP_TIMEOUT: 90 - - # Set manually RAFT voter nodes. - # RAFT_JOIN value is automatically generated by "raft_configuration" - # template, but if someone wants to set this value manually then it can be done - # by setting RAFT_JOIN environment variable, example: RAFT_JOIN: "weaviate-0,weaviate-1" - # Please notice that in this case RAFT_BOOTSTRAP_EXPECT setting needs to be also adjusted manually - # to match the number of RAFT voters, so if there are 2 nodes set using RAFT_JOIN variable - # then RAFT_BOOTSTRAP_EXPECT needs to be equal 2 also. - # RAFT_JOIN: "weaviate-0" - - # Set to true if voters nodes should handle only schema. With this setting enabled - # voter nodes will not accept any data, one needs to resize the cluster using replicas - # setting so that replicas > voters. - # RAFT_METADATA_ONLY_VOTERS: false - - # The aggressiveness of the Go Garbage Collector. 100 is the default value. - GOGC: 100 - - # Expose metrics on port 2112 for Prometheus to scrape - PROMETHEUS_MONITORING_ENABLED: false - PROMETHEUS_MONITORING_GROUP: false - - # Set a MEM limit for the Weaviate Pod so it can help you both increase GC-related - # performance as well as avoid GC-related out-of-memory (“OOM”) situations - # GOMEMLIMIT: 6GiB - - # Maximum results Weaviate can query with/without pagination - # NOTE: Affects performance, do NOT set to a very high value. - # The default is 100K - QUERY_MAXIMUM_RESULTS: 100000 - - # whether to enable vector dimensions tracking metric - TRACK_VECTOR_DIMENSIONS: false - - # whether to re-index/-compute the vector dimensions metric (needed if upgrading from weaviate < v1.16.0) - REINDEX_VECTOR_DIMENSIONS_AT_STARTUP: false - - ########################## - # API Keys with ENV Vars # - ########################## - # If using ENV Vars to set up API Keys make sure to have `authentication.apikey` block commented out - # to avoid any future changes. ENV Vars has priority over the config above `authentication.apikey`. - # If using `authentication.apikey `the below ENV Vars will be used because they have priority, - # so comment them out to avoid any future changes. - # Enables API key authentication. If it is set to 'false' the AUTHENTICATION_APIKEY_ALLOWED_KEYS - # and AUTHENTICATION_APIKEY_USERS will not have any effect. - # AUTHENTICATION_APIKEY_ENABLED: 'true' - - # List one or more keys, separated by commas. Each key corresponds to a specific user identity below. - # If you want to use a kubernetes secret for the API Keys comment out this Variable and use the one in `envSecrets` below - # AUTHENTICATION_APIKEY_ALLOWED_KEYS: 'jane-secret-key,ian-secret-key' (plain text) - - # List one or more user identities, separated by commas. You can have only one User for all the keys or one user per key. - # The User/s can be a simple name or an email, no matter if it exists or not. - # NOTE: Make sure to add the users to the authorization above overwise they will not be allowed to interact with Weaviate. - # AUTHENTICATION_APIKEY_USERS: 'jane@doe.com,ian-smith' - -envSecrets: - # create a Kubernetes secret with AUTHENTICATION_APIKEY_ALLOWED_KEYS key and its respective value - # AUTHENTICATION_APIKEY_ALLOWED_KEYS: name-of-the-k8s-secret-containing-the-comma-separated-api-keys - -# Configure offload providers -offload: - s3: - enabled: false - # If one is using AWS EKS and has already configured K8s Service Account - # that holds the AWS credentials one can pass a name of that service account - # here using this setting. - # NOTE: the root `serviceAccountName` config has priority over this one, and - # if the root one is set this one will NOT overwrite it. This one is here for - # backwards compatibility. - serviceAccountName: - - envconfig: - # Configure bucket where data should be saved, this setting is mandatory - OFFLOAD_S3_BUCKET: weaviate-offload - - # Optional setting. Defaults to AWS S3 (s3.amazonaws.com). - # Set this option if you have a MinIO storage configured in your environment - # and want to use it instead of the AWS S3. - # OFFLOAD_S3_ENDPOINT: custom.minio.endpoint.address - - # Optional setting. Defaults to true. - # Set this option if you don't want to use SSL. - # OFFLOAD_S3_USE_SSL: true - - # Optional setting. Defaults to false. - # Set this option if you wan't Weaviate to create - # the bucket used for offloading tenants. Otherwise, - # if set to false Weaviate expects the bucket to be - # already created with the OFFLOAD_S3_BUCKET name - # OFFLOAD_S3_BUCKET_AUTO_CREATE: true - - # You can pass environment AWS settings here: - # Define the region - # AWS_REGION: eu-west-1 - - # For Weaviate to be able to create bucket objects it needs a user credentials to authenticate to AWS. - # The User must have permissions to read/create/delete bucket objects. - # You can pass the User credentials (access-key id and access-secret-key) in 2 ways: - # 1. by setting the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY plain values in the `secrets` section below - # this chart will create a kubernetes secret for you with these key-values pairs - # 2. create Kubernetes secret/s with AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY keys and their respective values - # Set the Key and the secret where it is set in `envSecrets` section below - secrets: {} - # AWS_ACCESS_KEY_ID: access-key-id (plain text) - # AWS_SECRET_ACCESS_KEY: secret-access-key (plain text) - - # If one has already defined secrets with AWS credentials one can pass them using - # this setting: - envSecrets: {} - # AWS_ACCESS_KEY_ID: name-of-the-k8s-secret-containing-the-key-id - # AWS_SECRET_ACCESS_KEY: name-of-the-k8s-secret-containing-the-key - -# Configure backup providers -backups: - # The backup-filesystem module enables creation of the DB backups in - # the local filesystem - filesystem: - enabled: false - envconfig: - # Configure folder where backups should be saved - BACKUP_FILESYSTEM_PATH: /tmp/backups - - s3: - enabled: false - # If one is using AWS EKS and has already configured K8s Service Account - # that holds the AWS credentials one can pass a name of that service account - # here using this setting. - # NOTE: the root `serviceAccountName` config has priority over this one, and - # if the root one is set this one will NOT overwrite it. This one is here for - # backwards compatibility. - serviceAccountName: - - envconfig: - # Configure bucket where backups should be saved, this setting is mandatory - BACKUP_S3_BUCKET: weaviate-backups - - # Optional setting. Defaults to empty string. - # Set this option if you want to save backups to a given location - # inside the bucket - # BACKUP_S3_PATH: path/inside/bucket - - # Optional setting. Defaults to AWS S3 (s3.amazonaws.com). - # Set this option if you have a MinIO storage configured in your environment - # and want to use it instead of the AWS S3. - # BACKUP_S3_ENDPOINT: custom.minio.endpoint.address - - # Optional setting. Defaults to true. - # Set this option if you don't want to use SSL. - # BACKUP_S3_USE_SSL: true - - # You can pass environment AWS settings here: - # Define the region - # AWS_REGION: eu-west-1 - - # For Weaviate to be able to create bucket objects it needs a user credentials to authenticate to AWS. - # The User must have permissions to read/create/delete bucket objects. - # You can pass the User credentials (access-key id and access-secret-key) in 2 ways: - # 1. by setting the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY plain values in the `secrets` section below - # this chart will create a kubernetes secret for you with these key-values pairs - # 2. create Kubernetes secret/s with AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY keys and their respective values - # Set the Key and the secret where it is set in `envSecrets` section below - secrets: {} - # AWS_ACCESS_KEY_ID: access-key-id (plain text) - # AWS_SECRET_ACCESS_KEY: secret-access-key (plain text) - - # If one has already defined secrets with AWS credentials one can pass them using - # this setting: - envSecrets: {} - # AWS_ACCESS_KEY_ID: name-of-the-k8s-secret-containing-the-key-id - # AWS_SECRET_ACCESS_KEY: name-of-the-k8s-secret-containing-the-key - - gcs: - enabled: false - envconfig: - # Configure bucket where backups should be saved, this setting is mandatory - BACKUP_GCS_BUCKET: weaviate-backups - - # Optional setting. Defaults to empty string. - # Set this option if you want to save backups to a given location - # inside the bucket - # BACKUP_GCS_PATH: path/inside/bucket - - # You can pass environment Google settings here: - # Define the project - # GOOGLE_CLOUD_PROJECT: project-id - - # For Weaviate to be able to create bucket objects it needs a ServiceAccount credentials to authenticate to GCP. - # The ServiceAccount must have permissions to read/create/delete bucket objects. - # You can pass the ServiceAccount credentials (as JSON) in 2 ways: - # 1. by setting the GOOGLE_APPLICATION_CREDENTIALS json as plain text in the `secrets` section below - # this chart will create a kubernetes secret for you with this key-values pairs - # 2. create a Kubernetes secret with GOOGLE_APPLICATION_CREDENTIALS key and its respective value - # Set the Key and the secret where it is set in `envSecrets` section below - secrets: {} - # GOOGLE_APPLICATION_CREDENTIALS: credentials-json-string (plain text) - - # If one has already defined a secret with GOOGLE_APPLICATION_CREDENTIALS one can pass them using - # this setting: - envSecrets: {} - # GOOGLE_APPLICATION_CREDENTIALS: name-of-the-k8s-secret-containing-the-key - - azure: - enabled: false - envconfig: - # Configure container where backups should be saved, this setting is mandatory - BACKUP_AZURE_CONTAINER: weaviate-backups - - # Optional setting. Defaults to empty string. - # Set this option if you want to save backups to a given location - # inside the container - # BACKUP_AZURE_PATH: path/inside/container - - # For Weaviate to be able to create container objects it needs a user credentials to authenticate to Azure Storage. - # The User must have permissions to read/create/delete container objects. - # You can pass the User credentials (account-name id and account-key or connection-string) in 2 ways: - # 1. by setting the AZURE_STORAGE_ACCOUNT and AZURE_STORAGE_KEY - # or AZURE_STORAGE_CONNECTION_STRING plain values in the `secrets` section below - # this chart will create a kubernetes secret for you with these key-values pairs - # 2. create Kubernetes secret/s with AZURE_STORAGE_ACCOUNT and AZURE_STORAGE_KEY - # or AZURE_STORAGE_CONNECTION_STRING and their respective values - # Set the Key and the secret where it is set in `envSecrets` section below - secrets: {} - # AZURE_STORAGE_ACCOUNT: account-name (plain text) - # AZURE_STORAGE_KEY: account-key (plain text) - # AZURE_STORAGE_CONNECTION_STRING: connection-string (plain text) - - # If one has already defined secrets with Azure Storage credentials one can pass them using - # this setting: - envSecrets: {} - # AZURE_STORAGE_ACCOUNT: name-of-the-k8s-secret-containing-the-account-name - # AZURE_STORAGE_KEY: name-of-the-k8s-secret-containing-account-key - # AZURE_STORAGE_CONNECTION_STRING: name-of-the-k8s-secret-containing-connection-string - - -# modules are extensions to Weaviate, they can be used to support various -# ML-models, but also other features unrelated to model inference. -# An inference/vectorizer module is not required, you can also run without any -# modules and import your own vectors. -modules: - - # The text2vec-contextionary module uses a fastText-based vector-space to - # derive vector embeddings for your objects. It is very efficient on CPUs, - # but in some situations it cannot reach the same level of accuracy as - # transformers-based models. - text2vec-contextionary: - - # Enable deployment of this module - enabled: false - - # The configuration below is ignored if enabled==false - fullnameOverride: contextionary - tag: en0.16.0-v1.2.1 - repo: semitechnologies/contextionary - registry: cr.weaviate.io - replicas: 1 - strategy: - type: RollingUpdate - imagePullPolicy: IfNotPresent - imagePullSecrets: [] - priorityClassName: "" - livenessProbe: - initialDelaySeconds: 120 - periodSeconds: 3 - timeoutSeconds: 3 - readinessProbe: - initialDelaySeconds: 120 - periodSeconds: 3 - envconfig: - occurrence_weight_linear_factor: 0.75 - neighbor_occurrence_ignore_percentile: 5 - enable_compound_splitting: false - extensions_storage_mode: weaviate - resources: {} - # requests: - # cpu: '500m' - # memory: '500Mi' - # limits: - # cpu: '1000m' - # memory: '5000Mi' - - # security Context for the Contextionary Pods. The configurations are the same as setting them - # as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - securityContext: {} - - # It is possible to add a ServiceAccount to this module's Pods, it can be - # used in cases where the module is in a private registry and you want to - # give access to the registry only to this pod. - # NOTE: if not set the root `serviceAccountName` config will be used. - serviceAccountName: - - # You can guide where the pods are scheduled on a per-module basis, - # as well as for Weaviate overall. Each module accepts nodeSelector, - # tolerations, and affinity configuration. If it is set on a per- - # module basis, this configuration overrides the global config. - - nodeSelector: - tolerations: - affinity: - - # The text2vec-transformers modules uses neural networks, such as BERT, - # DistilBERT, etc. to dynamically compute vector embeddings based on the - # sentence's context. It is very slow on CPUs and should run with - # CUDA-enabled GPUs for optimal performance. - text2vec-transformers: - - # Enable deployment of this module - enabled: false - - # You can set directly an inference URL of this module without deploying it with this release. - # You can do so by setting a value for the `inferenceUrl` here AND by setting the `enable` to `false` - inferenceUrl: {} - - # The configuration below is ignored if enabled==false - - # replace with model of choice, see - # https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-transformers - # for all supported models or build your own container. - tag: sentence-transformers-paraphrase-multilingual-MiniLM-L12-v2 - repo: semitechnologies/transformers-inference - registry: cr.weaviate.io - replicas: 1 - strategy: - type: RollingUpdate - imagePullPolicy: IfNotPresent - imagePullSecrets: [] - priorityClassName: "" - fullnameOverride: transformers-inference - # Deprecated setting use initialDelaySeconds instead in each probe instead - # probeInitialDelaySeconds: 120 - livenessProbe: - initialDelaySeconds: 120 - periodSeconds: 3 - timeoutSeconds: 3 - readinessProbe: - initialDelaySeconds: 120 - periodSeconds: 3 - envconfig: - # enable for CUDA support. Your K8s cluster needs to be configured - # accordingly and you need to explicitly set GPU requests & limits below - enable_cuda: false - - # only used when CUDA is enabled - nvidia_visible_devices: all - nvidia_driver_capabilities: compute,utility - - # only used when CUDA is enabled - ld_library_path: /usr/local/nvidia/lib64 - - resources: {} - # requests: - # cpu: '1000m' - # memory: '3000Mi' - # enable if running with CUDA support - # nvidia.com/gpu: 1 - # limits: - # cpu: '1000m' - # memory: '5000Mi' - # enable if running with CUDA support - # nvidia.com/gpu: 1 - - # security Context for the Transformers Pods. The configurations are the same as setting them - # as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - # applies to passageQueryService below - securityContext: {} - - # It is possible to add a ServiceAccount to this module's Pods, it can be - # used in cases where the module is in a private registry and you want to - # give access to the registry only to this pod. - # NOTE: if not set the root `serviceAccountName` config will be used. - serviceAccountName: - - # You can guide where the pods are scheduled on a per-module basis, - # as well as for Weaviate overall. Each module accepts nodeSelector, - # tolerations, and affinity configuration. If it is set on a per- - # module basis, this configuration overrides the global config. - nodeSelector: - tolerations: - affinity: - - passageQueryServices: - passage: - enabled: false - # You can set directly an inference URL of this module without deploying it with this release. - # You can do so by setting a value for the `inferenceUrl` here AND by setting the `enable` to `false` - inferenceUrl: {} - - # The configuration below is ignored if enabled==false - tag: facebook-dpr-ctx_encoder-single-nq-base - repo: semitechnologies/transformers-inference - registry: cr.weaviate.io - imagePullPolicy: IfNotPresent - imagePullSecrets: [] - priorityClassName: "" - replicas: 1 - strategy: - type: RollingUpdate - fullnameOverride: transformers-inference-passage - livenessProbe: - initialDelaySeconds: 120 - periodSeconds: 3 - timeoutSeconds: 3 - readinessProbe: - initialDelaySeconds: 120 - periodSeconds: 3 - envconfig: - # enable for CUDA support. Your K8s cluster needs to be configured - # accordingly and you need to explicitly set GPU requests & limits below - enable_cuda: false - - # only used when CUDA is enabled - nvidia_visible_devices: all - nvidia_driver_capabilities: compute,utility - - # only used when CUDA is enabled - ld_library_path: /usr/local/nvidia/lib64 - - resources: {} - # requests: - # cpu: '1000m' - # memory: '3000Mi' - # enable if running with CUDA support - # nvidia.com/gpu: 1 - # limits: - # cpu: '1000m' - # memory: '5000Mi' - # enable if running with CUDA support - # nvidia.com/gpu: 1 - - # security Context for the Transformers Pods. The configurations are the same as setting them - # as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - # applies to passageQueryService below - securityContext: {} - - # It is possible to add a ServiceAccount to this module's Pods, it can be - # used in cases where the module is in a private registry and you want to - # give access to the registry only to this pod. - # NOTE: if not set the root `serviceAccountName` config will be used. - serviceAccountName: - - # You can guide where the pods are scheduled on a per-module basis, - # as well as for Weaviate overall. Each module accepts nodeSelector, - # tolerations, and affinity configuration. If it is set on a per- - # module basis, this configuration overrides the global config. - - nodeSelector: - tolerations: - affinity: - - query: - enabled: false - # You can set directly an inference URL of this module without deploying it with this release. - # You can do so by setting a value for the `inferenceUrl` here AND by setting the `enable` to `false` - inferenceUrl: {} - - # The configuration below is ignored if enabled==false - tag: facebook-dpr-question_encoder-single-nq-base - repo: semitechnologies/transformers-inference - registry: cr.weaviate.io - imagePullPolicy: IfNotPresent - imagePullSecrets: [] - priorityClassName: "" - replicas: 1 - strategy: - type: RollingUpdate - fullnameOverride: transformers-inference-query - livenessProbe: - initialDelaySeconds: 120 - periodSeconds: 3 - timeoutSeconds: 3 - readinessProbe: - initialDelaySeconds: 120 - periodSeconds: 3 - envconfig: - # enable for CUDA support. Your K8s cluster needs to be configured - # accordingly and you need to explicitly set GPU requests & limits below - enable_cuda: false - - # only used when CUDA is enabled - nvidia_visible_devices: all - nvidia_driver_capabilities: compute,utility - - # only used when CUDA is enabled - ld_library_path: /usr/local/nvidia/lib64 - - resources: {} - # requests: - # cpu: '1000m' - # memory: '3000Mi' - # enable if running with CUDA support - # nvidia.com/gpu: 1 - - # limits: - # cpu: '1000m' - # memory: '5000Mi' - # enable if running with CUDA support - # nvidia.com/gpu: 1 - - # security Context for the Transformers Pods. The configurations are the same as setting them - # as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - # applies to passageQueryService below - securityContext: {} - - # It is possible to add a ServiceAccount to this module's Pods, it can be - # used in cases where the module is in a private registry and you want to - # give access to the registry only to this pod. - # NOTE: if not set the root `serviceAccountName` config will be used. - serviceAccountName: - - # You can guide where the pods are scheduled on a per-module basis, - # as well as for Weaviate overall. Each module accepts nodeSelector, - # tolerations, and affinity configuration. If it is set on a per- - # module basis, this configuration overrides the global config. - - nodeSelector: - tolerations: - affinity: - - # The text2vec-gpt4all is a vectorizer module that allows to run ML models from - # nomic-ai/gpt4all: https://docs.gpt4all.io/gpt4all_python_embedding.html - # These models run only on CPU only and on x86_64 arch - # The ML model is containerized in a Weaviate compatible way. - # If you want to run a different model that published ones you can follow the - # tutorial from here on how to create such a container: https://github.com/weaviate/t2v-gpt4all-models - text2vec-gpt4all: - - # Enable deployment of this module - enabled: false - - # You can set directly an inference URL of this module without deploying it with this release. - # You can do so by setting a value for the `inferenceUrl` here AND by setting the `enable` to `false` - inferenceUrl: {} - - # The configuration below is ignored if enabled==false - tag: all-MiniLM-L6-v2 - repo: semitechnologies/gpt4all-inference - registry: cr.weaviate.io - replicas: 1 - strategy: - type: RollingUpdate - imagePullPolicy: IfNotPresent - imagePullSecrets: [] - priorityClassName: "" - fullnameOverride: gpt4all-inference - livenessProbe: - initialDelaySeconds: 120 - periodSeconds: 3 - timeoutSeconds: 3 - readinessProbe: - initialDelaySeconds: 120 - periodSeconds: 3 - - resources: {} - # requests: - # cpu: '1000m' - # memory: '3000Mi' - - # limits: - # cpu: '1000m' - # memory: '5000Mi' - - # security Context for the Contextionary Pods. The configurations are the same as setting them - # as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - securityContext: {} - - # It is possible to add a ServiceAccount to this module's Pods, it can be - # used in cases where the module is in a private registry and you want to - # give access to the registry only to this pod. - # NOTE: if not set the root `serviceAccountName` config will be used. - serviceAccountName: - - # You can guide where the pods are scheduled on a per-module basis, - # as well as for Weaviate overall. Each module accepts nodeSelector, - # tolerations, and affinity configuration. If it is set on a per- - # module basis, this configuration overrides the global config. - - nodeSelector: - tolerations: - affinity: - - # The text2vec-openai module uses OpenAI Embeddings API - # to dynamically compute vector embeddings based on the - # sentence's context. - # More information about OpenAI Embeddings API can be found here: - # https://beta.openai.com/docs/guides/embeddings/what-are-embeddings - text2vec-openai: - - # enable if you want to use OpenAI module - enabled: false - - # Set your OpenAI API Key to be passed to Weaviate pod as - # an environment variable. You can set either the OpenAI's Service API key - # OR the Azure OpenAI's Service Key. Note that you cannot set both of the at the same time. - apiKey: '' - azureApiKey: '' - - # The text2vec-huggingface module uses HuggingFace API - # to dynamically compute vector embeddings based on the - # sentence's context. - # More information about HuggingFace API can be found here: - # https://huggingface.co/docs/api-inference/detailed_parameters#feature-extraction-task - text2vec-huggingface: - - # enable if you want to use HuggingFace module - enabled: false - - # Set your HuggingFace API Key to be passed to Weaviate pod as - # an environment variable - apiKey: '' - - # The text2vec-cohere module uses Cohere API - # to dynamically compute vector embeddings based on the - # sentence's context. - # More information about Cohere API can be found here: https://docs.cohere.ai/ - text2vec-cohere: - - # enable if you want to use Cohere module - enabled: false - - # Set your Cohere API Key to be passed to Weaviate pod as - # an environment variable - apiKey: '' - - # The text2vec-palm module uses Google PaLM Embeddings API - # to dynamically compute vector embeddings based on the - # sentence's context. - # More information about Google PaLM Embeddings API can be found here: - # https://developers.generativeai.google/ - text2vec-palm: - - # enable if you want to use Google PaLM module - enabled: false - - # Set your Google PaLM API Key to be passed to Weaviate pod as - # an environment variable - apiKey: '' - - # The text2vec-jinaai module uses JinaAI Embeddings API - # to dynamically compute vector embeddings based on the - # sentence's context. - # More information about JinaAI Embeddings API can be found here: - # https://jina.ai/embeddings - text2vec-jinaai: - - # enable if you want to use JinaAI module - enabled: false - - # Set your JinaAI API Key to be passed to Weaviate pod as - # an environment variable - apiKey: '' - - # The ref2vec-centroid module - ref2vec-centroid: - - # enable if you want to use Centroid module - enabled: false - - # The text2vec-aws module uses AWS Bedrock service - # to dynamically compute vector embeddings based on the - # sentence's context. - # More information about AWS Bedrock service can be found here: https://docs.aws.amazon.com/bedrock/ - text2vec-aws: - - # enable if you want to use AWS module - enabled: false - - # For Weaviate to be able to use AWS Bedrock service it needs a user credentials to authenticate to AWS. - # The User must have permissions to use AWS Bedrock service AI. - # You can pass the User credentials (access-key id and access-secret-key) in 2 ways: - # 1. by setting the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY plain values in the `secrets` section below - # this chart will create a kubernetes secret for you with these key-values pairs - # 2. create Kubernetes secret/s with AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY keys and their respective values - # Set the Key and the secret where it is set in `envSecrets` section below - secrets: {} - # AWS_ACCESS_KEY_ID: access-key-id (plain text) - # AWS_SECRET_ACCESS_KEY: secret-access-key (plain text) - - # If one has already defined secrets with AWS credentials one can pass them using - # this setting: - envSecrets: {} - # AWS_ACCESS_KEY_ID: name-of-the-k8s-secret-containing-the-key-id - # AWS_SECRET_ACCESS_KEY: name-of-the-k8s-secret-containing-the-key - - # The text2vec-voyageai module uses VoaygeAI Embeddings API - # to dynamically compute vector embeddings based on the - # sentence's context. - # More information about VoaygeAI Embeddings API can be found here: - # https://www.voyageai.com/ - text2vec-voyageai: - - # enable if you want to use VoaygeAI module - enabled: false - - # Set your VoaygeAI API Key to be passed to Weaviate pod as - # an environment variable - apiKey: '' - - # The text2vec-ollama module uses Ollama Embeddings API - # to dynamically compute vector embeddings based on the - # sentence's context. - # More information about Ollama Embeddings API can be found here: - # https://github.com/ollama/ollama/blob/main/docs/api.md#generate-embeddings - # Please note that you have to deploy Ollama container by yourself - # and then properly point the Ollama endpoint using module settings. - # More about this configuration can be found here: - # https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-ollama#ollama-endpoint - text2vec-ollama: - - # enable if you want to use Ollama module - enabled: false - - # The text2vec-octoai module uses OctoAI Embeddings API - # to dynamically compute vector embeddings based on the - # sentence's context. - # More information about OctoAI Embeddings API can be found here: - # https://octo.ai/docs/text-gen-solution/getting-started - text2vec-octoai: - - # enable if you want to use OctoAI module - enabled: false - - # Set your OctoAI API Key to be passed to Weaviate pod as - # an environment variable - apiKey: '' - - # The text2vec-mistral module uses MISTRAL AI Embeddings API - # More information about MISTRAL AI's Embeddings API can be found here: - # https://docs.mistral.ai/api/#operation/createEmbedding - text2vec-mistral: - - # enable if you want to use MISTRAL AI module - enabled: false - - # Set your MISTRAL API Key to be passed to Weaviate pod as - # an environment variable - apiKey: '' - - # The text2vec-databricks module uses Databricks Embeddings API - # to dynamically compute vector embeddings based on the - # sentence's context. - # More information about Databricks Embeddings API can be found here: - # https://docs.databricks.com/en/machine-learning/foundation-models/api-reference.html#embedding-task - text2vec-databricks: - - # enable if you want to use Databricks module - enabled: false - - # Set your Databricks Token to be passed to Weaviate pod as - # an environment variable - token: '' - - # The multi2vec-clip modules uses CLIP transformers to vectorize both images - # and text in the same vector space. It is typically slow(er) on CPUs and should - # run with CUDA-enabled GPUs for optimal performance. - multi2vec-clip: - - # Enable deployment of this module - enabled: false - - # You can set directly an inference URL of this module without deploying it with this release. - # You can do so by setting a value for the `inferenceUrl` here AND by setting the `enable` to `false` - inferenceUrl: {} - - # The configuration below is ignored if enabled==false - - # replace with model of choice, see - # https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/multi2vec-clip - # for all supported models or build your own container. - tag: sentence-transformers-clip-ViT-B-32-multilingual-v1 - repo: semitechnologies/multi2vec-clip - registry: cr.weaviate.io - replicas: 1 - strategy: - type: RollingUpdate - imagePullPolicy: IfNotPresent - imagePullSecrets: [] - priorityClassName: "" - fullnameOverride: clip-inference - livenessProbe: - initialDelaySeconds: 120 - periodSeconds: 3 - timeoutSeconds: 3 - readinessProbe: - initialDelaySeconds: 120 - periodSeconds: 3 - envconfig: - # enable for CUDA support. Your K8s cluster needs to be configured - # accordingly and you need to explicitly set GPU requests & limits below - enable_cuda: false - - # only used when CUDA is enabled - nvidia_visible_devices: all - nvidia_driver_capabilities: compute,utility - - # only used when CUDA is enabled - ld_library_path: /usr/local/nvidia/lib64 - - resources: {} - # requests: - # cpu: '1000m' - # memory: '3000Mi' - # enable if running with CUDA support - # nvidia.com/gpu: 1 - # limits: - # cpu: '1000m' - # memory: '5000Mi' - # enable if running with CUDA support - # nvidia.com/gpu: 1 - - # security Context for the Contextionary Pods. The configurations are the same as setting them - # as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - securityContext: {} - - # It is possible to add a ServiceAccount to this module's Pods, it can be - # used in cases where the module is in a private registry and you want to - # give access to the registry only to this pod. - # NOTE: if not set the root `serviceAccountName` config will be used. - serviceAccountName: - - annotations: - nodeSelector: - tolerations: - - # The multi2vec-bind is a multi modal inference container it uses Meta's open - # source ImageBind implementation as base for this module: - # https://github.com/facebookresearch/ImageBind - # It is possible to create your own version of this module by following the instructions here: - # https://github.com/weaviate/multi2vec-bind-inference - multi2vec-bind: - - # Enable deployment of this module - enabled: false - - # You can set directly an inference URL of this module without deploying it with this release. - # You can do so by setting a value for the `inferenceUrl` here AND by setting the `enable` to `false` - inferenceUrl: {} - - # The configuration below is ignored if enabled==false - tag: imagebind - repo: semitechnologies/multi2vec-bind - registry: cr.weaviate.io - replicas: 1 - strategy: - type: RollingUpdate - imagePullPolicy: IfNotPresent - imagePullSecrets: [] - priorityClassName: "" - fullnameOverride: bind-inference - livenessProbe: - initialDelaySeconds: 120 - periodSeconds: 3 - timeoutSeconds: 3 - readinessProbe: - initialDelaySeconds: 120 - periodSeconds: 3 - envconfig: - # enable for CUDA support. Your K8s cluster needs to be configured - # accordingly and you need to explicitly set GPU requests & limits below - enable_cuda: false - - # only used when CUDA is enabled - nvidia_visible_devices: all - nvidia_driver_capabilities: compute,utility - - # only used when CUDA is enabled - ld_library_path: /usr/local/nvidia/lib64 - - resources: {} - # requests: - # cpu: '1000m' - # memory: '7000Mi' - # enable if running with CUDA support - # nvidia.com/gpu: 1 - # limits: - # cpu: '1000m' - # memory: '7000Mi' - # enable if running with CUDA support - # nvidia.com/gpu: 1 - - # security Context for the Contextionary Pods. The configurations are the same as setting them - # as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - securityContext: {} - - # It is possible to add a ServiceAccount to this module's Pods, it can be - # used in cases where the module is in a private registry and you want to - # give access to the registry only to this pod. - # NOTE: if not set the root `serviceAccountName` config will be used. - serviceAccountName: - - annotations: - nodeSelector: - tolerations: - - # The multi2vec-palm module uses Google PaLM Embeddings API - # to dynamically compute vector embeddings. - # More information about Google PaLM Embeddings API can be found here: - # https://cloud.google.com/vertex-ai/generative-ai/docs/embeddings/get-multimodal-embeddings - multi2vec-palm: - - # enable if you want to use Google Multimodal PaLM module - enabled: false - - # Set your Google PaLM API Key to be passed to Weaviate pod as - # an environment variable - apiKey: '' - - # The qna-transformers module uses neural networks, such as BERT, - # DistilBERT, to find an answer in text to a given question - qna-transformers: - - # Enable deployment of this module - enabled: false - - # You can set directly an inference URL of this module without deploying it with this release. - # You can do so by setting a value for the `inferenceUrl` here AND by setting the `enable` to `false` - inferenceUrl: {} - - # The configuration below is ignored if enabled==false - tag: bert-large-uncased-whole-word-masking-finetuned-squad - repo: semitechnologies/qna-transformers - registry: cr.weaviate.io - replicas: 1 - strategy: - type: RollingUpdate - imagePullPolicy: IfNotPresent - imagePullSecrets: [] - priorityClassName: "" - fullnameOverride: qna-transformers - livenessProbe: - initialDelaySeconds: 120 - periodSeconds: 3 - timeoutSeconds: 3 - readinessProbe: - initialDelaySeconds: 120 - periodSeconds: 3 - envconfig: - # enable for CUDA support. Your K8s cluster needs to be configured - # accordingly and you need to explicitly set GPU requests & limits below - enable_cuda: false - - # only used when CUDA is enabled - nvidia_visible_devices: all - nvidia_driver_capabilities: compute,utility - - # only used when CUDA is enabled - ld_library_path: /usr/local/nvidia/lib64 - - resources: {} - # requests: - # cpu: '1000m' - # memory: '3000Mi' - # enable if running with CUDA support - # nvidia.com/gpu: 1 - # limits: - # cpu: '1000m' - # memory: '5000Mi' - # enable if running with CUDA support - # nvidia.com/gpu: 1 - - # security Context for the Contextionary Pods. The configurations are the same as setting them - # as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - securityContext: {} - - # It is possible to add a ServiceAccount to this module's Pods, it can be - # used in cases where the module is in a private registry and you want to - # give access to the registry only to this pod. - # NOTE: if not set the root `serviceAccountName` config will be used. - serviceAccountName: - - # You can guide where the pods are scheduled on a per-module basis, - # as well as for Weaviate overall. Each module accepts nodeSelector, - # tolerations, and affinity configuration. If it is set on a per- - # module basis, this configuration overrides the global config. - - nodeSelector: - tolerations: - affinity: - - # The qna-openai module uses OpenAI Completions API - # to dynamically answer given questions. - # More information about OpenAI Completions API can be found here: - # https://beta.openai.com/docs/api-reference/completions - qna-openai: - - # enable if you want to use OpenAI module - enabled: false - - # Set your OpenAI API Key to be passed to Weaviate pod as - # an environment variable. You can set either the OpenAI's Service API key - # OR the Azure OpenAI's Service Key. Note that you cannot set both of the at the same time. - apiKey: '' - azureApiKey: '' - - # The generative-openai module uses OpenAI Completions API - # along with text-davinci-003 model to behave as ChatGPT. - # More information about OpenAI Completions API can be found here: - # https://beta.openai.com/docs/api-reference/completions - generative-openai: - - # enable if you want to use OpenAI module - enabled: false - - # Set your OpenAI API Key to be passed to Weaviate pod as - # an environment variable. You can set either the OpenAI's Service API key - # OR the Azure OpenAI's Service Key. Note that you cannot set both of the at the same time. - apiKey: '' - azureApiKey: '' - - # The generative-cohere module uses Cohere Generate API - # More information about Cohere's Generate API can be found here: - # https://docs.cohere.com/reference/generate - generative-cohere: - - # enable if you want to use Cohere generative module - enabled: false - - # Set your Cohere API Key to be passed to Weaviate pod as - # an environment variable - apiKey: '' - - # The generative-palm module uses Google PaLM API. - # More information about Google PaLM API can be found here: - # https://developers.generativeai.google/ - generative-palm: - - # enable if you want to use Google PaLM module - enabled: false - - # Set your Google PaLM API Key to be passed to Weaviate pod as - # an environment variable - apiKey: '' - - # The generative-aws module uses AWS Bedrock service. - # More information about AWS Bedrock service can be found here: - # https://docs.aws.amazon.com/bedrock/ - generative-aws: - - # enable if you want to use AWS module - enabled: false - - # For Weaviate to be able to use AWS Bedrock service it needs a user credentials to authenticate to AWS. - # The User must have permissions to use AWS Bedrock service AI. - # You can pass the User credentials (access-key id and access-secret-key) in 2 ways: - # 1. by setting the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY plain values in the `secrets` section below - # this chart will create a kubernetes secret for you with these key-values pairs - # 2. create Kubernetes secret/s with AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY keys and their respective values - # Set the Key and the secret where it is set in `envSecrets` section below - secrets: {} - # AWS_ACCESS_KEY_ID: access-key-id (plain text) - # AWS_SECRET_ACCESS_KEY: secret-access-key (plain text) - - # If one has already defined secrets with AWS credentials one can pass them using - # this setting: - envSecrets: {} - # AWS_ACCESS_KEY_ID: name-of-the-k8s-secret-containing-the-key-id - # AWS_SECRET_ACCESS_KEY: name-of-the-k8s-secret-containing-the-key - - # The generative-anyscale module uses Anyscale API - # More information about Anyscale API can be found here: - # https://docs.anyscale.com/endpoints/overview - generative-anyscale: - - # enable if you want to use Anyscale module - enabled: false - - # Set your Anyscale API Key to be passed to Weaviate pod as - # an environment variable. - apiKey: '' - - # The generative-mistral module uses MISTRAL AI Generative API - # More information about MISTRAL AI's Generative API can be found here: - # https://docs.mistral.ai/ - generative-mistral: - - # enable if you want to use MISTRAL AI generative module - enabled: false - - # Set your MISTRAL API Key to be passed to Weaviate pod as - # an environment variable - apiKey: '' - - # The generative-ollama module uses Ollama Generate API - # More information about Ollama's Generate API can be found here: - # https://github.com/ollama/ollama/blob/main/docs/api.md#generate-a-completion - # Please note that you have to deploy Ollama container by yourself - # and then properly point the Ollama endpoint using module settings. - # More about this configuration can be found here: - # https://weaviate.io/developers/weaviate/modules/reader-generator-modules/generative-ollama#ollama-endpoint - generative-ollama: - - # enable if you want to use Ollama module - enabled: false - - # The generative-octoai module uses OctoAI API. - # More information about OctoAI API can be found here: - # https://octo.ai/docs/text-gen-solution/getting-started - generative-octoai: - - # enable if you want to use OctoAI module - enabled: false - - # Set your OctoAI API Key to be passed to Weaviate pod as - # an environment variable - apiKey: '' - - # The generative-anthropic module uses Anthropic API. - # More information about Anthropic API can be found here: - # https://docs.anthropic.com/en/api/getting-started - generative-anthropic: - - # enable if you want to use Anthropic module - enabled: false - - # Set your Anthropic API Key to be passed to Weaviate pod as - # an environment variable - apiKey: '' - - # The generative-friendliai module uses FriendliAI API. - # More information about FriendliAI API can be found here: - # https://docs.friendli.ai/openapi/create-chat-completions - generative-friendliai: - - # enable if you want to use FriendliAI module - enabled: false - - # Set your FriendliAI Token to be passed to Weaviate pod as - # an environment variable - token: '' - - # The generative-databricks module uses Databricks API. - # More information about Databricks API can be found here: - # https://docs.databricks.com/en/machine-learning/foundation-models/api-reference.html#completion-task - generative-databricks: - - # enable if you want to use Databricks module - enabled: false - - # Set your Databricks Token to be passed to Weaviate pod as - # an environment variable - token: '' - - # The img2vec-neural module uses neural networks, to generate - # a vector representation of the image - img2vec-neural: - - # Enable deployment of this module - enabled: false - - # You can set directly an inference URL of this module without deploying it with this release. - # You can do so by setting a value for the `inferenceUrl` here AND by setting the `enable` to `false` - inferenceUrl: {} - - tag: resnet50 - repo: semitechnologies/img2vec-pytorch - registry: cr.weaviate.io - replicas: 1 - strategy: - type: RollingUpdate - imagePullPolicy: IfNotPresent - imagePullSecrets: [] - priorityClassName: "" - fullnameOverride: img2vec-neural - livenessProbe: - initialDelaySeconds: 120 - periodSeconds: 3 - timeoutSeconds: 3 - readinessProbe: - initialDelaySeconds: 120 - periodSeconds: 3 - envconfig: - # enable for CUDA support. Your K8s cluster needs to be configured - # accordingly and you need to explicitly set GPU requests & limits below - enable_cuda: false - - # only used when CUDA is enabled - nvidia_visible_devices: all - nvidia_driver_capabilities: compute,utility - - # only used when CUDA is enabled - ld_library_path: /usr/local/nvidia/lib64 - - resources: {} - # requests: - # cpu: '1000m' - # memory: '3000Mi' - # enable if running with CUDA support - # nvidia.com/gpu: 1 - # limits: - # cpu: '1000m' - # memory: '5000Mi' - # enable if running with CUDA support - # nvidia.com/gpu: 1 - - # security Context for the Contextionary Pods. The configurations are the same as setting them - # as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - securityContext: {} - - # It is possible to add a ServiceAccount to this module's Pods, it can be - # used in cases where the module is in a private registry and you want to - # give access to the registry only to this pod. - # NOTE: if not set the root `serviceAccountName` config will be used. - serviceAccountName: - - # You can guide where the pods are scheduled on a per-module basis, - # as well as for Weaviate overall. Each module accepts nodeSelector, - # tolerations, and affinity configuration. If it is set on a per- - # module basis, this configuration overrides the global config. - - nodeSelector: - tolerations: - affinity: - - # The reranker-cohere module uses Cohere API - # to dynamically compute a score for the relevance - # of the query with each of the initial search results. - # More information about Cohere API can be found here: https://docs.cohere.com/docs/rerank-guide - reranker-cohere: - - # enable if you want to use Cohere rerank module - enabled: false - - # Set your Cohere API Key to be passed to Weaviate pod as - # an environment variable - apiKey: '' - - # The reranker-voyageai module uses VoaygeAI API - # to dynamically compute a score for the relevance - # of the query with each of the initial search results. - # More information about Cohere API can be found here: https://www.voyageai.com/ - reranker-voyageai: - - # enable if you want to use VoaygeAI module - enabled: false - - # Set your VoaygeAI API Key to be passed to Weaviate pod as - # an environment variable - apiKey: '' - - # The reranker-jinaai module uses JinaAI API - # to dynamically compute a score for the relevance - # of the query with each of the initial search results. - # More information about Cohere API can be found here: https://jina.ai/reranker - reranker-jinaai: - - # enable if you want to use JinaAI module - enabled: false - - # Set your JinaAI API Key to be passed to Weaviate pod as - # an environment variable - apiKey: '' - - # The reranker-transformers module uses Cross-Encoders for - # sentence pair scoring and sentence pair classification tasks. - # More information about Cross-Encoders can be found here: - # https://www.sbert.net/examples/applications/cross-encoder/README.html - reranker-transformers: - - # Enable deployment of this module - enabled: false - - # You can set directly an inference URL of this module without deploying it with this release. - # You can do so by setting a value for the `inferenceUrl` here AND by setting the `enable` to `false` - inferenceUrl: {} - - # The configuration below is ignored if enabled==false - tag: cross-encoder-ms-marco-MiniLM-L-6-v2 - repo: semitechnologies/reranker-transformers - registry: cr.weaviate.io - replicas: 1 - strategy: - type: RollingUpdate - imagePullPolicy: IfNotPresent - imagePullSecrets: [] - priorityClassName: "" - fullnameOverride: reranker-transformers - livenessProbe: - initialDelaySeconds: 120 - periodSeconds: 3 - timeoutSeconds: 3 - readinessProbe: - initialDelaySeconds: 120 - periodSeconds: 3 - envconfig: - # enable for CUDA support. Your K8s cluster needs to be configured - # accordingly and you need to explicitly set GPU requests & limits below - enable_cuda: false - - # only used when CUDA is enabled - nvidia_visible_devices: all - nvidia_driver_capabilities: compute,utility - - # only used when CUDA is enabled - ld_library_path: /usr/local/nvidia/lib64 - - resources: {} - # requests: - # cpu: '1000m' - # memory: '3000Mi' - # enable if running with CUDA support - # nvidia.com/gpu: 1 - # limits: - # cpu: '1000m' - # memory: '5000Mi' - # enable if running with CUDA support - # nvidia.com/gpu: 1 - - # security Context for the Contextionary Pods. The configurations are the same as setting them - # as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - securityContext: {} - - # It is possible to add a ServiceAccount to this module's Pods, it can be - # used in cases where the module is in a private registry and you want to - # give access to the registry only to this pod. - # NOTE: if not set the root `serviceAccountName` config will be used. - serviceAccountName: - - # You can guide where the pods are scheduled on a per-module basis, - # as well as for Weaviate overall. Each module accepts nodeSelector, - # tolerations, and affinity configuration. If it is set on a per- - # module basis, this configuration overrides the global config. - - nodeSelector: - tolerations: - affinity: - - # The text-spellcheck module uses spellchecker library to check - # misspellings in a given text - text-spellcheck: - - # Enable deployment of this module - enabled: false - - # You can set directly an inference URL of this module without deploying it with this release. - # You can do so by setting a value for the `inferenceUrl` here AND by setting the `enable` to `false` - inferenceUrl: {} - - # The configuration below is ignored if enabled==false - tag: pyspellchecker-en - repo: semitechnologies/text-spellcheck-model - registry: cr.weaviate.io - replicas: 1 - strategy: - type: RollingUpdate - imagePullPolicy: IfNotPresent - imagePullSecrets: [] - priorityClassName: "" - fullnameOverride: text-spellcheck - livenessProbe: - initialDelaySeconds: 120 - periodSeconds: 3 - timeoutSeconds: 3 - readinessProbe: - initialDelaySeconds: 120 - periodSeconds: 3 - - resources: {} - # requests: - # cpu: '400m' - # memory: '400Mi' - # limits: - # cpu: '500m' - # memory: '500Mi' - - # security Context for the Contextionary Pods. The configurations are the same as setting them - # as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - securityContext: {} - - # It is possible to add a ServiceAccount to this module's Pods, it can be - # used in cases where the module is in a private registry and you want to - # give access to the registry only to this pod. - # NOTE: if not set the root `serviceAccountName` config will be used. - serviceAccountName: - - # You can guide where the pods are scheduled on a per-module basis, - # as well as for Weaviate overall. Each module accepts nodeSelector, - # tolerations, and affinity configuration. If it is set on a per- - # module basis, this configuration overrides the global config. - - nodeSelector: - tolerations: - affinity: - - # The ner-transformers module uses spellchecker library to check - # misspellings in a given text - ner-transformers: - - # Enable deployment of this module - enabled: false - - # You can set directly an inference URL of this module without deploying it with this release. - # You can do so by setting a value for the `inferenceUrl` here AND by setting the `enable` to `false` - inferenceUrl: {} - - # The configuration below is ignored if enabled==false - tag: dbmdz-bert-large-cased-finetuned-conll03-english - repo: semitechnologies/ner-transformers - registry: cr.weaviate.io - replicas: 1 - strategy: - type: RollingUpdate - imagePullPolicy: IfNotPresent - imagePullSecrets: [] - priorityClassName: "" - fullnameOverride: ner-transformers - livenessProbe: - initialDelaySeconds: 120 - periodSeconds: 3 - timeoutSeconds: 3 - readinessProbe: - initialDelaySeconds: 120 - periodSeconds: 3 - envconfig: - # enable for CUDA support. Your K8s cluster needs to be configured - # accordingly and you need to explicitly set GPU requests & limits below - enable_cuda: false - - # only used when CUDA is enabled - nvidia_visible_devices: all - nvidia_driver_capabilities: compute,utility - - # only used when CUDA is enabled - ld_library_path: /usr/local/nvidia/lib64 - - resources: {} - # requests: - # cpu: '1000m' - # memory: '3000Mi' - - # enable if running with CUDA support - # nvidia.com/gpu: 1 - # limits: - # cpu: '1000m' - # memory: '5000Mi' - - # enable if running with CUDA support - # nvidia.com/gpu: 1 - - # security Context for the Contextionary Pods. The configurations are the same as setting them - # as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - securityContext: {} - - # It is possible to add a ServiceAccount to this module's Pods, it can be - # used in cases where the module is in a private registry and you want to - # give access to the registry only to this pod. - # NOTE: if not set the root `serviceAccountName` config will be used. - serviceAccountName: - - # You can guide where the pods are scheduled on a per-module basis, - # as well as for Weaviate overall. Each module accepts nodeSelector, - # tolerations, and affinity configuration. If it is set on a per- - # module basis, this configuration overrides the global config. - - nodeSelector: - tolerations: - affinity: - - # The sum-transformers module makes result texts summarizations - sum-transformers: - - # Enable deployment of this module - enabled: false - - # You can set directly an inference URL of this module without deploying it with this release. - # You can do so by setting a value for the `inferenceUrl` here AND by setting the `enable` to `false` - inferenceUrl: {} - - # The configuration below is ignored if enabled==false - tag: facebook-bart-large-cnn - repo: semitechnologies/sum-transformers - registry: cr.weaviate.io - replicas: 1 - strategy: - type: RollingUpdate - imagePullPolicy: IfNotPresent - imagePullSecrets: [] - priorityClassName: "" - fullnameOverride: sum-transformers - livenessProbe: - initialDelaySeconds: 120 - periodSeconds: 3 - timeoutSeconds: 3 - readinessProbe: - initialDelaySeconds: 120 - periodSeconds: 3 - envconfig: - # enable for CUDA support. Your K8s cluster needs to be configured - # accordingly and you need to explicitly set GPU requests & limits below - enable_cuda: false - - # only used when CUDA is enabled - nvidia_visible_devices: all - nvidia_driver_capabilities: compute,utility - - # only used when CUDA is enabled - ld_library_path: /usr/local/nvidia/lib64 - - resources: {} - # requests: - # cpu: '1000m' - # memory: '3000Mi' - # enable if running with CUDA support - # nvidia.com/gpu: 1 - # limits: - # cpu: '1000m' - # memory: '5000Mi' - # enable if running with CUDA support - # nvidia.com/gpu: 1 - - # security Context for the Contextionary Pods. The configurations are the same as setting them - # as described here: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - securityContext: {} - - # It is possible to add a ServiceAccount to this module's Pods, it can be - # used in cases where the module is in a private registry and you want to - # give access to the registry only to this pod. - # NOTE: if not set the root `serviceAccountName` config will be used. - serviceAccountName: - - # You can guide where the pods are scheduled on a per-module basis, - # as well as for Weaviate overall. Each module accepts nodeSelector, - # tolerations, and affinity configuration. If it is set on a per- - # module basis, this configuration overrides the global config. - - nodeSelector: - tolerations: - affinity: - - # by choosing the default vectorizer module, you can tell Weaviate to always - # use this module as the vectorizer if nothing else is specified. Can be - # overwritten on a per-class basis. - # set to text2vec-transformers if running with transformers instead - default_vectorizer_module: none - -# It is also possible to configure authentication and authorization through a -# custom configmap The authorization and authentication values defined in -# values.yaml will be ignored when defining a custom config map. -custom_config_map: - enabled: false - name: 'custom-config' - -# Pass any annotations to Weaviate pods -annotations: - -extraVolumeMounts: - -extraVolumes: - -nodeSelector: - -tolerations: - -hostAliases: - -affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 1 - podAffinityTerm: - topologyKey: "kubernetes.io/hostname" - labelSelector: - matchExpressions: - - key: "app" - operator: In - values: - - weaviate - -## Optionally specify priorityClass name for the pod -## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#pod-priority -## -priorityClassName: "" -globalPriorityClassName: "" - From 7de7526abeed36cfb7878447ab7e695ded7595a9 Mon Sep 17 00:00:00 2001 From: mungitoperrito Date: Wed, 11 Sep 2024 18:12:08 -0400 Subject: [PATCH 05/67] sub pages --- .../weaviate/configuration/indexing/index.md | 55 ++------------ .../indexing/inverted-indexes.md | 26 +++++++ .../configuration/indexing/vector-indexes.md | 72 +++++++++++++++++++ 3 files changed, 105 insertions(+), 48 deletions(-) create mode 100644 developers/weaviate/configuration/indexing/inverted-indexes.md create mode 100644 developers/weaviate/configuration/indexing/vector-indexes.md diff --git a/developers/weaviate/configuration/indexing/index.md b/developers/weaviate/configuration/indexing/index.md index e526ca430a..46d80de1aa 100644 --- a/developers/weaviate/configuration/indexing/index.md +++ b/developers/weaviate/configuration/indexing/index.md @@ -9,55 +9,14 @@ This section provides examples of how to index Weaviate collections. Collection objects contain vectors and properties. Vector indexes let you search the collection's vectors quickly and efficiently. Inverted indexes make filtering and searching object properties faster. -## Vector indexes +- [Configure vector indexes](/developers/weaviate/configuration/indexing/vector-indexes.md) +- [Configure inverted indexes](/developers/weaviate/configuration/indexing/inverted-indexes.md) -import VectorIntro from '/_includes/indexes/vector-intro.mdx'; +## Related pages +- [Indexes overview](/developers/weaviate/starter-guides/managing-resources/indexing) - +## Questions and feedback -### HNSW indexes +import DocsFeedback from '/_includes/docs-feedback.mdx'; -import HNSWIntro from '/_includes/indexes/hnsw-intro.mdx'; - - - -See also: - - - [HNSW index parameters](/developers/weaviate/config-refs/schema/vector-index#hnsw-index-parameters) - -### Flat indexes - -import FlatIntro from '/_includes/indexes/flat-intro.mdx'; - - - -See also: - -- [Flat index parameters](/developers/weaviate/config-refs/schema/vector-index#flat-indexes) - -### Dynamic indexes - -import DynamicIntro from '/_includes/indexes/dynamic-intro.mdx'; - - - -See also: - -- [Dynamic index parameters](/developers/weaviate/config-refs/schema/vector-index#dynamic-index-parameters) - -## Inverted indexes - -- [Overview](add) -- [indexSearchable]](add) -- [indexFilterable]](add) -- [indexRangeFilters]](add) - -## Additional considerations - -- [Asynchronous indexing](add) - -import EnableAsynch from '/_includes/indexes/enable-async.mdx'; - - - -- [Availability levels](add) + \ No newline at end of file diff --git a/developers/weaviate/configuration/indexing/inverted-indexes.md b/developers/weaviate/configuration/indexing/inverted-indexes.md new file mode 100644 index 0000000000..1f31f73144 --- /dev/null +++ b/developers/weaviate/configuration/indexing/inverted-indexes.md @@ -0,0 +1,26 @@ +--- +title: Inverted indexes +sidebar_position: 20 +image: og/docs/indexes.jpg +# tags: ['configuration'] +--- + +- [Overview](add) +- [indexSearchable]](add) +- [indexFilterable]](add) +- [indexRangeFilters]](add) + +## Additional considerations + +These are some additional things to consider when you configure indexing. + +### Availability levels + +## Related pages +- [Indexes overview](/developers/weaviate/starter-guides/managing-resources/indexing) + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + \ No newline at end of file diff --git a/developers/weaviate/configuration/indexing/vector-indexes.md b/developers/weaviate/configuration/indexing/vector-indexes.md new file mode 100644 index 0000000000..c663ca934e --- /dev/null +++ b/developers/weaviate/configuration/indexing/vector-indexes.md @@ -0,0 +1,72 @@ +--- +title: Vector indexes +sidebar_position: 10 +image: og/docs/indexes.jpg +# tags: ['configuration'] +--- + +import VectorIntro from '/_includes/indexes/vector-intro.mdx'; + + + +### HNSW indexes + +import HNSWIntro from '/_includes/indexes/hnsw-intro.mdx'; + + + +See also: + + - [HNSW index parameters](/developers/weaviate/config-refs/schema/vector-index#hnsw-index-parameters) + +### Flat indexes + +import FlatIntro from '/_includes/indexes/flat-intro.mdx'; + + + +See also: + +- [Flat index parameters](/developers/weaviate/config-refs/schema/vector-index#flat-indexes) + +### Dynamic indexes + +import DynamicIntro from '/_includes/indexes/dynamic-intro.mdx'; + + + +See also: + +- [Dynamic index parameters](/developers/weaviate/config-refs/schema/vector-index#dynamic-index-parameters) + +## Additional considerations + +These are some additional things to consider when you configure indexing. + +### Compression + +Vector indexes can be large. Compressed vectors lose some information, but they use fewer resources and can be very cost effective. In most cases, + +See also: + +- [Compression overview](/developers/weaviate/starter-guides/managing-resources/compression) +- [Enable compression](/developers/weaviate/configuration/compression) + +### Asynchronous indexing + +Asynchronous indexing is a prerequisite for dynamic indexing and [AutoPQ](/developers/weaviate/configuration/compression/pq-compression#configure-autopq). + +import EnableAsynch from '/_includes/indexes/enable-async.mdx'; + + + +### Availability levels + +## Related pages +- [Indexes overview](/developers/weaviate/starter-guides/managing-resources/indexing) + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + \ No newline at end of file From 0ab0e7f25c360e3a65c83034979169366d358d23 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Thu, 12 Sep 2024 10:35:28 -0400 Subject: [PATCH 06/67] vector indexes --- _includes/configuration/compression-methods.mdx | 3 +++ _includes/indexes/dynamic-intro.mdx | 2 +- _includes/indexes/enable-async.mdx | 6 +++--- _includes/indexes/flat-intro.mdx | 2 +- _includes/indexes/vector-intro.mdx | 8 ++++---- developers/weaviate/configuration/compression/index.md | 8 ++++---- .../weaviate/configuration/indexing/_category_.json | 2 +- .../weaviate/configuration/indexing/vector-indexes.md | 10 +++++++++- developers/weaviate/configuration/tenant-offloading.md | 2 +- 9 files changed, 27 insertions(+), 16 deletions(-) create mode 100644 _includes/configuration/compression-methods.mdx diff --git a/_includes/configuration/compression-methods.mdx b/_includes/configuration/compression-methods.mdx new file mode 100644 index 0000000000..76ce28c530 --- /dev/null +++ b/_includes/configuration/compression-methods.mdx @@ -0,0 +1,3 @@ +- [Binary Quantization (BQ)](/developers/weaviate/configuration/compression/bq-compression) +- [Product Quantization (PQ)](/developers/weaviate/configuration/compression/pq-compression) +- [Scalar Quantization (SQ)](/developers/weaviate/configuration/compression/sq-compression) \ No newline at end of file diff --git a/_includes/indexes/dynamic-intro.mdx b/_includes/indexes/dynamic-intro.mdx index 6ff64b3ceb..98a656d3cb 100644 --- a/_includes/indexes/dynamic-intro.mdx +++ b/_includes/indexes/dynamic-intro.mdx @@ -1 +1 @@ -A [dynamic index](/developers/weaviate/concepts/vector-index#dynamic-index) offers a flexible approach to indexing. A dynamic index begins as a flat index, and converts automatically to an HNSW index upon reaching a threshold size. \ No newline at end of file +A [dynamic index](/developers/weaviate/concepts/vector-index#dynamic-index) offers a flexible approach to indexing. A dynamic index begins as a flat index, and converts automatically to an HNSW index when a [collection](/developers/weaviate/concepts/data#collections) or [tenant](/developers/weaviate/concepts/data#multi-tenancy) reaches a threshold size. \ No newline at end of file diff --git a/_includes/indexes/enable-async.mdx b/_includes/indexes/enable-async.mdx index 444a02b224..a462dff681 100644 --- a/_includes/indexes/enable-async.mdx +++ b/_includes/indexes/enable-async.mdx @@ -1,6 +1,6 @@ -To enable asynchronous indexing, set the `ASYNC_INDEXING` environment variable to `true` in your Weaviate configuration. +To enable asynchronous indexing, set the `ASYNC_INDEXING` parameter to `true` in your Weaviate configuration. -- [Docker Compose](developers/weaviate/installation/docker-compose): Add the variable to `docker-compose.yml`. -- [Kubernetes](/developers/weaviate/installation/kubernetes): Add the variable to `values.yaml`. +- [Docker Compose](developers/weaviate/installation/docker-compose): Add `ASYNC_INDEXING` to `docker-compose.yml`. +- [Kubernetes](/developers/weaviate/installation/kubernetes): Add `ASYNC_INDEXING` to `values.yaml`. When enabled, asynchronous indexing is available for all of the collections in your Weaviate instance. \ No newline at end of file diff --git a/_includes/indexes/flat-intro.mdx b/_includes/indexes/flat-intro.mdx index 538b194c3c..9881d90b31 100644 --- a/_includes/indexes/flat-intro.mdx +++ b/_includes/indexes/flat-intro.mdx @@ -1 +1 @@ -[Flat indexes](/developers/weaviate/concepts/vector-index#flat-index) are memory-efficient. They are disk based indexes that perform brute-force vector searches. Vector search speeds with flat indexes scale linearly with the number of objects. \ No newline at end of file +[Flat indexes](/developers/weaviate/concepts/vector-index#flat-index) are memory-efficient. They are disk based indexes that perform brute-force vector searches. Flat indexes work best with small collections, less than 10,000 objects. Vector search times increase linearly with the number of objects. \ No newline at end of file diff --git a/_includes/indexes/vector-intro.mdx b/_includes/indexes/vector-intro.mdx index c66ea4b859..f0d8afd9c2 100644 --- a/_includes/indexes/vector-intro.mdx +++ b/_includes/indexes/vector-intro.mdx @@ -1,5 +1,5 @@ -Weaviate offers three types of vector indexes, [Hierarchical Navigable Small World (HNSW)](#hnsw-indexes), [flat](#flat-indexes) and [dynamic](#dynamic-indexes). +Weaviate offers three types of vector indexes: -- HNSW indexes enable fast, scalable vector searching that works well even with very large data sets. -- Flat indexes are memory-efficient indexes that work best with small data sets. -- Dynamic indexes switch from flat indexes to HNSW indexes when a [collection](/developers/weaviate/concepts/data#collections) or [tenant](/developers/weaviate/concepts/data#multi-tenancy) reaches a threshold size. \ No newline at end of file +- [Hierarchical Navigable Small World (HNSW) indexes](#hnsw-indexes) +- [Flat indexes](#flat-indexes) +- [Dynamic indexes](#dynamic-indexes) \ No newline at end of file diff --git a/developers/weaviate/configuration/compression/index.md b/developers/weaviate/configuration/compression/index.md index c8e20a68e8..6838404a0a 100644 --- a/developers/weaviate/configuration/compression/index.md +++ b/developers/weaviate/configuration/compression/index.md @@ -5,10 +5,10 @@ image: og/docs/configuration.jpg # tags: ['configuration', 'compression', 'pq'] --- -Uncompressed vectors can be large. Compressed vectors lose some information, but they use fewer resources and can be very cost effective. +Uncompressed vectors can be large. Compressed vectors lose some information, but they use fewer resources and can be very cost effective. To balance resource costs and system performance, consider one of these options: -- [Binary Quantization (BQ)](/developers/weaviate/configuration/compression/bq-compression) -- [Product Quantization (PQ)](/developers/weaviate/configuration/compression/pq-compression) -- [Scalar Quantization (SQ)](/developers/weaviate/configuration/compression/sq-compression) +import CompressMethods from '/_includes/configuration/compression-methods.mdx'; + + \ No newline at end of file diff --git a/developers/weaviate/configuration/indexing/_category_.json b/developers/weaviate/configuration/indexing/_category_.json index 7225cca315..f762b60e46 100644 --- a/developers/weaviate/configuration/indexing/_category_.json +++ b/developers/weaviate/configuration/indexing/_category_.json @@ -1,4 +1,4 @@ { "label": "Indexing", - "position": 20 + "position": 10 } \ No newline at end of file diff --git a/developers/weaviate/configuration/indexing/vector-indexes.md b/developers/weaviate/configuration/indexing/vector-indexes.md index c663ca934e..ead46f4f49 100644 --- a/developers/weaviate/configuration/indexing/vector-indexes.md +++ b/developers/weaviate/configuration/indexing/vector-indexes.md @@ -9,6 +9,14 @@ import VectorIntro from '/_includes/indexes/vector-intro.mdx'; +HNSW indexes are the default index type. If your collection has more than 10,000 objects, you should use an HNSW index. A flat index works best for smaller collections. Collections that start small and grow beyond 10,000 objects should consider a dynamic index. + +In addition to indexes, consider compression as a way to manage resource usage. Weaviate provides these compression methods; + +import CompressMethods from '/_includes/configuration/compression-methods.mdx'; + + + ### HNSW indexes import HNSWIntro from '/_includes/indexes/hnsw-intro.mdx'; @@ -45,7 +53,7 @@ These are some additional things to consider when you configure indexing. ### Compression -Vector indexes can be large. Compressed vectors lose some information, but they use fewer resources and can be very cost effective. In most cases, +Vector indexes can be large. Compressed vectors lose some information, but they use fewer resources and can be very cost effective. In most cases the resource savings significantly outweigh the slight loss in performance. See also: diff --git a/developers/weaviate/configuration/tenant-offloading.md b/developers/weaviate/configuration/tenant-offloading.md index c79b740e6c..70ae860c63 100644 --- a/developers/weaviate/configuration/tenant-offloading.md +++ b/developers/weaviate/configuration/tenant-offloading.md @@ -1,6 +1,6 @@ --- title: Tenant Offloading -sidebar_position: 5 +sidebar_position: 15 image: og/docs/configuration.jpg --- From 333623c63aa8c1b5d253f067d9eb5393ef6fd390 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Thu, 12 Sep 2024 18:27:34 -0400 Subject: [PATCH 07/67] py code --- _includes/code/howto/indexes/indexes-v2.ts | 0 _includes/code/howto/indexes/indexes-v3.py | 118 +++++++++++++++++ _includes/code/howto/indexes/indexes-v3.ts | 0 _includes/code/howto/indexes/indexes-v4.py | 124 ++++++++++++++++++ .../configuration/indexing/vector-indexes.md | 88 ++++++++++++- 5 files changed, 329 insertions(+), 1 deletion(-) create mode 100644 _includes/code/howto/indexes/indexes-v2.ts create mode 100644 _includes/code/howto/indexes/indexes-v3.py create mode 100644 _includes/code/howto/indexes/indexes-v3.ts create mode 100644 _includes/code/howto/indexes/indexes-v4.py diff --git a/_includes/code/howto/indexes/indexes-v2.ts b/_includes/code/howto/indexes/indexes-v2.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/_includes/code/howto/indexes/indexes-v3.py b/_includes/code/howto/indexes/indexes-v3.py new file mode 100644 index 0000000000..3a7eb4bd9c --- /dev/null +++ b/_includes/code/howto/indexes/indexes-v3.py @@ -0,0 +1,118 @@ +# TODO: Configure as part of the test harness + +DEBUG = True +# DEBUG = False +class_name = "ConfigCollection" + +def print_response(title, response): + print(f"{title}: {response}") + +def print_response_iter(title, response): + for r in response: + print(f"{title}: {r}") + +######################## +### CLIENT CONNECTION ## +######################## + +import os +import weaviate + +client = weaviate.Client( + url="http://localhost:8080", + additional_headers={ + "X-Cohere-Api-Key": os.getenv("COHERE_API_KEY") + } +) + +if DEBUG : print(f"CLIENT IS READY: {client.is_ready()}") + +################################ +### ENABLE HNSW - COLLECTION ### +################################ + +# Delete data from prior runs +if client.schema.exists(class_name): + client.schema.delete_class(class_name) + +# START EnableHNSW +class_obj = { + 'class': class_name, + # Additional configuration not shown + "vectorIndexType": "hnsw", + "vectorIndexConfig": { + "distance_metric": "cosine", + "ef_construction": 256, # Dynamic list size during construction + "max_connections": 128, # Maximum number of connections per node + "pq": { "enabled": True, }, # Enable compression + "ef": -1, # Dynamic list size during search; -1 enables dynamic Ef + "dynamic_ef_factor": 15, # Multiplier for dynamic Ef + "dynamic_ef_min": 200, # Minimum threshold for dynamic Ef + "dynamic_ef_max": 1000, # Maximum threshold for dynamic Ef + } +} + +client.schema.create_class(class_obj) +# END EnableHNSW + +class_response = client.schema.get() +schema_response = client.schema.get(class_name) + +if DEBUG: print_response_iter("COLLECTIONS", class_response) +if DEBUG: print_response("SCHEMA", schema_response) + +classes = [] +for c in class_response['classes']: + classes.append(c['class']) +assert class_name in classes, "Class missing" + +correct_index = False +if ((schema_response['class'] == class_name) and (schema_response['vectorIndexType'] == "hnsw")): + correct_index = True +assert correct_index, "Wrong index type" + +# ############################## +# ### ENABLE HNSW - MULTIPLE ### +# ############################## + +# # Delete data from prior runs +# if (client.collections.exists(collection_name)): +# client.collections.delete(collection_name) + +# # START EnableMulti + +# from weaviate.classes.config import Configure, Property, DataType, Tokenization + +# client.collections.create( +# name=collection_name, +# vectorizer_config=[ +# # Define a named vector +# Configure.NamedVectors.text2vec_cohere( +# name="vectorForFieldOne", +# source_properties=["FieldOne"], +# vector_index_config=Configure.VectorIndex.hnsw( +# max_connections=128, +# ), +# ), +# # Define another named vector +# Configure.NamedVectors.text2vec_openai( +# name="vectorForFieldTwo", +# source_properties=["FieldTwo"], +# vector_index_config=Configure.VectorIndex.flat(), +# ), +# ], +# ) +# # END EnableMulti + +# if DEBUG: print("START: Enable HNSW for multiple named vectors") + +# collection = client.collections.get(collection_name) +# collections_response = client.collections.list_all() +# schema_response = collection.config.get() + +# if DEBUG: print_response_iter("COLLECTIONS", collections_response) +# if DEBUG: print_response("SCHEMA", schema_response) + +# assert collection_name in collections_response.keys(), "Collection missing" +# assert collection.config.get().vector_config['vectorForFieldOne'].vector_index_config.vector_index_type() == "hnsw", "Wrong index type" +# assert collection.config.get().vector_config['vectorForFieldTwo'].vector_index_config.vector_index_type() == "flat", "Wrong index type" diff --git a/_includes/code/howto/indexes/indexes-v3.ts b/_includes/code/howto/indexes/indexes-v3.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/_includes/code/howto/indexes/indexes-v4.py b/_includes/code/howto/indexes/indexes-v4.py new file mode 100644 index 0000000000..99e1307173 --- /dev/null +++ b/_includes/code/howto/indexes/indexes-v4.py @@ -0,0 +1,124 @@ +# TODO: Configure as part of the test harness + +# DEBUG = True +DEBUG = False +collection_name = "ConfigCollection" + +def print_response(title, response): + print(f"{title}: {response}") + +def print_response_iter(title, response): + for r in response: + print(f"{title}: {r}") + +######################## +### CLIENT CONNECTION ## +######################## + +import os +import weaviate + +cohere_api_key = os.environ["COHERE_API_KEY"] + +client = weaviate.connect_to_local( + headers={ + "X-Cohere-Api-Key": cohere_api_key + } +) + +if DEBUG : print(f"CLIENT IS READY: {client.is_ready()}") + +################################ +### ENABLE HNSW - COLLECTION ### +################################ + +# Delete data from prior runs +if (client.collections.exists(collection_name)): + client.collections.delete(collection_name) + +# START EnableHNSW +from weaviate.classes.config import Configure, VectorDistances + +client.collections.create( + name=collection_name, + vectorizer_config=Configure.Vectorizer.text2vec_cohere(), + + # This line enables the index + # vector_index_config=Configure.VectorIndex.hnsw() + + # These lines enable and configure the index + vector_index_config=Configure.VectorIndex.hnsw( + distance_metric=VectorDistances.COSINE, + ef_construction=256, # Dynamic list size during construction + max_connections=128, # Maximum number of connections per node + quantizer=Configure.VectorIndex.Quantizer.pq(), # Quantizer configuration + ef=-1, # Dynamic list size during search; -1 enables dynamic Ef + dynamic_ef_factor=15, # Multiplier for dynamic Ef + dynamic_ef_min=200, # Minimum threshold for dynamic Ef + dynamic_ef_max=1000, # Maximum threshold for dynamic Ef + ) +) + +# END EnableHNSW + +collection = client.collections.get(collection_name) +collections_response = client.collections.list_all() +schema_response = collection.config.get() + +if DEBUG: print_response_iter("COLLECTIONS", collections_response) +if DEBUG: print_response("SCHEMA", schema_response) + +assert collection_name in collections_response.keys(), "Collection missing" +assert str(schema_response.vector_index_type) == "VectorIndexType.HNSW", "Wrong index type" + +############################## +### ENABLE HNSW - MULTIPLE ### +############################## + +# Delete data from prior runs +if (client.collections.exists(collection_name)): + client.collections.delete(collection_name) + +# START EnableMulti + +from weaviate.classes.config import Configure, Property, DataType, Tokenization + +client.collections.create( + name=collection_name, + vectorizer_config=[ + # Define a named vector + Configure.NamedVectors.text2vec_cohere( + name="vectorForFieldOne", + source_properties=["FieldOne"], + vector_index_config=Configure.VectorIndex.hnsw( + max_connections=128, + ), + ), + # Define another named vector + Configure.NamedVectors.text2vec_openai( + name="vectorForFieldTwo", + source_properties=["FieldTwo"], + vector_index_config=Configure.VectorIndex.flat(), + ), + ], +) +# END EnableMulti + +if DEBUG: print("START: Enable HNSW for multiple named vectors") + +collection = client.collections.get(collection_name) +collections_response = client.collections.list_all() +schema_response = collection.config.get() + +if DEBUG: print_response_iter("COLLECTIONS", collections_response) +if DEBUG: print_response("SCHEMA", schema_response) + +assert collection_name in collections_response.keys(), "Collection missing" +assert collection.config.get().vector_config['vectorForFieldOne'].vector_index_config.vector_index_type() == "hnsw", "Wrong index type" +assert collection.config.get().vector_config['vectorForFieldTwo'].vector_index_config.vector_index_type() == "flat", "Wrong index type" + +################ +### CLEAN UP ### +################ + +client.close() \ No newline at end of file diff --git a/developers/weaviate/configuration/indexing/vector-indexes.md b/developers/weaviate/configuration/indexing/vector-indexes.md index ead46f4f49..33f897c06e 100644 --- a/developers/weaviate/configuration/indexing/vector-indexes.md +++ b/developers/weaviate/configuration/indexing/vector-indexes.md @@ -5,13 +5,21 @@ image: og/docs/indexes.jpg # tags: ['configuration'] --- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock'; +import PyCodeV4 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v4.py'; +import PyCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v3.py'; +import TSCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v3.ts'; +import TSCodeV2 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v2.ts'; + import VectorIntro from '/_includes/indexes/vector-intro.mdx'; HNSW indexes are the default index type. If your collection has more than 10,000 objects, you should use an HNSW index. A flat index works best for smaller collections. Collections that start small and grow beyond 10,000 objects should consider a dynamic index. -In addition to indexes, consider compression as a way to manage resource usage. Weaviate provides these compression methods; +When you configure your indexes, consider using compression to manage resource usage. Weaviate provides these compression methods; import CompressMethods from '/_includes/configuration/compression-methods.mdx'; @@ -23,6 +31,84 @@ import HNSWIntro from '/_includes/indexes/hnsw-intro.mdx'; +#### Collection configuration + +Configure an HNSW index for a collection this way: + + + + + + + + + + + + + + + + +#### Multiple named vectors configuration + +Configure an HNSW index for multiple named vectors this way: + + + + + + + + + + + + + + + + See also: - [HNSW index parameters](/developers/weaviate/config-refs/schema/vector-index#hnsw-index-parameters) From 5caa2d62c89f336b1e0a98ca07f6a11fe494a013 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Thu, 12 Sep 2024 18:42:21 -0400 Subject: [PATCH 08/67] python linter --- _includes/code/howto/indexes/indexes-v3.py | 97 +++++++------------ _includes/code/howto/indexes/indexes-v4.py | 71 ++++++++------ .../configuration/indexing/vector-indexes.md | 4 +- 3 files changed, 80 insertions(+), 92 deletions(-) diff --git a/_includes/code/howto/indexes/indexes-v3.py b/_includes/code/howto/indexes/indexes-v3.py index 3a7eb4bd9c..6b7f80919f 100644 --- a/_includes/code/howto/indexes/indexes-v3.py +++ b/_includes/code/howto/indexes/indexes-v3.py @@ -1,16 +1,19 @@ # TODO: Configure as part of the test harness -DEBUG = True -# DEBUG = False +# DEBUG = True +DEBUG = False class_name = "ConfigCollection" + def print_response(title, response): print(f"{title}: {response}") + def print_response_iter(title, response): for r in response: print(f"{title}: {r}") + ######################## ### CLIENT CONNECTION ## ######################## @@ -20,12 +23,11 @@ def print_response_iter(title, response): client = weaviate.Client( url="http://localhost:8080", - additional_headers={ - "X-Cohere-Api-Key": os.getenv("COHERE_API_KEY") - } + additional_headers={"X-Cohere-Api-Key": os.getenv("COHERE_API_KEY")}, ) -if DEBUG : print(f"CLIENT IS READY: {client.is_ready()}") +if DEBUG: + print(f"CLIENT IS READY: {client.is_ready()}") ################################ ### ENABLE HNSW - COLLECTION ### @@ -37,19 +39,21 @@ def print_response_iter(title, response): # START EnableHNSW class_obj = { - 'class': class_name, + "class": class_name, # Additional configuration not shown "vectorIndexType": "hnsw", "vectorIndexConfig": { "distance_metric": "cosine", - "ef_construction": 256, # Dynamic list size during construction - "max_connections": 128, # Maximum number of connections per node - "pq": { "enabled": True, }, # Enable compression - "ef": -1, # Dynamic list size during search; -1 enables dynamic Ef - "dynamic_ef_factor": 15, # Multiplier for dynamic Ef - "dynamic_ef_min": 200, # Minimum threshold for dynamic Ef - "dynamic_ef_max": 1000, # Maximum threshold for dynamic Ef - } + "ef_construction": 256, # Dynamic list size during construction + "max_connections": 128, # Maximum number of connections per node + "pq": { + "enabled": True, + }, # Enable compression + "ef": -1, # Dynamic list size during search; -1 enables dynamic Ef + "dynamic_ef_factor": 15, # Multiplier for dynamic Ef + "dynamic_ef_min": 200, # Minimum threshold for dynamic Ef + "dynamic_ef_max": 1000, # Maximum threshold for dynamic Ef + }, } client.schema.create_class(class_obj) @@ -58,61 +62,30 @@ def print_response_iter(title, response): class_response = client.schema.get() schema_response = client.schema.get(class_name) -if DEBUG: print_response_iter("COLLECTIONS", class_response) -if DEBUG: print_response("SCHEMA", schema_response) +if DEBUG: + print_response_iter("COLLECTIONS", class_response) +if DEBUG: + print_response("SCHEMA", schema_response) classes = [] -for c in class_response['classes']: - classes.append(c['class']) +for c in class_response["classes"]: + classes.append(c["class"]) assert class_name in classes, "Class missing" correct_index = False -if ((schema_response['class'] == class_name) and (schema_response['vectorIndexType'] == "hnsw")): - correct_index = True +if (schema_response["class"] == class_name) and ( + schema_response["vectorIndexType"] == "hnsw" +): + correct_index = True assert correct_index, "Wrong index type" # ############################## # ### ENABLE HNSW - MULTIPLE ### # ############################## -# # Delete data from prior runs -# if (client.collections.exists(collection_name)): -# client.collections.delete(collection_name) - -# # START EnableMulti - -# from weaviate.classes.config import Configure, Property, DataType, Tokenization - -# client.collections.create( -# name=collection_name, -# vectorizer_config=[ -# # Define a named vector -# Configure.NamedVectors.text2vec_cohere( -# name="vectorForFieldOne", -# source_properties=["FieldOne"], -# vector_index_config=Configure.VectorIndex.hnsw( -# max_connections=128, -# ), -# ), -# # Define another named vector -# Configure.NamedVectors.text2vec_openai( -# name="vectorForFieldTwo", -# source_properties=["FieldTwo"], -# vector_index_config=Configure.VectorIndex.flat(), -# ), -# ], -# ) -# # END EnableMulti - -# if DEBUG: print("START: Enable HNSW for multiple named vectors") - -# collection = client.collections.get(collection_name) -# collections_response = client.collections.list_all() -# schema_response = collection.config.get() - -# if DEBUG: print_response_iter("COLLECTIONS", collections_response) -# if DEBUG: print_response("SCHEMA", schema_response) - -# assert collection_name in collections_response.keys(), "Collection missing" -# assert collection.config.get().vector_config['vectorForFieldOne'].vector_index_config.vector_index_type() == "hnsw", "Wrong index type" -# assert collection.config.get().vector_config['vectorForFieldTwo'].vector_index_config.vector_index_type() == "flat", "Wrong index type" +# START EnableHNSW + +# The Python client v3 doesn't support multiple named vectors. To use +# multiple named vectors, upgrade to the Python client v4. +# +# END EnableMulti diff --git a/_includes/code/howto/indexes/indexes-v4.py b/_includes/code/howto/indexes/indexes-v4.py index 99e1307173..9360c17ba3 100644 --- a/_includes/code/howto/indexes/indexes-v4.py +++ b/_includes/code/howto/indexes/indexes-v4.py @@ -4,13 +4,16 @@ DEBUG = False collection_name = "ConfigCollection" + def print_response(title, response): print(f"{title}: {response}") + def print_response_iter(title, response): for r in response: print(f"{title}: {r}") + ######################## ### CLIENT CONNECTION ## ######################## @@ -20,20 +23,17 @@ def print_response_iter(title, response): cohere_api_key = os.environ["COHERE_API_KEY"] -client = weaviate.connect_to_local( - headers={ - "X-Cohere-Api-Key": cohere_api_key - } -) +client = weaviate.connect_to_local(headers={"X-Cohere-Api-Key": cohere_api_key}) -if DEBUG : print(f"CLIENT IS READY: {client.is_ready()}") +if DEBUG: + print(f"CLIENT IS READY: {client.is_ready()}") ################################ ### ENABLE HNSW - COLLECTION ### ################################ # Delete data from prior runs -if (client.collections.exists(collection_name)): +if client.collections.exists(collection_name): client.collections.delete(collection_name) # START EnableHNSW @@ -42,21 +42,19 @@ def print_response_iter(title, response): client.collections.create( name=collection_name, vectorizer_config=Configure.Vectorizer.text2vec_cohere(), - # This line enables the index # vector_index_config=Configure.VectorIndex.hnsw() - # These lines enable and configure the index vector_index_config=Configure.VectorIndex.hnsw( distance_metric=VectorDistances.COSINE, - ef_construction=256, # Dynamic list size during construction - max_connections=128, # Maximum number of connections per node - quantizer=Configure.VectorIndex.Quantizer.pq(), # Quantizer configuration - ef=-1, # Dynamic list size during search; -1 enables dynamic Ef - dynamic_ef_factor=15, # Multiplier for dynamic Ef - dynamic_ef_min=200, # Minimum threshold for dynamic Ef - dynamic_ef_max=1000, # Maximum threshold for dynamic Ef - ) + ef_construction=256, # Dynamic list size during construction + max_connections=128, # Maximum number of connections per node + quantizer=Configure.VectorIndex.Quantizer.pq(), # Quantizer configuration + ef=-1, # Dynamic list size during search; -1 enables dynamic Ef + dynamic_ef_factor=15, # Multiplier for dynamic Ef + dynamic_ef_min=200, # Minimum threshold for dynamic Ef + dynamic_ef_max=1000, # Maximum threshold for dynamic Ef + ), ) # END EnableHNSW @@ -65,18 +63,22 @@ def print_response_iter(title, response): collections_response = client.collections.list_all() schema_response = collection.config.get() -if DEBUG: print_response_iter("COLLECTIONS", collections_response) -if DEBUG: print_response("SCHEMA", schema_response) +if DEBUG: + print_response_iter("COLLECTIONS", collections_response) +if DEBUG: + print_response("SCHEMA", schema_response) assert collection_name in collections_response.keys(), "Collection missing" -assert str(schema_response.vector_index_type) == "VectorIndexType.HNSW", "Wrong index type" +assert ( + str(schema_response.vector_index_type) == "VectorIndexType.HNSW" +), "Wrong index type" ############################## ### ENABLE HNSW - MULTIPLE ### ############################## # Delete data from prior runs -if (client.collections.exists(collection_name)): +if client.collections.exists(collection_name): client.collections.delete(collection_name) # START EnableMulti @@ -92,7 +94,7 @@ def print_response_iter(title, response): source_properties=["FieldOne"], vector_index_config=Configure.VectorIndex.hnsw( max_connections=128, - ), + ), ), # Define another named vector Configure.NamedVectors.text2vec_openai( @@ -104,21 +106,34 @@ def print_response_iter(title, response): ) # END EnableMulti -if DEBUG: print("START: Enable HNSW for multiple named vectors") +if DEBUG: + print("START: Enable HNSW for multiple named vectors") collection = client.collections.get(collection_name) collections_response = client.collections.list_all() schema_response = collection.config.get() -if DEBUG: print_response_iter("COLLECTIONS", collections_response) -if DEBUG: print_response("SCHEMA", schema_response) +if DEBUG: + print_response_iter("COLLECTIONS", collections_response) +if DEBUG: + print_response("SCHEMA", schema_response) assert collection_name in collections_response.keys(), "Collection missing" -assert collection.config.get().vector_config['vectorForFieldOne'].vector_index_config.vector_index_type() == "hnsw", "Wrong index type" -assert collection.config.get().vector_config['vectorForFieldTwo'].vector_index_config.vector_index_type() == "flat", "Wrong index type" +assert ( + collection.config.get() + .vector_config["vectorForFieldOne"] + .vector_index_config.vector_index_type() + == "hnsw" +), "Wrong index type" +assert ( + collection.config.get() + .vector_config["vectorForFieldTwo"] + .vector_index_config.vector_index_type() + == "flat" +), "Wrong index type" ################ ### CLEAN UP ### ################ -client.close() \ No newline at end of file +client.close() diff --git a/developers/weaviate/configuration/indexing/vector-indexes.md b/developers/weaviate/configuration/indexing/vector-indexes.md index 33f897c06e..da02e76568 100644 --- a/developers/weaviate/configuration/indexing/vector-indexes.md +++ b/developers/weaviate/configuration/indexing/vector-indexes.md @@ -33,7 +33,7 @@ import HNSWIntro from '/_includes/indexes/hnsw-intro.mdx'; #### Collection configuration -Configure an HNSW index for a collection this way: +Configure an HNSW index for a collection: @@ -72,7 +72,7 @@ Configure an HNSW index for a collection this way: #### Multiple named vectors configuration -Configure an HNSW index for multiple named vectors this way: +Configure an HNSW index for multiple named vectors: From 74b4b4f03c13675040694fe4b956d7cd446ce0be Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Fri, 13 Sep 2024 11:19:30 -0400 Subject: [PATCH 09/67] ts code --- _includes/code/howto/indexes/indexes-v3.js | 72 +++++++++++++++++++ _includes/code/howto/indexes/indexes-v3.ts | 82 ++++++++++++++++++++++ _includes/code/howto/indexes/indexes-v4.py | 21 ------ 3 files changed, 154 insertions(+), 21 deletions(-) create mode 100644 _includes/code/howto/indexes/indexes-v3.js diff --git a/_includes/code/howto/indexes/indexes-v3.js b/_includes/code/howto/indexes/indexes-v3.js new file mode 100644 index 0000000000..07b52b29a0 --- /dev/null +++ b/_includes/code/howto/indexes/indexes-v3.js @@ -0,0 +1,72 @@ +"use strict"; +// TODO: Configure as part of the test harness +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype); + return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (g && (g = 0, op[0] && (_ = 0)), _) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +var DEBUG = true; +// var DEBUG = false; +var collection_name = "ConfigCollection"; +///////////// +// IMPORTS // +///////////// +var weaviate_client_1 = require("weaviate-client"); +////////// +// MAIN // +///////// +function main() { + return __awaiter(this, void 0, void 0, function () { + return __generator(this, function (_a) { + client = getClient(); + if (DEBUG) + console.log(client); + return [2 /*return*/]; + }); + }); +} +/////////////////////// +// CLIENT CONNECTION // +/////////////////////// +function getClient() { + return __awaiter(this, void 0, void 0, function () { + var client; + return __generator(this, function (_a) { + client = weaviate_client_1.default.connectToLocal(); + return [2 /*return*/, client]; + }); + }); +} +main(); diff --git a/_includes/code/howto/indexes/indexes-v3.ts b/_includes/code/howto/indexes/indexes-v3.ts index e69de29bb2..4895cc71df 100644 --- a/_includes/code/howto/indexes/indexes-v3.ts +++ b/_includes/code/howto/indexes/indexes-v3.ts @@ -0,0 +1,82 @@ +// TODO: Configure as part of the test harness +// TODO: Needs tests + +// Enable debug statements +var DEBUG = true; +// var DEBUG = false; + +// Imports +import weaviate from 'weaviate-client'; +import { vectorizer, dataType, configure } from 'weaviate-client'; + +// Delete pre-existing collections +function deletePrior(client, collectionName){ + if (client.collections.get(collectionName)) + client.collections.delete(collectionName); + + if(DEBUG) console.log("Deleted: " + collectionName); +} + +// Create client connection +async function getClient(){ + const client: WeaviateClient = weaviate.connectToLocal(); + + return client; +} + +////////////////////////////// +// ENABLE HNSW - COLLECTION // +////////////////////////////// + +// START EnableHNSW +// Add this import line +// import { vectorizer, dataType, configure } from 'weaviate-client'; + +async function createHNSWCollection(client, collectionName){ + if(DEBUG) console.log("Create HNSW: " + collectionName); + + await client.collections.create({ + name: collectionName, + vectorizers: vectorizer.text2VecOpenAI({ + vectorIndexConfig: configure.vectorIndex.hnsw(), + }), + }) +} +// END EnableHNSW + +////////////////////////////// +/// ENABLE HNSW - MULTIPLE /// +////////////////////////////// + +// START EnableMulti +async function createMultiCollection(client, collectionName){ + if(DEBUG) console.log("Create Multi: " + collectionName); + + const c = client; + const cn = collectionName; + + console.log("Multi no-op: " + cn); +} +// END EnableMulti + +///////////////////////////// +/// AVOID TOP LEVEL AWAIT /// +///////////////////////////// + +// Main +async function main(){ + const collectionName = "ConfigCollection"; + + const client = await getClient(); + if(DEBUG) console.log(client); + + // Run enable HNSW collection code + deletePrior(client, collectionName); + createHNSWCollection(client, collectionName); + + // Run multiple named vector collection code + deletePrior(client, collectionName); + createMultiCollection(client, collectionName); +} + +main() diff --git a/_includes/code/howto/indexes/indexes-v4.py b/_includes/code/howto/indexes/indexes-v4.py index 9360c17ba3..5149200a33 100644 --- a/_includes/code/howto/indexes/indexes-v4.py +++ b/_includes/code/howto/indexes/indexes-v4.py @@ -1,19 +1,14 @@ # TODO: Configure as part of the test harness -# DEBUG = True -DEBUG = False collection_name = "ConfigCollection" - def print_response(title, response): print(f"{title}: {response}") - def print_response_iter(title, response): for r in response: print(f"{title}: {r}") - ######################## ### CLIENT CONNECTION ## ######################## @@ -25,9 +20,6 @@ def print_response_iter(title, response): client = weaviate.connect_to_local(headers={"X-Cohere-Api-Key": cohere_api_key}) -if DEBUG: - print(f"CLIENT IS READY: {client.is_ready()}") - ################################ ### ENABLE HNSW - COLLECTION ### ################################ @@ -63,11 +55,6 @@ def print_response_iter(title, response): collections_response = client.collections.list_all() schema_response = collection.config.get() -if DEBUG: - print_response_iter("COLLECTIONS", collections_response) -if DEBUG: - print_response("SCHEMA", schema_response) - assert collection_name in collections_response.keys(), "Collection missing" assert ( str(schema_response.vector_index_type) == "VectorIndexType.HNSW" @@ -106,18 +93,10 @@ def print_response_iter(title, response): ) # END EnableMulti -if DEBUG: - print("START: Enable HNSW for multiple named vectors") - collection = client.collections.get(collection_name) collections_response = client.collections.list_all() schema_response = collection.config.get() -if DEBUG: - print_response_iter("COLLECTIONS", collections_response) -if DEBUG: - print_response("SCHEMA", schema_response) - assert collection_name in collections_response.keys(), "Collection missing" assert ( collection.config.get() From 110ecaa8b817e99badb67c7d7b2a6ef035afbcf6 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Fri, 13 Sep 2024 11:27:49 -0400 Subject: [PATCH 10/67] ts code --- _includes/code/howto/indexes/indexes-v3.ts | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/_includes/code/howto/indexes/indexes-v3.ts b/_includes/code/howto/indexes/indexes-v3.ts index 4895cc71df..22d411ca28 100644 --- a/_includes/code/howto/indexes/indexes-v3.ts +++ b/_includes/code/howto/indexes/indexes-v3.ts @@ -6,11 +6,11 @@ var DEBUG = true; // var DEBUG = false; // Imports -import weaviate from 'weaviate-client'; +import weaviate, { WeaviateClient } from 'weaviate-client'; import { vectorizer, dataType, configure } from 'weaviate-client'; // Delete pre-existing collections -function deletePrior(client, collectionName){ +function deletePrior(client: WeaviateClient, collectionName: string){ if (client.collections.get(collectionName)) client.collections.delete(collectionName); @@ -19,9 +19,9 @@ function deletePrior(client, collectionName){ // Create client connection async function getClient(){ - const client: WeaviateClient = weaviate.connectToLocal(); + const client: WeaviateClient = weaviate.connectToLocal(); - return client; + return client; } ////////////////////////////// @@ -32,7 +32,7 @@ async function getClient(){ // Add this import line // import { vectorizer, dataType, configure } from 'weaviate-client'; -async function createHNSWCollection(client, collectionName){ +async function createHNSWCollection(client: WeaviateClient, collectionName: string){ if(DEBUG) console.log("Create HNSW: " + collectionName); await client.collections.create({ @@ -49,13 +49,10 @@ async function createHNSWCollection(client, collectionName){ ////////////////////////////// // START EnableMulti -async function createMultiCollection(client, collectionName){ +async function createMultiCollection(client: WeaviateClient, collectionName: string){ if(DEBUG) console.log("Create Multi: " + collectionName); - const c = client; - const cn = collectionName; - - console.log("Multi no-op: " + cn); + console.log("Multi no-op: " + collectionName); } // END EnableMulti From 059ad738945399921d8acf2fd083201a9657d1a4 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Fri, 13 Sep 2024 14:43:41 -0400 Subject: [PATCH 11/67] ts v3 --- .../code/howto/indexes/indexes-v3 copy.ts | 100 ++++++++++++++++++ _includes/code/howto/indexes/indexes-v3.ts | 71 ++++++++----- _includes/code/howto/indexes/indexes-v4.py | 1 + 3 files changed, 147 insertions(+), 25 deletions(-) create mode 100644 _includes/code/howto/indexes/indexes-v3 copy.ts diff --git a/_includes/code/howto/indexes/indexes-v3 copy.ts b/_includes/code/howto/indexes/indexes-v3 copy.ts new file mode 100644 index 0000000000..21f5d25c08 --- /dev/null +++ b/_includes/code/howto/indexes/indexes-v3 copy.ts @@ -0,0 +1,100 @@ +// TODO: Configure as part of the test harness +// TODO: Needs tests + +// Imports +import weaviate, { WeaviateClient,vectorizer, configure } from 'weaviate-client'; + +// Delete pre-existing collections +function deleteCollection(client: WeaviateClient, collectionName: string){ + console.log("DEBUG 1") + + try { + client.collections.delete(collectionName) + console.log("DEBUG 2") + } catch (e) { + // ignore error if class doesn't exist + } + + console.log("DEBUG 3") +} + +// Create client connection +async function getClient(){ + const client: WeaviateClient = weaviate.connectToLocal(); + + return client; +} + +////////////////////////////// +// ENABLE HNSW - COLLECTION // +////////////////////////////// + +// START EnableHNSW +// Add this import line +// import { vectorizer, dataType, configure } from 'weaviate-client'; + +async function createHNSWCollection(client: WeaviateClient, collectionName: string){ + await client.collections.create({ + name: collectionName, + vectorizers: vectorizer.text2VecOpenAI({ + vectorIndexConfig: configure.vectorIndex.hnsw(), + }), + }) +} +// END EnableHNSW + +////////////////////////////// +/// ENABLE HNSW - MULTIPLE /// +////////////////////////////// + +async function createMultiCollection(client: WeaviateClient, collectionName: string){ +// START EnableMulti + + // Add this import line + // import { vectorizer, configure } from 'weaviate-client'; + + await client.collections.create({ + name: collectionName, + vectorizers: [ + // Define a named vector + vectorizer.text2VecOpenAI({ + name: "vectorForFieldOne", + sourceProperties: ['FieldOne'], + vectorIndexConfig: configure.vectorIndex.hnsw() + }), + // Define another named vector + vectorizer.text2VecOpenAI({ + name: "vectorForFieldTwo", + sourceProperties: ['FieldTwo'], + vectorIndexConfig: configure.vectorIndex.flat() + }) + ], + // Configure properties + }) +// END EnableMulti +} + +///////////////////////////// +/// AVOID TOP LEVEL AWAIT /// +///////////////////////////// + +// Main +async function main(){ + const collectionName = "ConfigCollection"; + + const client = await getClient(); + + // Only run one of these at a time. Async code causes race condition crashes + // // Run enable HNSW collection code + // deleteCollection(client, collectionName); + // createHNSWCollection(client, collectionName); + + // Run multiple named vector collection code + console.log("DEBUG 0") + deleteCollection(client, collectionName); + console.log("DEBUG 4") + createMultiCollection(client, collectionName); + console.log("DEBUG 5") +} + +main() diff --git a/_includes/code/howto/indexes/indexes-v3.ts b/_includes/code/howto/indexes/indexes-v3.ts index 22d411ca28..08bb2d040a 100644 --- a/_includes/code/howto/indexes/indexes-v3.ts +++ b/_includes/code/howto/indexes/indexes-v3.ts @@ -1,20 +1,18 @@ // TODO: Configure as part of the test harness // TODO: Needs tests -// Enable debug statements -var DEBUG = true; -// var DEBUG = false; - // Imports -import weaviate, { WeaviateClient } from 'weaviate-client'; -import { vectorizer, dataType, configure } from 'weaviate-client'; +import weaviate, { WeaviateClient,vectorizer, configure } from 'weaviate-client'; // Delete pre-existing collections -function deletePrior(client: WeaviateClient, collectionName: string){ - if (client.collections.get(collectionName)) - client.collections.delete(collectionName); +function deleteCollection(client: WeaviateClient, collectionName: string){ + try { + client.collections.delete(collectionName) +} catch (e) { + // ignore error if class doesn't exist +} - if(DEBUG) console.log("Deleted: " + collectionName); +return true } // Create client connection @@ -33,8 +31,6 @@ async function getClient(){ // import { vectorizer, dataType, configure } from 'weaviate-client'; async function createHNSWCollection(client: WeaviateClient, collectionName: string){ - if(DEBUG) console.log("Create HNSW: " + collectionName); - await client.collections.create({ name: collectionName, vectorizers: vectorizer.text2VecOpenAI({ @@ -48,13 +44,32 @@ async function createHNSWCollection(client: WeaviateClient, collectionName: stri /// ENABLE HNSW - MULTIPLE /// ////////////////////////////// -// START EnableMulti async function createMultiCollection(client: WeaviateClient, collectionName: string){ - if(DEBUG) console.log("Create Multi: " + collectionName); +// START EnableMulti - console.log("Multi no-op: " + collectionName); -} + // Add this import line + // import { vectorizer, configure } from 'weaviate-client'; + + await client.collections.create({ + name: collectionName, + vectorizers: [ + // Define a named vector + vectorizer.text2VecOpenAI({ + name: "vectorForFieldOne", + sourceProperties: ['FieldOne'], + vectorIndexConfig: configure.vectorIndex.hnsw() + }), + // Define another named vector + vectorizer.text2VecOpenAI({ + name: "vectorForFieldTwo", + sourceProperties: ['FieldTwo'], + vectorIndexConfig: configure.vectorIndex.flat() + }) + ], + // Configure properties + }) // END EnableMulti +} ///////////////////////////// /// AVOID TOP LEVEL AWAIT /// @@ -62,18 +77,24 @@ async function createMultiCollection(client: WeaviateClient, collectionName: str // Main async function main(){ - const collectionName = "ConfigCollection"; + const collectionName = "ConfigCollection"; + + const client = await getClient(); - const client = await getClient(); - if(DEBUG) console.log(client); + deleteCollection(client, collectionName) - // Run enable HNSW collection code - deletePrior(client, collectionName); - createHNSWCollection(client, collectionName); + // Only one create can run at a time due to aynsc code + // Run enable HNSW collection code + deleteCollection(client, collectionName) + if(await client.collections.get(collectionName).exists() != true){ + createHNSWCollection(client, collectionName); + } - // Run multiple named vector collection code - deletePrior(client, collectionName); - createMultiCollection(client, collectionName); + // // Run multiple named vector collection code + // deleteCollection(client, collectionName) + // if(await client.collections.get(collectionName).exists() != true){ + // createMultiCollection(client, collectionName); + // } } main() diff --git a/_includes/code/howto/indexes/indexes-v4.py b/_includes/code/howto/indexes/indexes-v4.py index 5149200a33..1a6639aa60 100644 --- a/_includes/code/howto/indexes/indexes-v4.py +++ b/_includes/code/howto/indexes/indexes-v4.py @@ -90,6 +90,7 @@ def print_response_iter(title, response): vector_index_config=Configure.VectorIndex.flat(), ), ], + # Configure properties ) # END EnableMulti From fe0b9c2822421e06866faf71e70ebd4a50f54ec1 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Fri, 13 Sep 2024 15:33:06 -0400 Subject: [PATCH 12/67] ts2 --- _includes/code/howto/indexes/indexes-v2.ts | 72 +++++++++++++ .../code/howto/indexes/indexes-v3 copy.ts | 100 ------------------ indexes-v2.ts | 72 +++++++++++++ 3 files changed, 144 insertions(+), 100 deletions(-) delete mode 100644 _includes/code/howto/indexes/indexes-v3 copy.ts create mode 100644 indexes-v2.ts diff --git a/_includes/code/howto/indexes/indexes-v2.ts b/_includes/code/howto/indexes/indexes-v2.ts index e69de29bb2..92457f6d82 100644 --- a/_includes/code/howto/indexes/indexes-v2.ts +++ b/_includes/code/howto/indexes/indexes-v2.ts @@ -0,0 +1,72 @@ +// TODO: Configure as part of the test harness +// TODO: Needs tests + + +var DEBUG = true; + +// Imports +import weaviate, { WeaviateClient } from 'weaviate-ts-client'; + +// Create client connection +function getClient(){ + const client: WeaviateClient = weaviate.client({ + scheme: 'http', + host: 'localhost:8080', + }); + + return client; +} + +// Delete pre-existing collections +function deleteClass(client, className: string){ + try { + client.schema.classDeleter().withClassName(className).do(); + } catch (e) { + // ignore error if class doesn't exist + } + return true +} + +////////////////////////////// +// ENABLE HNSW - COLLECTION // +////////////////////////////// + +async function createHNSWCollection(client, className: string){ + if(DEBUG) console.log("START HNSW: " + await client.schema.exists(className)) +// START EnableHNSW +// Add this import line +// import { vectorizer, dataType, configure } from 'weaviate-client'; + + if(DEBUG) console.log("END HNSW: " + await client.schema.exists(className)) +} + +///////////////////////////// +/// AVOID TOP LEVEL AWAIT /// +///////////////////////////// + +// Main +async function main(){ + const className = "ConfigCollection"; + + const client = await getClient(); + if(DEBUG) console.log(client) ; + + if(DEBUG) console.log("START: " + await client.schema.exists(className)) + deleteClass(client, className) + if(DEBUG) console.log("START +1: " + await client.schema.exists(className)) + + // Only one create can run at a time due to aynsc code + // Run enable HNSW collection code + deleteClass(client, className) + if(await client.schema.exists(className) != true){ + createHNSWCollection(client, className); + } + + // // Run multiple named vector collection code + // deleteClass(client, className) + // if(await client.schema.exists(className) != true){ + // createMultiCollection(client, className); + // } +} + +main() \ No newline at end of file diff --git a/_includes/code/howto/indexes/indexes-v3 copy.ts b/_includes/code/howto/indexes/indexes-v3 copy.ts deleted file mode 100644 index 21f5d25c08..0000000000 --- a/_includes/code/howto/indexes/indexes-v3 copy.ts +++ /dev/null @@ -1,100 +0,0 @@ -// TODO: Configure as part of the test harness -// TODO: Needs tests - -// Imports -import weaviate, { WeaviateClient,vectorizer, configure } from 'weaviate-client'; - -// Delete pre-existing collections -function deleteCollection(client: WeaviateClient, collectionName: string){ - console.log("DEBUG 1") - - try { - client.collections.delete(collectionName) - console.log("DEBUG 2") - } catch (e) { - // ignore error if class doesn't exist - } - - console.log("DEBUG 3") -} - -// Create client connection -async function getClient(){ - const client: WeaviateClient = weaviate.connectToLocal(); - - return client; -} - -////////////////////////////// -// ENABLE HNSW - COLLECTION // -////////////////////////////// - -// START EnableHNSW -// Add this import line -// import { vectorizer, dataType, configure } from 'weaviate-client'; - -async function createHNSWCollection(client: WeaviateClient, collectionName: string){ - await client.collections.create({ - name: collectionName, - vectorizers: vectorizer.text2VecOpenAI({ - vectorIndexConfig: configure.vectorIndex.hnsw(), - }), - }) -} -// END EnableHNSW - -////////////////////////////// -/// ENABLE HNSW - MULTIPLE /// -////////////////////////////// - -async function createMultiCollection(client: WeaviateClient, collectionName: string){ -// START EnableMulti - - // Add this import line - // import { vectorizer, configure } from 'weaviate-client'; - - await client.collections.create({ - name: collectionName, - vectorizers: [ - // Define a named vector - vectorizer.text2VecOpenAI({ - name: "vectorForFieldOne", - sourceProperties: ['FieldOne'], - vectorIndexConfig: configure.vectorIndex.hnsw() - }), - // Define another named vector - vectorizer.text2VecOpenAI({ - name: "vectorForFieldTwo", - sourceProperties: ['FieldTwo'], - vectorIndexConfig: configure.vectorIndex.flat() - }) - ], - // Configure properties - }) -// END EnableMulti -} - -///////////////////////////// -/// AVOID TOP LEVEL AWAIT /// -///////////////////////////// - -// Main -async function main(){ - const collectionName = "ConfigCollection"; - - const client = await getClient(); - - // Only run one of these at a time. Async code causes race condition crashes - // // Run enable HNSW collection code - // deleteCollection(client, collectionName); - // createHNSWCollection(client, collectionName); - - // Run multiple named vector collection code - console.log("DEBUG 0") - deleteCollection(client, collectionName); - console.log("DEBUG 4") - createMultiCollection(client, collectionName); - console.log("DEBUG 5") -} - -main() diff --git a/indexes-v2.ts b/indexes-v2.ts new file mode 100644 index 0000000000..92457f6d82 --- /dev/null +++ b/indexes-v2.ts @@ -0,0 +1,72 @@ +// TODO: Configure as part of the test harness +// TODO: Needs tests + + +var DEBUG = true; + +// Imports +import weaviate, { WeaviateClient } from 'weaviate-ts-client'; + +// Create client connection +function getClient(){ + const client: WeaviateClient = weaviate.client({ + scheme: 'http', + host: 'localhost:8080', + }); + + return client; +} + +// Delete pre-existing collections +function deleteClass(client, className: string){ + try { + client.schema.classDeleter().withClassName(className).do(); + } catch (e) { + // ignore error if class doesn't exist + } + return true +} + +////////////////////////////// +// ENABLE HNSW - COLLECTION // +////////////////////////////// + +async function createHNSWCollection(client, className: string){ + if(DEBUG) console.log("START HNSW: " + await client.schema.exists(className)) +// START EnableHNSW +// Add this import line +// import { vectorizer, dataType, configure } from 'weaviate-client'; + + if(DEBUG) console.log("END HNSW: " + await client.schema.exists(className)) +} + +///////////////////////////// +/// AVOID TOP LEVEL AWAIT /// +///////////////////////////// + +// Main +async function main(){ + const className = "ConfigCollection"; + + const client = await getClient(); + if(DEBUG) console.log(client) ; + + if(DEBUG) console.log("START: " + await client.schema.exists(className)) + deleteClass(client, className) + if(DEBUG) console.log("START +1: " + await client.schema.exists(className)) + + // Only one create can run at a time due to aynsc code + // Run enable HNSW collection code + deleteClass(client, className) + if(await client.schema.exists(className) != true){ + createHNSWCollection(client, className); + } + + // // Run multiple named vector collection code + // deleteClass(client, className) + // if(await client.schema.exists(className) != true){ + // createMultiCollection(client, className); + // } +} + +main() \ No newline at end of file From 448bfe05032245abe5d902d81c50782b335f50c0 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Fri, 13 Sep 2024 18:48:59 -0400 Subject: [PATCH 13/67] ts2 --- _includes/code/howto/indexes/indexes-v2.ts | 90 +++++++++++++++++----- _includes/code/howto/indexes/indexes-v3.js | 72 ----------------- _includes/code/howto/indexes/indexes-v3.ts | 7 +- 3 files changed, 74 insertions(+), 95 deletions(-) delete mode 100644 _includes/code/howto/indexes/indexes-v3.js diff --git a/_includes/code/howto/indexes/indexes-v2.ts b/_includes/code/howto/indexes/indexes-v2.ts index 92457f6d82..fb54dee68a 100644 --- a/_includes/code/howto/indexes/indexes-v2.ts +++ b/_includes/code/howto/indexes/indexes-v2.ts @@ -1,9 +1,6 @@ // TODO: Configure as part of the test harness // TODO: Needs tests - -var DEBUG = true; - // Imports import weaviate, { WeaviateClient } from 'weaviate-ts-client'; @@ -31,14 +28,72 @@ function deleteClass(client, className: string){ // ENABLE HNSW - COLLECTION // ////////////////////////////// -async function createHNSWCollection(client, className: string){ - if(DEBUG) console.log("START HNSW: " + await client.schema.exists(className)) // START EnableHNSW -// Add this import line -// import { vectorizer, dataType, configure } from 'weaviate-client'; +async function createHNSWCollection(client: WeaviateClient, className: string){ + + const setIndexType = { + class: className, + // Add property definitions + vectorizer: 'text2vec-openai', + vectorIndexType: 'hnsw', + vectorIndexConfig: { + distance: 'cosine', + ef_construction: '256', // Dynamic list size during construction + max_connections: '128', // Maximum number of connections per node + quantizer: 'Configure.VectorIndex.Quantizer.pq()', // Quantizer configuration + ef: '-1', // Dynamic list size during search; -1 enables dynamic Ef + dynamic_ef_factor: '15', // Multiplier for dynamic Ef + dynamic_ef_min: '200', // Minimum threshold for dynamic Ef + dynamic_ef_max: '1000', // Maximum threshold for dynamic Ef + pq: { + enabled: true, + trainingLimit: 100000, + segments: 96, + }, + }, + }; + + // Add the class to the schema + await client.schema.classCreator().withClass(setIndexType).do(); +} +// END EnableHNSW - if(DEBUG) console.log("END HNSW: " + await client.schema.exists(className)) +////////////////////////////// +/// ENABLE HNSW - MULTIPLE /// +////////////////////////////// + +// START EnableMulti +async function createMultiCollection(client: WeaviateClient, className: string){ + + const classWithNamedVectors = { + class: className, + vectorConfig: { + // Define a named vector + vectorForFieldOne: { + vectorizer: { + 'text2vec-cohere': { + properties: ['FieldOne'], + }, + }, + vectorIndexType: 'hnsw', + }, + // Define another named vector + vectorForFieldTwo: { + vectorizer: { + 'text2vec-openai': { + properties: ['FieldTwo'], + }, + }, + vectorIndexType: 'flat' + }, + } + // Configure properties + } + + // Add the class to the schema + await client.schema.classCreator().withClass(classWithNamedVectors).do(); } +// END EnableMulti ///////////////////////////// /// AVOID TOP LEVEL AWAIT /// @@ -49,24 +104,21 @@ async function main(){ const className = "ConfigCollection"; const client = await getClient(); - if(DEBUG) console.log(client) ; - - if(DEBUG) console.log("START: " + await client.schema.exists(className)) deleteClass(client, className) - if(DEBUG) console.log("START +1: " + await client.schema.exists(className)) // Only one create can run at a time due to aynsc code // Run enable HNSW collection code + // deleteClass(client, className) + // if(await client.schema.exists(className) != true){ + // createHNSWCollection(client, className); + // } + + // // Run multiple named vector collection code deleteClass(client, className) if(await client.schema.exists(className) != true){ - createHNSWCollection(client, className); - } + createMultiCollection(client, className); + } - // // Run multiple named vector collection code - // deleteClass(client, className) - // if(await client.schema.exists(className) != true){ - // createMultiCollection(client, className); - // } } main() \ No newline at end of file diff --git a/_includes/code/howto/indexes/indexes-v3.js b/_includes/code/howto/indexes/indexes-v3.js deleted file mode 100644 index 07b52b29a0..0000000000 --- a/_includes/code/howto/indexes/indexes-v3.js +++ /dev/null @@ -1,72 +0,0 @@ -"use strict"; -// TODO: Configure as part of the test harness -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; -var __generator = (this && this.__generator) || function (thisArg, body) { - var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype); - return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; - function verb(n) { return function (v) { return step([n, v]); }; } - function step(op) { - if (f) throw new TypeError("Generator is already executing."); - while (g && (g = 0, op[0] && (_ = 0)), _) try { - if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; - if (y = 0, t) op = [op[0] & 2, t.value]; - switch (op[0]) { - case 0: case 1: t = op; break; - case 4: _.label++; return { value: op[1], done: false }; - case 5: _.label++; y = op[1]; op = [0]; continue; - case 7: op = _.ops.pop(); _.trys.pop(); continue; - default: - if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } - if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } - if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } - if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } - if (t[2]) _.ops.pop(); - _.trys.pop(); continue; - } - op = body.call(thisArg, _); - } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } - if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; - } -}; -Object.defineProperty(exports, "__esModule", { value: true }); -var DEBUG = true; -// var DEBUG = false; -var collection_name = "ConfigCollection"; -///////////// -// IMPORTS // -///////////// -var weaviate_client_1 = require("weaviate-client"); -////////// -// MAIN // -///////// -function main() { - return __awaiter(this, void 0, void 0, function () { - return __generator(this, function (_a) { - client = getClient(); - if (DEBUG) - console.log(client); - return [2 /*return*/]; - }); - }); -} -/////////////////////// -// CLIENT CONNECTION // -/////////////////////// -function getClient() { - return __awaiter(this, void 0, void 0, function () { - var client; - return __generator(this, function (_a) { - client = weaviate_client_1.default.connectToLocal(); - return [2 /*return*/, client]; - }); - }); -} -main(); diff --git a/_includes/code/howto/indexes/indexes-v3.ts b/_includes/code/howto/indexes/indexes-v3.ts index 08bb2d040a..dcfb4092ec 100644 --- a/_includes/code/howto/indexes/indexes-v3.ts +++ b/_includes/code/howto/indexes/indexes-v3.ts @@ -44,9 +44,8 @@ async function createHNSWCollection(client: WeaviateClient, collectionName: stri /// ENABLE HNSW - MULTIPLE /// ////////////////////////////// -async function createMultiCollection(client: WeaviateClient, collectionName: string){ // START EnableMulti - +async function createMultiCollection(client: WeaviateClient, collectionName: string){ // Add this import line // import { vectorizer, configure } from 'weaviate-client'; @@ -54,7 +53,7 @@ async function createMultiCollection(client: WeaviateClient, collectionName: str name: collectionName, vectorizers: [ // Define a named vector - vectorizer.text2VecOpenAI({ + vectorizer.text2VecCohere({ name: "vectorForFieldOne", sourceProperties: ['FieldOne'], vectorIndexConfig: configure.vectorIndex.hnsw() @@ -68,8 +67,8 @@ async function createMultiCollection(client: WeaviateClient, collectionName: str ], // Configure properties }) -// END EnableMulti } +// END EnableMulti ///////////////////////////// /// AVOID TOP LEVEL AWAIT /// From 3b9cfff37bbb1a3bfe36d4dbf1847c00bba05f33 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Fri, 13 Sep 2024 19:01:10 -0400 Subject: [PATCH 14/67] staging --- _includes/code/howto/indexes/indexes-v3.py | 7 +-- _includes/code/howto/indexes/indexes-v3.ts | 6 +- .../configuration/indexing/vector-indexes.md | 56 +++++++++---------- 3 files changed, 34 insertions(+), 35 deletions(-) diff --git a/_includes/code/howto/indexes/indexes-v3.py b/_includes/code/howto/indexes/indexes-v3.py index 6b7f80919f..e2951b88fb 100644 --- a/_includes/code/howto/indexes/indexes-v3.py +++ b/_includes/code/howto/indexes/indexes-v3.py @@ -83,9 +83,8 @@ def print_response_iter(title, response): # ### ENABLE HNSW - MULTIPLE ### # ############################## -# START EnableHNSW +# START EnableMulti +# The Python client v3 doesn't support multiple named vectors. -# The Python client v3 doesn't support multiple named vectors. To use -# multiple named vectors, upgrade to the Python client v4. -# +# To use multiple named vectors, upgrade to the Python client v4. # END EnableMulti diff --git a/_includes/code/howto/indexes/indexes-v3.ts b/_includes/code/howto/indexes/indexes-v3.ts index dcfb4092ec..53af652a1f 100644 --- a/_includes/code/howto/indexes/indexes-v3.ts +++ b/_includes/code/howto/indexes/indexes-v3.ts @@ -45,10 +45,10 @@ async function createHNSWCollection(client: WeaviateClient, collectionName: stri ////////////////////////////// // START EnableMulti -async function createMultiCollection(client: WeaviateClient, collectionName: string){ - // Add this import line - // import { vectorizer, configure } from 'weaviate-client'; +// Add this import line +// import { vectorizer, configure } from 'weaviate-client'; +async function createMultiCollection(client: WeaviateClient, collectionName: string){ await client.collections.create({ name: collectionName, vectorizers: [ diff --git a/developers/weaviate/configuration/indexing/vector-indexes.md b/developers/weaviate/configuration/indexing/vector-indexes.md index da02e76568..40e006414d 100644 --- a/developers/weaviate/configuration/indexing/vector-indexes.md +++ b/developers/weaviate/configuration/indexing/vector-indexes.md @@ -25,13 +25,13 @@ import CompressMethods from '/_includes/configuration/compression-methods.mdx'; -### HNSW indexes +## HNSW indexes import HNSWIntro from '/_includes/indexes/hnsw-intro.mdx'; -#### Collection configuration +### Collection configuration Configure an HNSW index for a collection: @@ -70,9 +70,33 @@ Configure an HNSW index for a collection: -#### Multiple named vectors configuration +See also: + + - [HNSW index parameters](/developers/weaviate/config-refs/schema/vector-index#hnsw-index-parameters) + +## Flat indexes + +import FlatIntro from '/_includes/indexes/flat-intro.mdx'; + + + +See also: + +- [Flat index parameters](/developers/weaviate/config-refs/schema/vector-index#flat-indexes) + +## Dynamic indexes + +import DynamicIntro from '/_includes/indexes/dynamic-intro.mdx'; + + + +See also: + +- [Dynamic index parameters](/developers/weaviate/config-refs/schema/vector-index#dynamic-index-parameters) + +## Multiple named vectors -Configure an HNSW index for multiple named vectors: +Configure a collection to use [multiple named vectors](/developers/weaviate/config-refs/schema/multi-vector): @@ -109,30 +133,6 @@ Configure an HNSW index for multiple named vectors: -See also: - - - [HNSW index parameters](/developers/weaviate/config-refs/schema/vector-index#hnsw-index-parameters) - -### Flat indexes - -import FlatIntro from '/_includes/indexes/flat-intro.mdx'; - - - -See also: - -- [Flat index parameters](/developers/weaviate/config-refs/schema/vector-index#flat-indexes) - -### Dynamic indexes - -import DynamicIntro from '/_includes/indexes/dynamic-intro.mdx'; - - - -See also: - -- [Dynamic index parameters](/developers/weaviate/config-refs/schema/vector-index#dynamic-index-parameters) - ## Additional considerations These are some additional things to consider when you configure indexing. From d2bef9c5109c1036086c77fc34c8aa1697819f0b Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Fri, 13 Sep 2024 19:48:35 -0400 Subject: [PATCH 15/67] py4 --- _includes/code/howto/indexes/indexes-v4.py | 26 +++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/_includes/code/howto/indexes/indexes-v4.py b/_includes/code/howto/indexes/indexes-v4.py index 1a6639aa60..4cb2baa590 100644 --- a/_includes/code/howto/indexes/indexes-v4.py +++ b/_includes/code/howto/indexes/indexes-v4.py @@ -48,7 +48,6 @@ def print_response_iter(title, response): dynamic_ef_max=1000, # Maximum threshold for dynamic Ef ), ) - # END EnableHNSW collection = client.collections.get(collection_name) @@ -112,6 +111,31 @@ def print_response_iter(title, response): == "flat" ), "Wrong index type" +################### +### ENABLE FLAT ### +################### + +# Delete data from prior runs +if client.collections.exists(collection_name): + client.collections.delete(collection_name) + +# START EnableHNSW +from weaviate.classes.config import Configure, VectorDistances + +client.collections.create( + name=collection_name, + vectorizer_config=Configure.Vectorizer.text2vec_cohere(), + # This line enables the index + # vector_index_config=Configure.VectorIndex.hnsw() + # These lines enable and configure the index + vector_index_config=Configure.VectorIndex.flat( + distance_metric=VectorDistances.COSINE, + vector_cache_max_objects=100000, + quantizer=Configure.VectorIndex.Quantizer.bq() + ), +) +# END EnableHNSW + ################ ### CLEAN UP ### ################ From b631228cd88f23ab161a0f7f9d6ea6e8b66406d2 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Fri, 13 Sep 2024 19:57:39 -0400 Subject: [PATCH 16/67] staging and py3 --- _includes/code/howto/indexes/indexes-v3.py | 58 ++++++++++++++-------- 1 file changed, 38 insertions(+), 20 deletions(-) diff --git a/_includes/code/howto/indexes/indexes-v3.py b/_includes/code/howto/indexes/indexes-v3.py index e2951b88fb..4fecb415cc 100644 --- a/_includes/code/howto/indexes/indexes-v3.py +++ b/_includes/code/howto/indexes/indexes-v3.py @@ -1,19 +1,7 @@ # TODO: Configure as part of the test harness -# DEBUG = True -DEBUG = False class_name = "ConfigCollection" - -def print_response(title, response): - print(f"{title}: {response}") - - -def print_response_iter(title, response): - for r in response: - print(f"{title}: {r}") - - ######################## ### CLIENT CONNECTION ## ######################## @@ -26,9 +14,6 @@ def print_response_iter(title, response): additional_headers={"X-Cohere-Api-Key": os.getenv("COHERE_API_KEY")}, ) -if DEBUG: - print(f"CLIENT IS READY: {client.is_ready()}") - ################################ ### ENABLE HNSW - COLLECTION ### ################################ @@ -62,11 +47,6 @@ def print_response_iter(title, response): class_response = client.schema.get() schema_response = client.schema.get(class_name) -if DEBUG: - print_response_iter("COLLECTIONS", class_response) -if DEBUG: - print_response("SCHEMA", schema_response) - classes = [] for c in class_response["classes"]: classes.append(c["class"]) @@ -88,3 +68,41 @@ def print_response_iter(title, response): # To use multiple named vectors, upgrade to the Python client v4. # END EnableMulti + +# ################### +# ### ENABLE FLAT ### +# ################### + +# Delete data from prior runs +if client.schema.exists(class_name): + client.schema.delete_class(class_name) + +# START EnableFlat +class_obj = { + "class": class_name, + # Additional configuration not shown + "vectorIndexType": "flat", + "vectorIndexConfig": { + "distance_metric": "cosine", + "vector_cache_max_objects": 100000, + "bq": {"enabled": True} + }, +} + +client.schema.create_class(class_obj) +# END EnableFlat + +class_response = client.schema.get() +schema_response = client.schema.get(class_name) + +classes = [] +for c in class_response["classes"]: + classes.append(c["class"]) +assert class_name in classes, "Class missing" + +correct_index = False +if (schema_response["class"] == class_name) and ( + schema_response["vectorIndexType"] == "flat" +): + correct_index = True +assert correct_index, "Wrong index type" From f0f62347ed576c53ea78e8ca5dda135869fa7415 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Fri, 13 Sep 2024 20:19:26 -0400 Subject: [PATCH 17/67] ts3 --- _includes/code/howto/indexes/indexes-v2.ts | 33 ++++++++++++++- _includes/code/howto/indexes/indexes-v3.ts | 42 ++++++++++++++++--- .../configuration/indexing/vector-indexes.md | 35 ++++++++++++++++ 3 files changed, 103 insertions(+), 7 deletions(-) diff --git a/_includes/code/howto/indexes/indexes-v2.ts b/_includes/code/howto/indexes/indexes-v2.ts index fb54dee68a..effdf318cb 100644 --- a/_includes/code/howto/indexes/indexes-v2.ts +++ b/_includes/code/howto/indexes/indexes-v2.ts @@ -95,6 +95,30 @@ async function createMultiCollection(client: WeaviateClient, className: string){ } // END EnableMulti +///////////////// +// ENABLE FLAT // +///////////////// + +// START EnableFlat +async function createFlatCollection(client: WeaviateClient, className: string){ + + const setIndexType = { + class: className, + // Add property definitions + vectorizer: 'text2vec-openai', + vectorIndexType: 'flat', + vectorIndexConfig: { + distance: 'cosine', + vector_cache_max_objects: 100000, + bq: { enabled: true, }, + }, + }; + + // Add the class to the schema + await client.schema.classCreator().withClass(setIndexType).do(); +} +// END EnableFlat + ///////////////////////////// /// AVOID TOP LEVEL AWAIT /// ///////////////////////////// @@ -107,6 +131,7 @@ async function main(){ deleteClass(client, className) // Only one create can run at a time due to aynsc code + // Run enable HNSW collection code // deleteClass(client, className) // if(await client.schema.exists(className) != true){ @@ -114,9 +139,15 @@ async function main(){ // } // // Run multiple named vector collection code + // deleteClass(client, className) + // if(await client.schema.exists(className) != true){ + // createMultiCollection(client, className); + // } + + // Run enable flat collection code deleteClass(client, className) if(await client.schema.exists(className) != true){ - createMultiCollection(client, className); + createFlatCollection(client, className); } } diff --git a/_includes/code/howto/indexes/indexes-v3.ts b/_includes/code/howto/indexes/indexes-v3.ts index 53af652a1f..86ec393a73 100644 --- a/_includes/code/howto/indexes/indexes-v3.ts +++ b/_includes/code/howto/indexes/indexes-v3.ts @@ -70,6 +70,29 @@ async function createMultiCollection(client: WeaviateClient, collectionName: str } // END EnableMulti +///////////////// +// ENABLE FLAT // +///////////////// + +// START EnableFlat +// Add this import line +// import { vectorizer, dataType, configure } from 'weaviate-client'; + +async function createFlatCollection(client: WeaviateClient, collectionName: string){ + await client.collections.create({ + name: collectionName, + vectorizers: vectorizer.text2VecOpenAI({ + vectorIndexConfig: configure.vectorIndex.flat({ + quantizer: configure.vectorIndex.quantizer.bq({ + rescoreLimit: 200, + cache: true + }), + }), + }) + }) +} +// END EnableFlat + ///////////////////////////// /// AVOID TOP LEVEL AWAIT /// ///////////////////////////// @@ -83,17 +106,24 @@ async function main(){ deleteCollection(client, collectionName) // Only one create can run at a time due to aynsc code - // Run enable HNSW collection code - deleteCollection(client, collectionName) - if(await client.collections.get(collectionName).exists() != true){ - createHNSWCollection(client, collectionName); - } + + // // Run enable HNSW collection code + // deleteCollection(client, collectionName) + // if(await client.collections.get(collectionName).exists() != true){ + // createHNSWCollection(client, collectionName); + // } // // Run multiple named vector collection code // deleteCollection(client, collectionName) // if(await client.collections.get(collectionName).exists() != true){ // createMultiCollection(client, collectionName); // } -} + // Run enable Flat collection code + deleteCollection(client, collectionName) + if(await client.collections.get(collectionName).exists() != true){ + createFlatCollection(client, collectionName); + } +} +createFlatCollection main() diff --git a/developers/weaviate/configuration/indexing/vector-indexes.md b/developers/weaviate/configuration/indexing/vector-indexes.md index 40e006414d..ee1babe9de 100644 --- a/developers/weaviate/configuration/indexing/vector-indexes.md +++ b/developers/weaviate/configuration/indexing/vector-indexes.md @@ -80,6 +80,41 @@ import FlatIntro from '/_includes/indexes/flat-intro.mdx'; + + + + + + + + + + + + + + + See also: - [Flat index parameters](/developers/weaviate/config-refs/schema/vector-index#flat-indexes) From 70ad7defc7c06ca5e7efddf4bbd25c4b6c78c5be Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Fri, 13 Sep 2024 20:19:39 -0400 Subject: [PATCH 18/67] ts3 --- _includes/code/howto/indexes/indexes-v3.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_includes/code/howto/indexes/indexes-v3.ts b/_includes/code/howto/indexes/indexes-v3.ts index 86ec393a73..0cbbe9ca5f 100644 --- a/_includes/code/howto/indexes/indexes-v3.ts +++ b/_includes/code/howto/indexes/indexes-v3.ts @@ -125,5 +125,5 @@ async function main(){ createFlatCollection(client, collectionName); } } -createFlatCollection + main() From 4db98ffdb6dcb93c2602224f0f7c31a96d0406b6 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Fri, 13 Sep 2024 20:44:01 -0400 Subject: [PATCH 19/67] dynamic --- _includes/code/howto/indexes/indexes-v3.ts | 23 +++--- _includes/code/howto/indexes/indexes-v4.py | 4 +- .../configuration/indexing/vector-indexes.md | 35 +++++++++ indexes-v2.ts | 72 ------------------- 4 files changed, 49 insertions(+), 85 deletions(-) delete mode 100644 indexes-v2.ts diff --git a/_includes/code/howto/indexes/indexes-v3.ts b/_includes/code/howto/indexes/indexes-v3.ts index 0cbbe9ca5f..859de14873 100644 --- a/_includes/code/howto/indexes/indexes-v3.ts +++ b/_includes/code/howto/indexes/indexes-v3.ts @@ -103,27 +103,28 @@ async function main(){ const client = await getClient(); + // Clean up from earlier runs deleteCollection(client, collectionName) // Only one create can run at a time due to aynsc code - // // Run enable HNSW collection code - // deleteCollection(client, collectionName) - // if(await client.collections.get(collectionName).exists() != true){ - // createHNSWCollection(client, collectionName); - // } + // Run enable HNSW collection code + deleteCollection(client, collectionName) + if(await client.collections.get(collectionName).exists() != true){ + createHNSWCollection(client, collectionName); + } // // Run multiple named vector collection code // deleteCollection(client, collectionName) // if(await client.collections.get(collectionName).exists() != true){ // createMultiCollection(client, collectionName); - // } + // } - // Run enable Flat collection code - deleteCollection(client, collectionName) - if(await client.collections.get(collectionName).exists() != true){ - createFlatCollection(client, collectionName); - } + // // Run enable Flat collection code + // deleteCollection(client, collectionName) + // if(await client.collections.get(collectionName).exists() != true){ + // createFlatCollection(client, collectionName); + // } } main() diff --git a/_includes/code/howto/indexes/indexes-v4.py b/_includes/code/howto/indexes/indexes-v4.py index 4cb2baa590..77a6509baa 100644 --- a/_includes/code/howto/indexes/indexes-v4.py +++ b/_includes/code/howto/indexes/indexes-v4.py @@ -119,7 +119,7 @@ def print_response_iter(title, response): if client.collections.exists(collection_name): client.collections.delete(collection_name) -# START EnableHNSW +# START EnableFlat from weaviate.classes.config import Configure, VectorDistances client.collections.create( @@ -134,7 +134,7 @@ def print_response_iter(title, response): quantizer=Configure.VectorIndex.Quantizer.bq() ), ) -# END EnableHNSW +# END EnableFlat ################ ### CLEAN UP ### diff --git a/developers/weaviate/configuration/indexing/vector-indexes.md b/developers/weaviate/configuration/indexing/vector-indexes.md index ee1babe9de..7c4abe5622 100644 --- a/developers/weaviate/configuration/indexing/vector-indexes.md +++ b/developers/weaviate/configuration/indexing/vector-indexes.md @@ -125,6 +125,41 @@ import DynamicIntro from '/_includes/indexes/dynamic-intro.mdx'; + + + + + + + + + + + + + + + See also: - [Dynamic index parameters](/developers/weaviate/config-refs/schema/vector-index#dynamic-index-parameters) diff --git a/indexes-v2.ts b/indexes-v2.ts deleted file mode 100644 index 92457f6d82..0000000000 --- a/indexes-v2.ts +++ /dev/null @@ -1,72 +0,0 @@ -// TODO: Configure as part of the test harness -// TODO: Needs tests - - -var DEBUG = true; - -// Imports -import weaviate, { WeaviateClient } from 'weaviate-ts-client'; - -// Create client connection -function getClient(){ - const client: WeaviateClient = weaviate.client({ - scheme: 'http', - host: 'localhost:8080', - }); - - return client; -} - -// Delete pre-existing collections -function deleteClass(client, className: string){ - try { - client.schema.classDeleter().withClassName(className).do(); - } catch (e) { - // ignore error if class doesn't exist - } - return true -} - -////////////////////////////// -// ENABLE HNSW - COLLECTION // -////////////////////////////// - -async function createHNSWCollection(client, className: string){ - if(DEBUG) console.log("START HNSW: " + await client.schema.exists(className)) -// START EnableHNSW -// Add this import line -// import { vectorizer, dataType, configure } from 'weaviate-client'; - - if(DEBUG) console.log("END HNSW: " + await client.schema.exists(className)) -} - -///////////////////////////// -/// AVOID TOP LEVEL AWAIT /// -///////////////////////////// - -// Main -async function main(){ - const className = "ConfigCollection"; - - const client = await getClient(); - if(DEBUG) console.log(client) ; - - if(DEBUG) console.log("START: " + await client.schema.exists(className)) - deleteClass(client, className) - if(DEBUG) console.log("START +1: " + await client.schema.exists(className)) - - // Only one create can run at a time due to aynsc code - // Run enable HNSW collection code - deleteClass(client, className) - if(await client.schema.exists(className) != true){ - createHNSWCollection(client, className); - } - - // // Run multiple named vector collection code - // deleteClass(client, className) - // if(await client.schema.exists(className) != true){ - // createMultiCollection(client, className); - // } -} - -main() \ No newline at end of file From 64fcbf3ce5d5024412dab18dd11c6c35c48c1e55 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Fri, 13 Sep 2024 21:11:45 -0400 Subject: [PATCH 20/67] py4 --- _includes/code/howto/indexes/indexes-v4.py | 52 +++++++++++++++++-- .../configuration/indexing/vector-indexes.md | 2 + 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/_includes/code/howto/indexes/indexes-v4.py b/_includes/code/howto/indexes/indexes-v4.py index 77a6509baa..b6b8efa390 100644 --- a/_includes/code/howto/indexes/indexes-v4.py +++ b/_includes/code/howto/indexes/indexes-v4.py @@ -125,9 +125,6 @@ def print_response_iter(title, response): client.collections.create( name=collection_name, vectorizer_config=Configure.Vectorizer.text2vec_cohere(), - # This line enables the index - # vector_index_config=Configure.VectorIndex.hnsw() - # These lines enable and configure the index vector_index_config=Configure.VectorIndex.flat( distance_metric=VectorDistances.COSINE, vector_cache_max_objects=100000, @@ -136,6 +133,55 @@ def print_response_iter(title, response): ) # END EnableFlat +collection = client.collections.get(collection_name) +collections_response = client.collections.list_all() +schema_response = collection.config.get() + +assert collection_name in collections_response.keys(), "Collection missing" +assert ( + str(schema_response.vector_index_type) == "VectorIndexType.FLAT" +), "Wrong index type" + +###################### +### ENABLE DYNAMIC ### +###################### + +# Delete data from prior runs +if client.collections.exists(collection_name): + client.collections.delete(collection_name) + +# START EnableDynamic +from weaviate.classes.config import Configure, VectorDistances + +client.collections.create( + name=collection_name, + vector_index_config=Configure.VectorIndex.dynamic( + distance_metric=VectorDistances.COSINE, + threshold=20000, + hnsw=Configure.VectorIndex.hnsw( + # Any hnsw configuration parameters + dynamic_ef_factor=15, # Multiplier for dynamic Ef + dynamic_ef_min=200, # Minimum threshold for dynamic Ef + dynamic_ef_max=1000, # Maximum threshold for dynamic Ef + ), + flat=Configure.VectorIndex.flat( + # Any flat index configuration parameters + vector_cache_max_objects=100000, + quantizer=Configure.VectorIndex.Quantizer.bq() + ), + ) +) +# END EnableDynamic + +collection = client.collections.get(collection_name) +collections_response = client.collections.list_all() +schema_response = collection.config.get() + +assert collection_name in collections_response.keys(), "Collection missing" +assert ( + str(schema_response.vector_index_type) == "VectorIndexType.DYNAMIC" +), "Wrong index type" + ################ ### CLEAN UP ### ################ diff --git a/developers/weaviate/configuration/indexing/vector-indexes.md b/developers/weaviate/configuration/indexing/vector-indexes.md index 7c4abe5622..46c1dc6eee 100644 --- a/developers/weaviate/configuration/indexing/vector-indexes.md +++ b/developers/weaviate/configuration/indexing/vector-indexes.md @@ -125,6 +125,8 @@ import DynamicIntro from '/_includes/indexes/dynamic-intro.mdx'; +Dynamic indexes require [asynchronous indexing](/developers/weaviate/config-refs/schema/vector-index#asynchronous-indexing). Enable asynchronous indexing before you configure a collection to use dynamic indexing. + Date: Fri, 13 Sep 2024 21:14:53 -0400 Subject: [PATCH 21/67] dynamic py3 --- _includes/code/howto/indexes/indexes-v3.py | 71 ++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/_includes/code/howto/indexes/indexes-v3.py b/_includes/code/howto/indexes/indexes-v3.py index 4fecb415cc..ea785c02ce 100644 --- a/_includes/code/howto/indexes/indexes-v3.py +++ b/_includes/code/howto/indexes/indexes-v3.py @@ -106,3 +106,74 @@ ): correct_index = True assert correct_index, "Wrong index type" + +################### +### ENABLE FLAT ### +################### + +# Delete data from prior runs +if client.schema.exists(class_name): + client.schema.delete_class(class_name) + +# START EnableFlat +class_obj = { + "class": class_name, + # Additional configuration not shown + "vectorIndexType": "flat", + "vectorIndexConfig": { + "distance_metric": "cosine", + "vector_cache_max_objects": 100000, + "bq": {"enabled": True} + }, +} + +client.schema.create_class(class_obj) +# END EnableFlat + +class_response = client.schema.get() +schema_response = client.schema.get(class_name) + +classes = [] +for c in class_response["classes"]: + classes.append(c["class"]) +assert class_name in classes, "Class missing" + +correct_index = False +if (schema_response["class"] == class_name) and ( + schema_response["vectorIndexType"] == "flat" +): + correct_index = True +assert correct_index, "Wrong index type" + +###################### +### ENABLE DYNAMIC ### +###################### + +# Delete data from prior runs +if client.schema.exists(class_name): + client.schema.delete_class(class_name) + +# START EnableDynamic +class_obj = { + "class": class_name, + # Additional configuration not shown + "vectorIndexType": "dynamic", +} + +client.schema.create_class(class_obj) +# END EnableDynamic + +class_response = client.schema.get() +schema_response = client.schema.get(class_name) + +classes = [] +for c in class_response["classes"]: + classes.append(c["class"]) +assert class_name in classes, "Class missing" + +correct_index = False +if (schema_response["class"] == class_name) and ( + schema_response["vectorIndexType"] == "dynamic" +): + correct_index = True +assert correct_index, "Wrong index type" \ No newline at end of file From 6e4a16b26507fd30c04f82179504753a5ae04ad7 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Fri, 13 Sep 2024 21:20:41 -0400 Subject: [PATCH 22/67] ts3 --- _includes/code/howto/indexes/indexes-v3.ts | 34 ++++++++++++++++++---- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/_includes/code/howto/indexes/indexes-v3.ts b/_includes/code/howto/indexes/indexes-v3.ts index 859de14873..9c7ae56b0d 100644 --- a/_includes/code/howto/indexes/indexes-v3.ts +++ b/_includes/code/howto/indexes/indexes-v3.ts @@ -93,6 +93,24 @@ async function createFlatCollection(client: WeaviateClient, collectionName: stri } // END EnableFlat +//////////////////// +// ENABLE DYNAMIC // +//////////////////// + +// START EnableDynamic +// Add this import line +// import { vectorizer, dataType, configure } from 'weaviate-client'; + +async function createDynamicCollection(client: WeaviateClient, collectionName: string){ + await client.collections.create({ + name: collectionName, + vectorizers: vectorizer.text2VecOpenAI({ + vectorIndexConfig: configure.vectorIndex.dynamic(), + }), + }) +} +// END EnableDynamic + ///////////////////////////// /// AVOID TOP LEVEL AWAIT /// ///////////////////////////// @@ -108,11 +126,11 @@ async function main(){ // Only one create can run at a time due to aynsc code - // Run enable HNSW collection code - deleteCollection(client, collectionName) - if(await client.collections.get(collectionName).exists() != true){ - createHNSWCollection(client, collectionName); - } + // // Run enable HNSW collection code + // deleteCollection(client, collectionName) + // if(await client.collections.get(collectionName).exists() != true){ + // createHNSWCollection(client, collectionName); + // } // // Run multiple named vector collection code // deleteCollection(client, collectionName) @@ -125,6 +143,12 @@ async function main(){ // if(await client.collections.get(collectionName).exists() != true){ // createFlatCollection(client, collectionName); // } + + // // Run dynamic collection code + deleteCollection(client, collectionName) + if(await client.collections.get(collectionName).exists() != true){ + createDynamicCollection(client, collectionName); + } } main() From da9647f8c00858402184021e524299bc0bf405dc Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Fri, 13 Sep 2024 21:25:37 -0400 Subject: [PATCH 23/67] ts2 --- _includes/code/howto/indexes/indexes-v2.ts | 38 ++++++++++++++++++---- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/_includes/code/howto/indexes/indexes-v2.ts b/_includes/code/howto/indexes/indexes-v2.ts index effdf318cb..d0e0c55f18 100644 --- a/_includes/code/howto/indexes/indexes-v2.ts +++ b/_includes/code/howto/indexes/indexes-v2.ts @@ -119,6 +119,25 @@ async function createFlatCollection(client: WeaviateClient, className: string){ } // END EnableFlat +//////////////////// +// ENABLE DYNAMIC // +//////////////////// + +// START EnableDynamic +async function createDynamicCollection(client: WeaviateClient, className: string){ + + const setIndexType = { + class: className, + // Add property definitions + vectorizer: 'text2vec-openai', + vectorIndexType: 'dynamic', + }; + + // Add the class to the schema + await client.schema.classCreator().withClass(setIndexType).do(); +} +// END EnableDynamic + ///////////////////////////// /// AVOID TOP LEVEL AWAIT /// ///////////////////////////// @@ -136,20 +155,25 @@ async function main(){ // deleteClass(client, className) // if(await client.schema.exists(className) != true){ // createHNSWCollection(client, className); - // } + // } // // Run multiple named vector collection code // deleteClass(client, className) // if(await client.schema.exists(className) != true){ // createMultiCollection(client, className); - // } + // } - // Run enable flat collection code - deleteClass(client, className) - if(await client.schema.exists(className) != true){ - createFlatCollection(client, className); - } + // // Run enable flat collection code + // deleteClass(client, className) + // if(await client.schema.exists(className) != true){ + // createFlatCollection(client, className); + // } + // Run enable dynamic collection code + deleteClass(client, className) + if(await client.schema.exists(className) != true){ + createDynamicCollection(client, className); + } } main() \ No newline at end of file From 758baea0d36f516401fd88a516ff0cfa1464a2d2 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Fri, 13 Sep 2024 21:36:30 -0400 Subject: [PATCH 24/67] inverted indexes --- .../configuration/indexing/inverted-indexes.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/developers/weaviate/configuration/indexing/inverted-indexes.md b/developers/weaviate/configuration/indexing/inverted-indexes.md index 1f31f73144..30d6ca0d51 100644 --- a/developers/weaviate/configuration/indexing/inverted-indexes.md +++ b/developers/weaviate/configuration/indexing/inverted-indexes.md @@ -5,16 +5,18 @@ image: og/docs/indexes.jpg # tags: ['configuration'] --- -- [Overview](add) -- [indexSearchable]](add) -- [indexFilterable]](add) -- [indexRangeFilters]](add) +Properties use an [inverted indexes](/developers/weaviate/concepts/indexing#inverted-indexes), also known as keyword indexes, to make property searches more efficient. Weaviate provides different kinds to inverted index so you can match better match the index to your data: -## Additional considerations +- [indexSearchable](#indexSearchable) +- [indexFilterable](#indexfilterable) +- [indexRangeFilters](#indexrangefilters) -These are some additional things to consider when you configure indexing. +Use these indexes to improve your searches. -### Availability levels +## indexSearchable + +## indexFilterable +## indexRangeFilters ## Related pages - [Indexes overview](/developers/weaviate/starter-guides/managing-resources/indexing) From de36ec833cc72625ff9d536372ccf81bf7d44ffe Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Fri, 13 Sep 2024 21:54:07 -0400 Subject: [PATCH 25/67] fix range index copy --- developers/weaviate/concepts/prefiltering.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/developers/weaviate/concepts/prefiltering.md b/developers/weaviate/concepts/prefiltering.md index 2dbabaede9..ea06871c1e 100644 --- a/developers/weaviate/concepts/prefiltering.md +++ b/developers/weaviate/concepts/prefiltering.md @@ -60,11 +60,11 @@ To learn more about Weaviate's roaring bitmaps implementation, see the [in-line :::info Added in `1.26` ::: -Weaviate `1.26` introduces the `indexRangeFilters` index, which is a range-based index for filtering by numerical ranges. This index is available for `int`, `number`, or `date` properties. The index is not available for arrays of these data types. +Weaviate `1.26` introduces the `indexRangeFilters` index for filtering by numerical ranges. This index is available for properties of type `int`, `number`, or `date`. The index is not available for arrays of these data types. Internally, rangeable indexes are implemented as [roaring bitmap slices](https://www.featurebase.com/blog/range-encoded-bitmaps). This data structure limits the index to values that can be stored as 64 bit integers. -`indexRangeFilters` is only available for new properties. Existing properties cannot be converted to use the rangeable index. +The `indexRangeFilters` index is only available for new properties. Existing properties cannot be converted to use the rangeable index. ## Recall on Pre-Filtered Searches From c87c5fb7ba9e12d3b29d852475cc760f372f2f85 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Fri, 13 Sep 2024 21:58:10 -0400 Subject: [PATCH 26/67] indexing copy --- .../weaviate/starter-guides/managing-resources/indexing.mdx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/developers/weaviate/starter-guides/managing-resources/indexing.mdx b/developers/weaviate/starter-guides/managing-resources/indexing.mdx index 07e8a68c9c..826cb238a9 100644 --- a/developers/weaviate/starter-guides/managing-resources/indexing.mdx +++ b/developers/weaviate/starter-guides/managing-resources/indexing.mdx @@ -128,7 +128,9 @@ For details, see [set inverted index parameters](/developers/weaviate/manage-dat Property level configuration is more limited. Individual indexes can be turned on or off at the property level, and keyword tokenization options can be set. -`indexSearchable` determines whether a property is indexed for keyword searches. `indexFilterable` determines whether a property is indexed to speed up match-based filtering. `indexRangeFilters` determines whether a property is indexed for numerical range filters. +- `indexSearchable` indexes are used with keyword searches +- `indexFilterable` indexes are used for filtering +- `indexRangeFilters` indexes are used for numerical range filters For more on filters, see [Filtering](/developers/weaviate/concepts/prefiltering). From f50d1dff686fc766094c8dd950e2c57d1f58dca5 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Fri, 13 Sep 2024 22:06:32 -0400 Subject: [PATCH 27/67] empty table --- .../indexing/inverted-indexes.md | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/developers/weaviate/configuration/indexing/inverted-indexes.md b/developers/weaviate/configuration/indexing/inverted-indexes.md index 30d6ca0d51..18de8b490d 100644 --- a/developers/weaviate/configuration/indexing/inverted-indexes.md +++ b/developers/weaviate/configuration/indexing/inverted-indexes.md @@ -5,6 +5,14 @@ image: og/docs/indexes.jpg # tags: ['configuration'] --- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock'; +import PyCodeV4 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v4.py'; +import PyCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v3.py'; +import TSCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v3.ts'; +import TSCodeV2 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v2.ts'; + Properties use an [inverted indexes](/developers/weaviate/concepts/indexing#inverted-indexes), also known as keyword indexes, to make property searches more efficient. Weaviate provides different kinds to inverted index so you can match better match the index to your data: - [indexSearchable](#indexSearchable) @@ -15,8 +23,58 @@ Use these indexes to improve your searches. ## indexSearchable +The `indexSearchable` index improves property search times. This index is enabled by default. [Keyword search](/developers/weaviate/search/bm25) and [hybrid search](/developers/weaviate/search/hybrid) use this index. + +If you don't anticipate searching on a property field, you can disable this index to save disk space and import time. The property is still searchable. The search is just less efficient. + ## indexFilterable + +The `indexFilterable` index improves [filtering](/developers/weaviate/search/filters). This index is enabled by default. + +If you don't anticipate searching on a property field, you can disable this index to save disk space and import time. The property is still filterable. + ## indexRangeFilters +The `indexRangeFilters` is a range-based index for filtering by [numerical ranges](/developers/weaviate/release-notes/release_1_26#improved-range-queries). This index is not enabled by default. + +## Configuration example + +Set these indexes on the property level. + + + + + + + + + + + + + + + + ## Related pages - [Indexes overview](/developers/weaviate/starter-guides/managing-resources/indexing) From 959e650180fd6da5da0d31ada6887294f4e2ac01 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Fri, 13 Sep 2024 22:21:23 -0400 Subject: [PATCH 28/67] py4 --- _includes/code/howto/indexes/indexes-v4.py | 41 +++++++++++++++++++ .../indexing/inverted-indexes.md | 5 +++ 2 files changed, 46 insertions(+) diff --git a/_includes/code/howto/indexes/indexes-v4.py b/_includes/code/howto/indexes/indexes-v4.py index b6b8efa390..a39e47123e 100644 --- a/_includes/code/howto/indexes/indexes-v4.py +++ b/_includes/code/howto/indexes/indexes-v4.py @@ -182,6 +182,47 @@ def print_response_iter(title, response): str(schema_response.vector_index_type) == "VectorIndexType.DYNAMIC" ), "Wrong index type" +######################## +### Inverted Indexes ### +######################## + +# Delete data from prior runs +if client.collections.exists(collection_name): + client.collections.delete(collection_name) + +# START PropIndex +from weaviate.classes.config import Configure, Property, DataType + +client.collections.create( + name=collection_name, + properties=[ + Property( + name="TextProperty", + data_type=DataType.TEXT, + index_filterable=True, + index_searchable=True, + ), + Property( + name="NumericProperty", + data_type=DataType.INT, + index_range_filters=True, + ), + ], + inverted_index_config=Configure.inverted_index( + bm25_b=0.7, + bm25_k1=1.25, + index_null_state=True, + index_property_length=True, + index_timestamps=True + ) +) +# END PropIndex + +collection = client.collections.get(collection_name) +collections_response = client.collections.list_all() + +assert collection_name in collections_response.keys(), "Collection missing" + ################ ### CLEAN UP ### ################ diff --git a/developers/weaviate/configuration/indexing/inverted-indexes.md b/developers/weaviate/configuration/indexing/inverted-indexes.md index 18de8b490d..917c52497a 100644 --- a/developers/weaviate/configuration/indexing/inverted-indexes.md +++ b/developers/weaviate/configuration/indexing/inverted-indexes.md @@ -34,8 +34,13 @@ The `indexFilterable` index improves [filtering](/developers/weaviate/search/fil If you don't anticipate searching on a property field, you can disable this index to save disk space and import time. The property is still filterable. ## indexRangeFilters + The `indexRangeFilters` is a range-based index for filtering by [numerical ranges](/developers/weaviate/release-notes/release_1_26#improved-range-queries). This index is not enabled by default. +## bm25 index + +The [`bm25` index](/developers/weaviate/config-refs/schema#bm25) is configured on the collection level. These collection-wide settings can be overwritten on the property level. + ## Configuration example Set these indexes on the property level. From 3d1fac7fc496182d8409cbb5c1bff5b6d1e6bcee Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Fri, 13 Sep 2024 22:25:46 -0400 Subject: [PATCH 29/67] py3 --- _includes/code/howto/indexes/indexes-v3.py | 52 +++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/_includes/code/howto/indexes/indexes-v3.py b/_includes/code/howto/indexes/indexes-v3.py index ea785c02ce..f580651fcd 100644 --- a/_includes/code/howto/indexes/indexes-v3.py +++ b/_includes/code/howto/indexes/indexes-v3.py @@ -176,4 +176,54 @@ schema_response["vectorIndexType"] == "dynamic" ): correct_index = True -assert correct_index, "Wrong index type" \ No newline at end of file +assert correct_index, "Wrong index type" + +######################## +### Inverted Indexes ### +######################## + +# Delete data from prior runs +if client.schema.exists(class_name): + client.schema.delete_class(class_name) + +# START PropIndex +class_obj = { + "class": class_name, + "vectorizer": "text2vec-huggingface", + "properties": [ + { + "name": "title", + "dataType": ["text"], + "indexFilterable": True, + "indexSearchable": True, + "moduleConfig": { + "text2vec-huggingface": {} + } + }, + { + "name": "chunk", + "dataType": ["int"], + "indexRangeFilters": True, + }, + ], + "invertedIndexConfig": { + "bm25": { + "b": 0.7, + "k1": 1.25 + }, + "indexTimestamps": True, + "indexNullState": True, + "indexPropertyLength": True + } +} + +client.schema.create_class(class_obj) +# END PropIndex + +class_response = client.schema.get() +schema_response = client.schema.get(class_name) + +classes = [] +for c in class_response["classes"]: + classes.append(c["class"]) +assert class_name in classes, "Class missing" \ No newline at end of file From 5357a788491e4d620b6c588e30783292c56d70ff Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Fri, 13 Sep 2024 22:34:52 -0400 Subject: [PATCH 30/67] ts3 --- _includes/code/howto/indexes/indexes-v3.ts | 48 +++++++++++++++++++++- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/_includes/code/howto/indexes/indexes-v3.ts b/_includes/code/howto/indexes/indexes-v3.ts index 9c7ae56b0d..6a9939c4f2 100644 --- a/_includes/code/howto/indexes/indexes-v3.ts +++ b/_includes/code/howto/indexes/indexes-v3.ts @@ -2,7 +2,7 @@ // TODO: Needs tests // Imports -import weaviate, { WeaviateClient,vectorizer, configure } from 'weaviate-client'; +import weaviate, { dataType, WeaviateClient,vectorizer, configure } from 'weaviate-client'; // Delete pre-existing collections function deleteCollection(client: WeaviateClient, collectionName: string){ @@ -111,6 +111,44 @@ async function createDynamicCollection(client: WeaviateClient, collectionName: s } // END EnableDynamic + +////////////////////// +// Inverted Indexes // +////////////////////// + +// START PropIndex +// Add this import line +// import { dataType } from 'weaviate-client'; + +async function createInvertedCollection(client: WeaviateClient, collectionName: string){ + await client.collections.create({ + name: collectionName, + properties: [ + { + name: 'TextProperty', + dataType: dataType.TEXT, + indexFilterable: true, + indexSearchable: true, + }, + { + name: 'NumericProperty', + dataType: dataType.INT, + indexRangeFilters: true, + }, + ], + invertedIndex: { + bm25: { + b: 0.7, + k1: 1.25 + }, + indexNullState: true, + indexPropertyLength: true, + indexTimestamps: true + } + }) +} +// END PropIndex + ///////////////////////////// /// AVOID TOP LEVEL AWAIT /// ///////////////////////////// @@ -145,9 +183,15 @@ async function main(){ // } // // Run dynamic collection code + // deleteCollection(client, collectionName) + // if(await client.collections.get(collectionName).exists() != true){ + // createDynamicCollection(client, collectionName); + // } + + // Run inverted collection code deleteCollection(client, collectionName) if(await client.collections.get(collectionName).exists() != true){ - createDynamicCollection(client, collectionName); + createInvertedCollection(client, collectionName); } } From 7b47612da3b6d5cf172f66d3305fbc2ec453a879 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Mon, 16 Sep 2024 11:06:38 -0400 Subject: [PATCH 31/67] ts code --- _includes/code/howto/indexes/indexes-v2.ts | 60 ++++++++++++++++++++-- _includes/code/howto/indexes/indexes-v3.ts | 1 - 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/_includes/code/howto/indexes/indexes-v2.ts b/_includes/code/howto/indexes/indexes-v2.ts index d0e0c55f18..1e195ce3cc 100644 --- a/_includes/code/howto/indexes/indexes-v2.ts +++ b/_includes/code/howto/indexes/indexes-v2.ts @@ -138,6 +138,50 @@ async function createDynamicCollection(client: WeaviateClient, className: string } // END EnableDynamic +////////////////////// +// Inverted Indexes // +////////////////////// + +// START PropIndex +async function createInvertedCollection(client: WeaviateClient, className: string){ + const invertedIndexSettings = { + class: className, + vectorizer: 'text2vec-huggingface', + properties: [ + { + name: 'textProperty', + dataType: ['text'], + indexFilterable: true, + indexSearchable: true, + moduleConfig: { + 'text2vec-huggingface': {}, + }, + }, + { + name: 'numericProperty', + dataType: ['int'], + indexRangeFilters: true, + }, + ], + invertedIndexConfig: { + bm25: { + b: 0.7, + k1: 1.25 + }, + indexTimestamps: true, + indexNullState: true, + indexPropertyLength: true + } +}; + +// Add the class to the schema +const result = await client.schema + .classCreator() + .withClass(invertedIndexSettings) + .do(); +} +// END PropIndex + ///////////////////////////// /// AVOID TOP LEVEL AWAIT /// ///////////////////////////// @@ -169,11 +213,17 @@ async function main(){ // createFlatCollection(client, className); // } - // Run enable dynamic collection code - deleteClass(client, className) - if(await client.schema.exists(className) != true){ - createDynamicCollection(client, className); - } + // // Run enable dynamic collection code + // deleteClass(client, className) + // if(await client.schema.exists(className) != true){ + // createDynamicCollection(client, className); + // } + + // Run inverted collection code + deleteClass(client, className) + if(await client.schema.exists(className) != true){ + createInvertedCollection(client, className); + } } main() \ No newline at end of file diff --git a/_includes/code/howto/indexes/indexes-v3.ts b/_includes/code/howto/indexes/indexes-v3.ts index 6a9939c4f2..1f73d06009 100644 --- a/_includes/code/howto/indexes/indexes-v3.ts +++ b/_includes/code/howto/indexes/indexes-v3.ts @@ -111,7 +111,6 @@ async function createDynamicCollection(client: WeaviateClient, collectionName: s } // END EnableDynamic - ////////////////////// // Inverted Indexes // ////////////////////// From 23482528b879eb44dd133f95ff9229621373966b Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Mon, 16 Sep 2024 15:45:58 -0400 Subject: [PATCH 32/67] pre spilt up --- .../configuration/indexing/inverted-indexes.md | 2 +- .../configuration/indexing/vector-indexes.md | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/developers/weaviate/configuration/indexing/inverted-indexes.md b/developers/weaviate/configuration/indexing/inverted-indexes.md index 917c52497a..59684f5031 100644 --- a/developers/weaviate/configuration/indexing/inverted-indexes.md +++ b/developers/weaviate/configuration/indexing/inverted-indexes.md @@ -13,7 +13,7 @@ import PyCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v3.py'; import TSCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v3.ts'; import TSCodeV2 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v2.ts'; -Properties use an [inverted indexes](/developers/weaviate/concepts/indexing#inverted-indexes), also known as keyword indexes, to make property searches more efficient. Weaviate provides different kinds to inverted index so you can match better match the index to your data: +Properties use [inverted indexes](/developers/weaviate/concepts/indexing#inverted-indexes), also known as keyword indexes, to make property searches more efficient. Weaviate provides different kinds to inverted index so you can match better match the index to your data: - [indexSearchable](#indexSearchable) - [indexFilterable](#indexfilterable) diff --git a/developers/weaviate/configuration/indexing/vector-indexes.md b/developers/weaviate/configuration/indexing/vector-indexes.md index 46c1dc6eee..530f1fd950 100644 --- a/developers/weaviate/configuration/indexing/vector-indexes.md +++ b/developers/weaviate/configuration/indexing/vector-indexes.md @@ -19,11 +19,7 @@ import VectorIntro from '/_includes/indexes/vector-intro.mdx'; HNSW indexes are the default index type. If your collection has more than 10,000 objects, you should use an HNSW index. A flat index works best for smaller collections. Collections that start small and grow beyond 10,000 objects should consider a dynamic index. -When you configure your indexes, consider using compression to manage resource usage. Weaviate provides these compression methods; - -import CompressMethods from '/_includes/configuration/compression-methods.mdx'; - - +When you configure your indexes, consider using [compression](#compression) to manage resource usage. Some compression methods have to be enabled when you create your collection. ## HNSW indexes @@ -213,6 +209,12 @@ These are some additional things to consider when you configure indexing. Vector indexes can be large. Compressed vectors lose some information, but they use fewer resources and can be very cost effective. In most cases the resource savings significantly outweigh the slight loss in performance. +Weaviate provides these compression methods: + +import CompressMethods from '/_includes/configuration/compression-methods.mdx'; + + + See also: - [Compression overview](/developers/weaviate/starter-guides/managing-resources/compression) From df126cbc6ae63316bae470b835e5234456a4b320 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Mon, 16 Sep 2024 17:12:58 -0400 Subject: [PATCH 33/67] vector overview --- _includes/indexes/dynamic-intro.mdx | 2 +- _includes/indexes/hnsw-intro.mdx | 2 +- .../configuration/indexing/dynamic-indexes.md | 240 ++++++++++++++++++ .../configuration/indexing/flat-indexes.md | 240 ++++++++++++++++++ .../configuration/indexing/hnsw-indexes.md | 240 ++++++++++++++++++ .../indexing/inverted-indexes.md | 2 +- .../configuration/indexing/vector-indexes.md | 127 +-------- 7 files changed, 735 insertions(+), 118 deletions(-) create mode 100644 developers/weaviate/configuration/indexing/dynamic-indexes.md create mode 100644 developers/weaviate/configuration/indexing/flat-indexes.md create mode 100644 developers/weaviate/configuration/indexing/hnsw-indexes.md diff --git a/_includes/indexes/dynamic-intro.mdx b/_includes/indexes/dynamic-intro.mdx index 98a656d3cb..8adb6b4b01 100644 --- a/_includes/indexes/dynamic-intro.mdx +++ b/_includes/indexes/dynamic-intro.mdx @@ -1 +1 @@ -A [dynamic index](/developers/weaviate/concepts/vector-index#dynamic-index) offers a flexible approach to indexing. A dynamic index begins as a flat index, and converts automatically to an HNSW index when a [collection](/developers/weaviate/concepts/data#collections) or [tenant](/developers/weaviate/concepts/data#multi-tenancy) reaches a threshold size. \ No newline at end of file +[Dynamic index](/developers/weaviate/concepts/vector-index#dynamic-index) offer a flexible approach to indexing. A dynamic index begins as a flat index, and converts automatically to an HNSW index when a [collection](/developers/weaviate/concepts/data#collections) or [tenant](/developers/weaviate/concepts/data#multi-tenancy) reaches a threshold size. \ No newline at end of file diff --git a/_includes/indexes/hnsw-intro.mdx b/_includes/indexes/hnsw-intro.mdx index 624a618889..6be8fc9801 100644 --- a/_includes/indexes/hnsw-intro.mdx +++ b/_includes/indexes/hnsw-intro.mdx @@ -1 +1 @@ -[HNSW](/developers/weaviate/concepts/vector-index#hierarchical-navigable-small-world-hnsw-index) are high-performance, in-memory indexes. HNSW indexes scale well, meaning that vector searches remain fast even for very large data sets. \ No newline at end of file +[HNSW indexes](/developers/weaviate/concepts/vector-index#hierarchical-navigable-small-world-hnsw-index) are high-performance, in-memory indexes. HNSW indexes scale well, meaning that vector searches remain fast even for very large data sets. \ No newline at end of file diff --git a/developers/weaviate/configuration/indexing/dynamic-indexes.md b/developers/weaviate/configuration/indexing/dynamic-indexes.md new file mode 100644 index 0000000000..da53336e1c --- /dev/null +++ b/developers/weaviate/configuration/indexing/dynamic-indexes.md @@ -0,0 +1,240 @@ +--- +title: Dynamic indexes +sidebar_position: 30 +image: og/docs/indexes.jpg +# tags: ['configuration'] +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock'; +import PyCodeV4 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v4.py'; +import PyCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v3.py'; +import TSCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v3.ts'; +import TSCodeV2 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v2.ts'; + +import VectorIntro from '/_includes/indexes/vector-intro.mdx'; + + + +HNSW indexes are the default index type. If your collection has more than 10,000 objects, you should use an HNSW index. A flat index works best for smaller collections. Collections that start small and grow beyond 10,000 objects should consider a dynamic index. + +When you configure your indexes, consider using [compression](#compression) to manage resource usage. Some compression methods have to be enabled when you create your collection. + +## HNSW indexes + +import HNSWIntro from '/_includes/indexes/hnsw-intro.mdx'; + + + +### Collection configuration + +Configure an HNSW index for a collection: + + + + + + + + + + + + + + + + +See also: + + - [HNSW index parameters](/developers/weaviate/config-refs/schema/vector-index#hnsw-index-parameters) + +## Flat indexes + +import FlatIntro from '/_includes/indexes/flat-intro.mdx'; + + + + + + + + + + + + + + + + + + +See also: + +- [Flat index parameters](/developers/weaviate/config-refs/schema/vector-index#flat-indexes) + +## Dynamic indexes + +import DynamicIntro from '/_includes/indexes/dynamic-intro.mdx'; + + + +Dynamic indexes require [asynchronous indexing](/developers/weaviate/config-refs/schema/vector-index#asynchronous-indexing). Enable asynchronous indexing before you configure a collection to use dynamic indexing. + + + + + + + + + + + + + + + + +See also: + +- [Dynamic index parameters](/developers/weaviate/config-refs/schema/vector-index#dynamic-index-parameters) + +## Multiple named vectors + +Configure a collection to use [multiple named vectors](/developers/weaviate/config-refs/schema/multi-vector): + + + + + + + + + + + + + + + + +## Additional considerations + +These are some additional things to consider when you configure indexing. + +### Compression + +Vector indexes can be large. Compressed vectors lose some information, but they use fewer resources and can be very cost effective. In most cases the resource savings significantly outweigh the slight loss in performance. + +Weaviate provides these compression methods: + +import CompressMethods from '/_includes/configuration/compression-methods.mdx'; + + + +See also: + +- [Compression overview](/developers/weaviate/starter-guides/managing-resources/compression) +- [Enable compression](/developers/weaviate/configuration/compression) + +### Asynchronous indexing + +Asynchronous indexing is a prerequisite for dynamic indexing and [AutoPQ](/developers/weaviate/configuration/compression/pq-compression#configure-autopq). + +import EnableAsynch from '/_includes/indexes/enable-async.mdx'; + + + +### Availability levels + +## Related pages +- [Indexes overview](/developers/weaviate/starter-guides/managing-resources/indexing) + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + \ No newline at end of file diff --git a/developers/weaviate/configuration/indexing/flat-indexes.md b/developers/weaviate/configuration/indexing/flat-indexes.md new file mode 100644 index 0000000000..6bc904545d --- /dev/null +++ b/developers/weaviate/configuration/indexing/flat-indexes.md @@ -0,0 +1,240 @@ +--- +title: Flat indexes +sidebar_position: 20 +image: og/docs/indexes.jpg +# tags: ['configuration'] +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock'; +import PyCodeV4 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v4.py'; +import PyCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v3.py'; +import TSCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v3.ts'; +import TSCodeV2 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v2.ts'; + +import VectorIntro from '/_includes/indexes/vector-intro.mdx'; + + + +HNSW indexes are the default index type. If your collection has more than 10,000 objects, you should use an HNSW index. A flat index works best for smaller collections. Collections that start small and grow beyond 10,000 objects should consider a dynamic index. + +When you configure your indexes, consider using [compression](#compression) to manage resource usage. Some compression methods have to be enabled when you create your collection. + +## HNSW indexes + +import HNSWIntro from '/_includes/indexes/hnsw-intro.mdx'; + + + +### Collection configuration + +Configure an HNSW index for a collection: + + + + + + + + + + + + + + + + +See also: + + - [HNSW index parameters](/developers/weaviate/config-refs/schema/vector-index#hnsw-index-parameters) + +## Flat indexes + +import FlatIntro from '/_includes/indexes/flat-intro.mdx'; + + + + + + + + + + + + + + + + + + +See also: + +- [Flat index parameters](/developers/weaviate/config-refs/schema/vector-index#flat-indexes) + +## Dynamic indexes + +import DynamicIntro from '/_includes/indexes/dynamic-intro.mdx'; + + + +Dynamic indexes require [asynchronous indexing](/developers/weaviate/config-refs/schema/vector-index#asynchronous-indexing). Enable asynchronous indexing before you configure a collection to use dynamic indexing. + + + + + + + + + + + + + + + + +See also: + +- [Dynamic index parameters](/developers/weaviate/config-refs/schema/vector-index#dynamic-index-parameters) + +## Multiple named vectors + +Configure a collection to use [multiple named vectors](/developers/weaviate/config-refs/schema/multi-vector): + + + + + + + + + + + + + + + + +## Additional considerations + +These are some additional things to consider when you configure indexing. + +### Compression + +Vector indexes can be large. Compressed vectors lose some information, but they use fewer resources and can be very cost effective. In most cases the resource savings significantly outweigh the slight loss in performance. + +Weaviate provides these compression methods: + +import CompressMethods from '/_includes/configuration/compression-methods.mdx'; + + + +See also: + +- [Compression overview](/developers/weaviate/starter-guides/managing-resources/compression) +- [Enable compression](/developers/weaviate/configuration/compression) + +### Asynchronous indexing + +Asynchronous indexing is a prerequisite for dynamic indexing and [AutoPQ](/developers/weaviate/configuration/compression/pq-compression#configure-autopq). + +import EnableAsynch from '/_includes/indexes/enable-async.mdx'; + + + +### Availability levels + +## Related pages +- [Indexes overview](/developers/weaviate/starter-guides/managing-resources/indexing) + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + \ No newline at end of file diff --git a/developers/weaviate/configuration/indexing/hnsw-indexes.md b/developers/weaviate/configuration/indexing/hnsw-indexes.md new file mode 100644 index 0000000000..68d6dab244 --- /dev/null +++ b/developers/weaviate/configuration/indexing/hnsw-indexes.md @@ -0,0 +1,240 @@ +--- +title: HNSW indexes +sidebar_position: 13 +image: og/docs/indexes.jpg +# tags: ['configuration'] +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock'; +import PyCodeV4 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v4.py'; +import PyCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v3.py'; +import TSCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v3.ts'; +import TSCodeV2 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v2.ts'; + +import VectorIntro from '/_includes/indexes/vector-intro.mdx'; + + + +HNSW indexes are the default index type. If your collection has more than 10,000 objects, you should use an HNSW index. A flat index works best for smaller collections. Collections that start small and grow beyond 10,000 objects should consider a dynamic index. + +When you configure your indexes, consider using [compression](#compression) to manage resource usage. Some compression methods have to be enabled when you create your collection. + +## HNSW indexes + +import HNSWIntro from '/_includes/indexes/hnsw-intro.mdx'; + + + +### Collection configuration + +Configure an HNSW index for a collection: + + + + + + + + + + + + + + + + +See also: + + - [HNSW index parameters](/developers/weaviate/config-refs/schema/vector-index#hnsw-index-parameters) + +## Flat indexes + +import FlatIntro from '/_includes/indexes/flat-intro.mdx'; + + + + + + + + + + + + + + + + + + +See also: + +- [Flat index parameters](/developers/weaviate/config-refs/schema/vector-index#flat-indexes) + +## Dynamic indexes + +import DynamicIntro from '/_includes/indexes/dynamic-intro.mdx'; + + + +Dynamic indexes require [asynchronous indexing](/developers/weaviate/config-refs/schema/vector-index#asynchronous-indexing). Enable asynchronous indexing before you configure a collection to use dynamic indexing. + + + + + + + + + + + + + + + + +See also: + +- [Dynamic index parameters](/developers/weaviate/config-refs/schema/vector-index#dynamic-index-parameters) + +## Multiple named vectors + +Configure a collection to use [multiple named vectors](/developers/weaviate/config-refs/schema/multi-vector): + + + + + + + + + + + + + + + + +## Additional considerations + +These are some additional things to consider when you configure indexing. + +### Compression + +Vector indexes can be large. Compressed vectors lose some information, but they use fewer resources and can be very cost effective. In most cases the resource savings significantly outweigh the slight loss in performance. + +Weaviate provides these compression methods: + +import CompressMethods from '/_includes/configuration/compression-methods.mdx'; + + + +See also: + +- [Compression overview](/developers/weaviate/starter-guides/managing-resources/compression) +- [Enable compression](/developers/weaviate/configuration/compression) + +### Asynchronous indexing + +Asynchronous indexing is a prerequisite for dynamic indexing and [AutoPQ](/developers/weaviate/configuration/compression/pq-compression#configure-autopq). + +import EnableAsynch from '/_includes/indexes/enable-async.mdx'; + + + +### Availability levels + +## Related pages +- [Indexes overview](/developers/weaviate/starter-guides/managing-resources/indexing) + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + \ No newline at end of file diff --git a/developers/weaviate/configuration/indexing/inverted-indexes.md b/developers/weaviate/configuration/indexing/inverted-indexes.md index 59684f5031..93f4415125 100644 --- a/developers/weaviate/configuration/indexing/inverted-indexes.md +++ b/developers/weaviate/configuration/indexing/inverted-indexes.md @@ -1,6 +1,6 @@ --- title: Inverted indexes -sidebar_position: 20 +sidebar_position: 60 image: og/docs/indexes.jpg # tags: ['configuration'] --- diff --git a/developers/weaviate/configuration/indexing/vector-indexes.md b/developers/weaviate/configuration/indexing/vector-indexes.md index 530f1fd950..4686a98b3a 100644 --- a/developers/weaviate/configuration/indexing/vector-indexes.md +++ b/developers/weaviate/configuration/indexing/vector-indexes.md @@ -1,5 +1,5 @@ --- -title: Vector indexes +title: Vector index overview sidebar_position: 10 image: og/docs/indexes.jpg # tags: ['configuration'] @@ -27,44 +27,7 @@ import HNSWIntro from '/_includes/indexes/hnsw-intro.mdx'; -### Collection configuration - -Configure an HNSW index for a collection: - - - - - - - - - - - - - - - +[Configure an HNSW index](/developers/weaviate/configuration/indexing/hnsw-indexes.md). See also: @@ -76,40 +39,7 @@ import FlatIntro from '/_includes/indexes/flat-intro.mdx'; - - - - - - - - - - - - - - +[Configure a flat index](/developers/weaviate/configuration/indexing/flat-indexes.md). See also: @@ -123,40 +53,7 @@ import DynamicIntro from '/_includes/indexes/dynamic-intro.mdx'; Dynamic indexes require [asynchronous indexing](/developers/weaviate/config-refs/schema/vector-index#asynchronous-indexing). Enable asynchronous indexing before you configure a collection to use dynamic indexing. - - - - - - - - - - - - - - +[Configure a dynamic index](/developers/weaviate/configuration/indexing/dynamic-indexes.md). See also: @@ -164,7 +61,9 @@ See also: ## Multiple named vectors -Configure a collection to use [multiple named vectors](/developers/weaviate/config-refs/schema/multi-vector): +Items in a collection can have multiple named vectors. Each named vectors has it's own vector index. These vector indexes can be configured independently. + +Configure a collection to use named vectors: @@ -201,11 +100,11 @@ Configure a collection to use [multiple named vectors](/developers/weaviate/conf -## Additional considerations +See also: -These are some additional things to consider when you configure indexing. +- [Multiple named vectors](/developers/weaviate/config-refs/schema/multi-vector) -### Compression +## Compression Vector indexes can be large. Compressed vectors lose some information, but they use fewer resources and can be very cost effective. In most cases the resource savings significantly outweigh the slight loss in performance. @@ -220,16 +119,14 @@ See also: - [Compression overview](/developers/weaviate/starter-guides/managing-resources/compression) - [Enable compression](/developers/weaviate/configuration/compression) -### Asynchronous indexing +## Asynchronous indexing -Asynchronous indexing is a prerequisite for dynamic indexing and [AutoPQ](/developers/weaviate/configuration/compression/pq-compression#configure-autopq). +Asynchronous indexing is a prerequisite for dynamic indexing and for [AutoPQ](/developers/weaviate/configuration/compression/pq-compression#configure-autopq). import EnableAsynch from '/_includes/indexes/enable-async.mdx'; -### Availability levels - ## Related pages - [Indexes overview](/developers/weaviate/starter-guides/managing-resources/indexing) From e9ae768850854ba03620256339c19cb3edbeaaf8 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Mon, 16 Sep 2024 17:34:00 -0400 Subject: [PATCH 34/67] split pages --- _includes/indexes/async-indexing.mdx | 5 + _includes/indexes/consider-compression.mdx | 1 + _includes/indexes/index-compression.mdx | 12 ++ .../configuration/indexing/dynamic-indexes.md | 176 ++--------------- .../configuration/indexing/flat-indexes.md | 179 +----------------- .../configuration/indexing/hnsw-indexes.md | 176 ++--------------- .../indexing/inverted-indexes.md | 2 +- .../configuration/indexing/vector-indexes.md | 20 +- 8 files changed, 55 insertions(+), 516 deletions(-) create mode 100644 _includes/indexes/async-indexing.mdx create mode 100644 _includes/indexes/consider-compression.mdx create mode 100644 _includes/indexes/index-compression.mdx diff --git a/_includes/indexes/async-indexing.mdx b/_includes/indexes/async-indexing.mdx new file mode 100644 index 0000000000..5c74d88092 --- /dev/null +++ b/_includes/indexes/async-indexing.mdx @@ -0,0 +1,5 @@ +Asynchronous indexing is a prerequisite for dynamic indexing and [AutoPQ](/developers/weaviate/configuration/compression/pq-compression#configure-autopq). + +import EnableAsynch from '/_includes/indexes/enable-async.mdx'; + + \ No newline at end of file diff --git a/_includes/indexes/consider-compression.mdx b/_includes/indexes/consider-compression.mdx new file mode 100644 index 0000000000..a335d6b044 --- /dev/null +++ b/_includes/indexes/consider-compression.mdx @@ -0,0 +1 @@ +When you configure your indexes, consider using [compression](#compression) to manage resource usage. Some compression methods have to be enabled when you create your collection. \ No newline at end of file diff --git a/_includes/indexes/index-compression.mdx b/_includes/indexes/index-compression.mdx new file mode 100644 index 0000000000..c6b21433c6 --- /dev/null +++ b/_includes/indexes/index-compression.mdx @@ -0,0 +1,12 @@ +Vector indexes can be large. Compressed vectors lose some information, but they use fewer resources and can be very cost effective. In most cases the resource savings significantly outweigh the slight loss in performance. + +Weaviate provides these compression methods: + +import CompressMethods from '/_includes/configuration/compression-methods.mdx'; + + + +See also: + +- [Compression overview](/developers/weaviate/starter-guides/managing-resources/compression) +- [Enable compression](/developers/weaviate/configuration/compression) \ No newline at end of file diff --git a/developers/weaviate/configuration/indexing/dynamic-indexes.md b/developers/weaviate/configuration/indexing/dynamic-indexes.md index da53336e1c..68c58c55ad 100644 --- a/developers/weaviate/configuration/indexing/dynamic-indexes.md +++ b/developers/weaviate/configuration/indexing/dynamic-indexes.md @@ -13,109 +13,6 @@ import PyCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v3.py'; import TSCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v3.ts'; import TSCodeV2 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v2.ts'; -import VectorIntro from '/_includes/indexes/vector-intro.mdx'; - - - -HNSW indexes are the default index type. If your collection has more than 10,000 objects, you should use an HNSW index. A flat index works best for smaller collections. Collections that start small and grow beyond 10,000 objects should consider a dynamic index. - -When you configure your indexes, consider using [compression](#compression) to manage resource usage. Some compression methods have to be enabled when you create your collection. - -## HNSW indexes - -import HNSWIntro from '/_includes/indexes/hnsw-intro.mdx'; - - - -### Collection configuration - -Configure an HNSW index for a collection: - - - - - - - - - - - - - - - - -See also: - - - [HNSW index parameters](/developers/weaviate/config-refs/schema/vector-index#hnsw-index-parameters) - -## Flat indexes - -import FlatIntro from '/_includes/indexes/flat-intro.mdx'; - - - - - - - - - - - - - - - - - - -See also: - -- [Flat index parameters](/developers/weaviate/config-refs/schema/vector-index#flat-indexes) - -## Dynamic indexes import DynamicIntro from '/_includes/indexes/dynamic-intro.mdx'; @@ -123,6 +20,10 @@ import DynamicIntro from '/_includes/indexes/dynamic-intro.mdx'; Dynamic indexes require [asynchronous indexing](/developers/weaviate/config-refs/schema/vector-index#asynchronous-indexing). Enable asynchronous indexing before you configure a collection to use dynamic indexing. +import ConsiderComp from '/_includes/indexes/consider-compression.mdx'; + + + - - - - - - - - - - - - - - -## Additional considerations - -These are some additional things to consider when you configure indexing. - -### Compression - -Vector indexes can be large. Compressed vectors lose some information, but they use fewer resources and can be very cost effective. In most cases the resource savings significantly outweigh the slight loss in performance. +## Compression -Weaviate provides these compression methods: +import IndexCompression from '/_includes/indexes/index-compression.mdx'; -import CompressMethods from '/_includes/configuration/compression-methods.mdx'; + - +## Asynchronous indexing -See also: - -- [Compression overview](/developers/weaviate/starter-guides/managing-resources/compression) -- [Enable compression](/developers/weaviate/configuration/compression) - -### Asynchronous indexing - -Asynchronous indexing is a prerequisite for dynamic indexing and [AutoPQ](/developers/weaviate/configuration/compression/pq-compression#configure-autopq). +import AsynchIndexing from '/_includes/indexes/async-indexing.mdx'; -import EnableAsynch from '/_includes/indexes/enable-async.mdx'; - - - -### Availability levels + ## Related pages + - [Indexes overview](/developers/weaviate/starter-guides/managing-resources/indexing) ## Questions and feedback diff --git a/developers/weaviate/configuration/indexing/flat-indexes.md b/developers/weaviate/configuration/indexing/flat-indexes.md index 6bc904545d..91332e12ea 100644 --- a/developers/weaviate/configuration/indexing/flat-indexes.md +++ b/developers/weaviate/configuration/indexing/flat-indexes.md @@ -13,69 +13,14 @@ import PyCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v3.py'; import TSCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v3.ts'; import TSCodeV2 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v2.ts'; -import VectorIntro from '/_includes/indexes/vector-intro.mdx'; - - - -HNSW indexes are the default index type. If your collection has more than 10,000 objects, you should use an HNSW index. A flat index works best for smaller collections. Collections that start small and grow beyond 10,000 objects should consider a dynamic index. - -When you configure your indexes, consider using [compression](#compression) to manage resource usage. Some compression methods have to be enabled when you create your collection. - -## HNSW indexes - -import HNSWIntro from '/_includes/indexes/hnsw-intro.mdx'; - - - -### Collection configuration - -Configure an HNSW index for a collection: - - - - - - - - - - - - - - - - -See also: - - - [HNSW index parameters](/developers/weaviate/config-refs/schema/vector-index#hnsw-index-parameters) - -## Flat indexes - import FlatIntro from '/_includes/indexes/flat-intro.mdx'; +import ConsiderComp from '/_includes/indexes/consider-compression.mdx'; + + + - -Dynamic indexes require [asynchronous indexing](/developers/weaviate/config-refs/schema/vector-index#asynchronous-indexing). Enable asynchronous indexing before you configure a collection to use dynamic indexing. - - - - - - - - - - - - - - - - -See also: - -- [Dynamic index parameters](/developers/weaviate/config-refs/schema/vector-index#dynamic-index-parameters) +## Compression -## Multiple named vectors +import IndexCompression from '/_includes/indexes/index-compression.mdx'; -Configure a collection to use [multiple named vectors](/developers/weaviate/config-refs/schema/multi-vector): - - - - - - - - - - - - - - - - -## Additional considerations - -These are some additional things to consider when you configure indexing. - -### Compression - -Vector indexes can be large. Compressed vectors lose some information, but they use fewer resources and can be very cost effective. In most cases the resource savings significantly outweigh the slight loss in performance. - -Weaviate provides these compression methods: - -import CompressMethods from '/_includes/configuration/compression-methods.mdx'; - - - -See also: - -- [Compression overview](/developers/weaviate/starter-guides/managing-resources/compression) -- [Enable compression](/developers/weaviate/configuration/compression) - -### Asynchronous indexing - -Asynchronous indexing is a prerequisite for dynamic indexing and [AutoPQ](/developers/weaviate/configuration/compression/pq-compression#configure-autopq). - -import EnableAsynch from '/_includes/indexes/enable-async.mdx'; - - - -### Availability levels + ## Related pages + - [Indexes overview](/developers/weaviate/starter-guides/managing-resources/indexing) ## Questions and feedback diff --git a/developers/weaviate/configuration/indexing/hnsw-indexes.md b/developers/weaviate/configuration/indexing/hnsw-indexes.md index 68d6dab244..98cd48a1e7 100644 --- a/developers/weaviate/configuration/indexing/hnsw-indexes.md +++ b/developers/weaviate/configuration/indexing/hnsw-indexes.md @@ -13,21 +13,16 @@ import PyCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v3.py'; import TSCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v3.ts'; import TSCodeV2 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v2.ts'; -import VectorIntro from '/_includes/indexes/vector-intro.mdx'; - - - -HNSW indexes are the default index type. If your collection has more than 10,000 objects, you should use an HNSW index. A flat index works best for smaller collections. Collections that start small and grow beyond 10,000 objects should consider a dynamic index. - -When you configure your indexes, consider using [compression](#compression) to manage resource usage. Some compression methods have to be enabled when you create your collection. - -## HNSW indexes import HNSWIntro from '/_includes/indexes/hnsw-intro.mdx'; -### Collection configuration +import ConsiderComp from '/_includes/indexes/consider-compression.mdx'; + + + +## Collection configuration Configure an HNSW index for a collection: @@ -70,167 +65,20 @@ See also: - [HNSW index parameters](/developers/weaviate/config-refs/schema/vector-index#hnsw-index-parameters) -## Flat indexes - -import FlatIntro from '/_includes/indexes/flat-intro.mdx'; - - - - - - - - - - - - - - - - - - -See also: - -- [Flat index parameters](/developers/weaviate/config-refs/schema/vector-index#flat-indexes) - -## Dynamic indexes - -import DynamicIntro from '/_includes/indexes/dynamic-intro.mdx'; - - - -Dynamic indexes require [asynchronous indexing](/developers/weaviate/config-refs/schema/vector-index#asynchronous-indexing). Enable asynchronous indexing before you configure a collection to use dynamic indexing. - - - - - - - - - - - - - - - - -See also: - -- [Dynamic index parameters](/developers/weaviate/config-refs/schema/vector-index#dynamic-index-parameters) - -## Multiple named vectors - -Configure a collection to use [multiple named vectors](/developers/weaviate/config-refs/schema/multi-vector): - - - - - - - - - - - - - - - - -## Additional considerations - -These are some additional things to consider when you configure indexing. - -### Compression - -Vector indexes can be large. Compressed vectors lose some information, but they use fewer resources and can be very cost effective. In most cases the resource savings significantly outweigh the slight loss in performance. +## Compression -Weaviate provides these compression methods: +import IndexCompression from '/_includes/indexes/index-compression.mdx'; -import CompressMethods from '/_includes/configuration/compression-methods.mdx'; + - +## Asynchronous indexing -See also: - -- [Compression overview](/developers/weaviate/starter-guides/managing-resources/compression) -- [Enable compression](/developers/weaviate/configuration/compression) - -### Asynchronous indexing - -Asynchronous indexing is a prerequisite for dynamic indexing and [AutoPQ](/developers/weaviate/configuration/compression/pq-compression#configure-autopq). +import AsynchIndexing from '/_includes/indexes/async-indexing.mdx'; -import EnableAsynch from '/_includes/indexes/enable-async.mdx'; - - - -### Availability levels + ## Related pages + - [Indexes overview](/developers/weaviate/starter-guides/managing-resources/indexing) ## Questions and feedback diff --git a/developers/weaviate/configuration/indexing/inverted-indexes.md b/developers/weaviate/configuration/indexing/inverted-indexes.md index 93f4415125..c9b13b4977 100644 --- a/developers/weaviate/configuration/indexing/inverted-indexes.md +++ b/developers/weaviate/configuration/indexing/inverted-indexes.md @@ -80,8 +80,8 @@ Set these indexes on the property level. - ## Related pages + - [Indexes overview](/developers/weaviate/starter-guides/managing-resources/indexing) ## Questions and feedback diff --git a/developers/weaviate/configuration/indexing/vector-indexes.md b/developers/weaviate/configuration/indexing/vector-indexes.md index 4686a98b3a..7bd46b915a 100644 --- a/developers/weaviate/configuration/indexing/vector-indexes.md +++ b/developers/weaviate/configuration/indexing/vector-indexes.md @@ -106,28 +106,18 @@ See also: ## Compression -Vector indexes can be large. Compressed vectors lose some information, but they use fewer resources and can be very cost effective. In most cases the resource savings significantly outweigh the slight loss in performance. +import IndexCompression from '/_includes/indexes/index-compression.mdx'; -Weaviate provides these compression methods: - -import CompressMethods from '/_includes/configuration/compression-methods.mdx'; - - - -See also: - -- [Compression overview](/developers/weaviate/starter-guides/managing-resources/compression) -- [Enable compression](/developers/weaviate/configuration/compression) + ## Asynchronous indexing -Asynchronous indexing is a prerequisite for dynamic indexing and for [AutoPQ](/developers/weaviate/configuration/compression/pq-compression#configure-autopq). - -import EnableAsynch from '/_includes/indexes/enable-async.mdx'; +import AsynchIndexing from '/_includes/indexes/async-indexing.mdx'; - + ## Related pages + - [Indexes overview](/developers/weaviate/starter-guides/managing-resources/indexing) ## Questions and feedback From 5d65a82336b4d43bd02fc6109d5df26d60b7ecd2 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Tue, 17 Sep 2024 11:25:44 -0400 Subject: [PATCH 35/67] hnsw start --- _includes/code/howto/indexes/indexes-v3.py | 54 +++++++++++---- _includes/code/howto/indexes/indexes-v4.py | 65 +++++++++++++++++-- _includes/indexes/multiple-named-vectors.mdx | 50 ++++++++++++++ .../configuration/indexing/dynamic-indexes.md | 6 ++ .../configuration/indexing/flat-indexes.md | 49 ++++++++++++++ .../configuration/indexing/hnsw-indexes.md | 55 +++++++++++++--- .../configuration/indexing/vector-indexes.md | 45 ------------- 7 files changed, 250 insertions(+), 74 deletions(-) create mode 100644 _includes/indexes/multiple-named-vectors.mdx diff --git a/_includes/code/howto/indexes/indexes-v3.py b/_includes/code/howto/indexes/indexes-v3.py index f580651fcd..a1cf4a5118 100644 --- a/_includes/code/howto/indexes/indexes-v3.py +++ b/_includes/code/howto/indexes/indexes-v3.py @@ -25,7 +25,39 @@ # START EnableHNSW class_obj = { "class": class_name, - # Additional configuration not shown + "vectorIndexType": "hnsw", +} + +client.schema.create_class(class_obj) +# END EnableHNSW + +class_response = client.schema.get() +schema_response = client.schema.get(class_name) + +classes = [] +for c in class_response["classes"]: + classes.append(c["class"]) +assert class_name in classes, "Class missing" + +correct_index = False +if (schema_response["class"] == class_name) and ( + schema_response["vectorIndexType"] == "hnsw" +): + correct_index = True +assert correct_index, "Wrong index type" + + +################### +### CONFIG HNSW ### +################### + +# Delete data from prior runs +if client.schema.exists(class_name): + client.schema.delete_class(class_name) + +# START ConfigHNSW +class_obj = { + "class": class_name, "vectorIndexType": "hnsw", "vectorIndexConfig": { "distance_metric": "cosine", @@ -42,7 +74,7 @@ } client.schema.create_class(class_obj) -# END EnableHNSW +# END ConfigHNSW class_response = client.schema.get() schema_response = client.schema.get(class_name) @@ -80,13 +112,7 @@ # START EnableFlat class_obj = { "class": class_name, - # Additional configuration not shown "vectorIndexType": "flat", - "vectorIndexConfig": { - "distance_metric": "cosine", - "vector_cache_max_objects": 100000, - "bq": {"enabled": True} - }, } client.schema.create_class(class_obj) @@ -107,18 +133,17 @@ correct_index = True assert correct_index, "Wrong index type" -################### -### ENABLE FLAT ### -################### +###################### +### CONFIGURE FLAT ### +###################### # Delete data from prior runs if client.schema.exists(class_name): client.schema.delete_class(class_name) -# START EnableFlat +# START ConfigFlat class_obj = { "class": class_name, - # Additional configuration not shown "vectorIndexType": "flat", "vectorIndexConfig": { "distance_metric": "cosine", @@ -128,7 +153,7 @@ } client.schema.create_class(class_obj) -# END EnableFlat +# END ConfigFlat class_response = client.schema.get() schema_response = client.schema.get(class_name) @@ -143,6 +168,7 @@ schema_response["vectorIndexType"] == "flat" ): correct_index = True + assert correct_index, "Wrong index type" ###################### diff --git a/_includes/code/howto/indexes/indexes-v4.py b/_includes/code/howto/indexes/indexes-v4.py index a39e47123e..6e37782ef4 100644 --- a/_includes/code/howto/indexes/indexes-v4.py +++ b/_includes/code/howto/indexes/indexes-v4.py @@ -33,10 +33,33 @@ def print_response_iter(title, response): client.collections.create( name=collection_name, - vectorizer_config=Configure.Vectorizer.text2vec_cohere(), - # This line enables the index - # vector_index_config=Configure.VectorIndex.hnsw() - # These lines enable and configure the index + vector_index_config=Configure.VectorIndex.hnsw(), + # Configure properties, vectorizer +) +# END EnableHNSW + +collection = client.collections.get(collection_name) +collections_response = client.collections.list_all() +schema_response = collection.config.get() + +assert collection_name in collections_response.keys(), "Collection missing" +assert ( + str(schema_response.vector_index_type) == "VectorIndexType.HNSW" +), "Wrong index type" + +################################ +### CONFIG HNSW - COLLECTION ### +################################ + +# Delete data from prior runs +if client.collections.exists(collection_name): + client.collections.delete(collection_name) + +# START ConfigHNSW +from weaviate.classes.config import Configure, VectorDistances + +client.collections.create( + name=collection_name, vector_index_config=Configure.VectorIndex.hnsw( distance_metric=VectorDistances.COSINE, ef_construction=256, # Dynamic list size during construction @@ -47,8 +70,9 @@ def print_response_iter(title, response): dynamic_ef_min=200, # Minimum threshold for dynamic Ef dynamic_ef_max=1000, # Maximum threshold for dynamic Ef ), + # Configure properties, vectorizer ) -# END EnableHNSW +# END ConfigHNSW collection = client.collections.get(collection_name) collections_response = client.collections.list_all() @@ -124,14 +148,41 @@ def print_response_iter(title, response): client.collections.create( name=collection_name, - vectorizer_config=Configure.Vectorizer.text2vec_cohere(), + vector_index_config=Configure.VectorIndex.flat(), + # Configure properties, vectorizer +) +# END EnableFlat + +collection = client.collections.get(collection_name) +collections_response = client.collections.list_all() +schema_response = collection.config.get() + +assert collection_name in collections_response.keys(), "Collection missing" +assert ( + str(schema_response.vector_index_type) == "VectorIndexType.FLAT" +), "Wrong index type" + +################### +### CONFIG FLAT ### +################### + +# Delete data from prior runs +if client.collections.exists(collection_name): + client.collections.delete(collection_name) + +# START ConfigFlat +from weaviate.classes.config import Configure, VectorDistances + +client.collections.create( + name=collection_name, vector_index_config=Configure.VectorIndex.flat( distance_metric=VectorDistances.COSINE, vector_cache_max_objects=100000, quantizer=Configure.VectorIndex.Quantizer.bq() ), + # Configure properties, vectorizer ) -# END EnableFlat +# END ConfigFlat collection = client.collections.get(collection_name) collections_response = client.collections.list_all() diff --git a/_includes/indexes/multiple-named-vectors.mdx b/_includes/indexes/multiple-named-vectors.mdx new file mode 100644 index 0000000000..ca715eb75b --- /dev/null +++ b/_includes/indexes/multiple-named-vectors.mdx @@ -0,0 +1,50 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock'; +import PyCodeV4 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v4.py'; +import PyCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v3.py'; +import TSCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v3.ts'; +import TSCodeV2 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v2.ts'; + +Items in a collection can have multiple named vectors. Each named vectors has it's own vector index. These vector indexes can be configured independently. + +Configure a collection to use named vectors: + + + + + + + + + + + + + + + + +See also: + +- [Multiple named vectors](/developers/weaviate/config-refs/schema/multi-vector) \ No newline at end of file diff --git a/developers/weaviate/configuration/indexing/dynamic-indexes.md b/developers/weaviate/configuration/indexing/dynamic-indexes.md index 68c58c55ad..f67008020a 100644 --- a/developers/weaviate/configuration/indexing/dynamic-indexes.md +++ b/developers/weaviate/configuration/indexing/dynamic-indexes.md @@ -63,6 +63,12 @@ See also: - [Dynamic index parameters](/developers/weaviate/config-refs/schema/vector-index#dynamic-index-parameters) +## Multiple named vectors + +import MultiNameVec from '/_includes/indexes/multiple-named-vectors.mdx'; + + + ## Compression import IndexCompression from '/_includes/indexes/index-compression.mdx'; diff --git a/developers/weaviate/configuration/indexing/flat-indexes.md b/developers/weaviate/configuration/indexing/flat-indexes.md index 91332e12ea..641cbf9d00 100644 --- a/developers/weaviate/configuration/indexing/flat-indexes.md +++ b/developers/weaviate/configuration/indexing/flat-indexes.md @@ -21,6 +21,10 @@ import ConsiderComp from '/_includes/indexes/consider-compression.mdx'; +## Enable a flat index + +Enable a flat index for a collection: + +## Configure a flat index + +Configure a flat index for a collection: + + + + + + + + + + + + + + + + See also: - [Flat index parameters](/developers/weaviate/config-refs/schema/vector-index#flat-indexes) +## Multiple named vectors + +import MultiNameVec from '/_includes/indexes/multiple-named-vectors.mdx'; + + + ## Compression import IndexCompression from '/_includes/indexes/index-compression.mdx'; diff --git a/developers/weaviate/configuration/indexing/hnsw-indexes.md b/developers/weaviate/configuration/indexing/hnsw-indexes.md index 98cd48a1e7..ac9bf91585 100644 --- a/developers/weaviate/configuration/indexing/hnsw-indexes.md +++ b/developers/weaviate/configuration/indexing/hnsw-indexes.md @@ -22,9 +22,9 @@ import ConsiderComp from '/_includes/indexes/consider-compression.mdx'; -## Collection configuration +## Enable an HNSW index -Configure an HNSW index for a collection: +HNSW is the default vector index. You do not have to explicitly enable HNSW if you want to use the default vector index type. @@ -61,21 +61,60 @@ Configure an HNSW index for a collection: +## Configure an HNSW index + +Configure an HNSW index for a collection: + + + + + + + + + + + + + + + + See also: - [HNSW index parameters](/developers/weaviate/config-refs/schema/vector-index#hnsw-index-parameters) -## Compression +## Multiple named vectors -import IndexCompression from '/_includes/indexes/index-compression.mdx'; +import MultiNameVec from '/_includes/indexes/multiple-named-vectors.mdx'; - + -## Asynchronous indexing +## Compression -import AsynchIndexing from '/_includes/indexes/async-indexing.mdx'; +import IndexCompression from '/_includes/indexes/index-compression.mdx'; - + ## Related pages diff --git a/developers/weaviate/configuration/indexing/vector-indexes.md b/developers/weaviate/configuration/indexing/vector-indexes.md index 7bd46b915a..1cb4088e63 100644 --- a/developers/weaviate/configuration/indexing/vector-indexes.md +++ b/developers/weaviate/configuration/indexing/vector-indexes.md @@ -59,51 +59,6 @@ See also: - [Dynamic index parameters](/developers/weaviate/config-refs/schema/vector-index#dynamic-index-parameters) -## Multiple named vectors - -Items in a collection can have multiple named vectors. Each named vectors has it's own vector index. These vector indexes can be configured independently. - -Configure a collection to use named vectors: - - - - - - - - - - - - - - - - -See also: - -- [Multiple named vectors](/developers/weaviate/config-refs/schema/multi-vector) - ## Compression import IndexCompression from '/_includes/indexes/index-compression.mdx'; From 25e31c84dddd7699cd27f092142ae445b57fd7ab Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Tue, 17 Sep 2024 14:50:02 -0400 Subject: [PATCH 36/67] vector updates --- _includes/code/howto/indexes/indexes-v2.ts | 142 ++++++++++++++---- _includes/code/howto/indexes/indexes-v3.py | 80 +++++++++- _includes/code/howto/indexes/indexes-v3.ts | 136 +++++++++++++++-- _includes/code/howto/indexes/indexes-v4.py | 58 +++++++ .../configuration/indexing/flat-indexes.md | 39 +++++ .../configuration/indexing/hnsw-indexes.md | 39 +++++ 6 files changed, 453 insertions(+), 41 deletions(-) diff --git a/_includes/code/howto/indexes/indexes-v2.ts b/_includes/code/howto/indexes/indexes-v2.ts index 1e195ce3cc..da8033caaf 100644 --- a/_includes/code/howto/indexes/indexes-v2.ts +++ b/_includes/code/howto/indexes/indexes-v2.ts @@ -33,24 +33,8 @@ async function createHNSWCollection(client: WeaviateClient, className: string){ const setIndexType = { class: className, - // Add property definitions - vectorizer: 'text2vec-openai', vectorIndexType: 'hnsw', - vectorIndexConfig: { - distance: 'cosine', - ef_construction: '256', // Dynamic list size during construction - max_connections: '128', // Maximum number of connections per node - quantizer: 'Configure.VectorIndex.Quantizer.pq()', // Quantizer configuration - ef: '-1', // Dynamic list size during search; -1 enables dynamic Ef - dynamic_ef_factor: '15', // Multiplier for dynamic Ef - dynamic_ef_min: '200', // Minimum threshold for dynamic Ef - dynamic_ef_max: '1000', // Maximum threshold for dynamic Ef - pq: { - enabled: true, - trainingLimit: 100000, - segments: 96, - }, - }, + // Configure properties, vectorizer }; // Add the class to the schema @@ -58,6 +42,54 @@ async function createHNSWCollection(client: WeaviateClient, className: string){ } // END EnableHNSW + +//////////////////// +// CONFIGURE HNSW // +//////////////////// + +// START ConfigHNSW +async function configHNSWCollection(client: WeaviateClient, className: string){ + + const setIndexType = { + class: className, + vectorIndexType: 'hnsw', + vectorIndexConfig: { + distance: 'cosine', + ef_construction: '256', // Dynamic list size during construction + max_connections: '128', // Maximum number of connections per node + ef: '-1', // Dynamic list size during search; -1 enables dynamic Ef + dynamic_ef_factor: '15', // Multiplier for dynamic Ef + dynamic_ef_min: '200', // Minimum threshold for dynamic Ef + dynamic_ef_max: '1000', // Maximum threshold for dynamic Ef + quantizer: 'Configure.VectorIndex.Quantizer.pq()', // Quantizer configuration + }, + // Configure properties, vectorizer + }; + + // Add the class to the schema + await client.schema.classCreator().withClass(setIndexType).do(); +} +// END ConfigHNSW + +/////////////////// +// COMPRESS HNSW // +/////////////////// + +// START CompressHNSW +async function compressHNSWCollection(client: WeaviateClient, className: string){ + const setIndexType = { + class: className, + vectorIndexConfig: { + quantizer: 'Configure.VectorIndex.Quantizer.pq()', + }, + // Configure properties, vectorizer + }; + + // Add the class to the schema + await client.schema.classCreator().withClass(setIndexType).do(); +} +// END CompressHNSW + ////////////////////////////// /// ENABLE HNSW - MULTIPLE /// ////////////////////////////// @@ -86,8 +118,8 @@ async function createMultiCollection(client: WeaviateClient, className: string){ }, vectorIndexType: 'flat' }, - } - // Configure properties + }, + // Configure properties } // Add the class to the schema @@ -104,20 +136,58 @@ async function createFlatCollection(client: WeaviateClient, className: string){ const setIndexType = { class: className, - // Add property definitions - vectorizer: 'text2vec-openai', + vectorIndexType: 'flat', + // Configure properties, vectorizer + }; + + // Add the class to the schema + await client.schema.classCreator().withClass(setIndexType).do(); +} +// END EnableFlat + +//////////////////// +// CONFIGURE FLAT // +//////////////////// + +// START ConfigFlat +async function configureFlatCollection(client: WeaviateClient, className: string){ + + const setIndexType = { + class: className, vectorIndexType: 'flat', vectorIndexConfig: { distance: 'cosine', vector_cache_max_objects: 100000, bq: { enabled: true, }, }, + // Configure properties, vectorizer }; // Add the class to the schema await client.schema.classCreator().withClass(setIndexType).do(); } -// END EnableFlat +// END ConfigFlat + +/////////////////// +// COMPRESS FLAT // +/////////////////// + +// START CompressFlat +async function compressFlatCollection(client: WeaviateClient, className: string){ + + const setIndexType = { + class: className, + vectorIndexType: 'flat', + vectorIndexConfig: { + bq: { enabled: true, }, + }, + // Configure properties, vectorizer + }; + + // Add the class to the schema + await client.schema.classCreator().withClass(setIndexType).do(); +} +// END CompressFlat //////////////////// // ENABLE DYNAMIC // @@ -195,12 +265,18 @@ async function main(){ // Only one create can run at a time due to aynsc code - // Run enable HNSW collection code + // // Run enable HNSW collection code // deleteClass(client, className) // if(await client.schema.exists(className) != true){ // createHNSWCollection(client, className); // } + // Run configure HNSW collection code + deleteClass(client, className) + if(await client.schema.exists(className) != true){ + configHNSWCollection(client, className); + } + // // Run multiple named vector collection code // deleteClass(client, className) // if(await client.schema.exists(className) != true){ @@ -213,17 +289,29 @@ async function main(){ // createFlatCollection(client, className); // } - // // Run enable dynamic collection code + // // Run configure flat collection code // deleteClass(client, className) // if(await client.schema.exists(className) != true){ - // createDynamicCollection(client, className); + // configureFlatCollection(client, className); // } - // Run inverted collection code + // Run compress flat collection code deleteClass(client, className) if(await client.schema.exists(className) != true){ - createInvertedCollection(client, className); + compressFlatCollection(client, className); } + + // // Run enable dynamic collection code + // deleteClass(client, className) + // if(await client.schema.exists(className) != true){ + // createDynamicCollection(client, className); + // } + + // // Run inverted collection code + // deleteClass(client, className) + // if(await client.schema.exists(className) != true){ + // createInvertedCollection(client, className); + // } } main() \ No newline at end of file diff --git a/_includes/code/howto/indexes/indexes-v3.py b/_includes/code/howto/indexes/indexes-v3.py index a1cf4a5118..a4f7706468 100644 --- a/_includes/code/howto/indexes/indexes-v3.py +++ b/_includes/code/howto/indexes/indexes-v3.py @@ -63,13 +63,13 @@ "distance_metric": "cosine", "ef_construction": 256, # Dynamic list size during construction "max_connections": 128, # Maximum number of connections per node - "pq": { - "enabled": True, - }, # Enable compression "ef": -1, # Dynamic list size during search; -1 enables dynamic Ef "dynamic_ef_factor": 15, # Multiplier for dynamic Ef "dynamic_ef_min": 200, # Minimum threshold for dynamic Ef "dynamic_ef_max": 1000, # Maximum threshold for dynamic Ef + "pq": { + "enabled": True, + }, # Enable compression }, } @@ -91,6 +91,44 @@ correct_index = True assert correct_index, "Wrong index type" +##################### +### COMPRESS HNSW ### +##################### + +# Delete data from prior runs +if client.schema.exists(class_name): + client.schema.delete_class(class_name) + +# START CompressHNSW +class_obj = { + "class": class_name, + "vectorIndexType": "hnsw", + "vectorIndexConfig": { + "distance_metric": "cosine", + "pq": { + "enabled": True, + }, + }, +} + +client.schema.create_class(class_obj) +# END CompressHNSW + +class_response = client.schema.get() +schema_response = client.schema.get(class_name) + +classes = [] +for c in class_response["classes"]: + classes.append(c["class"]) +assert class_name in classes, "Class missing" + +correct_index = False +if (schema_response["class"] == class_name) and ( + schema_response["vectorIndexType"] == "hnsw" +): + correct_index = True +assert correct_index, "Wrong index type" + # ############################## # ### ENABLE HNSW - MULTIPLE ### # ############################## @@ -171,6 +209,42 @@ assert correct_index, "Wrong index type" +##################### +### COMPRESS FLAT ### +##################### + +# Delete data from prior runs +if client.schema.exists(class_name): + client.schema.delete_class(class_name) + +# START CompressFlat +class_obj = { + "class": class_name, + "vectorIndexType": "flat", + "vectorIndexConfig": { + "bq": {"enabled": True} + }, +} + +client.schema.create_class(class_obj) +# END CompressFlat + +class_response = client.schema.get() +schema_response = client.schema.get(class_name) + +classes = [] +for c in class_response["classes"]: + classes.append(c["class"]) +assert class_name in classes, "Class missing" + +correct_index = False +if (schema_response["class"] == class_name) and ( + schema_response["vectorIndexType"] == "flat" +): + correct_index = True + +assert correct_index, "Wrong index type" + ###################### ### ENABLE DYNAMIC ### ###################### diff --git a/_includes/code/howto/indexes/indexes-v3.ts b/_includes/code/howto/indexes/indexes-v3.ts index 1f73d06009..317f6e470d 100644 --- a/_includes/code/howto/indexes/indexes-v3.ts +++ b/_includes/code/howto/indexes/indexes-v3.ts @@ -40,6 +40,53 @@ async function createHNSWCollection(client: WeaviateClient, collectionName: stri } // END EnableHNSW +//////////////////// +// CONFIGURE HNSW // +//////////////////// + +// START ConfigHNSW +// Add this import line: +// import { vectorizer, dataType, configure } from 'weaviate-client'; + +async function configureHNSWCollection(client: WeaviateClient, collectionName: string){ + await client.collections.create({ + name: collectionName, + vectorizers: vectorizer.text2VecOpenAI({ + vectorIndexConfig: configure.vectorIndex.hnsw({ + distanceMetric: 'cosine', + efConstruction: 256, // Dynamic list size during construction + maxConnections: 128, // Maximum number of connections per node + ef: -1, // Dynamic list size during search; -1 enables dynamic Ef + dynamicEfFactor: 15, // Multiplier for dynamic Ef + dynamicEfMin: 200, // Minimum threshold for dynamic Ef + dynamicEfMax: 1000, // Maximum threshold for dynamic Ef + quantizer: configure.vectorIndex.quantizer.pq() // Compression + }), + }), + }) +} +// END ConfigHNSW + +//////////////////// +// COMPRESS HNSW // +//////////////////// + +// START CompressHNSW +// Add this import line: +// import { vectorizer, dataType, configure } from 'weaviate-client'; + +async function compressHNSWCollection(client: WeaviateClient, collectionName: string){ + await client.collections.create({ + name: collectionName, + vectorizers: vectorizer.text2VecOpenAI({ + vectorIndexConfig: configure.vectorIndex.hnsw({ + quantizer: configure.vectorIndex.quantizer.pq() // Compression + }), + }), + }) +} +// END CompressHNSW + ////////////////////////////// /// ENABLE HNSW - MULTIPLE /// ////////////////////////////// @@ -82,17 +129,60 @@ async function createFlatCollection(client: WeaviateClient, collectionName: stri await client.collections.create({ name: collectionName, vectorizers: vectorizer.text2VecOpenAI({ - vectorIndexConfig: configure.vectorIndex.flat({ - quantizer: configure.vectorIndex.quantizer.bq({ - rescoreLimit: 200, - cache: true - }), + vectorIndexConfig: configure.vectorIndex.flat(), }), - }) + // Configure properties }) } // END EnableFlat +//////////////////// +// CONFIGURE FLAT // +//////////////////// + +// START ConfigFlat +// Add this import line +// import { vectorizer, dataType, configure } from 'weaviate-client'; + +async function configureFlatCollection(client: WeaviateClient, collectionName: string){ + await client.collections.create({ + name: collectionName, + vectorizers: vectorizer.text2VecOpenAI({ + vectorIndexConfig: configure.vectorIndex.flat({ + distanceMetric: 'cosine', + vectorCacheMaxObjects: 1000000, + quantizer: configure.vectorIndex.quantizer.bq(), + }), + }) + // Configure properties + }) +} +// END ConfigFlat + +//////////////////// +// COMPRESS FLAT // +/////////////////// + +// START CompressFlat +// Add this import line +// import { vectorizer, dataType, configure } from 'weaviate-client'; + +async function compressFlatCollection(client: WeaviateClient, collectionName: string){ + await client.collections.create({ + name: collectionName, + vectorizers: vectorizer.text2VecOpenAI({ + vectorIndexConfig: configure.vectorIndex.flat({ + quantizer: configure.vectorIndex.quantizer.bq({ + rescoreLimit: 200, + cache: true + }), + }), + }) + // Configure properties + }) +} +// END CompressFlat + //////////////////// // ENABLE DYNAMIC // //////////////////// @@ -169,6 +259,18 @@ async function main(){ // createHNSWCollection(client, collectionName); // } + // // Run configure HNSW collection code + // deleteCollection(client, collectionName) + // if(await client.collections.get(collectionName).exists() != true){ + // configureHNSWCollection(client, collectionName); + // } + + // Run compress HNSW collection code + deleteCollection(client, collectionName) + if(await client.collections.get(collectionName).exists() != true){ + compressHNSWCollection(client, collectionName); + } + // // Run multiple named vector collection code // deleteCollection(client, collectionName) // if(await client.collections.get(collectionName).exists() != true){ @@ -181,17 +283,29 @@ async function main(){ // createFlatCollection(client, collectionName); // } + // // Run configure Flat collection code + // deleteCollection(client, collectionName) + // if(await client.collections.get(collectionName).exists() != true){ + // configureFlatCollection(client, collectionName); + // } + + // // Run compress Flat collection code + // deleteCollection(client, collectionName) + // if(await client.collections.get(collectionName).exists() != true){ + // compressFlatCollection(client, collectionName); + // } + // // Run dynamic collection code // deleteCollection(client, collectionName) // if(await client.collections.get(collectionName).exists() != true){ // createDynamicCollection(client, collectionName); // } - // Run inverted collection code - deleteCollection(client, collectionName) - if(await client.collections.get(collectionName).exists() != true){ - createInvertedCollection(client, collectionName); - } + // // Run inverted collection code + // deleteCollection(client, collectionName) + // if(await client.collections.get(collectionName).exists() != true){ + // createInvertedCollection(client, collectionName); + // } } main() diff --git a/_includes/code/howto/indexes/indexes-v4.py b/_includes/code/howto/indexes/indexes-v4.py index 6e37782ef4..53be130212 100644 --- a/_includes/code/howto/indexes/indexes-v4.py +++ b/_includes/code/howto/indexes/indexes-v4.py @@ -83,6 +83,35 @@ def print_response_iter(title, response): str(schema_response.vector_index_type) == "VectorIndexType.HNSW" ), "Wrong index type" +##################### +### COMPRESS HNSW ### +##################### + +# Delete data from prior runs +if client.collections.exists(collection_name): + client.collections.delete(collection_name) + +# START CompressHNSW +from weaviate.classes.config import Configure, VectorDistances + +client.collections.create( + name=collection_name, + vector_index_config=Configure.VectorIndex.hnsw( + quantizer=Configure.VectorIndex.Quantizer.pq(), # Quantizer configuration + ), + # Configure properties, vectorizer +) +# END CompressHNSW + +collection = client.collections.get(collection_name) +collections_response = client.collections.list_all() +schema_response = collection.config.get() + +assert collection_name in collections_response.keys(), "Collection missing" +assert ( + str(schema_response.vector_index_type) == "VectorIndexType.HNSW" +), "Wrong index type" + ############################## ### ENABLE HNSW - MULTIPLE ### ############################## @@ -193,6 +222,35 @@ def print_response_iter(title, response): str(schema_response.vector_index_type) == "VectorIndexType.FLAT" ), "Wrong index type" +################### +### COMPRESS FLAT ### +################### + +# Delete data from prior runs +if client.collections.exists(collection_name): + client.collections.delete(collection_name) + +# START CompressFlat +from weaviate.classes.config import Configure, VectorDistances + +client.collections.create( + name=collection_name, + vector_index_config=Configure.VectorIndex.flat( + quantizer=Configure.VectorIndex.Quantizer.bq() + ), + # Configure properties, vectorizer +) +# END CompressFlat + +collection = client.collections.get(collection_name) +collections_response = client.collections.list_all() +schema_response = collection.config.get() + +assert collection_name in collections_response.keys(), "Collection missing" +assert ( + str(schema_response.vector_index_type) == "VectorIndexType.FLAT" +), "Wrong index type" + ###################### ### ENABLE DYNAMIC ### ###################### diff --git a/developers/weaviate/configuration/indexing/flat-indexes.md b/developers/weaviate/configuration/indexing/flat-indexes.md index 641cbf9d00..3d5afeaf2f 100644 --- a/developers/weaviate/configuration/indexing/flat-indexes.md +++ b/developers/weaviate/configuration/indexing/flat-indexes.md @@ -115,6 +115,45 @@ import IndexCompression from '/_includes/indexes/index-compression.mdx'; +### Enable compression + +Enable compression on a flat index: + + + + + + + + + + + + + + + + ## Related pages - [Indexes overview](/developers/weaviate/starter-guides/managing-resources/indexing) diff --git a/developers/weaviate/configuration/indexing/hnsw-indexes.md b/developers/weaviate/configuration/indexing/hnsw-indexes.md index ac9bf91585..b94057a13b 100644 --- a/developers/weaviate/configuration/indexing/hnsw-indexes.md +++ b/developers/weaviate/configuration/indexing/hnsw-indexes.md @@ -116,6 +116,45 @@ import IndexCompression from '/_includes/indexes/index-compression.mdx'; +### Enable compression + +Enable compression on an HNSW index: + + + + + + + + + + + + + + + + ## Related pages - [Indexes overview](/developers/weaviate/starter-guides/managing-resources/indexing) From 20cc13c37fda70f29a30b71c3b847d85fe01f1b3 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Tue, 17 Sep 2024 15:14:07 -0400 Subject: [PATCH 37/67] start inverted --- .../code/howto/indexes/indexes-inverted-v2.ts | 91 ++++++++++++ .../code/howto/indexes/indexes-inverted-v3.py | 65 +++++++++ .../code/howto/indexes/indexes-inverted-v3.ts | 84 ++++++++++++ .../code/howto/indexes/indexes-inverted-v4.py | 68 +++++++++ _includes/code/howto/indexes/indexes-v2.ts | 12 +- _includes/code/howto/indexes/indexes-v3.ts | 2 +- .../indexing/inverted-indexes.md | 129 ++++++++++++++++-- 7 files changed, 432 insertions(+), 19 deletions(-) create mode 100644 _includes/code/howto/indexes/indexes-inverted-v2.ts create mode 100644 _includes/code/howto/indexes/indexes-inverted-v3.py create mode 100644 _includes/code/howto/indexes/indexes-inverted-v3.ts create mode 100644 _includes/code/howto/indexes/indexes-inverted-v4.py diff --git a/_includes/code/howto/indexes/indexes-inverted-v2.ts b/_includes/code/howto/indexes/indexes-inverted-v2.ts new file mode 100644 index 0000000000..635b0fb94a --- /dev/null +++ b/_includes/code/howto/indexes/indexes-inverted-v2.ts @@ -0,0 +1,91 @@ +// TODO: Configure as part of the test harness +// TODO: Needs tests + +// Imports +import weaviate, { WeaviateClient } from 'weaviate-ts-client'; + +// Create client connection +function getClient(){ + const client: WeaviateClient = weaviate.client({ + scheme: 'http', + host: 'localhost:8080', + }); + + return client; +} + +// Delete pre-existing collections +function deleteClass(client, className: string){ + try { + client.schema.classDeleter().withClassName(className).do(); + } catch (e) { + // ignore error if class doesn't exist + } + return true +} + +////////////////////// +// Inverted Indexes // +////////////////////// + +// START PropIndex +async function createInvertedCollection(client: WeaviateClient, className: string){ + const invertedIndexSettings = { + class: className, + vectorizer: 'text2vec-huggingface', + properties: [ + { + name: 'textProperty', + dataType: ['text'], + indexFilterable: true, + indexSearchable: true, + moduleConfig: { + 'text2vec-huggingface': {}, + }, + }, + { + name: 'numericProperty', + dataType: ['int'], + indexRangeFilters: true, + }, + ], + invertedIndexConfig: { + bm25: { + b: 0.7, + k1: 1.25 + }, + indexTimestamps: true, + indexNullState: true, + indexPropertyLength: true + } +}; + +// Add the class to the schema +const result = await client.schema + .classCreator() + .withClass(invertedIndexSettings) + .do(); +} +// END PropIndex + +///////////////////////////// +/// AVOID TOP LEVEL AWAIT /// +///////////////////////////// + +// Main +async function main(){ + const className = "ConfigCollection"; + + const client = await getClient(); + deleteClass(client, className) + + // Only safe to run one at a time due to aynsc code + + // // Run inverted collection code + // deleteClass(client, className) + // if(await client.schema.exists(className) != true){ + // createInvertedCollection(client, className); + // } +} + +main() \ No newline at end of file diff --git a/_includes/code/howto/indexes/indexes-inverted-v3.py b/_includes/code/howto/indexes/indexes-inverted-v3.py new file mode 100644 index 0000000000..5626567cb4 --- /dev/null +++ b/_includes/code/howto/indexes/indexes-inverted-v3.py @@ -0,0 +1,65 @@ +# TODO: Configure as part of the test harness + +class_name = "ConfigCollection" + +######################## +### CLIENT CONNECTION ## +######################## + +import os +import weaviate + +client = weaviate.Client( + url="http://localhost:8080", + additional_headers={"X-Cohere-Api-Key": os.getenv("COHERE_API_KEY")}, +) + +######################## +### Inverted Indexes ### +######################## + +# Delete data from prior runs +if client.schema.exists(class_name): + client.schema.delete_class(class_name) + +# START PropIndex +class_obj = { + "class": class_name, + "vectorizer": "text2vec-huggingface", + "properties": [ + { + "name": "title", + "dataType": ["text"], + "indexFilterable": True, + "indexSearchable": True, + "moduleConfig": { + "text2vec-huggingface": {} + } + }, + { + "name": "chunk", + "dataType": ["int"], + "indexRangeFilters": True, + }, + ], + "invertedIndexConfig": { + "bm25": { + "b": 0.7, + "k1": 1.25 + }, + "indexTimestamps": True, + "indexNullState": True, + "indexPropertyLength": True + } +} + +client.schema.create_class(class_obj) +# END PropIndex + +class_response = client.schema.get() +schema_response = client.schema.get(class_name) + +classes = [] +for c in class_response["classes"]: + classes.append(c["class"]) +assert class_name in classes, "Class missing" \ No newline at end of file diff --git a/_includes/code/howto/indexes/indexes-inverted-v3.ts b/_includes/code/howto/indexes/indexes-inverted-v3.ts new file mode 100644 index 0000000000..f856643670 --- /dev/null +++ b/_includes/code/howto/indexes/indexes-inverted-v3.ts @@ -0,0 +1,84 @@ +// TODO: Configure as part of the test harness +// TODO: Needs tests + +// Imports +import weaviate, { dataType, WeaviateClient,vectorizer, configure } from 'weaviate-client'; + +// Delete pre-existing collections +function deleteCollection(client: WeaviateClient, collectionName: string){ + try { + client.collections.delete(collectionName) +} catch (e) { + // ignore error if class doesn't exist +} + +return true +} + +// Create client connection +async function getClient(){ + const client: WeaviateClient = weaviate.connectToLocal(); + + return client; +} + +////////////////////// +// Inverted Indexes // +////////////////////// + +// START PropIndex +// Add this import line +// import { dataType } from 'weaviate-client'; + +async function createInvertedCollection(client: WeaviateClient, collectionName: string){ + await client.collections.create({ + name: collectionName, + properties: [ + { + name: 'TextProperty', + dataType: dataType.TEXT, + indexFilterable: true, + indexSearchable: true, + }, + { + name: 'NumericProperty', + dataType: dataType.INT, + indexRangeFilters: true, + }, + ], + invertedIndex: { + bm25: { + b: 0.7, + k1: 1.25 + }, + indexNullState: true, + indexPropertyLength: true, + indexTimestamps: true + } + }) +} +// END PropIndex + +///////////////////////////// +/// AVOID TOP LEVEL AWAIT /// +///////////////////////////// + +// Main +async function main(){ + const collectionName = "ConfigCollection"; + + const client = await getClient(); + + // Clean up from earlier runs + deleteCollection(client, collectionName) + + // Only safe to run one at a time due to aynsc code + + // // Run inverted collection code + // deleteCollection(client, collectionName) + // if(await client.collections.get(collectionName).exists() != true){ + // createInvertedCollection(client, collectionName); + // } +} + +main() diff --git a/_includes/code/howto/indexes/indexes-inverted-v4.py b/_includes/code/howto/indexes/indexes-inverted-v4.py new file mode 100644 index 0000000000..a2949519d7 --- /dev/null +++ b/_includes/code/howto/indexes/indexes-inverted-v4.py @@ -0,0 +1,68 @@ +# TODO: Configure as part of the test harness + +collection_name = "ConfigCollection" + +def print_response(title, response): + print(f"{title}: {response}") + +def print_response_iter(title, response): + for r in response: + print(f"{title}: {r}") + +######################## +### CLIENT CONNECTION ## +######################## + +import os +import weaviate + +cohere_api_key = os.environ["COHERE_API_KEY"] + +client = weaviate.connect_to_local(headers={"X-Cohere-Api-Key": cohere_api_key}) + +######################## +### Inverted Indexes ### +######################## + +# Delete data from prior runs +if client.collections.exists(collection_name): + client.collections.delete(collection_name) + +# START PropIndex +from weaviate.classes.config import Configure, Property, DataType + +client.collections.create( + name=collection_name, + properties=[ + Property( + name="TextProperty", + data_type=DataType.TEXT, + index_filterable=True, + index_searchable=True, + ), + Property( + name="NumericProperty", + data_type=DataType.INT, + index_range_filters=True, + ), + ], + inverted_index_config=Configure.inverted_index( + bm25_b=0.7, + bm25_k1=1.25, + index_null_state=True, + index_property_length=True, + index_timestamps=True + ) +) +# END PropIndex + +collection = client.collections.get(collection_name) +collections_response = client.collections.list_all() + +assert collection_name in collections_response.keys(), "Collection missing" + +################ +### CLEAN UP ### +################ + +client.close() diff --git a/_includes/code/howto/indexes/indexes-v2.ts b/_includes/code/howto/indexes/indexes-v2.ts index da8033caaf..e158de8b79 100644 --- a/_includes/code/howto/indexes/indexes-v2.ts +++ b/_includes/code/howto/indexes/indexes-v2.ts @@ -263,7 +263,7 @@ async function main(){ const client = await getClient(); deleteClass(client, className) - // Only one create can run at a time due to aynsc code + // Only safe to run one at a time due to aynsc code // // Run enable HNSW collection code // deleteClass(client, className) @@ -295,11 +295,11 @@ async function main(){ // configureFlatCollection(client, className); // } - // Run compress flat collection code - deleteClass(client, className) - if(await client.schema.exists(className) != true){ - compressFlatCollection(client, className); - } + // // Run compress flat collection code + // deleteClass(client, className) + // if(await client.schema.exists(className) != true){ + // compressFlatCollection(client, className); + // } // // Run enable dynamic collection code // deleteClass(client, className) diff --git a/_includes/code/howto/indexes/indexes-v3.ts b/_includes/code/howto/indexes/indexes-v3.ts index 317f6e470d..3e2167b687 100644 --- a/_includes/code/howto/indexes/indexes-v3.ts +++ b/_includes/code/howto/indexes/indexes-v3.ts @@ -251,7 +251,7 @@ async function main(){ // Clean up from earlier runs deleteCollection(client, collectionName) - // Only one create can run at a time due to aynsc code + // Only safe to run one at a time due to aynsc code // // Run enable HNSW collection code // deleteCollection(client, collectionName) diff --git a/developers/weaviate/configuration/indexing/inverted-indexes.md b/developers/weaviate/configuration/indexing/inverted-indexes.md index c9b13b4977..dfe8b7fc43 100644 --- a/developers/weaviate/configuration/indexing/inverted-indexes.md +++ b/developers/weaviate/configuration/indexing/inverted-indexes.md @@ -27,54 +27,159 @@ The `indexSearchable` index improves property search times. This index is enable If you don't anticipate searching on a property field, you can disable this index to save disk space and import time. The property is still searchable. The search is just less efficient. + + + + + + + + + + + + + + + ## indexFilterable The `indexFilterable` index improves [filtering](/developers/weaviate/search/filters). This index is enabled by default. If you don't anticipate searching on a property field, you can disable this index to save disk space and import time. The property is still filterable. +Set these indexes on the property level. + + + + + + + + + + + + + + + + ## indexRangeFilters The `indexRangeFilters` is a range-based index for filtering by [numerical ranges](/developers/weaviate/release-notes/release_1_26#improved-range-queries). This index is not enabled by default. -## bm25 index +Set these indexes on the property level. -The [`bm25` index](/developers/weaviate/config-refs/schema#bm25) is configured on the collection level. These collection-wide settings can be overwritten on the property level. + + + + + + + + + + + + + + -## Configuration example +## bm25 index -Set these indexes on the property level. +The [`bm25` index](/developers/weaviate/config-refs/schema#bm25) is configured on the collection level. These collection-wide settings can be overwritten on the property level. From ad31e2306289c5f21ae9334c25d40521ba2a9302 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Tue, 17 Sep 2024 16:26:54 -0400 Subject: [PATCH 38/67] py3 --- .../code/howto/indexes/indexes-inverted-v3.py | 127 ++++++++++++++++-- _includes/code/howto/indexes/indexes-v2.ts | 50 ------- _includes/code/howto/indexes/indexes-v3.py | 50 ------- _includes/code/howto/indexes/indexes-v3.ts | 42 ------ _includes/code/howto/indexes/indexes-v4.py | 41 ------ .../indexing/inverted-indexes.md | 53 +++++++- 6 files changed, 162 insertions(+), 201 deletions(-) diff --git a/_includes/code/howto/indexes/indexes-inverted-v3.py b/_includes/code/howto/indexes/indexes-inverted-v3.py index 5626567cb4..b34e7646e5 100644 --- a/_includes/code/howto/indexes/indexes-inverted-v3.py +++ b/_includes/code/howto/indexes/indexes-inverted-v3.py @@ -14,15 +14,47 @@ additional_headers={"X-Cohere-Api-Key": os.getenv("COHERE_API_KEY")}, ) -######################## -### Inverted Indexes ### -######################## +############## +### SEARCH ### +############## + +# Delete data from prior runs +if client.schema.exists(class_name): + client.schema.delete_class(class_name) + +# START SearchIndex +class_obj = { + "class": class_name, + "vectorizer": "text2vec-huggingface", + "properties": [ + { + "name": "title", + "dataType": ["text"], + "indexSearchable": False, + }, +} + +client.schema.create_class(class_obj) +# END SearchIndex + +class_response = client.schema.get() +schema_response = client.schema.get(class_name) + +classes = [] +for c in class_response["classes"]: + classes.append(c["class"]) +assert class_name in classes, "Class missing" + + +############## +### FILTER ### +############## # Delete data from prior runs if client.schema.exists(class_name): client.schema.delete_class(class_name) -# START PropIndex +# START FilterIndex class_obj = { "class": class_name, "vectorizer": "text2vec-huggingface", @@ -30,23 +62,96 @@ { "name": "title", "dataType": ["text"], - "indexFilterable": True, - "indexSearchable": True, - "moduleConfig": { - "text2vec-huggingface": {} - } + "indexFilterable": False, }, +} + +client.schema.create_class(class_obj) +# END FilterIndex + +class_response = client.schema.get() +schema_response = client.schema.get(class_name) + +classes = [] +for c in class_response["classes"]: + classes.append(c["class"]) +assert class_name in classes, "Class missing" + + +############# +### RANGE ### +############# + +# Delete data from prior runs +if client.schema.exists(class_name): + client.schema.delete_class(class_name) + +# START RangeIndex +class_obj = { + "class": class_name, + "vectorizer": "text2vec-huggingface", + "properties": [ { - "name": "chunk", + "name": "scores", "dataType": ["int"], "indexRangeFilters": True, }, ], +} + +client.schema.create_class(class_obj) +# END RangeIndex + +class_response = client.schema.get() +schema_response = client.schema.get(class_name) + +classes = [] +for c in class_response["classes"]: + classes.append(c["class"]) +assert class_name in classes, "Class missing" + +############ +### BM25 ### +############ + +# Delete data from prior runs +if client.schema.exists(class_name): + client.schema.delete_class(class_name) + +# START BM25Index +class_obj = { + "class": class_name, "invertedIndexConfig": { "bm25": { "b": 0.7, "k1": 1.25 }, + } +} + +client.schema.create_class(class_obj) +# END BM25Index + +class_response = client.schema.get() +schema_response = client.schema.get(class_name) + +classes = [] +for c in class_response["classes"]: + classes.append(c["class"]) +assert class_name in classes, "Class missing" + +######################## +### COLLECTION LEVEL ### +######################## + +# Delete data from prior runs +if client.schema.exists(class_name): + client.schema.delete_class(class_name) + +# START CollLevIndex +class_obj = { + "class": class_name, + "invertedIndexConfig": { "indexTimestamps": True, "indexNullState": True, "indexPropertyLength": True @@ -54,7 +159,7 @@ } client.schema.create_class(class_obj) -# END PropIndex +# END CollLevIndex class_response = client.schema.get() schema_response = client.schema.get(class_name) diff --git a/_includes/code/howto/indexes/indexes-v2.ts b/_includes/code/howto/indexes/indexes-v2.ts index e158de8b79..9fdb368607 100644 --- a/_includes/code/howto/indexes/indexes-v2.ts +++ b/_includes/code/howto/indexes/indexes-v2.ts @@ -208,50 +208,6 @@ async function createDynamicCollection(client: WeaviateClient, className: string } // END EnableDynamic -////////////////////// -// Inverted Indexes // -////////////////////// - -// START PropIndex -async function createInvertedCollection(client: WeaviateClient, className: string){ - const invertedIndexSettings = { - class: className, - vectorizer: 'text2vec-huggingface', - properties: [ - { - name: 'textProperty', - dataType: ['text'], - indexFilterable: true, - indexSearchable: true, - moduleConfig: { - 'text2vec-huggingface': {}, - }, - }, - { - name: 'numericProperty', - dataType: ['int'], - indexRangeFilters: true, - }, - ], - invertedIndexConfig: { - bm25: { - b: 0.7, - k1: 1.25 - }, - indexTimestamps: true, - indexNullState: true, - indexPropertyLength: true - } -}; - -// Add the class to the schema -const result = await client.schema - .classCreator() - .withClass(invertedIndexSettings) - .do(); -} -// END PropIndex - ///////////////////////////// /// AVOID TOP LEVEL AWAIT /// ///////////////////////////// @@ -306,12 +262,6 @@ async function main(){ // if(await client.schema.exists(className) != true){ // createDynamicCollection(client, className); // } - - // // Run inverted collection code - // deleteClass(client, className) - // if(await client.schema.exists(className) != true){ - // createInvertedCollection(client, className); - // } } main() \ No newline at end of file diff --git a/_includes/code/howto/indexes/indexes-v3.py b/_includes/code/howto/indexes/indexes-v3.py index a4f7706468..3206a36968 100644 --- a/_includes/code/howto/indexes/indexes-v3.py +++ b/_includes/code/howto/indexes/indexes-v3.py @@ -277,53 +277,3 @@ ): correct_index = True assert correct_index, "Wrong index type" - -######################## -### Inverted Indexes ### -######################## - -# Delete data from prior runs -if client.schema.exists(class_name): - client.schema.delete_class(class_name) - -# START PropIndex -class_obj = { - "class": class_name, - "vectorizer": "text2vec-huggingface", - "properties": [ - { - "name": "title", - "dataType": ["text"], - "indexFilterable": True, - "indexSearchable": True, - "moduleConfig": { - "text2vec-huggingface": {} - } - }, - { - "name": "chunk", - "dataType": ["int"], - "indexRangeFilters": True, - }, - ], - "invertedIndexConfig": { - "bm25": { - "b": 0.7, - "k1": 1.25 - }, - "indexTimestamps": True, - "indexNullState": True, - "indexPropertyLength": True - } -} - -client.schema.create_class(class_obj) -# END PropIndex - -class_response = client.schema.get() -schema_response = client.schema.get(class_name) - -classes = [] -for c in class_response["classes"]: - classes.append(c["class"]) -assert class_name in classes, "Class missing" \ No newline at end of file diff --git a/_includes/code/howto/indexes/indexes-v3.ts b/_includes/code/howto/indexes/indexes-v3.ts index 3e2167b687..529bc364fa 100644 --- a/_includes/code/howto/indexes/indexes-v3.ts +++ b/_includes/code/howto/indexes/indexes-v3.ts @@ -201,43 +201,6 @@ async function createDynamicCollection(client: WeaviateClient, collectionName: s } // END EnableDynamic -////////////////////// -// Inverted Indexes // -////////////////////// - -// START PropIndex -// Add this import line -// import { dataType } from 'weaviate-client'; - -async function createInvertedCollection(client: WeaviateClient, collectionName: string){ - await client.collections.create({ - name: collectionName, - properties: [ - { - name: 'TextProperty', - dataType: dataType.TEXT, - indexFilterable: true, - indexSearchable: true, - }, - { - name: 'NumericProperty', - dataType: dataType.INT, - indexRangeFilters: true, - }, - ], - invertedIndex: { - bm25: { - b: 0.7, - k1: 1.25 - }, - indexNullState: true, - indexPropertyLength: true, - indexTimestamps: true - } - }) -} -// END PropIndex - ///////////////////////////// /// AVOID TOP LEVEL AWAIT /// ///////////////////////////// @@ -301,11 +264,6 @@ async function main(){ // createDynamicCollection(client, collectionName); // } - // // Run inverted collection code - // deleteCollection(client, collectionName) - // if(await client.collections.get(collectionName).exists() != true){ - // createInvertedCollection(client, collectionName); - // } } main() diff --git a/_includes/code/howto/indexes/indexes-v4.py b/_includes/code/howto/indexes/indexes-v4.py index 53be130212..d4216faf3b 100644 --- a/_includes/code/howto/indexes/indexes-v4.py +++ b/_includes/code/howto/indexes/indexes-v4.py @@ -291,47 +291,6 @@ def print_response_iter(title, response): str(schema_response.vector_index_type) == "VectorIndexType.DYNAMIC" ), "Wrong index type" -######################## -### Inverted Indexes ### -######################## - -# Delete data from prior runs -if client.collections.exists(collection_name): - client.collections.delete(collection_name) - -# START PropIndex -from weaviate.classes.config import Configure, Property, DataType - -client.collections.create( - name=collection_name, - properties=[ - Property( - name="TextProperty", - data_type=DataType.TEXT, - index_filterable=True, - index_searchable=True, - ), - Property( - name="NumericProperty", - data_type=DataType.INT, - index_range_filters=True, - ), - ], - inverted_index_config=Configure.inverted_index( - bm25_b=0.7, - bm25_k1=1.25, - index_null_state=True, - index_property_length=True, - index_timestamps=True - ) -) -# END PropIndex - -collection = client.collections.get(collection_name) -collections_response = client.collections.list_all() - -assert collection_name in collections_response.keys(), "Collection missing" - ################ ### CLEAN UP ### ################ diff --git a/developers/weaviate/configuration/indexing/inverted-indexes.md b/developers/weaviate/configuration/indexing/inverted-indexes.md index dfe8b7fc43..e774126e50 100644 --- a/developers/weaviate/configuration/indexing/inverted-indexes.md +++ b/developers/weaviate/configuration/indexing/inverted-indexes.md @@ -8,18 +8,18 @@ image: og/docs/indexes.jpg import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock'; -import PyCodeV4 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v4.py'; -import PyCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v3.py'; -import TSCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v3.ts'; -import TSCodeV2 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v2.ts'; +import PyCodeV4 from '!!raw-loader!/_includes/code/howto/indexes/indexes-inverted-v4.py'; +import PyCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-inverted-v3.py'; +import TSCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-inverted-v3.ts'; +import TSCodeV2 from '!!raw-loader!/_includes/code/howto/indexes/indexes-inverted-v2.ts'; -Properties use [inverted indexes](/developers/weaviate/concepts/indexing#inverted-indexes), also known as keyword indexes, to make property searches more efficient. Weaviate provides different kinds to inverted index so you can match better match the index to your data: +Weaviate uses [inverted indexes](/developers/weaviate/concepts/indexing#inverted-indexes), also known as keyword indexes, to make textual and numeric searches more efficient. Weaviate provides different kinds to inverted index so you can match better match the index to your data. These indexes are normally configured on a property level: - [indexSearchable](#indexSearchable) - [indexFilterable](#indexfilterable) - [indexRangeFilters](#indexrangefilters) -Use these indexes to improve your searches. +To tune inverted indexes at the collection level, use the `invertedIndexConfig` to adjust the settings for the [BM25 inverted index](#bm25-index) and other collection level parameters. ## indexSearchable @@ -148,7 +148,7 @@ Set these indexes on the property level. ## bm25 index -The [`bm25` index](/developers/weaviate/config-refs/schema#bm25) is configured on the collection level. These collection-wide settings can be overwritten on the property level. +The [`bm25` index](/developers/weaviate/config-refs/schema#bm25) is configured on the collection level. @@ -185,6 +185,45 @@ The [`bm25` index](/developers/weaviate/config-refs/schema#bm25) is configured o +## Collection level properties + +These properties are configured on the collection level. + + + + + + + + + + + + + + + + ## Related pages - [Indexes overview](/developers/weaviate/starter-guides/managing-resources/indexing) From 8de50d36a362176af52c859cc6d7458444475e1d Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Tue, 17 Sep 2024 16:39:35 -0400 Subject: [PATCH 39/67] py4 --- .../code/howto/indexes/indexes-inverted-v3.py | 2 + .../code/howto/indexes/indexes-inverted-v4.py | 106 +++++++++++++++++- 2 files changed, 102 insertions(+), 6 deletions(-) diff --git a/_includes/code/howto/indexes/indexes-inverted-v3.py b/_includes/code/howto/indexes/indexes-inverted-v3.py index b34e7646e5..b1ff3b485c 100644 --- a/_includes/code/howto/indexes/indexes-inverted-v3.py +++ b/_includes/code/howto/indexes/indexes-inverted-v3.py @@ -32,6 +32,7 @@ "dataType": ["text"], "indexSearchable": False, }, + ] } client.schema.create_class(class_obj) @@ -64,6 +65,7 @@ "dataType": ["text"], "indexFilterable": False, }, + ] } client.schema.create_class(class_obj) diff --git a/_includes/code/howto/indexes/indexes-inverted-v4.py b/_includes/code/howto/indexes/indexes-inverted-v4.py index a2949519d7..e7594715c8 100644 --- a/_includes/code/howto/indexes/indexes-inverted-v4.py +++ b/_includes/code/howto/indexes/indexes-inverted-v4.py @@ -20,15 +20,15 @@ def print_response_iter(title, response): client = weaviate.connect_to_local(headers={"X-Cohere-Api-Key": cohere_api_key}) -######################## -### Inverted Indexes ### -######################## +############## +### SEARCH ### +############## # Delete data from prior runs if client.collections.exists(collection_name): client.collections.delete(collection_name) -# START PropIndex +# START SearchIndex from weaviate.classes.config import Configure, Property, DataType client.collections.create( @@ -37,24 +37,118 @@ def print_response_iter(title, response): Property( name="TextProperty", data_type=DataType.TEXT, - index_filterable=True, index_searchable=True, ), + ], +) +# END SearchIndex + +collection = client.collections.get(collection_name) +collections_response = client.collections.list_all() + +assert collection_name in collections_response.keys(), "Collection missing" + +############## +### FILTER ### +############## + +# Delete data from prior runs +if client.collections.exists(collection_name): + client.collections.delete(collection_name) + +# START FilterIndex +from weaviate.classes.config import Configure, Property, DataType + +client.collections.create( + name=collection_name, + properties=[ + Property( + name="TextProperty", + data_type=DataType.TEXT, + index_filterable=True, + ), + ], +) +# END FilterIndex + +collection = client.collections.get(collection_name) +collections_response = client.collections.list_all() + +assert collection_name in collections_response.keys(), "Collection missing" + +############# +### RANGE ### +############# + +# Delete data from prior runs +if client.collections.exists(collection_name): + client.collections.delete(collection_name) + +# START RangeIndex +from weaviate.classes.config import Configure, Property, DataType + +client.collections.create( + name=collection_name, + properties=[ Property( name="NumericProperty", data_type=DataType.INT, index_range_filters=True, ), ], +) +# END RangeIndex + +collection = client.collections.get(collection_name) +collections_response = client.collections.list_all() + +assert collection_name in collections_response.keys(), "Collection missing" + +############ +### BM25 ### +############ + +# Delete data from prior runs +if client.collections.exists(collection_name): + client.collections.delete(collection_name) + +# START BM25Index +from weaviate.classes.config import Configure, Property, DataType + +client.collections.create( + name=collection_name, inverted_index_config=Configure.inverted_index( bm25_b=0.7, bm25_k1=1.25, + ) +) +# END BM25Index + +collection = client.collections.get(collection_name) +collections_response = client.collections.list_all() + +assert collection_name in collections_response.keys(), "Collection missing" + +######################## +### COLLECTION LEVEL ### +######################## + +# Delete data from prior runs +if client.collections.exists(collection_name): + client.collections.delete(collection_name) + +# START CollLevIndex +from weaviate.classes.config import Configure, Property, DataType + +client.collections.create( + name=collection_name, + inverted_index_config=Configure.inverted_index( index_null_state=True, index_property_length=True, index_timestamps=True ) ) -# END PropIndex +# END CollLevIndex collection = client.collections.get(collection_name) collections_response = client.collections.list_all() From 0d44601415373aa9d785bb0ba80343ebc9cfc870 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Tue, 17 Sep 2024 16:49:29 -0400 Subject: [PATCH 40/67] ts3 --- .../code/howto/indexes/indexes-inverted-v3.ts | 139 +++++++++++++++--- 1 file changed, 117 insertions(+), 22 deletions(-) diff --git a/_includes/code/howto/indexes/indexes-inverted-v3.ts b/_includes/code/howto/indexes/indexes-inverted-v3.ts index f856643670..28124e5707 100644 --- a/_includes/code/howto/indexes/indexes-inverted-v3.ts +++ b/_includes/code/howto/indexes/indexes-inverted-v3.ts @@ -22,43 +22,113 @@ async function getClient(){ return client; } -////////////////////// -// Inverted Indexes // -////////////////////// +//////////// +// SEARCH // +//////////// -// START PropIndex +// START SearchIndex // Add this import line // import { dataType } from 'weaviate-client'; -async function createInvertedCollection(client: WeaviateClient, collectionName: string){ +async function searchInvertedCollection(client: WeaviateClient, collectionName: string){ await client.collections.create({ name: collectionName, properties: [ { name: 'TextProperty', dataType: dataType.TEXT, - indexFilterable: true, indexSearchable: true, }, - { - name: 'NumericProperty', - dataType: dataType.INT, - indexRangeFilters: true, - }, ], - invertedIndex: { - bm25: { - b: 0.7, - k1: 1.25 - }, - indexNullState: true, - indexPropertyLength: true, - indexTimestamps: true - } }) } +// END SearchIndex + +//////////// +// FILTER // +//////////// + +// START FilerIndex +// Add this import line +// import { dataType } from 'weaviate-client'; + +async function filterInvertedCollection(client: WeaviateClient, collectionName: string){ + await client.collections.create({ + name: collectionName, + properties: [ + { + name: 'TextProperty', + dataType: dataType.TEXT, + indexFilterable: true, + }, + ], + }) +} // END PropIndex +/////////// +// RANGE // +/////////// + +// START RangeIndex +// Add this import line +// import { dataType } from 'weaviate-client'; + +async function rangeInvertedCollection(client: WeaviateClient, collectionName: string){ + await client.collections.create({ + name: collectionName, + properties: [ + { + name: 'NumericProperty', + dataType: dataType.INT, + indexRangeFilters: true, + }, + ], + }) +} +// END RangeIndex + +////////// +// BM25 // +////////// + +// START BM25Index +// Add this import line +// import { dataType } from 'weaviate-client'; + +async function bm25InvertedCollection(client: WeaviateClient, collectionName: string){ + await client.collections.create({ + name: collectionName, + invertedIndex: { + bm25: { + b: 0.7, + k1: 1.25 + }, + } + }) +} +// END BM25Index + +////////////////////// +// COLLECTION LEVEL // +////////////////////// + +// START CollLevIndex +// Add this import line +// import { dataType } from 'weaviate-client'; + +async function collLevInvertedCollection(client: WeaviateClient, collectionName: string){ + await client.collections.create({ + name: collectionName, + invertedIndex: { + indexNullState: true, + indexPropertyLength: true, + indexTimestamps: true + } + }) +} +// END CollLevIndex + ///////////////////////////// /// AVOID TOP LEVEL AWAIT /// ///////////////////////////// @@ -74,11 +144,36 @@ async function main(){ // Only safe to run one at a time due to aynsc code - // // Run inverted collection code + // // Run search code + // deleteCollection(client, collectionName) + // if(await client.collections.get(collectionName).exists() != true){ + // searchInvertedCollection(client, collectionName); + // } + + // // Run filter code // deleteCollection(client, collectionName) // if(await client.collections.get(collectionName).exists() != true){ - // createInvertedCollection(client, collectionName); + // filterInvertedCollection(client, collectionName); // } + + // // Run range code + // deleteCollection(client, collectionName) + // if(await client.collections.get(collectionName).exists() != true){ + // rangeInvertedCollection(client, collectionName); + // } + + // // Run bm25 code + // deleteCollection(client, collectionName) + // if(await client.collections.get(collectionName).exists() != true){ + // bm25InvertedCollection(client, collectionName); + // } + + // Run collection level code + deleteCollection(client, collectionName) + if(await client.collections.get(collectionName).exists() != true){ + collLevInvertedCollection(client, collectionName); + } + } main() From 593b449453a22d28b593d2fc9f7f63399eb48e3f Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Tue, 17 Sep 2024 17:12:27 -0400 Subject: [PATCH 41/67] ts2 --- .../code/howto/indexes/indexes-inverted-v2.ts | 135 ++++++++++++++++-- .../code/howto/indexes/indexes-inverted-v3.ts | 4 +- 2 files changed, 123 insertions(+), 16 deletions(-) diff --git a/_includes/code/howto/indexes/indexes-inverted-v2.ts b/_includes/code/howto/indexes/indexes-inverted-v2.ts index 635b0fb94a..484ea2c2b1 100644 --- a/_includes/code/howto/indexes/indexes-inverted-v2.ts +++ b/_includes/code/howto/indexes/indexes-inverted-v2.ts @@ -24,12 +24,12 @@ function deleteClass(client, className: string){ return true } -////////////////////// -// Inverted Indexes // -////////////////////// +//////////// +// SEARCH // +/////////// -// START PropIndex -async function createInvertedCollection(client: WeaviateClient, className: string){ +// START SearchIndex +async function searchInvertedCollection(client: WeaviateClient, className: string){ const invertedIndexSettings = { class: className, vectorizer: 'text2vec-huggingface', @@ -37,23 +37,105 @@ async function createInvertedCollection(client: WeaviateClient, className: strin { name: 'textProperty', dataType: ['text'], - indexFilterable: true, indexSearchable: true, - moduleConfig: { - 'text2vec-huggingface': {}, - }, }, + ], +}; + +// Add the class to the schema +const result = await client.schema + .classCreator() + .withClass(invertedIndexSettings) + .do(); +} +// END SearchIndex + +//////////// +// FILTER // +//////////// + +// START FilterIndex +async function filterInvertedCollection(client: WeaviateClient, className: string){ + const invertedIndexSettings = { + class: className, + vectorizer: 'text2vec-huggingface', + properties: [ + { + name: 'textProperty', + dataType: ['text'], + indexFilterable: true, + }, + ], +}; + +// Add the class to the schema +const result = await client.schema + .classCreator() + .withClass(invertedIndexSettings) + .do(); +} +// END FilterIndex + +/////////// +// RANGE // +/////////// + +// START RangeIndex +async function rangeInvertedCollection(client: WeaviateClient, className: string){ + const invertedIndexSettings = { + class: className, + vectorizer: 'text2vec-huggingface', + properties: [ { name: 'numericProperty', dataType: ['int'], indexRangeFilters: true, }, ], +}; + +// Add the class to the schema +const result = await client.schema + .classCreator() + .withClass(invertedIndexSettings) + .do(); +} +// END RangeIndex + +////////// +// BM25 // +////////// + +// START BM25Index +async function bm25InvertedCollection(client: WeaviateClient, className: string){ + const invertedIndexSettings = { + class: className, + vectorizer: 'text2vec-huggingface', invertedIndexConfig: { bm25: { b: 0.7, k1: 1.25 }, +}; + +// Add the class to the schema +const result = await client.schema + .classCreator() + .withClass(invertedIndexSettings) + .do(); +} +// END BM25Index + +////////////////////// +// COLLECTION LEVEL // +////////////////////// + +// START CollLevIndex +async function collLevInvertedCollection(client: WeaviateClient, className: string){ + const invertedIndexSettings = { + class: className, + vectorizer: 'text2vec-huggingface', + invertedIndexConfig: { indexTimestamps: true, indexNullState: true, indexPropertyLength: true @@ -66,7 +148,7 @@ const result = await client.schema .withClass(invertedIndexSettings) .do(); } -// END PropIndex +// END CollLevIndex ///////////////////////////// /// AVOID TOP LEVEL AWAIT /// @@ -81,11 +163,36 @@ async function main(){ // Only safe to run one at a time due to aynsc code - // // Run inverted collection code - // deleteClass(client, className) - // if(await client.schema.exists(className) != true){ - // createInvertedCollection(client, className); + // Run search code + deleteCollection(client, collectionName) + if(await client.collections.get(collectionName).exists() != true){ + searchInvertedCollection(client, collectionName); + } + + // // Run filter code + // deleteCollection(client, collectionName) + // if(await client.collections.get(collectionName).exists() != true){ + // filterInvertedCollection(client, collectionName); // } + + // // Run range code + // deleteCollection(client, collectionName) + // if(await client.collections.get(collectionName).exists() != true){ + // rangeInvertedCollection(client, collectionName); + // } + + // // Run bm25 code + // deleteCollection(client, collectionName) + // if(await client.collections.get(collectionName).exists() != true){ + // bm25InvertedCollection(client, collectionName); + // } + + // // Run collection level code + // deleteCollection(client, collectionName) + // if(await client.collections.get(collectionName).exists() != true){ + // collLevInvertedCollection(client, collectionName); + // } + } main() \ No newline at end of file diff --git a/_includes/code/howto/indexes/indexes-inverted-v3.ts b/_includes/code/howto/indexes/indexes-inverted-v3.ts index 28124e5707..cdf1348986 100644 --- a/_includes/code/howto/indexes/indexes-inverted-v3.ts +++ b/_includes/code/howto/indexes/indexes-inverted-v3.ts @@ -48,7 +48,7 @@ async function searchInvertedCollection(client: WeaviateClient, collectionName: // FILTER // //////////// -// START FilerIndex +// START FilterIndex // Add this import line // import { dataType } from 'weaviate-client'; @@ -64,7 +64,7 @@ async function filterInvertedCollection(client: WeaviateClient, collectionName: ], }) } -// END PropIndex +// END FilterIndex /////////// // RANGE // From e1c6cd6cef5b2096421f45f82c3cf1d1e90b2ee7 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Wed, 18 Sep 2024 10:00:31 -0400 Subject: [PATCH 42/67] dynamic --- .../code/howto/indexes/indexes-dynamic-v2.ts | 66 ++++++++++++++++++ .../code/howto/indexes/indexes-dynamic-v3.py | 48 +++++++++++++ .../code/howto/indexes/indexes-dynamic-v3.ts | 66 ++++++++++++++++++ .../code/howto/indexes/indexes-dynamic-v4.py | 67 +++++++++++++++++++ .../configuration/indexing/dynamic-indexes.md | 12 ---- 5 files changed, 247 insertions(+), 12 deletions(-) create mode 100644 _includes/code/howto/indexes/indexes-dynamic-v2.ts create mode 100644 _includes/code/howto/indexes/indexes-dynamic-v3.py create mode 100644 _includes/code/howto/indexes/indexes-dynamic-v3.ts create mode 100644 _includes/code/howto/indexes/indexes-dynamic-v4.py diff --git a/_includes/code/howto/indexes/indexes-dynamic-v2.ts b/_includes/code/howto/indexes/indexes-dynamic-v2.ts new file mode 100644 index 0000000000..b7218011b3 --- /dev/null +++ b/_includes/code/howto/indexes/indexes-dynamic-v2.ts @@ -0,0 +1,66 @@ +// TODO: Configure as part of the test harness +// TODO: Needs tests + +// Imports +import weaviate, { WeaviateClient } from 'weaviate-ts-client'; + +// Create client connection +function getClient(){ + const client: WeaviateClient = weaviate.client({ + scheme: 'http', + host: 'localhost:8080', + }); + + return client; +} + +// Delete pre-existing collections +function deleteClass(client, className: string){ + try { + client.schema.classDeleter().withClassName(className).do(); + } catch (e) { + // ignore error if class doesn't exist + } + return true +} + +//////////////////// +// ENABLE DYNAMIC // +//////////////////// + +// START EnableDynamic +async function createDynamicCollection(client: WeaviateClient, className: string){ + + const setIndexType = { + class: className, + // Add property definitions + vectorizer: 'text2vec-openai', + vectorIndexType: 'dynamic', + }; + + // Add the class to the schema + await client.schema.classCreator().withClass(setIndexType).do(); +} +// END EnableDynamic + +///////////////////////////// +/// AVOID TOP LEVEL AWAIT /// +///////////////////////////// + +// Main +async function main(){ + const className = "ConfigCollection"; + + const client = await getClient(); + deleteClass(client, className) + + // Only safe to run one at a time due to aynsc code + + // // Run enable dynamic collection code + // deleteClass(client, className) + // if(await client.schema.exists(className) != true){ + // createDynamicCollection(client, className); + // } +} + +main() \ No newline at end of file diff --git a/_includes/code/howto/indexes/indexes-dynamic-v3.py b/_includes/code/howto/indexes/indexes-dynamic-v3.py new file mode 100644 index 0000000000..458fc79d34 --- /dev/null +++ b/_includes/code/howto/indexes/indexes-dynamic-v3.py @@ -0,0 +1,48 @@ +# TODO: Configure as part of the test harness + +class_name = "ConfigCollection" + +######################## +### CLIENT CONNECTION ## +######################## + +import os +import weaviate + +client = weaviate.Client( + url="http://localhost:8080", + additional_headers={"X-Cohere-Api-Key": os.getenv("COHERE_API_KEY")}, +) + +###################### +### ENABLE DYNAMIC ### +###################### + +# Delete data from prior runs +if client.schema.exists(class_name): + client.schema.delete_class(class_name) + +# START EnableDynamic +class_obj = { + "class": class_name, + # Additional configuration not shown + "vectorIndexType": "dynamic", +} + +client.schema.create_class(class_obj) +# END EnableDynamic + +class_response = client.schema.get() +schema_response = client.schema.get(class_name) + +classes = [] +for c in class_response["classes"]: + classes.append(c["class"]) +assert class_name in classes, "Class missing" + +correct_index = False +if (schema_response["class"] == class_name) and ( + schema_response["vectorIndexType"] == "dynamic" +): + correct_index = True +assert correct_index, "Wrong index type" diff --git a/_includes/code/howto/indexes/indexes-dynamic-v3.ts b/_includes/code/howto/indexes/indexes-dynamic-v3.ts new file mode 100644 index 0000000000..f52461aa81 --- /dev/null +++ b/_includes/code/howto/indexes/indexes-dynamic-v3.ts @@ -0,0 +1,66 @@ +// TODO: Configure as part of the test harness +// TODO: Needs tests + +// Imports +import weaviate, { dataType, WeaviateClient,vectorizer, configure } from 'weaviate-client'; + +// Delete pre-existing collections +function deleteCollection(client: WeaviateClient, collectionName: string){ + try { + client.collections.delete(collectionName) +} catch (e) { + // ignore error if class doesn't exist +} + +return true +} + +// Create client connection +async function getClient(){ + const client: WeaviateClient = weaviate.connectToLocal(); + + return client; +} + +//////////////////// +// ENABLE DYNAMIC // +//////////////////// + +// START EnableDynamic +// Add this import line +// import { vectorizer, dataType, configure } from 'weaviate-client'; + +async function createDynamicCollection(client: WeaviateClient, collectionName: string){ + await client.collections.create({ + name: collectionName, + vectorizers: vectorizer.text2VecOpenAI({ + vectorIndexConfig: configure.vectorIndex.dynamic(), + }), + }) +} +// END EnableDynamic + +///////////////////////////// +/// AVOID TOP LEVEL AWAIT /// +///////////////////////////// + +// Main +async function main(){ + const collectionName = "ConfigCollection"; + + const client = await getClient(); + + // Clean up from earlier runs + deleteCollection(client, collectionName) + + // Only safe to run one at a time due to aynsc code + + // // Run dynamic collection code + // deleteCollection(client, collectionName) + // if(await client.collections.get(collectionName).exists() != true){ + // createDynamicCollection(client, collectionName); + // } + +} + +main() diff --git a/_includes/code/howto/indexes/indexes-dynamic-v4.py b/_includes/code/howto/indexes/indexes-dynamic-v4.py new file mode 100644 index 0000000000..030aea46e5 --- /dev/null +++ b/_includes/code/howto/indexes/indexes-dynamic-v4.py @@ -0,0 +1,67 @@ +# TODO: Configure as part of the test harness + +collection_name = "ConfigCollection" + +def print_response(title, response): + print(f"{title}: {response}") + +def print_response_iter(title, response): + for r in response: + print(f"{title}: {r}") + +######################## +### CLIENT CONNECTION ## +######################## + +import os +import weaviate + +cohere_api_key = os.environ["COHERE_API_KEY"] + +client = weaviate.connect_to_local(headers={"X-Cohere-Api-Key": cohere_api_key}) + +###################### +### ENABLE DYNAMIC ### +###################### + +# Delete data from prior runs +if client.collections.exists(collection_name): + client.collections.delete(collection_name) + +# START EnableDynamic +from weaviate.classes.config import Configure, VectorDistances + +client.collections.create( + name=collection_name, + vector_index_config=Configure.VectorIndex.dynamic( + distance_metric=VectorDistances.COSINE, + threshold=20000, + hnsw=Configure.VectorIndex.hnsw( + # Any hnsw configuration parameters + dynamic_ef_factor=15, # Multiplier for dynamic Ef + dynamic_ef_min=200, # Minimum threshold for dynamic Ef + dynamic_ef_max=1000, # Maximum threshold for dynamic Ef + ), + flat=Configure.VectorIndex.flat( + # Any flat index configuration parameters + vector_cache_max_objects=100000, + quantizer=Configure.VectorIndex.Quantizer.bq() + ), + ) +) +# END EnableDynamic + +collection = client.collections.get(collection_name) +collections_response = client.collections.list_all() +schema_response = collection.config.get() + +assert collection_name in collections_response.keys(), "Collection missing" +assert ( + str(schema_response.vector_index_type) == "VectorIndexType.DYNAMIC" +), "Wrong index type" + +################ +### CLEAN UP ### +################ + +client.close() diff --git a/developers/weaviate/configuration/indexing/dynamic-indexes.md b/developers/weaviate/configuration/indexing/dynamic-indexes.md index f67008020a..adab7ea703 100644 --- a/developers/weaviate/configuration/indexing/dynamic-indexes.md +++ b/developers/weaviate/configuration/indexing/dynamic-indexes.md @@ -63,18 +63,6 @@ See also: - [Dynamic index parameters](/developers/weaviate/config-refs/schema/vector-index#dynamic-index-parameters) -## Multiple named vectors - -import MultiNameVec from '/_includes/indexes/multiple-named-vectors.mdx'; - - - -## Compression - -import IndexCompression from '/_includes/indexes/index-compression.mdx'; - - - ## Asynchronous indexing import AsynchIndexing from '/_includes/indexes/async-indexing.mdx'; From 0ab9af97b47b51c51e315df7ecc6d8a8b9799fc6 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Wed, 18 Sep 2024 10:04:09 -0400 Subject: [PATCH 43/67] dynamic --- .../code/howto/indexes/indexes-dynamic-v4.py | 40 ++++++++++++++++ .../configuration/indexing/dynamic-indexes.md | 47 +++++++++++++++++-- 2 files changed, 83 insertions(+), 4 deletions(-) diff --git a/_includes/code/howto/indexes/indexes-dynamic-v4.py b/_includes/code/howto/indexes/indexes-dynamic-v4.py index 030aea46e5..e512a594e4 100644 --- a/_includes/code/howto/indexes/indexes-dynamic-v4.py +++ b/_includes/code/howto/indexes/indexes-dynamic-v4.py @@ -60,6 +60,46 @@ def print_response_iter(title, response): str(schema_response.vector_index_type) == "VectorIndexType.DYNAMIC" ), "Wrong index type" +###################### +### ENABLE DYNAMIC ### +###################### + +# Delete data from prior runs +if client.collections.exists(collection_name): + client.collections.delete(collection_name) + +# START ConfigDynamic +from weaviate.classes.config import Configure, VectorDistances + +client.collections.create( + name=collection_name, + vector_index_config=Configure.VectorIndex.dynamic( + distance_metric=VectorDistances.COSINE, + threshold=20000, + hnsw=Configure.VectorIndex.hnsw( + # Any hnsw configuration parameters + dynamic_ef_factor=15, # Multiplier for dynamic Ef + dynamic_ef_min=200, # Minimum threshold for dynamic Ef + dynamic_ef_max=1000, # Maximum threshold for dynamic Ef + ), + flat=Configure.VectorIndex.flat( + # Any flat index configuration parameters + vector_cache_max_objects=100000, + quantizer=Configure.VectorIndex.Quantizer.bq() + ), + ) +) +# END ConfigDynamic + +collection = client.collections.get(collection_name) +collections_response = client.collections.list_all() +schema_response = collection.config.get() + +assert collection_name in collections_response.keys(), "Collection missing" +assert ( + str(schema_response.vector_index_type) == "VectorIndexType.DYNAMIC" +), "Wrong index type" + ################ ### CLEAN UP ### ################ diff --git a/developers/weaviate/configuration/indexing/dynamic-indexes.md b/developers/weaviate/configuration/indexing/dynamic-indexes.md index adab7ea703..a577fca884 100644 --- a/developers/weaviate/configuration/indexing/dynamic-indexes.md +++ b/developers/weaviate/configuration/indexing/dynamic-indexes.md @@ -8,10 +8,10 @@ image: og/docs/indexes.jpg import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock'; -import PyCodeV4 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v4.py'; -import PyCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v3.py'; -import TSCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v3.ts'; -import TSCodeV2 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v2.ts'; +import PyCodeV4 from '!!raw-loader!/_includes/code/howto/indexes/indexes-dynamic-v4.py'; +import PyCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-dynamic-v3.py'; +import TSCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-dynamic-v3.ts'; +import TSCodeV2 from '!!raw-loader!/_includes/code/howto/indexes/indexes-dynamic-v2.ts'; import DynamicIntro from '/_includes/indexes/dynamic-intro.mdx'; @@ -24,6 +24,8 @@ import ConsiderComp from '/_includes/indexes/consider-compression.mdx'; +## Enable dynamic indexes + +## Configure dynamic indexes + + + + + + + + + + + + + + + + See also: - [Dynamic index parameters](/developers/weaviate/config-refs/schema/vector-index#dynamic-index-parameters) From fa071f08b1fafb515b2d633a182b11b4d3324df0 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Wed, 18 Sep 2024 10:16:33 -0400 Subject: [PATCH 44/67] dynamic --- .../code/howto/indexes/indexes-dynamic-v2.ts | 26 ++++++++++++++ .../code/howto/indexes/indexes-dynamic-v3.py | 34 +++++++++++++++++++ .../code/howto/indexes/indexes-dynamic-v3.ts | 28 +++++++++++++-- .../code/howto/indexes/indexes-dynamic-v4.py | 20 +++-------- .../configuration/indexing/dynamic-indexes.md | 5 ++- 5 files changed, 94 insertions(+), 19 deletions(-) diff --git a/_includes/code/howto/indexes/indexes-dynamic-v2.ts b/_includes/code/howto/indexes/indexes-dynamic-v2.ts index b7218011b3..2afdb05fec 100644 --- a/_includes/code/howto/indexes/indexes-dynamic-v2.ts +++ b/_includes/code/howto/indexes/indexes-dynamic-v2.ts @@ -43,6 +43,25 @@ async function createDynamicCollection(client: WeaviateClient, className: string } // END EnableDynamic +//////////////////// +// CONFIGURE DYNAMIC // +//////////////////// + +// START ConfigDynamic +async function configureDynamicCollection(client: WeaviateClient, className: string){ + + const setIndexType = { + class: className, + // Add property definitions + vectorizer: 'text2vec-openai', + vectorIndexType: 'dynamic', + }; + + // Add the class to the schema + await client.schema.classCreator().withClass(setIndexType).do(); +} +// END ConfigDynamic + ///////////////////////////// /// AVOID TOP LEVEL AWAIT /// ///////////////////////////// @@ -61,6 +80,13 @@ async function main(){ // if(await client.schema.exists(className) != true){ // createDynamicCollection(client, className); // } + + // // Run configure dynamic collection code + // deleteClass(client, className) + // if(await client.schema.exists(className) != true){ + // configureDynamicCollection(client, className); + // } + } main() \ No newline at end of file diff --git a/_includes/code/howto/indexes/indexes-dynamic-v3.py b/_includes/code/howto/indexes/indexes-dynamic-v3.py index 458fc79d34..22442476b9 100644 --- a/_includes/code/howto/indexes/indexes-dynamic-v3.py +++ b/_includes/code/howto/indexes/indexes-dynamic-v3.py @@ -46,3 +46,37 @@ ): correct_index = True assert correct_index, "Wrong index type" + + +######################### +### CONFIGURE DYNAMIC ### +######################### + +# Delete data from prior runs +if client.schema.exists(class_name): + client.schema.delete_class(class_name) + +# START EnableDynamic +class_obj = { + "class": class_name, + # Additional configuration not shown + "vectorIndexType": "dynamic", +} + +client.schema.create_class(class_obj) +# END EnableDynamic + +class_response = client.schema.get() +schema_response = client.schema.get(class_name) + +classes = [] +for c in class_response["classes"]: + classes.append(c["class"]) +assert class_name in classes, "Class missing" + +correct_index = False +if (schema_response["class"] == class_name) and ( + schema_response["vectorIndexType"] == "dynamic" +): + correct_index = True +assert correct_index, "Wrong index type" diff --git a/_includes/code/howto/indexes/indexes-dynamic-v3.ts b/_includes/code/howto/indexes/indexes-dynamic-v3.ts index f52461aa81..2de2d5b837 100644 --- a/_includes/code/howto/indexes/indexes-dynamic-v3.ts +++ b/_includes/code/howto/indexes/indexes-dynamic-v3.ts @@ -40,6 +40,25 @@ async function createDynamicCollection(client: WeaviateClient, collectionName: s } // END EnableDynamic + +/////////////////////// +// CONFIGURE DYNAMIC // +/////////////////////// + +// START ConfigDynamic +// Add this import line +// import { vectorizer, dataType, configure } from 'weaviate-client'; + +async function configureDynamicCollection(client: WeaviateClient, collectionName: string){ + await client.collections.create({ + name: collectionName, + vectorizers: vectorizer.text2VecOpenAI({ + vectorIndexConfig: configure.vectorIndex.dynamic(), + }), + }) +} +// END ConfigDynamic + ///////////////////////////// /// AVOID TOP LEVEL AWAIT /// ///////////////////////////// @@ -55,12 +74,17 @@ async function main(){ // Only safe to run one at a time due to aynsc code - // // Run dynamic collection code + // // Run enable dynamic collection code // deleteCollection(client, collectionName) // if(await client.collections.get(collectionName).exists() != true){ // createDynamicCollection(client, collectionName); // } -} + // // Run configure dynamic collection code + // deleteCollection(client, collectionName) + // if(await client.collections.get(collectionName).exists() != true){ + // createDynamicCollection(client, collectionName); + // } + } main() diff --git a/_includes/code/howto/indexes/indexes-dynamic-v4.py b/_includes/code/howto/indexes/indexes-dynamic-v4.py index e512a594e4..8ea18664e9 100644 --- a/_includes/code/howto/indexes/indexes-dynamic-v4.py +++ b/_includes/code/howto/indexes/indexes-dynamic-v4.py @@ -33,21 +33,8 @@ def print_response_iter(title, response): client.collections.create( name=collection_name, - vector_index_config=Configure.VectorIndex.dynamic( - distance_metric=VectorDistances.COSINE, - threshold=20000, - hnsw=Configure.VectorIndex.hnsw( - # Any hnsw configuration parameters - dynamic_ef_factor=15, # Multiplier for dynamic Ef - dynamic_ef_min=200, # Minimum threshold for dynamic Ef - dynamic_ef_max=1000, # Maximum threshold for dynamic Ef - ), - flat=Configure.VectorIndex.flat( - # Any flat index configuration parameters - vector_cache_max_objects=100000, - quantizer=Configure.VectorIndex.Quantizer.bq() - ), - ) + vector_index_config=Configure.VectorIndex.dynamic(), + # Configure vectorizer, properties ) # END EnableDynamic @@ -87,7 +74,8 @@ def print_response_iter(title, response): vector_cache_max_objects=100000, quantizer=Configure.VectorIndex.Quantizer.bq() ), - ) + ), + # Configure vectorizer, properties ) # END ConfigDynamic diff --git a/developers/weaviate/configuration/indexing/dynamic-indexes.md b/developers/weaviate/configuration/indexing/dynamic-indexes.md index a577fca884..489a508751 100644 --- a/developers/weaviate/configuration/indexing/dynamic-indexes.md +++ b/developers/weaviate/configuration/indexing/dynamic-indexes.md @@ -13,7 +13,6 @@ import PyCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-dynamic import TSCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-dynamic-v3.ts'; import TSCodeV2 from '!!raw-loader!/_includes/code/howto/indexes/indexes-dynamic-v2.ts'; - import DynamicIntro from '/_includes/indexes/dynamic-intro.mdx'; @@ -26,6 +25,8 @@ import ConsiderComp from '/_includes/indexes/consider-compression.mdx'; ## Enable dynamic indexes +Enable a dynamic index that uses default parameters for the dynamic, HNSW, and flat indexes: + Date: Wed, 18 Sep 2024 16:48:37 -0400 Subject: [PATCH 45/67] ts2 async delete timing --- .../code/howto/indexes/indexes-dynamic-v2.ts | 53 ++++++------ .../code/howto/indexes/indexes-dynamic-v3.py | 76 ++++++++++------- .../code/howto/indexes/indexes-dynamic-v3.ts | 40 ++++++--- .../code/howto/indexes/indexes-dynamic-v4.py | 15 ++-- .../code/howto/indexes/indexes-inverted-v2.ts | 51 ++++------- .../code/howto/indexes/indexes-inverted-v3.ts | 2 +- .../code/howto/indexes/indexes-inverted-v4.py | 7 -- _includes/code/howto/indexes/indexes-v2.ts | 84 +++++-------------- _includes/code/howto/indexes/indexes-v3.py | 33 -------- _includes/code/howto/indexes/indexes-v3.ts | 26 +----- _includes/code/howto/indexes/indexes-v4.py | 47 ----------- _includes/code/howto/indexes/tmp-await.ts | 33 ++++++++ 12 files changed, 182 insertions(+), 285 deletions(-) create mode 100644 _includes/code/howto/indexes/tmp-await.ts diff --git a/_includes/code/howto/indexes/indexes-dynamic-v2.ts b/_includes/code/howto/indexes/indexes-dynamic-v2.ts index 2afdb05fec..5ea7a1c23d 100644 --- a/_includes/code/howto/indexes/indexes-dynamic-v2.ts +++ b/_includes/code/howto/indexes/indexes-dynamic-v2.ts @@ -15,13 +15,10 @@ function getClient(){ } // Delete pre-existing collections -function deleteClass(client, className: string){ - try { - client.schema.classDeleter().withClassName(className).do(); - } catch (e) { - // ignore error if class doesn't exist - } - return true +async function deleteClass(client: WeaviateClient, className: string){ + if (client.schema.exists(className)) { + await client.schema.classDeleter().withClassName(className).do(); + } } //////////////////// @@ -30,7 +27,6 @@ function deleteClass(client, className: string){ // START EnableDynamic async function createDynamicCollection(client: WeaviateClient, className: string){ - const setIndexType = { class: className, // Add property definitions @@ -49,12 +45,27 @@ async function createDynamicCollection(client: WeaviateClient, className: string // START ConfigDynamic async function configureDynamicCollection(client: WeaviateClient, className: string){ - const setIndexType = { class: className, // Add property definitions vectorizer: 'text2vec-openai', vectorIndexType: 'dynamic', + vectorIndexConfig: { + distance: 'cosine', + vector_cache_max_objects: 100000, + bq: { enabled: true, }, + }, + vectorIndexConfigDynamic: { + distance: 'cosine', + ef_construction: '256', // Dynamic list size during construction + max_connections: '128', // Maximum number of connections per node + ef: '-1', // Dynamic list size during search; -1 enables dynamic Ef + dynamic_ef_factor: '15', // Multiplier for dynamic Ef + dynamic_ef_min: '200', // Minimum threshold for dynamic Ef + dynamic_ef_max: '1000', // Maximum threshold for dynamic Ef + quantizer: 'Configure.VectorIndex.Quantizer.pq()', // Quantizer configuration + }, + }; // Add the class to the schema @@ -68,25 +79,17 @@ async function configureDynamicCollection(client: WeaviateClient, className: str // Main async function main(){ - const className = "ConfigCollection"; - - const client = await getClient(); - deleteClass(client, className) - - // Only safe to run one at a time due to aynsc code + const className = "ConfigCollection"; - // // Run enable dynamic collection code - // deleteClass(client, className) - // if(await client.schema.exists(className) != true){ - // createDynamicCollection(client, className); - // } + const client = await getClient(); - // // Run configure dynamic collection code - // deleteClass(client, className) - // if(await client.schema.exists(className) != true){ - // configureDynamicCollection(client, className); - // } + // Run enable dynamic collection code + await deleteClass(client, className) + createDynamicCollection(client, className); + // // Run configure dynamic collection code + // await deleteClass(client, className) + // configureDynamicCollection(client, className); } main() \ No newline at end of file diff --git a/_includes/code/howto/indexes/indexes-dynamic-v3.py b/_includes/code/howto/indexes/indexes-dynamic-v3.py index 22442476b9..77008c948b 100644 --- a/_includes/code/howto/indexes/indexes-dynamic-v3.py +++ b/_includes/code/howto/indexes/indexes-dynamic-v3.py @@ -25,7 +25,6 @@ # START EnableDynamic class_obj = { "class": class_name, - # Additional configuration not shown "vectorIndexType": "dynamic", } @@ -47,36 +46,53 @@ correct_index = True assert correct_index, "Wrong index type" - ######################### ### CONFIGURE DYNAMIC ### ######################### -# Delete data from prior runs -if client.schema.exists(class_name): - client.schema.delete_class(class_name) - -# START EnableDynamic -class_obj = { - "class": class_name, - # Additional configuration not shown - "vectorIndexType": "dynamic", -} - -client.schema.create_class(class_obj) -# END EnableDynamic - -class_response = client.schema.get() -schema_response = client.schema.get(class_name) - -classes = [] -for c in class_response["classes"]: - classes.append(c["class"]) -assert class_name in classes, "Class missing" - -correct_index = False -if (schema_response["class"] == class_name) and ( - schema_response["vectorIndexType"] == "dynamic" -): - correct_index = True -assert correct_index, "Wrong index type" +# START ConfigDynamic +# Configuration details coming soon +# END ConfigDynamic + +# # Delete data from prior runs +# if client.schema.exists(class_name): +# client.schema.delete_class(class_name) + +# #\ START ConfigDynamic +# class_obj = { +# "class": class_name, +# "vectorIndexType": "dynamic", +# "vectorIndexConfigDynamic": { +# "hnsw": { +# "vectorIndexConfig": { +# "dynamic_ef_factor": 15, # Multiplier for dynamic Ef +# "dynamic_ef_min": 200, # Minimum threshold for dynamic Ef +# "dynamic_ef_max": 1000, # Maximum threshold for dynamic Ef +# } +# }, +# "flat": { +# "vectorIndexConfig": { +# "vector_cache_max_objects": 100000, # Limit index cache size +# "bq": {"enabled": True} # Enable compression +# } +# }, +# } +# } + +# client.schema.create_class(class_obj) +# #\ END ConfigDynamic + +# class_response = client.schema.get() +# schema_response = client.schema.get(class_name) + +# classes = [] +# for c in class_response["classes"]: +# classes.append(c["class"]) +# assert class_name in classes, "Class missing" + +# correct_index = False +# if (schema_response["class"] == class_name) and ( +# schema_response["vectorIndexType"] == "dynamic" +# ): +# correct_index = True +# assert correct_index, "Wrong index type" diff --git a/_includes/code/howto/indexes/indexes-dynamic-v3.ts b/_includes/code/howto/indexes/indexes-dynamic-v3.ts index 2de2d5b837..d81d220fb4 100644 --- a/_includes/code/howto/indexes/indexes-dynamic-v3.ts +++ b/_includes/code/howto/indexes/indexes-dynamic-v3.ts @@ -6,19 +6,21 @@ import weaviate, { dataType, WeaviateClient,vectorizer, configure } from 'weavia // Delete pre-existing collections function deleteCollection(client: WeaviateClient, collectionName: string){ + +var success = false; try { client.collections.delete(collectionName) + success = true; } catch (e) { // ignore error if class doesn't exist } -return true +return success } // Create client connection async function getClient(){ const client: WeaviateClient = weaviate.connectToLocal(); - return client; } @@ -53,7 +55,24 @@ async function configureDynamicCollection(client: WeaviateClient, collectionName await client.collections.create({ name: collectionName, vectorizers: vectorizer.text2VecOpenAI({ - vectorIndexConfig: configure.vectorIndex.dynamic(), + vectorIndexConfig: configure.vectorIndex.dynamic({ + distanceMetric: 'cosine', + hnsw: { + distanceMetric: 'cosine', + efConstruction: 256, // Dynamic list size during construction + maxConnections: 128, // Maximum number of connections per node + ef: -1, // Dynamic list size during search; -1 enables dynamic Ef + dynamicEfFactor: 15, // Multiplier for dynamic Ef + dynamicEfMin: 200, // Minimum threshold for dynamic Ef + dynamicEfMax: 1000, // Maximum threshold for dynamic Ef + quantizer: configure.vectorIndex.quantizer.pq() // Compression + }, + flat: { + distanceMetric: 'cosine', + vectorCacheMaxObjects: 1000000, + quantizer: configure.vectorIndex.quantizer.bq(), + }, + }), }), }) } @@ -69,10 +88,7 @@ async function main(){ const client = await getClient(); - // Clean up from earlier runs - deleteCollection(client, collectionName) - - // Only safe to run one at a time due to aynsc code + // Only safe to run one at a time due to async code // // Run enable dynamic collection code // deleteCollection(client, collectionName) @@ -80,11 +96,11 @@ async function main(){ // createDynamicCollection(client, collectionName); // } - // // Run configure dynamic collection code - // deleteCollection(client, collectionName) - // if(await client.collections.get(collectionName).exists() != true){ - // createDynamicCollection(client, collectionName); - // } + // Run configure dynamic collection code + deleteCollection(client, collectionName) + if(await client.collections.get(collectionName).exists() != true){ + configureDynamicCollection(client, collectionName); + } } main() diff --git a/_includes/code/howto/indexes/indexes-dynamic-v4.py b/_includes/code/howto/indexes/indexes-dynamic-v4.py index 8ea18664e9..ccc2184ab4 100644 --- a/_includes/code/howto/indexes/indexes-dynamic-v4.py +++ b/_includes/code/howto/indexes/indexes-dynamic-v4.py @@ -2,13 +2,6 @@ collection_name = "ConfigCollection" -def print_response(title, response): - print(f"{title}: {response}") - -def print_response_iter(title, response): - for r in response: - print(f"{title}: {r}") - ######################## ### CLIENT CONNECTION ## ######################## @@ -17,7 +10,6 @@ def print_response_iter(title, response): import weaviate cohere_api_key = os.environ["COHERE_API_KEY"] - client = weaviate.connect_to_local(headers={"X-Cohere-Api-Key": cohere_api_key}) ###################### @@ -29,12 +21,17 @@ def print_response_iter(title, response): client.collections.delete(collection_name) # START EnableDynamic -from weaviate.classes.config import Configure, VectorDistances +from weaviate.classes.config import Configure, Property, DataType, VectorDistances client.collections.create( name=collection_name, + description="Configuration example", vector_index_config=Configure.VectorIndex.dynamic(), # Configure vectorizer, properties + properties=[ + Property(name="title", data_type=DataType.TEXT), + Property(name="body", data_type=DataType.TEXT), + ] ) # END EnableDynamic diff --git a/_includes/code/howto/indexes/indexes-inverted-v2.ts b/_includes/code/howto/indexes/indexes-inverted-v2.ts index 484ea2c2b1..d3c53e5bf8 100644 --- a/_includes/code/howto/indexes/indexes-inverted-v2.ts +++ b/_includes/code/howto/indexes/indexes-inverted-v2.ts @@ -15,13 +15,10 @@ function getClient(){ } // Delete pre-existing collections -function deleteClass(client, className: string){ - try { - client.schema.classDeleter().withClassName(className).do(); - } catch (e) { - // ignore error if class doesn't exist - } - return true +async function deleteClass(client: WeaviateClient, className: string){ + if (client.schema.exists(className)) { + await client.schema.classDeleter().withClassName(className).do(); + } } //////////// @@ -116,7 +113,8 @@ async function bm25InvertedCollection(client: WeaviateClient, className: string) b: 0.7, k1: 1.25 }, -}; + } +} // Add the class to the schema const result = await client.schema @@ -159,39 +157,26 @@ async function main(){ const className = "ConfigCollection"; const client = await getClient(); - deleteClass(client, className) - - // Only safe to run one at a time due to aynsc code - // Run search code - deleteCollection(client, collectionName) - if(await client.collections.get(collectionName).exists() != true){ - searchInvertedCollection(client, collectionName); - } + // // Run search code + // await deleteClass(client, className) + // searchInvertedCollection(client, className); // // Run filter code - // deleteCollection(client, collectionName) - // if(await client.collections.get(collectionName).exists() != true){ - // filterInvertedCollection(client, collectionName); - // } + // await deleteClass(client, className) + // filterInvertedCollection(client, className); // // Run range code - // deleteCollection(client, collectionName) - // if(await client.collections.get(collectionName).exists() != true){ - // rangeInvertedCollection(client, collectionName); - // } + // await deleteClass(client, className) + // rangeInvertedCollection(client, className); - // // Run bm25 code - // deleteCollection(client, collectionName) - // if(await client.collections.get(collectionName).exists() != true){ - // bm25InvertedCollection(client, collectionName); - // } + // Run bm25 code + await deleteClass(client, className) + bm25InvertedCollection(client, className); // // Run collection level code - // deleteCollection(client, collectionName) - // if(await client.collections.get(collectionName).exists() != true){ - // collLevInvertedCollection(client, collectionName); - // } + // await deleteClass(client, className) + // collLevInvertedCollection(client, className); } diff --git a/_includes/code/howto/indexes/indexes-inverted-v3.ts b/_includes/code/howto/indexes/indexes-inverted-v3.ts index cdf1348986..c6d24aebcd 100644 --- a/_includes/code/howto/indexes/indexes-inverted-v3.ts +++ b/_includes/code/howto/indexes/indexes-inverted-v3.ts @@ -142,7 +142,7 @@ async function main(){ // Clean up from earlier runs deleteCollection(client, collectionName) - // Only safe to run one at a time due to aynsc code + // Only safe to run one at a time due to async code // // Run search code // deleteCollection(client, collectionName) diff --git a/_includes/code/howto/indexes/indexes-inverted-v4.py b/_includes/code/howto/indexes/indexes-inverted-v4.py index e7594715c8..6a92aa0d8a 100644 --- a/_includes/code/howto/indexes/indexes-inverted-v4.py +++ b/_includes/code/howto/indexes/indexes-inverted-v4.py @@ -2,13 +2,6 @@ collection_name = "ConfigCollection" -def print_response(title, response): - print(f"{title}: {response}") - -def print_response_iter(title, response): - for r in response: - print(f"{title}: {r}") - ######################## ### CLIENT CONNECTION ## ######################## diff --git a/_includes/code/howto/indexes/indexes-v2.ts b/_includes/code/howto/indexes/indexes-v2.ts index 9fdb368607..0e1e2e9087 100644 --- a/_includes/code/howto/indexes/indexes-v2.ts +++ b/_includes/code/howto/indexes/indexes-v2.ts @@ -15,13 +15,10 @@ function getClient(){ } // Delete pre-existing collections -function deleteClass(client, className: string){ - try { - client.schema.classDeleter().withClassName(className).do(); - } catch (e) { - // ignore error if class doesn't exist - } - return true +async function deleteClass(client: WeaviateClient, className: string){ + if (client.schema.exists(className)) { + await client.schema.classDeleter().withClassName(className).do(); + } } ////////////////////////////// @@ -189,25 +186,6 @@ async function compressFlatCollection(client: WeaviateClient, className: string) } // END CompressFlat -//////////////////// -// ENABLE DYNAMIC // -//////////////////// - -// START EnableDynamic -async function createDynamicCollection(client: WeaviateClient, className: string){ - - const setIndexType = { - class: className, - // Add property definitions - vectorizer: 'text2vec-openai', - vectorIndexType: 'dynamic', - }; - - // Add the class to the schema - await client.schema.classCreator().withClass(setIndexType).do(); -} -// END EnableDynamic - ///////////////////////////// /// AVOID TOP LEVEL AWAIT /// ///////////////////////////// @@ -217,51 +195,31 @@ async function main(){ const className = "ConfigCollection"; const client = await getClient(); - deleteClass(client, className) - - // Only safe to run one at a time due to aynsc code - // // Run enable HNSW collection code - // deleteClass(client, className) - // if(await client.schema.exists(className) != true){ - // createHNSWCollection(client, className); - // } + // Run enable HNSW collection code + await deleteClass(client, className) + createHNSWCollection(client, className); - // Run configure HNSW collection code - deleteClass(client, className) - if(await client.schema.exists(className) != true){ - configHNSWCollection(client, className); - } + // // Run configure HNSW collection code + // await deleteClass(client, className) + // configHNSWCollection(client, className); // // Run multiple named vector collection code - // deleteClass(client, className) - // if(await client.schema.exists(className) != true){ - // createMultiCollection(client, className); - // } + // await deleteClass(client, className) + // createMultiCollection(client, className); // // Run enable flat collection code - // deleteClass(client, className) - // if(await client.schema.exists(className) != true){ - // createFlatCollection(client, className); - // } + // await deleteClass(client, className) + // createFlatCollection(client, className); // // Run configure flat collection code - // deleteClass(client, className) - // if(await client.schema.exists(className) != true){ - // configureFlatCollection(client, className); - // } - - // // Run compress flat collection code - // deleteClass(client, className) - // if(await client.schema.exists(className) != true){ - // compressFlatCollection(client, className); - // } - - // // Run enable dynamic collection code - // deleteClass(client, className) - // if(await client.schema.exists(className) != true){ - // createDynamicCollection(client, className); - // } + // await deleteClass(client, className) + // configureFlatCollection(client, className); + + // Run compress flat collection code + await deleteClass(client, className) + compressFlatCollection(client, className); + } main() \ No newline at end of file diff --git a/_includes/code/howto/indexes/indexes-v3.py b/_includes/code/howto/indexes/indexes-v3.py index 3206a36968..414e3dbead 100644 --- a/_includes/code/howto/indexes/indexes-v3.py +++ b/_includes/code/howto/indexes/indexes-v3.py @@ -244,36 +244,3 @@ correct_index = True assert correct_index, "Wrong index type" - -###################### -### ENABLE DYNAMIC ### -###################### - -# Delete data from prior runs -if client.schema.exists(class_name): - client.schema.delete_class(class_name) - -# START EnableDynamic -class_obj = { - "class": class_name, - # Additional configuration not shown - "vectorIndexType": "dynamic", -} - -client.schema.create_class(class_obj) -# END EnableDynamic - -class_response = client.schema.get() -schema_response = client.schema.get(class_name) - -classes = [] -for c in class_response["classes"]: - classes.append(c["class"]) -assert class_name in classes, "Class missing" - -correct_index = False -if (schema_response["class"] == class_name) and ( - schema_response["vectorIndexType"] == "dynamic" -): - correct_index = True -assert correct_index, "Wrong index type" diff --git a/_includes/code/howto/indexes/indexes-v3.ts b/_includes/code/howto/indexes/indexes-v3.ts index 529bc364fa..a04d59a9a6 100644 --- a/_includes/code/howto/indexes/indexes-v3.ts +++ b/_includes/code/howto/indexes/indexes-v3.ts @@ -183,24 +183,6 @@ async function compressFlatCollection(client: WeaviateClient, collectionName: st } // END CompressFlat -//////////////////// -// ENABLE DYNAMIC // -//////////////////// - -// START EnableDynamic -// Add this import line -// import { vectorizer, dataType, configure } from 'weaviate-client'; - -async function createDynamicCollection(client: WeaviateClient, collectionName: string){ - await client.collections.create({ - name: collectionName, - vectorizers: vectorizer.text2VecOpenAI({ - vectorIndexConfig: configure.vectorIndex.dynamic(), - }), - }) -} -// END EnableDynamic - ///////////////////////////// /// AVOID TOP LEVEL AWAIT /// ///////////////////////////// @@ -214,7 +196,7 @@ async function main(){ // Clean up from earlier runs deleteCollection(client, collectionName) - // Only safe to run one at a time due to aynsc code + // Only safe to run one at a time due to async code // // Run enable HNSW collection code // deleteCollection(client, collectionName) @@ -258,12 +240,6 @@ async function main(){ // compressFlatCollection(client, collectionName); // } - // // Run dynamic collection code - // deleteCollection(client, collectionName) - // if(await client.collections.get(collectionName).exists() != true){ - // createDynamicCollection(client, collectionName); - // } - } main() diff --git a/_includes/code/howto/indexes/indexes-v4.py b/_includes/code/howto/indexes/indexes-v4.py index d4216faf3b..4ca32974d2 100644 --- a/_includes/code/howto/indexes/indexes-v4.py +++ b/_includes/code/howto/indexes/indexes-v4.py @@ -2,13 +2,6 @@ collection_name = "ConfigCollection" -def print_response(title, response): - print(f"{title}: {response}") - -def print_response_iter(title, response): - for r in response: - print(f"{title}: {r}") - ######################## ### CLIENT CONNECTION ## ######################## @@ -251,46 +244,6 @@ def print_response_iter(title, response): str(schema_response.vector_index_type) == "VectorIndexType.FLAT" ), "Wrong index type" -###################### -### ENABLE DYNAMIC ### -###################### - -# Delete data from prior runs -if client.collections.exists(collection_name): - client.collections.delete(collection_name) - -# START EnableDynamic -from weaviate.classes.config import Configure, VectorDistances - -client.collections.create( - name=collection_name, - vector_index_config=Configure.VectorIndex.dynamic( - distance_metric=VectorDistances.COSINE, - threshold=20000, - hnsw=Configure.VectorIndex.hnsw( - # Any hnsw configuration parameters - dynamic_ef_factor=15, # Multiplier for dynamic Ef - dynamic_ef_min=200, # Minimum threshold for dynamic Ef - dynamic_ef_max=1000, # Maximum threshold for dynamic Ef - ), - flat=Configure.VectorIndex.flat( - # Any flat index configuration parameters - vector_cache_max_objects=100000, - quantizer=Configure.VectorIndex.Quantizer.bq() - ), - ) -) -# END EnableDynamic - -collection = client.collections.get(collection_name) -collections_response = client.collections.list_all() -schema_response = collection.config.get() - -assert collection_name in collections_response.keys(), "Collection missing" -assert ( - str(schema_response.vector_index_type) == "VectorIndexType.DYNAMIC" -), "Wrong index type" - ################ ### CLEAN UP ### ################ diff --git a/_includes/code/howto/indexes/tmp-await.ts b/_includes/code/howto/indexes/tmp-await.ts new file mode 100644 index 0000000000..3ed397c64b --- /dev/null +++ b/_includes/code/howto/indexes/tmp-await.ts @@ -0,0 +1,33 @@ + +const sleep = async(waitTime: number) => + new Promise(resolve => setTimeout(resolve, waitTime)); + +function fast01(){ + console.log("Start fast01") + console.log("End fast01") + return true +} + +async function slow01(){ + console.log("Start slow01") + await sleep(3000) + console.log("End slow01") + return true +} + +async function fast02(){ + console.log("Start fast02") + await sleep(1000) + console.log("End fast02") + return true +} + + +async function main(){ + + await slow01() + fast02() + fast01() +} + +main() \ No newline at end of file From 74ff3dbff252120a572f5c2cd586d65b2d8ba719 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Wed, 18 Sep 2024 17:19:18 -0400 Subject: [PATCH 46/67] ts3 async delete timing --- .../code/howto/indexes/indexes-dynamic-v3.ts | 36 +++------ .../code/howto/indexes/indexes-inverted-v3.ts | 54 +++++--------- _includes/code/howto/indexes/indexes-v3.ts | 73 +++++++------------ 3 files changed, 58 insertions(+), 105 deletions(-) diff --git a/_includes/code/howto/indexes/indexes-dynamic-v3.ts b/_includes/code/howto/indexes/indexes-dynamic-v3.ts index d81d220fb4..29b1a3e528 100644 --- a/_includes/code/howto/indexes/indexes-dynamic-v3.ts +++ b/_includes/code/howto/indexes/indexes-dynamic-v3.ts @@ -5,22 +5,15 @@ import weaviate, { dataType, WeaviateClient,vectorizer, configure } from 'weaviate-client'; // Delete pre-existing collections -function deleteCollection(client: WeaviateClient, collectionName: string){ - -var success = false; - try { - client.collections.delete(collectionName) - success = true; -} catch (e) { - // ignore error if class doesn't exist -} - -return success +async function deleteCollection(client: WeaviateClient, collectionName: string){ + if(client.collections.exists(collectionName)){ + await client.collections.delete(collectionName) + } } // Create client connection async function getClient(){ - const client: WeaviateClient = weaviate.connectToLocal(); + const client: WeaviateClient = await weaviate.connectToLocal(); return client; } @@ -88,19 +81,14 @@ async function main(){ const client = await getClient(); - // Only safe to run one at a time due to async code + // Run enable dynamic collection code + await deleteCollection(client, collectionName) + await createDynamicCollection(client, collectionName); - // // Run enable dynamic collection code - // deleteCollection(client, collectionName) - // if(await client.collections.get(collectionName).exists() != true){ - // createDynamicCollection(client, collectionName); - // } + // // Run configure dynamic collection code + // await deleteCollection(client, collectionName) + // await configureDynamicCollection(client, collectionName); - // Run configure dynamic collection code - deleteCollection(client, collectionName) - if(await client.collections.get(collectionName).exists() != true){ - configureDynamicCollection(client, collectionName); - } - } +} main() diff --git a/_includes/code/howto/indexes/indexes-inverted-v3.ts b/_includes/code/howto/indexes/indexes-inverted-v3.ts index c6d24aebcd..43267bc9a3 100644 --- a/_includes/code/howto/indexes/indexes-inverted-v3.ts +++ b/_includes/code/howto/indexes/indexes-inverted-v3.ts @@ -5,14 +5,10 @@ import weaviate, { dataType, WeaviateClient,vectorizer, configure } from 'weaviate-client'; // Delete pre-existing collections -function deleteCollection(client: WeaviateClient, collectionName: string){ - try { - client.collections.delete(collectionName) -} catch (e) { - // ignore error if class doesn't exist -} - -return true +async function deleteCollection(client: WeaviateClient, collectionName: string){ + if(client.collections.exists(collectionName)){ + await client.collections.delete(collectionName) + } } // Create client connection @@ -139,40 +135,26 @@ async function main(){ const client = await getClient(); - // Clean up from earlier runs - deleteCollection(client, collectionName) - - // Only safe to run one at a time due to async code - - // // Run search code - // deleteCollection(client, collectionName) - // if(await client.collections.get(collectionName).exists() != true){ - // searchInvertedCollection(client, collectionName); - // } + // Run search code + await deleteCollection(client, collectionName) + await searchInvertedCollection(client, collectionName); // // Run filter code - // deleteCollection(client, collectionName) - // if(await client.collections.get(collectionName).exists() != true){ - // filterInvertedCollection(client, collectionName); - // } + // await deleteCollection(client, collectionName) + // await filterInvertedCollection(client, collectionName); // // Run range code - // deleteCollection(client, collectionName) - // if(await client.collections.get(collectionName).exists() != true){ - // rangeInvertedCollection(client, collectionName); - // } + // await deleteCollection(client, collectionName) + // await rangeInvertedCollection(client, collectionName); // // Run bm25 code - // deleteCollection(client, collectionName) - // if(await client.collections.get(collectionName).exists() != true){ - // bm25InvertedCollection(client, collectionName); - // } - - // Run collection level code - deleteCollection(client, collectionName) - if(await client.collections.get(collectionName).exists() != true){ - collLevInvertedCollection(client, collectionName); - } + // await deleteCollection(client, collectionName) + // await bm25InvertedCollection(client, collectionName); + + // // Run collection level code + // await deleteCollection(client, collectionName) + // await collLevInvertedCollection(client, collectionName); + } diff --git a/_includes/code/howto/indexes/indexes-v3.ts b/_includes/code/howto/indexes/indexes-v3.ts index a04d59a9a6..b49ab236c0 100644 --- a/_includes/code/howto/indexes/indexes-v3.ts +++ b/_includes/code/howto/indexes/indexes-v3.ts @@ -5,14 +5,10 @@ import weaviate, { dataType, WeaviateClient,vectorizer, configure } from 'weaviate-client'; // Delete pre-existing collections -function deleteCollection(client: WeaviateClient, collectionName: string){ - try { - client.collections.delete(collectionName) -} catch (e) { - // ignore error if class doesn't exist -} - -return true +async function deleteCollection(client: WeaviateClient, collectionName: string){ + if(client.collections.exists(collectionName)){ + await client.collections.delete(collectionName) + } } // Create client connection @@ -193,52 +189,39 @@ async function main(){ const client = await getClient(); - // Clean up from earlier runs - deleteCollection(client, collectionName) - - // Only safe to run one at a time due to async code - // // Run enable HNSW collection code - // deleteCollection(client, collectionName) - // if(await client.collections.get(collectionName).exists() != true){ - // createHNSWCollection(client, collectionName); - // } + // await deleteCollection(client, collectionName) + // await createHNSWCollection(client, collectionName); + // // Run configure HNSW collection code - // deleteCollection(client, collectionName) - // if(await client.collections.get(collectionName).exists() != true){ - // configureHNSWCollection(client, collectionName); - // } - - // Run compress HNSW collection code - deleteCollection(client, collectionName) - if(await client.collections.get(collectionName).exists() != true){ - compressHNSWCollection(client, collectionName); - } + // await deleteCollection(client, collectionName) + // await configureHNSWCollection(client, collectionName); + + + // // Run compress HNSW collection code + // await deleteCollection(client, collectionName) + // await compressHNSWCollection(client, collectionName); + // // Run multiple named vector collection code - // deleteCollection(client, collectionName) - // if(await client.collections.get(collectionName).exists() != true){ - // createMultiCollection(client, collectionName); - // } + // await deleteCollection(client, collectionName) + // await createMultiCollection(client, collectionName); + // // Run enable Flat collection code - // deleteCollection(client, collectionName) - // if(await client.collections.get(collectionName).exists() != true){ - // createFlatCollection(client, collectionName); - // } + // await deleteCollection(client, collectionName) + // await createFlatCollection(client, collectionName); + // // Run configure Flat collection code - // deleteCollection(client, collectionName) - // if(await client.collections.get(collectionName).exists() != true){ - // configureFlatCollection(client, collectionName); - // } - - // // Run compress Flat collection code - // deleteCollection(client, collectionName) - // if(await client.collections.get(collectionName).exists() != true){ - // compressFlatCollection(client, collectionName); - // } + // await deleteCollection(client, collectionName) + // await configureFlatCollection(client, collectionName); + + + // Run compress Flat collection code + await deleteCollection(client, collectionName) + await compressFlatCollection(client, collectionName); } From 5824950fea64a6198853cff915cbfcded536086d Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Thu, 19 Sep 2024 09:43:10 -0400 Subject: [PATCH 47/67] dynamic --- .../code/howto/indexes/indexes-dynamic-v3.py | 89 ++++++++++--------- .../code/howto/indexes/indexes-dynamic-v3.ts | 14 +-- 2 files changed, 53 insertions(+), 50 deletions(-) diff --git a/_includes/code/howto/indexes/indexes-dynamic-v3.py b/_includes/code/howto/indexes/indexes-dynamic-v3.py index 77008c948b..62bf0abcde 100644 --- a/_includes/code/howto/indexes/indexes-dynamic-v3.py +++ b/_includes/code/howto/indexes/indexes-dynamic-v3.py @@ -50,49 +50,52 @@ ### CONFIGURE DYNAMIC ### ######################### +# Delete data from prior runs +if client.schema.exists(class_name): + client.schema.delete_class(class_name) + # START ConfigDynamic -# Configuration details coming soon +class_obj = { + "description": "Dynamic configuration example", + "class": class_name, + "vectorIndexType": "dynamic", + "vectorIndexConfig": { + "hnsw": { + "cleanupIntervalSeconds": 300, + "distanceMetric": "cosine", + "dynamicEfMin": 100, + "dynamicEfMax": 600, + "dynamicEfFactor": 8, + "ef": -1, + "efConstruction": 128, + "flatSearchCutoff": 40000, + "maxConnections": 32, + "skip": False, + "vectorCacheMaxObjects": 1000000000000, + "pq": { "enabled": True, }, + }, + "flat": { + "distanceMetric": "cosine", + "vector_cache_max_objects": 100000, + "bq": { "enabled": True, }, + }, + } +} + +client.schema.create_class(class_obj) # END ConfigDynamic -# # Delete data from prior runs -# if client.schema.exists(class_name): -# client.schema.delete_class(class_name) - -# #\ START ConfigDynamic -# class_obj = { -# "class": class_name, -# "vectorIndexType": "dynamic", -# "vectorIndexConfigDynamic": { -# "hnsw": { -# "vectorIndexConfig": { -# "dynamic_ef_factor": 15, # Multiplier for dynamic Ef -# "dynamic_ef_min": 200, # Minimum threshold for dynamic Ef -# "dynamic_ef_max": 1000, # Maximum threshold for dynamic Ef -# } -# }, -# "flat": { -# "vectorIndexConfig": { -# "vector_cache_max_objects": 100000, # Limit index cache size -# "bq": {"enabled": True} # Enable compression -# } -# }, -# } -# } - -# client.schema.create_class(class_obj) -# #\ END ConfigDynamic - -# class_response = client.schema.get() -# schema_response = client.schema.get(class_name) - -# classes = [] -# for c in class_response["classes"]: -# classes.append(c["class"]) -# assert class_name in classes, "Class missing" - -# correct_index = False -# if (schema_response["class"] == class_name) and ( -# schema_response["vectorIndexType"] == "dynamic" -# ): -# correct_index = True -# assert correct_index, "Wrong index type" +class_response = client.schema.get() +schema_response = client.schema.get(class_name) + +classes = [] +for c in class_response["classes"]: + classes.append(c["class"]) +assert class_name in classes, "Class missing" + +correct_index = False +if (schema_response["class"] == class_name) and ( + schema_response["vectorIndexType"] == "dynamic" +): + correct_index = True +assert correct_index, "Wrong index type" diff --git a/_includes/code/howto/indexes/indexes-dynamic-v3.ts b/_includes/code/howto/indexes/indexes-dynamic-v3.ts index 29b1a3e528..3ea92a669f 100644 --- a/_includes/code/howto/indexes/indexes-dynamic-v3.ts +++ b/_includes/code/howto/indexes/indexes-dynamic-v3.ts @@ -2,7 +2,7 @@ // TODO: Needs tests // Imports -import weaviate, { dataType, WeaviateClient,vectorizer, configure } from 'weaviate-client'; +import weaviate, { WeaviateClient,vectorizer, configure } from 'weaviate-client'; // Delete pre-existing collections async function deleteCollection(client: WeaviateClient, collectionName: string){ @@ -81,13 +81,13 @@ async function main(){ const client = await getClient(); - // Run enable dynamic collection code - await deleteCollection(client, collectionName) - await createDynamicCollection(client, collectionName); - - // // Run configure dynamic collection code + // // Run enable dynamic collection code // await deleteCollection(client, collectionName) - // await configureDynamicCollection(client, collectionName); + // await createDynamicCollection(client, collectionName); + + // Run configure dynamic collection code + await deleteCollection(client, collectionName) + await configureDynamicCollection(client, collectionName); } From 178ff2f7794ba9f075f9af9c00443527a0fb23ed Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Thu, 19 Sep 2024 10:10:01 -0400 Subject: [PATCH 48/67] reorder sections --- _includes/code/howto/indexes/tmp-await.ts | 33 ------------------- developers/weaviate/configuration/backups.md | 2 +- .../configuration/compression/_category_.json | 4 +++ .../indexing-keyword/_category_.json | 4 +++ .../{indexing => indexing-keyword}/index.md | 0 .../inverted-indexes.md | 0 .../indexing-vector/_category_.json | 4 +++ .../dynamic-indexes.md | 0 .../flat-indexes.md | 0 .../hnsw-indexes.md | 0 .../index.md} | 0 .../configuration/indexing/_category_.json | 4 --- developers/weaviate/configuration/modules.md | 2 +- .../configuration/tenant-offloading.md | 2 +- 14 files changed, 15 insertions(+), 40 deletions(-) delete mode 100644 _includes/code/howto/indexes/tmp-await.ts create mode 100644 developers/weaviate/configuration/compression/_category_.json create mode 100644 developers/weaviate/configuration/indexing-keyword/_category_.json rename developers/weaviate/configuration/{indexing => indexing-keyword}/index.md (100%) rename developers/weaviate/configuration/{indexing => indexing-keyword}/inverted-indexes.md (100%) create mode 100644 developers/weaviate/configuration/indexing-vector/_category_.json rename developers/weaviate/configuration/{indexing => indexing-vector}/dynamic-indexes.md (100%) rename developers/weaviate/configuration/{indexing => indexing-vector}/flat-indexes.md (100%) rename developers/weaviate/configuration/{indexing => indexing-vector}/hnsw-indexes.md (100%) rename developers/weaviate/configuration/{indexing/vector-indexes.md => indexing-vector/index.md} (100%) delete mode 100644 developers/weaviate/configuration/indexing/_category_.json diff --git a/_includes/code/howto/indexes/tmp-await.ts b/_includes/code/howto/indexes/tmp-await.ts deleted file mode 100644 index 3ed397c64b..0000000000 --- a/_includes/code/howto/indexes/tmp-await.ts +++ /dev/null @@ -1,33 +0,0 @@ - -const sleep = async(waitTime: number) => - new Promise(resolve => setTimeout(resolve, waitTime)); - -function fast01(){ - console.log("Start fast01") - console.log("End fast01") - return true -} - -async function slow01(){ - console.log("Start slow01") - await sleep(3000) - console.log("End slow01") - return true -} - -async function fast02(){ - console.log("Start fast02") - await sleep(1000) - console.log("End fast02") - return true -} - - -async function main(){ - - await slow01() - fast02() - fast01() -} - -main() \ No newline at end of file diff --git a/developers/weaviate/configuration/backups.md b/developers/weaviate/configuration/backups.md index 8b3ff79108..f5303653a6 100644 --- a/developers/weaviate/configuration/backups.md +++ b/developers/weaviate/configuration/backups.md @@ -1,6 +1,6 @@ --- title: Backups -sidebar_position: 12 +sidebar_position: 20 image: og/docs/configuration.jpg # tags: ['configuration', 'backups'] --- diff --git a/developers/weaviate/configuration/compression/_category_.json b/developers/weaviate/configuration/compression/_category_.json new file mode 100644 index 0000000000..8f1f43419e --- /dev/null +++ b/developers/weaviate/configuration/compression/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "Compression", + "position": 1 +} \ No newline at end of file diff --git a/developers/weaviate/configuration/indexing-keyword/_category_.json b/developers/weaviate/configuration/indexing-keyword/_category_.json new file mode 100644 index 0000000000..ea3db5c162 --- /dev/null +++ b/developers/weaviate/configuration/indexing-keyword/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "Keyword indexes", + "position": 7 +} \ No newline at end of file diff --git a/developers/weaviate/configuration/indexing/index.md b/developers/weaviate/configuration/indexing-keyword/index.md similarity index 100% rename from developers/weaviate/configuration/indexing/index.md rename to developers/weaviate/configuration/indexing-keyword/index.md diff --git a/developers/weaviate/configuration/indexing/inverted-indexes.md b/developers/weaviate/configuration/indexing-keyword/inverted-indexes.md similarity index 100% rename from developers/weaviate/configuration/indexing/inverted-indexes.md rename to developers/weaviate/configuration/indexing-keyword/inverted-indexes.md diff --git a/developers/weaviate/configuration/indexing-vector/_category_.json b/developers/weaviate/configuration/indexing-vector/_category_.json new file mode 100644 index 0000000000..f10274ad88 --- /dev/null +++ b/developers/weaviate/configuration/indexing-vector/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "Vector indexes", + "position": 3 +} \ No newline at end of file diff --git a/developers/weaviate/configuration/indexing/dynamic-indexes.md b/developers/weaviate/configuration/indexing-vector/dynamic-indexes.md similarity index 100% rename from developers/weaviate/configuration/indexing/dynamic-indexes.md rename to developers/weaviate/configuration/indexing-vector/dynamic-indexes.md diff --git a/developers/weaviate/configuration/indexing/flat-indexes.md b/developers/weaviate/configuration/indexing-vector/flat-indexes.md similarity index 100% rename from developers/weaviate/configuration/indexing/flat-indexes.md rename to developers/weaviate/configuration/indexing-vector/flat-indexes.md diff --git a/developers/weaviate/configuration/indexing/hnsw-indexes.md b/developers/weaviate/configuration/indexing-vector/hnsw-indexes.md similarity index 100% rename from developers/weaviate/configuration/indexing/hnsw-indexes.md rename to developers/weaviate/configuration/indexing-vector/hnsw-indexes.md diff --git a/developers/weaviate/configuration/indexing/vector-indexes.md b/developers/weaviate/configuration/indexing-vector/index.md similarity index 100% rename from developers/weaviate/configuration/indexing/vector-indexes.md rename to developers/weaviate/configuration/indexing-vector/index.md diff --git a/developers/weaviate/configuration/indexing/_category_.json b/developers/weaviate/configuration/indexing/_category_.json deleted file mode 100644 index f762b60e46..0000000000 --- a/developers/weaviate/configuration/indexing/_category_.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "label": "Indexing", - "position": 10 -} \ No newline at end of file diff --git a/developers/weaviate/configuration/modules.md b/developers/weaviate/configuration/modules.md index bb403fa34c..a8a7c0200b 100644 --- a/developers/weaviate/configuration/modules.md +++ b/developers/weaviate/configuration/modules.md @@ -1,6 +1,6 @@ --- title: Modules -sidebar_position: 11 +sidebar_position: 17 image: og/docs/configuration.jpg # tags: ['configuration', 'modules'] --- diff --git a/developers/weaviate/configuration/tenant-offloading.md b/developers/weaviate/configuration/tenant-offloading.md index 70ae860c63..dd4512452c 100644 --- a/developers/weaviate/configuration/tenant-offloading.md +++ b/developers/weaviate/configuration/tenant-offloading.md @@ -1,6 +1,6 @@ --- title: Tenant Offloading -sidebar_position: 15 +sidebar_position: 25 image: og/docs/configuration.jpg --- From cdf502258274a2256d1193e51c67fc5c20ca5627 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Thu, 19 Sep 2024 13:29:48 -0400 Subject: [PATCH 49/67] move around pages, update links --- .../concepts/indexing/dynamic-indexes.mdx | 0 .../concepts/indexing/flat-indexes.mdx | 0 .../concepts/indexing/hnsw-indexes.mdx | 0 .../{ => indexing}/img/supermarket.svg | 0 .../{ => indexing}/img/vectors-2d.png | Bin .../{ => indexing}/img/vectors-2d.svg | 0 .../weaviate/concepts/indexing/index.md | 6 +++++ .../inverted-indexes.md} | 0 .../vector-indexes.md} | 19 ++++++++------ .../indexing-keyword/_category_.json | 4 --- .../configuration/indexing-keyword/index.md | 22 ---------------- .../indexing-vector/dynamic-indexes.md | 4 +-- .../configuration/indexing-vector/index.md | 20 +++++---------- .../inverted-indexes.md | 24 +++++++++++++----- 14 files changed, 44 insertions(+), 55 deletions(-) create mode 100644 developers/weaviate/concepts/indexing/dynamic-indexes.mdx create mode 100644 developers/weaviate/concepts/indexing/flat-indexes.mdx create mode 100644 developers/weaviate/concepts/indexing/hnsw-indexes.mdx rename developers/weaviate/concepts/{ => indexing}/img/supermarket.svg (100%) rename developers/weaviate/concepts/{ => indexing}/img/vectors-2d.png (100%) rename developers/weaviate/concepts/{ => indexing}/img/vectors-2d.svg (100%) create mode 100644 developers/weaviate/concepts/indexing/index.md rename developers/weaviate/concepts/{indexing.md => indexing/inverted-indexes.md} (100%) rename developers/weaviate/concepts/{vector-index.md => indexing/vector-indexes.md} (95%) delete mode 100644 developers/weaviate/configuration/indexing-keyword/_category_.json delete mode 100644 developers/weaviate/configuration/indexing-keyword/index.md rename developers/weaviate/configuration/{indexing-keyword => }/inverted-indexes.md (85%) diff --git a/developers/weaviate/concepts/indexing/dynamic-indexes.mdx b/developers/weaviate/concepts/indexing/dynamic-indexes.mdx new file mode 100644 index 0000000000..e69de29bb2 diff --git a/developers/weaviate/concepts/indexing/flat-indexes.mdx b/developers/weaviate/concepts/indexing/flat-indexes.mdx new file mode 100644 index 0000000000..e69de29bb2 diff --git a/developers/weaviate/concepts/indexing/hnsw-indexes.mdx b/developers/weaviate/concepts/indexing/hnsw-indexes.mdx new file mode 100644 index 0000000000..e69de29bb2 diff --git a/developers/weaviate/concepts/img/supermarket.svg b/developers/weaviate/concepts/indexing/img/supermarket.svg similarity index 100% rename from developers/weaviate/concepts/img/supermarket.svg rename to developers/weaviate/concepts/indexing/img/supermarket.svg diff --git a/developers/weaviate/concepts/img/vectors-2d.png b/developers/weaviate/concepts/indexing/img/vectors-2d.png similarity index 100% rename from developers/weaviate/concepts/img/vectors-2d.png rename to developers/weaviate/concepts/indexing/img/vectors-2d.png diff --git a/developers/weaviate/concepts/img/vectors-2d.svg b/developers/weaviate/concepts/indexing/img/vectors-2d.svg similarity index 100% rename from developers/weaviate/concepts/img/vectors-2d.svg rename to developers/weaviate/concepts/indexing/img/vectors-2d.svg diff --git a/developers/weaviate/concepts/indexing/index.md b/developers/weaviate/concepts/indexing/index.md new file mode 100644 index 0000000000..d020446ba7 --- /dev/null +++ b/developers/weaviate/concepts/indexing/index.md @@ -0,0 +1,6 @@ +--- +title: Indexing +sidebar_position: 20 +image: og/docs/indexing.jpg +# tags: ['basics']['indexes'] +--- \ No newline at end of file diff --git a/developers/weaviate/concepts/indexing.md b/developers/weaviate/concepts/indexing/inverted-indexes.md similarity index 100% rename from developers/weaviate/concepts/indexing.md rename to developers/weaviate/concepts/indexing/inverted-indexes.md diff --git a/developers/weaviate/concepts/vector-index.md b/developers/weaviate/concepts/indexing/vector-indexes.md similarity index 95% rename from developers/weaviate/concepts/vector-index.md rename to developers/weaviate/concepts/indexing/vector-indexes.md index 31b8aa4a2a..0bb5a130a6 100644 --- a/developers/weaviate/concepts/vector-index.md +++ b/developers/weaviate/concepts/indexing/vector-indexes.md @@ -48,7 +48,10 @@ You might be also interested in our blog post [Why is vector search to fast?](ht **Hierarchical Navigable Small World (HNSW)** is an algorithm that works on multi-layered graphs. It is also an index type, and refers to vector indexes that are created using the HNSW algorithm. HNSW indexes enable very fast queries, but rebuilding the index when you add new vectors can be resource intensive. -Weaviate's `hnsw` index is a [custom implementation](../more-resources/faq.md#q-does-weaviate-use-hnswlib) of the Hierarchical Navigable Small World ([HNSW](https://arxiv.org/abs/1603.09320)) algorithm that offers full [CRUD-support](https://db-engines.com/en/blog_post/87). +Weaviate's `hnsw` index is a custom implementation of the Hierarchical Navigable Small World ([HNSW](https://arxiv.org/abs/1603.09320)) algorithm that offers full [CRUD-support](https://db-engines.com/en/blog_post/87). + + +DWCTODO: implementation https://github.com/weaviate/weaviate/tree/main/adapters/repos/db/vector/hnsw At build time, the HNSW algorithm creates a series of layers. At query time, the HNSW algorithm uses the layers to build a list of approximate nearest neighbors (ANN) quickly and efficiently. @@ -196,7 +199,7 @@ Vectors that aren't currently in cache are added to the cache if there is still ### Can I use vector indexing with vector quantization? -Yes, you can read more about it in [vector quantization (compression)](./vector-quantization.md). +Yes, you can read more about it in [vector quantization (compression)](DWCTODO). ### Which vector index is right for me? @@ -206,7 +209,7 @@ Note that the vector index type parameter only specifies how the vectors of data The `vectorizer` parameter determines how the data vectors are created (which numbers the vectors contain). `vectorizer` specifies a [module](/developers/weaviate/modules/index.md), such as `text2vec-contextionary`, that Weaviate uses to create the vectors. (You can also set to `vectorizer` to `none` if you want to import your own vectors). -To learn more about configuring the collection, see [this how-to page](../manage-data/collections.mdx). +To learn more about configuring the collection, see [this how-to page](/developers/weaviate/manage-data/collections). ### Which distance metrics can I use with vector indexing? @@ -214,7 +217,7 @@ All of [the distance metrics](/developers/weaviate/config-refs/distances.md), su ### How to configure the vector index type in Weaviate? -The index type can be specified per data collection via the [collection definition](/developers/weaviate/manage-data/collections.mdx#set-vector-index-type) settings, according to available [vector index settings](../config-refs/schema/vector-index.md). +The index type can be specified per data collection via the [collection definition](/developers/weaviate/manage-data/collections.mdx#set-vector-index-type) settings, according to available [vector index settings](/developers/weaviate/config-refs/schema/vector-index). ### When to skip indexing @@ -235,10 +238,10 @@ The [ANN benchmark page](/developers/weaviate/benchmarks/ann.md) contains a wide ## Further resources :::info Related pages -- [Concepts: Indexing](./indexing.md) -- [Concepts: Vector quantization (compression)](./vector-quantization.md) -- [Configuration: Vector index](../config-refs/schema/vector-index.md) -- [Configuration: Schema (Configure semantic indexing)](../config-refs/schema/index.md#configure-semantic-indexing) +- [Concepts: Inverted indexes](/developers/weaviate/concepts/indexing/inverted-indexes) +- [Concepts: Vector quantization (compression)](/developers/weaviate/concepts/vector-quantization) +- [Configuration: Vector index](/developers/weaviate/config-refs/schema/vector-index) +- [Configuration: Schema (Configure semantic indexing)](/developers/weaviate/config-refs/schema#configure-semantic-indexing) ::: ## Questions and feedback diff --git a/developers/weaviate/configuration/indexing-keyword/_category_.json b/developers/weaviate/configuration/indexing-keyword/_category_.json deleted file mode 100644 index ea3db5c162..0000000000 --- a/developers/weaviate/configuration/indexing-keyword/_category_.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "label": "Keyword indexes", - "position": 7 -} \ No newline at end of file diff --git a/developers/weaviate/configuration/indexing-keyword/index.md b/developers/weaviate/configuration/indexing-keyword/index.md deleted file mode 100644 index 46d80de1aa..0000000000 --- a/developers/weaviate/configuration/indexing-keyword/index.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -title: Indexing -sidebar_position: 0 -image: og/docs/indexes.jpg -# tags: ['configuration'] ---- - -This section provides examples of how to index Weaviate collections. - -Collection objects contain vectors and properties. Vector indexes let you search the collection's vectors quickly and efficiently. Inverted indexes make filtering and searching object properties faster. - -- [Configure vector indexes](/developers/weaviate/configuration/indexing/vector-indexes.md) -- [Configure inverted indexes](/developers/weaviate/configuration/indexing/inverted-indexes.md) - -## Related pages -- [Indexes overview](/developers/weaviate/starter-guides/managing-resources/indexing) - -## Questions and feedback - -import DocsFeedback from '/_includes/docs-feedback.mdx'; - - \ No newline at end of file diff --git a/developers/weaviate/configuration/indexing-vector/dynamic-indexes.md b/developers/weaviate/configuration/indexing-vector/dynamic-indexes.md index 489a508751..a3440d5c12 100644 --- a/developers/weaviate/configuration/indexing-vector/dynamic-indexes.md +++ b/developers/weaviate/configuration/indexing-vector/dynamic-indexes.md @@ -107,9 +107,9 @@ See also: ## Asynchronous indexing -import AsynchIndexing from '/_includes/indexes/async-indexing.mdx'; +import AsyncIndexing from '/_includes/indexes/async-indexing.mdx'; - + ## Related pages diff --git a/developers/weaviate/configuration/indexing-vector/index.md b/developers/weaviate/configuration/indexing-vector/index.md index 1cb4088e63..3e36104f5e 100644 --- a/developers/weaviate/configuration/indexing-vector/index.md +++ b/developers/weaviate/configuration/indexing-vector/index.md @@ -5,14 +5,6 @@ image: og/docs/indexes.jpg # tags: ['configuration'] --- -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock'; -import PyCodeV4 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v4.py'; -import PyCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v3.py'; -import TSCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v3.ts'; -import TSCodeV2 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v2.ts'; - import VectorIntro from '/_includes/indexes/vector-intro.mdx'; @@ -27,7 +19,7 @@ import HNSWIntro from '/_includes/indexes/hnsw-intro.mdx'; -[Configure an HNSW index](/developers/weaviate/configuration/indexing/hnsw-indexes.md). +[Configure an HNSW index](/developers/weaviate/configuration/indexing-vector/hnsw-indexes.md). See also: @@ -39,7 +31,7 @@ import FlatIntro from '/_includes/indexes/flat-intro.mdx'; -[Configure a flat index](/developers/weaviate/configuration/indexing/flat-indexes.md). +[Configure a flat index](/developers/weaviate/configuration/indexing-vector/flat-indexes.md). See also: @@ -53,7 +45,7 @@ import DynamicIntro from '/_includes/indexes/dynamic-intro.mdx'; Dynamic indexes require [asynchronous indexing](/developers/weaviate/config-refs/schema/vector-index#asynchronous-indexing). Enable asynchronous indexing before you configure a collection to use dynamic indexing. -[Configure a dynamic index](/developers/weaviate/configuration/indexing/dynamic-indexes.md). +[Configure a dynamic index](/developers/weaviate/configuration/indexing-vector/dynamic-indexes.md). See also: @@ -67,13 +59,15 @@ import IndexCompression from '/_includes/indexes/index-compression.mdx'; ## Asynchronous indexing -import AsynchIndexing from '/_includes/indexes/async-indexing.mdx'; +import AsyncIndexing from '/_includes/indexes/async-indexing.mdx'; - + ## Related pages - [Indexes overview](/developers/weaviate/starter-guides/managing-resources/indexing) +- [Configure inverted indexes](/developers/weaviate/configuration/inverted-indexes) + ## Questions and feedback diff --git a/developers/weaviate/configuration/indexing-keyword/inverted-indexes.md b/developers/weaviate/configuration/inverted-indexes.md similarity index 85% rename from developers/weaviate/configuration/indexing-keyword/inverted-indexes.md rename to developers/weaviate/configuration/inverted-indexes.md index e774126e50..d7f257e2af 100644 --- a/developers/weaviate/configuration/indexing-keyword/inverted-indexes.md +++ b/developers/weaviate/configuration/inverted-indexes.md @@ -1,6 +1,6 @@ --- title: Inverted indexes -sidebar_position: 60 +sidebar_position: 15 image: og/docs/indexes.jpg # tags: ['configuration'] --- @@ -13,13 +13,20 @@ import PyCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-inverte import TSCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-inverted-v3.ts'; import TSCodeV2 from '!!raw-loader!/_includes/code/howto/indexes/indexes-inverted-v2.ts'; -Weaviate uses [inverted indexes](/developers/weaviate/concepts/indexing#inverted-indexes), also known as keyword indexes, to make textual and numeric searches more efficient. Weaviate provides different kinds to inverted index so you can match better match the index to your data. These indexes are normally configured on a property level: +Weaviate uses [inverted indexes](/developers/weaviate/concepts/indexing#inverted-indexes), also known as keyword indexes, to make textual and numeric searches more efficient. Weaviate provides different kinds to inverted index so you can match better match the index to your data. + +These indexes are normally configured on a property level: - [indexSearchable](#indexSearchable) - [indexFilterable](#indexfilterable) - [indexRangeFilters](#indexrangefilters) -To tune inverted indexes at the collection level, use the `invertedIndexConfig` to adjust the settings for the [BM25 inverted index](#bm25-index) and other collection level parameters. +To tune inverted indexes at the collection level, adjust these configuration settings: + +- [BM25 search algorithm](#bm25) +- [indexNullState](#collection-level-properties) +- [indexPropertyLength](#collection-level-properties) +- [indexTimestamps](#collection-level-properties) ## indexSearchable @@ -66,7 +73,7 @@ If you don't anticipate searching on a property field, you can disable this inde The `indexFilterable` index improves [filtering](/developers/weaviate/search/filters). This index is enabled by default. -If you don't anticipate searching on a property field, you can disable this index to save disk space and import time. The property is still filterable. +If you don't anticipate searching on a property field, disable this index to save disk space and import time. The property is still filterable. Set these indexes on the property level. @@ -146,9 +153,13 @@ Set these indexes on the property level. -## bm25 index +## bm25 + +The [`bm25`](/developers/weaviate/config-refs/schema#bm25) ranking algorithm is configured on the collection level. The BM25 algorithm does not specify values for `b` or `k1`. Use these ["free parameters"](https://en.wikipedia.org/wiki/Free_parameter) to tailor the ranking algorithm for your data. + +To adjust for document length, modify `b`. Values range from 0 to 1. -The [`bm25` index](/developers/weaviate/config-refs/schema#bm25) is configured on the collection level. +To adjust for word frequency within a document, modify `k1`. Values are usually in the range from 0 to 3. There isn't an upper limit. @@ -227,6 +238,7 @@ These properties are configured on the collection level. ## Related pages - [Indexes overview](/developers/weaviate/starter-guides/managing-resources/indexing) +- [Configure vector indexes](/developers/weaviate/configuration/indexing-vector) ## Questions and feedback From 1ec1b36674010dae46e14495c10d420e66fb6ac9 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Thu, 19 Sep 2024 13:33:16 -0400 Subject: [PATCH 50/67] move around pages, update links --- developers/weaviate/concepts/indexing/vector-indexes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/developers/weaviate/concepts/indexing/vector-indexes.md b/developers/weaviate/concepts/indexing/vector-indexes.md index 0bb5a130a6..e51b399dfc 100644 --- a/developers/weaviate/concepts/indexing/vector-indexes.md +++ b/developers/weaviate/concepts/indexing/vector-indexes.md @@ -57,7 +57,7 @@ At build time, the HNSW algorithm creates a series of layers. At query time, the Consider this diagram of a vector search using HNSW. -![HNSW layers](./img/hnsw-layers.svg "HNSW layers") +![HNSW layers](../img/hnsw-layers.svg "HNSW layers") An individual object can exist in more than one layer, but every object in the database is represented in the lowest layer (layer zero in the picture). The layer zero data objects are very well connected to each other. Each layer above the lowest layer has fewer data object, and fewer connections. The data objects in the higher layers correspond to the objects in the lower layers, but each higher layer has exponentially fewer objects than the layer below. The HNSW algorithm takes advantage of the layers to efficiently process large amounts of data. From e484050ee17265684823dd10c4018a16be0c6487 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Thu, 19 Sep 2024 15:36:47 -0400 Subject: [PATCH 51/67] toc --- developers/weaviate/concepts/data.md | 2 +- .../concepts/indexing/dynamic-indexes.mdx | 6 +++ .../concepts/indexing/flat-indexes.mdx | 6 +++ .../concepts/indexing/hnsw-indexes.mdx | 6 +++ .../weaviate/concepts/indexing/index.md | 41 ++++++++++++++++++- .../concepts/indexing/inverted-indexes.md | 14 +++---- .../concepts/indexing/vector-indexes.md | 6 +-- developers/weaviate/concepts/interface.md | 4 +- developers/weaviate/concepts/modules.md | 2 +- developers/weaviate/concepts/prefiltering.md | 2 +- .../replication-architecture/_category_.json | 2 +- developers/weaviate/concepts/resources.md | 2 +- .../weaviate/concepts/search/_category_.json | 2 +- developers/weaviate/concepts/storage.md | 2 +- 14 files changed, 76 insertions(+), 21 deletions(-) diff --git a/developers/weaviate/concepts/data.md b/developers/weaviate/concepts/data.md index de6160a152..7ab1d9577f 100644 --- a/developers/weaviate/concepts/data.md +++ b/developers/weaviate/concepts/data.md @@ -1,5 +1,5 @@ --- -title: Data structure +title: Data structures sidebar_position: 10 image: og/docs/concepts.jpg # tags: ['basics'] diff --git a/developers/weaviate/concepts/indexing/dynamic-indexes.mdx b/developers/weaviate/concepts/indexing/dynamic-indexes.mdx index e69de29bb2..ea35eb4471 100644 --- a/developers/weaviate/concepts/indexing/dynamic-indexes.mdx +++ b/developers/weaviate/concepts/indexing/dynamic-indexes.mdx @@ -0,0 +1,6 @@ +--- +title: Dynamic indexed +sidebar_position: 40 +image: og/docs/indexing.jpg +# tags: ['basics']['indexes'] +--- \ No newline at end of file diff --git a/developers/weaviate/concepts/indexing/flat-indexes.mdx b/developers/weaviate/concepts/indexing/flat-indexes.mdx index e69de29bb2..dcfe86f362 100644 --- a/developers/weaviate/concepts/indexing/flat-indexes.mdx +++ b/developers/weaviate/concepts/indexing/flat-indexes.mdx @@ -0,0 +1,6 @@ +--- +title: Flat indexes +sidebar_position: 30 +image: og/docs/indexing.jpg +# tags: ['basics']['indexes'] +--- \ No newline at end of file diff --git a/developers/weaviate/concepts/indexing/hnsw-indexes.mdx b/developers/weaviate/concepts/indexing/hnsw-indexes.mdx index e69de29bb2..34440b6da1 100644 --- a/developers/weaviate/concepts/indexing/hnsw-indexes.mdx +++ b/developers/weaviate/concepts/indexing/hnsw-indexes.mdx @@ -0,0 +1,6 @@ +--- +title: HNSW Indexes +sidebar_position: 20 +image: og/docs/indexing.jpg +# tags: ['basics']['indexes'] +--- \ No newline at end of file diff --git a/developers/weaviate/concepts/indexing/index.md b/developers/weaviate/concepts/indexing/index.md index d020446ba7..e5287afffb 100644 --- a/developers/weaviate/concepts/indexing/index.md +++ b/developers/weaviate/concepts/indexing/index.md @@ -1,6 +1,43 @@ --- title: Indexing -sidebar_position: 20 +sidebar_position: 1 image: og/docs/indexing.jpg # tags: ['basics']['indexes'] ---- \ No newline at end of file +--- + +This section of the documentation discusses indexing concepts as they relate to Weaviate. + +For configuration examples without additional contextual material, see the [Vector index](/developers/weaviate/configuration/indexing-vector) and [Inverted index](/developers/weaviate/configuration/inverted-indexes) configuration pages. + +## Vector indexes + +Vector embeddings encapsulate meaning for the objects that they represent. Vector indexes make it easier efficiently search large vector spaces in order to find objects with similar vector embeddings. + +Weaviate offers these vector index types: + +- [Hierarchical Navigable Small World (HNSW) indexes](/developers/weaviate/concepts/indexing/hnsw-indexes) +- [Flat indexes](/developers/weaviate/concepts/indexing/flat-indexes) +- [Dynamic indexes](/developers/weaviate/configuration/indexing-vector/dynamic-indexes) + +## Inverted indexes + +Inverted indexes, also known as keyword indexes, make textual and numeric searches more efficient. Inverted indexes use the [BM25](/developers/weaviate/concepts/indexing/inverted-indexes#bm25-ranking-algorithm) ranking algorithm. + +Weaviate offers these inverted index types: + +- indexSearchable +- indexFilterable +- indexRangeFilters + +## Related pages + +For more information, see the following: + +- [Configure vector indexes](/developers/weaviate/configuration/indexing-vector) +- [Configure inverted indexes](/developers/weaviate/configuration/inverted-indexes) + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + \ No newline at end of file diff --git a/developers/weaviate/concepts/indexing/inverted-indexes.md b/developers/weaviate/concepts/indexing/inverted-indexes.md index b0be6b7ff3..219b0c2861 100644 --- a/developers/weaviate/concepts/indexing/inverted-indexes.md +++ b/developers/weaviate/concepts/indexing/inverted-indexes.md @@ -1,14 +1,10 @@ --- -title: Indexing -sidebar_position: 20 -image: og/docs/concepts.jpg +title: Inverted indexes +sidebar_position: 110 +image: og/docs/indexing.jpg # tags: ['basics'] --- -Weaviate supports several types of indexes. - -1. **Vector indexes** - a vector index (e.g. HNSW or flat) is used to serve all vector-search queries. -1. **Inverted indexes** - inverted indexes enable BM25 queries, or speed up filtering. You can configure indexes in Weaviate per collection. @@ -135,6 +131,10 @@ To create a collection without any indexes, skip indexing on the collection and } ``` +## BM25 ranking algorithm + +Some stuff + ## Further resources :::info Related pages diff --git a/developers/weaviate/concepts/indexing/vector-indexes.md b/developers/weaviate/concepts/indexing/vector-indexes.md index e51b399dfc..bad2783344 100644 --- a/developers/weaviate/concepts/indexing/vector-indexes.md +++ b/developers/weaviate/concepts/indexing/vector-indexes.md @@ -1,7 +1,7 @@ --- -title: Vector Indexing -sidebar_position: 23 -image: og/docs/concepts.jpg +title: Vector indexes overview +sidebar_position: 10 +image: og/docs/indexing.jpg # tags: ['vector index plugins'] --- diff --git a/developers/weaviate/concepts/interface.md b/developers/weaviate/concepts/interface.md index 4218ad5d28..9219de8f72 100644 --- a/developers/weaviate/concepts/interface.md +++ b/developers/weaviate/concepts/interface.md @@ -1,6 +1,6 @@ --- -title: Interface -sidebar_position: 85 +title: Interfaces +sidebar_position: 40 image: og/docs/concepts.jpg # tags: ['architecture', 'interface', 'API design'] --- diff --git a/developers/weaviate/concepts/modules.md b/developers/weaviate/concepts/modules.md index 1864e64f02..6992b90f29 100644 --- a/developers/weaviate/concepts/modules.md +++ b/developers/weaviate/concepts/modules.md @@ -1,6 +1,6 @@ --- title: Modules -sidebar_position: 15 +sidebar_position: 50 image: og/docs/concepts.jpg # tags: ['modules'] --- diff --git a/developers/weaviate/concepts/prefiltering.md b/developers/weaviate/concepts/prefiltering.md index ea06871c1e..0599f40379 100644 --- a/developers/weaviate/concepts/prefiltering.md +++ b/developers/weaviate/concepts/prefiltering.md @@ -1,6 +1,6 @@ --- title: Filtering -sidebar_position: 26 +sidebar_position: 20 image: og/docs/concepts.jpg # tags: ['architecture', 'filtered vector search', 'pre-filtering'] --- diff --git a/developers/weaviate/concepts/replication-architecture/_category_.json b/developers/weaviate/concepts/replication-architecture/_category_.json index 4cf28198bc..8c7776636f 100644 --- a/developers/weaviate/concepts/replication-architecture/_category_.json +++ b/developers/weaviate/concepts/replication-architecture/_category_.json @@ -1,4 +1,4 @@ { "label": "Replication Architecture", - "position": 35 + "position": 5 } \ No newline at end of file diff --git a/developers/weaviate/concepts/resources.md b/developers/weaviate/concepts/resources.md index 6b74daaf7d..84fd76a747 100644 --- a/developers/weaviate/concepts/resources.md +++ b/developers/weaviate/concepts/resources.md @@ -1,6 +1,6 @@ --- title: Resource Planning -sidebar_position: 90 +sidebar_position: 60 image: og/docs/concepts.jpg # tags: ['architecture', 'resource', 'cpu', 'memory', 'gpu'] --- diff --git a/developers/weaviate/concepts/search/_category_.json b/developers/weaviate/concepts/search/_category_.json index 47a6117993..eb356e15df 100644 --- a/developers/weaviate/concepts/search/_category_.json +++ b/developers/weaviate/concepts/search/_category_.json @@ -1,4 +1,4 @@ { "label": "Search", - "position": 5 + "position": 7 } diff --git a/developers/weaviate/concepts/storage.md b/developers/weaviate/concepts/storage.md index 6b358f0ff9..242a764bc2 100644 --- a/developers/weaviate/concepts/storage.md +++ b/developers/weaviate/concepts/storage.md @@ -1,6 +1,6 @@ --- title: Storage -sidebar_position: 18 +sidebar_position: 70 image: og/docs/concepts.jpg # tags: ['architecture', 'storage'] --- From bcea5b6cfd4a103662e0c5510f5470d64cf6b210 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Thu, 19 Sep 2024 17:14:25 -0400 Subject: [PATCH 52/67] tons of links --- _includes/dynamic-index-async-req.mdx | 4 +- _includes/indexes/dynamic-intro.mdx | 2 +- _includes/indexes/flat-intro.mdx | 2 +- _includes/indexes/hnsw-intro.mdx | 2 +- _includes/pq-compression/overview-text.mdx | 4 +- .../index.mdx | 2 +- .../index.mdx | 2 +- .../index.mdx | 2 +- .../index.mdx | 2 +- .../index.mdx | 2 +- .../index.mdx | 2 +- .../index.mdx | 2 +- .../_core-1-18-include.mdx | 2 +- .../index.mdx | 2 +- .../_core-1-20-include.mdx | 2 +- blog/2023-09-19-pq-rescoring/index.mdx | 2 +- .../_core-1-23-include.mdx | 2 +- .../index.mdx | 2 +- .../_core-1-24-include.mdx | 4 +- .../index.mdx | 2 +- blog/2024-04-02-binary-quantization/index.mdx | 2 +- developers/academy/py/compression/100_pq.mdx | 2 +- developers/academy/py/compression/200_bq.mdx | 3 +- .../academy/py/compression/900_next_steps.mdx | 2 +- .../academy/py/vector_index/100_overview.mdx | 3 +- .../academy/py/vector_index/200_hnsw.mdx | 2 +- .../academy/py/vector_index/220_flat.mdx | 3 +- .../academy/py/vector_index/250_dynamic.mdx | 3 +- .../py/vector_index/900_next_steps.mdx | 2 +- .../concepts/indexing/dynamic-indexes.mdx | 18 +++- .../concepts/indexing/flat-indexes.mdx | 11 ++- .../concepts/indexing/hnsw-indexes.mdx | 96 ++++++++++++++++++- .../concepts/indexing/inverted-indexes.md | 5 - .../concepts/indexing/vector-indexes.md | 94 +----------------- developers/weaviate/concepts/modules.md | 2 +- developers/weaviate/concepts/resources.md | 2 +- developers/weaviate/concepts/storage.md | 2 +- .../config-refs/schema/vector-index.md | 23 ++--- .../compression/bq-compression.md | 6 +- .../compression/pq-compression.md | 2 +- .../compression/sq-compression.md | 4 +- developers/weaviate/introduction.md | 4 +- .../weaviate/manage-data/collections.mdx | 4 +- developers/weaviate/more-resources/faq.md | 4 +- .../more-resources/migration/index.md | 2 +- .../weaviate/more-resources/performance.md | 2 +- .../managing-resources/index.md | 2 +- .../managing-resources/indexing.mdx | 18 ++-- developers/weaviate/starter-guides/schema.md | 4 +- site.redirects.js | 4 +- 50 files changed, 198 insertions(+), 179 deletions(-) diff --git a/_includes/dynamic-index-async-req.mdx b/_includes/dynamic-index-async-req.mdx index c0e1af1629..a6ba6feb4c 100644 --- a/_includes/dynamic-index-async-req.mdx +++ b/_includes/dynamic-index-async-req.mdx @@ -1,3 +1,3 @@ -:::info Dynamic index requires `ASYNC_INDEXING` -To use the `dynamic` vector index type, your Weaviate instance must have the `ASYNC_INDEXING` [environment variable](/developers/weaviate/config-refs/env-vars#general) enabled. This can be done by setting the `ASYNC_INDEXING` environment variable to `true`. For Weaviate Cloud users, this can be enabled through the Weaviate Cloud dashboard. +:::info Dynamic indexes require asynchronous indexing +The `dynamic` vector index type requires asynchronous indexing. To enable asynchronous indexing, set the `ASYNC_INDEXING` [environment variable](/developers/weaviate/config-refs/env-vars#general) to `true`. Weaviate Cloud users can enable asynchronous indexing in the Weaviate Cloud dashboard. ::: diff --git a/_includes/indexes/dynamic-intro.mdx b/_includes/indexes/dynamic-intro.mdx index 8adb6b4b01..04dd2e16a1 100644 --- a/_includes/indexes/dynamic-intro.mdx +++ b/_includes/indexes/dynamic-intro.mdx @@ -1 +1 @@ -[Dynamic index](/developers/weaviate/concepts/vector-index#dynamic-index) offer a flexible approach to indexing. A dynamic index begins as a flat index, and converts automatically to an HNSW index when a [collection](/developers/weaviate/concepts/data#collections) or [tenant](/developers/weaviate/concepts/data#multi-tenancy) reaches a threshold size. \ No newline at end of file +[Dynamic index](/developers/weaviate/concepts/indexing/vector-indexes/dynamic-indexes) offer a flexible approach to indexing. A dynamic index begins as a flat index, and converts automatically to an HNSW index when a [collection](/developers/weaviate/concepts/data#collections) or [tenant](/developers/weaviate/concepts/data#multi-tenancy) reaches a threshold size. \ No newline at end of file diff --git a/_includes/indexes/flat-intro.mdx b/_includes/indexes/flat-intro.mdx index 9881d90b31..231e72aeb2 100644 --- a/_includes/indexes/flat-intro.mdx +++ b/_includes/indexes/flat-intro.mdx @@ -1 +1 @@ -[Flat indexes](/developers/weaviate/concepts/vector-index#flat-index) are memory-efficient. They are disk based indexes that perform brute-force vector searches. Flat indexes work best with small collections, less than 10,000 objects. Vector search times increase linearly with the number of objects. \ No newline at end of file +[Flat indexes](/developers/weaviate/concepts/indexing/vector-indexes/flat-indexes) are memory-efficient. They are disk based indexes that perform brute-force vector searches. Flat indexes work best with small collections, less than 10,000 objects. Vector search times increase linearly with the number of objects. \ No newline at end of file diff --git a/_includes/indexes/hnsw-intro.mdx b/_includes/indexes/hnsw-intro.mdx index 6be8fc9801..9def68b57e 100644 --- a/_includes/indexes/hnsw-intro.mdx +++ b/_includes/indexes/hnsw-intro.mdx @@ -1 +1 @@ -[HNSW indexes](/developers/weaviate/concepts/vector-index#hierarchical-navigable-small-world-hnsw-index) are high-performance, in-memory indexes. HNSW indexes scale well, meaning that vector searches remain fast even for very large data sets. \ No newline at end of file +[HNSW indexes](/developers/weaviate/concepts/indexing/vector-indexes/hnsw-indexes) are high-performance, in-memory indexes. HNSW indexes scale well, meaning that vector searches remain fast even for very large data sets. \ No newline at end of file diff --git a/_includes/pq-compression/overview-text.mdx b/_includes/pq-compression/overview-text.mdx index c82e0dc4f8..aba30f05b8 100644 --- a/_includes/pq-compression/overview-text.mdx +++ b/_includes/pq-compression/overview-text.mdx @@ -1 +1,3 @@ -[Product quantization (PQ)](/developers/weaviate/concepts/vector-index#hnsw-with-product-quantizationpq) is a form of data compression for vectors. PQ reduces the memory footprint of a vector index, so enabling PQ for HNSW lets you work with larger datasets. For a discussion of how PQ saves memory, see [this page](/developers/weaviate/concepts/vector-quantization#product-quantization). +Product quantization (PQ) is a form of data compression for vectors. PQ reduces the memory requirements for vector indexes so you can use larger datasets. + +For a discussion of how PQ saves memory, see [Product quantization](/developers/weaviate/concepts/vector-quantization#product-quantization). diff --git a/blog/2022-07-04-weaviate-1-14-release/index.mdx b/blog/2022-07-04-weaviate-1-14-release/index.mdx index 63b3fbccc9..8349c88d51 100644 --- a/blog/2022-07-04-weaviate-1-14-release/index.mdx +++ b/blog/2022-07-04-weaviate-1-14-release/index.mdx @@ -193,7 +193,7 @@ The REST API CRUD operations now require you to use both an **object ID** and th This ensures that the operations are performed on the correct objects. ### Background -One of Weaviate's features is full CRUD support. CRUD operations enable the mutability of data objects and their vectors, which is a key difference between a vector database and an ANN library. In Weaviate, every data object has an ID (UUID). This ID is stored with the data object in a key-value store. IDs don't have to be globally unique, because in Weaviate [classes](/developers/weaviate/manage-data/collections) act as namespaces. While each class has a different [HNSW index](/developers/weaviate/concepts/vector-index#hnsw), including the store around it, which is isolated on disk. +One of Weaviate's features is full CRUD support. CRUD operations enable the mutability of data objects and their vectors, which is a key difference between a vector database and an ANN library. In Weaviate, every data object has an ID (UUID). This ID is stored with the data object in a key-value store. IDs don't have to be globally unique, because in Weaviate [classes](/developers/weaviate/manage-data/collections) act as namespaces. While each class has a different [HNSW index](/developers/weaviate/concepts/indexing/hnsw-indexes), including the store around it, which is isolated on disk. There was however one point in the API where reusing IDs between classes was causing serious issues. Most noticeable this was for the [v1/objects/{id}](/developers/weaviate/api/rest#tag/objects) REST endpoints. If you wanted to retrieve, modify or delete a data object by its ID, you would just need to specify the ID, without specifying the class name. So if the same ID exists for objects in multiple classes (which is fine because of the namespaces per class), Weaviate would not know which object to address and would address all objects with that ID instead. I.e. if you tried to delete an object by ID, this would result in the deletion of all objects with that ID. diff --git a/blog/2022-08-09-using-cross-encoders-as-reranker-in-multistage-vector-search/index.mdx b/blog/2022-08-09-using-cross-encoders-as-reranker-in-multistage-vector-search/index.mdx index 91af619455..2bf03b1ba2 100644 --- a/blog/2022-08-09-using-cross-encoders-as-reranker-in-multistage-vector-search/index.mdx +++ b/blog/2022-08-09-using-cross-encoders-as-reranker-in-multistage-vector-search/index.mdx @@ -25,7 +25,7 @@ Vector databases like [Weaviate](/developers/weaviate/) use Bi-Encoder machine l The models that compute **dense vector embeddings for data** (which later can be used for **search queries**) are so-called *Bi-Encoder models*. Data vectors and query vectors can be compared by computing the similarity (for example cosine similarity) between vectors. All data items in the database can be represented by a vector embedding using a Bi-Encoder model. -> Note. Vector databases **compute** dense vector embeddings **during import**. Calculating vectors up-front coupled with [ANN algorithms](/developers/weaviate/concepts/vector-index) (for faster retrieval) makes working with Bi-Encoder highly efficient. +> Note. Vector databases **compute** dense vector embeddings **during import**. Calculating vectors up-front coupled with [ANN algorithms](/developers/weaviate/concepts/indexing/vector-indexes) (for faster retrieval) makes working with Bi-Encoder highly efficient. ![Vector Database](./img/vector-database.png) diff --git a/blog/2022-09-13-why-is-vector-search-so-fast/index.mdx b/blog/2022-09-13-why-is-vector-search-so-fast/index.mdx index 1c338f38a7..e48e1d623f 100644 --- a/blog/2022-09-13-why-is-vector-search-so-fast/index.mdx +++ b/blog/2022-09-13-why-is-vector-search-so-fast/index.mdx @@ -98,7 +98,7 @@ Which algorithm works best depends on your project. Performance can be measured So, while ANN is not some magic method that will always find the true k nearest neighbors in a dataset, it can find a pretty good approximation of the true k neighbors. And it can do this in a fraction of the time! ## HNSW in Weaviate -[Weaviate](/developers/weaviate/) is a great example of a vector database that uses ANN algorithms to offer ultra-fast queries. The first ANN algorithm introduced to Weaviate is a custom implementation of [Hierarchical Navigable Small World graphs (HNSW)](/developers/weaviate/concepts/vector-index#hnsw). +[Weaviate](/developers/weaviate/) is a great example of a vector database that uses ANN algorithms to offer ultra-fast queries. The first ANN algorithm introduced to Weaviate is a custom implementation of [Hierarchical Navigable Small World graphs (HNSW)](/developers/weaviate/concepts/indexing/hnsw-indexes). :::tip Read more: -- [Concepts: Vector Indexing # Product Quantization](/developers/weaviate/concepts/vector-index#hnsw-with-product-quantizationpq) +- [Concepts: Vector Indexing # Product Quantization](/developers/weaviate/concepts/indexing/hnsw-indexes) - [How-to configure: Indexes # Vector index (see `pq`)](/developers/weaviate/config-refs/schema/vector-index#weaviates-vector-index) ::: diff --git a/blog/2023-09-19-pq-rescoring/index.mdx b/blog/2023-09-19-pq-rescoring/index.mdx index 2ce62b0a20..e77ee057ad 100644 --- a/blog/2023-09-19-pq-rescoring/index.mdx +++ b/blog/2023-09-19-pq-rescoring/index.mdx @@ -35,7 +35,7 @@ If you’d like to dig deeper into the inner workings of PQ and how these compre In this blog we cover how we can take this one step further, with 1.21 we introduced significant improvements to make PQ even more viable by reducing the recall price you pay in order to save on memory requirements. If you want to learn about these improvements and why using PQ should be a no-brainer now for most users, keep reading! :::info Enabling PQ -To learn how to enable and configure Weaviate to use PQ along with HNSW refer to the docs [here](/developers/weaviate/config-refs/schema/vector-index#how-to-configure-hnsw). You can read more about HNSW+PQ in the documentation [here](/developers/weaviate/concepts/vector-index#hnsw-with-product-quantizationpq). +To learn how to enable and configure Weaviate to use PQ along with HNSW refer to the docs [here](/developers/weaviate/config-refs/schema/vector-index#how-to-configure-hnsw). You can read more about HNSW+PQ in the documentation [here](/developers/weaviate/concepts/indexing/hnsw-indexes). ::: ## The problem with PQ diff --git a/blog/2023-12-19-weaviate-1-23-release/_core-1-23-include.mdx b/blog/2023-12-19-weaviate-1-23-release/_core-1-23-include.mdx index c698bec69e..3f53a87d6a 100644 --- a/blog/2023-12-19-weaviate-1-23-release/_core-1-23-include.mdx +++ b/blog/2023-12-19-weaviate-1-23-release/_core-1-23-include.mdx @@ -56,7 +56,7 @@ We expect that BQ will generally work better for vectors with higher dimensions. When BQ is enabled, a vector cache can be used to improve query performance by storing the quantized vectors of the most recently used data objects. Note that it must be balanced with memory usage considerations. -* Read more about the `flat` index [here](/developers/weaviate/concepts/vector-index#flat-index). +* Read more about the `flat` index [here](/developers/weaviate/concepts/indexing/flat-indexes). ## OSS LLM integration with `generative-anyscale` diff --git a/blog/2024-02-20-grpc-performance-improvements/index.mdx b/blog/2024-02-20-grpc-performance-improvements/index.mdx index ccc60a8279..920d73aa23 100644 --- a/blog/2024-02-20-grpc-performance-improvements/index.mdx +++ b/blog/2024-02-20-grpc-performance-improvements/index.mdx @@ -112,7 +112,7 @@ The chart shows that each query takes 40-70% less time to return results when us Query throughput is usually measured in queries per second (QPS). The chart shows query throughput values based on averages of 10,000 queries.
-The different settings relate to a [vector index (HNSW) parameter](https://weaviate.io/developers/weaviate/concepts/vector-index#hierarchical-navigable-small-world-hnsw-index) `ef`. +The different settings relate to a [vector index (HNSW) parameter](https://weaviate.io/developers/weaviate/concepts/indexing/hnsw-indexes) `ef`. This parameter sets the size of the dynamic candidate list used in the HNSW search algorithm, and drives the speed and accuracy trade-off. In this test, it will impact the ANN-search portion of each query. Regardless, you can see that even at high `ef` numbers the gRPC-driven improvements are significant. diff --git a/blog/2024-02-27-weaviate-1-24-release/_core-1-24-include.mdx b/blog/2024-02-27-weaviate-1-24-release/_core-1-24-include.mdx index 9c63f59dca..0bd37b9cd8 100644 --- a/blog/2024-02-27-weaviate-1-24-release/_core-1-24-include.mdx +++ b/blog/2024-02-27-weaviate-1-24-release/_core-1-24-include.mdx @@ -28,7 +28,7 @@ For details, see [Multiple vectors](/developers/weaviate/config-refs/schema/mult ![HNSW and binary quantization](./img/hnsw-and-bq.png) -Weaviate improves [binary quantization (BQ)](/developers/weaviate/configuration/compression/bq-compression) in 1.24 to be faster, more memory efficient, and more cost-effective. Use BQ vector compression with [HNSW indexes](/developers/weaviate/concepts/vector-index#hierarchical-navigable-small-world-hnsw-index) to dramatically improve your query speed. +Weaviate improves [binary quantization (BQ)](/developers/weaviate/configuration/compression/bq-compression) in 1.24 to be faster, more memory efficient, and more cost-effective. Use BQ vector compression with [HNSW indexes](/developers/weaviate/concepts/indexing/hnsw-indexes) to dramatically improve your query speed. BQ compresses vector representations while preserving essential information. Uncompressed, Weaviate uses a `float32` to store each dimension. BQ uses one bit per dimension to encode Vector directionality. This means BQ compresses vectors from 32 bits per dimension to 1 bit per dimension - a savings of 32 times the space. This compression significantly reduces storage requirements. @@ -52,7 +52,7 @@ Compare each value in the first vector one with the corresponding value in the s Efficient BQ processing compliments our HNSW index to enable mind dizzyingly fast vector search and throughput! -Let’s talk numbers! You might know from our v1.23 release that combining a [flat index](/developers/weaviate/concepts/vector-index#flat-index) with BQ enables real-time search capabilities. Even so, a brute force a search on 1 million vectors, where each vector has 768 dimensions, takes about 23 milliseconds with a flat index. When you use HNSW and BQ, even with vectors six times as big (4608 dimensions), the same brute force search only takes about 90 milliseconds. +Let’s talk numbers! You might know from our v1.23 release that combining a [flat index](/developers/weaviate/concepts/indexing/flat-indexes) with BQ enables real-time search capabilities. Even so, a brute force a search on 1 million vectors, where each vector has 768 dimensions, takes about 23 milliseconds with a flat index. When you use HNSW and BQ, even with vectors six times as big (4608 dimensions), the same brute force search only takes about 90 milliseconds. A flat index combined with BQ is fast, but compare a HNSW index combined with BQ. Together HNSW and BQ enable vector search capabilities that reach nearly 10,000 queries per second at 85% recall! diff --git a/blog/2024-03-12-enterprise-use-cases-weaviate/index.mdx b/blog/2024-03-12-enterprise-use-cases-weaviate/index.mdx index 49bad256c8..c82e91b110 100644 --- a/blog/2024-03-12-enterprise-use-cases-weaviate/index.mdx +++ b/blog/2024-03-12-enterprise-use-cases-weaviate/index.mdx @@ -54,7 +54,7 @@ These numerical representations are usually presented as an array of numbers and ![embeddings](./img/embeddings.png) -At Weaviate, we’ve implemented a custom [HNSW](https://weaviate.io/developers/weaviate/concepts/vector-index#hierarchical-navigable-small-world-hnsw-index) [indexing](https://weaviate.io/developers/weaviate/concepts/vector-index) algorithm to allow for highly performant querying over your dataset which enables all sorts of business use cases at scale. +At Weaviate, we’ve implemented a custom [HNSW](https://weaviate.io/developers/weaviate/concepts/indexing/hnsw-indexes) [indexing](https://weaviate.io/developers/weaviate/concepts/indexing/vector-indexes) algorithm to allow for highly performant querying over your dataset which enables all sorts of business use cases at scale. ## Features for Enterprise diff --git a/blog/2024-04-02-binary-quantization/index.mdx b/blog/2024-04-02-binary-quantization/index.mdx index bf2b5a516b..06a52a6994 100644 --- a/blog/2024-04-02-binary-quantization/index.mdx +++ b/blog/2024-04-02-binary-quantization/index.mdx @@ -160,7 +160,7 @@ So, what's the bottom line? Can Weaviate deliver lightning-fast brute-force sear There are several advantages to brute force searching your data. Firstly, you can bypass the need for data indexing, saving the time required to build the index. While indexing in Weaviate isn't overly sluggish, brute forcing allows you to skip this step entirely. Secondly, you no longer need to store a proximity graph, resulting in further memory savings. In fact, if you opt to brute force search your data directly from disk, memory usage becomes negligible – a mere 100MB is sufficient to host your application. -Weaviate recently introduced the [flat index](/developers/weaviate/concepts/vector-index#flat-index), offering the option to brute force data either from disk (the default behavior) or by retaining only compressed data in memory and fetching a small selection of full vectors from disk for final candidate rescoring. Both approaches expedite data ingestion compared to the traditional HNSW index while also reducing memory consumption. However, if your requirements demand high performance, HNSW remains the preferred choice. Nevertheless, the flat index presents a cost-effective, high-performing alternative. Furthermore, Weaviate now supports binary quantization (BQ) for use with both the flat and HNSW indexes. +Weaviate recently introduced the [flat index](/developers/weaviate/concepts/indexing/flat-indexes), offering the option to brute force data either from disk (the default behavior) or by retaining only compressed data in memory and fetching a small selection of full vectors from disk for final candidate rescoring. Both approaches expedite data ingestion compared to the traditional HNSW index while also reducing memory consumption. However, if your requirements demand high performance, HNSW remains the preferred choice. Nevertheless, the flat index presents a cost-effective, high-performing alternative. Furthermore, Weaviate now supports binary quantization (BQ) for use with both the flat and HNSW indexes. ### Indexing Time Improvements diff --git a/developers/academy/py/compression/100_pq.mdx b/developers/academy/py/compression/100_pq.mdx index fc37b833f8..691bf97094 100644 --- a/developers/academy/py/compression/100_pq.mdx +++ b/developers/academy/py/compression/100_pq.mdx @@ -8,7 +8,7 @@ import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBl import PyCode from '!!raw-loader!./_snippets/100_pq.py'; -[Product quantization](/developers/weaviate/concepts/vector-quantization#product-quantization) (PQ), is a technique used to compress vectors. In Weaviate, it can be used to reduce the size of the in-memory [HNSW index](/developers/weaviate/concepts/vector-index.md#hierarchical-navigable-small-world-hnsw-index), which can improve performance and reduce resource requirements as well as costs. +[Product quantization](/developers/weaviate/concepts/vector-quantization#product-quantization) (PQ), is a technique used to compress vectors. In Weaviate, it can be used to reduce the size of the in-memory [HNSW index](/developers/weaviate/concepts/indexing/hnsw-indexes), which can improve performance and reduce resource requirements as well as costs. ## What is product quantization? diff --git a/developers/academy/py/compression/200_bq.mdx b/developers/academy/py/compression/200_bq.mdx index 87985d0c40..c26c544f2e 100644 --- a/developers/academy/py/compression/200_bq.mdx +++ b/developers/academy/py/compression/200_bq.mdx @@ -8,7 +8,7 @@ import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBl import PyCode from '!!raw-loader!./_snippets/200_bq.py'; -[Binary quantization](/developers/weaviate/concepts/vector-quantization#binary-quantization) (BQ), is a technique used to compress vectors. In Weaviate, it can be used to reduce the size of the in-memory [HNSW index](/developers/weaviate/concepts/vector-index.md#hierarchical-navigable-small-world-hnsw-index) or the disk-based [flat index](/developers/weaviate/concepts/vector-index.md#flat-index). +[Binary quantization](/developers/weaviate/concepts/vector-quantization#binary-quantization) (BQ), is a technique used to compress vectors. In Weaviate, it can be used to reduce the size of the in-memory [HNSW index](/developers/weaviate/concepts/indexing/hnsw-indexes) or the disk-based [flat index](/developers/weaviate/concepts/indexing/flat-indexes). For HNSW, BQ can decrease its memory footprint and thus improve performance and reduce resource requirements as well as costs. For the flat index, BQ can reduce the size of the index on disk, which can improve performance. @@ -65,4 +65,3 @@ Some BQ parameters are configurable. An important one is `rescore_limit`, which import DocsFeedback from '/_includes/docs-feedback.mdx'; - diff --git a/developers/academy/py/compression/900_next_steps.mdx b/developers/academy/py/compression/900_next_steps.mdx index b0f61c9370..16718c6857 100644 --- a/developers/academy/py/compression/900_next_steps.mdx +++ b/developers/academy/py/compression/900_next_steps.mdx @@ -10,7 +10,7 @@ There are many more resources available to help you continue your learning journ - [How-to: Configure PQ](/developers/weaviate/configuration/compression/pq-compression.md) - [How-to: Configure BQ](/developers/weaviate/configuration/compression/bq-compression.md) -- [Concepts: vector indexes](/developers/weaviate/concepts/vector-index.md): Vector indexes are critical to how Weaviate works, and compression. Read more here. +- [Concepts: vector indexes](/developers/weaviate/concepts/indexing/vector-indexes): Vector indexes are critical to how Weaviate works, and compression. Read more here. - [Configuration references: Vector index](/developers/weaviate/config-refs/schema/vector-index.md): Configuration references for the vector index, including compression settings. import CTASocials from '../_snippets/cta_socials.mdx'; diff --git a/developers/academy/py/vector_index/100_overview.mdx b/developers/academy/py/vector_index/100_overview.mdx index eff6aed54e..1d0fbd8d25 100644 --- a/developers/academy/py/vector_index/100_overview.mdx +++ b/developers/academy/py/vector_index/100_overview.mdx @@ -16,7 +16,7 @@ A database index is a data structure that organizes data to make searches more e /> -A [vector index](/developers/weaviate/concepts/vector-index.md) is a specialized type of index that is designed to store and search vectors. +A [vector index](/developers/weaviate/concepts/indexing/vector-indexes) is a specialized type of index that is designed to store and search vectors. The choice and configuration of your vector index can significantly impact the performance of your imports as well as searches, and the resource requirements of your Weaviate instance. @@ -92,4 +92,3 @@ We will discuss these in more detail in the following sections. import DocsFeedback from '/_includes/docs-feedback.mdx'; - diff --git a/developers/academy/py/vector_index/200_hnsw.mdx b/developers/academy/py/vector_index/200_hnsw.mdx index 3b7c1c0bc9..3b73318446 100644 --- a/developers/academy/py/vector_index/200_hnsw.mdx +++ b/developers/academy/py/vector_index/200_hnsw.mdx @@ -183,7 +183,7 @@ There are more, advanced HNSW parameters that can be set in Weaviate. These are ## Further resources -- [Concepts: Vector index](/developers/weaviate/concepts/vector-index.md) +- [Concepts: Vector index](/developers/weaviate/concepts/indexing/vector-indexes) - [References: Vector index parameters](/developers/weaviate/config-refs/schema/vector-index.md) - [How-to manage collections](/developers/weaviate/manage-data/collections.mdx) - [Weaviate Academy: Compression](../compression/index.md) diff --git a/developers/academy/py/vector_index/220_flat.mdx b/developers/academy/py/vector_index/220_flat.mdx index ba9fcb984b..a1c2c0da67 100644 --- a/developers/academy/py/vector_index/220_flat.mdx +++ b/developers/academy/py/vector_index/220_flat.mdx @@ -62,7 +62,7 @@ Each of these parameters can be provided when creating a collection in Weaviate. ## Further resources -- [Concepts: Vector index](/developers/weaviate/concepts/vector-index.md) +- [Concepts: Vector index](/developers/weaviate/concepts/indexing/vector-indexes) - [References: Vector index parameters](/developers/weaviate/config-refs/schema/vector-index.md) - [How-to manage collections](/developers/weaviate/manage-data/collections.mdx) - [Weaviate Academy: Compression](../compression/index.md) @@ -72,4 +72,3 @@ Each of these parameters can be provided when creating a collection in Weaviate. import DocsFeedback from '/_includes/docs-feedback.mdx'; - diff --git a/developers/academy/py/vector_index/250_dynamic.mdx b/developers/academy/py/vector_index/250_dynamic.mdx index 8d8ab5f447..34d7dd8b03 100644 --- a/developers/academy/py/vector_index/250_dynamic.mdx +++ b/developers/academy/py/vector_index/250_dynamic.mdx @@ -67,7 +67,7 @@ Additionally, you can specify any of the `flat` and `hnsw` index parameters that ## Further resources -- [Concepts: Vector index](/developers/weaviate/concepts/vector-index.md) +- [Concepts: Vector index](/developers/weaviate/concepts/indexing/vector-indexes) - [References: Vector index parameters](/developers/weaviate/config-refs/schema/vector-index.md) - [How-to manage collections](/developers/weaviate/manage-data/collections.mdx) - [Weaviate Academy: Compression](../compression/index.md) @@ -77,4 +77,3 @@ Additionally, you can specify any of the `flat` and `hnsw` index parameters that import DocsFeedback from '/_includes/docs-feedback.mdx'; - diff --git a/developers/academy/py/vector_index/900_next_steps.mdx b/developers/academy/py/vector_index/900_next_steps.mdx index 72cb5819ba..efdbcf6972 100644 --- a/developers/academy/py/vector_index/900_next_steps.mdx +++ b/developers/academy/py/vector_index/900_next_steps.mdx @@ -18,7 +18,7 @@ Also, in a multi-tenant environment, the "dynamic" index may be a good default c These resources will help you continue your learning journey: -- [Concepts: Vector index](/developers/weaviate/concepts/vector-index.md) +- [Concepts: Vector index](/developers/weaviate/concepts/indexing/vector-indexes) - [References: Vector index parameters](/developers/weaviate/config-refs/schema/vector-index.md) - [How-to manage collections](/developers/weaviate/manage-data/collections.mdx) - [Weaviate Academy: Compression](../compression/index.md) diff --git a/developers/weaviate/concepts/indexing/dynamic-indexes.mdx b/developers/weaviate/concepts/indexing/dynamic-indexes.mdx index ea35eb4471..325107d5d2 100644 --- a/developers/weaviate/concepts/indexing/dynamic-indexes.mdx +++ b/developers/weaviate/concepts/indexing/dynamic-indexes.mdx @@ -3,4 +3,20 @@ title: Dynamic indexed sidebar_position: 40 image: og/docs/indexing.jpg # tags: ['basics']['indexes'] ---- \ No newline at end of file +--- + +:::caution Experimental feature +Available starting in `v1.25`. This is an experimental feature. Use with caution. +::: + +import DynamicAsyncRequirements from '/_includes/dynamic-index-async-req.mdx'; + + + +The flat index is ideal for use cases with a small object count and provides lower memory overhead and good latency. As the object count increases the HNSW index provides a more viable solution as HNSW speeds up search. The goal of the dynamic index is to shorten latencies during querying time at the cost of a larger memory footprint as you scale. + +By configuring a dynamic index, you can automatically switch from flat to HNSW indexes. This switch occurs when the object count exceeds a prespecified threshold (by default 10,000). This functionality only works with async indexing enabled. When the threshold is hit while importing, all the data piles up in the async queue, the HNSW index is constructed in the background and when ready the swap from flat to HNSW is completed. + +Currently, this is only a one-way upgrade from a flat to an HNSW index, it does not support changing back to a flat index even if the object count goes below the threshold due to deletion. + +This is particularly useful in a multi-tenant setup where building an HNSW index per tenant would introduce extra overhead. With a dynamic index, as individual tenants grow their index will switch from flat to HNSW, while smaller tenants' indexes remain flat. diff --git a/developers/weaviate/concepts/indexing/flat-indexes.mdx b/developers/weaviate/concepts/indexing/flat-indexes.mdx index dcfe86f362..68783801be 100644 --- a/developers/weaviate/concepts/indexing/flat-indexes.mdx +++ b/developers/weaviate/concepts/indexing/flat-indexes.mdx @@ -3,4 +3,13 @@ title: Flat indexes sidebar_position: 30 image: og/docs/indexing.jpg # tags: ['basics']['indexes'] ---- \ No newline at end of file +--- + +:::info Added in `v1.23` +::: + +The **flat index** is a simple, lightweight index that is fast to build and has a very small memory footprint. This index type is a good choice for use cases where each end user (i.e. tenant) has their own, isolated, dataset, such as in a SaaS product for example, or a database of isolated record sets. + +As the name suggests, the flat index is a single layer of disk-backed data objects and thus a very small memory footprint. The flat index is a good choice for small collections, such as for multi-tenancy use cases. + +A drawback of the flat index is that it does not scale well to large collections as it has a linear time complexity as a function of the number of data objects, unlike the `hnsw` index which has a logarithmic time complexity. diff --git a/developers/weaviate/concepts/indexing/hnsw-indexes.mdx b/developers/weaviate/concepts/indexing/hnsw-indexes.mdx index 34440b6da1..44ccfccd5a 100644 --- a/developers/weaviate/concepts/indexing/hnsw-indexes.mdx +++ b/developers/weaviate/concepts/indexing/hnsw-indexes.mdx @@ -3,4 +3,98 @@ title: HNSW Indexes sidebar_position: 20 image: og/docs/indexing.jpg # tags: ['basics']['indexes'] ---- \ No newline at end of file +--- + +**Hierarchical Navigable Small World (HNSW)** is an algorithm that works on multi-layered graphs. It is also an index type, and refers to vector indexes that are created using the HNSW algorithm. HNSW indexes enable very fast queries, but rebuilding the index when you add new vectors can be resource intensive. + +Weaviate's `hnsw` index is a custom implementation of the Hierarchical Navigable Small World ([HNSW](https://arxiv.org/abs/1603.09320)) algorithm that offers full [CRUD-support](https://db-engines.com/en/blog_post/87). + + +DWCTODO: implementation https://github.com/weaviate/weaviate/tree/main/adapters/repos/db/vector/hnsw + +At build time, the HNSW algorithm creates a series of layers. At query time, the HNSW algorithm uses the layers to build a list of approximate nearest neighbors (ANN) quickly and efficiently. + +Consider this diagram of a vector search using HNSW. + +![HNSW layers](../img/hnsw-layers.svg "HNSW layers") + +An individual object can exist in more than one layer, but every object in the database is represented in the lowest layer (layer zero in the picture). The layer zero data objects are very well connected to each other. Each layer above the lowest layer has fewer data object, and fewer connections. The data objects in the higher layers correspond to the objects in the lower layers, but each higher layer has exponentially fewer objects than the layer below. The HNSW algorithm takes advantage of the layers to efficiently process large amounts of data. + +When a search query comes in, the HNSW algorithm finds the closest matching data points in the highest layer. Then, HNSW goes one layer deeper, and finds the closest data points in that layer to the ones in the higher layer. These are the nearest neighbors. The algorithm searches the lower layer to create a new list of nearest neighbors. Then, HNSW uses the new list and repeats the process on the next layer down. When it gets to the deepest layer, the HNSW algorithm returns the data objects closest to the search query. + +Since there are relatively few data objects on the higher layers, HNSW has to search fewer objects. This means HNSW 'jumps' over large amounts of data that it doesn't need to search. When a data store has only one layer, the search algorithm can't skip unrelated objects. It has to search significantly more data objects even though they are unlikely to match. + +HNSW is very fast, memory efficient, approach to similarity search. The memory cache only stores the highest layer instead of storing all of the data objects in the lowest layer. When the search moves from a higher layer to a lower one, HNSW only adds the data objects that are closest to the search query. This means HNSW uses a relatively small amount of memory compared to other search algorithms. + +Have another look at the diagram; it demonstrates how the HNSW algorithm searches. The blue search vector in the top layer connects to a partial result in layer one. The objects in layer one lead HNSW to the result set in layer zero. HNSW makes three hops through the layers (the dotted blue lines) and skips objects that are unrelated to the search query. + +If your use case values fast data upload higher than super fast query time and high scalability, then other vector index types may be a better solution (e.g. [Spotify's Annoy](https://github.com/spotify/annoy)). + +### Managing search quality vs speed tradeoffs + +HNSW parameters can be adjusted to adjust search quality against speed. + +The `ef` parameter is a critical setting for balancing the trade-off between search speed and quality. + +The `ef` parameter dictates the size of the dynamic list used by the HNSW algorithm during the search process. A higher `ef` value results in a more extensive search, enhancing accuracy but potentially slowing down the query. + +In contrast, a lower `ef` makes the search faster but might compromise on accuracy. This balance is crucial in scenarios where either speed or accuracy is a priority. For instance, in applications where rapid responses are critical, a lower `ef` might be preferable, even at the expense of some accuracy. Conversely, in analytical or research contexts where precision is paramount, a higher `ef` would be more suitable, despite the increased query time. + +`ef` can be configured explicitly or dynamically. This feature is particularly beneficial in environments with varying query patterns. When `ef` is configured dynamically, Weaviate optimizes the balance between speed and recall based on real-time query requirements. + +To enable dynamic `ef`, set `ef`: -1. Weaviate adjusts the size of the ANN list based on the query response limit. The calculation also takes into account the values of `dynamicEfMin`, `dynamicEfMax`, and `dynamicEfFactor`. + +### Dynamic ef + +The `ef` parameter controls the size of the ANN list at query time. You can configure a specific list size or else let Weaviate configure the list dynamically. If you choose dynamic `ef`, Weaviate provides several options to control the size of the ANN list. + +The length of the list is determined by the query response limit that you set in your query. Weaviate uses the query limit as an anchor and modifies the size of ANN list according to the values you set for the `dynamicEf` parameters. + +- `dynamicEfMin` sets a lower bound on the list length. +- `dynamicEfMax` sets an upper bound on the list length. +- `dynamicEfFactor` sets a range for the list. + +To keep search recall high, the actual dynamic `ef` value stays above `dynamicEfMin` even if the query limit is small enough to suggest a lower value. + +To keep search speed reasonable even when retrieving large result sets, the dynamic `ef` value is limited to `dynamicEfMax`. Weaviate doesn't exceed `dynamicEfMax` even if the query limit is large enough to suggest a higher value. If the query limit is higher than `dynamicEfMax`, `dynamicEfMax` does not have any effect. In this case, dynamic `ef` value is equal to the query limit. + +To determine the length of the ANN list, Weaviate multiples the query limit by `dynamicEfFactor`. The list range is modified by `dynamicEfMin` and `dynamicEfMax`. + +Consider this GraphQL query that sets a limit of 4. + +```graphql +{ + Get { + JeopardyQuestion(limit: 4) { + answer + question + } + } +} +``` + +Imagine the collection has dynamic `ef` configured. + +```json + "vectorIndexConfig": { + "ef": -1, + "dynamicEfMin": 5 + "dynamicEfMax": 25 + "dynamicEfFactor": 10 + } +``` + +The resulting search list has these characteristics. + +- A potential length of 40 objects ( ("dynamicEfFactor": 10) * (limit: 4) ). +- A minimum length of 5 objects ("dynamicEfMin": 5). +- A maximum length of 25 objects ("dynamicEfMax": 25). +- An actual size of 5 to 25 objects. + +If you use the [`docker-compose.yml` file from Weaviate](/developers/weaviate/installation/docker-compose) to run your local instance, the `QUERY_DEFAULTS_LIMIT` environment variable sets a reasonable default query limit. To prevent out of memory errors,`QUERY_DEFAULTS_LIMIT` is significantly lower than `QUERY_MAXIMUM_RESULTS`. + +To change the default limit, edit the value for `QUERY_DEFAULTS_LIMIT` when you configure your Weaviate instance. + +### Deletions + +Cleanup is an async process runs that rebuilds the HNSW graph after deletes and updates. Prior to cleanup, objects are marked as deleted, but they are still connected to the HNSW graph. During cleanup, the edges are reassigned and the objects are deleted for good. diff --git a/developers/weaviate/concepts/indexing/inverted-indexes.md b/developers/weaviate/concepts/indexing/inverted-indexes.md index 219b0c2861..69084943fe 100644 --- a/developers/weaviate/concepts/indexing/inverted-indexes.md +++ b/developers/weaviate/concepts/indexing/inverted-indexes.md @@ -5,7 +5,6 @@ image: og/docs/indexing.jpg # tags: ['basics'] --- - You can configure indexes in Weaviate per collection. Some things to bear in mind: @@ -137,12 +136,8 @@ Some stuff ## Further resources -:::info Related pages - [Concepts: Vector Indexing](./vector-index.md) - [Configuration: Vector index](../config-refs/schema/vector-index.md) -::: - - ## Questions and feedback diff --git a/developers/weaviate/concepts/indexing/vector-indexes.md b/developers/weaviate/concepts/indexing/vector-indexes.md index bad2783344..9aa93d93a3 100644 --- a/developers/weaviate/concepts/indexing/vector-indexes.md +++ b/developers/weaviate/concepts/indexing/vector-indexes.md @@ -46,99 +46,7 @@ You might be also interested in our blog post [Why is vector search to fast?](ht ## Hierarchical Navigable Small World (HNSW) index -**Hierarchical Navigable Small World (HNSW)** is an algorithm that works on multi-layered graphs. It is also an index type, and refers to vector indexes that are created using the HNSW algorithm. HNSW indexes enable very fast queries, but rebuilding the index when you add new vectors can be resource intensive. - -Weaviate's `hnsw` index is a custom implementation of the Hierarchical Navigable Small World ([HNSW](https://arxiv.org/abs/1603.09320)) algorithm that offers full [CRUD-support](https://db-engines.com/en/blog_post/87). - - -DWCTODO: implementation https://github.com/weaviate/weaviate/tree/main/adapters/repos/db/vector/hnsw - -At build time, the HNSW algorithm creates a series of layers. At query time, the HNSW algorithm uses the layers to build a list of approximate nearest neighbors (ANN) quickly and efficiently. - -Consider this diagram of a vector search using HNSW. - -![HNSW layers](../img/hnsw-layers.svg "HNSW layers") - -An individual object can exist in more than one layer, but every object in the database is represented in the lowest layer (layer zero in the picture). The layer zero data objects are very well connected to each other. Each layer above the lowest layer has fewer data object, and fewer connections. The data objects in the higher layers correspond to the objects in the lower layers, but each higher layer has exponentially fewer objects than the layer below. The HNSW algorithm takes advantage of the layers to efficiently process large amounts of data. - -When a search query comes in, the HNSW algorithm finds the closest matching data points in the highest layer. Then, HNSW goes one layer deeper, and finds the closest data points in that layer to the ones in the higher layer. These are the nearest neighbors. The algorithm searches the lower layer to create a new list of nearest neighbors. Then, HNSW uses the new list and repeats the process on the next layer down. When it gets to the deepest layer, the HNSW algorithm returns the data objects closest to the search query. - -Since there are relatively few data objects on the higher layers, HNSW has to search fewer objects. This means HNSW 'jumps' over large amounts of data that it doesn't need to search. When a data store has only one layer, the search algorithm can't skip unrelated objects. It has to search significantly more data objects even though they are unlikely to match. - -HNSW is very fast, memory efficient, approach to similarity search. The memory cache only stores the highest layer instead of storing all of the data objects in the lowest layer. When the search moves from a higher layer to a lower one, HNSW only adds the data objects that are closest to the search query. This means HNSW uses a relatively small amount of memory compared to other search algorithms. - -Have another look at the diagram; it demonstrates how the HNSW algorithm searches. The blue search vector in the top layer connects to a partial result in layer one. The objects in layer one lead HNSW to the result set in layer zero. HNSW makes three hops through the layers (the dotted blue lines) and skips objects that are unrelated to the search query. - -If your use case values fast data upload higher than super fast query time and high scalability, then other vector index types may be a better solution (e.g. [Spotify's Annoy](https://github.com/spotify/annoy)). - -### Managing search quality vs speed tradeoffs - -HNSW parameters can be adjusted to adjust search quality against speed. - -The `ef` parameter is a critical setting for balancing the trade-off between search speed and quality. - -The `ef` parameter dictates the size of the dynamic list used by the HNSW algorithm during the search process. A higher `ef` value results in a more extensive search, enhancing accuracy but potentially slowing down the query. - -In contrast, a lower `ef` makes the search faster but might compromise on accuracy. This balance is crucial in scenarios where either speed or accuracy is a priority. For instance, in applications where rapid responses are critical, a lower `ef` might be preferable, even at the expense of some accuracy. Conversely, in analytical or research contexts where precision is paramount, a higher `ef` would be more suitable, despite the increased query time. - -`ef` can be configured explicitly or dynamically. This feature is particularly beneficial in environments with varying query patterns. When `ef` is configured dynamically, Weaviate optimizes the balance between speed and recall based on real-time query requirements. - -To enable dynamic `ef`, set `ef`: -1. Weaviate adjusts the size of the ANN list based on the query response limit. The calculation also takes into account the values of `dynamicEfMin`, `dynamicEfMax`, and `dynamicEfFactor`. - -### Dynamic ef - -The `ef` parameter controls the size of the ANN list at query time. You can configure a specific list size or else let Weaviate configure the list dynamically. If you choose dynamic `ef`, Weaviate provides several options to control the size of the ANN list. - -The length of the list is determined by the query response limit that you set in your query. Weaviate uses the query limit as an anchor and modifies the size of ANN list according to the values you set for the `dynamicEf` parameters. - -- `dynamicEfMin` sets a lower bound on the list length. -- `dynamicEfMax` sets an upper bound on the list length. -- `dynamicEfFactor` sets a range for the list. - -To keep search recall high, the actual dynamic `ef` value stays above `dynamicEfMin` even if the query limit is small enough to suggest a lower value. - -To keep search speed reasonable even when retrieving large result sets, the dynamic `ef` value is limited to `dynamicEfMax`. Weaviate doesn't exceed `dynamicEfMax` even if the query limit is large enough to suggest a higher value. If the query limit is higher than `dynamicEfMax`, `dynamicEfMax` does not have any effect. In this case, dynamic `ef` value is equal to the query limit. - -To determine the length of the ANN list, Weaviate multiples the query limit by `dynamicEfFactor`. The list range is modified by `dynamicEfMin` and `dynamicEfMax`. - -Consider this GraphQL query that sets a limit of 4. - -```graphql -{ - Get { - JeopardyQuestion(limit: 4) { - answer - question - } - } -} -``` - -Imagine the collection has dynamic `ef` configured. - -```json - "vectorIndexConfig": { - "ef": -1, - "dynamicEfMin": 5 - "dynamicEfMax": 25 - "dynamicEfFactor": 10 - } -``` - -The resulting search list has these characteristics. - -- A potential length of 40 objects ( ("dynamicEfFactor": 10) * (limit: 4) ). -- A minimum length of 5 objects ("dynamicEfMin": 5). -- A maximum length of 25 objects ("dynamicEfMax": 25). -- An actual size of 5 to 25 objects. - -If you use the [`docker-compose.yml` file from Weaviate](/developers/weaviate/installation/docker-compose) to run your local instance, the `QUERY_DEFAULTS_LIMIT` environment variable sets a reasonable default query limit. To prevent out of memory errors,`QUERY_DEFAULTS_LIMIT` is significantly lower than `QUERY_MAXIMUM_RESULTS`. - -To change the default limit, edit the value for `QUERY_DEFAULTS_LIMIT` when you configure your Weaviate instance. - -### Deletions - -Cleanup is an async process runs that rebuilds the HNSW graph after deletes and updates. Prior to cleanup, objects are marked as deleted, but they are still connected to the HNSW graph. During cleanup, the edges are reassigned and the objects are deleted for good. +Add summary ### Asynchronous indexing diff --git a/developers/weaviate/concepts/modules.md b/developers/weaviate/concepts/modules.md index 6992b90f29..ded297c75c 100644 --- a/developers/weaviate/concepts/modules.md +++ b/developers/weaviate/concepts/modules.md @@ -15,7 +15,7 @@ Weaviate has a modularized structure. Functionality such as vectorization or bac The core of Weaviate, without any modules attached, is a pure vector-native database. [![Weaviate modules introduction](./img/weaviate-module-diagram.svg "Weaviate Module Diagram")](./img/weaviate-module-diagram.svg) -Data is stored in Weaviate as the combination of an object and its vector, and these vectors are searchable by the provided [vector index algorithm](../concepts/vector-index.md). Without any vectorizer modules attached, Weaviate does not know how to *vectorize* an object, i.e. *how* to calculate the vectors given an object. +Data is stored in Weaviate as the combination of an object and its vector, and these vectors are searchable by the provided [vector index algorithm](../concepts/indexing/vector-indexes). Without any vectorizer modules attached, Weaviate does not know how to *vectorize* an object, i.e. *how* to calculate the vectors given an object. Depending on the type of data you want to store and search (text, images, etc.), and depending on the use case (like search, question answering, etc., depending on language, classification, ML model, training set, etc.), you can choose and attach a vectorizer module that best fits your use case. Or, you can "bring your own" vectors to Weaviate. diff --git a/developers/weaviate/concepts/resources.md b/developers/weaviate/concepts/resources.md index 84fd76a747..214965fed1 100644 --- a/developers/weaviate/concepts/resources.md +++ b/developers/weaviate/concepts/resources.md @@ -102,7 +102,7 @@ To avoid out-of-memory issues during imports, set `LIMIT_RESOURCES` to `True` or The following tactics can help to reduce Weaviate's memory usage: -- **Use vector compression**. Product quantization (PQ) is a technique that reduces the size of vectors. Vector compression impacts recall performance, so we recommend testing PQ on your dataset before using it in production.

For more information, see [Product Quantization](../concepts/vector-index.md#hnsw-with-product-quantization-pq).
To configure PQ, see [Compression](../configuration/compression/pq-compression.md). +- **Use vector compression**. Product quantization (PQ) is a technique that reduces the size of vectors. Vector compression impacts recall performance, so we recommend testing PQ on your dataset before using it in production.

For more information, see [Product Quantization](../concepts/indexing/hnsw-indexes).
To configure PQ, see [Compression](../configuration/compression/pq-compression.md). - **Reduce the dimensionality of your vectors.** The most effective approach to reducing memory size, is to reduce the number of dimensions per vector. If you have high dimension vectors, consider using a model that uses fewer dimensions. For example, a model that has 384 dimensions uses far less memory than a model with 1536 dimensions. diff --git a/developers/weaviate/concepts/storage.md b/developers/weaviate/concepts/storage.md index 242a764bc2..c0db13ceef 100644 --- a/developers/weaviate/concepts/storage.md +++ b/developers/weaviate/concepts/storage.md @@ -24,7 +24,7 @@ Each shard houses three main components: * An object store, essentially a key-value store * An [inverted index](https://en.wikipedia.org/wiki/Inverted_index) -* A vector index store (plugable, currently a [custom implementation of HNSW](/developers/weaviate/concepts/vector-index.md#hnsw)) +* A vector index store (plugable, currently a [custom implementation of HNSW](/developers/weaviate/concepts/indexing/hnsw-indexes)) #### Object and Inverted Index Store diff --git a/developers/weaviate/config-refs/schema/vector-index.md b/developers/weaviate/config-refs/schema/vector-index.md index 5866b3cc4f..0bd5d97b08 100644 --- a/developers/weaviate/config-refs/schema/vector-index.md +++ b/developers/weaviate/config-refs/schema/vector-index.md @@ -19,16 +19,16 @@ Some HNSW parameters are mutable, but others cannot be modified after you create | :-- | :-- | :-- | :-- | :-- | | `cleanupIntervalSeconds` | integer | 300 | Yes | Cleanup frequency. This value does not normally need to be adjusted. A higher value means cleanup runs less frequently, but it does more in a single batch. A lower value means cleanup is more frequent, but it may be less efficient on each run. | | `distance` | string | `cosine` | No | Distance metric. The metric that measures the distance between two arbitrary vectors. For available distance metrics, see [supported distance metrics](/developers/weaviate/config-refs/distances.md). | -| `ef` | integer | -1 | Yes | Balance search speed and recall. `ef` is the size of the dynamic list that the HNSW uses during search. Search is more accurate when `ef` is higher, but it is also slower. `ef` values greater than 512 show diminishing improvements in recall.

Dynamic `ef`. Weaviate automatically adjusts the `ef` value and creates a dynamic `ef` list when `ef` is set to -1. For more details, see [dynamic ef](../../concepts/vector-index.md#dynamic-ef). | +| `ef` | integer | -1 | Yes | Balance search speed and recall. `ef` is the size of the dynamic list that the HNSW uses during search. Search is more accurate when `ef` is higher, but it is also slower. `ef` values greater than 512 show diminishing improvements in recall.

Dynamic `ef`. Weaviate automatically adjusts the `ef` value and creates a dynamic `ef` list when `ef` is set to -1. For more details, see [dynamic ef](../../concepts/indexing/hnsw-indexes#dynamic-ef). | | `efConstruction` | integer | 128 | No | Balance index search speed and build speed. A high `efConstruction` value means you can lower your `ef` settings, but importing is slower.

`efConstruction` must be greater than 0. | | `maxConnections` | integer | 32 | No | Maximum number of connections per element. `maxConnections` is the connection limit per layer for layers above the zero layer. The zero layer can have (2 * maxConnections) connections.

`maxConnections` must be greater than 0. | -| `dynamicEfMin` | integer | 100 | Yes | *New in `v1.10.0`.*

Lower bound for [dynamic `ef`](../../concepts/vector-index.md#dynamic-ef). Protects against a creating search list that is too short.

This setting is only used when `ef` is -1. | -| `dynamicEfMax` | integer | 500 | Yes | *New in `v1.10.0`.*

Upper bound for [dynamic `ef`](../../concepts/vector-index.md#dynamic-ef). Protects against creating a search list that is too long.

If `dynamicEfMax` is higher than the limit, `dynamicEfMax` does not have any effect. In this case, `ef` is the limit.

This setting is only used when `ef` is -1. | -| `dynamicEfFactor` | integer | 8 | Yes | *New in `v1.10.0`.*

Multiplier for [dynamic `ef`](../../concepts/vector-index.md#dynamic-ef). Sets the potential length of the search list.

This setting is only used when `ef` is -1. | +| `dynamicEfMin` | integer | 100 | Yes | *New in `v1.10.0`.*

Lower bound for [dynamic `ef`](../../concepts/indexing/hnsw-indexes#dynamic-ef). Protects against a creating search list that is too short.

This setting is only used when `ef` is -1. | +| `dynamicEfMax` | integer | 500 | Yes | *New in `v1.10.0`.*

Upper bound for [dynamic `ef`](../../concepts/indexing/hnsw-indexes#dynamic-ef). Protects against creating a search list that is too long.

If `dynamicEfMax` is higher than the limit, `dynamicEfMax` does not have any effect. In this case, `ef` is the limit.

This setting is only used when `ef` is -1. | +| `dynamicEfFactor` | integer | 8 | Yes | *New in `v1.10.0`.*

Multiplier for [dynamic `ef`](../../concepts/indexing/hnsw-indexes#dynamic-ef). Sets the potential length of the search list.

This setting is only used when `ef` is -1. | | `flatSearchCutoff` | integer | 40000 | Yes | Optional. Threshold for the [flat-search cutoff](/developers/weaviate/concepts/prefiltering.md#flat-search-cutoff). To force a vector index search, set `"flatSearchCutoff": 0`. | -| `skip` | boolean | `false` | No | When true, do not index the collection.

Weaviate decouples vector creation and vector storage. If you skip vector indexing, but a vectorizer is configured (or a vector is provided manually), Weaviate logs a warning each import.

To skip indexing and vector generation, set `"vectorizer": "none"` when you set `"skip": true`.

See [When to skip indexing](../../concepts/vector-index.md#when-to-skip-indexing). | -| `vectorCacheMaxObjects`| integer | `1e12` | Yes | Maximum number of objects in the memory cache. By default, this limit is set to one trillion (`1e12`) objects when a new collection is created. For sizing recommendations, see [Vector cache considerations](../../concepts/vector-index.md#vector-cache-considerations). | -| `pq` | object | -- | Yes | Enable and configure [product quantization (PQ)](/developers/weaviate/concepts/vector-index.md#hnsw-with-product-quantizationpq) compression.

PQ assumes some data has already been loaded. You should have 10,000 to 100,000 vectors per shard loaded before you enable PQ.

For PQ configuration details, see [PQ configuration parameters](#pq-configuration-parameters). | +| `skip` | boolean | `false` | No | When true, do not index the collection.

Weaviate decouples vector creation and vector storage. If you skip vector indexing, but a vectorizer is configured (or a vector is provided manually), Weaviate logs a warning each import.

To skip indexing and vector generation, set `"vectorizer": "none"` when you set `"skip": true`.

See [When to skip indexing](../../concepts/indexing/hnsw-indexes#when-to-skip-indexing). | +| `vectorCacheMaxObjects`| integer | `1e12` | Yes | Maximum number of objects in the memory cache. By default, this limit is set to one trillion (`1e12`) objects when a new collection is created. For sizing recommendations, see [Vector cache considerations](../../concepts/indexing/hnsw-indexes). | +| `pq` | object | -- | Yes | Enable and configure [product quantization (PQ)](/developers/weaviate/indexing/hnsw-indexes) compression.

PQ assumes some data has already been loaded. You should have 10,000 to 100,000 vectors per shard loaded before you enable PQ.

For PQ configuration details, see [PQ configuration parameters](#pq-configuration-parameters). | ### Database parameters for HNSW @@ -140,7 +140,7 @@ Flat indexes are recommended for use cases where the number of objects per index | Parameter | Type | Default | Changeable | Details | | :-- | :-- | :-- | :-- | :-- | -| `vectorCacheMaxObjects`| integer | `1e12` | Yes | Maximum number of objects in the memory cache. By default, this limit is set to one trillion (`1e12`) objects when a new collection is created. For sizing recommendations, see [Vector cache considerations](../../concepts/vector-index.md#vector-cache-considerations). | +| `vectorCacheMaxObjects`| integer | `1e12` | Yes | Maximum number of objects in the memory cache. By default, this limit is set to one trillion (`1e12`) objects when a new collection is created. For sizing recommendations, see [Vector cache considerations](../../concepts/indexing/vector-indexes). | | `bq` | object | -- | No | Enable and configure [binary quantization (BQ)](../../concepts/vector-quantization.md#binary-quantization) compression.

For BQ configuration details, see [BQ configuration parameters](#bq-configuration-parameters). | ### BQ configuration parameters @@ -194,10 +194,11 @@ Use these parameters to configure the index type and their properties. They can
How to select the index type -Generally, the `hnsw` index type is recommended for most use cases. The `flat` index type is recommended for use cases where the data the number of objects per index is low, such as in multi-tenancy cases. You can also opt for the `dynamic` index which will initially configure a `flat` index and once the object count exceeds a specified threshold it will automatically convert to an `hnsw` index. +The `hnsw` index type is recommended for most use cases. -See [this section](../../concepts/vector-index.md#which-vector-index-is-right-for-me) for more information about the different index types and how to choose between them. +The `flat` index type is recommended for use cases where the number of objects per index is low, such as in multi-tenancy cases. +The `dynamic` index is useful if your collection size starts small and grows. A dynamic index starts as a flat index and converts to an hnsw index when the object count exceeds a specified threshold.
If faster import speeds are desired, [asynchronous indexing](#asynchronous-indexing) allows de-coupling of indexing from object creation. @@ -303,7 +304,7 @@ import MultiVectorSupport from '/_includes/multi-vector-support.mdx'; ## Related pages - [Concepts: Indexing](../../concepts/indexing.md) -- [Concepts: Vector Indexing](../../concepts/vector-index.md) +- [Concepts: Vector Indexing](../../concepts/indexing/vector-indexes) ## Questions and feedback diff --git a/developers/weaviate/configuration/compression/bq-compression.md b/developers/weaviate/configuration/compression/bq-compression.md index f48c7621bc..e86fef8a88 100644 --- a/developers/weaviate/configuration/compression/bq-compression.md +++ b/developers/weaviate/configuration/compression/bq-compression.md @@ -17,7 +17,7 @@ import GoCode from '!!raw-loader!/_includes/code/howto/configure.bq-compression. import JavaCode from '!!raw-loader!/_includes/code/howto/java/src/test/java/io/weaviate/docs/bq-compression.java'; :::info Added in `v1.23` -BQ is available for the [`flat` index](/developers/weaviate/concepts/vector-index.md#flat-index) type from `v1.23` onwards and for the [`hnsw` index](/developers/weaviate/concepts/vector-index.md#hnsw-index) type from `v1.24`. +BQ is available for the [`flat` index](/developers/weaviate/concepts/indexing/flat-indexes) type from `v1.23` onwards and for the [`hnsw` index](/developers/weaviate/concepts/indexing/hnsw-indexes) type from `v1.24`. ::: Binary quantization (BQ) is a vector compression technique that can reduce the size of a vector. @@ -104,7 +104,7 @@ The following parameters are available for BQ compression, under `vectorIndexCon | `bq` : `enabled` | boolean | `false` | Enable BQ. Weaviate uses binary quantization (BQ) compression when `true`.

The Python client v4 does not use the `enabled` parameter. To enable BQ with the v4 client, set a `quantizer` in the collection definition. | | `bq` : `rescoreLimit` | integer | -1 | The minimum number of candidates to fetch before rescoring. | | `bq` : `cache` | boolean | `false` | Whether to use the vector cache. | -| `vectorCacheMaxObjects` | integer | `1e12` | Maximum number of objects in the memory cache. By default, this limit is set to one trillion (`1e12`) objects when a new collection is created. For sizing recommendations, see [Vector cache considerations](/developers/weaviate/concepts/vector-index.md#vector-cache-considerations). | +| `vectorCacheMaxObjects` | integer | `1e12` | Maximum number of objects in the memory cache. By default, this limit is set to one trillion (`1e12`) objects when a new collection is created. For sizing recommendations, see [Vector cache considerations](/developers/weaviate/concepts/indexing/vector-indexes). | For example: @@ -173,7 +173,7 @@ import MultiVectorCompress from '/_includes/multi-vector-compress.mdx'; ## Related pages - [Configuration: Vector index](/developers/weaviate/config-refs/schema/vector-index.md) -- [Concepts: Vector index](/developers/weaviate/concepts/vector-index.md) +- [Concepts: Vector index](/developers/weaviate/concepts/indexing/vector-indexes) - [Concepts: Vector quantization](/developers/weaviate/concepts/vector-quantization.md) - [Tutorial: Schema](/developers/weaviate/starter-guides/schema) diff --git a/developers/weaviate/configuration/compression/pq-compression.md b/developers/weaviate/configuration/compression/pq-compression.md index 7b76bb99d6..0d2313b39d 100644 --- a/developers/weaviate/configuration/compression/pq-compression.md +++ b/developers/weaviate/configuration/compression/pq-compression.md @@ -375,7 +375,7 @@ import MultiVectorCompress from '/_includes/multi-vector-compress.mdx'; ## Related pages - [Configuration: Vector index](/developers/weaviate/config-refs/schema/vector-index.md) -- [Concepts: Vector index](/developers/weaviate/concepts/vector-index.md) +- [Concepts: Vector index](/developers/weaviate/concepts/indexing/vector-indexes) - [Concepts: Vector quantization](/developers/weaviate/concepts/vector-quantization.md) - [Guide: Schemas and collection definitions](/developers/weaviate/starter-guides/schema) diff --git a/developers/weaviate/configuration/compression/sq-compression.md b/developers/weaviate/configuration/compression/sq-compression.md index 985d4be8c7..41f40fd01f 100644 --- a/developers/weaviate/configuration/compression/sq-compression.md +++ b/developers/weaviate/configuration/compression/sq-compression.md @@ -60,7 +60,7 @@ To tune SQ, set these `vectorIndexConfig` parameters. | `sq`: `rescoreLimit` | integer | -1 | The minimum number of candidates to fetch before rescoring. | | `sq`: `trainingLimit` | integer | 100000 | The size of the training set to determine scalar bucket boundaries. | | `sq`: `cache` | boolean | `false` | Use the vector cache when true. | -| `vectorCacheMaxObjects` | integer | `1e12` | Maximum number of objects in the memory cache. By default, this limit is set to one trillion (`1e12`) objects when a new collection is created. For sizing recommendations, see [Vector cache considerations](/developers/weaviate/concepts/vector-index.md#vector-cache-considerations). | +| `vectorCacheMaxObjects` | integer | `1e12` | Maximum number of objects in the memory cache. By default, this limit is set to one trillion (`1e12`) objects when a new collection is created. For sizing recommendations, see [Vector cache considerations](/developers/weaviate/concepts/indexing/vector-indexes). | @@ -91,7 +91,7 @@ import MultiVectorCompress from '/_includes/multi-vector-compress.mdx'; ## Related pages - [Configuration: Vector index](/developers/weaviate/config-refs/schema/vector-index.md) -- [Concepts: Vector index](/developers/weaviate/concepts/vector-index.md) +- [Concepts: Vector index](/developers/weaviate/concepts/indexing/vector-indexes) - [Concepts: Vector quantization](/developers/weaviate/concepts/vector-quantization.md) - [Tutorial: Schema](/developers/weaviate/starter-guides/schema) diff --git a/developers/weaviate/introduction.md b/developers/weaviate/introduction.md index 5935218f42..87f639b80f 100644 --- a/developers/weaviate/introduction.md +++ b/developers/weaviate/introduction.md @@ -128,7 +128,7 @@ Weaviate is an open source vector database that stores both objects and vectors. **Weaviate in a nutshell**: * Weaviate is an open source [vector database](https://weaviate.io/blog/what-is-a-vector-database). -* Weaviate allows you to store and retrieve data objects based on their semantic properties by indexing them with [vectors](./concepts/vector-index.md). +* Weaviate allows you to store and retrieve data objects based on their semantic properties by indexing them with [vectors](./concepts/indexing/vector-indexes). * Weaviate can be used stand-alone (aka _bring your vectors_) or with a variety of [modules](./modules/index.md) that can do the vectorization for you and extend the core capabilities. * Weaviate has a [GraphQL-API](./api/graphql/index.md) to access your data easily. * Weaviate is fast (check our [open source benchmarks](./benchmarks/index.md)). @@ -186,7 +186,7 @@ Within Weaviate, all individual data objects are based on a class property struc You can add data to Weaviate through the [RESTful API](/developers/weaviate/api/rest) end-points and retrieve data through the [GraphQL interface](./api/graphql/index.md). -Weaviate's [vector indexing mechanism is modular](./concepts/vector-index.md), and the current available plugin is the Hierarchical Navigable Small World (HNSW) multilayered graph. +Weaviate's [vector indexing mechanism is modular](./concepts/indexing/vector-indexes), and the current available plugin is the Hierarchical Navigable Small World (HNSW) multilayered graph. ## What are Weaviate modules? diff --git a/developers/weaviate/manage-data/collections.mdx b/developers/weaviate/manage-data/collections.mdx index 0b171dd6d3..b63d7f3876 100644 --- a/developers/weaviate/manage-data/collections.mdx +++ b/developers/weaviate/manage-data/collections.mdx @@ -382,7 +382,7 @@ The vector index type can be set for each collection at creation time, between `
Additional information -- Read more about index types & compression in [Concepts: Vector index](../concepts/vector-index.md). +- Read more about index types & compression in [Concepts: Vector index](../concepts/indexing/vector-indexes).
@@ -440,7 +440,7 @@ Various vector index parameters are configurable at collection creation time, in
Additional information -- Read more about index types & compression in [Concepts: Vector index](../concepts/vector-index.md). +- Read more about index types & compression in [Concepts: Vector index](../concepts/indexing/vector-indexes).
diff --git a/developers/weaviate/more-resources/faq.md b/developers/weaviate/more-resources/faq.md index 1ab6f7cb1f..1869163a08 100644 --- a/developers/weaviate/more-resources/faq.md +++ b/developers/weaviate/more-resources/faq.md @@ -338,9 +338,9 @@ If you need a higher search quality for a given limit you can consider the follo > More information: > > - [Weaviate, an ANN Database with CRUD support – DB-Engines.com](https://db-engines.com/en/blog_post/87) ⬅️ best resource on the topic -> - [Weaviate's HNSW implementation in the docs](/developers/weaviate/concepts/vector-index.md#hnsw) +> - [Weaviate's HNSW implementation in the docs](/developers/weaviate/concepts/indexing/hnsw-indexes) > -> _Note I: HNSW is just one implementation in Weaviate, but Weaviate can support multiple indexing algoritmns as outlined [here](/developers/weaviate/concepts/vector-index.md)_ +> _Note I: HNSW is just one implementation in Weaviate, but Weaviate can support multiple indexing algoritmns as outlined [here](/developers/weaviate/concepts/indexing/vector-indexes)_
diff --git a/developers/weaviate/more-resources/migration/index.md b/developers/weaviate/more-resources/migration/index.md index 6ad50decab..548c74118b 100644 --- a/developers/weaviate/more-resources/migration/index.md +++ b/developers/weaviate/more-resources/migration/index.md @@ -624,7 +624,7 @@ With the modularization, it becomes possible to vectorize non-text objects. Sear * The `vectorizer` indicates which module (if any) are responsible for vectorization. * The `moduleConfig` allows configuration per module (by name). * See [here](#text2vec-contextionary) for Contextionary specific property configuration. - * The `vectorIndexType` allows the choosing the vector index (defaults to [HNSW](/developers/weaviate/concepts/vector-index.md#hnsw)) + * The `vectorIndexType` allows the choosing the vector index (defaults to [HNSW](/developers/weaviate/concepts/indexing/hnsw-indexes)) * The `vectorIndexConfig` is an arbitrary object passed to the index for config (defaults can be found [here](/developers/weaviate/config-refs/schema/vector-index.md#how-to-configure-hnsw) ) All changes are in this example: diff --git a/developers/weaviate/more-resources/performance.md b/developers/weaviate/more-resources/performance.md index ea2e17fad2..89c6a30055 100644 --- a/developers/weaviate/more-resources/performance.md +++ b/developers/weaviate/more-resources/performance.md @@ -28,7 +28,7 @@ The inverted index currently does not do any weighing (e.g. tf-idf) for sorting, ## Vector index Everything that has a vector, thus every data object in Weaviate, is also indexed in the vector index. Weaviate currently supports [HNSW](https://arxiv.org/abs/1603.09320) and flat vector indexes. -See [Concepts: vector index](../concepts/vector-index.md) for more information about the vector index. +See [Concepts: vector index](../concepts/indexing/vector-indexes) for more information about the vector index. ## Costs of queries and operations diff --git a/developers/weaviate/starter-guides/managing-resources/index.md b/developers/weaviate/starter-guides/managing-resources/index.md index bba5048aea..59b47a6ca6 100644 --- a/developers/weaviate/starter-guides/managing-resources/index.md +++ b/developers/weaviate/starter-guides/managing-resources/index.md @@ -168,7 +168,7 @@ Consider a strategy of deactivating tenants that are not frequently accessed, an - [Starter guide: Compression](./compression.mdx) - [Starter guide: Indexing](./indexing.mdx) - [Starter guide: Tenant states](./tenant-states.mdx) -- [Concepts: Vector Index](../../concepts/vector-index.md) +- [Concepts: Vector Index](../../concepts/indexing/vector-indexes) - [Concepts: Vector Quantization](../../concepts/vector-quantization.md) - [Concepts: Multi-Tenancy](../../concepts/data.md#multi-tenancy) - [How-to: Set the vector index type](../../manage-data/collections.mdx#set-vector-index-type) diff --git a/developers/weaviate/starter-guides/managing-resources/indexing.mdx b/developers/weaviate/starter-guides/managing-resources/indexing.mdx index 826cb238a9..607d3494ae 100644 --- a/developers/weaviate/starter-guides/managing-resources/indexing.mdx +++ b/developers/weaviate/starter-guides/managing-resources/indexing.mdx @@ -65,19 +65,21 @@ As a result, flat indexes are best suited for cases where the number of objects :::info Added in `v1.25` ::: -import DynamicAsyncRequirements from '/_includes/dynamic-index-async-req.mdx'; - - - import DynamicIntro from '/_includes/indexes/dynamic-intro.mdx'; This can be particularly useful in multi-tenant configurations, where different tenants may have different numbers of objects. With a dynamic index, you can avoid the overhead of an HNSW index when it's not needed. -The threshold size is 10,000 objects by default. You can configure the threshold size when you create the dynamic index. +The `dynamic` vector index type requires asynchronous indexing. To enable asynchronous indexing, set the `ASYNC_INDEXING` [environment variable](/developers/weaviate/config-refs/env-vars#general) to `true`. + +Configure the settings for the flat and HNSW indexes when you create the collection definition. The dynamic index uses the flat index configuration when the collection is small. It converts the flat index to an HNSW index when the collection size reaches the dynamic index threshold. + +The threshold size is 10,000 objects by default. Configure the threshold size when you create the dynamic index. -This table shows how a dynamic index changes as the number of objects in a collection grows. The assumed set up is a dynamic index with: +##### Dynamic index behavior + +This table shows how a dynamic index changes as the number of objects in a collection grows. The table assumes a dynamic index configure like this: - A threshold of 10,000 objects. - Flat index + BQ configuration. @@ -90,10 +92,6 @@ This table shows how a dynamic index changes as the number of objects in a colle | 100,000 | HNSW | Training | The collection object count == PQ/SQ training threshold. | | 100,001 | HNSW | PQ/SQ | PQ/SQ is active. | -:::info Dynamic index requires flat and HNSW index settings -A dynamic index requires its flat and HNSW index settings at creation time. The dynamic index will use the flat index settings initially, then automatically switch to the HNSW index with provided settings when the threshold is reached. -::: - ### Asynchronous vector indexing :::info Added in `v1.22` diff --git a/developers/weaviate/starter-guides/schema.md b/developers/weaviate/starter-guides/schema.md index a6dd0d96e4..e5e30451cc 100644 --- a/developers/weaviate/starter-guides/schema.md +++ b/developers/weaviate/starter-guides/schema.md @@ -211,9 +211,9 @@ import SchemaWithMT from '/_includes/code/tutorial.schema.multi-tenancy.mdx'; ### Index settings -Weaviate uses two types of indexes: [vector indexes](../concepts/vector-index.md) and [inverted indexes](../concepts/indexing.md#inverted-indexes). Vector indexes are used to store and organize vectors for fast vector similarity-based searches. Inverted indexes are used to store data for fast filtering and keyword searches. +Weaviate uses two types of indexes: [vector indexes](../concepts/indexing/vector-indexes) and [inverted indexes](../concepts/indexing.md#inverted-indexes). Vector indexes are used to store and organize vectors for fast vector similarity-based searches. Inverted indexes are used to store data for fast filtering and keyword searches. -The default vector index type is [HNSW](../concepts/vector-index.md#hierarchical-navigable-small-world-hnsw-index). The other options are [flat](../concepts/vector-index.md#flat-index), which is suitable for small collections, such as those in a multi-tenancy collection, or [dynamic](../concepts/vector-index.md#dynamic-index), which starts as a flat index before switching to an HNSW index if its size grows beyond a predetermined threshold. +The default vector index type is [HNSW](../concepts/indexing/hnsw-indexes). The other options are [flat](../concepts/indexing/flat-indexes), which is suitable for small collections, such as those in a multi-tenancy collection, or [dynamic](../concepts/indexing/dynamic-indexes), which starts as a flat index before switching to an HNSW index if its size grows beyond a predetermined threshold. import SchemaWithIndexSettings from '/_includes/code/tutorial.schema.index-settings.mdx'; diff --git a/site.redirects.js b/site.redirects.js index 61a530528c..6c5745b48d 100644 --- a/site.redirects.js +++ b/site.redirects.js @@ -19,7 +19,7 @@ const siteRedirects = { from: '/developers/weaviate/current/core-knowledge/basics', }, { - to: '/developers/weaviate/concepts/vector-index', + to: '/developers/weaviate/concepts/indexing/vector-indexes', from: [ '/developers/weaviate/current/vector-index-plugins', '/developers/weaviate/current/vector-index-plugins/hnsw', @@ -219,7 +219,7 @@ const siteRedirects = { // Remove BPR page { - to: '/developers/weaviate/concepts/vector-index', + to: '/developers/weaviate/concepts/indexing/vector-indexes', from: '/developers/weaviate/concepts/binary-passage-retrieval', }, From e9bd2b2ad6f84887f336f13963aac0c228aa34ce Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Thu, 19 Sep 2024 22:13:40 -0400 Subject: [PATCH 53/67] more links --- _includes/indexes/dynamic-intro.mdx | 2 +- _includes/indexes/flat-intro.mdx | 2 +- _includes/indexes/hnsw-intro.mdx | 2 +- developers/weaviate/concepts/modules.md | 2 +- developers/weaviate/concepts/resources.md | 2 +- .../config-refs/schema/vector-index.md | 18 +++++++++--------- .../weaviate/manage-data/collections.mdx | 4 ++-- .../weaviate/more-resources/performance.md | 2 +- .../starter-guides/managing-resources/index.md | 2 +- developers/weaviate/starter-guides/schema.md | 4 ++-- 10 files changed, 20 insertions(+), 20 deletions(-) diff --git a/_includes/indexes/dynamic-intro.mdx b/_includes/indexes/dynamic-intro.mdx index 04dd2e16a1..f61d013ddd 100644 --- a/_includes/indexes/dynamic-intro.mdx +++ b/_includes/indexes/dynamic-intro.mdx @@ -1 +1 @@ -[Dynamic index](/developers/weaviate/concepts/indexing/vector-indexes/dynamic-indexes) offer a flexible approach to indexing. A dynamic index begins as a flat index, and converts automatically to an HNSW index when a [collection](/developers/weaviate/concepts/data#collections) or [tenant](/developers/weaviate/concepts/data#multi-tenancy) reaches a threshold size. \ No newline at end of file +[Dynamic index](/developers/weaviate/concepts/indexing/dynamic-indexes) offer a flexible approach to indexing. A dynamic index begins as a flat index, and converts automatically to an HNSW index when a [collection](/developers/weaviate/concepts/data#collections) or [tenant](/developers/weaviate/concepts/data#multi-tenancy) reaches a threshold size. \ No newline at end of file diff --git a/_includes/indexes/flat-intro.mdx b/_includes/indexes/flat-intro.mdx index 231e72aeb2..a06fd93829 100644 --- a/_includes/indexes/flat-intro.mdx +++ b/_includes/indexes/flat-intro.mdx @@ -1 +1 @@ -[Flat indexes](/developers/weaviate/concepts/indexing/vector-indexes/flat-indexes) are memory-efficient. They are disk based indexes that perform brute-force vector searches. Flat indexes work best with small collections, less than 10,000 objects. Vector search times increase linearly with the number of objects. \ No newline at end of file +[Flat indexes](/developers/weaviate/concepts/indexing/flat-indexes) are memory-efficient. They are disk based indexes that perform brute-force vector searches. Flat indexes work best with small collections, less than 10,000 objects. Vector search times increase linearly with the number of objects. \ No newline at end of file diff --git a/_includes/indexes/hnsw-intro.mdx b/_includes/indexes/hnsw-intro.mdx index 9def68b57e..c93d28fb8e 100644 --- a/_includes/indexes/hnsw-intro.mdx +++ b/_includes/indexes/hnsw-intro.mdx @@ -1 +1 @@ -[HNSW indexes](/developers/weaviate/concepts/indexing/vector-indexes/hnsw-indexes) are high-performance, in-memory indexes. HNSW indexes scale well, meaning that vector searches remain fast even for very large data sets. \ No newline at end of file +[HNSW indexes](/developers/weaviate/concepts/indexing/hnsw-indexes) are high-performance, in-memory indexes. HNSW indexes scale well, meaning that vector searches remain fast even for very large data sets. \ No newline at end of file diff --git a/developers/weaviate/concepts/modules.md b/developers/weaviate/concepts/modules.md index ded297c75c..2da6295da2 100644 --- a/developers/weaviate/concepts/modules.md +++ b/developers/weaviate/concepts/modules.md @@ -15,7 +15,7 @@ Weaviate has a modularized structure. Functionality such as vectorization or bac The core of Weaviate, without any modules attached, is a pure vector-native database. [![Weaviate modules introduction](./img/weaviate-module-diagram.svg "Weaviate Module Diagram")](./img/weaviate-module-diagram.svg) -Data is stored in Weaviate as the combination of an object and its vector, and these vectors are searchable by the provided [vector index algorithm](../concepts/indexing/vector-indexes). Without any vectorizer modules attached, Weaviate does not know how to *vectorize* an object, i.e. *how* to calculate the vectors given an object. +Data is stored in Weaviate as the combination of an object and its vector, and these vectors are searchable by the provided [vector index algorithm](/developers/weaviate/concepts/indexing/vector-indexes). Without any vectorizer modules attached, Weaviate does not know how to *vectorize* an object, i.e. *how* to calculate the vectors given an object. Depending on the type of data you want to store and search (text, images, etc.), and depending on the use case (like search, question answering, etc., depending on language, classification, ML model, training set, etc.), you can choose and attach a vectorizer module that best fits your use case. Or, you can "bring your own" vectors to Weaviate. diff --git a/developers/weaviate/concepts/resources.md b/developers/weaviate/concepts/resources.md index 214965fed1..a1bfe789a2 100644 --- a/developers/weaviate/concepts/resources.md +++ b/developers/weaviate/concepts/resources.md @@ -102,7 +102,7 @@ To avoid out-of-memory issues during imports, set `LIMIT_RESOURCES` to `True` or The following tactics can help to reduce Weaviate's memory usage: -- **Use vector compression**. Product quantization (PQ) is a technique that reduces the size of vectors. Vector compression impacts recall performance, so we recommend testing PQ on your dataset before using it in production.

For more information, see [Product Quantization](../concepts/indexing/hnsw-indexes).
To configure PQ, see [Compression](../configuration/compression/pq-compression.md). +- **Use vector compression**. Product quantization (PQ) is a technique that reduces the size of vectors. Vector compression impacts recall performance, so we recommend testing PQ on your dataset before using it in production.

For more information, see [Product Quantization](/developers/weaviate/concepts/indexing/hnsw-indexes).
To configure PQ, see [Compression](../configuration/compression/pq-compression.md). - **Reduce the dimensionality of your vectors.** The most effective approach to reducing memory size, is to reduce the number of dimensions per vector. If you have high dimension vectors, consider using a model that uses fewer dimensions. For example, a model that has 384 dimensions uses far less memory than a model with 1536 dimensions. diff --git a/developers/weaviate/config-refs/schema/vector-index.md b/developers/weaviate/config-refs/schema/vector-index.md index 0bd5d97b08..e63343958e 100644 --- a/developers/weaviate/config-refs/schema/vector-index.md +++ b/developers/weaviate/config-refs/schema/vector-index.md @@ -19,15 +19,15 @@ Some HNSW parameters are mutable, but others cannot be modified after you create | :-- | :-- | :-- | :-- | :-- | | `cleanupIntervalSeconds` | integer | 300 | Yes | Cleanup frequency. This value does not normally need to be adjusted. A higher value means cleanup runs less frequently, but it does more in a single batch. A lower value means cleanup is more frequent, but it may be less efficient on each run. | | `distance` | string | `cosine` | No | Distance metric. The metric that measures the distance between two arbitrary vectors. For available distance metrics, see [supported distance metrics](/developers/weaviate/config-refs/distances.md). | -| `ef` | integer | -1 | Yes | Balance search speed and recall. `ef` is the size of the dynamic list that the HNSW uses during search. Search is more accurate when `ef` is higher, but it is also slower. `ef` values greater than 512 show diminishing improvements in recall.

Dynamic `ef`. Weaviate automatically adjusts the `ef` value and creates a dynamic `ef` list when `ef` is set to -1. For more details, see [dynamic ef](../../concepts/indexing/hnsw-indexes#dynamic-ef). | +| `ef` | integer | -1 | Yes | Balance search speed and recall. `ef` is the size of the dynamic list that the HNSW uses during search. Search is more accurate when `ef` is higher, but it is also slower. `ef` values greater than 512 show diminishing improvements in recall.

Dynamic `ef`. Weaviate automatically adjusts the `ef` value and creates a dynamic `ef` list when `ef` is set to -1. For more details, see [dynamic ef](/developers/weaviate/concepts/indexing/hnsw-indexes#dynamic-ef). | | `efConstruction` | integer | 128 | No | Balance index search speed and build speed. A high `efConstruction` value means you can lower your `ef` settings, but importing is slower.

`efConstruction` must be greater than 0. | | `maxConnections` | integer | 32 | No | Maximum number of connections per element. `maxConnections` is the connection limit per layer for layers above the zero layer. The zero layer can have (2 * maxConnections) connections.

`maxConnections` must be greater than 0. | -| `dynamicEfMin` | integer | 100 | Yes | *New in `v1.10.0`.*

Lower bound for [dynamic `ef`](../../concepts/indexing/hnsw-indexes#dynamic-ef). Protects against a creating search list that is too short.

This setting is only used when `ef` is -1. | -| `dynamicEfMax` | integer | 500 | Yes | *New in `v1.10.0`.*

Upper bound for [dynamic `ef`](../../concepts/indexing/hnsw-indexes#dynamic-ef). Protects against creating a search list that is too long.

If `dynamicEfMax` is higher than the limit, `dynamicEfMax` does not have any effect. In this case, `ef` is the limit.

This setting is only used when `ef` is -1. | -| `dynamicEfFactor` | integer | 8 | Yes | *New in `v1.10.0`.*

Multiplier for [dynamic `ef`](../../concepts/indexing/hnsw-indexes#dynamic-ef). Sets the potential length of the search list.

This setting is only used when `ef` is -1. | +| `dynamicEfMin` | integer | 100 | Yes | *New in `v1.10.0`.*

Lower bound for [dynamic `ef`](/developers/weaviate/concepts/indexing/hnsw-indexes#dynamic-ef). Protects against a creating search list that is too short.

This setting is only used when `ef` is -1. | +| `dynamicEfMax` | integer | 500 | Yes | *New in `v1.10.0`.*

Upper bound for [dynamic `ef`](/developers/weaviate/concepts/indexing/hnsw-indexes#dynamic-ef). Protects against creating a search list that is too long.

If `dynamicEfMax` is higher than the limit, `dynamicEfMax` does not have any effect. In this case, `ef` is the limit.

This setting is only used when `ef` is -1. | +| `dynamicEfFactor` | integer | 8 | Yes | *New in `v1.10.0`.*

Multiplier for [dynamic `ef`](/developers/weaviate/concepts/indexing/hnsw-indexes#dynamic-ef). Sets the potential length of the search list.

This setting is only used when `ef` is -1. | | `flatSearchCutoff` | integer | 40000 | Yes | Optional. Threshold for the [flat-search cutoff](/developers/weaviate/concepts/prefiltering.md#flat-search-cutoff). To force a vector index search, set `"flatSearchCutoff": 0`. | -| `skip` | boolean | `false` | No | When true, do not index the collection.

Weaviate decouples vector creation and vector storage. If you skip vector indexing, but a vectorizer is configured (or a vector is provided manually), Weaviate logs a warning each import.

To skip indexing and vector generation, set `"vectorizer": "none"` when you set `"skip": true`.

See [When to skip indexing](../../concepts/indexing/hnsw-indexes#when-to-skip-indexing). | -| `vectorCacheMaxObjects`| integer | `1e12` | Yes | Maximum number of objects in the memory cache. By default, this limit is set to one trillion (`1e12`) objects when a new collection is created. For sizing recommendations, see [Vector cache considerations](../../concepts/indexing/hnsw-indexes). | +| `skip` | boolean | `false` | No | When true, do not index the collection.

Weaviate decouples vector creation and vector storage. If you skip vector indexing, but a vectorizer is configured (or a vector is provided manually), Weaviate logs a warning each import.

To skip indexing and vector generation, set `"vectorizer": "none"` when you set `"skip": true`.

See [When to skip indexing](/developers/weaviate/concepts/indexing/hnsw-indexes#when-to-skip-indexing). | +| `vectorCacheMaxObjects`| integer | `1e12` | Yes | Maximum number of objects in the memory cache. By default, this limit is set to one trillion (`1e12`) objects when a new collection is created. For sizing recommendations, see [Vector cache considerations](/developers/weaviate/concepts/indexing/hnsw-indexes). | | `pq` | object | -- | Yes | Enable and configure [product quantization (PQ)](/developers/weaviate/indexing/hnsw-indexes) compression.

PQ assumes some data has already been loaded. You should have 10,000 to 100,000 vectors per shard loaded before you enable PQ.

For PQ configuration details, see [PQ configuration parameters](#pq-configuration-parameters). | ### Database parameters for HNSW @@ -140,7 +140,7 @@ Flat indexes are recommended for use cases where the number of objects per index | Parameter | Type | Default | Changeable | Details | | :-- | :-- | :-- | :-- | :-- | -| `vectorCacheMaxObjects`| integer | `1e12` | Yes | Maximum number of objects in the memory cache. By default, this limit is set to one trillion (`1e12`) objects when a new collection is created. For sizing recommendations, see [Vector cache considerations](../../concepts/indexing/vector-indexes). | +| `vectorCacheMaxObjects`| integer | `1e12` | Yes | Maximum number of objects in the memory cache. By default, this limit is set to one trillion (`1e12`) objects when a new collection is created. For sizing recommendations, see [Vector cache considerations](/developers/weaviate/concepts/indexing/vector-indexes). | | `bq` | object | -- | No | Enable and configure [binary quantization (BQ)](../../concepts/vector-quantization.md#binary-quantization) compression.

For BQ configuration details, see [BQ configuration parameters](#bq-configuration-parameters). | ### BQ configuration parameters @@ -303,8 +303,8 @@ import MultiVectorSupport from '/_includes/multi-vector-support.mdx'; ## Related pages -- [Concepts: Indexing](../../concepts/indexing.md) -- [Concepts: Vector Indexing](../../concepts/indexing/vector-indexes) +- [Concepts: Indexing](/developers/weaviate/concepts/indexing) +- [Concepts: Vector Indexing](/developers/weaviate/concepts/indexing/vector-indexes) ## Questions and feedback diff --git a/developers/weaviate/manage-data/collections.mdx b/developers/weaviate/manage-data/collections.mdx index b63d7f3876..faeda92722 100644 --- a/developers/weaviate/manage-data/collections.mdx +++ b/developers/weaviate/manage-data/collections.mdx @@ -382,7 +382,7 @@ The vector index type can be set for each collection at creation time, between `
Additional information -- Read more about index types & compression in [Concepts: Vector index](../concepts/indexing/vector-indexes). +- Read more about index types & compression in [Concepts: Vector index](/developers/weaviate/concepts/indexing/vector-indexes).
@@ -440,7 +440,7 @@ Various vector index parameters are configurable at collection creation time, in
Additional information -- Read more about index types & compression in [Concepts: Vector index](../concepts/indexing/vector-indexes). +- Read more about index types & compression in [Concepts: Vector index](/developers/weaviate/concepts/indexing/vector-indexes).
diff --git a/developers/weaviate/more-resources/performance.md b/developers/weaviate/more-resources/performance.md index 89c6a30055..502929bc44 100644 --- a/developers/weaviate/more-resources/performance.md +++ b/developers/weaviate/more-resources/performance.md @@ -28,7 +28,7 @@ The inverted index currently does not do any weighing (e.g. tf-idf) for sorting, ## Vector index Everything that has a vector, thus every data object in Weaviate, is also indexed in the vector index. Weaviate currently supports [HNSW](https://arxiv.org/abs/1603.09320) and flat vector indexes. -See [Concepts: vector index](../concepts/indexing/vector-indexes) for more information about the vector index. +See [Concepts: vector index](/developers/weaviate/concepts/indexing/vector-indexes) for more information about the vector index. ## Costs of queries and operations diff --git a/developers/weaviate/starter-guides/managing-resources/index.md b/developers/weaviate/starter-guides/managing-resources/index.md index 59b47a6ca6..f7945fcc61 100644 --- a/developers/weaviate/starter-guides/managing-resources/index.md +++ b/developers/weaviate/starter-guides/managing-resources/index.md @@ -168,7 +168,7 @@ Consider a strategy of deactivating tenants that are not frequently accessed, an - [Starter guide: Compression](./compression.mdx) - [Starter guide: Indexing](./indexing.mdx) - [Starter guide: Tenant states](./tenant-states.mdx) -- [Concepts: Vector Index](../../concepts/indexing/vector-indexes) +- [Concepts: Vector Index](/developers/weaviate/concepts/indexing/vector-indexes) - [Concepts: Vector Quantization](../../concepts/vector-quantization.md) - [Concepts: Multi-Tenancy](../../concepts/data.md#multi-tenancy) - [How-to: Set the vector index type](../../manage-data/collections.mdx#set-vector-index-type) diff --git a/developers/weaviate/starter-guides/schema.md b/developers/weaviate/starter-guides/schema.md index e5e30451cc..96f86fd96d 100644 --- a/developers/weaviate/starter-guides/schema.md +++ b/developers/weaviate/starter-guides/schema.md @@ -211,9 +211,9 @@ import SchemaWithMT from '/_includes/code/tutorial.schema.multi-tenancy.mdx'; ### Index settings -Weaviate uses two types of indexes: [vector indexes](../concepts/indexing/vector-indexes) and [inverted indexes](../concepts/indexing.md#inverted-indexes). Vector indexes are used to store and organize vectors for fast vector similarity-based searches. Inverted indexes are used to store data for fast filtering and keyword searches. +Weaviate uses two types of indexes: [vector indexes](/developers/weaviate/concepts/indexing/vector-indexes) and [inverted indexes](../concepts/indexing.md#inverted-indexes). Vector indexes are used to store and organize vectors for fast vector similarity-based searches. Inverted indexes are used to store data for fast filtering and keyword searches. -The default vector index type is [HNSW](../concepts/indexing/hnsw-indexes). The other options are [flat](../concepts/indexing/flat-indexes), which is suitable for small collections, such as those in a multi-tenancy collection, or [dynamic](../concepts/indexing/dynamic-indexes), which starts as a flat index before switching to an HNSW index if its size grows beyond a predetermined threshold. +The default vector index type is [HNSW](/developers/weaviate/concepts/indexing/hnsw-indexes). The other options are [flat](/developers/weaviate/concepts/indexing/flat-indexes), which is suitable for small collections, such as those in a multi-tenancy collection, or [dynamic](/developers/weaviate/concepts/indexing/dynamic-indexes), which starts as a flat index before switching to an HNSW index if its size grows beyond a predetermined threshold. import SchemaWithIndexSettings from '/_includes/code/tutorial.schema.index-settings.mdx'; From a7bdb86912f41ff7a3f7f3e53ce4190ce0325a27 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Thu, 19 Sep 2024 22:37:27 -0400 Subject: [PATCH 54/67] more links --- .../py/tokenization/900_next_steps.mdx | 2 +- developers/weaviate/concepts/index.md | 14 ++-------- .../concepts/indexing/inverted-indexes.md | 14 +++++----- .../concepts/indexing/vector-indexes.md | 27 +++---------------- developers/weaviate/concepts/resources.md | 2 +- .../weaviate/concepts/vector-quantization.md | 8 +++--- .../weaviate/config-refs/schema/index.md | 4 +-- .../config-refs/schema/vector-index.md | 2 +- developers/weaviate/introduction.md | 4 +-- .../managing-resources/indexing.mdx | 4 +-- 10 files changed, 25 insertions(+), 56 deletions(-) diff --git a/developers/academy/py/tokenization/900_next_steps.mdx b/developers/academy/py/tokenization/900_next_steps.mdx index 64c17c05b5..a61113e313 100644 --- a/developers/academy/py/tokenization/900_next_steps.mdx +++ b/developers/academy/py/tokenization/900_next_steps.mdx @@ -10,7 +10,7 @@ There are many more resources available to help you continue your learning journ - [Refereces: Configuration: Tokenization](/developers/weaviate/config-refs/schema/index.md#tokenization) - [Refereces: Configuration: Stopwords](/developers/weaviate/config-refs/schema/index.md#invertedindexconfig--stopwords-stopword-lists) -- [Concepts: Inverted index](/developers/weaviate/concepts/indexing.md#inverted-indexes) +- [Concepts: Inverted index](/developers/weaviate/concepts/indexing/inverted-indexes) - [Concepts: Filtering](/developers/weaviate/concepts/prefiltering.md) :::note diff --git a/developers/weaviate/concepts/index.md b/developers/weaviate/concepts/index.md index 7325a98385..772c45d283 100644 --- a/developers/weaviate/concepts/index.md +++ b/developers/weaviate/concepts/index.md @@ -5,16 +5,6 @@ image: og/docs/concepts.jpg # tags: ['getting started'] --- - - - The **Concepts** section explains various aspects related to Weaviate and its architecture to help you get the most out of it. You can read these sections in any order. :::info @@ -31,11 +21,11 @@ If you are after a practical guide, try the [quickstart tutorial](/developers/we - An overview of Weaviate's module system, including what can be done with modules, existing module types, and custom modules. -**[Indexing](./indexing.md)** +**[Indexing](/developers/weaviate/concepts/indexing)** - Read how data is indexed within Weaviate using inverted and ANN indexes, and about configurable settings. -**[Vector indexing](./vector-index.md)** +**[Vector indexing](/developers/weaviate/concepts/indexing/vector-indexes)** - Read more about Weaviate's vector indexing architecture, such as the HNSW algorithm, distance metrics, and configurable settings. diff --git a/developers/weaviate/concepts/indexing/inverted-indexes.md b/developers/weaviate/concepts/indexing/inverted-indexes.md index 69084943fe..116352a73c 100644 --- a/developers/weaviate/concepts/indexing/inverted-indexes.md +++ b/developers/weaviate/concepts/indexing/inverted-indexes.md @@ -11,7 +11,7 @@ Some things to bear in mind: * Especially for large datasets, configuring the indexes is important because the more you index, the more storage is needed. * A rule of thumb -- if you don't query over a specific field or vector space, don't index it. -* One of Weaviate's unique features is how the indexes are configured (learn more about this [here](../concepts/prefiltering.md)). +* One of Weaviate's unique features is how the indexes are configured (learn more about this [here](/developers/weaviate/concepts/prefiltering.md)). ## Vector indexes @@ -21,7 +21,7 @@ A vector index is used to serve all vector-search queries. Weaviate supports mul 2. **Flat** - a vector index that is used for brute-force searches. This is useful for small datasets. 2. **Dynamic** - a vector index that is flat when the dataset is small and switches to HNSW when the dataset is large. -For more information on vector indexes, see the [Vector Indexing](./vector-index.md) page. +For more information on vector indexes, see the [Vector Indexing](/developers/weaviate/concepts/indexing/vector-indexes) page. ## Inverted indexes @@ -30,16 +30,16 @@ For more information on vector indexes, see the [Vector Indexing](./vector-index There are three inverted index types in Weaviate: - `indexSearchable` - a searchable index for BM25 or hybrid search -- `indexFilterable` - a match-based index for fast [filtering](./prefiltering.md) by matching criteria -- `indexRangeFilters` - a range-based index for [filtering](./prefiltering.md) by numerical ranges +- `indexFilterable` - a match-based index for fast [filtering](/developers/weaviate/concepts/prefiltering.md) by matching criteria +- `indexRangeFilters` - a range-based index for [filtering](/developers/weaviate/concepts/prefiltering.md) by numerical ranges Each inverted index can be set to `true` (on) or `false` (off) on a property level. The `indexSearchable` and `indexFilterable` indexes are on by default, while the `indexRangeFilters` index is off by default. -The filterable indexes are only capable of [filtering](./prefiltering.md), while the searchable index can be used for both searching and filtering (though not as fast as the filterable index). +The filterable indexes are only capable of [filtering](/developers/weaviate/concepts/prefiltering.md), while the searchable index can be used for both searching and filtering (though not as fast as the filterable index). So, setting `"indexFilterable": false` and `"indexSearchable": true` (or not setting it at all) will have the trade-off of worse filtering performance but faster imports (due to only needing to update one index) and lower disk usage. -See the [related how-to section](../manage-data/collections.mdx#property-level-settings) to learn how to enable or disable inverted indexes on a property level. +See the [related how-to section](/developers/weaviate/manage-data/collections.mdx#property-level-settings) to learn how to enable or disable inverted indexes on a property level. A rule of thumb to follow when determining whether to switch off indexing is: _if you will never perform queries based on this property, you can turn it off._ @@ -137,7 +137,7 @@ Some stuff ## Further resources - [Concepts: Vector Indexing](./vector-index.md) -- [Configuration: Vector index](../config-refs/schema/vector-index.md) +- [Configuration: Vector index](/developers/weaviate/config-refs/schema/vector-index.md) ## Questions and feedback diff --git a/developers/weaviate/concepts/indexing/vector-indexes.md b/developers/weaviate/concepts/indexing/vector-indexes.md index 9aa93d93a3..0cb9d7e764 100644 --- a/developers/weaviate/concepts/indexing/vector-indexes.md +++ b/developers/weaviate/concepts/indexing/vector-indexes.md @@ -66,32 +66,11 @@ You might be also interested in our blog post [Vamana vs. HNSW - Exploring ANN a ## Flat index -:::info Added in `v1.23` -::: - -The **flat index** is a simple, lightweight index that is fast to build and has a very small memory footprint. This index type is a good choice for use cases where each end user (i.e. tenant) has their own, isolated, dataset, such as in a SaaS product for example, or a database of isolated record sets. - -As the name suggests, the flat index is a single layer of disk-backed data objects and thus a very small memory footprint. The flat index is a good choice for small collections, such as for multi-tenancy use cases. - -A drawback of the flat index is that it does not scale well to large collections as it has a linear time complexity as a function of the number of data objects, unlike the `hnsw` index which has a logarithmic time complexity. +Add summary ## Dynamic index -:::caution Experimental feature -Available starting in `v1.25`. This is an experimental feature. Use with caution. -::: - -import DynamicAsyncRequirements from '/_includes/dynamic-index-async-req.mdx'; - - - -The flat index is ideal for use cases with a small object count and provides lower memory overhead and good latency. As the object count increases the HNSW index provides a more viable solution as HNSW speeds up search. The goal of the dynamic index is to shorten latencies during querying time at the cost of a larger memory footprint as you scale. - -By configuring a dynamic index, you can automatically switch from flat to HNSW indexes. This switch occurs when the object count exceeds a prespecified threshold (by default 10,000). This functionality only works with async indexing enabled. When the threshold is hit while importing, all the data piles up in the async queue, the HNSW index is constructed in the background and when ready the swap from flat to HNSW is completed. - -Currently, this is only a one-way upgrade from a flat to an HNSW index, it does not support changing back to a flat index even if the object count goes below the threshold due to deletion. - -This is particularly useful in a multi-tenant setup where building an HNSW index per tenant would introduce extra overhead. With a dynamic index, as individual tenants grow their index will switch from flat to HNSW, while smaller tenants' indexes remain flat. +Add summary ## Vector cache considerations @@ -107,7 +86,7 @@ Vectors that aren't currently in cache are added to the cache if there is still ### Can I use vector indexing with vector quantization? -Yes, you can read more about it in [vector quantization (compression)](DWCTODO). +Yes, you can read more about it in [vector quantization (compression)]DWCTODO. ### Which vector index is right for me? diff --git a/developers/weaviate/concepts/resources.md b/developers/weaviate/concepts/resources.md index a1bfe789a2..26408aee41 100644 --- a/developers/weaviate/concepts/resources.md +++ b/developers/weaviate/concepts/resources.md @@ -45,7 +45,7 @@ When search throughput is limited, add CPUs to increase the number of queries pe Memory determines the maximum supported dataset size. Memory does not directly influence query speed. ::: -The HNSW index must be stored in memory. The memory required is directly related to the size of your dataset. There is no correlation between the size of your dataset and the current query load. You can use [`product quantization (PQ)`](/developers/weaviate/concepts/vector-index#hnsw-with-product-quantization-pq) to compress the vectors in your dataset in increase the number of vectors your can hold in memory. +The HNSW index must be stored in memory. The memory required is directly related to the size of your dataset. There is no correlation between the size of your dataset and the current query load. You can use [`product quantization (PQ)`](/developers/weaviate/concepts/indexing/hnsw-indexes) to compress the vectors in your dataset in increase the number of vectors your can hold in memory. Weaviate let's you configure a limit to the number of vectors held in memory in order to prevent unexpected Out-of-Memory ("OOM") situations. The default value is one trillion (`1e12`) objects. per collection. To adjust the number of objects, update the value of [`vectorCacheMaxObjects`](../config-refs/schema/vector-index.md) in your index settings. diff --git a/developers/weaviate/concepts/vector-quantization.md b/developers/weaviate/concepts/vector-quantization.md index 4e6105c131..c51858fd4c 100644 --- a/developers/weaviate/concepts/vector-quantization.md +++ b/developers/weaviate/concepts/vector-quantization.md @@ -5,7 +5,7 @@ image: og/docs/concepts.jpg # tags: ['vector compression', 'quantization'] --- -**Vector quantization** reduces the memory footprint of the [vector index](./vector-index.md) by compressing the vector embeddings, and thus reduces deployment costs and improves the speed of the vector similarity search process. +**Vector quantization** reduces the memory footprint of the [vector index](/developers/weaviate/concepts/indexing/vector-indexes) by compressing the vector embeddings, and thus reduces deployment costs and improves the speed of the vector similarity search process. Weaviate currently offers two vector quantization techniques: @@ -131,9 +131,9 @@ In some cases, rescoring also includes over-fetching, whereby additional candida ## Further resources :::info Related pages -- [Concepts: Indexing](./indexing.md) -- [Concepts: Vector Indexing](./vector-index.md) -- [Configuration: Vector index](../config-refs/schema/vector-index.md) +- [Concepts: Indexing](/developers/weaviate/concepts/indexing) +- [Concepts: Vector Indexing](/developers/weaviate/concepts/indexing/vector-indexes) +- [Configuration: Vector index](/developers/weaviate/concepts/config-refs/schema/vector-index.md) - [Configuration: Schema (Configure semantic indexing)](../config-refs/schema/index.md#configure-semantic-indexing) - [How to configure: Binary quantization (compression)](../configuration/compression/bq-compression.md) - [How to configure: Product quantization (compression)](../configuration/compression/pq-compression.md) diff --git a/developers/weaviate/config-refs/schema/index.md b/developers/weaviate/config-refs/schema/index.md index 7b15809cb4..9a0a01104c 100644 --- a/developers/weaviate/config-refs/schema/index.md +++ b/developers/weaviate/config-refs/schema/index.md @@ -323,7 +323,7 @@ The `vectorIndexType` parameter controls the type of vector index that is used f The `vectorIndexConfig` parameter controls the configuration of the vector index. The available parameters depend on the `vectorIndexType` that is used. -See the [vector index configuration](./vector-index.md) page for more details. +See the [vector index configuration](/developers/weaviate/concepts/indexing/vector-indexes) page for more details. ### `shardingConfig` @@ -567,7 +567,7 @@ The `kagome_kr` tokenizer is not loaded by default to save resources. To use it, The `indexInverted` parameter has been deprecated from Weaviate `v1.19` onwards. ::: -Multiple [inverted index types](../../concepts/indexing.md#inverted-indexes) are available in Weaviate. Not all inverted index types are available for all data types. The available inverted index types are: +Multiple [inverted index types](/developers/weaviate/concepts/indexing/inverted-indexes) are available in Weaviate. Not all inverted index types are available for all data types. The available inverted index types are: import InvertedIndexTypesSummary from '/_includes/inverted-index-types-summary.mdx'; diff --git a/developers/weaviate/config-refs/schema/vector-index.md b/developers/weaviate/config-refs/schema/vector-index.md index e63343958e..e53e71cfc0 100644 --- a/developers/weaviate/config-refs/schema/vector-index.md +++ b/developers/weaviate/config-refs/schema/vector-index.md @@ -28,7 +28,7 @@ Some HNSW parameters are mutable, but others cannot be modified after you create | `flatSearchCutoff` | integer | 40000 | Yes | Optional. Threshold for the [flat-search cutoff](/developers/weaviate/concepts/prefiltering.md#flat-search-cutoff). To force a vector index search, set `"flatSearchCutoff": 0`. | | `skip` | boolean | `false` | No | When true, do not index the collection.

Weaviate decouples vector creation and vector storage. If you skip vector indexing, but a vectorizer is configured (or a vector is provided manually), Weaviate logs a warning each import.

To skip indexing and vector generation, set `"vectorizer": "none"` when you set `"skip": true`.

See [When to skip indexing](/developers/weaviate/concepts/indexing/hnsw-indexes#when-to-skip-indexing). | | `vectorCacheMaxObjects`| integer | `1e12` | Yes | Maximum number of objects in the memory cache. By default, this limit is set to one trillion (`1e12`) objects when a new collection is created. For sizing recommendations, see [Vector cache considerations](/developers/weaviate/concepts/indexing/hnsw-indexes). | -| `pq` | object | -- | Yes | Enable and configure [product quantization (PQ)](/developers/weaviate/indexing/hnsw-indexes) compression.

PQ assumes some data has already been loaded. You should have 10,000 to 100,000 vectors per shard loaded before you enable PQ.

For PQ configuration details, see [PQ configuration parameters](#pq-configuration-parameters). | +| `pq` | object | -- | Yes | Enable and configure [product quantization (PQ)](/developers/weaviate/concepts/indexing/hnsw-indexes) compression.

PQ assumes some data has already been loaded. You should have 10,000 to 100,000 vectors per shard loaded before you enable PQ.

For PQ configuration details, see [PQ configuration parameters](#pq-configuration-parameters). | ### Database parameters for HNSW diff --git a/developers/weaviate/introduction.md b/developers/weaviate/introduction.md index 87f639b80f..5a141d6aee 100644 --- a/developers/weaviate/introduction.md +++ b/developers/weaviate/introduction.md @@ -128,7 +128,7 @@ Weaviate is an open source vector database that stores both objects and vectors. **Weaviate in a nutshell**: * Weaviate is an open source [vector database](https://weaviate.io/blog/what-is-a-vector-database). -* Weaviate allows you to store and retrieve data objects based on their semantic properties by indexing them with [vectors](./concepts/indexing/vector-indexes). +* Weaviate allows you to store and retrieve data objects based on their semantic properties by indexing them with [vectors](/developers/weaviate/concepts/indexing/vector-indexes). * Weaviate can be used stand-alone (aka _bring your vectors_) or with a variety of [modules](./modules/index.md) that can do the vectorization for you and extend the core capabilities. * Weaviate has a [GraphQL-API](./api/graphql/index.md) to access your data easily. * Weaviate is fast (check our [open source benchmarks](./benchmarks/index.md)). @@ -186,7 +186,7 @@ Within Weaviate, all individual data objects are based on a class property struc You can add data to Weaviate through the [RESTful API](/developers/weaviate/api/rest) end-points and retrieve data through the [GraphQL interface](./api/graphql/index.md). -Weaviate's [vector indexing mechanism is modular](./concepts/indexing/vector-indexes), and the current available plugin is the Hierarchical Navigable Small World (HNSW) multilayered graph. +Weaviate's [vector indexing mechanism is modular](/developers/weaviate/concepts/indexing/vector-indexes), and the current available plugin is the Hierarchical Navigable Small World (HNSW) multilayered graph. ## What are Weaviate modules? diff --git a/developers/weaviate/starter-guides/managing-resources/indexing.mdx b/developers/weaviate/starter-guides/managing-resources/indexing.mdx index 607d3494ae..47fdda309f 100644 --- a/developers/weaviate/starter-guides/managing-resources/indexing.mdx +++ b/developers/weaviate/starter-guides/managing-resources/indexing.mdx @@ -145,8 +145,8 @@ To configure indexing, follow the steps on these pages: For more documentation details, see: -- [Vector indexing](/developers/weaviate/concepts/vector-index) -- [Inverted indexes](/developers/weaviate/concepts/indexing) +- [Vector indexing](/developers/weaviate/concepts/indexing/vector-indexes) +- [Inverted indexes](/developers/weaviate/concepts/indexing/inverted-indexes) ### Weaviate academy From 9c6418b4dbe6d5c0cccd18616511a05567cbaa37 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Thu, 19 Sep 2024 22:45:02 -0400 Subject: [PATCH 55/67] links --- developers/weaviate/concepts/indexing/index.md | 1 + developers/weaviate/concepts/indexing/inverted-indexes.md | 2 +- developers/weaviate/concepts/vector-quantization.md | 2 +- .../starter-guides/managing-resources/compression.mdx | 4 ++-- developers/weaviate/starter-guides/schema.md | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/developers/weaviate/concepts/indexing/index.md b/developers/weaviate/concepts/indexing/index.md index e5287afffb..2b596af039 100644 --- a/developers/weaviate/concepts/indexing/index.md +++ b/developers/weaviate/concepts/indexing/index.md @@ -35,6 +35,7 @@ For more information, see the following: - [Configure vector indexes](/developers/weaviate/configuration/indexing-vector) - [Configure inverted indexes](/developers/weaviate/configuration/inverted-indexes) +- [Indexing starter guide](/developers/weaviate/starter-guides/managing-resources/indexing) ## Questions and feedback diff --git a/developers/weaviate/concepts/indexing/inverted-indexes.md b/developers/weaviate/concepts/indexing/inverted-indexes.md index 116352a73c..4f45ef6df9 100644 --- a/developers/weaviate/concepts/indexing/inverted-indexes.md +++ b/developers/weaviate/concepts/indexing/inverted-indexes.md @@ -136,7 +136,7 @@ Some stuff ## Further resources -- [Concepts: Vector Indexing](./vector-index.md) +- [Concepts: Vector Indexing](/developers/weaviate/concepts/indexing/vector-indexes) - [Configuration: Vector index](/developers/weaviate/config-refs/schema/vector-index.md) ## Questions and feedback diff --git a/developers/weaviate/concepts/vector-quantization.md b/developers/weaviate/concepts/vector-quantization.md index c51858fd4c..53be89bec0 100644 --- a/developers/weaviate/concepts/vector-quantization.md +++ b/developers/weaviate/concepts/vector-quantization.md @@ -133,7 +133,7 @@ In some cases, rescoring also includes over-fetching, whereby additional candida :::info Related pages - [Concepts: Indexing](/developers/weaviate/concepts/indexing) - [Concepts: Vector Indexing](/developers/weaviate/concepts/indexing/vector-indexes) -- [Configuration: Vector index](/developers/weaviate/concepts/config-refs/schema/vector-index.md) +- [Configuration: Vector index](/developers/weaviate/config-refs/schema/vector-index.md) - [Configuration: Schema (Configure semantic indexing)](../config-refs/schema/index.md#configure-semantic-indexing) - [How to configure: Binary quantization (compression)](../configuration/compression/bq-compression.md) - [How to configure: Product quantization (compression)](../configuration/compression/pq-compression.md) diff --git a/developers/weaviate/starter-guides/managing-resources/compression.mdx b/developers/weaviate/starter-guides/managing-resources/compression.mdx index ea11df9ce9..4a58a65636 100644 --- a/developers/weaviate/starter-guides/managing-resources/compression.mdx +++ b/developers/weaviate/starter-guides/managing-resources/compression.mdx @@ -11,11 +11,11 @@ Use compression to lower system requirements and save on infrastructure costs. Weaviate stores objects and vector representations of those objects (vectors). Vectors can be very large. Vector dimensions are stored as 32 bit floats. A single vector with 1536 dimensions uses about 6 KB of storage. When collections have millions of objects, the resulting size can lead to significant costs, especially where an in-memory vector index is used. -Weaviate creates indexes to search the vector space for your collection. By default, the vector index is an [Hierarchical Navigable Small World (HNSW)](/developers/weaviate/concepts/vector-index#hierarchical-navigable-small-world-hnsw-index) index which includes the vector as well as a graph structure. HNSW indexes allow fast vector searches while maintaining excellent recall, but they can be expensive to use as they are stored in memory. +Weaviate creates indexes to search the vector space for your collection. By default, the vector index is an [Hierarchical Navigable Small World (HNSW)](/developers/weaviate/concepts/indexing/hnsw-indexes) index which includes the vector as well as a graph structure. HNSW indexes allow fast vector searches while maintaining excellent recall, but they can be expensive to use as they are stored in memory. In many cases, you can use compression or a different index type to change the way Weaviate stores and searches your data, and still maintain high levels of recall. Updating the default settings can result in significant cost savings and performance improvements. -This page discusses compression algorithms. For more on indexes, see [Vector indexing](/developers/weaviate/concepts/vector-index). +This page discusses compression algorithms. For more on indexes, see [Vector indexing](/developers/weaviate/concepts/indexing/vector-index). ## Compression algorithms diff --git a/developers/weaviate/starter-guides/schema.md b/developers/weaviate/starter-guides/schema.md index 96f86fd96d..d774a7703e 100644 --- a/developers/weaviate/starter-guides/schema.md +++ b/developers/weaviate/starter-guides/schema.md @@ -211,7 +211,7 @@ import SchemaWithMT from '/_includes/code/tutorial.schema.multi-tenancy.mdx'; ### Index settings -Weaviate uses two types of indexes: [vector indexes](/developers/weaviate/concepts/indexing/vector-indexes) and [inverted indexes](../concepts/indexing.md#inverted-indexes). Vector indexes are used to store and organize vectors for fast vector similarity-based searches. Inverted indexes are used to store data for fast filtering and keyword searches. +Weaviate uses two types of indexes: [vector indexes](/developers/weaviate/concepts/indexing/vector-indexes) and [inverted indexes](/developers/weaviate/concepts/indexing/inverted-indexes). Vector indexes are used to store and organize vectors for fast vector similarity-based searches. Inverted indexes are used to store data for fast filtering and keyword searches. The default vector index type is [HNSW](/developers/weaviate/concepts/indexing/hnsw-indexes). The other options are [flat](/developers/weaviate/concepts/indexing/flat-indexes), which is suitable for small collections, such as those in a multi-tenancy collection, or [dynamic](/developers/weaviate/concepts/indexing/dynamic-indexes), which starts as a flat index before switching to an HNSW index if its size grows beyond a predetermined threshold. From 7640b58a5bb4866567d868932d3d10c0f19fcc57 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Fri, 20 Sep 2024 12:54:39 -0400 Subject: [PATCH 56/67] tsc updates --- .../code/howto/indexes/indexes-dynamic-v2.ts | 11 +++-- .../code/howto/indexes/indexes-dynamic-v3.ts | 2 +- .../code/howto/indexes/indexes-inverted-v2.ts | 2 +- .../code/howto/indexes/indexes-inverted-v3.ts | 2 +- _includes/code/howto/indexes/indexes-v2.ts | 2 +- _includes/code/howto/indexes/indexes-v3.ts | 2 +- .../concepts/indexing/dynamic-indexes.mdx | 19 +++++---- .../concepts/indexing/flat-indexes.mdx | 16 +++++++ .../concepts/indexing/hnsw-indexes.mdx | 22 ++++++++++ .../weaviate/concepts/indexing/index.md | 2 +- .../concepts/indexing/vector-indexes.md | 42 +++++++++++++++++++ 11 files changed, 103 insertions(+), 19 deletions(-) diff --git a/_includes/code/howto/indexes/indexes-dynamic-v2.ts b/_includes/code/howto/indexes/indexes-dynamic-v2.ts index 5ea7a1c23d..94ec70debe 100644 --- a/_includes/code/howto/indexes/indexes-dynamic-v2.ts +++ b/_includes/code/howto/indexes/indexes-dynamic-v2.ts @@ -7,16 +7,16 @@ import weaviate, { WeaviateClient } from 'weaviate-ts-client'; // Create client connection function getClient(){ const client: WeaviateClient = weaviate.client({ - scheme: 'http', - host: 'localhost:8080', - }); + scheme: 'http', + host: 'localhost:8080', +}) return client; } // Delete pre-existing collections async function deleteClass(client: WeaviateClient, className: string){ - if (client.schema.exists(className)) { + if (await client.schema.exists(className)) { await client.schema.classDeleter().withClassName(className).do(); } } @@ -64,8 +64,7 @@ async function configureDynamicCollection(client: WeaviateClient, className: str dynamic_ef_min: '200', // Minimum threshold for dynamic Ef dynamic_ef_max: '1000', // Maximum threshold for dynamic Ef quantizer: 'Configure.VectorIndex.Quantizer.pq()', // Quantizer configuration - }, - + }, }; // Add the class to the schema diff --git a/_includes/code/howto/indexes/indexes-dynamic-v3.ts b/_includes/code/howto/indexes/indexes-dynamic-v3.ts index 3ea92a669f..828388925d 100644 --- a/_includes/code/howto/indexes/indexes-dynamic-v3.ts +++ b/_includes/code/howto/indexes/indexes-dynamic-v3.ts @@ -6,7 +6,7 @@ import weaviate, { WeaviateClient,vectorizer, configure } from 'weaviate-client' // Delete pre-existing collections async function deleteCollection(client: WeaviateClient, collectionName: string){ - if(client.collections.exists(collectionName)){ + if(await client.collections.exists(collectionName)){ await client.collections.delete(collectionName) } } diff --git a/_includes/code/howto/indexes/indexes-inverted-v2.ts b/_includes/code/howto/indexes/indexes-inverted-v2.ts index d3c53e5bf8..54b9140704 100644 --- a/_includes/code/howto/indexes/indexes-inverted-v2.ts +++ b/_includes/code/howto/indexes/indexes-inverted-v2.ts @@ -16,7 +16,7 @@ function getClient(){ // Delete pre-existing collections async function deleteClass(client: WeaviateClient, className: string){ - if (client.schema.exists(className)) { + if (await client.schema.exists(className)) { await client.schema.classDeleter().withClassName(className).do(); } } diff --git a/_includes/code/howto/indexes/indexes-inverted-v3.ts b/_includes/code/howto/indexes/indexes-inverted-v3.ts index 43267bc9a3..1c8bc6d866 100644 --- a/_includes/code/howto/indexes/indexes-inverted-v3.ts +++ b/_includes/code/howto/indexes/indexes-inverted-v3.ts @@ -6,7 +6,7 @@ import weaviate, { dataType, WeaviateClient,vectorizer, configure } from 'weavia // Delete pre-existing collections async function deleteCollection(client: WeaviateClient, collectionName: string){ - if(client.collections.exists(collectionName)){ + if(await client.collections.exists(collectionName)){ await client.collections.delete(collectionName) } } diff --git a/_includes/code/howto/indexes/indexes-v2.ts b/_includes/code/howto/indexes/indexes-v2.ts index 0e1e2e9087..afba33e40b 100644 --- a/_includes/code/howto/indexes/indexes-v2.ts +++ b/_includes/code/howto/indexes/indexes-v2.ts @@ -16,7 +16,7 @@ function getClient(){ // Delete pre-existing collections async function deleteClass(client: WeaviateClient, className: string){ - if (client.schema.exists(className)) { + if (await client.schema.exists(className)) { await client.schema.classDeleter().withClassName(className).do(); } } diff --git a/_includes/code/howto/indexes/indexes-v3.ts b/_includes/code/howto/indexes/indexes-v3.ts index b49ab236c0..c6284d4b8e 100644 --- a/_includes/code/howto/indexes/indexes-v3.ts +++ b/_includes/code/howto/indexes/indexes-v3.ts @@ -6,7 +6,7 @@ import weaviate, { dataType, WeaviateClient,vectorizer, configure } from 'weavia // Delete pre-existing collections async function deleteCollection(client: WeaviateClient, collectionName: string){ - if(client.collections.exists(collectionName)){ + if(await client.collections.exists(collectionName)){ await client.collections.delete(collectionName) } } diff --git a/developers/weaviate/concepts/indexing/dynamic-indexes.mdx b/developers/weaviate/concepts/indexing/dynamic-indexes.mdx index 325107d5d2..9422fe2b29 100644 --- a/developers/weaviate/concepts/indexing/dynamic-indexes.mdx +++ b/developers/weaviate/concepts/indexing/dynamic-indexes.mdx @@ -1,22 +1,27 @@ --- -title: Dynamic indexed +title: Dynamic indexes sidebar_position: 40 image: og/docs/indexing.jpg # tags: ['basics']['indexes'] --- -:::caution Experimental feature -Available starting in `v1.25`. This is an experimental feature. Use with caution. +:::info Added in `v1.25` ::: -import DynamicAsyncRequirements from '/_includes/dynamic-index-async-req.mdx'; +import DynamicIntro from '/_includes/indexes/dynamic-intro.mdx'; - + -The flat index is ideal for use cases with a small object count and provides lower memory overhead and good latency. As the object count increases the HNSW index provides a more viable solution as HNSW speeds up search. The goal of the dynamic index is to shorten latencies during querying time at the cost of a larger memory footprint as you scale. +The flat index is ideal for collections with low object counts, less than 10,000 objects. Even though they are disk based, at that scale flat indexes provide lower memory overhead and good latency. When the number of objects increases, so does search latency. For collections with more than 10,000 objects, an HNSW index is usually a better indexing solution. The dynamic index balances search latency costs against in-memory costs. Memory costs are low until search latencies grow too large, then the index type switches to preserve low search latencies as your collection grows in size. -By configuring a dynamic index, you can automatically switch from flat to HNSW indexes. This switch occurs when the object count exceeds a prespecified threshold (by default 10,000). This functionality only works with async indexing enabled. When the threshold is hit while importing, all the data piles up in the async queue, the HNSW index is constructed in the background and when ready the swap from flat to HNSW is completed. +The dynamic index automatically switches from a flat index to an HNSW index when the object count exceeds a pre-specified threshold. (The default is 10,000 objects.) This functionality only works with async indexing enabled. When the threshold is hit while importing, all the data piles up in the async queue, the HNSW index is constructed in the background and when ready the swap from flat to HNSW is completed. Currently, this is only a one-way upgrade from a flat to an HNSW index, it does not support changing back to a flat index even if the object count goes below the threshold due to deletion. This is particularly useful in a multi-tenant setup where building an HNSW index per tenant would introduce extra overhead. With a dynamic index, as individual tenants grow their index will switch from flat to HNSW, while smaller tenants' indexes remain flat. + + + +import DynamicAsyncRequirements from '/_includes/dynamic-index-async-req.mdx'; + + \ No newline at end of file diff --git a/developers/weaviate/concepts/indexing/flat-indexes.mdx b/developers/weaviate/concepts/indexing/flat-indexes.mdx index 68783801be..16642f0fdb 100644 --- a/developers/weaviate/concepts/indexing/flat-indexes.mdx +++ b/developers/weaviate/concepts/indexing/flat-indexes.mdx @@ -5,6 +5,22 @@ image: og/docs/indexing.jpg # tags: ['basics']['indexes'] --- + + +#### Flat indexes + +import FlatIntro from '/_includes/indexes/flat-intro.mdx'; + + + +As a result, flat indexes are best suited for cases where the number of objects is low and will not grow significantly. + +[Binary quantization (BQ)](/developers/weaviate/configuration/compression/bq-compression) can improve flat indexes' search speeds. BQ improves search time by reducing the amount of data to read, and speeding up time taken to calculate the distance between vectors. + + + + + :::info Added in `v1.23` ::: diff --git a/developers/weaviate/concepts/indexing/hnsw-indexes.mdx b/developers/weaviate/concepts/indexing/hnsw-indexes.mdx index 44ccfccd5a..bfff1f71f4 100644 --- a/developers/weaviate/concepts/indexing/hnsw-indexes.mdx +++ b/developers/weaviate/concepts/indexing/hnsw-indexes.mdx @@ -5,6 +5,28 @@ image: og/docs/indexing.jpg # tags: ['basics']['indexes'] --- + + +#### HNSW indexes + +import HNSWIntro from '/_includes/indexes/hnsw-intro.mdx'; + + + +HNSW indexes achieve this by building a multi-layered graph of objects, allowing for fast, approximate nearest neighbor searches. + +While HNSW indexes enable fast searches, they use a lot of [*hot* resources](./index.md#-hot), as they load the graph structure and vectors into memory. + +Consider using [compression](./compression.mdx) to reduce the size of for your HNSW indexes. Weaviate offers several ways to compress your data: + +import CompressionAlgorithms from '/_includes/starter-guides/compression-types.mdx'; + + + + + + + **Hierarchical Navigable Small World (HNSW)** is an algorithm that works on multi-layered graphs. It is also an index type, and refers to vector indexes that are created using the HNSW algorithm. HNSW indexes enable very fast queries, but rebuilding the index when you add new vectors can be resource intensive. Weaviate's `hnsw` index is a custom implementation of the Hierarchical Navigable Small World ([HNSW](https://arxiv.org/abs/1603.09320)) algorithm that offers full [CRUD-support](https://db-engines.com/en/blog_post/87). diff --git a/developers/weaviate/concepts/indexing/index.md b/developers/weaviate/concepts/indexing/index.md index 2b596af039..414e7ae39b 100644 --- a/developers/weaviate/concepts/indexing/index.md +++ b/developers/weaviate/concepts/indexing/index.md @@ -11,7 +11,7 @@ For configuration examples without additional contextual material, see the [Vect ## Vector indexes -Vector embeddings encapsulate meaning for the objects that they represent. Vector indexes make it easier efficiently search large vector spaces in order to find objects with similar vector embeddings. +Vector embeddings encapsulate meaning for the objects that they represent. Vector indexes make it easier to efficiently search large vector spaces in order to find objects with similar vector embeddings. Weaviate offers these vector index types: diff --git a/developers/weaviate/concepts/indexing/vector-indexes.md b/developers/weaviate/concepts/indexing/vector-indexes.md index 0cb9d7e764..d2a3ba7425 100644 --- a/developers/weaviate/concepts/indexing/vector-indexes.md +++ b/developers/weaviate/concepts/indexing/vector-indexes.md @@ -5,6 +5,48 @@ image: og/docs/indexing.jpg # tags: ['vector index plugins'] --- +## Vector indexes + +import VectorIntro from '/_includes/indexes/vector-intro.mdx'; + + + +#### HNSW indexes + +import HNSWIntro from '/_includes/indexes/hnsw-intro.mdx'; + + + +HNSW indexes achieve this by building a multi-layered graph of objects, allowing for fast, approximate nearest neighbor searches. + +While HNSW indexes enable fast searches, they use a lot of [*hot* resources](./index.md#-hot), as they load the graph structure and vectors into memory. + +Consider using [compression](./compression.mdx) to reduce the size of for your HNSW indexes. Weaviate offers several ways to compress your data: + +import CompressionAlgorithms from '/_includes/starter-guides/compression-types.mdx'; + + + +#### Flat indexes + +import FlatIntro from '/_includes/indexes/flat-intro.mdx'; + + + +As a result, flat indexes are best suited for cases where the number of objects is low and will not grow significantly. + +[Binary quantization (BQ)](/developers/weaviate/configuration/compression/bq-compression) can improve flat indexes' search speeds. BQ improves search time by reducing the amount of data to read, and speeding up time taken to calculate the distance between vectors. + +#### Dynamic indexes + +:::info Added in `v1.25` +::: + +import DynamicIntro from '/_includes/indexes/dynamic-intro.mdx'; + + + + Vector indexing is a key component of vector databases. It can help to [significantly **increase the speed** of the search process of similarity search](https://weaviate.io/blog/why-is-vector-search-so-fast) with only a minimal tradeoff in search accuracy ([HNSW index](#hierarchical-navigable-small-world-hnsw-index)), or efficiently store many subsets of data in a small memory footprint ([flat index](#flat-index)). The [dynamic index](#dynamic-index) can even start off as a flat index and then dynamically switch to the HNSW index as it scales past a threshold. Weaviate's vector-first storage system takes care of all storage operations with a vector index. Storing data in a vector-first manner not only allows for semantic or context-based search, but also makes it possible to store *very* large amounts of data without decreasing performance (assuming scaled well horizontally or having sufficient shards for the indexes). From 238098a463a78e27e511b496dc74b4ec51f081c1 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Sun, 22 Sep 2024 19:48:40 -0400 Subject: [PATCH 57/67] dynamic --- _includes/indexes/dynamic-intro.mdx | 2 +- .../concepts/indexing/bm25-ranking.md | 146 ++++++++++++++++++ .../concepts/indexing/dynamic-indexes.mdx | 39 +++-- .../concepts/indexing/indexFilterable.md | 65 ++++++++ .../concepts/indexing/indexRangeFilters.md | 24 +++ .../concepts/indexing/indexSearchable.md | 67 ++++++++ .../concepts/indexing/inverted-indexes.md | 2 +- 7 files changed, 334 insertions(+), 11 deletions(-) create mode 100644 developers/weaviate/concepts/indexing/bm25-ranking.md create mode 100644 developers/weaviate/concepts/indexing/indexFilterable.md create mode 100644 developers/weaviate/concepts/indexing/indexRangeFilters.md create mode 100644 developers/weaviate/concepts/indexing/indexSearchable.md diff --git a/_includes/indexes/dynamic-intro.mdx b/_includes/indexes/dynamic-intro.mdx index f61d013ddd..a09dc0c162 100644 --- a/_includes/indexes/dynamic-intro.mdx +++ b/_includes/indexes/dynamic-intro.mdx @@ -1 +1 @@ -[Dynamic index](/developers/weaviate/concepts/indexing/dynamic-indexes) offer a flexible approach to indexing. A dynamic index begins as a flat index, and converts automatically to an HNSW index when a [collection](/developers/weaviate/concepts/data#collections) or [tenant](/developers/weaviate/concepts/data#multi-tenancy) reaches a threshold size. \ No newline at end of file +[Dynamic indexes](/developers/weaviate/concepts/indexing/dynamic-indexes) offer a flexible approach to indexing. A dynamic index starts as a flat index and converts automatically to an HNSW index when a [collection](/developers/weaviate/concepts/data#collections) or [tenant](/developers/weaviate/concepts/data#multi-tenancy) reaches a threshold size. \ No newline at end of file diff --git a/developers/weaviate/concepts/indexing/bm25-ranking.md b/developers/weaviate/concepts/indexing/bm25-ranking.md new file mode 100644 index 0000000000..4dd057e2e5 --- /dev/null +++ b/developers/weaviate/concepts/indexing/bm25-ranking.md @@ -0,0 +1,146 @@ +--- +title: BM25 ranking +sidebar_position: 110 +image: og/docs/indexing.jpg +# tags: ['basics'] +--- + +You can configure indexes in Weaviate per collection. + +Some things to bear in mind: + +* Especially for large datasets, configuring the indexes is important because the more you index, the more storage is needed. +* A rule of thumb -- if you don't query over a specific field or vector space, don't index it. +* One of Weaviate's unique features is how the indexes are configured (learn more about this [here](/developers/weaviate/concepts/prefiltering.md)). + +## Vector indexes + +A vector index is used to serve all vector-search queries. Weaviate supports multiple types of vector indexes: + +1. **HNSW** - an approximate nearest neighbor (ANN) search based vector index. HNSW indexes scale well with large datasets. +2. **Flat** - a vector index that is used for brute-force searches. This is useful for small datasets. +2. **Dynamic** - a vector index that is flat when the dataset is small and switches to HNSW when the dataset is large. + +For more information on vector indexes, see the [Vector Indexing](/developers/weaviate/concepts/indexing/vector-indexes) page. + +## Inverted indexes + +### Configure the inverted index + +There are three inverted index types in Weaviate: + +- `indexSearchable` - a searchable index for BM25 or hybrid search +- `indexFilterable` - a match-based index for fast [filtering](/developers/weaviate/concepts/prefiltering.md) by matching criteria +- `indexRangeFilters` - a range-based index for [filtering](/developers/weaviate/concepts/prefiltering.md) by numerical ranges + +Each inverted index can be set to `true` (on) or `false` (off) on a property level. The `indexSearchable` and `indexFilterable` indexes are on by default, while the `indexRangeFilters` index is off by default. + +The filterable indexes are only capable of [filtering](/developers/weaviate/concepts/prefiltering.md), while the searchable index can be used for both searching and filtering (though not as fast as the filterable index). + +So, setting `"indexFilterable": false` and `"indexSearchable": true` (or not setting it at all) will have the trade-off of worse filtering performance but faster imports (due to only needing to update one index) and lower disk usage. + +See the [related how-to section](/developers/weaviate/manage-data/collections.mdx#property-level-settings) to learn how to enable or disable inverted indexes on a property level. + +A rule of thumb to follow when determining whether to switch off indexing is: _if you will never perform queries based on this property, you can turn it off._ + +#### Inverted index types summary + +import InvertedIndexTypesSummary from '/_includes/inverted-index-types-summary.mdx'; + + + +- Enable one or both of `indexFilterable` and `indexRangeFilters` to index a property for faster filtering. + - If only one is enabled, the respective index is used for filtering. + - If both are enabled, `indexRangeFilters` is used for operations involving comparison operators, and `indexFilterable` is used for equality and inequality operations. + +This chart shows which filter makes the comparison when one or both index type is `true` for an applicable property. + +| Operator | `indexRangeFilters` only | `indexFilterable` only | Both enabled | +| :- | :- | :- | :- | +| Equal | `indexRangeFilters` | `indexFilterable` | `indexFilterable` | +| Not equal | `indexRangeFilters` | `indexFilterable` | `indexFilterable` | +| Greater than | `indexRangeFilters` | `indexFilterable` | `indexRangeFilters` | +| Greater than equal | `indexRangeFilters` | `indexFilterable` | `indexRangeFilters` | +| Less than | `indexRangeFilters` | `indexFilterable` | `indexRangeFilters` | +| Less than equal | `indexRangeFilters` | `indexFilterable` | `indexRangeFilters` | + +#### Inverted index for timestamps + +You can also enable an inverted index to search [based on timestamps](/developers/weaviate/config-refs/schema/index.md#invertedindexconfig--indextimestamps). + +Timestamps are currently indexed using the `indexFilterable` index. + +## Collections without indexes + +If you don't want to set an index at all, this is possible too. + +To create a collection without any indexes, skip indexing on the collection and on the properties. + +```js +{ + "class": "Author", + "description": "A description of this collection, in this case, it's about authors", + "vectorIndexConfig": { + "skip": true // <== disable vector index + }, + "properties": [ + { + "indexFilterable": false, // <== disable filterable index for this property + "indexSearchable": false, // <== disable searchable index for this property + "dataType": [ + "text" + ], + "description": "The name of the Author", + "name": "name" + }, + { + "indexFilterable": false, // <== disable filterable index for this property + "dataType": [ + "int" + ], + "description": "The age of the Author", + "name": "age" + }, + { + "indexFilterable": false, // <== disable filterable index for this property + "dataType": [ + "date" + ], + "description": "The date of birth of the Author", + "name": "born" + }, + { + "indexFilterable": false, // <== disable filterable index for this property + "dataType": [ + "boolean" + ], + "description": "A boolean value if the Author won a nobel prize", + "name": "wonNobelPrize" + }, + { + "indexFilterable": false, // <== disable filterable index for this property + "indexSearchable": false, // <== disable searchable index for this property + "dataType": [ + "text" + ], + "description": "A description of the author", + "name": "description" + } + ] +} +``` + +## BM25 ranking algorithm + +Some stuff + +## Further resources + +- [Concepts: Vector Indexing](/developers/weaviate/concepts/indexing/vector-indexes) +- [Configuration: Vector index](/developers/weaviate/config-refs/schema/vector-index.md) + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + diff --git a/developers/weaviate/concepts/indexing/dynamic-indexes.mdx b/developers/weaviate/concepts/indexing/dynamic-indexes.mdx index 9422fe2b29..3dab61bd01 100644 --- a/developers/weaviate/concepts/indexing/dynamic-indexes.mdx +++ b/developers/weaviate/concepts/indexing/dynamic-indexes.mdx @@ -5,23 +5,44 @@ image: og/docs/indexing.jpg # tags: ['basics']['indexes'] --- -:::info Added in `v1.25` -::: - import DynamicIntro from '/_includes/indexes/dynamic-intro.mdx'; -The flat index is ideal for collections with low object counts, less than 10,000 objects. Even though they are disk based, at that scale flat indexes provide lower memory overhead and good latency. When the number of objects increases, so does search latency. For collections with more than 10,000 objects, an HNSW index is usually a better indexing solution. The dynamic index balances search latency costs against in-memory costs. Memory costs are low until search latencies grow too large, then the index type switches to preserve low search latencies as your collection grows in size. +## Overview + +Dynamic indexes are flat indexed collections that Weaviate converts to HNSW indexed collections when the collection reaches a certain size. Flat indexes work well for collections with less than 10,000 objects. At that size, flat indexes have low memory overhead and good latency. But, search latency increases as the number of objects in a collection increases. When the collection grows to about 10,000 objects, an HNSW index usually has better latency than a flat index. + +The dynamic index helps to balance resource costs against search latency. Flat indexes are disc-based. HNSW indexes reside in RAM. Disk storage is orders of magnitude cheaper than RAM memory so hosting an index on disc is cheaper than hosting it in RAM. + +If your collection size grows over time, or if you have a mix of smaller and larger tenants, dynamic indexes let you take advantage of lower cost flat indexes while search latency times are low. When the object count increase and latencies grow larger, converting the flat index to an HNSW index preserves low search latencies at the expense of increased RAM memory costs. + +The dynamic index automatically switches from a flat index to an HNSW index when the object count exceeds a pre-specified threshold. (The default is 10,000 objects.) + +Dynamic indexing requires [asynchronous indexing](/developers/weaviate/configuration/indexing-vector/dynamic-indexes#asynchronous-indexing). After the dynamic index reaches the threshold value to change from a flat index to an HNSW index, newly added objects are sent to the async queue while Weaviate converts the indexes. When the HNSW index is ready, Weaviate processes the objects in the async queue and adds them to the index. + +## Index transformations + +The dynamic index only changes the index type once. The change only happens in one direction. The dynamic index converts a flat index to an HNSW index. If the size of a collection drops below the threshold value, the dynamic index does not revert the HNSW index back to a flat index. + +## Multi-tenant collections + +The size of tenants in a multi-tenant collection can have vary considerably. If you anticipate having tenants of different sizes, consider a dynamic index. The flat index reduces memory overhead for small tenants. If individual tenants grow larger than others, dynamically switching those tenants to HNSW preserves latency while saving on overall resource usage. + +## Other considerations -The dynamic index automatically switches from a flat index to an HNSW index when the object count exceeds a pre-specified threshold. (The default is 10,000 objects.) This functionality only works with async indexing enabled. When the threshold is hit while importing, all the data piles up in the async queue, the HNSW index is constructed in the background and when ready the swap from flat to HNSW is completed. +- Asynchronous indexing is a prerequisite for dynamic indexing. [Enable asynchronous indexing.](/developers/weaviate/configuration/indexing-vector/dynamic-indexes#asynchronous-indexing) +- Configure the flat index and HNSW index when you [define the collection](/developers/weaviate/configuration/indexing-vector/dynamic-indexes#configure-dynamic-indexes). Many [index parameters](/developers/weaviate/config-refs/schema/vector-index) cannot be modified after the collection is created. -Currently, this is only a one-way upgrade from a flat to an HNSW index, it does not support changing back to a flat index even if the object count goes below the threshold due to deletion. +## Related pages -This is particularly useful in a multi-tenant setup where building an HNSW index per tenant would introduce extra overhead. With a dynamic index, as individual tenants grow their index will switch from flat to HNSW, while smaller tenants' indexes remain flat. +For more information, see the following: +- [Configure dynamic indexes](/developers/weaviate/configuration/indexing-vector/dynamic-indexes) +- [Indexing starter guide](/developers/weaviate/starter-guides/managing-resources/indexing) +## Questions and feedback -import DynamicAsyncRequirements from '/_includes/dynamic-index-async-req.mdx'; +import DocsFeedback from '/_includes/docs-feedback.mdx'; - \ No newline at end of file + diff --git a/developers/weaviate/concepts/indexing/indexFilterable.md b/developers/weaviate/concepts/indexing/indexFilterable.md new file mode 100644 index 0000000000..a66200b615 --- /dev/null +++ b/developers/weaviate/concepts/indexing/indexFilterable.md @@ -0,0 +1,65 @@ +--- +title: indexFilterable indexes +sidebar_position: 130 +image: og/docs/indexing.jpg +# tags: ['basics'] +--- + +You can configure indexes in Weaviate per collection. + +Some things to bear in mind: + +* Especially for large datasets, configuring the indexes is important because the more you index, the more storage is needed. +* A rule of thumb -- if you don't query over a specific field or vector space, don't index it. +* One of Weaviate's unique features is how the indexes are configured (learn more about this [here](/developers/weaviate/concepts/prefiltering.md)). + + + +There are three inverted index types in Weaviate: + +- `indexSearchable` - a searchable index for BM25 or hybrid search +- `indexFilterable` - a match-based index for fast [filtering](/developers/weaviate/concepts/prefiltering.md) by matching criteria +- `indexRangeFilters` - a range-based index for [filtering](/developers/weaviate/concepts/prefiltering.md) by numerical ranges + +Each inverted index can be set to `true` (on) or `false` (off) on a property level. The `indexSearchable` and `indexFilterable` indexes are on by default, while the `indexRangeFilters` index is off by default. + +The filterable indexes are only capable of [filtering](/developers/weaviate/concepts/prefiltering.md), while the searchable index can be used for both searching and filtering (though not as fast as the filterable index). + +So, setting `"indexFilterable": false` and `"indexSearchable": true` (or not setting it at all) will have the trade-off of worse filtering performance but faster imports (due to only needing to update one index) and lower disk usage. + +See the [related how-to section](/developers/weaviate/manage-data/collections.mdx#property-level-settings) to learn how to enable or disable inverted indexes on a property level. + +A rule of thumb to follow when determining whether to switch off indexing is: _if you will never perform queries based on this property, you can turn it off._ + +#### Inverted index types summary + +import InvertedIndexTypesSummary from '/_includes/inverted-index-types-summary.mdx'; + + + +- Enable one or both of `indexFilterable` and `indexRangeFilters` to index a property for faster filtering. + - If only one is enabled, the respective index is used for filtering. + - If both are enabled, `indexRangeFilters` is used for operations involving comparison operators, and `indexFilterable` is used for equality and inequality operations. + +This chart shows which filter makes the comparison when one or both index type is `true` for an applicable property. + +| Operator | `indexRangeFilters` only | `indexFilterable` only | Both enabled | +| :- | :- | :- | :- | +| Equal | `indexRangeFilters` | `indexFilterable` | `indexFilterable` | +| Not equal | `indexRangeFilters` | `indexFilterable` | `indexFilterable` | +| Greater than | `indexRangeFilters` | `indexFilterable` | `indexRangeFilters` | +| Greater than equal | `indexRangeFilters` | `indexFilterable` | `indexRangeFilters` | +| Less than | `indexRangeFilters` | `indexFilterable` | `indexRangeFilters` | +| Less than equal | `indexRangeFilters` | `indexFilterable` | `indexRangeFilters` | + + +## Further resources + +- [Concepts: Vector Indexing](/developers/weaviate/concepts/indexing/vector-indexes) +- [Configuration: Vector index](/developers/weaviate/config-refs/schema/vector-index.md) + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + diff --git a/developers/weaviate/concepts/indexing/indexRangeFilters.md b/developers/weaviate/concepts/indexing/indexRangeFilters.md new file mode 100644 index 0000000000..56664aad5e --- /dev/null +++ b/developers/weaviate/concepts/indexing/indexRangeFilters.md @@ -0,0 +1,24 @@ +--- +title: indexRangeFilter indexes +sidebar_position: 140 +image: og/docs/indexing.jpg +# tags: ['basics'] +--- + +#### Inverted index for timestamps + +You can also enable an inverted index to search [based on timestamps](/developers/weaviate/config-refs/schema/index.md#invertedindexconfig--indextimestamps). + +Timestamps are currently indexed using the `indexFilterable` index. + + +## Further resources + +- [Concepts: Vector Indexing](/developers/weaviate/concepts/indexing/vector-indexes) +- [Configuration: Vector index](/developers/weaviate/config-refs/schema/vector-index.md) + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + diff --git a/developers/weaviate/concepts/indexing/indexSearchable.md b/developers/weaviate/concepts/indexing/indexSearchable.md new file mode 100644 index 0000000000..9d543007b6 --- /dev/null +++ b/developers/weaviate/concepts/indexing/indexSearchable.md @@ -0,0 +1,67 @@ +--- +title: indeSearchable indexes +sidebar_position: 120 +image: og/docs/indexing.jpg +# tags: ['basics'] +--- + +You can configure indexes in Weaviate per collection. + +Some things to bear in mind: + +* Especially for large datasets, configuring the indexes is important because the more you index, the more storage is needed. +* A rule of thumb -- if you don't query over a specific field or vector space, don't index it. +* One of Weaviate's unique features is how the indexes are configured (learn more about this [here](/developers/weaviate/concepts/prefiltering.md)). + + +## Inverted indexes + +### Configure the inverted index + +There are three inverted index types in Weaviate: + +- `indexSearchable` - a searchable index for BM25 or hybrid search +- `indexFilterable` - a match-based index for fast [filtering](/developers/weaviate/concepts/prefiltering.md) by matching criteria +- `indexRangeFilters` - a range-based index for [filtering](/developers/weaviate/concepts/prefiltering.md) by numerical ranges + +Each inverted index can be set to `true` (on) or `false` (off) on a property level. The `indexSearchable` and `indexFilterable` indexes are on by default, while the `indexRangeFilters` index is off by default. + +The filterable indexes are only capable of [filtering](/developers/weaviate/concepts/prefiltering.md), while the searchable index can be used for both searching and filtering (though not as fast as the filterable index). + +So, setting `"indexFilterable": false` and `"indexSearchable": true` (or not setting it at all) will have the trade-off of worse filtering performance but faster imports (due to only needing to update one index) and lower disk usage. + +See the [related how-to section](/developers/weaviate/manage-data/collections.mdx#property-level-settings) to learn how to enable or disable inverted indexes on a property level. + +A rule of thumb to follow when determining whether to switch off indexing is: _if you will never perform queries based on this property, you can turn it off._ + +#### Inverted index types summary + +import InvertedIndexTypesSummary from '/_includes/inverted-index-types-summary.mdx'; + + + +- Enable one or both of `indexFilterable` and `indexRangeFilters` to index a property for faster filtering. + - If only one is enabled, the respective index is used for filtering. + - If both are enabled, `indexRangeFilters` is used for operations involving comparison operators, and `indexFilterable` is used for equality and inequality operations. + +This chart shows which filter makes the comparison when one or both index type is `true` for an applicable property. + +| Operator | `indexRangeFilters` only | `indexFilterable` only | Both enabled | +| :- | :- | :- | :- | +| Equal | `indexRangeFilters` | `indexFilterable` | `indexFilterable` | +| Not equal | `indexRangeFilters` | `indexFilterable` | `indexFilterable` | +| Greater than | `indexRangeFilters` | `indexFilterable` | `indexRangeFilters` | +| Greater than equal | `indexRangeFilters` | `indexFilterable` | `indexRangeFilters` | +| Less than | `indexRangeFilters` | `indexFilterable` | `indexRangeFilters` | +| Less than equal | `indexRangeFilters` | `indexFilterable` | `indexRangeFilters` | + +## Further resources + +- [Concepts: Vector Indexing](/developers/weaviate/concepts/indexing/vector-indexes) +- [Configuration: Vector index](/developers/weaviate/config-refs/schema/vector-index.md) + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + diff --git a/developers/weaviate/concepts/indexing/inverted-indexes.md b/developers/weaviate/concepts/indexing/inverted-indexes.md index 4f45ef6df9..90f20d17cc 100644 --- a/developers/weaviate/concepts/indexing/inverted-indexes.md +++ b/developers/weaviate/concepts/indexing/inverted-indexes.md @@ -1,6 +1,6 @@ --- title: Inverted indexes -sidebar_position: 110 +sidebar_position: 100 image: og/docs/indexing.jpg # tags: ['basics'] --- From 9fb689bf4037bace8087a3939795032efb0be6d2 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Sun, 22 Sep 2024 20:16:58 -0400 Subject: [PATCH 58/67] dynamic indexes --- developers/weaviate/concepts/indexing/dynamic-indexes.mdx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/developers/weaviate/concepts/indexing/dynamic-indexes.mdx b/developers/weaviate/concepts/indexing/dynamic-indexes.mdx index 3dab61bd01..ae81e970e5 100644 --- a/developers/weaviate/concepts/indexing/dynamic-indexes.mdx +++ b/developers/weaviate/concepts/indexing/dynamic-indexes.mdx @@ -13,9 +13,9 @@ import DynamicIntro from '/_includes/indexes/dynamic-intro.mdx'; Dynamic indexes are flat indexed collections that Weaviate converts to HNSW indexed collections when the collection reaches a certain size. Flat indexes work well for collections with less than 10,000 objects. At that size, flat indexes have low memory overhead and good latency. But, search latency increases as the number of objects in a collection increases. When the collection grows to about 10,000 objects, an HNSW index usually has better latency than a flat index. -The dynamic index helps to balance resource costs against search latency. Flat indexes are disc-based. HNSW indexes reside in RAM. Disk storage is orders of magnitude cheaper than RAM memory so hosting an index on disc is cheaper than hosting it in RAM. +The dynamic index helps to balance resource costs against search latency times. Flat indexes are disc-based. They are responsive at low object counts, but get slower as object counts grow. HNSW indexes reside in RAM. They are very fast, but RAM is expensive. Disk storage is orders of magnitude cheaper than RAM memory, so hosting an index on disc is significantly cheaper than hosting it in RAM. -If your collection size grows over time, or if you have a mix of smaller and larger tenants, dynamic indexes let you take advantage of lower cost flat indexes while search latency times are low. When the object count increase and latencies grow larger, converting the flat index to an HNSW index preserves low search latencies at the expense of increased RAM memory costs. +If your collection size grows over time, or if you have a mix of smaller and larger tenants, dynamic indexes let you take advantage of lower cost flat indexes while object counts and search latency times are low. When the object count increase, and latencies grow larger, converting the flat index to an HNSW index preserves low search latencies at the expense of increased RAM memory costs. The dynamic index automatically switches from a flat index to an HNSW index when the object count exceeds a pre-specified threshold. (The default is 10,000 objects.) @@ -27,7 +27,7 @@ The dynamic index only changes the index type once. The change only happens in o ## Multi-tenant collections -The size of tenants in a multi-tenant collection can have vary considerably. If you anticipate having tenants of different sizes, consider a dynamic index. The flat index reduces memory overhead for small tenants. If individual tenants grow larger than others, dynamically switching those tenants to HNSW preserves latency while saving on overall resource usage. +The size of tenants in a multi-tenant collection can vary considerably. If you anticipate having tenants of different sizes, consider a dynamic index. The flat index reduces memory overhead for small tenants. If individual tenants grow larger than others, dynamically switching those tenants to HNSW preserves latency while saving on overall resource usage. ## Other considerations From 516cd52e65c5c69828dfd8c6c534b3afac3b9a28 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Sun, 22 Sep 2024 20:22:16 -0400 Subject: [PATCH 59/67] dynamic indexes --- developers/weaviate/concepts/indexing/flat-indexes.mdx | 2 -- 1 file changed, 2 deletions(-) diff --git a/developers/weaviate/concepts/indexing/flat-indexes.mdx b/developers/weaviate/concepts/indexing/flat-indexes.mdx index 16642f0fdb..4ccba3b398 100644 --- a/developers/weaviate/concepts/indexing/flat-indexes.mdx +++ b/developers/weaviate/concepts/indexing/flat-indexes.mdx @@ -5,8 +5,6 @@ image: og/docs/indexing.jpg # tags: ['basics']['indexes'] --- - - #### Flat indexes import FlatIntro from '/_includes/indexes/flat-intro.mdx'; From 3243c83c1d21dd5229a94b7d47abf7420f93b2f7 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Sun, 22 Sep 2024 22:54:39 -0400 Subject: [PATCH 60/67] vector overview --- _includes/indexes/flat-intro.mdx | 4 +- _includes/indexes/hnsw-intro.mdx | 2 +- .../concepts/indexing/inverted-indexes.md | 2 +- .../concepts/indexing/vector-indexes.md | 92 +++++++++---------- 4 files changed, 51 insertions(+), 49 deletions(-) diff --git a/_includes/indexes/flat-intro.mdx b/_includes/indexes/flat-intro.mdx index a06fd93829..151fd1e8cb 100644 --- a/_includes/indexes/flat-intro.mdx +++ b/_includes/indexes/flat-intro.mdx @@ -1 +1,3 @@ -[Flat indexes](/developers/weaviate/concepts/indexing/flat-indexes) are memory-efficient. They are disk based indexes that perform brute-force vector searches. Flat indexes work best with small collections, less than 10,000 objects. Vector search times increase linearly with the number of objects. \ No newline at end of file +[Flat indexes](/developers/weaviate/concepts/indexing/flat-indexes) are memory-efficient. Flat indexes are disk based, rather than loaded into RAM. + +Flat indexes do brute-force vector searches. The search latency increases linearly with the number of objects. For that reason, flat indexes work best with small collections, less than 10,000 objects. \ No newline at end of file diff --git a/_includes/indexes/hnsw-intro.mdx b/_includes/indexes/hnsw-intro.mdx index c93d28fb8e..5e200d147d 100644 --- a/_includes/indexes/hnsw-intro.mdx +++ b/_includes/indexes/hnsw-intro.mdx @@ -1 +1 @@ -[HNSW indexes](/developers/weaviate/concepts/indexing/hnsw-indexes) are high-performance, in-memory indexes. HNSW indexes scale well, meaning that vector searches remain fast even for very large data sets. \ No newline at end of file +[Hierarchical Navigable Small World (HNSW) indexes](/developers/weaviate/concepts/indexing/hnsw-indexes) are high-performance, in-memory indexes. HNSW indexes scale well; vector searches are fast, even for very large data sets. \ No newline at end of file diff --git a/developers/weaviate/concepts/indexing/inverted-indexes.md b/developers/weaviate/concepts/indexing/inverted-indexes.md index 90f20d17cc..744f719e99 100644 --- a/developers/weaviate/concepts/indexing/inverted-indexes.md +++ b/developers/weaviate/concepts/indexing/inverted-indexes.md @@ -1,5 +1,5 @@ --- -title: Inverted indexes +title: Inverted indexes overview sidebar_position: 100 image: og/docs/indexing.jpg # tags: ['basics'] diff --git a/developers/weaviate/concepts/indexing/vector-indexes.md b/developers/weaviate/concepts/indexing/vector-indexes.md index d2a3ba7425..127acb74c1 100644 --- a/developers/weaviate/concepts/indexing/vector-indexes.md +++ b/developers/weaviate/concepts/indexing/vector-indexes.md @@ -5,90 +5,91 @@ image: og/docs/indexing.jpg # tags: ['vector index plugins'] --- -## Vector indexes +Weaviate is a vector database. Most objects in Weaviate collections have one or more vectors. Individual vectors can have thousands of dimensions. Collections can have millions of objects. The resulting vector space can be exceedingly large. + +Weaviate uses vector indexes to [efficiently search]((https://weaviate.io/blog/why-is-vector-search-so-fast)) the vector space. Different vector index types offer trade-offs in resource use, speed, and accuracy. import VectorIntro from '/_includes/indexes/vector-intro.mdx'; -#### HNSW indexes +## Vector indexing -import HNSWIntro from '/_includes/indexes/hnsw-intro.mdx'; +[Vector embeddings](https://weaviate.io/blog/vector-embeddings-explained) are arrays of elements that can capture meaning. The original data can come from text, images, videos, or other content types. A model transforms the underlying data into an embedding. The elements in the embedding are called dimensions. High dimension vectors, with thousands of elements, capture more information, but they are harder to work with. - +Vector databases make it easier to work with high dimensional vector embeddings. Embeddings that capture similar meanings are closer to each other than embeddings that capture different meanings. To find objects that have similar semantics, vector databases must efficiently calculate the "distance" between the objects' embeddings. -HNSW indexes achieve this by building a multi-layered graph of objects, allowing for fast, approximate nearest neighbor searches. +In Weaviate, the [distance calculation method](/developers/weaviate/manage-data/collections#specify-a-distance-metric) is configurable. The [distance threshold](/developers/weaviate/search/similarity#set-a-similarity-threshold) is also configurable. A lower threshold returns more specific results. -While HNSW indexes enable fast searches, they use a lot of [*hot* resources](./index.md#-hot), as they load the graph structure and vectors into memory. +### Example - relative dimensions in space -Consider using [compression](./compression.mdx) to reduce the size of for your HNSW indexes. Weaviate offers several ways to compress your data: +The elements in a vector define a point in a multi-dimensional space, similar to how the coordinates (X,Y) define a point on a graph and (X,Y,Z) define a point in three dimensional space. -import CompressionAlgorithms from '/_includes/starter-guides/compression-types.mdx'; +Consider a very simple vector embedding that uses two elements to represent the meanings of some English words. Taken together, the vectors in the collection define a two dimensional space like this one: - - -#### Flat indexes +![2D Vector embedding visualization](./img/vectors-2d.png "2D Vectors visualization") -import FlatIntro from '/_includes/indexes/flat-intro.mdx'; +Apples and bananas are both fruits. Newspapers and magazines are both publications. In the representation, the fruits are close to each other. The publications are also close to each other, but relatively far from the fruits. The vector distances are smaller between items with similar semantics and larger between objects that don't share similar meanings. - +In a real example, the embeddings would have hundreds or thousands of elements. The vector space is difficult to visualize, but the concept is the same. Similar embeddings capture similar meanings and are closer to each other than to embeddings that capture different meanings. -As a result, flat indexes are best suited for cases where the number of objects is low and will not grow significantly. +For more details on this representation, see the [GloVe model](https://github.com/stanfordnlp/GloVe) from Stanford or our [vector embeddings blog post](https://weaviate.io/blog/vector-embeddings-explained#what-exactly-are-vector-embeddings). -[Binary quantization (BQ)](/developers/weaviate/configuration/compression/bq-compression) can improve flat indexes' search speeds. BQ improves search time by reducing the amount of data to read, and speeding up time taken to calculate the distance between vectors. +### Example - supermarket layout -#### Dynamic indexes +Consider how items are arranged in a supermarket. Similar products are close to each other. Apples and bananas are close to each other in the fruit section. Newspapers and magazines aren't fruit. They aren't even food. They are displayed in a different section of the supermarket. -:::info Added in `v1.25` -::: +![Supermarket map visualization as analogy for vector indexing](./img/supermarket.svg "Supermarket map visualization") -import DynamicIntro from '/_includes/indexes/dynamic-intro.mdx'; +If you walk into the supermarket to buy an apple, you move away from the publications and move towards the fruits. You minimize the distance from where you are to the object you are searching for. - +Vector search works the same way. Weaviate turns search queries into vector embeddings and then searches the collection to find objects that are close to the query in the vector space. +## HNSW indexes -Vector indexing is a key component of vector databases. It can help to [significantly **increase the speed** of the search process of similarity search](https://weaviate.io/blog/why-is-vector-search-so-fast) with only a minimal tradeoff in search accuracy ([HNSW index](#hierarchical-navigable-small-world-hnsw-index)), or efficiently store many subsets of data in a small memory footprint ([flat index](#flat-index)). The [dynamic index](#dynamic-index) can even start off as a flat index and then dynamically switch to the HNSW index as it scales past a threshold. +import HNSWIntro from '/_includes/indexes/hnsw-intro.mdx'; -Weaviate's vector-first storage system takes care of all storage operations with a vector index. Storing data in a vector-first manner not only allows for semantic or context-based search, but also makes it possible to store *very* large amounts of data without decreasing performance (assuming scaled well horizontally or having sufficient shards for the indexes). + -Weaviate supports these vector index types: -* [flat index](#flat-index): a simple, lightweight index that is designed for small datasets. -* [HNSW index](#hierarchical-navigable-small-world-hnsw-index): a more complex index that is slower to build, but it scales well to large datasets as queries have a logarithmic time complexity. -* [dynamic index](#dynamic-index): allows you to automatically switch from a flat index to an HNSW index as object count scales +HNSW indexes build a multi-layered object graph. The graph structure and HNSW algorithm result in fast, approximate nearest neighbor [(ANN)](https://en.wikipedia.org/wiki/Nearest_neighbor_search) searches. -:::caution Experimental feature -Available starting in `v1.25`. This is an experimental feature. Use with caution. -::: +The index and graph structure are stored in RAM memory. This makes HNSW indexes fast, but RAM is an expensive resource. Consider using [compression](./compression.mdx) to reduce the size of for your HNSW indexes. -This page explains what vector indexes are, and what purpose they serve in the Weaviate vector database. +Weaviate offers these methods to compress ("quantize") your HNSW index: -## Why do you need vector indexing? +import CompressionAlgorithms from '/_includes/starter-guides/compression-types.mdx'; -[Vector embeddings](https://weaviate.io/blog/vector-embeddings-explained) are a great way to represent meaning. Vectors embeddings are arrays of elements that can capture meaning from different data types, such as texts, images, videos, and other content. The number of elements are called dimensions. High dimension vectors capture more information, but they are harder to work with. + -Vector databases make it easier to work with high dimensional vectors. Consider search; Vector databases efficiently measure semantic similarity between data objects. When you run a [similarity search](https://weaviate.io/developers/weaviate/search/similarity), a vector database like Weaviate uses a vectorized version of the query to find objects in the database that have vectors similar to the query vector. +For more details, see [HNSW indexes](/developers/weaviate/concepts/indexing/hnsw-indexes). -Vectors are like coordinates in a multi-dimensional space. A very simple vector might represent objects, *words* in this case, in a 2-dimensional space. +## Flat indexes -In the graph below, the words `Apple` and `Banana` are shown close to each other. `Newspaper` and `Magazine` are also close to each other, but they are far away from `Apple` and `Banana` in the same vector space. +import FlatIntro from '/_includes/indexes/flat-intro.mdx'; -Within each pair, the distance between words is small because the objects have similar vector representations. The distance between the pairs is larger because the difference between the vectors is larger. Intuitively, fruits are similar to each other, but fruits are not similar to reading material. + -For more details of this representation, see: ([GloVe](https://github.com/stanfordnlp/GloVe)) and [vector embeddings](https://weaviate.io/blog/vector-embeddings-explained#what-exactly-are-vector-embeddings). +Flat indexes are best suited for collections that have relatively small object counts. If you expect the object count to grow significantly, consider using a [dynamic index](#dynamic-indexes). -![2D Vector embedding visualization](./img/vectors-2d.png "2D Vectors visualization") +[Binary quantization (BQ)](/developers/weaviate/configuration/compression/bq-compression) is a compression technique that also improves search speed for flat indexes. BQ reduces the amount of data the search engine reads. It also permits efficient binary calculations. These benefits of compression shorten the time needed to calculate vector distances during search. -Another way to think of this is how products are placed in a supermarket. You'd expect to find `Apples` close to `Bananas`, because they are both fruit. But when you are searching for a `Magazine`, you would move away from the `Apples` and `Bananas`, more towards the aisle with, for example, `Newspapers`. This is how the semantics of concepts can be stored in Weaviate as well, depending on the module you're using to calculate the numbers in the vectors. Not only words or text can be indexed as vectors, but also images, video, DNA sequences, etc. Read more about which model to use [here](/developers/weaviate/modules/index.md). +For more details, see [flat indexes](/developers/weaviate/concepts/indexing/flat-indexes). -![Supermarket map visualization as analogy for vector indexing](./img/supermarket.svg "Supermarket map visualization") +## Dynamic indexes -:::tip -You might be also interested in our blog post [Why is vector search to fast?](https://weaviate.io/blog/why-is-vector-search-so-fast). +:::info Added in `v1.25` ::: -## Hierarchical Navigable Small World (HNSW) index +import DynamicIntro from '/_includes/indexes/dynamic-intro.mdx'; -Add summary + + +For more details, see [dynamic indexes](/developers/weaviate/concepts/indexing/dynamic-indexes). + + +# # # # # # # # # # NEXT + +Not only words or text can be indexed as vectors, but also images, video, DNA sequences, etc. Read more about which model to use [here](/developers/weaviate/modules/index.md). ### Asynchronous indexing @@ -166,12 +167,11 @@ The [ANN benchmark page](/developers/weaviate/benchmarks/ann.md) contains a wide ## Further resources -:::info Related pages - [Concepts: Inverted indexes](/developers/weaviate/concepts/indexing/inverted-indexes) - [Concepts: Vector quantization (compression)](/developers/weaviate/concepts/vector-quantization) - [Configuration: Vector index](/developers/weaviate/config-refs/schema/vector-index) - [Configuration: Schema (Configure semantic indexing)](/developers/weaviate/config-refs/schema#configure-semantic-indexing) -::: +- [Compression overview](/developers/weaviate/starter-guides/managing-resources/compression) ## Questions and feedback From 5cd9a038adfe547eb3961eb3ec12b4b83bf59461 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Mon, 23 Sep 2024 00:00:10 -0400 Subject: [PATCH 61/67] vector overview --- .../concepts/indexing/dynamic-indexes.mdx | 2 +- .../concepts/indexing/vector-indexes.md | 75 ++++++------------- .../weaviate/concepts/vector-quantization.md | 2 - 3 files changed, 25 insertions(+), 54 deletions(-) diff --git a/developers/weaviate/concepts/indexing/dynamic-indexes.mdx b/developers/weaviate/concepts/indexing/dynamic-indexes.mdx index ae81e970e5..721edc8ba1 100644 --- a/developers/weaviate/concepts/indexing/dynamic-indexes.mdx +++ b/developers/weaviate/concepts/indexing/dynamic-indexes.mdx @@ -31,7 +31,7 @@ The size of tenants in a multi-tenant collection can vary considerably. If you a ## Other considerations -- Asynchronous indexing is a prerequisite for dynamic indexing. [Enable asynchronous indexing.](/developers/weaviate/configuration/indexing-vector/dynamic-indexes#asynchronous-indexing) +- Asynchronous indexing is a prerequisite for dynamic indexing. [Enable asynchronous indexing](/developers/weaviate/configuration/indexing-vector/dynamic-indexes#asynchronous-indexing). - Configure the flat index and HNSW index when you [define the collection](/developers/weaviate/configuration/indexing-vector/dynamic-indexes#configure-dynamic-indexes). Many [index parameters](/developers/weaviate/config-refs/schema/vector-index) cannot be modified after the collection is created. ## Related pages diff --git a/developers/weaviate/concepts/indexing/vector-indexes.md b/developers/weaviate/concepts/indexing/vector-indexes.md index 127acb74c1..d912da3be2 100644 --- a/developers/weaviate/concepts/indexing/vector-indexes.md +++ b/developers/weaviate/concepts/indexing/vector-indexes.md @@ -86,84 +86,56 @@ import DynamicIntro from '/_includes/indexes/dynamic-intro.mdx'; For more details, see [dynamic indexes](/developers/weaviate/concepts/indexing/dynamic-indexes). +## Asynchronous indexing -# # # # # # # # # # NEXT - -Not only words or text can be indexed as vectors, but also images, video, DNA sequences, etc. Read more about which model to use [here](/developers/weaviate/modules/index.md). - -### Asynchronous indexing - -:::caution Experimental -Available starting in `v1.22`. This is an experimental feature. Use with caution. +:::info Added in `v1.22` ::: -Starting in Weaviate `1.22`, you can use asynchronous indexing by opting in. +Asynchronous indexing separates indexing and object creation. Weaviate creates new objects faster. The vector index updates in the background. -Asynchronous indexing decouples object creation from vector index updates. Objects are created faster, and the vector index updates in the background. Asynchronous indexing is especially useful for importing large amounts of data. +While the vector index is updating, Weaviate can search a maximum of 100,000 un-indexed objects by brute force. Brute force searches are slower, so search performance drops during indexing. The 100,000 object limit means that any un-indexed objects beyond the limit are not included in the search. -While the vector index is updating, Weaviate can search a maximum of 100,000 un-indexed objects by brute force, that is, without using the vector index. This means that the search performance is slower until the vector index has been fully updated. Also, any additional new objects beyond the first 100,000 in the queue are not include in the search. +Asynchronous indexing is not enabled by default. [Enable asynchronous indexing](/developers/weaviate/configuration/indexing-vector/dynamic-indexes#asynchronous-indexing). -:::tip -You might be also interested in our blog post [Vamana vs. HNSW - Exploring ANN algorithms Part 1](https://weaviate.io/blog/ann-algorithms-vamana-vs-hnsw). -::: +## Compression -## Flat index +Weaviate stores objects and vector representations of those objects (vectors). Vectors can be very large. Vector dimensions are stored as 32 bit floats. A single vector with 1536 dimensions uses about 6 KB of storage. When collections have millions of objects, the resulting size can lead to significant costs, especially where an in-memory vector index is used. -Add summary - -## Dynamic index - -Add summary +Consider [enabling compression](/developers/weaviate/configuration/compression) to manage system resources. ## Vector cache considerations -For optimal search and import performance, previously imported vectors need to be in memory. A disk lookup for a vector is orders of magnitudes slower than memory lookup, so the disk cache should be used sparingly. However, Weaviate can limit the number of vectors in memory. By default, this limit is set to one trillion (`1e12`) objects when a new collection is created. +Weaviate uses an in-memory cache to hold vectors in RAM. -During import set `vectorCacheMaxObjects` high enough that all vectors can be held in memory. Each import requires multiple searches. Import performance drops drastically when there isn't enough memory to hold all of the vectors in the cache. +Each import requires multiple searches for indexing. By default, the cache is set to one trillion (`1e12`) objects when a new collection is created. During data import, set [`vectorCacheMaxObjects`](/developers/weaviate/configuration/indexing-vector) high enough that all of the vectors can be held in memory. Import performance drops drastically when there isn't enough memory to hold all of the vectors in the cache. After import, when your workload is mostly querying, experiment with vector cache limits that are less than your total dataset size. -Vectors that aren't currently in cache are added to the cache if there is still room. If the cache fills, Weaviate drops the whole cache. All future vectors have to be read from disk for the first time. Then, subsequent queries run against the cache until it fills again and the procedure repeats. Note that the cache can be a very valuable tool if you have a large dataset, and a large percentage of users only query a specific subset of vectors. In this case you might be able to serve the largest user group from cache while requiring disk lookups for "irregular" queries. - -## Vector indexing FAQ - -### Can I use vector indexing with vector quantization? - -Yes, you can read more about it in [vector quantization (compression)]DWCTODO. - -### Which vector index is right for me? - -A simple heuristic is that for use cases such as SaaS products where each end user (i.e. tenant) has their own, isolated, dataset, the `flat` index is a good choice. For use cases with large collections, the `hnsw` index may be a better choice. - -Note that the vector index type parameter only specifies how the vectors of data objects are *indexed*. The index is used for data retrieval and similarity search. - -The `vectorizer` parameter determines how the data vectors are created (which numbers the vectors contain). `vectorizer` specifies a [module](/developers/weaviate/modules/index.md), such as `text2vec-contextionary`, that Weaviate uses to create the vectors. (You can also set to `vectorizer` to `none` if you want to import your own vectors). - -To learn more about configuring the collection, see [this how-to page](/developers/weaviate/manage-data/collections). +Vectors that aren't currently in cache are added to the cache until it reaches the maximum size. If the cache fills, Weaviate drops the whole cache. Subsequent queries force data to be read from disk until the cache fills up again. -### Which distance metrics can I use with vector indexing? +Depending of query patterns, you may be able to tune the cache to hold frequently queried vectors in memory. Then, search only has to do slower disk lookups for less frequent queries. -All of [the distance metrics](/developers/weaviate/config-refs/distances.md), such as cosine similarity, can be used with any vector index type. +## Benchmarks -### How to configure the vector index type in Weaviate? +The [ANN benchmark page](/developers/weaviate/benchmarks/ann.md) has a wide variety of vector search use cases and relative benchmarks. -The index type can be specified per data collection via the [collection definition](/developers/weaviate/manage-data/collections.mdx#set-vector-index-type) settings, according to available [vector index settings](/developers/weaviate/config-refs/schema/vector-index). +Consult the page to find a dataset similar to yours. The benchmarks are a good starting point to learn to optimal configuration settings for each type of workload. -### When to skip indexing +## Recommendations -There are situations where it doesn't make sense to vectorize a collection. For example, if the collection consists solely of references between two other collections, or if the collection contains mostly duplicate elements. +- Large collections or tenants with more than 10,000 objects should use the HNSW index. -Importing duplicate vectors into HNSW is very expensive. The import algorithm checks early on if a candidate vector's distance is greater than the worst candidate's distance. When there are lots of duplicate vectors, this early exit condition is never met so each import or query results in an exhaustive search. +- Collections or tenants with less than 10,000 objects should use the flat index. -To avoid indexing a collection, set `"skip"` to `"true"`. By default, collections are indexed. +- Collections, especially multi-tenant collections with a variety of tenant sizes, should use a dynamic index. -### What ANN algorithms exist? +## Other considerations -There are different ANN algorithms, you can find a nice overview of them on this website. +- If a collection consists of references between two other collections, don't index it. -### Are there indicative benchmarks for Weaviate's ANN performance? +- Duplicate vectors are expensive to import. Avoid importing duplicate vectors since import speeds are very slow. -The [ANN benchmark page](/developers/weaviate/benchmarks/ann.md) contains a wide variety of vector search use cases and relative benchmarks. This page is ideal for finding a dataset similar to yours and learning what the most optimal settings are. +- The vector index type specifies how vectors are indexed. The `vectorizer` parameter specifies how to create a vector embedding. See [model providers](/developers/weaviate/model-providers) or set `vectorizer` to `none` to import your own vectors. ## Further resources @@ -172,6 +144,7 @@ The [ANN benchmark page](/developers/weaviate/benchmarks/ann.md) contains a wide - [Configuration: Vector index](/developers/weaviate/config-refs/schema/vector-index) - [Configuration: Schema (Configure semantic indexing)](/developers/weaviate/config-refs/schema#configure-semantic-indexing) - [Compression overview](/developers/weaviate/starter-guides/managing-resources/compression) +- [Blog post: Exploring ANN algorithms Part 1](https://weaviate.io/blog/ann-algorithms-vamana-vs-hnsw) ## Questions and feedback diff --git a/developers/weaviate/concepts/vector-quantization.md b/developers/weaviate/concepts/vector-quantization.md index 53be89bec0..3df051d2b3 100644 --- a/developers/weaviate/concepts/vector-quantization.md +++ b/developers/weaviate/concepts/vector-quantization.md @@ -130,7 +130,6 @@ In some cases, rescoring also includes over-fetching, whereby additional candida ## Further resources -:::info Related pages - [Concepts: Indexing](/developers/weaviate/concepts/indexing) - [Concepts: Vector Indexing](/developers/weaviate/concepts/indexing/vector-indexes) - [Configuration: Vector index](/developers/weaviate/config-refs/schema/vector-index.md) @@ -138,7 +137,6 @@ In some cases, rescoring also includes over-fetching, whereby additional candida - [How to configure: Binary quantization (compression)](../configuration/compression/bq-compression.md) - [How to configure: Product quantization (compression)](../configuration/compression/pq-compression.md) - [Weaviate Academy: 250 Vector Compression](../../academy/py/compression/index.md) -::: ## Questions and feedback From 7bbca9738b056f73c057b63b15aaac43fedc7578 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Mon, 23 Sep 2024 01:57:00 -0400 Subject: [PATCH 62/67] inverted --- .../concepts/indexing/flat-indexes.mdx | 13 ++++ .../concepts/indexing/hnsw-indexes.mdx | 13 ++++ .../concepts/indexing/indexFilterable.md | 65 ------------------ .../concepts/indexing/indexRangeFilters.md | 24 ------- .../concepts/indexing/indexSearchable.md | 67 ------------------- .../concepts/indexing/inverted-indexes.md | 49 +++++++++----- 6 files changed, 58 insertions(+), 173 deletions(-) delete mode 100644 developers/weaviate/concepts/indexing/indexFilterable.md delete mode 100644 developers/weaviate/concepts/indexing/indexRangeFilters.md delete mode 100644 developers/weaviate/concepts/indexing/indexSearchable.md diff --git a/developers/weaviate/concepts/indexing/flat-indexes.mdx b/developers/weaviate/concepts/indexing/flat-indexes.mdx index 4ccba3b398..3e09b50ffc 100644 --- a/developers/weaviate/concepts/indexing/flat-indexes.mdx +++ b/developers/weaviate/concepts/indexing/flat-indexes.mdx @@ -27,3 +27,16 @@ The **flat index** is a simple, lightweight index that is fast to build and has As the name suggests, the flat index is a single layer of disk-backed data objects and thus a very small memory footprint. The flat index is a good choice for small collections, such as for multi-tenancy use cases. A drawback of the flat index is that it does not scale well to large collections as it has a linear time complexity as a function of the number of data objects, unlike the `hnsw` index which has a logarithmic time complexity. + +## Related pages + +For more information, see the following: + +- [Configure dynamic indexes](/developers/weaviate/configuration/indexing-vector/dynamic-indexes) +- [Indexing starter guide](/developers/weaviate/starter-guides/managing-resources/indexing) + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + \ No newline at end of file diff --git a/developers/weaviate/concepts/indexing/hnsw-indexes.mdx b/developers/weaviate/concepts/indexing/hnsw-indexes.mdx index bfff1f71f4..522fe789c8 100644 --- a/developers/weaviate/concepts/indexing/hnsw-indexes.mdx +++ b/developers/weaviate/concepts/indexing/hnsw-indexes.mdx @@ -120,3 +120,16 @@ To change the default limit, edit the value for `QUERY_DEFAULTS_LIMIT` when you ### Deletions Cleanup is an async process runs that rebuilds the HNSW graph after deletes and updates. Prior to cleanup, objects are marked as deleted, but they are still connected to the HNSW graph. During cleanup, the edges are reassigned and the objects are deleted for good. + +## Related pages + +For more information, see the following: + +- [Configure dynamic indexes](/developers/weaviate/configuration/indexing-vector/dynamic-indexes) +- [Indexing starter guide](/developers/weaviate/starter-guides/managing-resources/indexing) + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + \ No newline at end of file diff --git a/developers/weaviate/concepts/indexing/indexFilterable.md b/developers/weaviate/concepts/indexing/indexFilterable.md deleted file mode 100644 index a66200b615..0000000000 --- a/developers/weaviate/concepts/indexing/indexFilterable.md +++ /dev/null @@ -1,65 +0,0 @@ ---- -title: indexFilterable indexes -sidebar_position: 130 -image: og/docs/indexing.jpg -# tags: ['basics'] ---- - -You can configure indexes in Weaviate per collection. - -Some things to bear in mind: - -* Especially for large datasets, configuring the indexes is important because the more you index, the more storage is needed. -* A rule of thumb -- if you don't query over a specific field or vector space, don't index it. -* One of Weaviate's unique features is how the indexes are configured (learn more about this [here](/developers/weaviate/concepts/prefiltering.md)). - - - -There are three inverted index types in Weaviate: - -- `indexSearchable` - a searchable index for BM25 or hybrid search -- `indexFilterable` - a match-based index for fast [filtering](/developers/weaviate/concepts/prefiltering.md) by matching criteria -- `indexRangeFilters` - a range-based index for [filtering](/developers/weaviate/concepts/prefiltering.md) by numerical ranges - -Each inverted index can be set to `true` (on) or `false` (off) on a property level. The `indexSearchable` and `indexFilterable` indexes are on by default, while the `indexRangeFilters` index is off by default. - -The filterable indexes are only capable of [filtering](/developers/weaviate/concepts/prefiltering.md), while the searchable index can be used for both searching and filtering (though not as fast as the filterable index). - -So, setting `"indexFilterable": false` and `"indexSearchable": true` (or not setting it at all) will have the trade-off of worse filtering performance but faster imports (due to only needing to update one index) and lower disk usage. - -See the [related how-to section](/developers/weaviate/manage-data/collections.mdx#property-level-settings) to learn how to enable or disable inverted indexes on a property level. - -A rule of thumb to follow when determining whether to switch off indexing is: _if you will never perform queries based on this property, you can turn it off._ - -#### Inverted index types summary - -import InvertedIndexTypesSummary from '/_includes/inverted-index-types-summary.mdx'; - - - -- Enable one or both of `indexFilterable` and `indexRangeFilters` to index a property for faster filtering. - - If only one is enabled, the respective index is used for filtering. - - If both are enabled, `indexRangeFilters` is used for operations involving comparison operators, and `indexFilterable` is used for equality and inequality operations. - -This chart shows which filter makes the comparison when one or both index type is `true` for an applicable property. - -| Operator | `indexRangeFilters` only | `indexFilterable` only | Both enabled | -| :- | :- | :- | :- | -| Equal | `indexRangeFilters` | `indexFilterable` | `indexFilterable` | -| Not equal | `indexRangeFilters` | `indexFilterable` | `indexFilterable` | -| Greater than | `indexRangeFilters` | `indexFilterable` | `indexRangeFilters` | -| Greater than equal | `indexRangeFilters` | `indexFilterable` | `indexRangeFilters` | -| Less than | `indexRangeFilters` | `indexFilterable` | `indexRangeFilters` | -| Less than equal | `indexRangeFilters` | `indexFilterable` | `indexRangeFilters` | - - -## Further resources - -- [Concepts: Vector Indexing](/developers/weaviate/concepts/indexing/vector-indexes) -- [Configuration: Vector index](/developers/weaviate/config-refs/schema/vector-index.md) - -## Questions and feedback - -import DocsFeedback from '/_includes/docs-feedback.mdx'; - - diff --git a/developers/weaviate/concepts/indexing/indexRangeFilters.md b/developers/weaviate/concepts/indexing/indexRangeFilters.md deleted file mode 100644 index 56664aad5e..0000000000 --- a/developers/weaviate/concepts/indexing/indexRangeFilters.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -title: indexRangeFilter indexes -sidebar_position: 140 -image: og/docs/indexing.jpg -# tags: ['basics'] ---- - -#### Inverted index for timestamps - -You can also enable an inverted index to search [based on timestamps](/developers/weaviate/config-refs/schema/index.md#invertedindexconfig--indextimestamps). - -Timestamps are currently indexed using the `indexFilterable` index. - - -## Further resources - -- [Concepts: Vector Indexing](/developers/weaviate/concepts/indexing/vector-indexes) -- [Configuration: Vector index](/developers/weaviate/config-refs/schema/vector-index.md) - -## Questions and feedback - -import DocsFeedback from '/_includes/docs-feedback.mdx'; - - diff --git a/developers/weaviate/concepts/indexing/indexSearchable.md b/developers/weaviate/concepts/indexing/indexSearchable.md deleted file mode 100644 index 9d543007b6..0000000000 --- a/developers/weaviate/concepts/indexing/indexSearchable.md +++ /dev/null @@ -1,67 +0,0 @@ ---- -title: indeSearchable indexes -sidebar_position: 120 -image: og/docs/indexing.jpg -# tags: ['basics'] ---- - -You can configure indexes in Weaviate per collection. - -Some things to bear in mind: - -* Especially for large datasets, configuring the indexes is important because the more you index, the more storage is needed. -* A rule of thumb -- if you don't query over a specific field or vector space, don't index it. -* One of Weaviate's unique features is how the indexes are configured (learn more about this [here](/developers/weaviate/concepts/prefiltering.md)). - - -## Inverted indexes - -### Configure the inverted index - -There are three inverted index types in Weaviate: - -- `indexSearchable` - a searchable index for BM25 or hybrid search -- `indexFilterable` - a match-based index for fast [filtering](/developers/weaviate/concepts/prefiltering.md) by matching criteria -- `indexRangeFilters` - a range-based index for [filtering](/developers/weaviate/concepts/prefiltering.md) by numerical ranges - -Each inverted index can be set to `true` (on) or `false` (off) on a property level. The `indexSearchable` and `indexFilterable` indexes are on by default, while the `indexRangeFilters` index is off by default. - -The filterable indexes are only capable of [filtering](/developers/weaviate/concepts/prefiltering.md), while the searchable index can be used for both searching and filtering (though not as fast as the filterable index). - -So, setting `"indexFilterable": false` and `"indexSearchable": true` (or not setting it at all) will have the trade-off of worse filtering performance but faster imports (due to only needing to update one index) and lower disk usage. - -See the [related how-to section](/developers/weaviate/manage-data/collections.mdx#property-level-settings) to learn how to enable or disable inverted indexes on a property level. - -A rule of thumb to follow when determining whether to switch off indexing is: _if you will never perform queries based on this property, you can turn it off._ - -#### Inverted index types summary - -import InvertedIndexTypesSummary from '/_includes/inverted-index-types-summary.mdx'; - - - -- Enable one or both of `indexFilterable` and `indexRangeFilters` to index a property for faster filtering. - - If only one is enabled, the respective index is used for filtering. - - If both are enabled, `indexRangeFilters` is used for operations involving comparison operators, and `indexFilterable` is used for equality and inequality operations. - -This chart shows which filter makes the comparison when one or both index type is `true` for an applicable property. - -| Operator | `indexRangeFilters` only | `indexFilterable` only | Both enabled | -| :- | :- | :- | :- | -| Equal | `indexRangeFilters` | `indexFilterable` | `indexFilterable` | -| Not equal | `indexRangeFilters` | `indexFilterable` | `indexFilterable` | -| Greater than | `indexRangeFilters` | `indexFilterable` | `indexRangeFilters` | -| Greater than equal | `indexRangeFilters` | `indexFilterable` | `indexRangeFilters` | -| Less than | `indexRangeFilters` | `indexFilterable` | `indexRangeFilters` | -| Less than equal | `indexRangeFilters` | `indexFilterable` | `indexRangeFilters` | - -## Further resources - -- [Concepts: Vector Indexing](/developers/weaviate/concepts/indexing/vector-indexes) -- [Configuration: Vector index](/developers/weaviate/config-refs/schema/vector-index.md) - -## Questions and feedback - -import DocsFeedback from '/_includes/docs-feedback.mdx'; - - diff --git a/developers/weaviate/concepts/indexing/inverted-indexes.md b/developers/weaviate/concepts/indexing/inverted-indexes.md index 744f719e99..309414d883 100644 --- a/developers/weaviate/concepts/indexing/inverted-indexes.md +++ b/developers/weaviate/concepts/indexing/inverted-indexes.md @@ -5,33 +5,48 @@ image: og/docs/indexing.jpg # tags: ['basics'] --- -You can configure indexes in Weaviate per collection. +[Inverted indexes](https://en.wikipedia.org/wiki/Inverted_index) map the contents of documents to improve search performance. The Weaviate database uses inverted indexes to search object properties efficiently. -Some things to bear in mind: +There are three inverted index types in Weaviate: -* Especially for large datasets, configuring the indexes is important because the more you index, the more storage is needed. -* A rule of thumb -- if you don't query over a specific field or vector space, don't index it. -* One of Weaviate's unique features is how the indexes are configured (learn more about this [here](/developers/weaviate/concepts/prefiltering.md)). +- [`indexSearchable`](#indexsearchable) A default index. BM25 (keyword) and hybrid search use `indexSearchable`. +- [`indexFilterable`](#indexfilterable) A default index. Filters use the `indexFilterable` index. +- [`indexRangeFilters](#indexrangefilters) An optional index. Numerical filters use the `indexRangeFilter`. -## Vector indexes +## Configuration -A vector index is used to serve all vector-search queries. Weaviate supports multiple types of vector indexes: +These indexes are set on the property level. `indexSearchable` and `indexFilterable` indexes are enabled by default. The `indexRangeFilters` index is off by default. -1. **HNSW** - an approximate nearest neighbor (ANN) search based vector index. HNSW indexes scale well with large datasets. -2. **Flat** - a vector index that is used for brute-force searches. This is useful for small datasets. -2. **Dynamic** - a vector index that is flat when the dataset is small and switches to HNSW when the dataset is large. +If you do not plan to search or filter on an object property, disable the index. Unused indexes waste space and they slow import processing. -For more information on vector indexes, see the [Vector Indexing](/developers/weaviate/concepts/indexing/vector-indexes) page. +To configure an inverted index see these examples: -## Inverted indexes +- [`indexSearchable`](/developers/weaviate/configuration/inverted-indexes#indexsearchable) +- [`indexFilterable`](/developers/weaviate/configuration/inverted-indexes#indexfilterable) +- [`indexRangeFilters](/developers/weaviate/configuration/inverted-indexes#indexrangefilters) + +Keyword searches and hybrid searches use the BM25 ranking algorithm in conjunction with the `indexSearchable` index. To configure the BM25 algorithm, see these examples: + +- [BM25]/developers/weaviate/configuration/inverted-indexes#bm25) + +## indexSearchable + +## indexFilterable +[Filters](/developers/weaviate/concepts/prefiltering.md) + +## indexRangeFilters + +:::info Added in `1.26` +::: + +Weaviate `1.26` introduces the `indexRangeFilters` index for filtering by numerical ranges. This index is available for properties of type `int`, `number`, or `date`. The index is not available for arrays of these data types. + +Internally, rangeable indexes are implemented as [roaring bitmap slices](https://www.featurebase.com/blog/range-encoded-bitmaps). This data structure limits the index to values that can be stored as 64 bit integers. + +The `indexRangeFilters` index is only available for new properties. Existing properties cannot be converted to use the rangeable index. -### Configure the inverted index -There are three inverted index types in Weaviate: -- `indexSearchable` - a searchable index for BM25 or hybrid search -- `indexFilterable` - a match-based index for fast [filtering](/developers/weaviate/concepts/prefiltering.md) by matching criteria -- `indexRangeFilters` - a range-based index for [filtering](/developers/weaviate/concepts/prefiltering.md) by numerical ranges Each inverted index can be set to `true` (on) or `false` (off) on a property level. The `indexSearchable` and `indexFilterable` indexes are on by default, while the `indexRangeFilters` index is off by default. From 6cadbbee7997420f4ea9fb6b3a67f39fc36dd924 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Mon, 23 Sep 2024 02:25:49 -0400 Subject: [PATCH 63/67] inverted --- .../concepts/indexing/inverted-indexes.md | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/developers/weaviate/concepts/indexing/inverted-indexes.md b/developers/weaviate/concepts/indexing/inverted-indexes.md index 309414d883..0eed45a8cf 100644 --- a/developers/weaviate/concepts/indexing/inverted-indexes.md +++ b/developers/weaviate/concepts/indexing/inverted-indexes.md @@ -31,34 +31,36 @@ Keyword searches and hybrid searches use the BM25 ranking algorithm in conjuncti ## indexSearchable -## indexFilterable -[Filters](/developers/weaviate/concepts/prefiltering.md) +[Keyword searches](/developers/weaviate/search/bm25) and [hybrid searches](/developers/weaviate/search/hybrid) use the BM25 ranking algorithm in conjunction with the `indexSearchable` index. These searches match the search terms in a query to the same term in the database. They only match terms, not meanings. To match meanings, a semantic search, use a [vector similarity search](/developers/weaviate/search/similarity). -## indexRangeFilters +`indexSearchable` is enabled by default. This index is required for BM25 search. -:::info Added in `1.26` -::: +For simple (non-BM25) filters, the `indexFilterable` index enables faster filtering. However, if the `indexFilterable` index is not enabled, Weaviate uses the `indexSearchable` index to [filter](/developers/weaviate/concepts/prefiltering). -Weaviate `1.26` introduces the `indexRangeFilters` index for filtering by numerical ranges. This index is available for properties of type `int`, `number`, or `date`. The index is not available for arrays of these data types. +For configuration examples, see [`indexSearchable`](/developers/weaviate/configuration/inverted-indexes#indexsearchable). -Internally, rangeable indexes are implemented as [roaring bitmap slices](https://www.featurebase.com/blog/range-encoded-bitmaps). This data structure limits the index to values that can be stored as 64 bit integers. +## indexFilterable -The `indexRangeFilters` index is only available for new properties. Existing properties cannot be converted to use the rangeable index. +`indexFilterable` is enabled by default. This index is not required for filtering of BM25 search. However, this index is much faster for filtering than the `indexSearchable` index. +If you do not expect to filter on a property often, disable this filter to save space and to improve import times. For occasional filtered queries, the `indexSearchable` index is sufficient. If you expect to filter on a property regularly, leave this index enabled. +For configuration examples, see [`indexFilterable`](/developers/weaviate/configuration/inverted-indexes#indexfilterable). +## indexRangeFilters -Each inverted index can be set to `true` (on) or `false` (off) on a property level. The `indexSearchable` and `indexFilterable` indexes are on by default, while the `indexRangeFilters` index is off by default. +:::info Added in `1.26` +::: -The filterable indexes are only capable of [filtering](/developers/weaviate/concepts/prefiltering.md), while the searchable index can be used for both searching and filtering (though not as fast as the filterable index). +The `indexRangeFilters` index is for filtering by numerical ranges. This index is not enabled by default. -So, setting `"indexFilterable": false` and `"indexSearchable": true` (or not setting it at all) will have the trade-off of worse filtering performance but faster imports (due to only needing to update one index) and lower disk usage. +Internally, rangeable indexes are implemented as [roaring bitmap slices](https://www.featurebase.com/blog/range-encoded-bitmaps). The roaring bitmap data structure limits the `indexRangeFilters` index to values that can be stored as 64 bit integers. This index is available for properties of type `int`, `number`, or `date`. The `indexRangeFilters` index is not available for arrays of these data types. -See the [related how-to section](/developers/weaviate/manage-data/collections.mdx#property-level-settings) to learn how to enable or disable inverted indexes on a property level. +The `indexRangeFilters` index is only available for new properties. Existing properties cannot be converted to use the rangeable index. -A rule of thumb to follow when determining whether to switch off indexing is: _if you will never perform queries based on this property, you can turn it off._ +For configuration examples, see [`indexRangeFilters](/developers/weaviate/configuration/inverted-indexes#indexrangefilters). -#### Inverted index types summary +## Inverted index types summary import InvertedIndexTypesSummary from '/_includes/inverted-index-types-summary.mdx'; From bfc0a024e5aa00cf78ed55f00c4f8f7a22c3583b Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Mon, 23 Sep 2024 02:55:09 -0400 Subject: [PATCH 64/67] inverted --- .../concepts/indexing/bm25-ranking.md | 136 +----------------- .../concepts/indexing/hnsw-indexes.mdx | 2 +- .../concepts/indexing/inverted-indexes.md | 84 ++--------- .../concepts/indexing/vector-indexes.md | 4 +- .../managing-resources/compression.mdx | 2 +- 5 files changed, 23 insertions(+), 205 deletions(-) diff --git a/developers/weaviate/concepts/indexing/bm25-ranking.md b/developers/weaviate/concepts/indexing/bm25-ranking.md index 4dd057e2e5..23b0de576c 100644 --- a/developers/weaviate/concepts/indexing/bm25-ranking.md +++ b/developers/weaviate/concepts/indexing/bm25-ranking.md @@ -1,143 +1,17 @@ --- -title: BM25 ranking +title: BM25 ranking algorithm sidebar_position: 110 image: og/docs/indexing.jpg # tags: ['basics'] --- -You can configure indexes in Weaviate per collection. - -Some things to bear in mind: - -* Especially for large datasets, configuring the indexes is important because the more you index, the more storage is needed. -* A rule of thumb -- if you don't query over a specific field or vector space, don't index it. -* One of Weaviate's unique features is how the indexes are configured (learn more about this [here](/developers/weaviate/concepts/prefiltering.md)). - -## Vector indexes - -A vector index is used to serve all vector-search queries. Weaviate supports multiple types of vector indexes: - -1. **HNSW** - an approximate nearest neighbor (ANN) search based vector index. HNSW indexes scale well with large datasets. -2. **Flat** - a vector index that is used for brute-force searches. This is useful for small datasets. -2. **Dynamic** - a vector index that is flat when the dataset is small and switches to HNSW when the dataset is large. - -For more information on vector indexes, see the [Vector Indexing](/developers/weaviate/concepts/indexing/vector-indexes) page. - -## Inverted indexes - -### Configure the inverted index - -There are three inverted index types in Weaviate: - -- `indexSearchable` - a searchable index for BM25 or hybrid search -- `indexFilterable` - a match-based index for fast [filtering](/developers/weaviate/concepts/prefiltering.md) by matching criteria -- `indexRangeFilters` - a range-based index for [filtering](/developers/weaviate/concepts/prefiltering.md) by numerical ranges - -Each inverted index can be set to `true` (on) or `false` (off) on a property level. The `indexSearchable` and `indexFilterable` indexes are on by default, while the `indexRangeFilters` index is off by default. - -The filterable indexes are only capable of [filtering](/developers/weaviate/concepts/prefiltering.md), while the searchable index can be used for both searching and filtering (though not as fast as the filterable index). - -So, setting `"indexFilterable": false` and `"indexSearchable": true` (or not setting it at all) will have the trade-off of worse filtering performance but faster imports (due to only needing to update one index) and lower disk usage. - -See the [related how-to section](/developers/weaviate/manage-data/collections.mdx#property-level-settings) to learn how to enable or disable inverted indexes on a property level. - -A rule of thumb to follow when determining whether to switch off indexing is: _if you will never perform queries based on this property, you can turn it off._ - -#### Inverted index types summary - -import InvertedIndexTypesSummary from '/_includes/inverted-index-types-summary.mdx'; - - - -- Enable one or both of `indexFilterable` and `indexRangeFilters` to index a property for faster filtering. - - If only one is enabled, the respective index is used for filtering. - - If both are enabled, `indexRangeFilters` is used for operations involving comparison operators, and `indexFilterable` is used for equality and inequality operations. - -This chart shows which filter makes the comparison when one or both index type is `true` for an applicable property. - -| Operator | `indexRangeFilters` only | `indexFilterable` only | Both enabled | -| :- | :- | :- | :- | -| Equal | `indexRangeFilters` | `indexFilterable` | `indexFilterable` | -| Not equal | `indexRangeFilters` | `indexFilterable` | `indexFilterable` | -| Greater than | `indexRangeFilters` | `indexFilterable` | `indexRangeFilters` | -| Greater than equal | `indexRangeFilters` | `indexFilterable` | `indexRangeFilters` | -| Less than | `indexRangeFilters` | `indexFilterable` | `indexRangeFilters` | -| Less than equal | `indexRangeFilters` | `indexFilterable` | `indexRangeFilters` | - -#### Inverted index for timestamps - -You can also enable an inverted index to search [based on timestamps](/developers/weaviate/config-refs/schema/index.md#invertedindexconfig--indextimestamps). - -Timestamps are currently indexed using the `indexFilterable` index. - -## Collections without indexes - -If you don't want to set an index at all, this is possible too. - -To create a collection without any indexes, skip indexing on the collection and on the properties. - -```js -{ - "class": "Author", - "description": "A description of this collection, in this case, it's about authors", - "vectorIndexConfig": { - "skip": true // <== disable vector index - }, - "properties": [ - { - "indexFilterable": false, // <== disable filterable index for this property - "indexSearchable": false, // <== disable searchable index for this property - "dataType": [ - "text" - ], - "description": "The name of the Author", - "name": "name" - }, - { - "indexFilterable": false, // <== disable filterable index for this property - "dataType": [ - "int" - ], - "description": "The age of the Author", - "name": "age" - }, - { - "indexFilterable": false, // <== disable filterable index for this property - "dataType": [ - "date" - ], - "description": "The date of birth of the Author", - "name": "born" - }, - { - "indexFilterable": false, // <== disable filterable index for this property - "dataType": [ - "boolean" - ], - "description": "A boolean value if the Author won a nobel prize", - "name": "wonNobelPrize" - }, - { - "indexFilterable": false, // <== disable filterable index for this property - "indexSearchable": false, // <== disable searchable index for this property - "dataType": [ - "text" - ], - "description": "A description of the author", - "name": "description" - } - ] -} -``` - -## BM25 ranking algorithm - -Some stuff ## Further resources -- [Concepts: Vector Indexing](/developers/weaviate/concepts/indexing/vector-indexes) -- [Configuration: Vector index](/developers/weaviate/config-refs/schema/vector-index.md) +- [Concepts: Inverted Indexes](/developers/weaviate/concepts/indexing/inverted-indexes) +- [Configuration: BM25 algorithm](/developers/weaviate/configuration/inverted-indexes#bm25) +- [Blog: Ranking models](https://weaviate.io/blog/ranking-models-for-better-search) +- [Blog: Hybrid search](https://weaviate.io/blog/hybrid-search-for-web-developers) ## Questions and feedback diff --git a/developers/weaviate/concepts/indexing/hnsw-indexes.mdx b/developers/weaviate/concepts/indexing/hnsw-indexes.mdx index 522fe789c8..4438b718a3 100644 --- a/developers/weaviate/concepts/indexing/hnsw-indexes.mdx +++ b/developers/weaviate/concepts/indexing/hnsw-indexes.mdx @@ -17,7 +17,7 @@ HNSW indexes achieve this by building a multi-layered graph of objects, allowing While HNSW indexes enable fast searches, they use a lot of [*hot* resources](./index.md#-hot), as they load the graph structure and vectors into memory. -Consider using [compression](./compression.mdx) to reduce the size of for your HNSW indexes. Weaviate offers several ways to compress your data: +Consider using [compression](/developers/weaviate/starter-guides/managing-resources/compression) to reduce the size of for your HNSW indexes. Weaviate offers several ways to compress your data: import CompressionAlgorithms from '/_includes/starter-guides/compression-types.mdx'; diff --git a/developers/weaviate/concepts/indexing/inverted-indexes.md b/developers/weaviate/concepts/indexing/inverted-indexes.md index 0eed45a8cf..32ecf31db4 100644 --- a/developers/weaviate/concepts/indexing/inverted-indexes.md +++ b/developers/weaviate/concepts/indexing/inverted-indexes.md @@ -11,7 +11,7 @@ There are three inverted index types in Weaviate: - [`indexSearchable`](#indexsearchable) A default index. BM25 (keyword) and hybrid search use `indexSearchable`. - [`indexFilterable`](#indexfilterable) A default index. Filters use the `indexFilterable` index. -- [`indexRangeFilters](#indexrangefilters) An optional index. Numerical filters use the `indexRangeFilter`. +- [`indexRangeFilters`](#indexrangefilters) An optional index. Numerical filters use the `indexRangeFilter`. ## Configuration @@ -27,7 +27,7 @@ To configure an inverted index see these examples: Keyword searches and hybrid searches use the BM25 ranking algorithm in conjunction with the `indexSearchable` index. To configure the BM25 algorithm, see these examples: -- [BM25]/developers/weaviate/configuration/inverted-indexes#bm25) +- [BM25](/developers/weaviate/configuration/inverted-indexes#bm25) ## indexSearchable @@ -43,6 +43,7 @@ For configuration examples, see [`indexSearchable`](/developers/weaviate/configu `indexFilterable` is enabled by default. This index is not required for filtering of BM25 search. However, this index is much faster for filtering than the `indexSearchable` index. +You can also enable an inverted index to search [based on timestamps](/developers/weaviate/config-refs/schema/index.md#invertedindexconfig--indextimestamps). If you do not expect to filter on a property often, disable this filter to save space and to improve import times. For occasional filtered queries, the `indexSearchable` index is sufficient. If you expect to filter on a property regularly, leave this index enabled. For configuration examples, see [`indexFilterable`](/developers/weaviate/configuration/inverted-indexes#indexfilterable). @@ -66,11 +67,14 @@ import InvertedIndexTypesSummary from '/_includes/inverted-index-types-summary.m -- Enable one or both of `indexFilterable` and `indexRangeFilters` to index a property for faster filtering. - - If only one is enabled, the respective index is used for filtering. - - If both are enabled, `indexRangeFilters` is used for operations involving comparison operators, and `indexFilterable` is used for equality and inequality operations. +## Multiple filter indexes + +`indexFilterable` and `indexRangeFilters` both enable faster filtering. -This chart shows which filter makes the comparison when one or both index type is `true` for an applicable property. +- If only one of them is enabled on a property, that index is used for filtering. +- If both are enabled on a property, `indexRangeFilters` is used with numeric comparison operators, and `indexFilterable` is used for equality and inequality operations. + +This chart shows Weaviate applies the filters: | Operator | `indexRangeFilters` only | `indexFilterable` only | Both enabled | | :- | :- | :- | :- | @@ -81,71 +85,11 @@ This chart shows which filter makes the comparison when one or both index type i | Less than | `indexRangeFilters` | `indexFilterable` | `indexRangeFilters` | | Less than equal | `indexRangeFilters` | `indexFilterable` | `indexRangeFilters` | -#### Inverted index for timestamps +## Collection level settings -You can also enable an inverted index to search [based on timestamps](/developers/weaviate/config-refs/schema/index.md#invertedindexconfig--indextimestamps). +Some inverted index values are configured at the collection level. These settings allow you do create indexes for [timestamps](/developers/weaviate/config-refs/schema#indextimestamps), filter on [`null`](/developers/weaviate/config-refs/schema#indexnullstate) and filter on [property length](/developers/weaviate/config-refs/schema#indexpropertylength). -Timestamps are currently indexed using the `indexFilterable` index. - -## Collections without indexes - -If you don't want to set an index at all, this is possible too. - -To create a collection without any indexes, skip indexing on the collection and on the properties. - -```js -{ - "class": "Author", - "description": "A description of this collection, in this case, it's about authors", - "vectorIndexConfig": { - "skip": true // <== disable vector index - }, - "properties": [ - { - "indexFilterable": false, // <== disable filterable index for this property - "indexSearchable": false, // <== disable searchable index for this property - "dataType": [ - "text" - ], - "description": "The name of the Author", - "name": "name" - }, - { - "indexFilterable": false, // <== disable filterable index for this property - "dataType": [ - "int" - ], - "description": "The age of the Author", - "name": "age" - }, - { - "indexFilterable": false, // <== disable filterable index for this property - "dataType": [ - "date" - ], - "description": "The date of birth of the Author", - "name": "born" - }, - { - "indexFilterable": false, // <== disable filterable index for this property - "dataType": [ - "boolean" - ], - "description": "A boolean value if the Author won a nobel prize", - "name": "wonNobelPrize" - }, - { - "indexFilterable": false, // <== disable filterable index for this property - "indexSearchable": false, // <== disable searchable index for this property - "dataType": [ - "text" - ], - "description": "A description of the author", - "name": "description" - } - ] -} -``` +If you so not intend to filter on these properties, do not enable these indexes. ## BM25 ranking algorithm @@ -154,7 +98,7 @@ Some stuff ## Further resources - [Concepts: Vector Indexing](/developers/weaviate/concepts/indexing/vector-indexes) -- [Configuration: Vector index](/developers/weaviate/config-refs/schema/vector-index.md) +- [Configuration: Inverted indexes](/developers/weaviate/configuration/inverted-indexes) ## Questions and feedback diff --git a/developers/weaviate/concepts/indexing/vector-indexes.md b/developers/weaviate/concepts/indexing/vector-indexes.md index d912da3be2..6c68de345f 100644 --- a/developers/weaviate/concepts/indexing/vector-indexes.md +++ b/developers/weaviate/concepts/indexing/vector-indexes.md @@ -7,7 +7,7 @@ image: og/docs/indexing.jpg Weaviate is a vector database. Most objects in Weaviate collections have one or more vectors. Individual vectors can have thousands of dimensions. Collections can have millions of objects. The resulting vector space can be exceedingly large. -Weaviate uses vector indexes to [efficiently search]((https://weaviate.io/blog/why-is-vector-search-so-fast)) the vector space. Different vector index types offer trade-offs in resource use, speed, and accuracy. +Weaviate uses vector indexes to [efficiently search](https://weaviate.io/blog/why-is-vector-search-so-fast) the vector space. Different vector index types offer trade-offs in resource use, speed, and accuracy. import VectorIntro from '/_includes/indexes/vector-intro.mdx'; @@ -53,7 +53,7 @@ import HNSWIntro from '/_includes/indexes/hnsw-intro.mdx'; HNSW indexes build a multi-layered object graph. The graph structure and HNSW algorithm result in fast, approximate nearest neighbor [(ANN)](https://en.wikipedia.org/wiki/Nearest_neighbor_search) searches. -The index and graph structure are stored in RAM memory. This makes HNSW indexes fast, but RAM is an expensive resource. Consider using [compression](./compression.mdx) to reduce the size of for your HNSW indexes. +The index and graph structure are stored in RAM memory. This makes HNSW indexes fast, but RAM is an expensive resource. Consider using [compression](/developers/weaviate/starter-guides/managing-resources/compression) to reduce the size of for your HNSW indexes. Weaviate offers these methods to compress ("quantize") your HNSW index: diff --git a/developers/weaviate/starter-guides/managing-resources/compression.mdx b/developers/weaviate/starter-guides/managing-resources/compression.mdx index 4a58a65636..4b40adf30c 100644 --- a/developers/weaviate/starter-guides/managing-resources/compression.mdx +++ b/developers/weaviate/starter-guides/managing-resources/compression.mdx @@ -15,7 +15,7 @@ Weaviate creates indexes to search the vector space for your collection. By defa In many cases, you can use compression or a different index type to change the way Weaviate stores and searches your data, and still maintain high levels of recall. Updating the default settings can result in significant cost savings and performance improvements. -This page discusses compression algorithms. For more on indexes, see [Vector indexing](/developers/weaviate/concepts/indexing/vector-index). +This page discusses compression algorithms. For more on indexes, see [Vector indexing](/developers/weaviate/concepts/indexing/vector-indexes). ## Compression algorithms From 1600c5f4fd8769593c1a6b758d8cda5f2ade3ca3 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Mon, 23 Sep 2024 04:53:18 -0400 Subject: [PATCH 65/67] hnsw --- _includes/indexes/flat-intro.mdx | 6 +- _includes/indexes/hnsw-how.mdx | 3 + .../concepts/indexing/bm25-ranking.md | 20 ---- .../concepts/indexing/dynamic-indexes.mdx | 2 + .../concepts/indexing/flat-indexes.mdx | 18 +--- .../concepts/indexing/hnsw-indexes.mdx | 95 +++++++++++-------- .../concepts/indexing/inverted-indexes.md | 19 +++- .../concepts/indexing/vector-indexes.md | 6 +- 8 files changed, 89 insertions(+), 80 deletions(-) create mode 100644 _includes/indexes/hnsw-how.mdx delete mode 100644 developers/weaviate/concepts/indexing/bm25-ranking.md diff --git a/_includes/indexes/flat-intro.mdx b/_includes/indexes/flat-intro.mdx index 151fd1e8cb..3560af2e0e 100644 --- a/_includes/indexes/flat-intro.mdx +++ b/_includes/indexes/flat-intro.mdx @@ -1,3 +1,5 @@ -[Flat indexes](/developers/weaviate/concepts/indexing/flat-indexes) are memory-efficient. Flat indexes are disk based, rather than loaded into RAM. +[Flat indexes](/developers/weaviate/concepts/indexing/flat-indexes) are disk based. Compared to HNSW indexes, flat indexes have very small RAM requirements. -Flat indexes do brute-force vector searches. The search latency increases linearly with the number of objects. For that reason, flat indexes work best with small collections, less than 10,000 objects. \ No newline at end of file +Flat indexes do brute-force vector searches. The search latency increases linearly with the number of objects. For that reason, flat indexes work best with small collections, less than 10,000 objects. + +Flat indexes are best suited for collections that have relatively small object counts. If you expect the object count to grow significantly, consider using a [dynamic index](#dynamic-indexes). \ No newline at end of file diff --git a/_includes/indexes/hnsw-how.mdx b/_includes/indexes/hnsw-how.mdx new file mode 100644 index 0000000000..809f66234f --- /dev/null +++ b/_includes/indexes/hnsw-how.mdx @@ -0,0 +1,3 @@ +HNSW indexes build a multi-layered object graph. The graph structure and HNSW algorithm result in efficient, approximate nearest neighbor [(ANN)](https://en.wikipedia.org/wiki/Nearest_neighbor_search) searches. + +The index and graph structure are stored in RAM memory. This makes HNSW indexes fast, but RAM is an expensive resource. Consider using [compression](/developers/weaviate/starter-guides/managing-resources/compression) to reduce the size of for your HNSW indexes. \ No newline at end of file diff --git a/developers/weaviate/concepts/indexing/bm25-ranking.md b/developers/weaviate/concepts/indexing/bm25-ranking.md deleted file mode 100644 index 23b0de576c..0000000000 --- a/developers/weaviate/concepts/indexing/bm25-ranking.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -title: BM25 ranking algorithm -sidebar_position: 110 -image: og/docs/indexing.jpg -# tags: ['basics'] ---- - - -## Further resources - -- [Concepts: Inverted Indexes](/developers/weaviate/concepts/indexing/inverted-indexes) -- [Configuration: BM25 algorithm](/developers/weaviate/configuration/inverted-indexes#bm25) -- [Blog: Ranking models](https://weaviate.io/blog/ranking-models-for-better-search) -- [Blog: Hybrid search](https://weaviate.io/blog/hybrid-search-for-web-developers) - -## Questions and feedback - -import DocsFeedback from '/_includes/docs-feedback.mdx'; - - diff --git a/developers/weaviate/concepts/indexing/dynamic-indexes.mdx b/developers/weaviate/concepts/indexing/dynamic-indexes.mdx index 721edc8ba1..9ab7f02c7b 100644 --- a/developers/weaviate/concepts/indexing/dynamic-indexes.mdx +++ b/developers/weaviate/concepts/indexing/dynamic-indexes.mdx @@ -9,6 +9,8 @@ import DynamicIntro from '/_includes/indexes/dynamic-intro.mdx'; +To configure a dynamic index, see: [Configure dynamic indexes](/developers/weaviate/configuration/indexing-vector/dynamic-indexes). + ## Overview Dynamic indexes are flat indexed collections that Weaviate converts to HNSW indexed collections when the collection reaches a certain size. Flat indexes work well for collections with less than 10,000 objects. At that size, flat indexes have low memory overhead and good latency. But, search latency increases as the number of objects in a collection increases. When the collection grows to about 10,000 objects, an HNSW index usually has better latency than a flat index. diff --git a/developers/weaviate/concepts/indexing/flat-indexes.mdx b/developers/weaviate/concepts/indexing/flat-indexes.mdx index 3e09b50ffc..92351e75c6 100644 --- a/developers/weaviate/concepts/indexing/flat-indexes.mdx +++ b/developers/weaviate/concepts/indexing/flat-indexes.mdx @@ -5,28 +5,16 @@ image: og/docs/indexing.jpg # tags: ['basics']['indexes'] --- -#### Flat indexes +:::info Added in `v1.23` +::: import FlatIntro from '/_includes/indexes/flat-intro.mdx'; -As a result, flat indexes are best suited for cases where the number of objects is low and will not grow significantly. - [Binary quantization (BQ)](/developers/weaviate/configuration/compression/bq-compression) can improve flat indexes' search speeds. BQ improves search time by reducing the amount of data to read, and speeding up time taken to calculate the distance between vectors. - - - - -:::info Added in `v1.23` -::: - -The **flat index** is a simple, lightweight index that is fast to build and has a very small memory footprint. This index type is a good choice for use cases where each end user (i.e. tenant) has their own, isolated, dataset, such as in a SaaS product for example, or a database of isolated record sets. - -As the name suggests, the flat index is a single layer of disk-backed data objects and thus a very small memory footprint. The flat index is a good choice for small collections, such as for multi-tenancy use cases. - -A drawback of the flat index is that it does not scale well to large collections as it has a linear time complexity as a function of the number of data objects, unlike the `hnsw` index which has a logarithmic time complexity. +To configure a flat index, see: [Configure flat indexes](/developers/weaviate/configuration/indexing-vector/flat-indexes). ## Related pages diff --git a/developers/weaviate/concepts/indexing/hnsw-indexes.mdx b/developers/weaviate/concepts/indexing/hnsw-indexes.mdx index 4438b718a3..50fccdadae 100644 --- a/developers/weaviate/concepts/indexing/hnsw-indexes.mdx +++ b/developers/weaviate/concepts/indexing/hnsw-indexes.mdx @@ -5,68 +5,61 @@ image: og/docs/indexing.jpg # tags: ['basics']['indexes'] --- - - -#### HNSW indexes - import HNSWIntro from '/_includes/indexes/hnsw-intro.mdx'; -HNSW indexes achieve this by building a multi-layered graph of objects, allowing for fast, approximate nearest neighbor searches. - -While HNSW indexes enable fast searches, they use a lot of [*hot* resources](./index.md#-hot), as they load the graph structure and vectors into memory. - -Consider using [compression](/developers/weaviate/starter-guides/managing-resources/compression) to reduce the size of for your HNSW indexes. Weaviate offers several ways to compress your data: - -import CompressionAlgorithms from '/_includes/starter-guides/compression-types.mdx'; - - - - +import HNSWHow from '/_includes/indexes/hnsw-how.mdx'; + +To configure an HNSW index, see: [Configure HNSW indexes](/developers/weaviate/configuration/indexing-vector/hnsw-indexes). -**Hierarchical Navigable Small World (HNSW)** is an algorithm that works on multi-layered graphs. It is also an index type, and refers to vector indexes that are created using the HNSW algorithm. HNSW indexes enable very fast queries, but rebuilding the index when you add new vectors can be resource intensive. +## Overview -Weaviate's `hnsw` index is a custom implementation of the Hierarchical Navigable Small World ([HNSW](https://arxiv.org/abs/1603.09320)) algorithm that offers full [CRUD-support](https://db-engines.com/en/blog_post/87). +HNSW is an algorithm that works on multi-layered graphs. It is also an index. An HNSW index is the vector index that Weaviate creates when it applies the HNSW algorithm to the vectors in a collection. Weaviate's [implementation](https://github.com/weaviate/weaviate/tree/main/adapters/repos/db/vector/hnsw) offers full [CRUD-support](https://db-engines.com/en/blog_post/87). +When Weaviate builds the index, the HNSW algorithm creates a series of layers. The layers improve search because the database engine doesn't have to scan the entire vector space at query time. Instead, the HNSW algorithm uses the layers to build a list of approximate nearest neighbors (ANN) quickly and efficiently. -DWCTODO: implementation https://github.com/weaviate/weaviate/tree/main/adapters/repos/db/vector/hnsw +### Example - HNSW search -At build time, the HNSW algorithm creates a series of layers. At query time, the HNSW algorithm uses the layers to build a list of approximate nearest neighbors (ANN) quickly and efficiently. - -Consider this diagram of a vector search using HNSW. +Consider this diagram of a vector search that uses HNSW. ![HNSW layers](../img/hnsw-layers.svg "HNSW layers") -An individual object can exist in more than one layer, but every object in the database is represented in the lowest layer (layer zero in the picture). The layer zero data objects are very well connected to each other. Each layer above the lowest layer has fewer data object, and fewer connections. The data objects in the higher layers correspond to the objects in the lower layers, but each higher layer has exponentially fewer objects than the layer below. The HNSW algorithm takes advantage of the layers to efficiently process large amounts of data. +Layer zero is the lowest layer. Layer zero contains every object in the database, and the objects are well connected to each other. + +Some of the objects are also represented in the layers above layer zero. Each layer above layer zero has fewer objects and fewer connections. -When a search query comes in, the HNSW algorithm finds the closest matching data points in the highest layer. Then, HNSW goes one layer deeper, and finds the closest data points in that layer to the ones in the higher layer. These are the nearest neighbors. The algorithm searches the lower layer to create a new list of nearest neighbors. Then, HNSW uses the new list and repeats the process on the next layer down. When it gets to the deepest layer, the HNSW algorithm returns the data objects closest to the search query. +When HNSW searches the graph, it starts at the highest layer. The algorithm finds the closest matching data points in the highest layer. Then, HNSW goes one layer deeper, and finds the closest matching data points in the lower layer that correspond to the objects in the higher layer. These are the nearest neighbors. -Since there are relatively few data objects on the higher layers, HNSW has to search fewer objects. This means HNSW 'jumps' over large amounts of data that it doesn't need to search. When a data store has only one layer, the search algorithm can't skip unrelated objects. It has to search significantly more data objects even though they are unlikely to match. +The HNSW algorithm searches the lower layer and creates list of nearest neighbors. The nearest neighbors list is the starting point for a similar search on the next layer down. The process repeats until the search reaches the lowest (deepest) layer. Finally, the HNSW algorithm returns the data objects that are closest to the search query. -HNSW is very fast, memory efficient, approach to similarity search. The memory cache only stores the highest layer instead of storing all of the data objects in the lowest layer. When the search moves from a higher layer to a lower one, HNSW only adds the data objects that are closest to the search query. This means HNSW uses a relatively small amount of memory compared to other search algorithms. +Since there are relatively few data objects on the higher layers, HNSW 'jumps' over large amounts of data that it doesn't need to search. In contrast, when a data store has only one layer, the search algorithm can't skip unrelated objects. Flat hierarchies mean the search engine has to scan significantly more data objects even though those object are unlikely to match the search criteria. -Have another look at the diagram; it demonstrates how the HNSW algorithm searches. The blue search vector in the top layer connects to a partial result in layer one. The objects in layer one lead HNSW to the result set in layer zero. HNSW makes three hops through the layers (the dotted blue lines) and skips objects that are unrelated to the search query. +Weaviate's HNSW implementation is a very fast, memory efficient, approach to similarity search. The memory cache only stores the highest layer of the index instead of storing all of the data objects from the lowest layer. As a search moves from a higher layer to a lower one, HNSW only adds the data objects that are closest to the search query. This means HNSW uses a relatively small amount of memory compared to other search algorithms. -If your use case values fast data upload higher than super fast query time and high scalability, then other vector index types may be a better solution (e.g. [Spotify's Annoy](https://github.com/spotify/annoy)). +The diagram demonstrates how the HNSW algorithm searches. The blue search vector in the top layer connects to a partial result in layer one. The objects in layer one lead HNSW to the result set in layer zero. In this example, HNSW makes three hops through the layers (the dotted blue lines) and skips the objects that are unrelated to the search query. -### Managing search quality vs speed tradeoffs +## Search quality vs search speed HNSW parameters can be adjusted to adjust search quality against speed. The `ef` parameter is a critical setting for balancing the trade-off between search speed and quality. -The `ef` parameter dictates the size of the dynamic list used by the HNSW algorithm during the search process. A higher `ef` value results in a more extensive search, enhancing accuracy but potentially slowing down the query. +The `ef` parameter dictates the size of the dynamic list (the nearest neighbors list in the example above) that the HNSW algorithm uses during the search process. A higher `ef` value results in a more extensive search. A higher `ef` value improves accuracy but it can also slow down the query. + +A lower value for `ef` makes the search faster, but it might compromise on accuracy. + +The balance is crucial in scenarios where either speed or accuracy is a priority. For instance, in applications where rapid responses are critical, a lower `ef` might be preferable, even at the expense of some accuracy. In analytical or research contexts where precision is paramount, a higher `ef` might be better, despite the increased query time. -In contrast, a lower `ef` makes the search faster but might compromise on accuracy. This balance is crucial in scenarios where either speed or accuracy is a priority. For instance, in applications where rapid responses are critical, a lower `ef` might be preferable, even at the expense of some accuracy. Conversely, in analytical or research contexts where precision is paramount, a higher `ef` would be more suitable, despite the increased query time. +### Configure ef -`ef` can be configured explicitly or dynamically. This feature is particularly beneficial in environments with varying query patterns. When `ef` is configured dynamically, Weaviate optimizes the balance between speed and recall based on real-time query requirements. +You can set `ef` explicitly or allow Weaviate to set it dynamically. When `ef` is configured dynamically, Weaviate optimizes the balance between speed and recall based on real-time query requirements. If your query patterns are variable, consider setting dynamic `ef`. -To enable dynamic `ef`, set `ef`: -1. Weaviate adjusts the size of the ANN list based on the query response limit. The calculation also takes into account the values of `dynamicEfMin`, `dynamicEfMax`, and `dynamicEfFactor`. +To enable dynamic `ef`, set `ef: -1` in your [collection configuration](/developers/weaviate/configuration/indexing-vector/hnsw-indexes#configure-an-hnsw-index). When dynamic `ef` is enabled, Weaviate adjusts the size of the ANN list based on the query response limit. The calculation also takes into account the values of `dynamicEfMin`, `dynamicEfMax`, and `dynamicEfFactor`. -### Dynamic ef +### Configure dynamic ef The `ef` parameter controls the size of the ANN list at query time. You can configure a specific list size or else let Weaviate configure the list dynamically. If you choose dynamic `ef`, Weaviate provides several options to control the size of the ANN list. @@ -78,10 +71,14 @@ The length of the list is determined by the query response limit that you set in To keep search recall high, the actual dynamic `ef` value stays above `dynamicEfMin` even if the query limit is small enough to suggest a lower value. -To keep search speed reasonable even when retrieving large result sets, the dynamic `ef` value is limited to `dynamicEfMax`. Weaviate doesn't exceed `dynamicEfMax` even if the query limit is large enough to suggest a higher value. If the query limit is higher than `dynamicEfMax`, `dynamicEfMax` does not have any effect. In this case, dynamic `ef` value is equal to the query limit. +To keep search speed reasonable even when retrieving large result sets, the dynamic `ef` value is limited to `dynamicEfMax`. Weaviate doesn't exceed `dynamicEfMax` even if the query limit is large enough to suggest a higher value. + +If the query limit is higher than `dynamicEfMax`, `dynamicEfMax` does not have any effect. In this case, the dynamic `ef` value is equal to the query limit. To determine the length of the ANN list, Weaviate multiples the query limit by `dynamicEfFactor`. The list range is modified by `dynamicEfMin` and `dynamicEfMax`. +### Dynamic ef example + Consider this GraphQL query that sets a limit of 4. ```graphql @@ -113,13 +110,37 @@ The resulting search list has these characteristics. - A maximum length of 25 objects ("dynamicEfMax": 25). - An actual size of 5 to 25 objects. -If you use the [`docker-compose.yml` file from Weaviate](/developers/weaviate/installation/docker-compose) to run your local instance, the `QUERY_DEFAULTS_LIMIT` environment variable sets a reasonable default query limit. To prevent out of memory errors,`QUERY_DEFAULTS_LIMIT` is significantly lower than `QUERY_MAXIMUM_RESULTS`. +#### Related environment variables + +If you use the [`docker-compose.yml` file from Weaviate](/developers/weaviate/installation/docker-compose) to run your local instance, the `QUERY_DEFAULTS_LIMIT` environment variable sets a reasonable default query limit. + +To prevent out of memory errors, the default setting for `QUERY_DEFAULTS_LIMIT` is significantly lower than `QUERY_MAXIMUM_RESULTS`. To change the default limit, edit the value for `QUERY_DEFAULTS_LIMIT` when you configure your Weaviate instance. -### Deletions +## Deletions + +Deleted objects are marked as deleted, but Weaviate doesn't remove them immediately. Cleanup is an async process runs that rebuilds the HNSW graph after deletes and updates. During cleanup, Weaviate updates the graph and actually removes the deleted objects from the collection. + +Cleanup can affect system performance if your instance has a large number of deletes. These environment variables tune the cleanup process: + +- `TOMBSTONE_DELETION_CONCURRENCY` +- `TOMBSTONE_DELETION_MAX_PER_CYCLE` +- `TOMBSTONE_DELETION_MIN_PER_CYCLE` + +For more details, see [environment variables](/developers/weaviate/config-refs/env-vars). + +## Compression + +Consider using [compression](/developers/weaviate/starter-guides/managing-resources/compression) to reduce the size of for your HNSW indexes. Weaviate offers several ways to compress your data: + +import CompressionAlgorithms from '/_includes/starter-guides/compression-types.mdx'; + + + +## Other considerations -Cleanup is an async process runs that rebuilds the HNSW graph after deletes and updates. Prior to cleanup, objects are marked as deleted, but they are still connected to the HNSW graph. During cleanup, the edges are reassigned and the objects are deleted for good. +HNSW indexes enable very fast queries, but they are not as fast at import time. Rebuilding the index when you add new vectors can be resource intensive. If you use HNSW, consider [enabling asynchronous indexing](/developers/weaviate/configuration/indexing-vector/dynamic-indexes#asynchronous-indexing) to improve system response during imports. ## Related pages diff --git a/developers/weaviate/concepts/indexing/inverted-indexes.md b/developers/weaviate/concepts/indexing/inverted-indexes.md index 32ecf31db4..2657c52dea 100644 --- a/developers/weaviate/concepts/indexing/inverted-indexes.md +++ b/developers/weaviate/concepts/indexing/inverted-indexes.md @@ -1,5 +1,5 @@ --- -title: Inverted indexes overview +title: Inverted indexes sidebar_position: 100 image: og/docs/indexing.jpg # tags: ['basics'] @@ -93,12 +93,27 @@ If you so not intend to filter on these properties, do not enable these indexes. ## BM25 ranking algorithm -Some stuff +The [BM25 ranking algorithm](https://en.wikipedia.org/wiki/Okapi_BM25) ranks documents based on how frequently particular words appear in the document. Weaviate uses BM25 with [keyword searches](/developers/weaviate/search/bm25) and [hybrid searches](/developers/weaviate/search/hybrid). + +The `indexSearchable` index enables BM25 on a property. You cannot do a hybrid search or a keyword search on a property if `indexSearchable` is disabled. + +The BM25 algorithm has two parameters that change how the `indexSearchable` index is created: + +- To adjust for document length, modify `b`. Values range from 0 to 1. +- To adjust for word frequency within a document, modify `k1`. Values are usually in the range from 0 to 3. There isn't an upper limit. + +For configuration examples, see [BM25](/developers/weaviate/configuration/inverted-indexes#bm25). + ## Further resources - [Concepts: Vector Indexing](/developers/weaviate/concepts/indexing/vector-indexes) - [Configuration: Inverted indexes](/developers/weaviate/configuration/inverted-indexes) +- [Concepts: Inverted Indexes](/developers/weaviate/concepts/indexing/inverted-indexes) +- [Configuration: BM25 algorithm](/developers/weaviate/configuration/inverted-indexes#bm25) +- [Blog: Ranking models](https://weaviate.io/blog/ranking-models-for-better-search) +- [Blog: Hybrid search](https://weaviate.io/blog/hybrid-search-for-web-developers) +- [Blog: Fusion algorithms](https://weaviate.io/blog/hybrid-search-fusion-algorithms) ## Questions and feedback diff --git a/developers/weaviate/concepts/indexing/vector-indexes.md b/developers/weaviate/concepts/indexing/vector-indexes.md index 6c68de345f..afe3f74ac6 100644 --- a/developers/weaviate/concepts/indexing/vector-indexes.md +++ b/developers/weaviate/concepts/indexing/vector-indexes.md @@ -51,9 +51,9 @@ import HNSWIntro from '/_includes/indexes/hnsw-intro.mdx'; -HNSW indexes build a multi-layered object graph. The graph structure and HNSW algorithm result in fast, approximate nearest neighbor [(ANN)](https://en.wikipedia.org/wiki/Nearest_neighbor_search) searches. +import HNSWHow from '/_includes/indexes/hnsw-how.mdx'; -The index and graph structure are stored in RAM memory. This makes HNSW indexes fast, but RAM is an expensive resource. Consider using [compression](/developers/weaviate/starter-guides/managing-resources/compression) to reduce the size of for your HNSW indexes. + Weaviate offers these methods to compress ("quantize") your HNSW index: @@ -69,8 +69,6 @@ import FlatIntro from '/_includes/indexes/flat-intro.mdx'; -Flat indexes are best suited for collections that have relatively small object counts. If you expect the object count to grow significantly, consider using a [dynamic index](#dynamic-indexes). - [Binary quantization (BQ)](/developers/weaviate/configuration/compression/bq-compression) is a compression technique that also improves search speed for flat indexes. BQ reduces the amount of data the search engine reads. It also permits efficient binary calculations. These benefits of compression shorten the time needed to calculate vector distances during search. For more details, see [flat indexes](/developers/weaviate/concepts/indexing/flat-indexes). From 3dcd799f94263b34b92b676cfcfaaf7b3b301f04 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Wed, 25 Sep 2024 09:53:06 -0400 Subject: [PATCH 66/67] review feedback --- _includes/indexes/flat-intro.mdx | 4 +--- _includes/indexes/hnsw-how.mdx | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/_includes/indexes/flat-intro.mdx b/_includes/indexes/flat-intro.mdx index 3560af2e0e..b023e5a8d0 100644 --- a/_includes/indexes/flat-intro.mdx +++ b/_includes/indexes/flat-intro.mdx @@ -1,5 +1,3 @@ [Flat indexes](/developers/weaviate/concepts/indexing/flat-indexes) are disk based. Compared to HNSW indexes, flat indexes have very small RAM requirements. -Flat indexes do brute-force vector searches. The search latency increases linearly with the number of objects. For that reason, flat indexes work best with small collections, less than 10,000 objects. - -Flat indexes are best suited for collections that have relatively small object counts. If you expect the object count to grow significantly, consider using a [dynamic index](#dynamic-indexes). \ No newline at end of file +Flat indexes do brute-force vector searches. The search latency increases linearly with the number of objects. For that reason, flat indexes work best with small collections, less than 10,000 objects. If you expect the object count to grow significantly, consider using a [dynamic index](#dynamic-indexes). \ No newline at end of file diff --git a/_includes/indexes/hnsw-how.mdx b/_includes/indexes/hnsw-how.mdx index 809f66234f..46b45bb7b9 100644 --- a/_includes/indexes/hnsw-how.mdx +++ b/_includes/indexes/hnsw-how.mdx @@ -1,3 +1,3 @@ HNSW indexes build a multi-layered object graph. The graph structure and HNSW algorithm result in efficient, approximate nearest neighbor [(ANN)](https://en.wikipedia.org/wiki/Nearest_neighbor_search) searches. -The index and graph structure are stored in RAM memory. This makes HNSW indexes fast, but RAM is an expensive resource. Consider using [compression](/developers/weaviate/starter-guides/managing-resources/compression) to reduce the size of for your HNSW indexes. \ No newline at end of file +The index and graph structure are stored in RAM memory. The vectors are also stored in RAM. This makes HNSW indexes fast, but RAM is an expensive resource. Consider using [compression](/developers/weaviate/starter-guides/managing-resources/compression) to reduce the size of for your HNSW indexes. \ No newline at end of file From c565e72e1210a669793eb519891ed44eaaaa2890 Mon Sep 17 00:00:00 2001 From: daveatweaviate Date: Wed, 25 Sep 2024 13:35:15 -0400 Subject: [PATCH 67/67] review feedback --- _includes/code/howto/indexes/indexes-dynamic-v2.ts | 2 +- _includes/indexes/hnsw-intro.mdx | 2 +- _includes/indexes/multiple-named-vectors.mdx | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/_includes/code/howto/indexes/indexes-dynamic-v2.ts b/_includes/code/howto/indexes/indexes-dynamic-v2.ts index 94ec70debe..d0420283f3 100644 --- a/_includes/code/howto/indexes/indexes-dynamic-v2.ts +++ b/_includes/code/howto/indexes/indexes-dynamic-v2.ts @@ -9,7 +9,7 @@ function getClient(){ const client: WeaviateClient = weaviate.client({ scheme: 'http', host: 'localhost:8080', -}) +}); return client; } diff --git a/_includes/indexes/hnsw-intro.mdx b/_includes/indexes/hnsw-intro.mdx index 5e200d147d..95e20c7679 100644 --- a/_includes/indexes/hnsw-intro.mdx +++ b/_includes/indexes/hnsw-intro.mdx @@ -1 +1 @@ -[Hierarchical Navigable Small World (HNSW) indexes](/developers/weaviate/concepts/indexing/hnsw-indexes) are high-performance, in-memory indexes. HNSW indexes scale well; vector searches are fast, even for very large data sets. \ No newline at end of file +[Hierarchical Navigable Small World (HNSW) indexes](/developers/weaviate/concepts/indexing/hnsw-indexes) are high-performance, in-memory indexes. HNSW indexes scale well. Search times grow logarithmically, so vector searches are fast, even for very large data sets. \ No newline at end of file diff --git a/_includes/indexes/multiple-named-vectors.mdx b/_includes/indexes/multiple-named-vectors.mdx index ca715eb75b..77512b5dad 100644 --- a/_includes/indexes/multiple-named-vectors.mdx +++ b/_includes/indexes/multiple-named-vectors.mdx @@ -6,7 +6,7 @@ import PyCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v3.py'; import TSCodeV3 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v3.ts'; import TSCodeV2 from '!!raw-loader!/_includes/code/howto/indexes/indexes-v2.ts'; -Items in a collection can have multiple named vectors. Each named vectors has it's own vector index. These vector indexes can be configured independently. +Items in a collection can have multiple named vectors. Each named vectors has its own vector index. These vector indexes can be configured independently. Configure a collection to use named vectors: