From 96d0b285c9a3b0fdfd654f0d164310fa79fe4a7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Schaeffer?= Date: Thu, 13 Feb 2025 13:44:54 +0900 Subject: [PATCH] anthropic: Add support for multi content part and images content in human messages. (#1141) * anthropic: Add support for multi content part in human messages. * examples: Add Anthropic Vision example * fix: move base64 encoding in the lib --- examples/anthropic-vision-example/README.md | 34 ++++++++++ .../anthropic_vision_example.go | 59 ++++++++++++++++++ examples/anthropic-vision-example/go.mod | 13 ++++ examples/anthropic-vision-example/go.sum | 22 +++++++ examples/anthropic-vision-example/image.png | Bin 0 -> 8192 bytes llms/anthropic/anthropicllm.go | 37 +++++++++-- .../internal/anthropicclient/messages.go | 15 +++++ 7 files changed, 174 insertions(+), 6 deletions(-) create mode 100644 examples/anthropic-vision-example/README.md create mode 100644 examples/anthropic-vision-example/anthropic_vision_example.go create mode 100644 examples/anthropic-vision-example/go.mod create mode 100644 examples/anthropic-vision-example/go.sum create mode 100644 examples/anthropic-vision-example/image.png diff --git a/examples/anthropic-vision-example/README.md b/examples/anthropic-vision-example/README.md new file mode 100644 index 000000000..56a192917 --- /dev/null +++ b/examples/anthropic-vision-example/README.md @@ -0,0 +1,34 @@ +# Anthropic Vision Example + +Hello there! 👋 This example demonstrates how to use the Anthropic Claude 3 Claude 3 Sonnet model for image analysis using Go and the LangChain Go library. Let's break down what this exciting code does! + +## What This Example Does + +1. **Sets Up Anthropic**: The code initializes an Anthropic client to interact with the Claude 3 Sonnet model. + +2. **Loads an Image**: An image file (`image.png`) is embedded into the binary using Go's `embed` package. This image will be analyzed by the AI model. + +3. **Sends a Request**: The code constructs a request to the Claude 3 model, including: + - The image data in a base64 encoded string (in PNG format) + - A text prompt asking to identify the string on a box in the image + +4. **Processes the Response**: After sending the request, the code handles the response from the AI model, extracting the generated content and some metadata about token usage. + +5. **Outputs Results**: Finally, it prints out the AI's interpretation of what string is on the box in the image. + +## Key Features + +- **Multimodal AI**: This example showcases how to work with both image and text inputs in a single AI request. +- **Error Handling**: Includes basic error checking to ensure the process runs smoothly. +- **Token Usage Tracking**: Logs the number of input and output tokens used, which can be helpful for monitoring usage and costs. + +## Running the Example + +To run this example, you'll need: + +1. An Anthropic API KEY set up in your environment variables +2. The required Go dependencies installed + +Once everything is set up, simply run the Go file, and it should output the AI's interpretation of the text on the box in the image! + +Happy coding, and enjoy exploring the fascinating world of multimodal AI with Claude 3! 🚀🖼️🤖 diff --git a/examples/anthropic-vision-example/anthropic_vision_example.go b/examples/anthropic-vision-example/anthropic_vision_example.go new file mode 100644 index 000000000..30634f338 --- /dev/null +++ b/examples/anthropic-vision-example/anthropic_vision_example.go @@ -0,0 +1,59 @@ +package main + +import ( + "context" + _ "embed" + "fmt" + "log" + + "github.com/tmc/langchaingo/llms" + "github.com/tmc/langchaingo/llms/anthropic" +) + +//go:embed image.png +var image []byte + +func main() { + llm, err := anthropic.New( + anthropic.WithModel("claude-3-5-sonnet-20240620"), + ) + if err != nil { + log.Fatal(err) + } + ctx := context.Background() + resp, err := llm.GenerateContent( + ctx, + []llms.MessageContent{ + { + Role: llms.ChatMessageTypeHuman, + Parts: []llms.ContentPart{ + // For images, you can use image formats such as image/png, image/jpeg, image/gif, image/webp. + // Please change according to the actual byte array to be given. + // for more detailes, see this https://docs.anthropic.com/claude/reference/messages_post + llms.BinaryPart("image/png", image), + llms.TextPart("Please tell me the string on the box."), + }, + }, + }, + llms.WithMaxTokens(1000), + llms.WithTemperature(0.1), + llms.WithTopP(1.0), + llms.WithTopK(100), + ) + if err != nil { + log.Fatal(err) + } + choices := resp.Choices + if len(choices) < 1 { + log.Fatal("empty response from model") + } + + log.Printf( + "input_tokens: %d, output_tokens: %d", + choices[0].GenerationInfo["InputTokens"], + choices[0].GenerationInfo["OutputTokens"], + ) + fmt.Println(choices[0].Content) + // Output: + // The string on the box in the image is "LGTM". +} diff --git a/examples/anthropic-vision-example/go.mod b/examples/anthropic-vision-example/go.mod new file mode 100644 index 000000000..43d84a051 --- /dev/null +++ b/examples/anthropic-vision-example/go.mod @@ -0,0 +1,13 @@ +module github.com/tmc/langchaingo/examples/bedrock-claude3-vision-example + +go 1.22.0 + +toolchain go1.22.1 + +require github.com/tmc/langchaingo v0.1.13-pre.1 + +require ( + github.com/dlclark/regexp2 v1.10.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/pkoukk/tiktoken-go v0.1.6 // indirect +) diff --git a/examples/anthropic-vision-example/go.sum b/examples/anthropic-vision-example/go.sum new file mode 100644 index 000000000..e92326ed5 --- /dev/null +++ b/examples/anthropic-vision-example/go.sum @@ -0,0 +1,22 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0= +github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/pkoukk/tiktoken-go v0.1.6 h1:JF0TlJzhTbrI30wCvFuiw6FzP2+/bR+FIxUdgEAcUsw= +github.com/pkoukk/tiktoken-go v0.1.6/go.mod h1:9NiV+i9mJKGj1rYOT+njbv+ZwA/zJxYdewGl6qVatpg= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tmc/langchaingo v0.1.13-pre.1 h1:r+ma9kl0NuFJGtIrnMPFjEn4RhXktwSI31fIpgiiMm4= +github.com/tmc/langchaingo v0.1.13-pre.1/go.mod h1:vpQ5NOIhpzxDfTZK9B6tf2GM/MoaHewPWM5KXXGh7hg= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/examples/anthropic-vision-example/image.png b/examples/anthropic-vision-example/image.png new file mode 100644 index 0000000000000000000000000000000000000000..27da75c582e5e876ab29b564bd78280bced06ab5 GIT binary patch literal 8192 zcmb_>WmFv7wr)d^#;tJ)?$$V@acc+=oW?D zf`q^$d!KjC9`C;U>)xtSRdY@GzPaXFwZ^DgF`636c-U0f0000_1*)L^@ci;{`2sN> z?jO!RNj^N#>}1tt0f4$BoLfuuhc>-6R9hVY@Mi@8Lc#%nn+H|MHUQww4*=|1000tM z0022W_p=t{!JugaQ?XT72XH>fKmaNVG2qdIgz|6#q8tEdf5`yA1Jc9sC?DlN)_l}| zTfgK#`nT-+#}OJgI}8Az3OMM(ykP2T64q`myq3@1tZaDwT;P8k01!Wk2hqjG%aY#D z#o5(U!cUU%FAs?a`Hz~9k^V0iFDFSxn7Sstyqkv&y$CNqFF&IcHa$H(#N)ZGgtmgx zzrr6{l8p9VUT_IMK3`v7USC08HxD~Lu(-H5AHM*ffB?^f2ao3qS1(IH9#>DMzd`;l zj)IM+wTA=T%fZc+{tvFDm7BMhBqQS=qJJNM=V{~T@E;~u&wq96p@V#XEPP;Ie!l-- zn3sd?e-rk{@^{!@{rZ~@@&`;p)4|Wi*-*j3#m3e1AvGy5SRC>f&401{C(*wn<=vd! zJapie);3b$e}()*_5XVR#RK~fj{v{mKY9LP`A6iRyd|D_IM_Ut^iSdb5d9b3zx*M5 zf2#VAD*ru%e`z1aPzoEu_wV^2g{>Qdj0ON`PE-_Rb^TBdOdZ@zW?xA8{-op;G_h-r zBj(WgXg`g0BmoLB0*c-0VJl)kz9SUN)@q|oB36sXzI}rCTdCVj)@2^QvvSU z(i(C*37*^%Xu|F}s?AD9j6HCqT;tu^u8v>Yo`A5*iKKoLzsZ~3+J|V?U+rN905MrMpIrO{-&DWjG>g@x{I@R=c!}j7gXaCCID6^xB zt$78SmAm_g+OAEdgjYLGARr21l%d|ivxG!kYO@+X^wn-EewSt5ee`5T)dU#5SvkFb z!k3Nf$b|#fz}4(6Q{6=%L z--ZXfu@mce7?%^`#DtrmE3EEc=k|b_dA3VNDCH$!>f1 z4(G9IpreC>79O6NC|~Xqm-wUvs?d3=Ui6dCjn!K3F{6s;D~7*bXQ0--R-YFJUxse50=)Ya9# zIG;VDKDu9x{?Gxohuodx{2Y$uobIT6%0dzowpXmrMwatw+*ETt_25TrMPp8gw63v% zIgz8I`;=&S{BT7@*sF|A8|@!W-7 zn<1Pfg~PR^3N@WCbgI<2nbUmEF*2V@I@ODt?{05b-Q0-Yj}V;Iw%eK4n1-Ao9cmVQ zWB>|Qk#4ws1IoR-b~X11iqBFOQ&Bn2e77s2&c|3~KNB^G8V>M*)O9^$-zpGt$T-9V z09lDw(!<-jh&0}~Txh-A_eKYpsGGKXw3hY%8LM$5G-~biG(}v5N~E_o5D)(+wP<8i zLjNXNk+ogsN@ba5$1rE~e4pSCSTYRCY`V=LM$@@=aqW*UeeJxxT=dMppok*CKfuMc z*0Qv|cG$F>56YGF(%rqArbCMGsMTNP_kO2Yy%z4?{%+lo^m!I?V&3&rqc3*yH|M@- zl@CyEE_~%D={K(Ob3c0~Hs%-W2h2=Oi%{>34wUOohnEoFw1?h{i;}A(Bt;}R8Sup= zPE7tHjD8-aWjvc7NE)Ba6I@SR_k9yyg@0v?FM0FaNuGau{(f9Gz{xmDwgwW$5|@-i z*DcBP`s+$_bFIbr%Jzw$-a?|Ol8e0Nb=wS5<%0rulQ_*H-7_G*@=M>b-_;q7K1|=H zaFGruqkuZa-_`PQ$tw8@(v&vX(w%eG@tUfIp$?2{(Ol zJ?>NM_fFz8PEOQ_6{!1yCD+f1HtI2}q-boQC(*p6)N@`{AS)vFIp+?Pc~yy7py>GS zP;2j*Dp@E@s3H;*a|-b-ORo91%O9!)F&Z4&{uHjPK2|bOwXP^x=9dvD2FgWsIYTk- zznrA62wJ}!=u0I%i!*QeF+_Y#+gDrMhhFtWUbw-VU5q9nzBl>u`>?h`T}&Uo92TSd z=ULbKImr&}8`u=P9Ie;FTi|kVu8z7Rcw)lvd-Hd zOXaGt4zGm{U_7p=0r^;7Xomn^qIU5vwLFWU;lT=5wsA40ku#c59q1SS_Fg6~Y9D!T z6+S6QdmRSdq17dpozT*rfE{SaV2UsFIZL()e3ofCNC;ug@IK`GAc}+I$w?N+f#4kq z0Qu?++Gp&KR8WBiWim=gOi4y0EHbWUGg!zJ3M=O3C`ktxZD@!N&OW70Pt9zxWKBHZ zcy??3nNuF|LVev}`LhaXsKDp8Hnsb%14P&=7Blk=24`FPsp?TZ$(j)En=o;cW)0hf z<6LLC_UbiYtts(6P5wnguhz4*-4rbUKi`)ts&dm(lr?Sz1#Zo zSJ1|Vo-0@=@x$9mQI>t zTwm#w1Un@ZuJl=KQoiQzui(7TL>Gc5*rXk;gavIKh)%mL1sRjg>uVrl{3=S#{Z!E9 zzd9<ri2BtkH~epQx}Lt&s5FB^@ux@KU0XMF(*9;^Ep*^N!Q0eP?UF1Ro; z=qi-(&f;D#>!l0l)I$4Wj5%A9T)lh(oUB#M~VLS-XA z*B3ebxz8L~+^1M~fA+avaB??pl5Lt;tP5347DAMnM=~E{^7e_)o5_QoRnH0qq3Vi9 z!o=q)OR^QLF~_H|=U}en31(2DH165XoDN%f@1V-Z+kvyQOXqB-I=XqI!nf1~2d#Al z@Rdo>*cw^)HSas=O8bv^Nw`Bal<%RVlfTA$Ib$VEM#NZ@2le)b;e#hgBJY)b4#g#` zn(nW!6ozG>pzbdMl@oV^9}(I>m^ulIZ|&qqA6`ezavVD;1MYdB&2_A8cCC#t*HttW zFRv0V6YaJ_c4!?FSwbi^%M@-fUrm{LI>%MN;@m`syyV^aSq1hg&{JR^22C*o5EVCe z5M_s4erU;uwxzCF_nb5zLz&fiTzd4brKpk6?EFY2LKhi^K88KFXs!XLezhwJR#}-| zP7Ey`3TL_C0Q|X9`Ysp^4Kam+E$cLUtZIIIh3d-LfvXXZo0Up&oZ48ZQ{MIYUM+4PkGREv%NYFQlYW43QZi*aQWF^ zb(-ytZb$MJzdC-|GqP*NKv|Ec4kHt zh_v~#SCBYC;iw%tFR^jrKTA&r7lZToBcs4(9zQo5=^m|YD z%oiYHZB;Q5ac@_#`eIJ%h#1mF3eQ8Ry^iKE(VJC;nU<5VD>2ibw`oSV2(~UtQ9Q1F z?dvVCYaEg#7ST7Gze)d*C+xMRkM3euOUol2Hn{BmN=wTxfrI>Z&rHHzM}!=$H4rCx zSfB+M^Qw?EweZTYu?KHTaj+rlUY^&J?UDh^Vyb)Uz`s&xj$xOpkfNtIidt&(RumPcz1PfQZxKlMA4^$@1tL8f{m+OpGjT{uW*}W znQSxO$$TVtiD=~*eWxmg6U17854j277C(}Gaep;?R8bHrbuoJ1QIJ#HY_2FwRfmsc zd)0WqT(oF#t-dyGF=MA3Gxp_26=oJx|r zpbFp7npK!B+8^Fd9XkdO52SS0i3Jm^CYi~!9Ztva(^8lSKkl&CNu1u+BR;Ux)Yy<1 zhg6qye@bt&?&*Y|j8?dZl(=$!qZxYbuln=Rn+#F*E3u4WN#B<_)*PsF`ywl1afHZ} zW->)m%{Ns{O5dx;zgFxGqic;u=H+a4hsu@dj%2^%MWBYwK~uTbZghK0HRQ$J4|Rzn znk)g(H!m&Knr-4M-%@uPn&grn_Lwk@rVoT_F)))+4_9Dj+>zU;aC`CMTWpmAeeU+R zc6T>C3Ed>7R;2y#V=rAVM$Y)Kkoh1T4d1NXsDWhQPyypI#=Z_o5e&*9jG4?Go0t3J zZuOuIl-h-|Oj1!vP=S$^>DE;hczDi1Sg8KB7h;YQWN{dWO$PK0YLroL zcXW10U&G_pkS4R>jrFX~{M@uu+$XqeRz)C5)`8M)gVpgK9n3IbG5vAU19P>rGBki$ z_KKNILQhLyFP_eeIJGFn+D2{Im$p@1iv6io!Nu;xwP26o)myr&%|;*Z?K-yFXsY$2 zuRoUB+X*TxJnN&q%inVdNLEaOvMU3{sfp#7Vng;9UIH(|dx4qKJUE??N6k5Jg;I6W z+Uk*-vVCv$B|P5m1sNc|)gE;*PzYdzY#F-G|9J7;=cbiACZd4y+_wHqQgST*0$aGa zee5=%KH;+l%Po!WM~9%#lZ*{>XRg!kpcV?^UI;OAfL;O#k;JQ{5YSVk`Rc=C)IAun zvz|MEAxMCnKyjT1+~fY15t-tiKb$5W&VVt(2kUg0i;CInLlEo!1Z1?tRFd1eULVT# zJ+x9i!*h%o1ZF}l0t>gJ`pi?bO&0?`B4d{k{(?^k}=DP*OkS0!0M2U|Qn6UQ&G6lx2J+3>Vv6qH032g-?SJ{4D0mF1h z>~=wzfA zv+a;du$)ZKJ#lC1tMfmh>~2& za?3)Tt?v$N+h3MxGU_Xv&3)pjecpz#?el>jE!=2caUJyiqeaH-3((5?!vuDSlEFB@ z^|UXH*!FSq0TD)JduZI zi8vRp70F$BKr;aqKJSg|8~PyW(ikqv)Fq zE8PnOE#saQe~#5mYg)REfkn+oL@udl7L8RB6RuEMid!XcSAN?3iJvZ{HZ&yezAqp1 ze7iN!tOeo<&l8!(4hlyTLeE1Rd#5~%ASIH1_j1Rk!ReAtLEjf