diff --git a/app/jobs/get_next_ai_message_job.rb b/app/jobs/get_next_ai_message_job.rb index 74a68b1d9..0ad348334 100644 --- a/app/jobs/get_next_ai_message_job.rb +++ b/app/jobs/get_next_ai_message_job.rb @@ -1,3 +1,4 @@ +require "open-uri" include ActionView::RecordIdentifier require "nokogiri/xml/node" @@ -190,6 +191,7 @@ def call_tools_before_wrapping_up end index = @message.index + url_of_dalle_generated_image = nil msgs.each do |tool_message| # one message for each tool executed @conversation.messages.create!( assistant: @assistant, @@ -200,6 +202,13 @@ def call_tools_before_wrapping_up index: index += 1, processed_at: Time.current, ) + + parsed = JSON.parse(tool_message[:content]) rescue nil + + if parsed.is_a?(Hash) && parsed.has_key?("url_of_dalle_generated_image") + url_of_dalle_generated_image = parsed["url_of_dalle_generated_image"] + end + end assistant_reply = @conversation.messages.create!( @@ -210,6 +219,12 @@ def call_tools_before_wrapping_up index: index += 1 ) + unless url_of_dalle_generated_image.nil? + d = Document.new + d.file.attach(io: URI.open(url_of_dalle_generated_image), filename: "image.png") + assistant_reply.documents << d + end + GetNextAIMessageJob.perform_later( @user.id, assistant_reply.id, diff --git a/app/services/toolbox.rb b/app/services/toolbox.rb index b86ed99f5..54dab1358 100644 --- a/app/services/toolbox.rb +++ b/app/services/toolbox.rb @@ -6,6 +6,7 @@ def self.descendants [ test_env && Toolbox::HelloWorld, Toolbox::OpenMeteo, + Toolbox::Dalle, Toolbox::Memory, gmail_active && Toolbox::Gmail, tasks_active && Toolbox::GoogleTasks, diff --git a/app/services/toolbox/dalle.rb b/app/services/toolbox/dalle.rb new file mode 100644 index 000000000..8cf85c3d9 --- /dev/null +++ b/app/services/toolbox/dalle.rb @@ -0,0 +1,33 @@ +class Toolbox::Dalle < Toolbox + + describe :generate_an_image, <<~S + Generate an image based on what the user asks you to generate. You will pass the user's prompt and will get back a URL to an image. + S + + def generate_an_image(image_generation_prompt_s:) + response = client.images.generate( + parameters: { + prompt: image_generation_prompt_s, + model: "dall-e-3", + size: "1024x1792", + quality: "standard" + } + ) + + dalle_url = response.dig("data", 0, "url") + + { + prompt_given: image_generation_prompt_s, + url_of_dalle_generated_image: dalle_url, + note_to_assistant: "The image at the URL is already being shown on screen so reply with a nice message confirming the image has been generated, maybe re-describing it, but don't include the link to it." + } + end + + private + + def client + OpenAI::Client.new( + access_token: Current.message.assistant.api_service.effective_token + ) + end +end diff --git a/test/controllers/active_storage/postgresql_controller_test.rb b/test/controllers/active_storage/postgresql_controller_test.rb index 365641294..275a7150a 100644 --- a/test/controllers/active_storage/postgresql_controller_test.rb +++ b/test/controllers/active_storage/postgresql_controller_test.rb @@ -52,6 +52,7 @@ class ActiveStorage::PostgresqlControllerTest < ActionDispatch::IntegrationTest blob.delete get blob.send(url_method) + assert_response :not_found end test "showing blob with invalid key" do diff --git a/test/fixtures/conversations.yml b/test/fixtures/conversations.yml index 5631c0eca..d0e301697 100644 --- a/test/fixtures/conversations.yml +++ b/test/fixtures/conversations.yml @@ -80,6 +80,12 @@ weather: title: Weather last_assistant_message: weather_explained +image_generation: + user: christoph + assistant: samantha + title: Generating an image + last_assistant_message: image_generation_explained + trees: user: keith assistant: keith_gpt3 diff --git a/test/fixtures/messages.yml b/test/fixtures/messages.yml index 274504320..7f6fe0b0c 100644 --- a/test/fixtures/messages.yml +++ b/test/fixtures/messages.yml @@ -668,6 +668,45 @@ weather_explained: version: 1 +image_generation_tool_call: + assistant: samantha + conversation: image_generation + role: assistant + tool_call_id: + content_text: + content_tool_calls: '[{"index": 0, "id": "def456", "type": "function", "function": {"name": "generate_an_image", "arguments": {"image_generation_prompt_s": "World"}}}]' + content_document: + created_at: 2024-08-25 1:01:00 + processed_at: 2024-08-25 1:01:00 + index: 1 + version: 1 + +image_generation_tool_result: + assistant: samantha + conversation: image_generation + role: tool + tool_call_id: def456 + content_text: weather is + content_tool_calls: + content_document: + created_at: 2024-08-25 1:02:00 + processed_at: 2024-08-25 1:02:00 + index: 2 + version: 1 + +image_generation_explained: + assistant: samantha + conversation: image_generation + role: assistant + tool_call_id: + content_text: The weather in Austin is + content_tool_calls: + content_document: + created_at: 2024-08-25 1:03:00 + processed_at: 2024-08-25 1:03:00 + index: 3 + version: 1 + # Next conversation trees_explained: diff --git a/test/fixtures/users.yml b/test/fixtures/users.yml index cb56107ee..c8fa13c09 100644 --- a/test/fixtures/users.yml +++ b/test/fixtures/users.yml @@ -14,3 +14,8 @@ taylor: first_name: Taylor registered_at: 2024-05-31 08:40:05 preferences: {} + +christoph: + first_name: Christoph + registered_at: 2024-08-25 08:08:08 + preferences: {} diff --git a/test/jobs/get_next_ai_message_job_openai_test.rb b/test/jobs/get_next_ai_message_job_openai_test.rb index aac8fe17c..333ecf6f3 100644 --- a/test/jobs/get_next_ai_message_job_openai_test.rb +++ b/test/jobs/get_next_ai_message_job_openai_test.rb @@ -56,6 +56,66 @@ class GetNextAIMessageJobOpenaiTest < ActiveJob::TestCase refute second_new_message.finished?, "This message SHOULD NOT be considered finished yet" end + test "properly handles a tool response call from the assistant when images are included" do + @image_generation = conversations(:image_generation) + @image_generation.messages.create! role: :user, content_text: "Generate an image", assistant: @image_generation.assistant + @image_generation_message = @image_generation.latest_message_for_version(:latest) + + @image_generation.assistant.language_model.update!(supports_tools: true) + + image_generation_prompt = "Kitten" + + response = { + data: [{ + url: "https://example.com/image.jpg" + }] + } + + images_mock = Minitest::Mock.new + images_mock.expect :generate, response, parameters: { + prompt: image_generation_prompt, + model: "dall-e-3", + size: "1024x1792", + quality: "standard" + } + + assert_difference "@image_generation.messages.reload.length", 2 do + OpenAI::Client.stub_any_instance :images, images_mock do + TestClient::OpenAI.stub :function, "dalle_generate_an_image" do + TestClient::OpenAI.stub :arguments, { :image_generation_prompt=>image_generation_prompt } do + assert GetNextAIMessageJob.perform_now(@user.id, @image_generation_message.id, @image_generation.assistant.id) + end + end + end + end + + @image_generation_message.reload + assert @image_generation_message.content_text.blank? + assert @image_generation_message.tool_call_id.nil? + assert @image_generation_message.content_tool_calls.present?, "Assistant should have decided to call a tool" + + @new_messages = @image_generation.messages.where("id > ?", @image_generation_message.id).order(:created_at) + + # first + first_new_message = @new_messages.first + assert first_new_message.tool? + content_text = first_new_message.content_text + json_content_text = JSON.parse(content_text) + assert_equal image_generation_prompt, json_content_text["prompt_given"], "First new message should have the result of calling the tool" + assert first_new_message.tool_call_id.present? + assert first_new_message.content_tool_calls.blank? + assert_equal @image_generation_message.content_tool_calls.dig(0, :id), first_new_message.tool_call_id, "ID of tool execution should have matched decision to call the tool" + assert first_new_message.finished?, "This message SHOULD HAVE been considered finished" + + # second + second_new_message = @new_messages.second + assert second_new_message.assistant?, "Second new message should be queued up for the assistant to reply" + assert second_new_message.content_text.nil?, "The content should be nil to indicate that it hasn't even started processing" + assert second_new_message.tool_call_id.nil? + assert second_new_message.content_tool_calls.blank? + refute second_new_message.finished?, "This message SHOULD NOT be considered finished yet" + end + test "returns early if the message id was invalid" do refute GetNextAIMessageJob.perform_now(@user.id, 0, @assistant.id) end diff --git a/test/models/api_service_test.rb b/test/models/api_service_test.rb index da2ab9198..81088cd52 100644 --- a/test/models/api_service_test.rb +++ b/test/models/api_service_test.rb @@ -57,7 +57,9 @@ class APIServiceTest < ActiveSupport::TestCase end test "can create record" do - APIService.create!(create_params) + assert_nothing_raised do + APIService.create!(create_params) + end end test "soft delete also soft deletes language_models" do diff --git a/test/models/document_test.rb b/test/models/document_test.rb index c6229a1fc..b36372f9a 100644 --- a/test/models/document_test.rb +++ b/test/models/document_test.rb @@ -55,9 +55,8 @@ class DocumentTest < ActiveSupport::TestCase end test "fully_processed_url" do - assert documents(:cat_photo).fully_processed_url(:small).starts_with?("http") - assert documents(:cat_photo).fully_processed_url(:small).include?("rails/active_storage/postgresql") - assert documents(:cat_photo).fully_processed_url(:small).exclude?("/redirect") + assert documents(:cat_photo).fully_processed_url(:small).include?('rails/active_storage/postgresql') + assert documents(:cat_photo).fully_processed_url(:small).exclude?('/redirect') end test "redirect_to_processed_path" do diff --git a/test/models/message/version_test.rb b/test/models/message/version_test.rb index 018144269..16b31ae75 100644 --- a/test/models/message/version_test.rb +++ b/test/models/message/version_test.rb @@ -65,8 +65,10 @@ class Message::VersionTest < ActiveSupport::TestCase end test "creating a message with branched true AND branched_from_version specified SUCCEEDS" do - Current.user = users(:keith) - conversations(:versioned).messages.create!(assistant: assistants(:samantha), content_text: "What is your name?", index: 2, version: 3, branched: true, branched_from_version: 2) + assert_nothing_raised do + Current.user = users(:keith) + conversations(:versioned).messages.create!(assistant: assistants(:samantha), content_text: "What is your name?", index: 2, version: 3, branched: true, branched_from_version: 2) + end end test "creating a new messages for a SPECIFIC INDEX and SPECIFIC VERSION fails if the VERSION is SKIPPING a number" do diff --git a/test/services/ai_backend/anthropic_test.rb b/test/services/ai_backend/anthropic_test.rb index 7791ae20d..ed38a6014 100644 --- a/test/services/ai_backend/anthropic_test.rb +++ b/test/services/ai_backend/anthropic_test.rb @@ -48,11 +48,11 @@ class AIBackend::AnthropicTest < ActiveSupport::TestCase end end - test "preceding_conversation_messages only considers messages on the intended conversation version and includes the correct names" do - # TODO - end + # test "preceding_conversation_messages only considers messages on the intended conversation version and includes the correct names" do + # # TODO + # end - test "preceding_conversation_messages includes the appropriate tool details" do - # TODO - end + # test "preceding_conversation_messages includes the appropriate tool details" do + # # TODO + # end end