diff --git a/packages/client-twitter/src/base.ts b/packages/client-twitter/src/base.ts index d9f10281cb5..1055f501933 100644 --- a/packages/client-twitter/src/base.ts +++ b/packages/client-twitter/src/base.ts @@ -284,12 +284,13 @@ export class ClientBase extends EventEmitter { hashtags: tweet.hashtags ?? tweet.legacy?.entities.hashtags, mentions: tweet.mentions ?? tweet.legacy?.entities.user_mentions, - photos: - tweet.photos ?? - tweet.legacy?.entities.media?.filter( + photos: tweet.legacy?.entities?.media?.filter( (media) => media.type === "photo" - ) ?? - [], + ).map(media => ({ + id: media.id_str, + url: media.media_url_https, // Store media_url_https as url + alt_text: media.alt_text + })) || [], thread: tweet.thread || [], urls: tweet.urls ?? tweet.legacy?.entities.urls, videos: @@ -312,7 +313,6 @@ export class ClientBase extends EventEmitter { count, [] ); - return homeTimeline.map((tweet) => ({ id: tweet.rest_id, name: tweet.core?.user_results?.result?.legacy?.name, @@ -325,10 +325,13 @@ export class ClientBase extends EventEmitter { permanentUrl: `https://twitter.com/${tweet.core?.user_results?.result?.legacy?.screen_name}/status/${tweet.rest_id}`, hashtags: tweet.legacy?.entities?.hashtags || [], mentions: tweet.legacy?.entities?.user_mentions || [], - photos: - tweet.legacy?.entities?.media?.filter( - (media) => media.type === "photo" - ) || [], + photos: tweet.legacy?.entities?.media?.filter( + (media) => media.type === "photo" + ).map(media => ({ + id: media.id_str, + url: media.media_url_https, // Store media_url_https as url + alt_text: media.alt_text + })) || [], thread: tweet.thread || [], urls: tweet.legacy?.entities?.urls || [], videos: diff --git a/packages/client-twitter/src/post.ts b/packages/client-twitter/src/post.ts index 540e27603a2..77fbd4003db 100644 --- a/packages/client-twitter/src/post.ts +++ b/packages/client-twitter/src/post.ts @@ -53,16 +53,21 @@ Guidelines: - Only interact with post that are related to the league of legend or arcane universe Actions (respond only with tags): -[LIKE] - Resonates with interests (10/10) -[RETWEET] - Has to be related to the arcane or league of legend universe (10/10) -[QUOTE] - Can add unique value (10/10) -[REPLY] - Something related to jinx and the arcane universe (10/10) +[LIKE] - Resonates with interests (9.5/10) +[RETWEET] - Perfect character alignment (9/10) +[QUOTE] - Can add unique value (8/10) +[REPLY] - Memetic opportunity (9/10) Tweet: {{currentTweet}} # Respond with qualifying action tags only.` + postActionResponseFooter; +//Actions (respond only with tags): +//[LIKE] - Resonates with interests (10/10) +//[RETWEET] - Has to be related to the arcane or league of legend universe (10/10) +//[QUOTE] - Can add unique value (9/0) +//[REPLY] - Something related to jinx and the arcane universe (8/10) /** * Truncate text to fit within the Twitter character limit, ensuring it ends at a complete sentence. */ @@ -458,12 +463,28 @@ export class TwitterPostClient { ); } + private async processTweetActions() { if (this.isProcessing) { elizaLogger.log("Already processing tweet actions, skipping"); return null; } try { + //image test function + /*const imageDescriptions = []; + + elizaLogger.log( + "Processing TEST images" + ); + const description = await this.runtime + .getService( + ServiceType.IMAGE_DESCRIPTION + ) + .describeImage("https://x.com/lesly_oh/status/1872738371988275557"); + imageDescriptions.push(description); + elizaLogger.log( + "Processing TEST images"+imageDescriptions + );*/ this.isProcessing = true; this.lastProcessTime = Date.now(); @@ -483,7 +504,22 @@ export class TwitterPostClient { for (const tweet of homeTimeline) { elizaLogger.log(`Processing tweet ID: ${tweet.id}`); - + //test image description function + /* const imageDescriptions = []; + if (tweet.photos?.length > 0) { + elizaLogger.log( + "Processing images in tweet for context" + ); + for (const photo of tweet.photos) { + const description = await this.runtime + .getService( + ServiceType.IMAGE_DESCRIPTION + ) + .describeImage(photo.url); + imageDescriptions.push(description); + console.log("photo.url :", photo.url); + } + }*/ try { // Skip if we've already processed this tweet const memory = diff --git a/packages/plugin-node/src/services/image.ts b/packages/plugin-node/src/services/image.ts index 03dacb92b0f..07d034f22c8 100644 --- a/packages/plugin-node/src/services/image.ts +++ b/packages/plugin-node/src/services/image.ts @@ -134,36 +134,34 @@ export class ImageDescriptionService private async recognizeWithOpenAI( imageUrl: string ): Promise<{ title: string; description: string }> { - const isGif = imageUrl.toLowerCase().endsWith(".gif"); - let imageData: Buffer | null = null; - try { - elizaLogger.log("Recognizing image with OpenAI URL:"+imageUrl); - if (isGif) { - const { filePath } = - await this.extractFirstFrameFromGif(imageUrl); - imageData = fs.readFileSync(filePath); - } else { - const response = await fetch(imageUrl); - if (!response.ok) { - throw new Error( - `Failed to fetch image: ${response.statusText}` - ); - } - imageData = Buffer.from(await response.arrayBuffer()); - } + elizaLogger.log("Recognizing image with OpenAI URL:" + imageUrl); - if (!imageData || imageData.length === 0) { - throw new Error("Failed to fetch image data"); + // Fetch the image first + const response = await fetch(imageUrl); + if (!response.ok) { + throw new Error(`Failed to fetch image: ${response.status} ${response.statusText}`); } - const prompt = - "Describe this image and give it a title. The first line should be the title, and then a line break, then a detailed description of the image. Respond with the format 'title\ndescription'"; + // Convert to buffer + const imageBuffer = Buffer.from(await response.arrayBuffer()); + + // Convert to base64 and ensure it's a supported format + const base64Image = imageBuffer.toString('base64'); + const imageType = response.headers.get('content-type') || 'image/jpeg'; + + // Use data URL with proper MIME type + const finalImageUrl = `data:${imageType};base64,${base64Image}`; + + elizaLogger.log("Image type:", imageType); + + const prompt = "Describe this image and give it a title, try to identify characters from the show Arcane, especially jinx, vi and ekko. The characters might have different visuals from the original show but try to identify them by using general characteristics. Respond with the format 'title\ndescription'"; + const text = await this.requestOpenAI( - imageUrl, - imageData, + finalImageUrl, + imageBuffer, prompt, - isGif + false ); const [title, ...descriptionParts] = text.split("\n"); @@ -172,7 +170,11 @@ export class ImageDescriptionService description: descriptionParts.join("\n"), }; } catch (error) { - elizaLogger.error("Error in recognizeWithOpenAI:", error); + elizaLogger.error("Error in recognizeWithOpenAI:", { + message: error.message, + stack: error.stack, + cause: error.cause + }); throw error; } }