Skip to content

Commit

Permalink
fixed issue of image generation
Browse files Browse the repository at this point in the history
  • Loading branch information
@nusk0
nusk0 committed Dec 28, 2024
1 parent 870a98e commit 4c566b3
Show file tree
Hide file tree
Showing 3 changed files with 81 additions and 40 deletions.
23 changes: 13 additions & 10 deletions packages/client-twitter/src/base.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -284,12 +284,13 @@ export class ClientBase extends EventEmitter {
hashtags: tweet.hashtags ?? tweet.legacy?.entities.hashtags,
mentions:
tweet.mentions ?? tweet.legacy?.entities.user_mentions,
photos:
tweet.photos ??
tweet.legacy?.entities.media?.filter(
photos: tweet.legacy?.entities?.media?.filter(
(media) => media.type === "photo"
) ??
[],
).map(media => ({
id: media.id_str,
url: media.media_url_https, // Store media_url_https as url
alt_text: media.alt_text
})) || [],
thread: tweet.thread || [],
urls: tweet.urls ?? tweet.legacy?.entities.urls,
videos:
Expand All @@ -312,7 +313,6 @@ export class ClientBase extends EventEmitter {
count,
[]
);

return homeTimeline.map((tweet) => ({
id: tweet.rest_id,
name: tweet.core?.user_results?.result?.legacy?.name,
Expand All @@ -325,10 +325,13 @@ export class ClientBase extends EventEmitter {
permanentUrl: `https://twitter.com/${tweet.core?.user_results?.result?.legacy?.screen_name}/status/${tweet.rest_id}`,
hashtags: tweet.legacy?.entities?.hashtags || [],
mentions: tweet.legacy?.entities?.user_mentions || [],
photos:
tweet.legacy?.entities?.media?.filter(
(media) => media.type === "photo"
) || [],
photos: tweet.legacy?.entities?.media?.filter(
(media) => media.type === "photo"
).map(media => ({
id: media.id_str,
url: media.media_url_https, // Store media_url_https as url
alt_text: media.alt_text
})) || [],
thread: tweet.thread || [],
urls: tweet.legacy?.entities?.urls || [],
videos:
Expand Down
46 changes: 41 additions & 5 deletions packages/client-twitter/src/post.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,16 +53,21 @@ Guidelines:
- Only interact with post that are related to the league of legend or arcane universe
Actions (respond only with tags):
[LIKE] - Resonates with interests (10/10)
[RETWEET] - Has to be related to the arcane or league of legend universe (10/10)
[QUOTE] - Can add unique value (10/10)
[REPLY] - Something related to jinx and the arcane universe (10/10)
[LIKE] - Resonates with interests (9.5/10)
[RETWEET] - Perfect character alignment (9/10)
[QUOTE] - Can add unique value (8/10)
[REPLY] - Memetic opportunity (9/10)
Tweet:
{{currentTweet}}
# Respond with qualifying action tags only.` + postActionResponseFooter;

//Actions (respond only with tags):
//[LIKE] - Resonates with interests (10/10)
//[RETWEET] - Has to be related to the arcane or league of legend universe (10/10)
//[QUOTE] - Can add unique value (9/0)
//[REPLY] - Something related to jinx and the arcane universe (8/10)
/**
* Truncate text to fit within the Twitter character limit, ensuring it ends at a complete sentence.
*/
Expand Down Expand Up @@ -458,12 +463,28 @@ export class TwitterPostClient {
);
}


private async processTweetActions() {
if (this.isProcessing) {
elizaLogger.log("Already processing tweet actions, skipping");
return null;
}
try {
//image test function
/*const imageDescriptions = [];
elizaLogger.log(
"Processing TEST images"
);
const description = await this.runtime
.getService<IImageDescriptionService>(
ServiceType.IMAGE_DESCRIPTION
)
.describeImage("https://x.com/lesly_oh/status/1872738371988275557");
imageDescriptions.push(description);
elizaLogger.log(
"Processing TEST images"+imageDescriptions
);*/
this.isProcessing = true;
this.lastProcessTime = Date.now();

Expand All @@ -483,7 +504,22 @@ export class TwitterPostClient {

for (const tweet of homeTimeline) {
elizaLogger.log(`Processing tweet ID: ${tweet.id}`);

//test image description function
/* const imageDescriptions = [];
if (tweet.photos?.length > 0) {
elizaLogger.log(
"Processing images in tweet for context"
);
for (const photo of tweet.photos) {
const description = await this.runtime
.getService<IImageDescriptionService>(
ServiceType.IMAGE_DESCRIPTION
)
.describeImage(photo.url);
imageDescriptions.push(description);
console.log("photo.url :", photo.url);
}
}*/
try {
// Skip if we've already processed this tweet
const memory =
Expand Down
52 changes: 27 additions & 25 deletions packages/plugin-node/src/services/image.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,36 +134,34 @@ export class ImageDescriptionService
private async recognizeWithOpenAI(
imageUrl: string
): Promise<{ title: string; description: string }> {
const isGif = imageUrl.toLowerCase().endsWith(".gif");
let imageData: Buffer | null = null;

try {
elizaLogger.log("Recognizing image with OpenAI URL:"+imageUrl);
if (isGif) {
const { filePath } =
await this.extractFirstFrameFromGif(imageUrl);
imageData = fs.readFileSync(filePath);
} else {
const response = await fetch(imageUrl);
if (!response.ok) {
throw new Error(
`Failed to fetch image: ${response.statusText}`
);
}
imageData = Buffer.from(await response.arrayBuffer());
}
elizaLogger.log("Recognizing image with OpenAI URL:" + imageUrl);

if (!imageData || imageData.length === 0) {
throw new Error("Failed to fetch image data");
// Fetch the image first
const response = await fetch(imageUrl);
if (!response.ok) {
throw new Error(`Failed to fetch image: ${response.status} ${response.statusText}`);
}

const prompt =
"Describe this image and give it a title. The first line should be the title, and then a line break, then a detailed description of the image. Respond with the format 'title\ndescription'";
// Convert to buffer
const imageBuffer = Buffer.from(await response.arrayBuffer());

// Convert to base64 and ensure it's a supported format
const base64Image = imageBuffer.toString('base64');
const imageType = response.headers.get('content-type') || 'image/jpeg';

// Use data URL with proper MIME type
const finalImageUrl = `data:${imageType};base64,${base64Image}`;

elizaLogger.log("Image type:", imageType);

const prompt = "Describe this image and give it a title, try to identify characters from the show Arcane, especially jinx, vi and ekko. The characters might have different visuals from the original show but try to identify them by using general characteristics. Respond with the format 'title\ndescription'";

const text = await this.requestOpenAI(
imageUrl,
imageData,
finalImageUrl,
imageBuffer,
prompt,
isGif
false
);

const [title, ...descriptionParts] = text.split("\n");
Expand All @@ -172,7 +170,11 @@ export class ImageDescriptionService
description: descriptionParts.join("\n"),
};
} catch (error) {
elizaLogger.error("Error in recognizeWithOpenAI:", error);
elizaLogger.error("Error in recognizeWithOpenAI:", {
message: error.message,
stack: error.stack,
cause: error.cause
});
throw error;
}
}
Expand Down

0 comments on commit 4c566b3

Please sign in to comment.