Twitter proxy url

agent/src/index.ts

@@ -620,6 +620,7 @@ export async function initializeClients(
             password: getSecret(character, "TWITTER_PASSWORD"),
             email: getSecret(character, "TWITTER_EMAIL"),
             twitter2faSecret: getSecret(character, "TWITTER_2FA_SECRET"),
+            proxyUrl: getSecret(character, "TWITTER_PROXY_URL"),
         });
         if (isValidKey.success) {
             const twitterClient = await TwitterClientInterface.start(runtime);

packages/client-twitter/package.json

@@ -21,6 +21,7 @@
     "dependencies": {
         "@elizaos/core": "workspace:*",
         "agent-twitter-client": "0.0.18",
+        "undici": "7.3.0",
         "glob": "11.0.0",
         "zod": "3.23.8",
         "discord.js": "14.16.3"

packages/client-twitter/src/base.ts

@@ -18,6 +18,7 @@ import {
 } from "agent-twitter-client";
 import { EventEmitter } from "events";
 import { TwitterConfig } from "./environment.ts";
+import { fetch, ProxyAgent, setGlobalDispatcher } from "undici";
 
 export function extractAnswer(text: string): string {
     const startIndex = text.indexOf("Answer: ") + 8;
@@ -60,8 +61,8 @@ class RequestQueue {
         while (this.queue.length > 0) {
             const request = this.queue.shift()!;
             try {
-                await request().catch(e => {
-                  console.error('client.twitter.base - request err', e)
+                await request().catch((e) => {
+                    console.error("client.twitter.base - request err", e);
                 });
             } catch (error) {
                 console.error("Error processing request:", error);
@@ -85,35 +86,80 @@ class RequestQueue {
     }
 }
 
-let lastStart = Date.now()
+let lastStart = Date.now();
 
-function doLogin(username, cb) {
-    const ts = Date.now()
-    const since = ts - lastStart
-    elizaLogger.log('last twitter scrapper created', since, 'ms ago')
-    const delay = 5 * 1000
+function doLogin(username, cb, proxyUrl?: string) {
+    const ts = Date.now();
+    const since = ts - lastStart;
+    elizaLogger.log("last twitter scrapper created", since, "ms ago");
+    const delay = 5 * 1000;
     if (since > delay) {
-        const twitterClient = new Scraper();
+        let agent;
+
+        // Add proxy configuration if TWITTER_PROXY_URL exists
+        if (proxyUrl) {
+            elizaLogger.log("Using proxy", proxyUrl);
+            const url = new URL(proxyUrl);
+            const username = url.username;
+            const password = url.password;
+
+            // Strip auth from URL if present
+            url.username = "";
+            url.password = "";
+
+            const agentOptions: any = {
+                uri: url.toString(),
+                requestTls: {
+                    rejectUnauthorized: false,
+                },
+            };
+
+            // Add Basic auth if credentials exist
+            if (username && password) {
+                agentOptions.token = `Basic ${Buffer.from(
+                    `${username}:${password}`
+                ).toString("base64")}`;
+            }
+
+            agent = new ProxyAgent(agentOptions);
+            setGlobalDispatcher(agent);
+        }
+
+        const twitterClient = new Scraper({
+            fetch: fetch,
+            transform: {
+                request: (input: any, init: any) => {
+                    if (agent) {
+                        return [input, { ...init, dispatcher: agent }];
+                    }
+                    return [input, init];
+                },
+            },
+        });
+
         ClientBase._twitterClients[username] = twitterClient;
-        lastStart = ts
-        cb(twitterClient)
+        lastStart = ts;
+        cb(twitterClient);
     } else {
-        elizaLogger.log('Delaying twitter scrapper creation for', username)
+        elizaLogger.log("Delaying twitter scrapper creation for", username);
         setTimeout(() => {
-          doLogin(username, cb)
-        }, delay)
+            doLogin(username, cb);
+        }, delay);
     }
 }
 
-export function getScrapper(username:string):twitterClient {
-    return new Promise(resolve => {
+export function getScraper(
+    username: string,
+    proxyUrl?: string
+): Promise<Scraper> {
+    return new Promise((resolve) => {
         if (ClientBase._twitterClients[username]) {
             const twitterClient = ClientBase._twitterClients[username];
-            resolve(twitterClient)
+            resolve(twitterClient);
         } else {
-            doLogin(username, resolve)
+            doLogin(username, resolve, proxyUrl);
         }
-    })
+    });
 }
 
 export class ClientBase extends EventEmitter {
@@ -174,10 +220,13 @@ export class ClientBase extends EventEmitter {
         super();
         this.runtime = runtime;
         this.twitterConfig = twitterConfig;
+
+        // Store proxy URL statically so it's available to doLogin
+        const proxyUrl = twitterConfig.TWITTER_PROXY_URL;
         const username = twitterConfig.TWITTER_USERNAME;
-        getScrapper(username).then(tc => {
-          this.twitterClient = tc;
-        })
+        getScraper(username, proxyUrl).then((tc) => {
+            this.twitterClient = tc;
+        });
 
         this.directions =
             "- " +
@@ -193,7 +242,7 @@ export class ClientBase extends EventEmitter {
         let retries = this.twitterConfig.TWITTER_RETRY_LIMIT;
         const twitter2faSecret = this.twitterConfig.TWITTER_2FA_SECRET;
         // if twitter says its bad, trust twitter
-        retries = 1 // mee.fun, lets no hammer this, it should work or not
+        retries = 1; // mee.fun, lets no hammer this, it should work or not
 
         if (!username) {
             throw new Error("Twitter username not configured");
@@ -221,7 +270,7 @@ export class ClientBase extends EventEmitter {
                         twitter2faSecret
                     );
                     if (await this.twitterClient.isLoggedIn()) {
-                        lastStart = Date.now()
+                        lastStart = Date.now();
                         // fresh login, store new cookies
                         elizaLogger.info("Successfully logged in.");
                         elizaLogger.info("Caching cookies");
@@ -233,7 +282,9 @@ export class ClientBase extends EventEmitter {
                     }
                 }
             } catch (error) {
-                elizaLogger.error(`${this.runtime.character.name}(${this.runtime.agentId}): Login attempt failed: ${error.message} for twitter @${username}`);
+                elizaLogger.error(
+                    `${this.runtime.character.name}(${this.runtime.agentId}): Login attempt failed: ${error.message} for twitter @${username}`
+                );
             }
 
             retries--;
@@ -255,7 +306,7 @@ export class ClientBase extends EventEmitter {
         this.profile = await this.fetchProfile(username);
 
         if (!this.profile) {
-            elizaLogger.error('cl-tw::init - profile did not load')
+            elizaLogger.error("cl-tw::init - profile did not load");
             return false;
         }
         elizaLogger.log("Twitter user ID:", this.profile.id);
@@ -724,8 +775,8 @@ export class ClientBase extends EventEmitter {
             if (latestCheckedTweetId) {
                 this.lastCheckedTweetId = BigInt(latestCheckedTweetId);
             }
-        } catch(e) {
-            elizaLogger.error('cl-tw::loadLatestCheckedTweetId - err', e)
+        } catch (e) {
+            elizaLogger.error("cl-tw::loadLatestCheckedTweetId - err", e);
         }
     }
 

packages/client-twitter/src/environment.ts

@@ -76,6 +76,7 @@ export const twitterEnvSchema = z.object({
     ACTION_TIMELINE_TYPE: z
         .nativeEnum(ActionTimelineType)
         .default(ActionTimelineType.ForYou),
+    TWITTER_PROXY_URL: z.string().optional(),
 });
 
 export type TwitterConfig = z.infer<typeof twitterEnvSchema>;
@@ -224,6 +225,11 @@ export async function validateTwitterConfig(
             ACTION_TIMELINE_TYPE:
                 runtime.getSetting("ACTION_TIMELINE_TYPE") ||
                 process.env.ACTION_TIMELINE_TYPE,
+
+            TWITTER_PROXY_URL:
+                (runtime.getSetting("TWITTER_PROXY_URL") ||
+                    process.env.TWITTER_PROXY_URL) ??
+                "",
         };
 
         return twitterEnvSchema.parse(twitterConfig);
@@ -232,7 +238,9 @@ export async function validateTwitterConfig(
             const errorMessages = error.errors
                 .map((err) => `${err.path.join(".")}: ${err.message}`)
                 .join("\n");
-            elizaLogger.error(`Twitter configuration validation failed:\n${errorMessages}`)
+            elizaLogger.error(
+                `Twitter configuration validation failed:\n${errorMessages}`
+            );
             /*
             throw new Error(
                 `X/Twitter configuration validation failed:\n${errorMessages}`

packages/client-twitter/src/index.ts

@@ -1,5 +1,5 @@
 import { Client, elizaLogger, IAgentRuntime } from "@elizaos/core";
-import { ClientBase, getScrapper } from "./base.ts";
+import { ClientBase, getScraper } from "./base.ts";
 import { validateTwitterConfig, TwitterConfig } from "./environment.ts";
 import { TwitterInteractionClient } from "./interactions.ts";
 import { TwitterPostClient } from "./post.ts";
@@ -65,13 +65,15 @@ export const TwitterClientInterface: Client = {
         let twitterConfig: TwitterConfig;
         try {
             twitterConfig = await validateTwitterConfig(runtime);
-        } catch (e) {
+        } catch (error) {
             elizaLogger.error(
                 "TwitterConfig validation failed for",
                 runtime.getSetting("TWITTER_USERNAME") ||
                     process.env.TWITTER_USERNAME,
                 "email",
-                runtime.getSetting("TWITTER_EMAIL") || process.env.TWITTER_EMAIL
+                runtime.getSetting("TWITTER_EMAIL") ||
+                    process.env.TWITTER_EMAIL,
+                error
             );
             return;
         }
@@ -115,7 +117,10 @@ export const TwitterClientInterface: Client = {
     },
     validate: async (secrets) => {
         try {
-            const twClient = await getScrapper(secrets.username);
+            const twClient = await getScraper(
+                secrets.username,
+                secrets.proxyUrl
+            );
             // try logging in
             console.log("trying to log in");
             await twClient.login(

packages/client-twitter/src/post.ts

@@ -130,6 +130,9 @@ export class TwitterPostClient {
         elizaLogger.log(
             `- Search Enabled: ${this.client.twitterConfig.TWITTER_SEARCH_ENABLE ? "enabled" : "disabled"}`
         );
+        elizaLogger.log(
+            `- Proxy URL: ${this.client.twitterConfig.TWITTER_PROXY_URL}`
+        );
 
         const targetUsers = this.client.twitterConfig.TWITTER_TARGET_USERS;
         if (targetUsers) {
@@ -251,7 +254,11 @@ export class TwitterPostClient {
             const actionInterval = this.client.twitterConfig.ACTION_INTERVAL; // Defaults to 5 minutes
 
             while (!this.stopProcessingActions) {
-                elizaLogger.log('ACTION_INTERVAL', actionInterval, this.client.twitterConfig.TWITTER_USERNAME)
+                elizaLogger.log(
+                    "ACTION_INTERVAL",
+                    actionInterval,
+                    this.client.twitterConfig.TWITTER_USERNAME
+                );
                 try {
                     const results = await this.processTweetActions();
                     if (results) {
@@ -414,27 +421,30 @@ export class TwitterPostClient {
             const body = await standardTweetResult.json();
             if (!body?.data?.create_tweet?.tweet_results?.result) {
                 elizaLogger.error("Error sending tweet; Bad response:", body);
-                if (body?.errors?.[0].message === 'Authorization: Denied by access control: Missing TwitterUserNotSuspended') {
-                  elizaLogger.error("Account suspended");
-                  //this.client is base
-                  //this.runtime needs a stop client
-                  // this is
-                  const manager = this.runtime.clients.twitter
-                  // stop post/search/interaction
-                  if (manager) {
-                      if (manager.client.twitterClient) {
-                          await manager.post.stop();
-                          await manager.interaction.stop();
-                          if (manager.search) {
-                              await manager.search.stop();
-                          }
-                      } else {
-                          // it's still starting up
-                      }
-                  } // already stoped
-
-                  // mark it offline
-                  delete runtime.clients.twitter;
+                if (
+                    body?.errors?.[0].message ===
+                    "Authorization: Denied by access control: Missing TwitterUserNotSuspended"
+                ) {
+                    elizaLogger.error("Account suspended");
+                    //this.client is base
+                    //this.runtime needs a stop client
+                    // this is
+                    const manager = this.runtime.clients.twitter;
+                    // stop post/search/interaction
+                    if (manager) {
+                        if (manager.client.twitterClient) {
+                            await manager.post.stop();
+                            await manager.interaction.stop();
+                            if (manager.search) {
+                                await manager.search.stop();
+                            }
+                        } else {
+                            // it's still starting up
+                        }
+                    } // already stoped
+
+                    // mark it offline
+                    delete runtime.clients.twitter;
                 }
                 return;
             }
@@ -531,7 +541,7 @@ export class TwitterPostClient {
                 modelClass: ModelClass.SMALL,
             });
 
-             const newTweetContent = response
+            const newTweetContent = response
                 .replace(/```json\s*/g, "") // Remove ```json
                 .replace(/```\s*/g, "") // Remove any remaining ```
                 .replace(/(\r\n|\n|\r)/g, "") // Remove line break
@@ -618,10 +628,16 @@ export class TwitterPostClient {
                     );
                 }
             } catch (error) {
-                elizaLogger.error("generateNewTweet Error sending tweet:", error);
+                elizaLogger.error(
+                    "generateNewTweet Error sending tweet:",
+                    error
+                );
             }
         } catch (error) {
-            elizaLogger.error("generateNewTweet Error generating new tweet:", error);
+            elizaLogger.error(
+                "generateNewTweet Error generating new tweet:",
+                error
+            );
         }
     }