Skip to content

Commit

Permalink
Merge pull request #8 from Manifest-Holdings/twitter_proxy_url
Browse files Browse the repository at this point in the history 
Twitter proxy url
  • Loading branch information
@odilitime
odilitime authored Feb 4, 2025
2 parents b6680af + 4333ed6 commit 7820624
Show file tree
Hide file tree
Showing 7 changed files with 708 additions and 601 deletions.
1 change: 1 addition & 0 deletions agent/src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -621,6 +621,7 @@ export async function initializeClients(
password: getSecret(character, "TWITTER_PASSWORD"),
email: getSecret(character, "TWITTER_EMAIL"),
twitter2faSecret: getSecret(character, "TWITTER_2FA_SECRET"),
proxyUrl: getSecret(character, "TWITTER_PROXY_URL"),
});
if (isValidKey.success) {
const twitterClient = await TwitterClientInterface.start(runtime);
Expand Down
1 change: 1 addition & 0 deletions packages/client-twitter/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
"dependencies": {
"@elizaos/core": "workspace:*",
"agent-twitter-client": "0.0.18",
"undici": "7.3.0",
"glob": "11.0.0",
"zod": "3.23.8",
"discord.js": "14.16.3"
Expand Down
107 changes: 79 additions & 28 deletions packages/client-twitter/src/base.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import {
} from "agent-twitter-client";
import { EventEmitter } from "events";
import { TwitterConfig } from "./environment.ts";
import { fetch, ProxyAgent, setGlobalDispatcher } from "undici";

export function extractAnswer(text: string): string {
const startIndex = text.indexOf("Answer: ") + 8;
Expand Down Expand Up @@ -60,8 +61,8 @@ class RequestQueue {
while (this.queue.length > 0) {
const request = this.queue.shift()!;
try {
await request().catch(e => {
console.error('client.twitter.base - request err', e)
await request().catch((e) => {
console.error("client.twitter.base - request err", e);
});
} catch (error) {
console.error("Error processing request:", error);
Expand All @@ -85,35 +86,80 @@ class RequestQueue {
}
}

let lastStart = Date.now()
let lastStart = Date.now();

function doLogin(username, cb) {
const ts = Date.now()
const since = ts - lastStart
elizaLogger.log('last twitter scrapper created', since, 'ms ago')
const delay = 5 * 1000
function doLogin(username, cb, proxyUrl?: string) {
const ts = Date.now();
const since = ts - lastStart;
elizaLogger.log("last twitter scrapper created", since, "ms ago");
const delay = 5 * 1000;
if (since > delay) {
const twitterClient = new Scraper();
let agent;

// Add proxy configuration if TWITTER_PROXY_URL exists
if (proxyUrl) {
elizaLogger.log("Using proxy", proxyUrl);
const url = new URL(proxyUrl);
const username = url.username;
const password = url.password;

// Strip auth from URL if present
url.username = "";
url.password = "";

const agentOptions: any = {
uri: url.toString(),
requestTls: {
rejectUnauthorized: false,
},
};

// Add Basic auth if credentials exist
if (username && password) {
agentOptions.token = `Basic ${Buffer.from(
`${username}:${password}`
).toString("base64")}`;
}

agent = new ProxyAgent(agentOptions);
setGlobalDispatcher(agent);
}

const twitterClient = new Scraper({
fetch: fetch,
transform: {
request: (input: any, init: any) => {
if (agent) {
return [input, { ...init, dispatcher: agent }];
}
return [input, init];
},
},
});

ClientBase._twitterClients[username] = twitterClient;
lastStart = ts
cb(twitterClient)
lastStart = ts;
cb(twitterClient);
} else {
elizaLogger.log('Delaying twitter scrapper creation for', username)
elizaLogger.log("Delaying twitter scrapper creation for", username);
setTimeout(() => {
doLogin(username, cb)
}, delay)
doLogin(username, cb);
}, delay);
}
}

export function getScrapper(username:string):twitterClient {
return new Promise(resolve => {
export function getScraper(
username: string,
proxyUrl?: string
): Promise<Scraper> {
return new Promise((resolve) => {
if (ClientBase._twitterClients[username]) {
const twitterClient = ClientBase._twitterClients[username];
resolve(twitterClient)
resolve(twitterClient);
} else {
doLogin(username, resolve)
doLogin(username, resolve, proxyUrl);
}
})
});
}

export class ClientBase extends EventEmitter {
Expand Down Expand Up @@ -174,10 +220,13 @@ export class ClientBase extends EventEmitter {
super();
this.runtime = runtime;
this.twitterConfig = twitterConfig;

// Store proxy URL statically so it's available to doLogin
const proxyUrl = twitterConfig.TWITTER_PROXY_URL;
const username = twitterConfig.TWITTER_USERNAME;
getScrapper(username).then(tc => {
this.twitterClient = tc;
})
getScraper(username, proxyUrl).then((tc) => {
this.twitterClient = tc;
});

this.directions =
"- " +
Expand All @@ -193,7 +242,7 @@ export class ClientBase extends EventEmitter {
let retries = this.twitterConfig.TWITTER_RETRY_LIMIT;
const twitter2faSecret = this.twitterConfig.TWITTER_2FA_SECRET;
// if twitter says its bad, trust twitter
retries = 1 // mee.fun, lets no hammer this, it should work or not
retries = 1; // mee.fun, lets no hammer this, it should work or not

if (!username) {
throw new Error("Twitter username not configured");
Expand Down Expand Up @@ -221,7 +270,7 @@ export class ClientBase extends EventEmitter {
twitter2faSecret
);
if (await this.twitterClient.isLoggedIn()) {
lastStart = Date.now()
lastStart = Date.now();
// fresh login, store new cookies
elizaLogger.info("Successfully logged in.");
elizaLogger.info("Caching cookies");
Expand All @@ -233,7 +282,9 @@ export class ClientBase extends EventEmitter {
}
}
} catch (error) {
elizaLogger.error(`${this.runtime.character.name}(${this.runtime.agentId}): Login attempt failed: ${error.message} for twitter @${username}`);
elizaLogger.error(
`${this.runtime.character.name}(${this.runtime.agentId}): Login attempt failed: ${error.message} for twitter @${username}`
);
}

retries--;
Expand All @@ -255,7 +306,7 @@ export class ClientBase extends EventEmitter {
this.profile = await this.fetchProfile(username);

if (!this.profile) {
elizaLogger.error('cl-tw::init - profile did not load')
elizaLogger.error("cl-tw::init - profile did not load");
return false;
}
elizaLogger.log("Twitter user ID:", this.profile.id);
Expand Down Expand Up @@ -724,8 +775,8 @@ export class ClientBase extends EventEmitter {
if (latestCheckedTweetId) {
this.lastCheckedTweetId = BigInt(latestCheckedTweetId);
}
} catch(e) {
elizaLogger.error('cl-tw::loadLatestCheckedTweetId - err', e)
} catch (e) {
elizaLogger.error("cl-tw::loadLatestCheckedTweetId - err", e);
}
}

Expand Down
10 changes: 9 additions & 1 deletion packages/client-twitter/src/environment.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ export const twitterEnvSchema = z.object({
ACTION_TIMELINE_TYPE: z
.nativeEnum(ActionTimelineType)
.default(ActionTimelineType.ForYou),
TWITTER_PROXY_URL: z.string().optional(),
});

export type TwitterConfig = z.infer<typeof twitterEnvSchema>;
Expand Down Expand Up @@ -224,6 +225,11 @@ export async function validateTwitterConfig(
ACTION_TIMELINE_TYPE:
runtime.getSetting("ACTION_TIMELINE_TYPE") ||
process.env.ACTION_TIMELINE_TYPE,

TWITTER_PROXY_URL:
(runtime.getSetting("TWITTER_PROXY_URL") ||
process.env.TWITTER_PROXY_URL) ??
"",
};

return twitterEnvSchema.parse(twitterConfig);
Expand All @@ -232,7 +238,9 @@ export async function validateTwitterConfig(
const errorMessages = error.errors
.map((err) => `${err.path.join(".")}: ${err.message}`)
.join("\n");
elizaLogger.error(`Twitter configuration validation failed:\n${errorMessages}`)
elizaLogger.error(
`Twitter configuration validation failed:\n${errorMessages}`
);
/*
throw new Error(
`X/Twitter configuration validation failed:\n${errorMessages}`
Expand Down
13 changes: 9 additions & 4 deletions packages/client-twitter/src/index.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { Client, elizaLogger, IAgentRuntime } from "@elizaos/core";
import { ClientBase, getScrapper } from "./base.ts";
import { ClientBase, getScraper } from "./base.ts";
import { validateTwitterConfig, TwitterConfig } from "./environment.ts";
import { TwitterInteractionClient } from "./interactions.ts";
import { TwitterPostClient } from "./post.ts";
Expand Down Expand Up @@ -65,13 +65,15 @@ export const TwitterClientInterface: Client = {
let twitterConfig: TwitterConfig;
try {
twitterConfig = await validateTwitterConfig(runtime);
} catch (e) {
} catch (error) {
elizaLogger.error(
"TwitterConfig validation failed for",
runtime.getSetting("TWITTER_USERNAME") ||
process.env.TWITTER_USERNAME,
"email",
runtime.getSetting("TWITTER_EMAIL") || process.env.TWITTER_EMAIL
runtime.getSetting("TWITTER_EMAIL") ||
process.env.TWITTER_EMAIL,
error
);
return;
}
Expand Down Expand Up @@ -115,7 +117,10 @@ export const TwitterClientInterface: Client = {
},
validate: async (secrets) => {
try {
const twClient = await getScrapper(secrets.username);
const twClient = await getScraper(
secrets.username,
secrets.proxyUrl
);
// try logging in
console.log("trying to log in");
await twClient.login(
Expand Down
66 changes: 41 additions & 25 deletions packages/client-twitter/src/post.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,9 @@ export class TwitterPostClient {
elizaLogger.log(
`- Search Enabled: ${this.client.twitterConfig.TWITTER_SEARCH_ENABLE ? "enabled" : "disabled"}`
);
elizaLogger.log(
`- Proxy URL: ${this.client.twitterConfig.TWITTER_PROXY_URL}`
);

const targetUsers = this.client.twitterConfig.TWITTER_TARGET_USERS;
if (targetUsers) {
Expand Down Expand Up @@ -251,7 +254,11 @@ export class TwitterPostClient {
const actionInterval = this.client.twitterConfig.ACTION_INTERVAL; // Defaults to 5 minutes

while (!this.stopProcessingActions) {
elizaLogger.log('ACTION_INTERVAL', actionInterval, this.client.twitterConfig.TWITTER_USERNAME)
elizaLogger.log(
"ACTION_INTERVAL",
actionInterval,
this.client.twitterConfig.TWITTER_USERNAME
);
try {
const results = await this.processTweetActions();
if (results) {
Expand Down Expand Up @@ -414,27 +421,30 @@ export class TwitterPostClient {
const body = await standardTweetResult.json();
if (!body?.data?.create_tweet?.tweet_results?.result) {
elizaLogger.error("Error sending tweet; Bad response:", body);
if (body?.errors?.[0].message === 'Authorization: Denied by access control: Missing TwitterUserNotSuspended') {
elizaLogger.error("Account suspended");
//this.client is base
//this.runtime needs a stop client
// this is
const manager = this.runtime.clients.twitter
// stop post/search/interaction
if (manager) {
if (manager.client.twitterClient) {
await manager.post.stop();
await manager.interaction.stop();
if (manager.search) {
await manager.search.stop();
}
} else {
// it's still starting up
}
} // already stoped

// mark it offline
delete runtime.clients.twitter;
if (
body?.errors?.[0].message ===
"Authorization: Denied by access control: Missing TwitterUserNotSuspended"
) {
elizaLogger.error("Account suspended");
//this.client is base
//this.runtime needs a stop client
// this is
const manager = this.runtime.clients.twitter;
// stop post/search/interaction
if (manager) {
if (manager.client.twitterClient) {
await manager.post.stop();
await manager.interaction.stop();
if (manager.search) {
await manager.search.stop();
}
} else {
// it's still starting up
}
} // already stoped

// mark it offline
delete runtime.clients.twitter;
}
return;
}
Expand Down Expand Up @@ -531,7 +541,7 @@ export class TwitterPostClient {
modelClass: ModelClass.SMALL,
});

const newTweetContent = response
const newTweetContent = response
.replace(/```json\s*/g, "") // Remove ```json
.replace(/```\s*/g, "") // Remove any remaining ```
.replace(/(\r\n|\n|\r)/g, "") // Remove line break
Expand Down Expand Up @@ -618,10 +628,16 @@ export class TwitterPostClient {
);
}
} catch (error) {
elizaLogger.error("generateNewTweet Error sending tweet:", error);
elizaLogger.error(
"generateNewTweet Error sending tweet:",
error
);
}
} catch (error) {
elizaLogger.error("generateNewTweet Error generating new tweet:", error);
elizaLogger.error(
"generateNewTweet Error generating new tweet:",
error
);
}
}

Expand Down
Loading

0 comments on commit 7820624

Please sign in to comment.