Skip to content

Commit

Permalink
Merge pull request #2818 from elizaOS/tcm-post-media
Browse files Browse the repository at this point in the history 
feat: twitter post media
  • Loading branch information
@tcm390
tcm390 authored Jan 27, 2025
2 parents 872b13e + b2e2c56 commit 8536985
Show file tree
Hide file tree
Showing 3 changed files with 89 additions and 72 deletions.
92 changes: 51 additions & 41 deletions packages/client-twitter/src/post.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import type { ClientBase } from "./base.ts";
import { postActionResponseFooter } from "@elizaos/core";
import { generateTweetActions } from "@elizaos/core";
import { type IImageDescriptionService, ServiceType } from "@elizaos/core";
import { buildConversationThread } from "./utils.ts";
import { buildConversationThread, fetchMediaData } from "./utils.ts";
import { twitterMessageHandlerTemplate } from "./interactions.ts";
import { DEFAULT_MAX_TWEET_LENGTH } from "./environment.ts";
import {
Expand All @@ -30,6 +30,7 @@ import {
} from "discord.js";
import type { State } from "@elizaos/core";
import type { ActionResponse } from "@elizaos/core";
import { MediaData } from "./types.ts";

const MAX_TIMELINES_TO_FETCH = 15;

Expand Down Expand Up @@ -82,9 +83,9 @@ Tweet:
postActionResponseFooter;

interface PendingTweet {
cleanedContent: string;
tweetTextForPosting: string;
roomId: UUID;
newTweetContent: string;
rawTweetContent: string;
discordMessageId: string;
channelId: string;
timestamp: number;
Expand Down Expand Up @@ -338,7 +339,7 @@ export class TwitterPostClient {
client: ClientBase,
tweet: Tweet,
roomId: UUID,
newTweetContent: string,
rawTweetContent: string,
) {
// Cache the last post details
await runtime.cacheManager.set(
Expand All @@ -365,7 +366,7 @@ export class TwitterPostClient {
userId: runtime.agentId,
agentId: runtime.agentId,
content: {
text: newTweetContent.trim(),
text: rawTweetContent.trim(),
url: tweet.permanentUrl,
source: "twitter",
},
Expand All @@ -379,11 +380,12 @@ export class TwitterPostClient {
client: ClientBase,
content: string,
tweetId?: string,
mediaData?: MediaData[]
) {
try {
const noteTweetResult = await client.requestQueue.add(
async () =>
await client.twitterClient.sendNoteTweet(content, tweetId),
await client.twitterClient.sendNoteTweet(content, tweetId, mediaData),
);

if (noteTweetResult.errors && noteTweetResult.errors.length > 0) {
Expand All @@ -410,11 +412,12 @@ export class TwitterPostClient {
client: ClientBase,
content: string,
tweetId?: string,
mediaData?: MediaData[]
) {
try {
const standardTweetResult = await client.requestQueue.add(
async () =>
await client.twitterClient.sendTweet(content, tweetId),
await client.twitterClient.sendTweet(content, tweetId, mediaData),
);
const body = await standardTweetResult.json();
if (!body?.data?.create_tweet?.tweet_results?.result) {
Expand All @@ -431,20 +434,21 @@ export class TwitterPostClient {
async postTweet(
runtime: IAgentRuntime,
client: ClientBase,
cleanedContent: string,
tweetTextForPosting: string,
roomId: UUID,
newTweetContent: string,
rawTweetContent: string,
twitterUsername: string,
mediaData?: MediaData[]
) {
try {
elizaLogger.log(`Posting new tweet:\n`);

let result;

if (cleanedContent.length > DEFAULT_MAX_TWEET_LENGTH) {
result = await this.handleNoteTweet(client, cleanedContent);
if (tweetTextForPosting.length > DEFAULT_MAX_TWEET_LENGTH) {
result = await this.handleNoteTweet(client, tweetTextForPosting, undefined, mediaData);
} else {
result = await this.sendStandardTweet(client, cleanedContent);
result = await this.sendStandardTweet(client, tweetTextForPosting, undefined, mediaData);
}

const tweet = this.createTweetObject(
Expand All @@ -458,7 +462,7 @@ export class TwitterPostClient {
client,
tweet,
roomId,
newTweetContent,
rawTweetContent,
);
} catch (error) {
elizaLogger.error("Error sending tweet:", error);
Expand Down Expand Up @@ -514,40 +518,45 @@ export class TwitterPostClient {
modelClass: ModelClass.SMALL,
});

const newTweetContent = cleanJsonResponse(response);
const rawTweetContent = cleanJsonResponse(response);

// First attempt to clean content
let cleanedContent = "";
let tweetTextForPosting = "";
let mediaData = null;

// Try parsing as JSON first
const parsedResponse = parseJSONObjectFromText(newTweetContent);
const parsedResponse = parseJSONObjectFromText(rawTweetContent);
if (parsedResponse.text) {
cleanedContent = parsedResponse.text;
tweetTextForPosting = parsedResponse.text;
}

if (parsedResponse.attachments && parsedResponse.attachments.length > 0) {
mediaData = await fetchMediaData(parsedResponse.attachments);
}

// Try extracting text attribute
if (!cleanedContent) {
const parsingText = extractAttributes(newTweetContent, [
if (!tweetTextForPosting) {
const parsingText = extractAttributes(rawTweetContent, [
"text",
]).text;
if (parsingText) {
cleanedContent = truncateToCompleteSentence(
extractAttributes(newTweetContent, ["text"]).text,
tweetTextForPosting = truncateToCompleteSentence(
extractAttributes(rawTweetContent, ["text"]).text,
this.client.twitterConfig.MAX_TWEET_LENGTH,
);
}
}

// Use the raw text
if (!cleanedContent) {
cleanedContent = newTweetContent;
if (!tweetTextForPosting) {
tweetTextForPosting = rawTweetContent;
}

// Truncate the content to the maximum tweet length specified in the environment settings, ensuring the truncation respects sentence boundaries.
const maxTweetLength = this.client.twitterConfig.MAX_TWEET_LENGTH;
if (maxTweetLength) {
cleanedContent = truncateToCompleteSentence(
cleanedContent,
tweetTextForPosting = truncateToCompleteSentence(
tweetTextForPosting,
maxTweetLength,
);
}
Expand All @@ -558,11 +567,11 @@ export class TwitterPostClient {
const fixNewLines = (str: string) => str.replaceAll(/\\n/g, "\n\n"); //ensures double spaces

// Final cleaning
cleanedContent = removeQuotes(fixNewLines(cleanedContent));
tweetTextForPosting = removeQuotes(fixNewLines(tweetTextForPosting));

if (this.isDryRun) {
elizaLogger.info(
`Dry run: would have posted tweet: ${cleanedContent}`,
`Dry run: would have posted tweet: ${tweetTextForPosting}`,
);
return;
}
Expand All @@ -571,23 +580,24 @@ export class TwitterPostClient {
if (this.approvalRequired) {
// Send for approval instead of posting directly
elizaLogger.log(
`Sending Tweet For Approval:\n ${cleanedContent}`,
`Sending Tweet For Approval:\n ${tweetTextForPosting}`,
);
await this.sendForApproval(
cleanedContent,
tweetTextForPosting,
roomId,
newTweetContent,
rawTweetContent,
);
elizaLogger.log("Tweet sent for approval");
} else {
elizaLogger.log(`Posting new tweet:\n ${cleanedContent}`);
elizaLogger.log(`Posting new tweet:\n ${tweetTextForPosting}`);
this.postTweet(
this.runtime,
this.client,

Check warning on line 595 in packages/client-twitter/src/post.ts

View check run for this annotation

codefactor.io / CodeFactor

packages/client-twitter/src/post.ts#L595 

Unexpected any. Specify a different type. (@typescript-eslint/no-explicit-any)
cleanedContent,
tweetTextForPosting,
roomId,
newTweetContent,
rawTweetContent,
this.twitterUsername,
mediaData,
);
}
} catch (error) {
Expand Down Expand Up @@ -1216,14 +1226,14 @@ export class TwitterPostClient {
}

private async sendForApproval(
cleanedContent: string,
tweetTextForPosting: string,
roomId: UUID,
newTweetContent: string,
rawTweetContent: string,
): Promise<string | null> {
try {
const embed = {
title: "New Tweet Pending Approval",
description: cleanedContent,
description: tweetTextForPosting,
fields: [
{
name: "Character",
Expand All @@ -1232,7 +1242,7 @@ export class TwitterPostClient {
},
{
name: "Length",
value: cleanedContent.length.toString(),
value: tweetTextForPosting.length.toString(),
inline: true,
},
],
Expand Down Expand Up @@ -1260,9 +1270,9 @@ export class TwitterPostClient {
)) || [];
// Add new pending tweet
currentPendingTweets.push({
cleanedContent,
tweetTextForPosting,
roomId,
newTweetContent,
rawTweetContent,
discordMessageId: message.id,
channelId: this.discordApprovalChannelId,
timestamp: Date.now(),
Expand Down Expand Up @@ -1411,9 +1421,9 @@ export class TwitterPostClient {
await this.postTweet(
this.runtime,
this.client,
pendingTweet.cleanedContent,
pendingTweet.tweetTextForPosting,
pendingTweet.roomId,
pendingTweet.newTweetContent,
pendingTweet.rawTweetContent,
this.twitterUsername,
);

Expand Down
4 changes: 4 additions & 0 deletions packages/client-twitter/src/types.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
export type MediaData = {
data: Buffer;
mediaType: string;
};
65 changes: 34 additions & 31 deletions packages/client-twitter/src/utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import { elizaLogger } from "@elizaos/core";
import type { Media } from "@elizaos/core";
import fs from "fs";
import path from "path";
import { MediaData } from "./types";

export const wait = (minTime = 1000, maxTime = 3000) => {
const waitTime =
Expand Down Expand Up @@ -164,6 +165,36 @@ export async function buildConversationThread(
return thread;
}

export async function fetchMediaData(
attachments: Media[]
): Promise<MediaData[]> {
return Promise.all(
attachments.map(async (attachment: Media) => {
if (/^(http|https):\/\//.test(attachment.url)) {
// Handle HTTP URLs
const response = await fetch(attachment.url);
if (!response.ok) {
throw new Error(`Failed to fetch file: ${attachment.url}`);
}
const mediaBuffer = Buffer.from(await response.arrayBuffer());
const mediaType = attachment.contentType;
return { data: mediaBuffer, mediaType };
} else if (fs.existsSync(attachment.url)) {
// Handle local file paths
const mediaBuffer = await fs.promises.readFile(
path.resolve(attachment.url)
);
const mediaType = attachment.contentType;
return { data: mediaBuffer, mediaType };
} else {
throw new Error(
`File not found: ${attachment.url}. Make sure the path is correct.`
);
}
})
);
}

export async function sendTweet(
client: ClientBase,
content: Content,
Expand All @@ -179,38 +210,10 @@ export async function sendTweet(
let previousTweetId = inReplyTo;

for (const chunk of tweetChunks) {
let mediaData: { data: Buffer; mediaType: string }[] | undefined;
let mediaData = null;

if (content.attachments && content.attachments.length > 0) {
mediaData = await Promise.all(
content.attachments.map(async (attachment: Media) => {
if (/^(http|https):\/\//.test(attachment.url)) {
// Handle HTTP URLs
const response = await fetch(attachment.url);
if (!response.ok) {
throw new Error(
`Failed to fetch file: ${attachment.url}`
);
}
const mediaBuffer = Buffer.from(
await response.arrayBuffer()
);
const mediaType = attachment.contentType;
return { data: mediaBuffer, mediaType };
} else if (fs.existsSync(attachment.url)) {
// Handle local file paths
const mediaBuffer = await fs.promises.readFile(
path.resolve(attachment.url)
);
const mediaType = attachment.contentType;
return { data: mediaBuffer, mediaType };
} else {
throw new Error(
`File not found: ${attachment.url}. Make sure the path is correct.`
);
}
})
);
mediaData = await fetchMediaData(content.attachments);
}

const cleanChunk = deduplicateMentions(chunk.trim())
Expand Down Expand Up @@ -282,7 +285,7 @@ export async function sendTweet(
},
roomId,
embedding: getEmbeddingZeroVector(),
createdAt: tweet.timestamp * 1000,
createdAt: tweet.timestamp * 1000,
}));

return memories;
Expand Down

0 comments on commit 8536985

Please sign in to comment.