server.ts

import OAuth from "oauth";
import "dotenv/config";
import { setTimeout } from "timers/promises";

const { API_KEY, API_SECRET, BEARER_TOKEN, ACCESS_TOKEN, ACCESS_TOKEN_SECRET } = process.env;

function getAuthHeader(oauth: OAuth.OAuth, url: string) {
	return oauth.authHeader(url, ACCESS_TOKEN as string, ACCESS_TOKEN_SECRET as string, "post");
}

const client = new OAuth.OAuth(
	"https://api.twitter.com/oauth/request_token",
	"https://api.twitter.com/oauth/access_token",
	API_KEY as string,
	API_SECRET as string,
	"1.0A",
	null,
	"HMAC-SHA1"
);

const BOT_ID = "1612094318906417152";

interface TweetMentions {
	data: Array<{ id: string; text: string }>;
	meta: {
		result_count: number;
		newest_id: string;
		oldest_id: string;
	};
}

interface TweetLookups {
	data: Array<{
		id: string;
		conversation_id: "string";
		text: string;
		created_at: string;
		attachments?: { media_keys: string[] };
	}>;
	includes: { media: Array<{ media_key: string; url: string }> };
}

async function ff(url: string) {
	const resp = await fetch(`https://api.twitter.com/2/${url}`, {
		headers: { Authorization: `Bearer ${BEARER_TOKEN}` },
	});

	if (resp.status !== 200) {
		throw new Error("invalid status: " + resp.status + "- " + (await resp.text()));
	}

	return await resp.json();
}

let lastMention = "";

async function lookupTweets() {
	const data: TweetMentions = await ff(
		`users/${BOT_ID}/mentions?${lastMention && `start_time=${new Date(new Date(lastMention).getTime() + 1).toJSON()}`}`
	);

	let lookups: TweetLookups = await ff(
		`tweets?ids=${data.data
			.map((t) => t.id)
			.join(
				","
			)}&tweet.fields=created_at&expansions=attachments.media_keys&media.fields=duration_ms,height,media_key,preview_image_url,public_metrics,type,url,width,alt_text`
	);

	if (!lastMention) {
		console.log("added mention", lookups.data[0].created_at);
		lastMention = lookups.data[0].created_at;
		return lookupTweets();
	}
	const tweets = lookups.data.filter((tweet) => tweet.attachments?.media_keys.length === 1);
	console.log(lastMention);

	for (const tweet of tweets) {
		const imageUrl = lookups.includes.media.find((media) => media.media_key === tweet?.attachments!.media_keys[0])
			?.url!;

		console.log("imageUrl", imageUrl);

		const imageResp = await fetch(imageUrl);
		const contentType = imageResp.headers.get("Content-Type");
		const image = await imageResp.arrayBuffer();

		console.log(contentType, image);

		const altText = await fetch("https://olivierdehaene-git-large-coco.hf.space/run/predict", {
			method: "POST",
			headers: { "Content-Type": "application/json" },
			body: JSON.stringify({
				data: [`data:${contentType};base64,${Buffer.from(image).toString("base64")}`],
			}),
		})
			.then((r) => r.json())
			.then((r) => r.data);

		console.log(altText[0]);

		const header = getAuthHeader(client, "https://api.twitter.com/2/tweets");

		const r = await fetch("https://api.twitter.com/2/tweets", {
			headers: {
				Authorization: header,
				"user-agent": "v3CreateTweetJS",
				"content-type": "application/json",
				accept: "application/json",
			},
			body: JSON.stringify({
				text: altText[0],
				reply: { in_reply_to_tweet_id: tweet!.id, conversation_id: tweet!.conversation_id },
			}),
			method: "post",
		});

		try {
			console.log("end", await r.json());
		} catch {}
	}
}

process.on("unhandledRejection", async (err) => {
	console.error("unhandled rejection", err);
});

async function run() {
	while (1) {
		console.log("looking up");
		await lookupTweets();
		await setTimeout(5_000);
	}
}

run();