Skip to content

Commit

Permalink
moves cache to own file
Browse files Browse the repository at this point in the history
  • Loading branch information
elliotBraem committed Dec 17, 2024
1 parent 3be1dfb commit 9ba5bdb
Show file tree
Hide file tree
Showing 2 changed files with 117 additions and 116 deletions.
130 changes: 14 additions & 116 deletions src/services/twitter/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,14 @@ import { TwitterSubmission, Moderation, TwitterConfig } from "../../types/twitte
import { ADMIN_ACCOUNTS } from "../../config/admins";
import { logger } from "../../utils/logger";
import { db } from "../db";

interface TwitterCookie {
key: string;
value: string;
domain: string;
path: string;
secure: boolean;
httpOnly: boolean;
sameSite?: string;
}

interface CookieCache {
[username: string]: TwitterCookie[];
}
import {
TwitterCookie,
ensureCacheDirectory,
getCachedCookies,
cacheCookies,
getLastCheckedTweetId,
saveLastCheckedTweetId
} from "../../utils/cache";

export class TwitterService {
private client: Scraper;
Expand All @@ -26,33 +20,12 @@ export class TwitterService {
private isInitialized = false;
private checkInterval: NodeJS.Timeout | null = null;
private lastCheckedTweetId: string | null = null;
private cacheDir: string;
private adminIdCache: Map<string, string> = new Map();

constructor(config: TwitterConfig) {
this.client = new Scraper();
this.twitterUsername = config.username;
this.config = config;
this.cacheDir = '.cache';
}

private async ensureCacheDirectory() {
try {
const fs = await import('fs/promises');
const path = await import('path');
const cacheDir = path.join(process.cwd(), this.cacheDir);

try {
await fs.access(cacheDir);
} catch {
// Directory doesn't exist, create it
await fs.mkdir(cacheDir, { recursive: true });
logger.info('Created cache directory');
}
} catch (error) {
logger.error('Failed to create cache directory:', error);
throw error;
}
}

private async setCookiesFromArray(cookiesArray: TwitterCookie[]) {
Expand All @@ -65,82 +38,6 @@ export class TwitterService {
await this.client.setCookies(cookieStrings);
}

private async getCachedCookies(username: string): Promise<TwitterCookie[] | null> {
try {
// Try to read cookies from a local cache file
const fs = await import('fs/promises');
const path = await import('path');
const cookiePath = path.join(process.cwd(), this.cacheDir, '.twitter-cookies.json');

const data = await fs.readFile(cookiePath, 'utf-8');
const cache: CookieCache = JSON.parse(data);

if (cache[username]) {
return cache[username];
}
} catch (error) {
// If file doesn't exist or is invalid, return null
return null;
}
return null;
}

private async cacheCookies(username: string, cookies: TwitterCookie[]) {
try {
const fs = await import('fs/promises');
const path = await import('path');
const cookiePath = path.join(process.cwd(), this.cacheDir, '.twitter-cookies.json');

let cache: CookieCache = {};
try {
const data = await fs.readFile(cookiePath, 'utf-8');
cache = JSON.parse(data);
} catch (error) {
// If file doesn't exist, start with empty cache
}

cache[username] = cookies;
await fs.writeFile(cookiePath, JSON.stringify(cache, null, 2));
} catch (error) {
logger.error('Failed to cache cookies:', error);
}
}

private async getLastCheckedTweetId(): Promise<string | null> {
try {
const fs = await import('fs/promises');
const path = await import('path');
const statePath = path.join(process.cwd(), this.cacheDir, '.twitter-state.json');

const data = await fs.readFile(statePath, 'utf-8');
const state = JSON.parse(data);
return state.lastCheckedTweetId || null;
} catch (error) {
return null;
}
}

private async saveLastCheckedTweetId(tweetId: string) {
try {
const fs = await import('fs/promises');
const path = await import('path');
const statePath = path.join(process.cwd(), this.cacheDir, '.twitter-state.json');

let state = { lastCheckedTweetId: tweetId };
try {
const data = await fs.readFile(statePath, 'utf-8');
state = { ...JSON.parse(data), lastCheckedTweetId: tweetId };
} catch (error) {
// If file doesn't exist, use the default state
}

await fs.writeFile(statePath, JSON.stringify(state, null, 2));
this.lastCheckedTweetId = tweetId;
} catch (error) {
logger.error('Failed to save last checked tweet ID:', error);
}
}

private async initializeAdminIds() {
for (const handle of ADMIN_ACCOUNTS) {
try {
Expand All @@ -160,16 +57,16 @@ export class TwitterService {
async initialize() {
try {
// Ensure cache directory exists
await this.ensureCacheDirectory();
await ensureCacheDirectory();

// Check for cached cookies
const cachedCookies = await this.getCachedCookies(this.twitterUsername);
const cachedCookies = await getCachedCookies(this.twitterUsername);
if (cachedCookies) {
await this.setCookiesFromArray(cachedCookies);
}

// Load last checked tweet ID
this.lastCheckedTweetId = await this.getLastCheckedTweetId();
this.lastCheckedTweetId = await getLastCheckedTweetId();

// Try to login with retries
logger.info('Attempting Twitter login...');
Expand All @@ -184,7 +81,7 @@ export class TwitterService {
if (await this.client.isLoggedIn()) {
// Cache the new cookies
const cookies = await this.client.getCookies();
await this.cacheCookies(this.config.username, cookies);
await cacheCookies(this.config.username, cookies);
break;
}
} catch (error) {
Expand Down Expand Up @@ -284,7 +181,8 @@ export class TwitterService {
// Update the last checked tweet ID to the most recent one
const latestTweetId = sortedTweets[sortedTweets.length - 1].id;
if (latestTweetId) {
await this.saveLastCheckedTweetId(latestTweetId);
await saveLastCheckedTweetId(latestTweetId);
this.lastCheckedTweetId = latestTweetId;
}
}
} else {
Expand Down
103 changes: 103 additions & 0 deletions src/utils/cache.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
import { logger } from "./logger";
import path from "path";
import fs from "fs/promises";

export interface TwitterCookie {
key: string;
value: string;
domain: string;
path: string;
secure: boolean;
httpOnly: boolean;
sameSite?: string;
}

interface CookieCache {
[username: string]: TwitterCookie[];
}

const CACHE_DIR = '.cache';

export async function ensureCacheDirectory() {
try {
const cacheDir = path.join(process.cwd(), CACHE_DIR);

try {
await fs.access(cacheDir);
} catch {
// Directory doesn't exist, create it
await fs.mkdir(cacheDir, { recursive: true });
logger.info('Created cache directory');
}
} catch (error) {
logger.error('Failed to create cache directory:', error);
throw error;
}
}

export async function getCachedCookies(username: string): Promise<TwitterCookie[] | null> {
try {
// Try to read cookies from a local cache file
const cookiePath = path.join(process.cwd(), CACHE_DIR, '.twitter-cookies.json');

const data = await fs.readFile(cookiePath, 'utf-8');
const cache: CookieCache = JSON.parse(data);

if (cache[username]) {
return cache[username];
}
} catch (error) {
// If file doesn't exist or is invalid, return null
return null;
}
return null;
}

export async function cacheCookies(username: string, cookies: TwitterCookie[]) {
try {
const cookiePath = path.join(process.cwd(), CACHE_DIR, '.twitter-cookies.json');

let cache: CookieCache = {};
try {
const data = await fs.readFile(cookiePath, 'utf-8');
cache = JSON.parse(data);
} catch (error) {
// If file doesn't exist, start with empty cache
}

cache[username] = cookies;
await fs.writeFile(cookiePath, JSON.stringify(cache, null, 2));
} catch (error) {
logger.error('Failed to cache cookies:', error);
}
}

export async function getLastCheckedTweetId(): Promise<string | null> {
try {
const statePath = path.join(process.cwd(), CACHE_DIR, '.twitter-state.json');

const data = await fs.readFile(statePath, 'utf-8');
const state = JSON.parse(data);
return state.lastCheckedTweetId || null;
} catch (error) {
return null;
}
}

export async function saveLastCheckedTweetId(tweetId: string) {
try {
const statePath = path.join(process.cwd(), CACHE_DIR, '.twitter-state.json');

let state = { lastCheckedTweetId: tweetId };
try {
const data = await fs.readFile(statePath, 'utf-8');
state = { ...JSON.parse(data), lastCheckedTweetId: tweetId };
} catch (error) {
// If file doesn't exist, use the default state
}

await fs.writeFile(statePath, JSON.stringify(state, null, 2));
} catch (error) {
logger.error('Failed to save last checked tweet ID:', error);
}
}

0 comments on commit 9ba5bdb

Please sign in to comment.