[WIP] Added Rettiwt API package

ghost/core/core/server/services/oembed/RettiwtOEmbedProvider.js

@@ -0,0 +1,140 @@
+const logging = require('@tryghost/logging');
+
+/**
+ * @typedef {import('./oembed').ICustomProvider} ICustomProvider
+ * @typedef {import('./oembed').IExternalRequest} IExternalRequest
+ */
+
+const TWITTER_PATH_REGEX = /\/status\/(\d+)/;
+
+function mapTweetEntity(tweet) {
+    return {
+        id: tweet?.id,
+        created_at: new Date(tweet?.createdAt),
+        text: tweet?.fullText,
+        public_metrics: {
+            retweet_count: tweet?.retweetCount || 0,
+            like_count: tweet?.likeCount || 0,
+            reply_count: tweet?.replyCount || 0,
+            view_count: tweet?.viewCount || 0
+        },
+        author_id: tweet?.tweetBy?.id,
+        entities: {
+            mentions: (tweet?.entities?.mentionedUsers || []).map(user => ({
+                start: 0, // Update with actual start index if available
+                end: 0, // Update with actual end index if available
+                username: user?.userName
+            })),
+            hashtags: (tweet?.entities?.hashtags || []).map(hashtag => ({
+                start: 0, // Update with actual start index if available
+                end: 0, // Update with actual end index if available
+                tag: hashtag?.tag || hashtag
+            })),
+            urls: (tweet?.entities?.urls || []).map(url => ({
+                start: 0, // Update with actual start index if available
+                end: 0, // Update with actual end index if available
+                url: url?.url,
+                display_url: url?.displayUrl || url?.url,
+                expanded_url: url?.expandedUrl || url?.url
+            }))
+        },
+        users: [
+            {
+                id: tweet?.tweetBy?.id,
+                name: tweet?.tweetBy?.fullName,
+                username: tweet?.tweetBy?.userName,
+                profile_image_url: tweet?.tweetBy?.profileImage,
+                description: tweet?.tweetBy?.description,
+                verified: tweet?.tweetBy?.isVerified,
+                location: tweet?.tweetBy?.location
+            }
+        ],
+        attachments: {
+            media_keys: tweet?.media ? tweet?.media.map((_, index) => `media_${index + 1}`) : []
+        },
+        includes: {
+            media: (tweet?.media || []).map((media, index) => ({
+                media_key: `media_${index + 1}`,
+                type: media?.type,
+                url: media?.url,
+                preview_image_url: media?.previewUrl || media?.url
+            }))
+        }
+    };
+}
+
+/**
+ * @implements ICustomProvider
+ */
+
+class RettiwtOEmbedProvider {
+    /**
+     * @param {object} dependencies
+     */
+    constructor(dependencies) {
+        this.dependencies = dependencies;
+    }
+
+    /**
+     * @param {URL} url
+     * @returns {Promise<boolean>}
+     */
+    async canSupportRequest(url) {
+        return (url.host === 'twitter.com' || url.host === 'x.com') && TWITTER_PATH_REGEX.test(url.pathname);
+    }
+
+    /**
+     * @param {URL} url
+     *
+     * @returns {Promise<object>}
+     */
+    async getOEmbedData(url) {
+        if (url.host === 'x.com') { // api is still at twitter.com... also not certain how people are getting x urls because twitter currently redirects every x host to twitter
+            url = new URL('https://twitter.com' + url.pathname);
+        }
+
+        const [match, tweetId] = url.pathname.match(TWITTER_PATH_REGEX);
+        if (!match) {
+            return null;
+        }
+
+        const {extract} = require('@extractus/oembed-extractor');
+
+        /** @type {object} */
+        const oembedData = await extract(url.href);
+        const query = {
+            expansions: ['attachments.poll_ids', 'attachments.media_keys', 'author_id', 'entities.mentions.username', 'geo.place_id', 'in_reply_to_user_id', 'referenced_tweets.id', 'referenced_tweets.id.author_id'],
+            'media.fields': ['duration_ms', 'height', 'media_key', 'preview_image_url', 'type', 'url', 'width', 'public_metrics', 'alt_text'],
+            'place.fields': ['contained_within', 'country', 'country_code', 'full_name', 'geo', 'id', 'name', 'place_type'],
+            'poll.fields': ['duration_minutes', 'end_datetime', 'id', 'options', 'voting_status'],
+            'tweet.fields': ['attachments', 'author_id', 'context_annotations', 'conversation_id', 'created_at', 'entities', 'geo', 'id', 'in_reply_to_user_id', 'lang', 'public_metrics', 'possibly_sensitive', 'referenced_tweets', 'reply_settings', 'source', 'text', 'withheld'],
+            'user.fields': ['created_at', 'description', 'entities', 'id', 'location', 'name', 'pinned_tweet_id', 'profile_image_url', 'protected', 'public_metrics', 'url', 'username', 'verified', 'withheld']
+        };
+
+        const queryString = Object.keys(query).map((key) => {
+            return `${key}=${query[key].join(',')}`;
+        }).join('&');
+
+        try {
+            // const tweet = await .request(EResourceType.TWEET_DETAILS, {id: tweetId, query: queryString});
+            const tweet = await this.dependencies.externalRequest.tweet.details(tweetId, queryString);
+            console.log(tweet);
+            oembedData.tweet_data = mapTweetEntity(tweet);
+        } catch (err) {
+            if (err.response?.body) {
+                try {
+                    const parsed = JSON.parse(err.response.body);
+                    err.context = parsed;
+                } catch (e) {
+                    err.context = err.response.body;
+                }
+            }
+            logging.error(err);
+        }
+
+        oembedData.type = 'twitter';
+        return oembedData;
+    }
+}
+
+module.exports = RettiwtOEmbedProvider;

ghost/core/core/server/services/oembed/service.js

@@ -1,7 +1,9 @@
 const config = require('../../../shared/config');
 const storage = require('../../adapters/storage');
 const externalRequest = require('../../lib/request-external');
-
+// const {FetcherService, EResourceType} = require('rettiwt-api');
+const {Rettiwt} = require('rettiwt-api');
+const {XEmbedProvider} = require('@tryghost/x-embed-provider');
 const OEmbed = require('@tryghost/oembed-service');
 const oembed = new OEmbed({config, externalRequest, storage});
 
@@ -12,10 +14,20 @@ const nft = new NFT({
     }
 });
 
-const Twitter = require('./TwitterOEmbedProvider');
-const twitter = new Twitter({
-    config: {
-        bearerToken: config.get('twitter').privateReadOnlyToken
+// const Twitter = require('./TwitterOEmbedProvider');
+// const twitter = new Twitter({
+//     config: {
+//         bearerToken: config.get('twitter').privateReadOnlyToken
+//     }
+// });
+
+const rettiwt = new Rettiwt({apiKey: config.get('twitter').privateReadOnlyToken || ''});
+
+// Wrapping FetcherService to conform to the expected dependency structure
+const twitter = new XEmbedProvider({
+    _fetcher: async (tweetId) => {
+        const response = await rettiwt.tweet.details(tweetId);
+        return response;
     }
 });
 

ghost/core/package.json

@@ -165,6 +165,7 @@
     "@tryghost/verification-trigger": "0.0.0",
     "@tryghost/version": "0.1.30",
     "@tryghost/webmentions": "0.0.0",
+    "@tryghost/x-embed-provider": "0.0.0",
     "@tryghost/zip": "1.1.46",
     "amperize": "0.6.1",
     "body-parser": "1.20.3",
@@ -215,6 +216,7 @@
     "node-jose": "2.2.0",
     "path-match": "1.2.4",
     "probe-image-size": "7.2.3",
+    "rettiwt-api": "4.1.4",
     "rss": "1.2.2",
     "sanitize-html": "2.13.1",
     "semver": "7.6.3",

ghost/oembed-service/lib/OEmbedService.js

@@ -65,7 +65,7 @@ const findUrlWithProvider = (url) => {
 /**
  * @typedef {object} ICustomProvider
  * @prop {(url: URL) => Promise<boolean>} canSupportRequest
- * @prop {(url: URL, externalRequest: IExternalRequest) => Promise<import('@extractus/oembed-extractor').OembedData>} getOEmbedData
+ * @prop {(url: URL, externalRequest: IExternalRequest) => Promise<import('@extractus/oembed-extractor').OembedData>} [getOEmbedData]
  */
 
 class OEmbedService {

ghost/x-embed-provider/.eslintrc.js

@@ -0,0 +1,7 @@
+module.exports = {
+    parser: '@typescript-eslint/parser',
+    plugins: ['ghost'],
+    extends: [
+        'plugin:ghost/node'
+    ]
+};

ghost/x-embed-provider/README.md

@@ -0,0 +1,23 @@
+# X Embed Provider
+
+Embed Provider for Twitter / X
+
+
+## Usage
+
+
+## Develop
+
+This is a monorepo package.
+
+Follow the instructions for the top-level repo.
+1. `git clone` this repo & `cd` into it as usual
+2. Run `yarn` to install top-level dependencies.
+
+
+
+## Test
+
+- `yarn lint` run just eslint
+- `yarn test` run lint and tests
+

ghost/x-embed-provider/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "@tryghost/x-embed-provider",
+  "version": "0.0.0",
+  "repository": "https://github.com/TryGhost/Ghost/tree/main/packages/x-embed-provider",
+  "author": "Ghost Foundation",
+  "private": true,
+  "main": "build/index.js",
+  "types": "build/index.d.ts",
+  "scripts": {
+    "dev": "tsc --watch --preserveWatchOutput --sourceMap",
+    "build": "tsc",
+    "build:ts": "tsc",
+    "prepare": "tsc",
+    "test:unit": "NODE_ENV=testing c8 --src src --all --check-coverage --100 --reporter text --reporter cobertura mocha -r ts-node/register './test/**/*.test.ts'",
+    "test": "yarn test:types && yarn test:unit",
+    "test:types": "tsc --noEmit",
+    "lint:code": "eslint src/ --ext .ts --cache",
+    "lint": "yarn lint:code && yarn lint:test",
+    "lint:test": "eslint -c test/.eslintrc.js test/ --ext .ts --cache"
+  },
+  "files": [
+    "build"
+  ],
+  "devDependencies": {
+    "c8": "10.1.3",
+    "mocha": "11.0.1",
+    "sinon": "19.0.2",
+    "ts-node": "10.9.2",
+    "typescript": "5.7.2"
+  }
+}