Extend social domains (#70)

* Update process_batch.py * Update process_batch.py * Update setup.py * Update spotting.py * Update spotting.py
exorde-labs · Nov 19, 2024 · 19d571c · 19d571c
1 parent f1244b2
commit 19d571c
Show file tree

Hide file tree

Showing 3 changed files with 46 additions and 4 deletions.
diff --git a/exorde/process_batch.py b/exorde/process_batch.py
@@ -191,8 +191,9 @@ def merge_chunks(chunks: list[ProcessedItem]) -> ProcessedItem:
         merged_item = None
     return merged_item
 
+
 SOCIAL_DOMAINS = [
-    "4chan",
+    "4chan.org",
     "4channel.org",
     "reddit.com",
     "twitter.com",
@@ -203,7 +204,6 @@ def merge_chunks(chunks: list[ProcessedItem]) -> ProcessedItem:
     "yt.co",
     "lemmy.world",
     "mastodon.social",
-    "mastodon",
     "weibo.com",
     "nostr.social",
     "nostr.com",
@@ -214,7 +214,45 @@ def merge_chunks(chunks: list[ProcessedItem]) -> ProcessedItem:
     "news.ycombinator.com",
     "tradingview.com",
     "followin.in",
-    "seekingalpha.io"
+    "seekingalpha.io",
+    "threads.net",
+    "telegram.org",
+    "tumblr.com",
+    "facebook.com",
+    "instagram.com",
+    "tiktok.com",
+    "whatsapp.com",
+    "9gag.com",
+    "techhaven.org",
+    "hey.xyz",
+    "dscvr.one",
+    "warpcast.com",
+    "discord.com",
+    "matrix.org",
+    "gab.com",
+    "parler.com",
+    "truthsocial.com",
+    "vk.com",
+    "vero.co",
+    "substack.com",
+    "8kun.top",
+    "ello.co",
+    "minds.com",
+    "mewe.com",
+    "livejournal.com",
+    "plurk.com",
+    "friendica.net",
+    "aminoapps.com",
+    "bitchute.com",
+    "gettr.com",
+    "odysee.com",
+    "poal.co",
+    "rumble.com",
+    "wimkin.com",
+    "twitch.tv",
+    "rocket.chat",
+    "snap.com",
+    "vidlii.com"
 ]
 
 def get_source_type(item: ProtocolItem) -> SourceType:

diff --git a/exorde/spotting.py b/exorde/spotting.py
@@ -63,6 +63,10 @@ async def _get_alias() -> dict[str, str]:
                     "tradingview.com": "tradingview",
                     "followin.in": "followin",
                     "seekingalpha.io": "seekingalpha",
+                    "threads.net":"threads",
+                    "bsky.app":"bluesky",
+                    "bsky.social":"bluesky",
+                    "bluesky.social":"bluesky"                    
                 }
             return json_data
 

diff --git a/setup.py b/setup.py
@@ -5,7 +5,7 @@
 
 setup(
     name="exorde",
-    version="v2.5.15",
+    version="v2.5.16",
     author="Exorde Labs",
     author_email="hello@exordelabs.com",
     description="The AI-based client to mine data and power the Exorde Network",