+ content field (optional) (#61)

* + content field (optional)

* added translated and raw_content

---------

Co-authored-by: patrick borowy <patrick.borowy@protonmail.com>

Dockerfile

@@ -1,6 +1,5 @@
 FROM python:3.10.11
 
-
 # Update and install dependencies
 RUN apt-get update \
     && apt-get upgrade -y \

exorde/ipfs.py

@@ -13,7 +13,7 @@ class EnumEncoder(json.JSONEncoder):
     def default(self, obj):
         if isinstance(obj, Enum):
             return obj.name  # Serialize Enum value as its name
-        return super().default(obj)
+        return str(super().default(obj))
 
 
 async def upload_to_ipfs(

exorde/models.py

@@ -15,6 +15,8 @@
     ExternalId,
     ExternalParentId,
     Domain,
+    Content,
+    Item
 )
 from dataclasses import dataclass
 
@@ -200,6 +202,7 @@ class LiveConfiguration(dict):
 
 class Processed(dict, metaclass=MadType):
     translation: Translation
+    raw_content: Content
     top_keywords: Keywords
     classification: Classification
     item: Item
@@ -233,6 +236,8 @@ class ProtocolItem(dict, metaclass=MadType):
 
     created_at: CreatedAt
     title: Optional[Title]  # titre obligatoire si pas de contenu
+    raw_content: Optional[Content]
+    translated_content: Optional[Content]
     summary: Optional[Summary]  # <- description or summary available
     picture: Optional[Url]
     author: Optional[Author]

exorde/process.py

@@ -69,6 +69,7 @@ async def process(
             raise err
         return Processed(
             item=item,
+            raw_content=item.content,
             translation=translation,
             top_keywords=top_keywords,
             classification=classification,

exorde/process_batch.py

@@ -29,7 +29,7 @@
     Gender,
     Analysis
 )
-from exorde_data import Url
+from exorde_data import Url, Content
 
 from exorde.tag import tag
 from collections import Counter
@@ -236,6 +236,8 @@ async def process_batch(
     complete_processes: dict[int, list[ProcessedItem]] = {}
     for (id, processed), analysis in zip(batch, analysis_results):
         prot_item: ProtocolItem = ProtocolItem(
+            raw_content=Content(processed.raw_content),
+            translated_content=Content(processed.translation.translation),
             created_at=processed.item.created_at,
             domain=processed.item.domain,
             url=Url(processed.item.url),

exorde/spotting.py

@@ -9,10 +9,15 @@
 from exorde.spot_data import spot_data
 
 from exorde.get_transaction_receipt import get_transaction_receipt
-from exorde.ipfs import download_ipfs_file, upload_to_ipfs
+from exorde.ipfs import download_ipfs_file, upload_to_ipfs, EnumEncoder
 from exorde.models import LiveConfiguration, StaticConfiguration
 from exorde.counter import AsyncItemCounter
 
+import json
+import os
+import hashlib
+import random
+import string
 import json
 import logging
 import argparse
@@ -99,6 +104,22 @@ async def count_rep_for_each_domain(
         await counter.increment(f"rep_{alias}")
 
 
+def save_json_to_file(data, folder_path='output_folder'):
+    """Save JSON data to a file with a random hash filename in the specified folder."""
+    # Ensure the folder exists
+    if not os.path.exists(folder_path):
+        os.makedirs(folder_path)
+
+    # Generate a random hash for the filename
+    filename = f"{generate_random_hash()}.json"
+    file_path = os.path.join(folder_path, filename)
+
+    # Write JSON data to the file
+    with open(file_path, 'w') as file:
+        json.dump(data, file, indent=4, cls=EnumEncoder)
+
+    return file_path
+
 async def spotting(
     live_configuration: LiveConfiguration,
     static_configuration: StaticConfiguration,