cleanup zomato

src/event-fetcher.py

@@ -202,6 +202,7 @@ def find_event(l):
                                 return (url, d)
 
                     data = JsonLdExtractor().extract(r.text)
+                    print(data)
                     m = None
                     for x in data:
                         if x.get("@graph"):

src/mmb.py

@@ -1,10 +1,9 @@
-from common.session import get_cached_session
 import json
 import datetime
-
+from requests import Session
 
 def fetch_events():
-    session = get_cached_session()
+    session = Session()
     payload = {
         "langId": "1",
         "filterData": json.dumps(
@@ -24,6 +23,5 @@ def fetch_events():
     for event in response.json()["eventItems"]:
         print(f"https://www.goethe.de/ins/in/en/ver.cfm?event_id={event['object_id']}")
 
-
 if __name__ == "__main__":
     fetch_events()

src/zomato.py

@@ -9,10 +9,15 @@
 from common import USER_AGENT_HEADERS
 import os
 
+EVENT_URL_REGEX = r"(https:\/\/www\.zomato\.com\/events\/[\w|-]+-et\d+)"
+
 # Public Key, not-logged-in API key
 ZOMATO_API_KEY = os.environ.get("ZOMATO_PUBLIC_API_KEY")
 BASE_URL = "https://zoma.to/live-event/"
-KNOWN_BAD_EVENTS = ["43612"]  # SkyJumper Trampoline Park, too long
+KNOWN_BAD_EVENTS = [
+    "43612", # SkyJumper Trampoline Park, too long
+    "44864", # Wonderla Amusement Park
+]
 
 
 def fix_date(date_str):
@@ -25,10 +30,10 @@ def get_events(session, event_id):
     url = f"{BASE_URL}{event_id}"
     session.browser = "chrome"
     r = session.get(url, cache=True)
-    if "window.location.replace" in r.text:
-        redirect_url = r.text.split('window.location.replace("')[1].split('")')[0]
-        r = session.get(redirect_url, cache=True)
-    # print(r.__class__)
+    # search for event_url_regex in r.text
+    if re.search(EVENT_URL_REGEX, r.text):
+        event_url = re.search(EVENT_URL_REGEX, r.text).group(1)
+        r = session.get(event_url, cache=True)
     from bs4 import BeautifulSoup
 
     soup = BeautifulSoup(r.text, "html.parser")
@@ -96,14 +101,14 @@ def get_event_ids(session):
     if "ZOMATO_PUBLIC_API_KEY" not in os.environ:
         raise Exception("ZOMATO_PUBLIC_API_KEY not set")
     events = []
-    limit = sys.argv[1] if len(sys.argv) > 1 else None
+    limit = int(sys.argv[1]) if len(sys.argv) > 1 else 1000
     session = Fetch(cache={"serializer": "json"})
     for event_id in sorted(get_event_ids(session)):
         for event in get_events(session, event_id):
             print(event["url"])
             events.append(event)
-            if len(events) == limit:
-                break
+        if len(events) >= limit:
+            break
 
     if len(events) == 0:
         import sys