fixup! Add retry logic with detailled logs to extraction of video dat…

…a from HTML page
openzim · Jul 10, 2024 · 669da9b · 669da9b
1 parent 2829651
commit 669da9b
Showing 1 changed file with 3 additions and 17 deletions.
diff --git a/src/ted2zim/scraper.py b/src/ted2zim/scraper.py
@@ -840,23 +840,9 @@ def extract_info_from_video_page(
                     url, retry_count=retry_count + 1
                 )
 
-            json_data = json.loads(next_data_tag.string)
-
-            if (
-                "props" not in json_data
-                or "pageProps" not in json_data["props"]
-                or "videoData" not in json_data["props"]["pageProps"]
-            ):
-                logger.debug(
-                    "Inproper data returned by server in __NEXT_DATA__ JSON. Retrying "
-                    "in 5 seconds..."
-                )
-                time.sleep(5)
-                return self.extract_info_from_video_page(
-                    url, retry_count=retry_count + 1
-                )
-
-            json_data = json_data["props"]["pageProps"]["videoData"]
+            json_data = json.loads(next_data_tag.string)["props"]["pageProps"][
+                "videoData"
+            ]
 
             requested_lang_code = self.get_lang_code_from_url(url)
             if requested_lang_code and json_data["language"] != requested_lang_code: