From dc4d5775f438c686fc010c32d6e8b89512115f7b Mon Sep 17 00:00:00 2001 From: BishopRed Date: Tue, 14 Jan 2025 16:02:58 -0600 Subject: [PATCH] [kemonoparty] Support /posts endpoint and Creator Tag Calls - Adding support for calling a creator with a tag selected. It is using a legacy endpoint but there is no other way currently documented to get the users post filtered by a tag. - Fixing the User Tags feature to be paginated offset is not defined in the API but it is supported. - Fixed the `/posts` endpoint not working: 1. Added check along with metadata to make sure there is a creator/service information as that is a requirement 2. Fixed the parameter from tags -> tag. 3. Fixed the _paginate call to exit correctly when there is a key required for the data (it was prematurely exiting) - Adding a type of caching mechanism for the metadata/user information. The current logic would work just fine if looking up for a singular user, however for the multiple posts via normal filtering would cause it to either: This builds a local cache during the process so it should only make a call for the user info once during the process. - Updating to meet standards Fixes 1. Reset formatting for unnecessary line changes 2. Removed Type Hinting 3.Replaced f-string with "".format Updates Renamed function creator_posts_tags -> creator_tagged_posts for clarity of what it does (get posts tags vs get tagged posts) - Fixing check for the length of response: 1. If it is list - just check len 2. If there is a key - check that the key length is less than the batch. - add test for '?tag=...' user URLs plus some code simplifications --- gallery_dl/extractor/kemonoparty.py | 51 ++++++++++++++++++----------- test/results/kemonoparty.py | 17 +++++++++- 2 files changed, 48 insertions(+), 20 deletions(-) diff --git a/gallery_dl/extractor/kemonoparty.py b/gallery_dl/extractor/kemonoparty.py index 66bbab573c7..788b5d94449 100644 --- a/gallery_dl/extractor/kemonoparty.py +++ b/gallery_dl/extractor/kemonoparty.py @@ -54,26 +54,19 @@ def _init(self): sort_keys=True, separators=(",", ":")).encode def items(self): - service = self.groups[2] - creator_id = self.groups[3] - find_hash = re.compile(HASH_PATTERN).match generators = self._build_file_generators(self.config("files")) announcements = True if self.config("announcements") else None comments = True if self.config("comments") else False duplicates = True if self.config("duplicates") else False dms = True if self.config("dms") else None - profile = username = None + max_posts = self.config("max-posts") + creator_info = {} if self.config("metadata") else None # prevent files from being sent with gzip compression headers = {"Accept-Encoding": "identity"} - if self.config("metadata"): - profile = self.api.creator_profile(service, creator_id) - username = profile["name"] - posts = self.posts() - max_posts = self.config("max-posts") if max_posts: posts = itertools.islice(posts, max_posts) if self.revisions: @@ -85,10 +78,20 @@ def items(self): post["_http_headers"] = headers post["date"] = self._parse_datetime( post.get("published") or post.get("added") or "") + service = post["service"] + creator_id = post["user"] + + if creator_info is not None: + key = "{}_{}".format(service, creator_id) + if key not in creator_info: + creator = creator_info[key] = self.api.creator_profile( + service, creator_id) + else: + creator = creator_info[key] + + post["user_profile"] = creator + post["username"] = creator["name"] - if profile is not None: - post["username"] = username - post["user_profile"] = profile if comments: try: post["comments"] = self.api.creator_post_comments( @@ -171,7 +174,7 @@ def _login_impl(self, username, password): try: msg = '"' + response.json()["error"] + '"' except Exception: - msg = '"0/1 Username or password is incorrect"' + msg = '"Username or password is incorrect"' raise exception.AuthenticationError(msg) return {c.name: c.value for c in response.cookies} @@ -296,8 +299,12 @@ def __init__(self, match): def posts(self): _, _, service, creator_id, query = self.groups params = text.parse_query(query) - return self.api.creator_posts( - service, creator_id, params.get("o"), params.get("q")) + if params.get("tag"): + return self.api.creator_tagged_posts( + service, creator_id, params.get("tag"), params.get("o")) + else: + return self.api.creator_posts( + service, creator_id, params.get("o"), params.get("q")) class KemonopartyPostsExtractor(KemonopartyExtractor): @@ -493,7 +500,7 @@ def __init__(self, extractor): def posts(self, offset=0, query=None, tags=None): endpoint = "/posts" - params = {"q": query, "o": offset, "tags": tags} + params = {"q": query, "o": offset, "tag": tags} return self._pagination(endpoint, params, 50, "posts") def creator_posts(self, service, creator_id, offset=0, query=None): @@ -501,6 +508,11 @@ def creator_posts(self, service, creator_id, offset=0, query=None): params = {"q": query, "o": offset} return self._pagination(endpoint, params, 50) + def creator_tagged_posts(self, service, creator_id, tags, offset=0): + endpoint = "/{}/user/{}/posts-legacy".format(service, creator_id) + params = {"o": offset, "tag": tags} + return self._pagination(endpoint, params, 50, "results") + def creator_announcements(self, service, creator_id): endpoint = "/{}/user/{}/announcements".format(service, creator_id) return self._call(endpoint) @@ -565,9 +577,10 @@ def _pagination(self, endpoint, params, batch=50, key=False): data = self._call(endpoint, params) if key: - yield from data[key] - else: - yield from data + data = data.get(key) + if not data: + return + yield from data if len(data) < batch: return diff --git a/test/results/kemonoparty.py b/test/results/kemonoparty.py index ccf9d072210..846f16092f2 100644 --- a/test/results/kemonoparty.py +++ b/test/results/kemonoparty.py @@ -39,6 +39,21 @@ "id": "8779", }, +{ + "#url" : "https://kemono.su/patreon/user/3161935?tag=pin-up", + "#comment" : "'tag' query parameter", + "#category": ("", "kemonoparty", "fanbox"), + "#class" : kemonoparty.KemonopartyUserExtractor, + "#urls" : ( + "https://kemono.su/data/03/e6/03e62592c3b616b8906c1aaa130bd9ceaa24d7f601b31f90cc11956a57ca1d82.png", + "https://kemono.su/data/6a/9b/6a9b6d93dcb86c24a48def1bb93ce2a9ad77393941f3469d87d39400433cf825.png", + "https://kemono.su/data/2a/b8/2ab8ba30644249e9516afaea05d61c0de14591cb9d232a2dc249650eb1a9a759.jpg", + "https://kemono.su/data/b0/38/b03882c8b0ab3b1cf9fc658a2bb2f9ac6ad4f3449015311dcd2d7ee7f748db31.png", + ), + + "tags": r"\bpin-up\b", +}, + { "#url" : "https://kemono.su/subscribestar/user/alcorart", "#category": ("", "kemonoparty", "subscribestar"), @@ -379,7 +394,7 @@ "#category": ("", "kemonoparty", "discord-server"), "#class" : kemonoparty.KemonopartyDiscordServerExtractor, "#pattern" : kemonoparty.KemonopartyDiscordExtractor.pattern, - "#count" : 15, + "#count" : 26, }, {