From 78c3467686c38ed92d1d715d9f4cf1c5c770a9c2 Mon Sep 17 00:00:00 2001 From: missionfloyd Date: Wed, 18 Sep 2024 23:27:27 -0600 Subject: [PATCH] Add pagination Add tests --- gallery_dl/extractor/girlsreleased.py | 51 ++++++++++++-------------- test/results/girlsreleased.py | 53 +++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 28 deletions(-) create mode 100644 test/results/girlsreleased.py diff --git a/gallery_dl/extractor/girlsreleased.py b/gallery_dl/extractor/girlsreleased.py index 59b36055f46..f367b69dca6 100644 --- a/gallery_dl/extractor/girlsreleased.py +++ b/gallery_dl/extractor/girlsreleased.py @@ -21,10 +21,9 @@ class GirlsreleasedExtractor(Extractor): request_interval = 0.5 request_interval_min = 0.2 - def _init(self): - domain = self.config("domain") - if domain: - self.root = text.ensure_http_scheme(domain) + def __init__(self, match): + Extractor.__init__(self, match) + self.id = match.group(1) def _images(self, json): data = { @@ -37,14 +36,24 @@ def _images(self, json): for image in json["images"]: yield Message.Queue, image[3], data + def _pagination(self, url): + sets = [] + page = 0 + while True: + json = self.request(f"{url}/page/{page}").json()["sets"] + if not json: + return sets + sets += json[1:] + page += 1 + def items(self): - posts = self.posts() + sets = self.sets() - if "images" in posts: - yield from self._images(posts) + if "images" in sets: + yield from self._images(sets) else: - for gallery in posts: - url = "{}/set/{}".format(self.root, gallery[0]) + for set in sets: + url = "{}/set/{}".format(self.root, set[0]) yield from self._images(self.request(url).json()["set"]) @@ -54,11 +63,7 @@ class GirlsreleasedSetExtractor(GirlsreleasedExtractor): pattern = BASE_PATTERN + r"/set/(\d+)" example = "https://girlsreleased.com/set/12345" - def __init__(self, match): - GirlsreleasedExtractor.__init__(self, match) - self.id = match.group(1) - - def posts(self): + def sets(self): url = "{}/set/{}".format(self.root, self.id) return self.request(url).json()["set"] @@ -69,13 +74,8 @@ class GirlsreleasedModelExtractor(GirlsreleasedExtractor): pattern = BASE_PATTERN + r"/model/(\d+(?:/?.+)?)" example = "https://girlsreleased.com/model/12345/MODEL" - def __init__(self, match): - GirlsreleasedExtractor.__init__(self, match) - self.id = match.group(1) - - def posts(self): - url = "{}/sets/model/{}".format(self.root, self.id) - return self.request(url).json()["sets"] + def sets(self): + return self._pagination(f"{self.root}/sets/model/{self.id}") class GirlsreleasedSiteExtractor(GirlsreleasedExtractor): @@ -84,10 +84,5 @@ class GirlsreleasedSiteExtractor(GirlsreleasedExtractor): pattern = BASE_PATTERN + r"/site/(.+(?:/model/\d+(?:/?.+)?)?)" example = "https://girlsreleased.com/site/SITE" - def __init__(self, match): - GirlsreleasedExtractor.__init__(self, match) - self.id = match.group(1) - - def posts(self): - url = "{}/sets/site/{}".format(self.root, self.id) - return self.request(url).json()["sets"] + def sets(self): + return self._pagination(f"{self.root}/sets/site/{self.id}") diff --git a/test/results/girlsreleased.py b/test/results/girlsreleased.py new file mode 100644 index 00000000000..f3f90c4845e --- /dev/null +++ b/test/results/girlsreleased.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import girlsreleased + + +__tests__ = ( +{ + "#url" : "https://girlsreleased.com/set/32332", + "#category": ("", "girlsreleased", "set"), + "#class" : girlsreleased.GirlsreleasedSetExtractor, + "#count" : 122, + + "id" : "32332", + "title" : "Monadiko", + "model" : "Mia Sollis", + "site" : "sexart.com" +}, + +{ + "#url" : "https://girlsreleased.com/set/124943", + "#category": ("", "girlsreleased", "set"), + "#class" : girlsreleased.GirlsreleasedSetExtractor, + "#count" : 79, + + "id" : "124943", + "title" : "124943", + "model" : "Iveta", + "site" : "errotica-archives.com" +}, + +{ + "#url" : "https://girlsreleased.com/model/11545/Ariana%20Regent", + "#category": ("", "girlsreleased", "model"), + "#class" : girlsreleased.GirlsreleasedModelExtractor, +}, + +{ + "#url" : "https://girlsreleased.com/site/amourangels.com", + "#category": ("", "girlsreleased", "model"), + "#class" : girlsreleased.GirlsreleasedSiteExtractor, +}, + +{ + "#url" : "https://girlsreleased.com/site/femjoy.com/model/854/Gabi", + "#category": ("", "girlsreleased", "model"), + "#class" : girlsreleased.GirlsreleasedSiteExtractor, +}, + +)