Skip to content

Commit

Permalink
add missing extractor info when spawning new ones (fixes #1051)
Browse files Browse the repository at this point in the history
Not having this information causes the blacklist/whitelist logic to
trigger and prevents things from functioning as intended when using
default settings.

Fixes issues for 8muses, deviantart, exhentai, and mangoxo.
  • Loading branch information
mikf committed Oct 8, 2020
1 parent f9c1684 commit 3ebb174
Show file tree
Hide file tree
Showing 4 changed files with 18 additions and 12 deletions.
10 changes: 5 additions & 5 deletions gallery_dl/extractor/8muses.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,12 +94,12 @@ def items(self):
if albums:
for album in albums:
url = self.root + "/comics/album/" + album["permalink"]
album = {
"url" : url,
"name" : album["name"],
"private": album["isPrivate"],
yield Message.Queue, url, {
"url" : url,
"name" : album["name"],
"private" : album["isPrivate"],
"_extractor": _8musesAlbumExtractor,
}
yield Message.Queue, url, album

if data["page"] >= data["pages"]:
return
Expand Down
13 changes: 8 additions & 5 deletions gallery_dl/extractor/deviantart.py
Original file line number Diff line number Diff line change
Expand Up @@ -262,9 +262,11 @@ def _find_folder(folders, name):
return folder
raise exception.NotFoundError("folder")

def _folder_urls(self, folders, category):
url = "{}/{}/{}/0/".format(self.root, self.user, category)
return [(url + folder["name"], folder) for folder in folders]
def _folder_urls(self, folders, category, extractor):
base = "{}/{}/{}/0/".format(self.root, self.user, category)
for folder in folders:
folder["_extractor"] = extractor
yield base + folder["name"], folder

def _update_content_default(self, deviation, content):
public = "premium_folder_data" not in deviation
Expand Down Expand Up @@ -450,7 +452,7 @@ def deviations(self):
if self.flat and not self.group:
return self.api.gallery_all(self.user, self.offset)
folders = self.api.gallery_folders(self.user)
return self._folder_urls(folders, "gallery")
return self._folder_urls(folders, "gallery", DeviantartFolderExtractor)


class DeviantartFolderExtractor(DeviantartExtractor):
Expand Down Expand Up @@ -589,7 +591,8 @@ def deviations(self):
self.api.collections(self.user, folder["folderid"])
for folder in folders
)
return self._folder_urls(folders, "favourites")
return self._folder_urls(
folders, "favourites", DeviantartCollectionExtractor)


class DeviantartCollectionExtractor(DeviantartExtractor):
Expand Down
3 changes: 2 additions & 1 deletion gallery_dl/extractor/exhentai.py
Original file line number Diff line number Diff line change
Expand Up @@ -392,6 +392,7 @@ def __init__(self, match):
def items(self):
self.login()
yield Message.Version, 1
data = {"_extractor": ExhentaiGalleryExtractor}

while True:
last = None
Expand All @@ -402,7 +403,7 @@ def items(self):
if url == last:
continue
last = url
yield Message.Queue, url, {}
yield Message.Queue, url, data

if 'class="ptdd">&gt;<' in page or ">No hits found</p>" in page:
return
Expand Down
4 changes: 3 additions & 1 deletion gallery_dl/extractor/mangoxo.py
Original file line number Diff line number Diff line change
Expand Up @@ -167,14 +167,16 @@ def items(self):
self.login()
num = total = 1
url = "{}/channel/{}/album/".format(self.root, self.channel_id)
data = {"_extractor": MangoxoAlbumExtractor}

yield Message.Version, 1

while True:
page = self.request(url + str(num)).text

for album in text.extract_iter(
page, '<a class="link black" href="', '"'):
yield Message.Queue, album, {}
yield Message.Queue, album, data

if num == 1:
total = self._total_pages(page)
Expand Down

0 comments on commit 3ebb174

Please sign in to comment.