From 0bd52575380da34be2f70956e4075110fd2901fd Mon Sep 17 00:00:00 2001 From: benoit74 Date: Mon, 10 Jul 2023 09:11:17 +0200 Subject: [PATCH 1/5] Fix changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6730270..7d29bd9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,11 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## Fixed +- Fix issue with ePub rendering which was outside the iframe + ### Changed - Using zimscraperlib 2.1 - Updated image to `python:3.11-bullseye` - Retrying video reencoding up to three times +- Move inline javascript to dedicated files +- Move huge inline CSS to dedicated file +- Add `--node-ids` CLI parameter to process only few nodes (useful for debugging) ## [1.0.1] - 2023-02-22 From 61defd6ee12f951fe465996d23a0cb1414191820 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Tue, 11 Jul 2023 08:53:22 +0200 Subject: [PATCH 2/5] Upgrade Zimscraperlib to 3.1.0 --- CHANGELOG.md | 4 ++-- kolibri2zim/scraper.py | 37 +++++++++++++++++++++++++------------ requirements.txt | 4 ++-- 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d29bd9..6eb19cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,9 +12,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- Using zimscraperlib 2.1 +- Using zimscraperlib 3.1.0 - Updated image to `python:3.11-bullseye` -- Retrying video reencoding up to three times +- Retry video reencoding up to three times - Move inline javascript to dedicated files - Move huge inline CSS to dedicated file - Add `--node-ids` CLI parameter to process only few nodes (useful for debugging) diff --git a/kolibri2zim/scraper.py b/kolibri2zim/scraper.py index 287d73e..6588572 100644 --- a/kolibri2zim/scraper.py +++ b/kolibri2zim/scraper.py @@ -756,6 +756,9 @@ def run(self): f" tags: {';'.join(self.tags)}" ) + logger.info("Retrieving favicon") + self.retrieve_favicon() + logger.info("Setup Zim Creator") self.output_dir.mkdir(parents=True, exist_ok=True) @@ -764,15 +767,21 @@ def run(self): filename=self.output_dir.joinpath(self.fname), main_path=self.root_id, ignore_duplicates=True, - language="eng", - title=self.title, - description=self.description, - creator=self.author, - publisher=self.publisher, - name=self.name, - tags=";".join(self.tags), - date=datetime.date.today().strftime("%Y-%d-%m"), - ).start() + ) + with open(self.favicon_fpath, "rb") as fh: + favicon_bytes = fh.read() + self.creator.config_metadata( + Name=self.name, + Language="eng", + Title=self.title, + Description=self.description, + Creator=self.author, + Publisher=self.publisher, + Date=datetime.date.today().strftime("%Y-%d-%m"), + Illustration_48x48_at_1= favicon_bytes, + ) + del favicon_bytes + self.creator.start() succeeded = False try: @@ -901,7 +910,7 @@ def sanitize_inputs(self): self.tags = list(set(self.tags + ["_category:other", "kolibri", "_videos:yes"])) - def add_favicon(self): + def retrieve_favicon(self): favicon_orig = self.build_dir / "favicon" # if user provided a custom favicon, retrieve that if self.favicon: @@ -944,8 +953,12 @@ def add_favicon(self): favicon_ico_path = favicon_fpath.with_suffix(".ico") create_favicon(src=favicon_fpath, dst=favicon_ico_path) - self.creator.add_item_for("favicon.png", fpath=favicon_fpath) - self.creator.add_item_for("favicon.ico", fpath=favicon_ico_path) + self.favicon_fpath = favicon_fpath + self.favicon_ico_path = favicon_ico_path + + def add_favicon(self): + self.creator.add_item_for("favicon.png", fpath=self.favicon_fpath) + self.creator.add_item_for("favicon.ico", fpath=self.favicon_ico_path) def add_custom_about_and_css(self): channel_meta = self.db.get_channel_metadata(self.channel_id) diff --git a/requirements.txt b/requirements.txt index 4e1719f..3558586 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ -zimscraperlib>=2.1.0,<2.2 +zimscraperlib>=3.1.0,<3.2 kiwixstorage>=0.8.3,<0.9 jinja2>=3.1.2<3.2 pif==0.8.2 -beautifulsoup4>=4.9.0,<5.0 +beautifulsoup4>=4.9.3,<4.10 retrying>=1.3.4,<1.4 From 1549d026dc696ac23d30b96c387991c88d132818 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Thu, 13 Jul 2023 09:01:22 +0200 Subject: [PATCH 3/5] Pin all dependencies --- requirements.txt | 44 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3558586..ca82fad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,40 @@ -zimscraperlib>=3.1.0,<3.2 -kiwixstorage>=0.8.3,<0.9 -jinja2>=3.1.2<3.2 +## Direct dependencies +zimscraperlib==3.1.0 +kiwixstorage==0.8.3 +Jinja2==3.1.2 pif==0.8.2 -beautifulsoup4>=4.9.3,<4.10 -retrying>=1.3.4,<1.4 +beautifulsoup4==4.9.3 +retrying==1.3.4 + +## The following requirements were added by pip freeze: +aws-requests-auth==0.4.3 +Babel==2.12.1 +boto3==1.28.2 +botocore==1.31.2 +Brotli==1.0.9 +certifi==2023.5.7 +charset-normalizer==3.2.0 +colorthief==0.2.1 +file-magic==0.4.1 +idna==3.4 +iso-639==0.4.5 +jmespath==1.0.1 +libzim==3.1.0 +lxml==4.9.3 +MarkupSafe==2.1.3 +mutagen==1.46.0 +optimize-images==1.5.1 +piexif==1.1.3 +Pillow==10.0.0 +pycryptodomex==3.18.0 +python-dateutil==2.8.2 +python-resize-image==1.1.20 +pytz==2023.3 +requests==2.31.0 +s3transfer==0.6.1 +six==1.16.0 +soupsieve==2.4.1 +urllib3==1.26.16 +watchdog==3.0.0 +websockets==11.0.3 +yt-dlp==2023.7.6 \ No newline at end of file From 9cf1f95a60a1f5a98da42352868168c77a3393f0 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Thu, 13 Jul 2023 09:04:51 +0200 Subject: [PATCH 4/5] Simplify reading favicon bytes --- kolibri2zim/scraper.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/kolibri2zim/scraper.py b/kolibri2zim/scraper.py index 6588572..28d59c1 100644 --- a/kolibri2zim/scraper.py +++ b/kolibri2zim/scraper.py @@ -768,8 +768,6 @@ def run(self): main_path=self.root_id, ignore_duplicates=True, ) - with open(self.favicon_fpath, "rb") as fh: - favicon_bytes = fh.read() self.creator.config_metadata( Name=self.name, Language="eng", @@ -778,9 +776,8 @@ def run(self): Creator=self.author, Publisher=self.publisher, Date=datetime.date.today().strftime("%Y-%d-%m"), - Illustration_48x48_at_1= favicon_bytes, + Illustration_48x48_at_1=self.favicon_fpath.read_bytes(), ) - del favicon_bytes self.creator.start() succeeded = False From 4ee48102fe3c8ed16a566ba79525f6b1dca2dc7b Mon Sep 17 00:00:00 2001 From: benoit74 Date: Thu, 13 Jul 2023 21:35:50 +0200 Subject: [PATCH 5/5] Pin only direct dependencies --- requirements.txt | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/requirements.txt b/requirements.txt index ca82fad..fe0d483 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,36 +5,3 @@ Jinja2==3.1.2 pif==0.8.2 beautifulsoup4==4.9.3 retrying==1.3.4 - -## The following requirements were added by pip freeze: -aws-requests-auth==0.4.3 -Babel==2.12.1 -boto3==1.28.2 -botocore==1.31.2 -Brotli==1.0.9 -certifi==2023.5.7 -charset-normalizer==3.2.0 -colorthief==0.2.1 -file-magic==0.4.1 -idna==3.4 -iso-639==0.4.5 -jmespath==1.0.1 -libzim==3.1.0 -lxml==4.9.3 -MarkupSafe==2.1.3 -mutagen==1.46.0 -optimize-images==1.5.1 -piexif==1.1.3 -Pillow==10.0.0 -pycryptodomex==3.18.0 -python-dateutil==2.8.2 -python-resize-image==1.1.20 -pytz==2023.3 -requests==2.31.0 -s3transfer==0.6.1 -six==1.16.0 -soupsieve==2.4.1 -urllib3==1.26.16 -watchdog==3.0.0 -websockets==11.0.3 -yt-dlp==2023.7.6 \ No newline at end of file