From d3f26ecbc925a09a09cda19e97ecb0294b6fc037 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sat, 6 Mar 2021 15:04:44 +0000 Subject: [PATCH 01/20] Move the page margin config to frontend --- public/js/reader.js | 12 +++++++++++- src/config.cr | 1 - src/routes/api.cr | 2 -- src/views/reader.html.ecr | 10 +++++++++- 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/public/js/reader.js b/public/js/reader.js index 1aa1bd13..f2279728 100644 --- a/public/js/reader.js +++ b/public/js/reader.js @@ -10,6 +10,7 @@ const readerComponent = () => { longPages: false, lastSavedPage: page, selectedIndex: 0, // 0: not selected; 1: the first page + margin: 30, /** * Initialize the component by fetching the page dimensions @@ -27,7 +28,6 @@ const readerComponent = () => { url: `${base_url}api/page/${tid}/${eid}/${i+1}`, width: d.width, height: d.height, - style: `margin-top: ${data.margin}px; margin-bottom: ${data.margin}px;` }; }); @@ -47,6 +47,11 @@ const readerComponent = () => { const mode = this.mode; this.updateMode(this.mode, page, nextTick); $('#mode-select').val(mode); + + const savedMargin = localStorage.getItem('margin'); + if (savedMargin) { + this.margin = savedMargin; + } }) .catch(e => { const errMsg = `Failed to get the page dimensions. ${e}`; @@ -277,6 +282,11 @@ const readerComponent = () => { entryChanged() { const id = $('#entry-select').val(); this.redirect(`${base_url}reader/${tid}/${id}`); + }, + + marginChanged() { + localStorage.setItem('margin', this.margin); + this.toPage(this.selectedIndex); } }; } diff --git a/src/config.cr b/src/config.cr index abacbb3f..dc7e35c3 100644 --- a/src/config.cr +++ b/src/config.cr @@ -20,7 +20,6 @@ class Config property plugin_path : String = File.expand_path "~/mango/plugins", home: true property download_timeout_seconds : Int32 = 30 - property page_margin : Int32 = 30 property disable_login = false property default_username = "" property auth_proxy_header_name = "" diff --git a/src/routes/api.cr b/src/routes/api.cr index 7d2af128..34e2857a 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -612,7 +612,6 @@ struct APIRouter "width" => Int32, "height" => Int32, }], - "margin" => Int32?, } get "/api/dimensions/:tid/:eid" do |env| begin @@ -628,7 +627,6 @@ struct APIRouter send_json env, { "success" => true, "dimensions" => sizes, - "margin" => Config.current.page_margin, }.to_json rescue e send_json env, { diff --git a/src/views/reader.html.ecr b/src/views/reader.html.ecr index 0b4ee4de..9604c614 100644 --- a/src/views/reader.html.ecr +++ b/src/views/reader.html.ecr @@ -25,11 +25,11 @@ @@ -80,6 +80,7 @@ +
@@ -90,6 +91,13 @@
+
+ +
+ +
+
+
From 757f7c821428a4dd9de08c55cfca817957d20576 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Fri, 12 Mar 2021 13:41:24 +0000 Subject: [PATCH 02/20] Upgrade Crystal to 0.36.1 --- .github/workflows/build.yml | 2 +- Dockerfile | 2 +- Dockerfile.arm32v7 | 2 +- Dockerfile.arm64v8 | 2 +- shard.lock | 4 ++-- shard.yml | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1967b06d..5a7ef42b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest container: - image: crystallang/crystal:0.35.1-alpine + image: crystallang/crystal:0.36.1-alpine steps: - uses: actions/checkout@v2 diff --git a/Dockerfile b/Dockerfile index f203c6c0..6e5f3d8e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM crystallang/crystal:0.35.1-alpine AS builder +FROM crystallang/crystal:0.36.1-alpine AS builder WORKDIR /Mango diff --git a/Dockerfile.arm32v7 b/Dockerfile.arm32v7 index 65abb908..5a31a264 100644 --- a/Dockerfile.arm32v7 +++ b/Dockerfile.arm32v7 @@ -2,7 +2,7 @@ FROM arm32v7/ubuntu:18.04 RUN apt-get update && apt-get install -y wget git make llvm-8 llvm-8-dev g++ libsqlite3-dev libyaml-dev libgc-dev libssl-dev libcrypto++-dev libevent-dev libgmp-dev zlib1g-dev libpcre++-dev pkg-config libarchive-dev libxml2-dev libacl1-dev nettle-dev liblzo2-dev liblzma-dev libbz2-dev libjpeg-turbo8-dev libpng-dev libtiff-dev -RUN git clone https://github.com/crystal-lang/crystal && cd crystal && git checkout 0.35.1 && make deps && cd .. +RUN git clone https://github.com/crystal-lang/crystal && cd crystal && git checkout 0.36.1 && make deps && cd .. RUN git clone https://github.com/kostya/myhtml && cd myhtml/src/ext && git checkout v1.5.0 && make && cd .. RUN git clone https://github.com/jessedoyle/duktape.cr && cd duktape.cr/ext && git checkout v0.20.0 && make && cd .. RUN git clone https://github.com/hkalexling/image_size.cr && cd image_size.cr && git checkout v0.2.0 && make && cd .. diff --git a/Dockerfile.arm64v8 b/Dockerfile.arm64v8 index d9b41862..3cd59396 100644 --- a/Dockerfile.arm64v8 +++ b/Dockerfile.arm64v8 @@ -2,7 +2,7 @@ FROM arm64v8/ubuntu:18.04 RUN apt-get update && apt-get install -y wget git make llvm-8 llvm-8-dev g++ libsqlite3-dev libyaml-dev libgc-dev libssl-dev libcrypto++-dev libevent-dev libgmp-dev zlib1g-dev libpcre++-dev pkg-config libarchive-dev libxml2-dev libacl1-dev nettle-dev liblzo2-dev liblzma-dev libbz2-dev libjpeg-turbo8-dev libpng-dev libtiff-dev -RUN git clone https://github.com/crystal-lang/crystal && cd crystal && git checkout 0.35.1 && make deps && cd .. +RUN git clone https://github.com/crystal-lang/crystal && cd crystal && git checkout 0.36.1 && make deps && cd .. RUN git clone https://github.com/kostya/myhtml && cd myhtml/src/ext && git checkout v1.5.0 && make && cd .. RUN git clone https://github.com/jessedoyle/duktape.cr && cd duktape.cr/ext && git checkout v0.20.0 && make && cd .. RUN git clone https://github.com/hkalexling/image_size.cr && cd image_size.cr && git checkout v0.2.0 && make && cd .. diff --git a/shard.lock b/shard.lock index 254f3a5a..6e434219 100644 --- a/shard.lock +++ b/shard.lock @@ -30,7 +30,7 @@ shards: http_proxy: git: https://github.com/mamantoha/http_proxy.git - version: 0.7.1 + version: 0.8.0 image_size: git: https://github.com/hkalexling/image_size.cr.git @@ -42,7 +42,7 @@ shards: kemal-session: git: https://github.com/kemalcr/kemal-session.git - version: 0.12.1 + version: 0.13.0 kilt: git: https://github.com/jeromegn/kilt.git diff --git a/shard.yml b/shard.yml index 390ddc69..3ed69c7d 100644 --- a/shard.yml +++ b/shard.yml @@ -8,7 +8,7 @@ targets: mango: main: src/mango.cr -crystal: 0.35.1 +crystal: 0.36.1 license: MIT From e9a490676b30133e8bceeb8f9c990391b1ba1730 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Fri, 12 Mar 2021 13:59:11 +0000 Subject: [PATCH 03/20] Update the mangadex shard --- shard.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shard.lock b/shard.lock index 6e434219..776587e1 100644 --- a/shard.lock +++ b/shard.lock @@ -54,7 +54,7 @@ shards: mangadex: git: https://github.com/hkalexling/mangadex.git - version: 0.8.0+git.commit.24e6fb51afd043721139355854e305b43bf98c43 + version: 0.9.0+git.commit.a8e5deb3e6f882f5bc0f4de66e0f6c20aa98a8a6 mg: git: https://github.com/hkalexling/mg.git From daec2bdac68631c4337986ca4f2af005ed9c6192 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Fri, 12 Mar 2021 14:06:20 +0000 Subject: [PATCH 04/20] Update ameba --- shard.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shard.lock b/shard.lock index 776587e1..c22f839b 100644 --- a/shard.lock +++ b/shard.lock @@ -2,7 +2,7 @@ version: 2.0 shards: ameba: git: https://github.com/crystal-ameba/ameba.git - version: 0.12.1 + version: 0.14.0 archive: git: https://github.com/hkalexling/archive.cr.git From ee52c52f465a3d4b8f0dbe16dd74fe430a097028 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Fri, 12 Mar 2021 15:03:12 +0000 Subject: [PATCH 05/20] Fix new linter errors --- spec/storage_spec.cr | 4 +--- spec/util_spec.cr | 6 +++--- src/library/entry.cr | 2 +- src/library/library.cr | 18 +++++++++--------- src/library/title.cr | 36 +++++++++++++++--------------------- src/mangadex/ext.cr | 2 +- src/plugin/plugin.cr | 2 +- src/queue.cr | 4 ++-- src/rename.cr | 10 +++++----- src/routes/api.cr | 2 +- src/routes/main.cr | 2 +- src/storage.cr | 4 ++-- src/util/chapter_sort.cr | 2 +- src/util/numeric_sort.cr | 2 +- src/util/util.cr | 2 +- src/util/web.cr | 2 +- 16 files changed, 46 insertions(+), 54 deletions(-) diff --git a/spec/storage_spec.cr b/spec/storage_spec.cr index 44bfb5a0..ad10de59 100644 --- a/spec/storage_spec.cr +++ b/spec/storage_spec.cr @@ -8,9 +8,7 @@ describe Storage do end it "deletes user" do - with_storage do |storage| - storage.delete_user "admin" - end + with_storage &.delete_user "admin" end it "creates new user" do diff --git a/spec/util_spec.cr b/spec/util_spec.cr index 3ee4aac3..27d97c2a 100644 --- a/spec/util_spec.cr +++ b/spec/util_spec.cr @@ -21,7 +21,7 @@ describe "compare_numerically" do it "sorts like the stack exchange post" do ary = ["2", "12", "200000", "1000000", "a", "a12", "b2", "text2", "text2a", "text2a2", "text2a12", "text2ab", "text12", "text12a"] - ary.reverse.sort { |a, b| + ary.reverse.sort! { |a, b| compare_numerically a, b }.should eq ary end @@ -29,7 +29,7 @@ describe "compare_numerically" do # https://github.com/hkalexling/Mango/issues/22 it "handles numbers larger than Int32" do ary = ["14410155591588.jpg", "21410155591588.png", "104410155591588.jpg"] - ary.reverse.sort { |a, b| + ary.reverse.sort! { |a, b| compare_numerically a, b }.should eq ary end @@ -56,7 +56,7 @@ describe "chapter_sort" do it "sorts correctly" do ary = ["Vol.1 Ch.01", "Vol.1 Ch.02", "Vol.2 Ch. 2.5", "Ch. 3", "Ch.04"] sorter = ChapterSorter.new ary - ary.reverse.sort do |a, b| + ary.reverse.sort! do |a, b| sorter.compare a, b end.should eq ary end diff --git a/src/library/entry.cr b/src/library/entry.cr index cb458959..4ff01eed 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -86,7 +86,7 @@ class Entry SUPPORTED_IMG_TYPES.includes? \ MIME.from_filename? e.filename } - .sort { |a, b| + .sort! { |a, b| compare_numerically a.filename, b.filename } yield file, entries diff --git a/src/library/library.cr b/src/library/library.cr index 7e97e85c..e673c192 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -63,7 +63,7 @@ class Library end def deep_titles - titles + titles.map { |t| t.deep_titles }.flatten + titles + titles.flat_map &.deep_titles end def to_json(json : JSON::Builder) @@ -98,7 +98,7 @@ class Library .select { |path| File.directory? path } .map { |path| Title.new path, "" } .select { |title| !(title.entries.empty? && title.titles.empty?) } - .sort { |a, b| a.title <=> b.title } + .sort! { |a, b| a.title <=> b.title } .tap { |_| @title_ids.clear } .each do |title| @title_hash[title.id] = title @@ -114,7 +114,7 @@ class Library def get_continue_reading_entries(username) cr_entries = deep_titles - .map { |t| t.get_last_read_entry username } + .map(&.get_last_read_entry username) # Select elements with type `Entry` from the array and ignore all `Nil`s .select(Entry)[0...ENTRIES_IN_HOME_SECTIONS] .map { |e| @@ -150,9 +150,9 @@ class Library recently_added = [] of RA last_date_added = nil - titles.map { |t| t.deep_entries_with_date_added }.flatten - .select { |e| e[:date_added] > 1.month.ago } - .sort { |a, b| b[:date_added] <=> a[:date_added] } + titles.flat_map(&.deep_entries_with_date_added) + .select(&.[:date_added].> 1.month.ago) + .sort! { |a, b| b[:date_added] <=> a[:date_added] } .each do |e| break if recently_added.size > 12 last = recently_added.last? @@ -188,9 +188,9 @@ class Library # If we use `deep_titles`, the start reading section might include `Vol. 2` # when the user hasn't started `Vol. 1` yet titles - .select { |t| t.load_percentage(username) == 0 } + .select(&.load_percentage(username).== 0) .sample(ENTRIES_IN_HOME_SECTIONS) - .shuffle + .shuffle! end def thumbnail_generation_progress @@ -205,7 +205,7 @@ class Library end Logger.info "Starting thumbnail generation" - entries = deep_titles.map(&.deep_entries).flatten.reject &.err_msg + entries = deep_titles.flat_map(&.deep_entries).reject &.err_msg @entries_count = entries.size @thumbnails_count = 0 diff --git a/src/library/title.cr b/src/library/title.cr index 4c439e72..a757d42c 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -44,14 +44,14 @@ class Title mtimes = [@mtime] mtimes += @title_ids.map { |e| Library.default.title_hash[e].mtime } - mtimes += @entries.map { |e| e.mtime } + mtimes += @entries.map &.mtime @mtime = mtimes.max @title_ids.sort! do |a, b| compare_numerically Library.default.title_hash[a].title, Library.default.title_hash[b].title end - sorter = ChapterSorter.new @entries.map { |e| e.title } + sorter = ChapterSorter.new @entries.map &.title @entries.sort! do |a, b| sorter.compare a.title, b.title end @@ -92,12 +92,12 @@ class Title # Get all entries, including entries in nested titles def deep_entries return @entries if title_ids.empty? - @entries + titles.map { |t| t.deep_entries }.flatten + @entries + titles.flat_map &.deep_entries end def deep_titles return [] of Title if titles.empty? - titles + titles.map { |t| t.deep_titles }.flatten + titles + titles.flat_map &.deep_titles end def parents @@ -138,7 +138,7 @@ class Title end def get_entry(eid) - @entries.find { |e| e.id == eid } + @entries.find &.id.== eid end def display_name @@ -217,29 +217,23 @@ class Title @entries.each do |e| e.save_progress username, e.pages end - titles.each do |t| - t.read_all username - end + titles.each &.read_all username end # Set the reading progress of all entries and nested libraries to 0% def unread_all(username) - @entries.each do |e| - e.save_progress username, 0 - end - titles.each do |t| - t.unread_all username - end + @entries.each &.save_progress(username, 0) + titles.each &.unread_all username end def deep_read_page_count(username) : Int32 load_progress_for_all_entries(username).sum + - titles.map { |t| t.deep_read_page_count username }.flatten.sum + titles.flat_map(&.deep_read_page_count username).sum end def deep_total_page_count : Int32 - entries.map { |e| e.pages }.sum + - titles.map { |t| t.deep_total_page_count }.flatten.sum + entries.sum(&.pages) + + titles.flat_map(&.deep_total_page_count).sum end def load_percentage(username) @@ -311,13 +305,13 @@ class Title ary = @entries.zip(percentage_ary) .sort { |a_tp, b_tp| (a_tp[1] <=> b_tp[1]).or \ compare_numerically a_tp[0].title, b_tp[0].title } - .map { |tp| tp[0] } + .map &.[0] else unless opt.method.auto? Logger.warn "Unknown sorting method #{opt.not_nil!.method}. Using " \ "Auto instead" end - sorter = ChapterSorter.new @entries.map { |e| e.title } + sorter = ChapterSorter.new @entries.map &.title ary = @entries.sort do |a, b| sorter.compare(a.title, b.title).or \ compare_numerically a.title, b.title @@ -383,13 +377,13 @@ class Title {entry: e, date_added: da_ary[i]} end return zip if title_ids.empty? - zip + titles.map { |t| t.deep_entries_with_date_added }.flatten + zip + titles.flat_map &.deep_entries_with_date_added end def bulk_progress(action, ids : Array(String), username) selected_entries = ids .map { |id| - @entries.find { |e| e.id == id } + @entries.find &.id.==(id) } .select(Entry) diff --git a/src/mangadex/ext.cr b/src/mangadex/ext.cr index dfb302c5..deb09c89 100644 --- a/src/mangadex/ext.cr +++ b/src/mangadex/ext.cr @@ -35,7 +35,7 @@ module MangaDex struct Chapter def rename(rule : Rename::Rule) hash = properties_to_hash %w(id title volume chapter lang_code language) - hash["groups"] = groups.map(&.name).join "," + hash["groups"] = groups.join(",", &.name) rule.render hash end diff --git a/src/plugin/plugin.cr b/src/plugin/plugin.cr index baa77d1a..6bedea17 100644 --- a/src/plugin/plugin.cr +++ b/src/plugin/plugin.cr @@ -117,7 +117,7 @@ class Plugin def initialize(id : String) Plugin.build_info_ary - @info = @@info_ary.find { |i| i.id == id } + @info = @@info_ary.find &.id.== id if @info.nil? raise Error.new "Plugin with ID #{id} not found" end diff --git a/src/queue.cr b/src/queue.cr index c9f805c1..381441b6 100644 --- a/src/queue.cr +++ b/src/queue.cr @@ -303,12 +303,12 @@ class Queue end def pause - @downloaders.each { |d| d.stopped = true } + @downloaders.each &.stopped=(true) @paused = true end def resume - @downloaders.each { |d| d.stopped = false } + @downloaders.each &.stopped=(false) @paused = false end diff --git a/src/rename.cr b/src/rename.cr index 1fc7693d..3e00fdcb 100644 --- a/src/rename.cr +++ b/src/rename.cr @@ -35,15 +35,15 @@ module Rename class Group < Base(Pattern | String) def render(hash : VHash) - return "" if @ary.select(&.is_a? Pattern) + return "" if @ary.select(Pattern) .any? &.as(Pattern).render(hash).empty? - @ary.map do |e| + @ary.join do |e| if e.is_a? Pattern e.render hash else e end - end.join + end end end @@ -129,13 +129,13 @@ module Rename end def render(hash : VHash) - str = @ary.map do |e| + str = @ary.join do |e| if e.is_a? String e else e.render hash end - end.join.strip + end.strip post_process str end diff --git a/src/routes/api.cr b/src/routes/api.cr index 34e2857a..be7fc1dd 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -339,7 +339,7 @@ struct APIRouter } post "/api/admin/mangadex/download" do |env| begin - chapters = env.params.json["chapters"].as(Array).map { |c| c.as_h } + chapters = env.params.json["chapters"].as(Array).map &.as_h jobs = chapters.map { |chapter| Queue::Job.new( chapter["id"].as_i64.to_s, diff --git a/src/routes/main.cr b/src/routes/main.cr index 65048005..2497c2f6 100644 --- a/src/routes/main.cr +++ b/src/routes/main.cr @@ -103,7 +103,7 @@ struct MainRouter recently_added = Library.default.get_recently_added_entries username start_reading = Library.default.get_start_reading_titles username titles = Library.default.titles - new_user = !titles.any? { |t| t.load_percentage(username) > 0 } + new_user = !titles.any? &.load_percentage(username).> 0 empty_library = titles.size == 0 layout "home" rescue e diff --git a/src/storage.cr b/src/storage.cr index 971bba36..39116b98 100644 --- a/src/storage.cr +++ b/src/storage.cr @@ -445,7 +445,7 @@ class Storage Logger.debug "Marking #{trash_ids.size} entries as unavailable" end db.exec "update ids set unavailable = 1 where id in " \ - "(#{trash_ids.map { |i| "'#{i}'" }.join ","})" + "(#{trash_ids.join "," { |i| "'#{i}'" }})" # Detect dangling title IDs trash_titles = [] of String @@ -461,7 +461,7 @@ class Storage Logger.debug "Marking #{trash_titles.size} titles as unavailable" end db.exec "update titles set unavailable = 1 where id in " \ - "(#{trash_titles.map { |i| "'#{i}'" }.join ","})" + "(#{trash_titles.join "," { |i| "'#{i}'" }})" end end end diff --git a/src/util/chapter_sort.cr b/src/util/chapter_sort.cr index 44dfb4e1..eed7a3ec 100644 --- a/src/util/chapter_sort.cr +++ b/src/util/chapter_sort.cr @@ -73,7 +73,7 @@ class ChapterSorter .select do |key| keys[key].count >= str_ary.size / 2 end - .sort do |a_key, b_key| + .sort! do |a_key, b_key| a = keys[a_key] b = keys[b_key] # Sort keys by the number of times they appear diff --git a/src/util/numeric_sort.cr b/src/util/numeric_sort.cr index 7365a9f7..c455b47a 100644 --- a/src/util/numeric_sort.cr +++ b/src/util/numeric_sort.cr @@ -11,7 +11,7 @@ end def split_by_alphanumeric(str) arr = [] of String str.scan(/([^\d\n\r]*)(\d*)([^\d\n\r]*)/) do |match| - arr += match.captures.select { |s| s != "" } + arr += match.captures.select &.!= "" end arr end diff --git a/src/util/util.cr b/src/util/util.cr index 8903f5ef..c4e168a7 100644 --- a/src/util/util.cr +++ b/src/util/util.cr @@ -114,7 +114,7 @@ class String def components_similarity(other : String) : Float64 s, l = [self, other] .map { |str| Path.new(str).parts } - .sort_by &.size + .sort_by! &.size match = s.reverse.zip(l.reverse).count { |a, b| a == b } match / s.size diff --git a/src/util/web.cr b/src/util/web.cr index 67227c71..12459e53 100644 --- a/src/util/web.cr +++ b/src/util/web.cr @@ -72,7 +72,7 @@ def redirect(env, path) end def hash_to_query(hash) - hash.map { |k, v| "#{k}=#{v}" }.join("&") + hash.join "&" { |k, v| "#{k}=#{v}" } end def request_path_startswith(env, ary) From 9bb7144479d823d41bdfeadf28cb34671f0746ac Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Fri, 12 Mar 2021 15:28:39 +0000 Subject: [PATCH 06/20] Fix warning --- src/library/library.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/library/library.cr b/src/library/library.cr index e673c192..48bd619a 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -157,7 +157,7 @@ class Library break if recently_added.size > 12 last = recently_added.last? if last && e[:entry].book.id == last[:entry].book.id && - (e[:date_added] - last_date_added.not_nil!).duration < 1.day + (e[:date_added] - last_date_added.not_nil!).abs < 1.day # A NamedTuple is immutable, so we have to cast it to a Hash first last_hash = last.to_h count = last_hash[:grouped_count].as(Int32) From a612500b0fabf7259a5ee0c841b0157d191e5bdd Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sat, 13 Mar 2021 10:26:17 +0000 Subject: [PATCH 07/20] Subscription manager --- migration/subscription.12.cr | 31 +++ public/js/download.js | 93 +++++++++ public/js/subscription.js | 73 +++++++ shard.lock | 2 +- src/config.cr | 6 +- src/library/library.cr | 19 ++ src/mangadex/ext.cr | 34 ++++ src/routes/api.cr | 92 ++++++++- src/routes/main.cr | 6 + src/storage.cr | 68 +++++++ src/subscription.cr | 83 ++++++++ src/util/web.cr | 19 ++ src/views/download-manager.html.ecr | 100 +++++----- src/views/download.html.ecr | 288 ++++++++++++++-------------- src/views/layout.html.ecr | 158 +++++++-------- src/views/missing-items.html.ecr | 56 +++--- src/views/plugin-download.html.ecr | 6 +- src/views/subscription.html.ecr | 58 ++++++ 18 files changed, 883 insertions(+), 309 deletions(-) create mode 100644 migration/subscription.12.cr create mode 100644 public/js/subscription.js create mode 100644 src/subscription.cr create mode 100644 src/views/subscription.html.ecr diff --git a/migration/subscription.12.cr b/migration/subscription.12.cr new file mode 100644 index 00000000..3810755c --- /dev/null +++ b/migration/subscription.12.cr @@ -0,0 +1,31 @@ +class CreateSubscription < MG::Base + def up : String + # We allow multiple subscriptions for the same manga. + # This can be useful for example when you want to download from multiple + # groups. + <<-SQL + CREATE TABLE subscription ( + id INTEGER PRIMARY KEY, + manga_id INTEGER NOT NULL, + language TEXT, + group_id INTEGER, + min_volume INTEGER, + max_volume INTEGER, + min_chapter INTEGER, + max_chapter INTEGER, + last_checked INTEGER NOT NULL, + created_at INTEGER NOT NULL, + username TEXT NOT NULL, + FOREIGN KEY (username) REFERENCES users (username) + ON UPDATE CASCADE + ON DELETE CASCADE + ); + SQL + end + + def down : String + <<-SQL + DROP TABLE subscription; + SQL + end +end diff --git a/public/js/download.js b/public/js/download.js index 74fab76a..957767d4 100644 --- a/public/js/download.js +++ b/public/js/download.js @@ -282,6 +282,99 @@ const downloadComponent = () => { UIkit.modal($('#modal').get(0)).hide(); this.searchInput = id; this.search(); + }, + + subscribe(langConfirmed = false, groupConfirmed = false) { + const filters = { + manga: this.data.id, + language: this.langChoice === 'All' ? null : this.langChoice, + group: this.groupChoice === 'All' ? null : this.groupChoice, + volume: this.volumeRange === '' ? null : this.volumeRange, + chapter: this.chapterRange === '' ? null : this.chapterRange + }; + + // Get group ID + if (filters.group) { + this.data.chapters.forEach(chp => { + const gid = chp.groups[filters.group]; + if (gid) { + filters.groupId = gid; + return; + } + }); + } + + // Parse range values + if (filters.volume) { + [filters.volumeMin, filters.volumeMax] = this.parseRange(filters.volume); + } + if (filters.chapter) { + [filters.chapterMin, filters.chapterMax] = this.parseRange(filters.chapter); + } + + if (!filters.language && !langConfirmed) { + UIkit.modal.confirm('You didn\'t specify a language in the filtering rules. This might cause Mango to download chapters that are not in your preferred language. Are you sure you want to continue?', { + labels: { + ok: 'Yes', + cancel: 'Cancel' + } + }).then(() => { + this.subscribe(true, groupConfirmed); + }); + return; + } + + if (!filters.group && !groupConfirmed) { + UIkit.modal.confirm('You didn\'t specify a group in the filtering rules. This might cause Mango to download multiple versions of the same chapter. Are you sure you want to continue?', { + labels: { + ok: 'Yes', + cancel: 'Cancel' + } + }).then(() => { + this.subscribe(langConfirmed, true); + }); + return; + } + + const mangaURL = `${mangadex_base_url}/manga/${filters.manga}`; + + console.log(filters); + UIkit.modal.confirm(`All FUTURE chapters matching the following filters will be downloaded:
+
    +
  • Manga ID: ${filters.manga}
    • +
  • Language: ${filters.language || 'all'}
    • +
  • Group: ${filters.group || 'all'}
    • +
  • Volume: ${filters.volume || 'all'}
    • +
  • Chapter: ${filters.chapter || 'all'}
    • +
+ + IMPORTANT: Please make sure you are following the manga on MangaDex, otherwise Mango won't be able to receive any updates. To follow it, visit ${mangaURL} and click "Follow". + `, { + labels: { + ok: 'Confirm', + cancel: 'Cancel' + } + }).then(() => { + $.ajax({ + type: 'POST', + url: `${base_url}api/admin/mangadex/subscriptions`, + data: JSON.stringify({ + subscription: filters + }), + contentType: "application/json", + dataType: 'json' + }) + .done(data => { + console.log(data); + if (data.error) { + alert('danger', `Failed to subscribe. Error: ${data.error}`); + return; + } + }) + .fail((jqXHR, status) => { + alert('danger', `Failed to subscribe. Error: [${jqXHR.status}] ${jqXHR.statusText}`); + }); + }); } }; }; diff --git a/public/js/subscription.js b/public/js/subscription.js new file mode 100644 index 00000000..e6e4909b --- /dev/null +++ b/public/js/subscription.js @@ -0,0 +1,73 @@ +const component = () => { + return { + available: undefined, + subscriptions: [], + + init() { + $.getJSON(`${base_url}api/admin/mangadex/expires`) + .done((data) => { + if (data.error) { + alert('danger', 'Failed to check MangaDex integration status. Error: ' + data.error); + return; + } + this.available = Boolean(data.expires && data.expires > Math.floor(Date.now() / 1000)); + + if (this.available) this.getSubscriptions(); + }) + .fail((jqXHR, status) => { + alert('danger', `Failed to check MangaDex integration status. Error: [${jqXHR.status}] ${jqXHR.statusText}`); + }) + }, + + getSubscriptions() { + $.getJSON(`${base_url}api/admin/mangadex/subscriptions`) + .done(data => { + if (data.error) { + alert('danger', 'Failed to get subscriptions. Error: ' + data.error); + return; + } + this.subscriptions = data.subscriptions; + }) + .fail((jqXHR, status) => { + alert('danger', `Failed to get subscriptions. Error: [${jqXHR.status}] ${jqXHR.statusText}`); + }) + }, + + rm(event) { + const id = event.currentTarget.parentNode.getAttribute('data-id'); + $.ajax({ + type: 'DELETE', + url: `${base_url}api/admin/mangadex/subscriptions/${id}`, + contentType: 'application/json' + }) + .done(data => { + if (data.error) { + alert('danger', `Failed to delete subscription. Error: ${data.error}`); + } + this.getSubscriptions(); + }) + .fail((jqXHR, status) => { + alert('danger', `Failed to delete subscription. Error: [${jqXHR.status}] ${jqXHR.statusText}`); + }); + }, + + check(event) { + const id = event.currentTarget.parentNode.getAttribute('data-id'); + $.ajax({ + type: 'POST', + url: `${base_url}api/admin/mangadex/subscriptions/check/${id}`, + contentType: 'application/json' + }) + .done(data => { + if (data.error) { + alert('danger', `Failed to check subscription. Error: ${data.error}`); + return; + } + alert('success', 'Mango is now checking the subscription for updates. This might take a while, but you can safely leave the page.'); + }) + .fail((jqXHR, status) => { + alert('danger', `Failed to check subscription. Error: [${jqXHR.status}] ${jqXHR.statusText}`); + }); + } + }; +}; diff --git a/shard.lock b/shard.lock index c22f839b..13ce55dc 100644 --- a/shard.lock +++ b/shard.lock @@ -54,7 +54,7 @@ shards: mangadex: git: https://github.com/hkalexling/mangadex.git - version: 0.9.0+git.commit.a8e5deb3e6f882f5bc0f4de66e0f6c20aa98a8a6 + version: 0.11.0+git.commit.f5b0d64fbb138879fb9228b6e9ff34ec97c3e824 mg: git: https://github.com/hkalexling/mg.git diff --git a/src/config.cr b/src/config.cr index 332a159e..6de78a79 100644 --- a/src/config.cr +++ b/src/config.cr @@ -33,8 +33,10 @@ class Config "download_retries" => 4, "download_queue_db_path" => File.expand_path("~/mango/queue.db", home: true), - "chapter_rename_rule" => "[Vol.{volume} ][Ch.{chapter} ]{title|id}", - "manga_rename_rule" => "{title}", + "chapter_rename_rule" => "[Vol.{volume} ]" \ + "[Ch.{chapter} ]{title|id}", + "manga_rename_rule" => "{title}", + "subscription_update_interval_hours" => 24, } @@singlet : Config? diff --git a/src/library/library.cr b/src/library/library.cr index 48bd619a..8fad451b 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -42,6 +42,25 @@ class Library end end end + + subscription_interval = Config.current + .mangadex["subscription_update_interval_hours"].as Int32 + unless subscription_interval < 1 + spawn do + loop do + subscriptions = Storage.default.subscriptions + Logger.info "Checking MangaDex for updates on " \ + "#{subscriptions.size} subscriptions" + added_count = 0 + subscriptions.each do |sub| + added_count += sub.check_for_updates + end + Logger.info "Subscription update completed. Added #{added_count} " \ + "chapters to the download queue" + sleep subscription_interval.hours + end + end + end end def titles diff --git a/src/mangadex/ext.cr b/src/mangadex/ext.cr index deb09c89..e919d970 100644 --- a/src/mangadex/ext.cr +++ b/src/mangadex/ext.cr @@ -56,5 +56,39 @@ module MangaDex hash["full_title"] = JSON::Any.new full_title hash.to_json end + + # We don't need to rename the manga title here. It will be renamed in + # src/mangadex/downloader.cr + def to_job : Queue::Job + Queue::Job.new( + id.to_s, + manga_id.to_s, + full_title, + manga_title, + Queue::JobStatus::Pending, + Time.unix timestamp + ) + end + end + + struct User + def updates_after(time : Time, &block : Chapter ->) + page = 1 + stopped = false + until stopped + chapters = followed_updates(page: page).chapters + return if chapters.empty? + chapters.each do |c| + if time > Time.unix c.timestamp + stopped = true + break + end + yield c + end + page += 1 + # Let's not DDOS MangaDex :) + sleep 5.seconds + end + end end end diff --git a/src/routes/api.cr b/src/routes/api.cr index be7fc1dd..8a43b9d3 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -961,23 +961,95 @@ struct APIRouter Koa.tags ["admin", "mangadex"] get "/api/admin/mangadex/search" do |env| begin - username = get_username env - token, expires = Storage.default.get_md_token username + query = env.params.query["query"] - unless expires && token - raise "No token found for user #{username}" - end + send_json env, { + "success" => true, + "error" => nil, + "manga" => get_client(env).partial_search query, + }.to_json + rescue e + Logger.error e + send_json env, { + "success" => false, + "error" => e.message, + }.to_json + end + end - client = MangaDex::Client.from_config - client.token = token - client.token_expires = expires + get "/api/admin/mangadex/subscriptions" do |env| + begin + send_json env, { + "success" => true, + "error" => nil, + "subscriptions" => Storage.default.subscriptions, + }.to_json + rescue e + Logger.error e + send_json env, { + "success" => false, + "error" => e.message, + }.to_json + end + end - query = env.params.query["query"] + post "/api/admin/mangadex/subscriptions" do |env| + begin + json = env.params.json["subscription"].as Hash(String, JSON::Any) + sub = Subscription.new json["manga"].as_i64, get_username env + sub.language = json["language"]?.try &.as_s? + sub.group_id = json["groupId"]?.try &.as_i64? + sub.min_volume = json["volumeMin"]?.try &.as_i64? + sub.max_volume = json["volumeMax"]?.try &.as_i64? + sub.min_chapter = json["chapterMin"]?.try &.as_i64? + sub.max_chapter = json["chapterMax"]?.try &.as_i64? + + Storage.default.save_subscription sub send_json env, { "success" => true, "error" => nil, - "manga" => client.partial_search query, + }.to_json + rescue e + Logger.error e + send_json env, { + "success" => false, + "error" => e.message, + }.to_json + end + end + + delete "/api/admin/mangadex/subscriptions/:id" do |env| + begin + id = env.params.url["id"].to_i64 + Storage.default.delete_subscription id, get_username env + send_json env, { + "success" => true, + "error" => nil, + }.to_json + rescue e + Logger.error e + send_json env, { + "success" => false, + "error" => e.message, + }.to_json + end + end + + post "/api/admin/mangadex/subscriptions/check/:id" do |env| + begin + id = env.params.url["id"].to_i64 + username = get_username env + sub = Storage.default.get_subscription id, username + unless sub + raise "Subscription with id #{id} not found under user #{username}" + end + spawn do + sub.check_for_updates + end + send_json env, { + "success" => true, + "error" => nil, }.to_json rescue e Logger.error e diff --git a/src/routes/main.cr b/src/routes/main.cr index 2497c2f6..2993db7a 100644 --- a/src/routes/main.cr +++ b/src/routes/main.cr @@ -95,6 +95,12 @@ struct MainRouter end end + get "/download/subscription" do |env| + mangadex_base_url = Config.current.mangadex["base_url"] + username = get_username env + layout "subscription" + end + get "/" do |env| begin username = get_username env diff --git a/src/storage.cr b/src/storage.cr index 39116b98..164ce401 100644 --- a/src/storage.cr +++ b/src/storage.cr @@ -5,6 +5,7 @@ require "base64" require "./util/*" require "mg" require "../migration/*" +require "./subscription" def hash_password(pw) Crypto::Bcrypt::Password.create(pw).to_s @@ -14,6 +15,9 @@ def verify_password(hash, pw) (Crypto::Bcrypt::Password.new hash).verify pw end +SUB_ATTR = %w(manga_id language group_id min_volume max_volume min_chapter + max_chapter username) + class Storage @@insert_entry_ids = [] of IDTuple @@insert_title_ids = [] of IDTuple @@ -545,6 +549,70 @@ class Storage {token, expires} end + def save_subscription(sub : Subscription) + MainFiber.run do + get_db do |db| + {% begin %} + db.exec "insert into subscription (#{SUB_ATTR.join ","}, " \ + "last_checked, created_at) values " \ + "(#{Array.new(SUB_ATTR.size + 2, "?").join ","})", + {% for type in SUB_ATTR %} + sub.{{type.id}}, + {% end %} + sub.last_checked.to_unix, sub.created_at.to_unix + {% end %} + end + end + end + + def subscriptions : Array(Subscription) + subs = [] of Subscription + MainFiber.run do + get_db do |db| + db.query "select * from subscription" do |rs| + subs += Subscription.from_rs rs + end + end + end + subs + end + + def delete_subscription(id : Int64, username : String) + MainFiber.run do + get_db do |db| + db.exec "delete from subscription where id = (?) and username = (?)", + id, username + end + end + end + + def get_subscription(id : Int64, username : String) : Subscription? + sub = nil + MainFiber.run do + get_db do |db| + db.query "select * from subscription where id = (?) and " \ + "username = (?) limit 1", id, username do |rs| + sub = Subscription.from_rs(rs).first? + end + end + end + sub + end + + def update_subscription_last_checked(id : Int64? = nil) + MainFiber.run do + get_db do |db| + if id + db.exec "update subscription set last_checked = (?) where id = (?)", + Time.utc.to_unix, id + else + db.exec "update subscription set last_checked = (?)", + Time.utc.to_unix + end + end + end + end + def close MainFiber.run do unless @db.nil? diff --git a/src/subscription.cr b/src/subscription.cr new file mode 100644 index 00000000..1c0764a1 --- /dev/null +++ b/src/subscription.cr @@ -0,0 +1,83 @@ +require "db" +require "json" + +struct Subscription + include DB::Serializable + include JSON::Serializable + + getter id : Int64 = 0 + getter username : String + getter manga_id : Int64 + property language : String? + property group_id : Int64? + property min_volume : Int64? + property max_volume : Int64? + property min_chapter : Int64? + property max_chapter : Int64? + @[DB::Field(key: "last_checked")] + @[JSON::Field(key: "last_checked")] + @raw_last_checked : Int64 + @[DB::Field(key: "created_at")] + @[JSON::Field(key: "created_at")] + @raw_created_at : Int64 + + def last_checked : Time + Time.unix @raw_last_checked + end + + def created_at : Time + Time.unix @raw_created_at + end + + def initialize(@manga_id, @username) + @raw_created_at = Time.utc.to_unix + @raw_last_checked = Time.utc.to_unix + end + + def in_range?(value : String, lowerbound : Int64?, + upperbound : Int64?) : Bool + lb = lowerbound.try &.to_f64 + ub = upperbound.try &.to_f64 + + return true if lb.nil? && ub.nil? + + v = value.to_f64? + return false unless v + + if lb.nil? + v <= ub.not_nil! + elsif ub.nil? + v >= lb.not_nil! + else + v >= lb.not_nil! && v <= ub.not_nil! + end + end + + def match?(chapter : MangaDex::Chapter) : Bool + if chapter.manga_id != manga_id || + (language && chapter.language != language) || + (group_id && !chapter.groups.map(&.id).includes? group_id) + return false + end + + in_range?(chapter.volume, min_volume, max_volume) && + in_range?(chapter.chapter, min_chapter, max_chapter) + end + + def check_for_updates : Int32 + Logger.debug "Checking updates for subscription with ID #{id}" + jobs = [] of Queue::Job + get_client(username).user.updates_after last_checked do |chapter| + next unless match? chapter + jobs << chapter.to_job + end + Storage.default.update_subscription_last_checked id + count = Queue.default.push jobs + Logger.debug "#{count}/#{jobs.size} of updates added to queue" + count + rescue e + Logger.error "Error occurred when checking updates for " \ + "subscription with ID #{id}. #{e}" + 0 + end +end diff --git a/src/util/web.cr b/src/util/web.cr index 12459e53..efa269d8 100644 --- a/src/util/web.cr +++ b/src/util/web.cr @@ -107,6 +107,25 @@ macro get_sort_opt end end +# Returns an authorized client +def get_client(username : String) : MangaDex::Client + token, expires = Storage.default.get_md_token username + + unless expires && token + raise "No token found for user #{username}" + end + + client = MangaDex::Client.from_config + client.token = token + client.token_expires = expires + + client +end + +def get_client(env) : MangaDex::Client + get_client get_username env +end + module HTTP class Client private def self.exec(uri : URI, tls : TLSContext = nil) diff --git a/src/views/download-manager.html.ecr b/src/views/download-manager.html.ecr index 2b6e4348..73a5445d 100644 --- a/src/views/download-manager.html.ecr +++ b/src/views/download-manager.html.ecr @@ -5,61 +5,63 @@
- - - - - - - - - - - - - - + +
ChapterMangaProgressTimeStatusPluginActions
+ <% content_for "script" do %> diff --git a/src/views/download.html.ecr b/src/views/download.html.ecr index 0ea85276..983cae00 100644 --- a/src/views/download.html.ecr +++ b/src/views/download.html.ecr @@ -1,162 +1,170 @@

Download from MangaDex

-
-
- -
-
-
- +
+
+ +
+
+
+ +
-
- -
- -
- -
-
+
+
+
+ +
+
+

Title:

+

+

+
+
+

+ Filter Chapters + +

+

+
+ +
+ +
+
-
- -
- -
+
+ +
+ +
+
+ +
+ +
+ +
+
+ +
+ +
+ +
+
+
- -
- -
+
+ + + +
+
+

Click on a table row to select the chapter. Drag your mouse over multiple rows to select them all. Hold Ctrl to make multiple non-adjacent selections.

-
-
- -
-
- - - -
-
-

Click on a table row to select the chapter. Drag your mouse over multiple rows to select them all. Hold Ctrl to make multiple non-adjacent selections.

-
-

- - - - - - - - - - - - +

+
+
IDTitleLanguageGroupVolumeChapterTimestamp
+ + + + + + + + + + + -
IDTitleLanguageGroupVolumeChapterTimestamp