diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1967b06d..bbf13951 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,12 +12,12 @@ jobs: runs-on: ubuntu-latest container: - image: crystallang/crystal:0.35.1-alpine + image: crystallang/crystal:0.36.1-alpine steps: - uses: actions/checkout@v2 - name: Install dependencies - run: apk add --no-cache yarn yaml sqlite-static libarchive-dev libarchive-static acl-static expat-static zstd-static lz4-static bzip2-static libjpeg-turbo-dev libpng-dev tiff-dev + run: apk add --no-cache yarn yaml-static sqlite-static libarchive-dev libarchive-static acl-static expat-static zstd-static lz4-static bzip2-static libjpeg-turbo-dev libpng-dev tiff-dev - name: Build run: make static || make static - name: Linter diff --git a/Dockerfile b/Dockerfile index f203c6c0..adc82dd6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,9 @@ -FROM crystallang/crystal:0.35.1-alpine AS builder +FROM crystallang/crystal:0.36.1-alpine AS builder WORKDIR /Mango COPY . . -RUN apk add --no-cache yarn yaml sqlite-static libarchive-dev libarchive-static acl-static expat-static zstd-static lz4-static bzip2-static libjpeg-turbo-dev libpng-dev tiff-dev +RUN apk add --no-cache yarn yaml-static sqlite-static libarchive-dev libarchive-static acl-static expat-static zstd-static lz4-static bzip2-static libjpeg-turbo-dev libpng-dev tiff-dev RUN make static || make static FROM library/alpine diff --git a/Dockerfile.arm32v7 b/Dockerfile.arm32v7 index 65abb908..5a31a264 100644 --- a/Dockerfile.arm32v7 +++ b/Dockerfile.arm32v7 @@ -2,7 +2,7 @@ FROM arm32v7/ubuntu:18.04 RUN apt-get update && apt-get install -y wget git make llvm-8 llvm-8-dev g++ libsqlite3-dev libyaml-dev libgc-dev libssl-dev libcrypto++-dev libevent-dev libgmp-dev zlib1g-dev libpcre++-dev pkg-config libarchive-dev libxml2-dev libacl1-dev nettle-dev liblzo2-dev liblzma-dev libbz2-dev libjpeg-turbo8-dev libpng-dev libtiff-dev -RUN git clone https://github.com/crystal-lang/crystal && cd crystal && git checkout 0.35.1 && make deps && cd .. +RUN git clone https://github.com/crystal-lang/crystal && cd crystal && git checkout 0.36.1 && make deps && cd .. RUN git clone https://github.com/kostya/myhtml && cd myhtml/src/ext && git checkout v1.5.0 && make && cd .. RUN git clone https://github.com/jessedoyle/duktape.cr && cd duktape.cr/ext && git checkout v0.20.0 && make && cd .. RUN git clone https://github.com/hkalexling/image_size.cr && cd image_size.cr && git checkout v0.2.0 && make && cd .. diff --git a/Dockerfile.arm64v8 b/Dockerfile.arm64v8 index d9b41862..3cd59396 100644 --- a/Dockerfile.arm64v8 +++ b/Dockerfile.arm64v8 @@ -2,7 +2,7 @@ FROM arm64v8/ubuntu:18.04 RUN apt-get update && apt-get install -y wget git make llvm-8 llvm-8-dev g++ libsqlite3-dev libyaml-dev libgc-dev libssl-dev libcrypto++-dev libevent-dev libgmp-dev zlib1g-dev libpcre++-dev pkg-config libarchive-dev libxml2-dev libacl1-dev nettle-dev liblzo2-dev liblzma-dev libbz2-dev libjpeg-turbo8-dev libpng-dev libtiff-dev -RUN git clone https://github.com/crystal-lang/crystal && cd crystal && git checkout 0.35.1 && make deps && cd .. +RUN git clone https://github.com/crystal-lang/crystal && cd crystal && git checkout 0.36.1 && make deps && cd .. RUN git clone https://github.com/kostya/myhtml && cd myhtml/src/ext && git checkout v1.5.0 && make && cd .. RUN git clone https://github.com/jessedoyle/duktape.cr && cd duktape.cr/ext && git checkout v0.20.0 && make && cd .. RUN git clone https://github.com/hkalexling/image_size.cr && cd image_size.cr && git checkout v0.2.0 && make && cd .. diff --git a/README.md b/README.md index e1545d68..3f1a2746 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ The official docker images are available on [Dockerhub](https://hub.docker.com/r ### CLI ``` - Mango - Manga Server and Web Reader. Version 0.21.0 + Mango - Manga Server and Web Reader. Version 0.22.0 Usage: @@ -99,6 +99,7 @@ mangadex: download_queue_db_path: ~/mango/queue.db chapter_rename_rule: '[Vol.{volume} ][Ch.{chapter} ]{title|id}' manga_rename_rule: '{title}' + subscription_update_interval_hours: 24 ``` - `scan_interval_minutes`, `thumbnail_generation_interval_hours` and `db_optimization_interval_hours` can be any non-negative integer. Setting them to `0` disables the periodic tasks diff --git a/migration/subscription.12.cr b/migration/subscription.12.cr new file mode 100644 index 00000000..3810755c --- /dev/null +++ b/migration/subscription.12.cr @@ -0,0 +1,31 @@ +class CreateSubscription < MG::Base + def up : String + # We allow multiple subscriptions for the same manga. + # This can be useful for example when you want to download from multiple + # groups. + <<-SQL + CREATE TABLE subscription ( + id INTEGER PRIMARY KEY, + manga_id INTEGER NOT NULL, + language TEXT, + group_id INTEGER, + min_volume INTEGER, + max_volume INTEGER, + min_chapter INTEGER, + max_chapter INTEGER, + last_checked INTEGER NOT NULL, + created_at INTEGER NOT NULL, + username TEXT NOT NULL, + FOREIGN KEY (username) REFERENCES users (username) + ON UPDATE CASCADE + ON DELETE CASCADE + ); + SQL + end + + def down : String + <<-SQL + DROP TABLE subscription; + SQL + end +end diff --git a/public/js/download.js b/public/js/download.js index 74fab76a..31a803eb 100644 --- a/public/js/download.js +++ b/public/js/download.js @@ -260,9 +260,7 @@ const downloadComponent = () => { } const successCount = parseInt(data.success); const failCount = parseInt(data.fail); - UIkit.modal.confirm(`${successCount} of ${successCount + failCount} chapters added to the download queue. Proceed to the download manager?`).then(() => { - window.location.href = base_url + 'admin/downloads'; - }); + alert('success', `${successCount} of ${successCount + failCount} chapters added to the download queue. You can view and manage your download queue on the download manager page.`); }) .fail((jqXHR, status) => { alert('danger', `Failed to add chapters to the download queue. Error: [${jqXHR.status}] ${jqXHR.statusText}`); @@ -282,6 +280,100 @@ const downloadComponent = () => { UIkit.modal($('#modal').get(0)).hide(); this.searchInput = id; this.search(); + }, + + subscribe(langConfirmed = false, groupConfirmed = false) { + const filters = { + manga: this.data.id, + language: this.langChoice === 'All' ? null : this.langChoice, + group: this.groupChoice === 'All' ? null : this.groupChoice, + volume: this.volumeRange === '' ? null : this.volumeRange, + chapter: this.chapterRange === '' ? null : this.chapterRange + }; + + // Get group ID + if (filters.group) { + this.data.chapters.forEach(chp => { + const gid = chp.groups[filters.group]; + if (gid) { + filters.groupId = gid; + return; + } + }); + } + + // Parse range values + if (filters.volume) { + [filters.volumeMin, filters.volumeMax] = this.parseRange(filters.volume); + } + if (filters.chapter) { + [filters.chapterMin, filters.chapterMax] = this.parseRange(filters.chapter); + } + + if (!filters.language && !langConfirmed) { + UIkit.modal.confirm('You didn\'t specify a language in the filtering rules. This might cause Mango to download chapters that are not in your preferred language. Are you sure you want to continue?', { + labels: { + ok: 'Yes', + cancel: 'Cancel' + } + }).then(() => { + this.subscribe(true, groupConfirmed); + }); + return; + } + + if (!filters.group && !groupConfirmed) { + UIkit.modal.confirm('You didn\'t specify a group in the filtering rules. This might cause Mango to download multiple versions of the same chapter. Are you sure you want to continue?', { + labels: { + ok: 'Yes', + cancel: 'Cancel' + } + }).then(() => { + this.subscribe(langConfirmed, true); + }); + return; + } + + const mangaURL = `${mangadex_base_url}/manga/${filters.manga}`; + + console.log(filters); + UIkit.modal.confirm(`All FUTURE chapters matching the following filters will be downloaded:
+ + + IMPORTANT: Please make sure you are following the manga on MangaDex, otherwise Mango won't be able to receive any updates. To follow it, visit ${mangaURL} and click "Follow". + `, { + labels: { + ok: 'Confirm', + cancel: 'Cancel' + } + }).then(() => { + $.ajax({ + type: 'POST', + url: `${base_url}api/admin/mangadex/subscriptions`, + data: JSON.stringify({ + subscription: filters + }), + contentType: "application/json", + dataType: 'json' + }) + .done(data => { + console.log(data); + if (data.error) { + alert('danger', `Failed to subscribe. Error: ${data.error}`); + return; + } + alert('success', `You've successfully subscribed to this manga! You can view and manage your subscriptions on the subscription manager page.`); + }) + .fail((jqXHR, status) => { + alert('danger', `Failed to subscribe. Error: [${jqXHR.status}] ${jqXHR.statusText}`); + }); + }); } }; }; diff --git a/public/js/plugin-download.js b/public/js/plugin-download.js index e1fba961..a335e038 100644 --- a/public/js/plugin-download.js +++ b/public/js/plugin-download.js @@ -126,9 +126,7 @@ const download = () => { } const successCount = parseInt(data.success); const failCount = parseInt(data.fail); - UIkit.modal.confirm(`${successCount} of ${successCount + failCount} chapters added to the download queue. Proceed to the download manager?`).then(() => { - window.location.href = base_url + 'admin/downloads'; - }); + alert('success', `${successCount} of ${successCount + failCount} chapters added to the download queue. You can view and manage your download queue on the download manager page.`); }) .fail((jqXHR, status) => { alert('danger', `Failed to add chapters to the download queue. Error: [${jqXHR.status}] ${jqXHR.statusText}`); diff --git a/public/js/reader.js b/public/js/reader.js index 1aa1bd13..f2279728 100644 --- a/public/js/reader.js +++ b/public/js/reader.js @@ -10,6 +10,7 @@ const readerComponent = () => { longPages: false, lastSavedPage: page, selectedIndex: 0, // 0: not selected; 1: the first page + margin: 30, /** * Initialize the component by fetching the page dimensions @@ -27,7 +28,6 @@ const readerComponent = () => { url: `${base_url}api/page/${tid}/${eid}/${i+1}`, width: d.width, height: d.height, - style: `margin-top: ${data.margin}px; margin-bottom: ${data.margin}px;` }; }); @@ -47,6 +47,11 @@ const readerComponent = () => { const mode = this.mode; this.updateMode(this.mode, page, nextTick); $('#mode-select').val(mode); + + const savedMargin = localStorage.getItem('margin'); + if (savedMargin) { + this.margin = savedMargin; + } }) .catch(e => { const errMsg = `Failed to get the page dimensions. ${e}`; @@ -277,6 +282,11 @@ const readerComponent = () => { entryChanged() { const id = $('#entry-select').val(); this.redirect(`${base_url}reader/${tid}/${id}`); + }, + + marginChanged() { + localStorage.setItem('margin', this.margin); + this.toPage(this.selectedIndex); } }; } diff --git a/public/js/subscription.js b/public/js/subscription.js new file mode 100644 index 00000000..ed2cb174 --- /dev/null +++ b/public/js/subscription.js @@ -0,0 +1,82 @@ +const component = () => { + return { + available: undefined, + subscriptions: [], + + init() { + $.getJSON(`${base_url}api/admin/mangadex/expires`) + .done((data) => { + if (data.error) { + alert('danger', 'Failed to check MangaDex integration status. Error: ' + data.error); + return; + } + this.available = Boolean(data.expires && data.expires > Math.floor(Date.now() / 1000)); + + if (this.available) this.getSubscriptions(); + }) + .fail((jqXHR, status) => { + alert('danger', `Failed to check MangaDex integration status. Error: [${jqXHR.status}] ${jqXHR.statusText}`); + }) + }, + + getSubscriptions() { + $.getJSON(`${base_url}api/admin/mangadex/subscriptions`) + .done(data => { + if (data.error) { + alert('danger', 'Failed to get subscriptions. Error: ' + data.error); + return; + } + this.subscriptions = data.subscriptions; + }) + .fail((jqXHR, status) => { + alert('danger', `Failed to get subscriptions. Error: [${jqXHR.status}] ${jqXHR.statusText}`); + }) + }, + + rm(event) { + const id = event.currentTarget.parentNode.getAttribute('data-id'); + $.ajax({ + type: 'DELETE', + url: `${base_url}api/admin/mangadex/subscriptions/${id}`, + contentType: 'application/json' + }) + .done(data => { + if (data.error) { + alert('danger', `Failed to delete subscription. Error: ${data.error}`); + } + this.getSubscriptions(); + }) + .fail((jqXHR, status) => { + alert('danger', `Failed to delete subscription. Error: [${jqXHR.status}] ${jqXHR.statusText}`); + }); + }, + + check(event) { + const id = event.currentTarget.parentNode.getAttribute('data-id'); + $.ajax({ + type: 'POST', + url: `${base_url}api/admin/mangadex/subscriptions/check/${id}`, + contentType: 'application/json' + }) + .done(data => { + if (data.error) { + alert('danger', `Failed to check subscription. Error: ${data.error}`); + return; + } + alert('success', 'Mango is now checking the subscription for updates. This might take a while, but you can safely leave the page.'); + }) + .fail((jqXHR, status) => { + alert('danger', `Failed to check subscription. Error: [${jqXHR.status}] ${jqXHR.statusText}`); + }); + }, + + formatRange(min, max) { + if (!isNaN(min) && isNaN(max)) return `≥ ${min}`; + if (isNaN(min) && !isNaN(max)) return `≤ ${max}`; + if (isNaN(min) && isNaN(max)) return 'All'; + + if (min === max) return `= ${min}`; + return `${min} - ${max}`; + } + }; +}; diff --git a/shard.lock b/shard.lock index 254f3a5a..13ce55dc 100644 --- a/shard.lock +++ b/shard.lock @@ -2,7 +2,7 @@ version: 2.0 shards: ameba: git: https://github.com/crystal-ameba/ameba.git - version: 0.12.1 + version: 0.14.0 archive: git: https://github.com/hkalexling/archive.cr.git @@ -30,7 +30,7 @@ shards: http_proxy: git: https://github.com/mamantoha/http_proxy.git - version: 0.7.1 + version: 0.8.0 image_size: git: https://github.com/hkalexling/image_size.cr.git @@ -42,7 +42,7 @@ shards: kemal-session: git: https://github.com/kemalcr/kemal-session.git - version: 0.12.1 + version: 0.13.0 kilt: git: https://github.com/jeromegn/kilt.git @@ -54,7 +54,7 @@ shards: mangadex: git: https://github.com/hkalexling/mangadex.git - version: 0.8.0+git.commit.24e6fb51afd043721139355854e305b43bf98c43 + version: 0.11.0+git.commit.f5b0d64fbb138879fb9228b6e9ff34ec97c3e824 mg: git: https://github.com/hkalexling/mg.git diff --git a/shard.yml b/shard.yml index 390ddc69..3991a13f 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: mango -version: 0.21.0 +version: 0.22.0 authors: - Alex Ling @@ -8,7 +8,7 @@ targets: mango: main: src/mango.cr -crystal: 0.35.1 +crystal: 0.36.1 license: MIT diff --git a/spec/storage_spec.cr b/spec/storage_spec.cr index 44bfb5a0..ad10de59 100644 --- a/spec/storage_spec.cr +++ b/spec/storage_spec.cr @@ -8,9 +8,7 @@ describe Storage do end it "deletes user" do - with_storage do |storage| - storage.delete_user "admin" - end + with_storage &.delete_user "admin" end it "creates new user" do diff --git a/spec/util_spec.cr b/spec/util_spec.cr index 3ee4aac3..27d97c2a 100644 --- a/spec/util_spec.cr +++ b/spec/util_spec.cr @@ -21,7 +21,7 @@ describe "compare_numerically" do it "sorts like the stack exchange post" do ary = ["2", "12", "200000", "1000000", "a", "a12", "b2", "text2", "text2a", "text2a2", "text2a12", "text2ab", "text12", "text12a"] - ary.reverse.sort { |a, b| + ary.reverse.sort! { |a, b| compare_numerically a, b }.should eq ary end @@ -29,7 +29,7 @@ describe "compare_numerically" do # https://github.com/hkalexling/Mango/issues/22 it "handles numbers larger than Int32" do ary = ["14410155591588.jpg", "21410155591588.png", "104410155591588.jpg"] - ary.reverse.sort { |a, b| + ary.reverse.sort! { |a, b| compare_numerically a, b }.should eq ary end @@ -56,7 +56,7 @@ describe "chapter_sort" do it "sorts correctly" do ary = ["Vol.1 Ch.01", "Vol.1 Ch.02", "Vol.2 Ch. 2.5", "Ch. 3", "Ch.04"] sorter = ChapterSorter.new ary - ary.reverse.sort do |a, b| + ary.reverse.sort! do |a, b| sorter.compare a, b end.should eq ary end diff --git a/src/config.cr b/src/config.cr index a05c3a76..6de78a79 100644 --- a/src/config.cr +++ b/src/config.cr @@ -20,7 +20,6 @@ class Config property plugin_path : String = File.expand_path "~/mango/plugins", home: true property download_timeout_seconds : Int32 = 30 - property page_margin : Int32 = 30 property disable_login = false property default_username = "" property auth_proxy_header_name = "" @@ -34,8 +33,10 @@ class Config "download_retries" => 4, "download_queue_db_path" => File.expand_path("~/mango/queue.db", home: true), - "chapter_rename_rule" => "[Vol.{volume} ][Ch.{chapter} ]{title|id}", - "manga_rename_rule" => "{title}", + "chapter_rename_rule" => "[Vol.{volume} ]" \ + "[Ch.{chapter} ]{title|id}", + "manga_rename_rule" => "{title}", + "subscription_update_interval_hours" => 24, } @@singlet : Config? diff --git a/src/library/entry.cr b/src/library/entry.cr index cb458959..4ff01eed 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -86,7 +86,7 @@ class Entry SUPPORTED_IMG_TYPES.includes? \ MIME.from_filename? e.filename } - .sort { |a, b| + .sort! { |a, b| compare_numerically a.filename, b.filename } yield file, entries diff --git a/src/library/library.cr b/src/library/library.cr index 7e97e85c..8fad451b 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -42,6 +42,25 @@ class Library end end end + + subscription_interval = Config.current + .mangadex["subscription_update_interval_hours"].as Int32 + unless subscription_interval < 1 + spawn do + loop do + subscriptions = Storage.default.subscriptions + Logger.info "Checking MangaDex for updates on " \ + "#{subscriptions.size} subscriptions" + added_count = 0 + subscriptions.each do |sub| + added_count += sub.check_for_updates + end + Logger.info "Subscription update completed. Added #{added_count} " \ + "chapters to the download queue" + sleep subscription_interval.hours + end + end + end end def titles @@ -63,7 +82,7 @@ class Library end def deep_titles - titles + titles.map { |t| t.deep_titles }.flatten + titles + titles.flat_map &.deep_titles end def to_json(json : JSON::Builder) @@ -98,7 +117,7 @@ class Library .select { |path| File.directory? path } .map { |path| Title.new path, "" } .select { |title| !(title.entries.empty? && title.titles.empty?) } - .sort { |a, b| a.title <=> b.title } + .sort! { |a, b| a.title <=> b.title } .tap { |_| @title_ids.clear } .each do |title| @title_hash[title.id] = title @@ -114,7 +133,7 @@ class Library def get_continue_reading_entries(username) cr_entries = deep_titles - .map { |t| t.get_last_read_entry username } + .map(&.get_last_read_entry username) # Select elements with type `Entry` from the array and ignore all `Nil`s .select(Entry)[0...ENTRIES_IN_HOME_SECTIONS] .map { |e| @@ -150,14 +169,14 @@ class Library recently_added = [] of RA last_date_added = nil - titles.map { |t| t.deep_entries_with_date_added }.flatten - .select { |e| e[:date_added] > 1.month.ago } - .sort { |a, b| b[:date_added] <=> a[:date_added] } + titles.flat_map(&.deep_entries_with_date_added) + .select(&.[:date_added].> 1.month.ago) + .sort! { |a, b| b[:date_added] <=> a[:date_added] } .each do |e| break if recently_added.size > 12 last = recently_added.last? if last && e[:entry].book.id == last[:entry].book.id && - (e[:date_added] - last_date_added.not_nil!).duration < 1.day + (e[:date_added] - last_date_added.not_nil!).abs < 1.day # A NamedTuple is immutable, so we have to cast it to a Hash first last_hash = last.to_h count = last_hash[:grouped_count].as(Int32) @@ -188,9 +207,9 @@ class Library # If we use `deep_titles`, the start reading section might include `Vol. 2` # when the user hasn't started `Vol. 1` yet titles - .select { |t| t.load_percentage(username) == 0 } + .select(&.load_percentage(username).== 0) .sample(ENTRIES_IN_HOME_SECTIONS) - .shuffle + .shuffle! end def thumbnail_generation_progress @@ -205,7 +224,7 @@ class Library end Logger.info "Starting thumbnail generation" - entries = deep_titles.map(&.deep_entries).flatten.reject &.err_msg + entries = deep_titles.flat_map(&.deep_entries).reject &.err_msg @entries_count = entries.size @thumbnails_count = 0 diff --git a/src/library/title.cr b/src/library/title.cr index 4c439e72..a757d42c 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -44,14 +44,14 @@ class Title mtimes = [@mtime] mtimes += @title_ids.map { |e| Library.default.title_hash[e].mtime } - mtimes += @entries.map { |e| e.mtime } + mtimes += @entries.map &.mtime @mtime = mtimes.max @title_ids.sort! do |a, b| compare_numerically Library.default.title_hash[a].title, Library.default.title_hash[b].title end - sorter = ChapterSorter.new @entries.map { |e| e.title } + sorter = ChapterSorter.new @entries.map &.title @entries.sort! do |a, b| sorter.compare a.title, b.title end @@ -92,12 +92,12 @@ class Title # Get all entries, including entries in nested titles def deep_entries return @entries if title_ids.empty? - @entries + titles.map { |t| t.deep_entries }.flatten + @entries + titles.flat_map &.deep_entries end def deep_titles return [] of Title if titles.empty? - titles + titles.map { |t| t.deep_titles }.flatten + titles + titles.flat_map &.deep_titles end def parents @@ -138,7 +138,7 @@ class Title end def get_entry(eid) - @entries.find { |e| e.id == eid } + @entries.find &.id.== eid end def display_name @@ -217,29 +217,23 @@ class Title @entries.each do |e| e.save_progress username, e.pages end - titles.each do |t| - t.read_all username - end + titles.each &.read_all username end # Set the reading progress of all entries and nested libraries to 0% def unread_all(username) - @entries.each do |e| - e.save_progress username, 0 - end - titles.each do |t| - t.unread_all username - end + @entries.each &.save_progress(username, 0) + titles.each &.unread_all username end def deep_read_page_count(username) : Int32 load_progress_for_all_entries(username).sum + - titles.map { |t| t.deep_read_page_count username }.flatten.sum + titles.flat_map(&.deep_read_page_count username).sum end def deep_total_page_count : Int32 - entries.map { |e| e.pages }.sum + - titles.map { |t| t.deep_total_page_count }.flatten.sum + entries.sum(&.pages) + + titles.flat_map(&.deep_total_page_count).sum end def load_percentage(username) @@ -311,13 +305,13 @@ class Title ary = @entries.zip(percentage_ary) .sort { |a_tp, b_tp| (a_tp[1] <=> b_tp[1]).or \ compare_numerically a_tp[0].title, b_tp[0].title } - .map { |tp| tp[0] } + .map &.[0] else unless opt.method.auto? Logger.warn "Unknown sorting method #{opt.not_nil!.method}. Using " \ "Auto instead" end - sorter = ChapterSorter.new @entries.map { |e| e.title } + sorter = ChapterSorter.new @entries.map &.title ary = @entries.sort do |a, b| sorter.compare(a.title, b.title).or \ compare_numerically a.title, b.title @@ -383,13 +377,13 @@ class Title {entry: e, date_added: da_ary[i]} end return zip if title_ids.empty? - zip + titles.map { |t| t.deep_entries_with_date_added }.flatten + zip + titles.flat_map &.deep_entries_with_date_added end def bulk_progress(action, ids : Array(String), username) selected_entries = ids .map { |id| - @entries.find { |e| e.id == id } + @entries.find &.id.==(id) } .select(Entry) diff --git a/src/mangadex/downloader.cr b/src/mangadex/downloader.cr index c0b50c72..e677a719 100644 --- a/src/mangadex/downloader.cr +++ b/src/mangadex/downloader.cr @@ -49,6 +49,9 @@ module MangaDex @queue.set_status Queue::JobStatus::Downloading, job begin chapter = @client.chapter job.id + # We must put the `.pages` call in a rescue block to handle external + # chapters. + pages = chapter.pages rescue e Logger.error e @queue.set_status Queue::JobStatus::Error, job @@ -58,7 +61,7 @@ module MangaDex @downloading = false return end - @queue.set_pages chapter.pages.size, job + @queue.set_pages pages.size, job lib_dir = @library_path rename_rule = Rename::Rule.new \ Config.current.mangadex["manga_rename_rule"].to_s @@ -69,13 +72,13 @@ module MangaDex zip_path = File.join manga_dir, "#{job.title}.cbz.part" # Find the number of digits needed to store the number of pages - len = Math.log10(chapter.pages.size).to_i + 1 + len = Math.log10(pages.size).to_i + 1 writer = Compress::Zip::Writer.new zip_path # Create a buffered channel. It works as an FIFO queue - channel = Channel(PageJob).new chapter.pages.size + channel = Channel(PageJob).new pages.size spawn do - chapter.pages.each_with_index do |url, i| + pages.each_with_index do |url, i| fn = Path.new(URI.parse(url).path).basename ext = File.extname fn fn = "#{i.to_s.rjust len, '0'}#{ext}" @@ -99,7 +102,7 @@ module MangaDex spawn do page_jobs = [] of PageJob - chapter.pages.size.times do + pages.size.times do page_job = channel.receive break unless @queue.exists? job diff --git a/src/mangadex/ext.cr b/src/mangadex/ext.cr index dfb302c5..e919d970 100644 --- a/src/mangadex/ext.cr +++ b/src/mangadex/ext.cr @@ -35,7 +35,7 @@ module MangaDex struct Chapter def rename(rule : Rename::Rule) hash = properties_to_hash %w(id title volume chapter lang_code language) - hash["groups"] = groups.map(&.name).join "," + hash["groups"] = groups.join(",", &.name) rule.render hash end @@ -56,5 +56,39 @@ module MangaDex hash["full_title"] = JSON::Any.new full_title hash.to_json end + + # We don't need to rename the manga title here. It will be renamed in + # src/mangadex/downloader.cr + def to_job : Queue::Job + Queue::Job.new( + id.to_s, + manga_id.to_s, + full_title, + manga_title, + Queue::JobStatus::Pending, + Time.unix timestamp + ) + end + end + + struct User + def updates_after(time : Time, &block : Chapter ->) + page = 1 + stopped = false + until stopped + chapters = followed_updates(page: page).chapters + return if chapters.empty? + chapters.each do |c| + if time > Time.unix c.timestamp + stopped = true + break + end + yield c + end + page += 1 + # Let's not DDOS MangaDex :) + sleep 5.seconds + end + end end end diff --git a/src/mango.cr b/src/mango.cr index b5fd1684..768058ff 100644 --- a/src/mango.cr +++ b/src/mango.cr @@ -8,7 +8,7 @@ require "option_parser" require "clim" require "tallboy" -MANGO_VERSION = "0.21.0" +MANGO_VERSION = "0.22.0" # From http://www.network-science.de/ascii/ BANNER = %{ diff --git a/src/plugin/plugin.cr b/src/plugin/plugin.cr index baa77d1a..6bedea17 100644 --- a/src/plugin/plugin.cr +++ b/src/plugin/plugin.cr @@ -117,7 +117,7 @@ class Plugin def initialize(id : String) Plugin.build_info_ary - @info = @@info_ary.find { |i| i.id == id } + @info = @@info_ary.find &.id.== id if @info.nil? raise Error.new "Plugin with ID #{id} not found" end diff --git a/src/queue.cr b/src/queue.cr index c9f805c1..381441b6 100644 --- a/src/queue.cr +++ b/src/queue.cr @@ -303,12 +303,12 @@ class Queue end def pause - @downloaders.each { |d| d.stopped = true } + @downloaders.each &.stopped=(true) @paused = true end def resume - @downloaders.each { |d| d.stopped = false } + @downloaders.each &.stopped=(false) @paused = false end diff --git a/src/rename.cr b/src/rename.cr index 1fc7693d..3e00fdcb 100644 --- a/src/rename.cr +++ b/src/rename.cr @@ -35,15 +35,15 @@ module Rename class Group < Base(Pattern | String) def render(hash : VHash) - return "" if @ary.select(&.is_a? Pattern) + return "" if @ary.select(Pattern) .any? &.as(Pattern).render(hash).empty? - @ary.map do |e| + @ary.join do |e| if e.is_a? Pattern e.render hash else e end - end.join + end end end @@ -129,13 +129,13 @@ module Rename end def render(hash : VHash) - str = @ary.map do |e| + str = @ary.join do |e| if e.is_a? String e else e.render hash end - end.join.strip + end.strip post_process str end diff --git a/src/routes/api.cr b/src/routes/api.cr index 7d2af128..0102e0db 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -339,7 +339,7 @@ struct APIRouter } post "/api/admin/mangadex/download" do |env| begin - chapters = env.params.json["chapters"].as(Array).map { |c| c.as_h } + chapters = env.params.json["chapters"].as(Array).map &.as_h jobs = chapters.map { |chapter| Queue::Job.new( chapter["id"].as_i64.to_s, @@ -366,7 +366,7 @@ struct APIRouter interval = (interval_raw.to_i? if interval_raw) || 5 loop do socket.send({ - "jobs" => Queue.default.get_all, + "jobs" => Queue.default.get_all.reverse, "paused" => Queue.default.paused?, }.to_json) sleep interval.seconds @@ -390,13 +390,13 @@ struct APIRouter } get "/api/admin/mangadex/queue" do |env| begin - jobs = Queue.default.get_all send_json env, { - "jobs" => jobs, + "jobs" => Queue.default.get_all.reverse, "paused" => Queue.default.paused?, "success" => true, }.to_json rescue e + Logger.error e send_json env, { "success" => false, "error" => e.message, @@ -444,6 +444,7 @@ struct APIRouter send_json env, {"success" => true}.to_json rescue e + Logger.error e send_json env, { "success" => false, "error" => e.message, @@ -516,6 +517,7 @@ struct APIRouter raise "No part with name `file` found" rescue e + Logger.error e send_json env, { "success" => false, "error" => e.message, @@ -551,6 +553,7 @@ struct APIRouter "title" => title, }.to_json rescue e + Logger.error e send_json env, { "success" => false, "error" => e.message, @@ -594,6 +597,7 @@ struct APIRouter "fail": jobs.size - inserted_count, }.to_json rescue e + Logger.error e send_json env, { "success" => false, "error" => e.message, @@ -612,7 +616,6 @@ struct APIRouter "width" => Int32, "height" => Int32, }], - "margin" => Int32?, } get "/api/dimensions/:tid/:eid" do |env| begin @@ -628,9 +631,9 @@ struct APIRouter send_json env, { "success" => true, "dimensions" => sizes, - "margin" => Config.current.page_margin, }.to_json rescue e + Logger.error e send_json env, { "success" => false, "error" => e.message, @@ -770,6 +773,7 @@ struct APIRouter "titles" => Storage.default.missing_titles, }.to_json rescue e + Logger.error e send_json env, { "success" => false, "error" => e.message, @@ -796,6 +800,7 @@ struct APIRouter "entries" => Storage.default.missing_entries, }.to_json rescue e + Logger.error e send_json env, { "success" => false, "error" => e.message, @@ -814,6 +819,7 @@ struct APIRouter "error" => nil, }.to_json rescue e + Logger.error e send_json env, { "success" => false, "error" => e.message, @@ -832,6 +838,7 @@ struct APIRouter "error" => nil, }.to_json rescue e + Logger.error e send_json env, { "success" => false, "error" => e.message, @@ -853,6 +860,7 @@ struct APIRouter "error" => nil, }.to_json rescue e + Logger.error e send_json env, { "success" => false, "error" => e.message, @@ -874,6 +882,7 @@ struct APIRouter "error" => nil, }.to_json rescue e + Logger.error e send_json env, { "success" => false, "error" => e.message, @@ -963,23 +972,147 @@ struct APIRouter Koa.tags ["admin", "mangadex"] get "/api/admin/mangadex/search" do |env| begin - username = get_username env - token, expires = Storage.default.get_md_token username + query = env.params.query["query"] - unless expires && token - raise "No token found for user #{username}" - end + send_json env, { + "success" => true, + "error" => nil, + "manga" => get_client(env).partial_search query, + }.to_json + rescue e + Logger.error e + send_json env, { + "success" => false, + "error" => e.message, + }.to_json + end + end - client = MangaDex::Client.from_config - client.token = token - client.token_expires = expires + Koa.describe "Lists all MangaDex subscriptions" + Koa.response 200, schema: { + "success" => Bool, + "error" => String?, + "subscriptions?" => [{ + "id" => Int64, + "username" => String, + "manga_id" => Int64, + "language" => String?, + "group_id" => Int64?, + "min_volume" => Int64?, + "max_volume" => Int64?, + "min_chapter" => Int64?, + "max_chapter" => Int64?, + "last_checked" => Int64, + "created_at" => Int64, + }], + } + Koa.tags ["admin", "mangadex", "subscriptions"] + get "/api/admin/mangadex/subscriptions" do |env| + begin + send_json env, { + "success" => true, + "error" => nil, + "subscriptions" => Storage.default.subscriptions, + }.to_json + rescue e + Logger.error e + send_json env, { + "success" => false, + "error" => e.message, + }.to_json + end + end - query = env.params.query["query"] + Koa.describe "Creates a new MangaDex subscription" + Koa.body schema: { + "subscription" => { + "manga" => Int64, + "language" => String?, + "groupId" => Int64?, + "volumeMin" => Int64?, + "volumeMax" => Int64?, + "chapterMin" => Int64?, + "chapterMax" => Int64?, + }, + } + Koa.response 200, schema: { + "success" => Bool, + "error" => String?, + } + Koa.tags ["admin", "mangadex", "subscriptions"] + post "/api/admin/mangadex/subscriptions" do |env| + begin + json = env.params.json["subscription"].as Hash(String, JSON::Any) + sub = Subscription.new json["manga"].as_i64, get_username env + sub.language = json["language"]?.try &.as_s? + sub.group_id = json["groupId"]?.try &.as_i64? + sub.min_volume = json["volumeMin"]?.try &.as_i64? + sub.max_volume = json["volumeMax"]?.try &.as_i64? + sub.min_chapter = json["chapterMin"]?.try &.as_i64? + sub.max_chapter = json["chapterMax"]?.try &.as_i64? + + Storage.default.save_subscription sub + + send_json env, { + "success" => true, + "error" => nil, + }.to_json + rescue e + Logger.error e + send_json env, { + "success" => false, + "error" => e.message, + }.to_json + end + end + + Koa.describe "Deletes a MangaDex subscription identified by `id`", <<-MD + Does nothing if the subscription was not created by the current user. + MD + Koa.response 200, schema: { + "success" => Bool, + "error" => String?, + } + Koa.tags ["admin", "mangadex", "subscriptions"] + delete "/api/admin/mangadex/subscriptions/:id" do |env| + begin + id = env.params.url["id"].to_i64 + Storage.default.delete_subscription id, get_username env + send_json env, { + "success" => true, + "error" => nil, + }.to_json + rescue e + Logger.error e + send_json env, { + "success" => false, + "error" => e.message, + }.to_json + end + end + Koa.describe "Triggers an update for a MangaDex subscription identified by `id`", <<-MD + Does nothing if the subscription was not created by the current user. + MD + Koa.response 200, schema: { + "success" => Bool, + "error" => String?, + } + Koa.tags ["admin", "mangadex", "subscriptions"] + post "/api/admin/mangadex/subscriptions/check/:id" do |env| + begin + id = env.params.url["id"].to_i64 + username = get_username env + sub = Storage.default.get_subscription id, username + unless sub + raise "Subscription with id #{id} not found under user #{username}" + end + spawn do + sub.check_for_updates + end send_json env, { "success" => true, "error" => nil, - "manga" => client.partial_search query, }.to_json rescue e Logger.error e diff --git a/src/routes/main.cr b/src/routes/main.cr index 65048005..2993db7a 100644 --- a/src/routes/main.cr +++ b/src/routes/main.cr @@ -95,6 +95,12 @@ struct MainRouter end end + get "/download/subscription" do |env| + mangadex_base_url = Config.current.mangadex["base_url"] + username = get_username env + layout "subscription" + end + get "/" do |env| begin username = get_username env @@ -103,7 +109,7 @@ struct MainRouter recently_added = Library.default.get_recently_added_entries username start_reading = Library.default.get_start_reading_titles username titles = Library.default.titles - new_user = !titles.any? { |t| t.load_percentage(username) > 0 } + new_user = !titles.any? &.load_percentage(username).> 0 empty_library = titles.size == 0 layout "home" rescue e diff --git a/src/storage.cr b/src/storage.cr index 971bba36..164ce401 100644 --- a/src/storage.cr +++ b/src/storage.cr @@ -5,6 +5,7 @@ require "base64" require "./util/*" require "mg" require "../migration/*" +require "./subscription" def hash_password(pw) Crypto::Bcrypt::Password.create(pw).to_s @@ -14,6 +15,9 @@ def verify_password(hash, pw) (Crypto::Bcrypt::Password.new hash).verify pw end +SUB_ATTR = %w(manga_id language group_id min_volume max_volume min_chapter + max_chapter username) + class Storage @@insert_entry_ids = [] of IDTuple @@insert_title_ids = [] of IDTuple @@ -445,7 +449,7 @@ class Storage Logger.debug "Marking #{trash_ids.size} entries as unavailable" end db.exec "update ids set unavailable = 1 where id in " \ - "(#{trash_ids.map { |i| "'#{i}'" }.join ","})" + "(#{trash_ids.join "," { |i| "'#{i}'" }})" # Detect dangling title IDs trash_titles = [] of String @@ -461,7 +465,7 @@ class Storage Logger.debug "Marking #{trash_titles.size} titles as unavailable" end db.exec "update titles set unavailable = 1 where id in " \ - "(#{trash_titles.map { |i| "'#{i}'" }.join ","})" + "(#{trash_titles.join "," { |i| "'#{i}'" }})" end end end @@ -545,6 +549,70 @@ class Storage {token, expires} end + def save_subscription(sub : Subscription) + MainFiber.run do + get_db do |db| + {% begin %} + db.exec "insert into subscription (#{SUB_ATTR.join ","}, " \ + "last_checked, created_at) values " \ + "(#{Array.new(SUB_ATTR.size + 2, "?").join ","})", + {% for type in SUB_ATTR %} + sub.{{type.id}}, + {% end %} + sub.last_checked.to_unix, sub.created_at.to_unix + {% end %} + end + end + end + + def subscriptions : Array(Subscription) + subs = [] of Subscription + MainFiber.run do + get_db do |db| + db.query "select * from subscription" do |rs| + subs += Subscription.from_rs rs + end + end + end + subs + end + + def delete_subscription(id : Int64, username : String) + MainFiber.run do + get_db do |db| + db.exec "delete from subscription where id = (?) and username = (?)", + id, username + end + end + end + + def get_subscription(id : Int64, username : String) : Subscription? + sub = nil + MainFiber.run do + get_db do |db| + db.query "select * from subscription where id = (?) and " \ + "username = (?) limit 1", id, username do |rs| + sub = Subscription.from_rs(rs).first? + end + end + end + sub + end + + def update_subscription_last_checked(id : Int64? = nil) + MainFiber.run do + get_db do |db| + if id + db.exec "update subscription set last_checked = (?) where id = (?)", + Time.utc.to_unix, id + else + db.exec "update subscription set last_checked = (?)", + Time.utc.to_unix + end + end + end + end + def close MainFiber.run do unless @db.nil? diff --git a/src/subscription.cr b/src/subscription.cr new file mode 100644 index 00000000..e3913601 --- /dev/null +++ b/src/subscription.cr @@ -0,0 +1,83 @@ +require "db" +require "json" + +struct Subscription + include DB::Serializable + include JSON::Serializable + + getter id : Int64 = 0 + getter username : String + getter manga_id : Int64 + property language : String? + property group_id : Int64? + property min_volume : Int64? + property max_volume : Int64? + property min_chapter : Int64? + property max_chapter : Int64? + @[DB::Field(key: "last_checked")] + @[JSON::Field(key: "last_checked")] + @raw_last_checked : Int64 + @[DB::Field(key: "created_at")] + @[JSON::Field(key: "created_at")] + @raw_created_at : Int64 + + def last_checked : Time + Time.unix @raw_last_checked + end + + def created_at : Time + Time.unix @raw_created_at + end + + def initialize(@manga_id, @username) + @raw_created_at = Time.utc.to_unix + @raw_last_checked = Time.utc.to_unix + end + + private def in_range?(value : String, lowerbound : Int64?, + upperbound : Int64?) : Bool + lb = lowerbound.try &.to_f64 + ub = upperbound.try &.to_f64 + + return true if lb.nil? && ub.nil? + + v = value.to_f64? + return false unless v + + if lb.nil? + v <= ub.not_nil! + elsif ub.nil? + v >= lb.not_nil! + else + v >= lb.not_nil! && v <= ub.not_nil! + end + end + + def match?(chapter : MangaDex::Chapter) : Bool + if chapter.manga_id != manga_id || + (language && chapter.language != language) || + (group_id && !chapter.groups.map(&.id).includes? group_id) + return false + end + + in_range?(chapter.volume, min_volume, max_volume) && + in_range?(chapter.chapter, min_chapter, max_chapter) + end + + def check_for_updates : Int32 + Logger.debug "Checking updates for subscription with ID #{id}" + jobs = [] of Queue::Job + get_client(username).user.updates_after last_checked do |chapter| + next unless match? chapter + jobs << chapter.to_job + end + Storage.default.update_subscription_last_checked id + count = Queue.default.push jobs + Logger.debug "#{count}/#{jobs.size} of updates added to queue" + count + rescue e + Logger.error "Error occurred when checking updates for " \ + "subscription with ID #{id}. #{e}" + 0 + end +end diff --git a/src/util/chapter_sort.cr b/src/util/chapter_sort.cr index 44dfb4e1..eed7a3ec 100644 --- a/src/util/chapter_sort.cr +++ b/src/util/chapter_sort.cr @@ -73,7 +73,7 @@ class ChapterSorter .select do |key| keys[key].count >= str_ary.size / 2 end - .sort do |a_key, b_key| + .sort! do |a_key, b_key| a = keys[a_key] b = keys[b_key] # Sort keys by the number of times they appear diff --git a/src/util/numeric_sort.cr b/src/util/numeric_sort.cr index 7365a9f7..c455b47a 100644 --- a/src/util/numeric_sort.cr +++ b/src/util/numeric_sort.cr @@ -11,7 +11,7 @@ end def split_by_alphanumeric(str) arr = [] of String str.scan(/([^\d\n\r]*)(\d*)([^\d\n\r]*)/) do |match| - arr += match.captures.select { |s| s != "" } + arr += match.captures.select &.!= "" end arr end diff --git a/src/util/util.cr b/src/util/util.cr index 8903f5ef..c4e168a7 100644 --- a/src/util/util.cr +++ b/src/util/util.cr @@ -114,7 +114,7 @@ class String def components_similarity(other : String) : Float64 s, l = [self, other] .map { |str| Path.new(str).parts } - .sort_by &.size + .sort_by! &.size match = s.reverse.zip(l.reverse).count { |a, b| a == b } match / s.size diff --git a/src/util/web.cr b/src/util/web.cr index 67227c71..efa269d8 100644 --- a/src/util/web.cr +++ b/src/util/web.cr @@ -72,7 +72,7 @@ def redirect(env, path) end def hash_to_query(hash) - hash.map { |k, v| "#{k}=#{v}" }.join("&") + hash.join "&" { |k, v| "#{k}=#{v}" } end def request_path_startswith(env, ary) @@ -107,6 +107,25 @@ macro get_sort_opt end end +# Returns an authorized client +def get_client(username : String) : MangaDex::Client + token, expires = Storage.default.get_md_token username + + unless expires && token + raise "No token found for user #{username}" + end + + client = MangaDex::Client.from_config + client.token = token + client.token_expires = expires + + client +end + +def get_client(env) : MangaDex::Client + get_client get_username env +end + module HTTP class Client private def self.exec(uri : URI, tls : TLSContext = nil) diff --git a/src/views/download-manager.html.ecr b/src/views/download-manager.html.ecr index 2b6e4348..2075e01e 100644 --- a/src/views/download-manager.html.ecr +++ b/src/views/download-manager.html.ecr @@ -5,61 +5,63 @@ - - - - - - - - - - - - - - + +
ChapterMangaProgressTimeStatusPluginActions
+ <% content_for "script" do %> diff --git a/src/views/download.html.ecr b/src/views/download.html.ecr index 0ea85276..983cae00 100644 --- a/src/views/download.html.ecr +++ b/src/views/download.html.ecr @@ -1,162 +1,170 @@

Download from MangaDex

-
-
- -
-
-
- +
+
+ +
+
+
+ +
-
- -
- -
- -
-
+
+
+
+ +
+
+

Title:

+

+

+
+
+

+ Filter Chapters + +

+

+
+ +
+ +
+
-
- -
- -
+
+ +
+ +
+
+ +
+ +
+ +
+
+ +
+ +
+ +
+
+
- -
- -
+
+ + + +
+
+

Click on a table row to select the chapter. Drag your mouse over multiple rows to select them all. Hold Ctrl to make multiple non-adjacent selections.

-
-
- -
-
- - - -
-
-

Click on a table row to select the chapter. Drag your mouse over multiple rows to select them all. Hold Ctrl to make multiple non-adjacent selections.

-
-

- - - - - - - - - - - - +

+
+
IDTitleLanguageGroupVolumeChapterTimestamp
+ + + + + + + + + + + - @@ -80,6 +80,7 @@ +
@@ -90,6 +91,13 @@
+
+ +
+ +
+
+
@@ -110,12 +118,12 @@
<% if previous_entry_url %> - Previous Entry + Previous Entry <% end %> <% if next_entry_url %> - Next Entry + Next Entry <% end %> - Exit Reader + Exit Reader
diff --git a/src/views/subscription.html.ecr b/src/views/subscription.html.ecr new file mode 100644 index 00000000..cc96c471 --- /dev/null +++ b/src/views/subscription.html.ecr @@ -0,0 +1,54 @@ +

MangaDex Subscription Manager

+ +
+

The subscription manager uses a MangaDex API that requires authentication. Please connect to MangaDex before using this feature.

+ +

No subscription found. Go to the MangaDex download page and start subscribing.

+ + +
+ +<% content_for "script" do %> + <%= render_component "moment" %> + + +<% end %>
IDTitleLanguageGroupVolumeChapterTimestamp