Merge pull request #734 from toyhammered/my-anime-list-importer

MyAnimeList Importer

server/lib/data_import/my_anime_list.rb

@@ -0,0 +1,45 @@
+module DataImport
+  class MyAnimeList
+    ATARASHII_API_HOST = 'https://hbv3-mal-api.herokuapp.com/2.1/'.freeze
+
+    include DataImport::Media
+    include DataImport::HTTP
+
+    attr_reader :opts
+
+    def initialize(opts = {})
+      @opts = opts.with_indifferent_access
+      super()
+    end
+
+    def get_media(external_id) # anime/1234 or manga/1234
+      media = Mapping.lookup('myanimelist', external_id)
+      # should return Anime or Manga
+      klass = external_id.split('/').first.classify.constantize
+      # initialize the class
+      media ||= klass.new
+
+      get(external_id) do |response|
+        details = Extractor::Media.new(response)
+
+        media.assign_attributes(details.to_h.compact)
+        media.genres = details.genres.map { |genre|
+          Genre.find_by(name: genre)
+        }.compact
+
+        yield media
+      end
+    end
+
+    private
+
+    def get(url, opts = {})
+      super(build_url(url), opts)
+    end
+
+    def build_url(path)
+      return path if path.include?('://')
+      "#{ATARASHII_API_HOST}#{path}"
+    end
+  end
+end

server/lib/data_import/my_anime_list/extractor/media.rb

@@ -0,0 +1,160 @@
+module DataImport
+  class MyAnimeList
+    module Extractor
+      class Media
+        attr_reader :data
+
+        def initialize(json)
+          @data = JSON.parse(json)
+        end
+
+        def age_rating
+          return unless data['classification']
+          rating = data['classification'].split(' - ')
+
+          case rating[0]
+          when 'G', 'TV-Y7' then :G
+          when 'PG', 'PG13' then :PG
+          when 'R', 'R+' then :R
+          when 'Rx' then :R18
+          end
+        end
+
+        def episode_count
+          data['episodes']
+        end
+
+        def episode_length
+          data['duration']
+        end
+
+        def synopsis
+          clean_desc(data['synopsis'])
+        end
+
+        def youtube_video_id
+          data['preview']&.split('/')&.last
+        end
+
+        def poster_image
+          data['image_url']
+        end
+
+        def age_rating_guide
+          return unless data['classification']
+          rating = data['classification'].split(' - ')
+
+          return 'Violence, Profanity' if rating[0] == 'R'
+          return rating[1] if rating[1].present?
+
+          # fallback
+          case rating[0]
+          when 'G' then 'All Ages'
+          when 'PG' then 'Children'
+          when 'PG13', 'PG-13' then 'Teens 13 or older'
+          # when 'R' then 'Violence, Profanity'
+          # this will NEVER happen because of return
+          when 'R+' then 'Mild Nudity'
+          when 'Rx' then 'Hentai'
+          end
+        end
+
+        def subtype # will be renamed to this hopefully
+          # anime matches [TV special OVA ONA movie music]
+          # manga matches [manga novel manhua oneshot doujin]
+          type = data['type'].downcase
+
+          case type
+          when 'tv' then :TV
+          when 'ova' then :OVA
+          when 'ona' then :ONA
+          else type.to_sym
+          end
+        end
+
+        def start_date
+          data['start_date']&.to_date
+        end
+
+        def end_date
+          data['end_date']&.to_date
+        end
+
+        def titles
+          {
+            en_jp: data['title'],
+            en_us: data['other_titles']['english'].try(:first),
+            ja_jp: data['other_titles']['japanese'].try(:first)
+          }
+        end
+
+        def abbreviated_titles
+          data['other_titles']['synonyms']
+        end
+
+        # Manga Specific
+
+        def chapters
+          data['chapters']
+        end
+
+        def volumes
+          data['volumes']
+        end
+
+        # TODO: removed subtype (show_type, manga_type issue)
+        # TODO: missing status on manga (anime does automagically)
+        def to_h
+          %i[age_rating episode_count episode_length synopsis youtube_video_id
+             poster_image age_rating_guide start_date end_date
+             titles abbreviated_titles chapters volumes]
+            .map { |k|
+              [k, send(k)]
+            }.to_h
+        end
+
+        def genres
+          data['genres']
+        end
+
+        # synopsis: seriously don't touch this unless you are Nuck.
+        def br_to_p(src)
+          src = '<p>' + src.gsub(/<br>\s*<br>/, '</p><p>') + '</p>'
+          doc = Nokogiri::HTML.fragment src
+          doc.traverse do |x|
+            next x.remove if x.name == 'br' && x.previous.nil?
+            next x.remove if x.name == 'br' && x.next.nil?
+            next x.remove if x.name == 'br' && x.next.name == 'p' && x.previous.name == 'p'
+            next x.remove if x.name == 'p' && x.content.blank?
+          end
+          doc.inner_html.gsub(/[\r\n\t]/, '')
+        end
+
+        # synopsis: seriously don't touch this unless you are Nuck.
+        def clean_desc(desc)
+          desc = Nokogiri::HTML.fragment br_to_p(desc)
+          desc.css('.spoiler').each do |x|
+            x.name = 'span'
+            x.inner_html = x.css('.spoiler_content').inner_html
+            x.css('input').remove
+          end
+          desc.css('.spoiler').wrap('<p></p>')
+          desc.xpath('descendant::comment()').remove
+          desc.css('b').each { |b| b.replace(b.content) }
+          desc.traverse do |node|
+            next unless node.text?
+            t = node.content.split(/: ?/).map { |x| x.split(' ') }
+            if t.length >= 2
+              if t[0].length <= 3 && t[1].length <= 20
+                node.remove
+              end
+            else
+              node.remove if /^\s+\*\s+.*/ =~ node.content
+            end
+          end
+          desc.inner_html
+        end
+      end
+    end
+  end
+end

server/spec/fixtures/my_anime_list/berserk-manga.json

@@ -0,0 +1,79 @@
+{
+  "id": 2,
+  "title": "Berserk",
+  "other_titles": {
+    "english": [
+      "Berserk"
+    ],
+    "synonyms": [
+      "Berserk: The Prototype"
+    ],
+    "japanese": [
+      "\u30d9\u30eb\u30bb\u30eb\u30af"
+    ]
+  },
+  "rank": 1,
+  "popularity_rank": 8,
+  "image_url": "http:\/\/cdn.myanimelist.net\/images\/manga\/1\/157931.jpg",
+  "type": "Manga",
+  "status": "publishing",
+  "members_score": 9.24,
+  "members_count": 104891,
+  "favorited_count": 24537,
+  "synopsis": "Guts, a former mercenary now known as the \"Black Swordsman,\" is out for revenge.",
+  "genres": [
+    "Action",
+    "Adventure",
+    "Demons",
+    "Drama",
+    "Fantasy",
+    "Horror",
+    "Supernatural",
+    "Military",
+    "Psychological",
+    "Seinen"
+  ],
+  "tags": [
+
+  ],
+  "anime_adaptations": [
+    {
+      "anime_id": 33,
+      "title": "Berserk",
+      "url": "http:\/\/myanimelist.net\/anime\/33\/Berserk"
+    },
+    {
+      "anime_id": 10218,
+      "title": "Berserk: Ougon Jidai-hen I - Haou no Tamago",
+      "url": "http:\/\/myanimelist.net\/anime\/10218\/Berserk__Ougon_Jidai-hen_I_-_Haou_no_Tamago"
+    },
+    {
+      "anime_id": 12113,
+      "title": "Berserk: Ougon Jidai-hen II - Doldrey Kouryaku",
+      "url": "http:\/\/myanimelist.net\/anime\/12113\/Berserk__Ougon_Jidai-hen_II_-_Doldrey_Kouryaku"
+    },
+    {
+      "anime_id": 12115,
+      "title": "Berserk: Ougon Jidai-hen III - Kourin",
+      "url": "http:\/\/myanimelist.net\/anime\/12115\/Berserk__Ougon_Jidai-hen_III_-_Kourin"
+    },
+    {
+      "anime_id": 32379,
+      "title": "Berserk (2016)",
+      "url": "http:\/\/myanimelist.net\/anime\/32379\/Berserk_2016"
+    }
+  ],
+  "related_manga": [
+    {
+      "manga_id": 92299,
+      "title": "Berserk: Shinen no Kami 2",
+      "url": "http:\/\/myanimelist.net\/manga\/92299\/Berserk__Shinen_no_Kami_2"
+    }
+  ],
+  "alternative_versions": [
+
+  ],
+  "personal_tags": [
+
+  ]
+}