Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/korean drama engine #34

Merged
merged 5 commits into from
Aug 19, 2020
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -36,6 +36,10 @@ Gophie is a tool to help you search, stream and download movies from movie sites
- AnimeOut
- Takanimelist

### Korean

- KDramaHood

Gophie also has [mobile](https://github.com/Go-phie/gophie-mobile) and [web](https://github.com/Go-phie/gophie-web) clients.

## Installation
27 changes: 27 additions & 0 deletions cmd/cmd_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
package cmd

import (
. "github.com/smartystreets/goconvey/convey"
"testing"
)

func TestCmd(t *testing.T) {
Convey("CLI [Test listing engines]\n", t, func() {
args := []string{"engines", "list"}
rootCmd.SetArgs(args)
So(rootCmd.Execute(), ShouldBeNil)
})

Convey("CLI [Test current version]\n", t, func() {
args := []string{"version"}
rootCmd.SetArgs(args)
So(rootCmd.Execute(), ShouldBeNil)
})

Convey("CLI [Test clearing cache]\n", t, func() {
args := []string{"clear-cache"}
rootCmd.SetArgs(args)
So(rootCmd.Execute(), ShouldBeNil)
})

}
2 changes: 1 addition & 1 deletion cmd/list.go
Original file line number Diff line number Diff line change
@@ -61,7 +61,7 @@ func listPager(pageNum int) {
index++
}
searchResult := engine.SearchResult{
Query: selectedMovie.Title + "EPISODES",
Query: selectedMovie.Title + " EPISODES",
Movies: movieArray,
}
selectedMovie = processList(pageNum, selectedEngine, searchResult)
2 changes: 1 addition & 1 deletion cmd/search.go
Original file line number Diff line number Diff line change
@@ -83,7 +83,7 @@ func searchPager(param ...string) {
index++
}
searchResult := engine.SearchResult{
Query: selectedMovie.Title + "EPISODES",
Query: selectedMovie.Title + " EPISODES",
Movies: movieArray,
}
selectedMovie = processSearch(selectedEngine, searchResult, param...)
3 changes: 3 additions & 0 deletions cmd/version_num.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
package cmd

const Version = "0.3.6"
21 changes: 11 additions & 10 deletions engine/engine_test.go
Original file line number Diff line number Diff line change
@@ -4,23 +4,24 @@ import (
"fmt"
"strings"
"testing"
// _ "github.com/go-phie/gophie/cmd"
)

func testResults(t *testing.T, engine Engine) {
counter := map[string]int{}
var result SearchResult
var searchTerm string
fmt.Println(engine.String())
if !strings.HasPrefix(engine.String(), "TvSeries") {
if strings.HasPrefix(engine.String(), "Anime") || strings.HasPrefix(engine.String(), "Takanime") {
searchTerm = "attack on titan"
} else {
searchTerm = "jumanji"
}
} else {
// search for the flash for movie series
// different search terms on engines
switch {
case strings.HasPrefix(engine.String(), "TvSeries"):
searchTerm = "devs"
case strings.HasPrefix(engine.String(), "TakanimeList"),
strings.HasPrefix(engine.String(), "AnimeOut"):
searchTerm = "attack on titans"
case strings.HasPrefix(engine.String(), "KDramaHood"):
searchTerm = "flower of evil"
default:
searchTerm = "jumanji"
}
result = engine.Search(searchTerm)

@@ -37,7 +38,7 @@ func testResults(t *testing.T, engine Engine) {
}
if movie.IsSeries == false {
downloadlink := strings.ToLower(movie.DownloadLink.String())
if !(strings.HasSuffix(downloadlink, "1") || strings.HasSuffix(downloadlink, ".mp4") || strings.Contains(downloadlink, ".mkv") || strings.Contains(downloadlink, ".avi") || strings.Contains(downloadlink, ".webm") || strings.Contains(downloadlink, "freeload") || strings.Contains(downloadlink, "download_token=") || strings.Contains(downloadlink, "mycoolmoviez") || strings.Contains(downloadlink, "server")) {
if !(strings.HasSuffix(downloadlink, "1") || strings.HasSuffix(downloadlink, ".mp4") || strings.Contains(downloadlink, ".mkv") || strings.Contains(downloadlink, ".avi") || strings.Contains(downloadlink, ".webm") || strings.Contains(downloadlink, "freeload") || strings.Contains(downloadlink, "download_token=") || strings.Contains(downloadlink, "mycoolmoviez") || strings.Contains(downloadlink, "server") || strings.Contains(downloadlink, "kdramahood")) {
t.Errorf("Could not obtain link for single movie, linked returned is %v", downloadlink)
}
}
15 changes: 14 additions & 1 deletion engine/engines.go
Original file line number Diff line number Diff line change
@@ -100,6 +100,10 @@ func Scrape(engine Engine) ([]Movie, error) {
log.Fatal(err)
}

// c.OnHTML("div", func(e *colly.HTMLElement) {
// log.Debugf("%#v", e)
// })

c.OnHTML(main, func(e *colly.HTMLElement) {
e.ForEach(article, func(_ int, el *colly.HTMLElement) {
movie, err := engine.parseSingleMovie(el, movieIndex)
@@ -167,14 +171,17 @@ type Movie struct {
Category string // csv of categories
Cast string // csv of actors in movie
UploadDate string
Source string // The Engine From which it is gotten from
Source string // The Engine From which it is gotten from
SubtitleLink *url.URL // single subtitle link
SubtitleLinks map[string]*url.URL // Subtitle links for a series
}

// MovieJSON : JSON structure of all downloadable movies
type MovieJSON struct {
Movie
DownloadLink string
SDownloadLink map[string]string
SubtitleLinks map[string]string
}

func (m *Movie) String() string {
@@ -187,11 +194,16 @@ func (m *Movie) MarshalJSON() ([]byte, error) {
for key, val := range m.SDownloadLink {
sDownloadLink[key] = val.String()
}
subtitleLinks := make(map[string]string)
for key, val := range m.SubtitleLinks {
subtitleLinks[key] = val.String()
}

movie := MovieJSON{
Movie: *m,
DownloadLink: m.DownloadLink.String(),
SDownloadLink: sDownloadLink,
SubtitleLinks: subtitleLinks,
}

return json.Marshal(movie)
@@ -244,6 +256,7 @@ func GetEngines() map[string]Engine {
engines["coolmoviez"] = NewCoolMoviezEngine()
engines["animeout"] = NewAnimeOutEngine()
engines["takanimelist"] = NewTakanimeListEngine()
engines["kdramahood"] = NewKDramaHoodEngine()
return engines
}

165 changes: 165 additions & 0 deletions engine/kdramahood.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,165 @@
package engine

import (
"fmt"
"net/url"
"path"
"strconv"
"strings"

"github.com/gocolly/colly/v2"
log "github.com/sirupsen/logrus"
)

// KDramaHood : An Engine for KDramaHood
type KDramaHood struct {
Props
}

// NewKDramaHoodEngine : create a new engine for scraping latest korean drama
func NewKDramaHoodEngine() *KDramaHood {
base := "https://kdramahood.com"
baseURL, err := url.Parse(base)
if err != nil {
log.Fatal(err)
}
// Search URL
searchURL, err := url.Parse(base)
if err != nil {
log.Fatal(err)
}
searchURL.Path = "/"

// List URL
listURL, err := url.Parse(base)
if err != nil {
log.Fatal(err)
}
listURL.Path = "/home2/"

dramaFeverEngine := KDramaHood{}
dramaFeverEngine.Name = "KDramaHood"
dramaFeverEngine.BaseURL = baseURL
dramaFeverEngine.Description = `Watch your favourite korean movie all in one place`
dramaFeverEngine.SearchURL = searchURL
dramaFeverEngine.ListURL = listURL
return &dramaFeverEngine
}

// Engine Interface Methods

func (engine *KDramaHood) String() string {
st := fmt.Sprintf("%s (%s)", engine.Name, engine.BaseURL)
return st
}

func (engine *KDramaHood) getParseAttrs() (string, string, error) {
return "div.items", "div.item", nil
}

func (engine *KDramaHood) parseSingleMovie(el *colly.HTMLElement, index int) (Movie, error) {
movie := Movie{
Index: index,
IsSeries: true,
Source: engine.Name,
Size: "---MB",
}
switch engine.mode {
case SearchMode:
movie.Title = strings.TrimSpace(el.ChildText("span.tt"))
movie.Description = strings.TrimSpace(el.ChildText("span.ttx"))
case ListMode:
movie.Title = strings.TrimSpace(el.ChildAttr("img", "alt"))
movie.Description = strings.TrimSpace(el.ChildText("div.contenido"))
}
movie.CoverPhotoLink = el.ChildAttr("img", "src")
link := el.Request.AbsoluteURL(el.ChildAttr("a", "href"))
downloadLink, err := url.Parse(link)

if err != nil {
log.Fatal(err)
}
movie.DownloadLink = downloadLink
movie.Category = "kdrama"
return movie, nil
}

func (engine *KDramaHood) updateDownloadProps(downloadCollector *colly.Collector, movies *[]Movie) {
innerCollector := downloadCollector.Clone()
episodeMap := map[string]*url.URL{}
subtitleMap := map[string]*url.URL{}
downloadCollector.OnHTML("ul.episodios", func(e *colly.HTMLElement) {
// clear existing map
for k := range episodeMap {
delete(episodeMap, k)
}
for k := range subtitleMap {
delete(subtitleMap, k)
}
// create local targets
targetepisode := make(map[string]*url.URL)
targetsub := make(map[string]*url.URL)
movie := &(*movies)[getMovieIndexFromCtx(e.Request)]
e.ForEach("li", func(_ int, inn *colly.HTMLElement) {
innerCollector.Visit(inn.ChildAttr("a", "href"))
})

// deepcopy to localtargets
for k, v := range episodeMap {
targetepisode[k] = v
}
for k, v := range subtitleMap {
targetsub[k] = v
}

movie.SDownloadLink = targetepisode
movie.SubtitleLinks = targetsub
})

innerCollector.OnHTML("div.linkstv", func(e *colly.HTMLElement) {
name := e.ChildAttr("a", "download")
links := e.ChildAttrs("a", "href")
if len(links) > 1 {
// select first link
movieLink, _ := url.Parse(links[0])
// subtitle is always the last link
subLink, _ := url.Parse(links[len(links)-1])
episodeMap[name] = movieLink
subtitleMap[name] = subLink
}
})
}

// List : list all the movies on a page
func (engine *KDramaHood) List(page int) SearchResult {
engine.mode = ListMode
result := SearchResult{
Query: "List of Recent Uploads - Page " + strconv.Itoa(page),
}
pageParam := fmt.Sprintf("page/%v", strconv.Itoa(page))
engine.ListURL.Path = path.Join(engine.ListURL.Path, pageParam)
movies, err := Scrape(engine)
if err != nil {
log.Fatal(err)
}
result.Movies = movies
return result
}

// Search : Searches fzmovies for a particular query and return an array of movies
func (engine *KDramaHood) Search(param ...string) SearchResult {
query := param[0]
engine.mode = SearchMode
result := SearchResult{
Query: query,
}
q := engine.SearchURL.Query()
q.Set("s", query)
engine.SearchURL.RawQuery = q.Encode()
movies, err := Scrape(engine)
if err != nil {
log.Fatal(err)
}
result.Movies = movies
return result
}
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
@@ -11,6 +11,7 @@ require (
github.com/manifoldco/promptui v0.7.0
github.com/mitchellh/go-homedir v1.1.0
github.com/sirupsen/logrus v1.6.0
github.com/smartystreets/goconvey v1.6.4
github.com/spf13/cobra v1.0.0
github.com/spf13/viper v1.7.0
github.com/tebeka/selenium v0.9.9
5 changes: 5 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
@@ -170,6 +170,7 @@ github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OI
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
@@ -207,6 +208,7 @@ github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6Pyu
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a h1:FaWFmfWdAUKbSCtOU2QjDaorUexogfaMgbipgYATUMU=
github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a/go.mod h1:UJSiEoRfvx3hP73CvoARgeLjaIOjybY9vj8PUPPFGeU=
@@ -319,7 +321,9 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
@@ -493,6 +497,7 @@ golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtn
golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc h1:NCy3Ohtk6Iny5V/reW2Ktypo4zIpWBdRJ1uFMjBxdg8=
golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
4 changes: 4 additions & 0 deletions reference/Gophie.v1.yaml
Original file line number Diff line number Diff line change
@@ -39,6 +39,10 @@ info:

- [AnimeOut](https://www.animeout.xyz)
- [TakanimeList] (https://takanimelist.best)

### Korean

- [KDramaHood](https://kdramahood.com)
servers:
- url: 'https://deploy-gophie.herokuapp.com'
description: Heroku server