From 1dd26b95cb1a22274f47a601982709ebeb18ac53 Mon Sep 17 00:00:00 2001 From: Nyah Check Date: Sun, 19 Jan 2020 12:57:12 -0800 Subject: [PATCH] Remove ffmpeg requirement and simplify downloader interface - Add goutube package - remove go-lame - update unit tests --- Dockerfile | 4 +- README.md | 11 +-- download.go | 221 ++++------------------------------------------- download_test.go | 39 ++------- go.mod | 7 +- go.sum | 26 ++---- main.go | 19 ++-- 7 files changed, 48 insertions(+), 279 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0c176f5..c138849 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,5 @@ + +# Build container FROM golang:1.13-alpine AS go-base RUN apk add --no-cache git @@ -11,7 +13,7 @@ COPY . . RUN CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-extldflags "-static"' -o ./youtube-dl download.go main.go -#runtime container +# Runtime container FROM scratch RUN echo "Runtime container" COPY --from=go-base /app/youtube-dl /youtube-dl diff --git a/README.md b/README.md index 533d2c9..f42149c 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,6 @@ Downloaded videos could be converted to `flv` or `mp3` formats. -## Pre-requisites - -- Install [FFMPEG](https://github.com/adaptlearning/adapt_authoring/wiki/Installing-FFmpeg) - - ## Build ```bash @@ -50,6 +45,12 @@ Flags: -h Help page ``` +### Example + +```console +$ ./youtube-dl -format mp3 https://www.youtube.com/watch?v=jOWsu8ePrbE +``` + ## Roadmap * Download youtube video with video id or link and converts to flv or mp3. diff --git a/download.go b/download.go index 00f6cae..9c950e0 100644 --- a/download.go +++ b/download.go @@ -1,13 +1,8 @@ package main import ( - "bufio" - "errors" - "fmt" - "io/ioutil" - "log" - "net/http" - "net/url" + "context" + "io" "os" "os/user" "path/filepath" @@ -15,26 +10,9 @@ import ( "unicode" "github.com/sirupsen/logrus" - "github.com/viert/go-lame" + "github.com/wader/goutubedl" ) -const ( - audioBitRate = 123 - - streamApiUrl = "http://youtube.com/get_video_info?video_id=" -) - -type stream map[string]string - -type RawVideoStream struct { - VideoId string - VideoInfo string - Title string `json:"title"` - Author string `json:"author"` - URLEncodedFmtStreamMap []stream `json:"url_encoded_fmt_stream_map"` - Status string `json:"status"` -} - // removeWhiteSpace removes white spaces from string // removeWhiteSpace returns a filename without whitespaces func removeWhiteSpace(str string) string { @@ -58,61 +36,21 @@ func fixExtension(str string) string { return str } -// encodeAudioStream consumes a raw data stream and -// encodeAudioStream encodes the data stream in mp3 -func encodeAudioStream(file, path, surl string, bitrate uint) error { - data, err := downloadVideoStream(surl) - if err != nil { - log.Printf("Http.Get\nerror: %s\nURL: %s\n", err, surl) - return err - } - - tmp, _ := os.OpenFile("_temp_", os.O_CREATE, 0755) - defer tmp.Close() - if _, err := tmp.Write(data); err != nil { - logrus.Errorf("Failed to read response body: %v", err) - return err - } - - // Create output file - currentDirectory, err := user.Current() - if err != nil { - logrus.Errorf("Error getting current user directory: %v", err) - return err - } - - outputDirectory := currentDirectory.HomeDir + "/Downloads/" + path - outputFile := filepath.Join(outputDirectory, file) - if err := os.MkdirAll(filepath.Dir(outputFile), 0775); err != nil { - logrus.Errorf("Unable to create output directory: %v", err) - } +// decodeVideoStream processes downloaded video stream and +// decodeVideoStream calls helper functions and writes the +// output in the required format +func decodeVideoStream(videoUrl, path, format string) error { - fp, err := os.OpenFile(outputFile, os.O_CREATE, 0755) + // Get video data + res, err := goutubedl.New(context.Background(), videoUrl, goutubedl.Options{}) if err != nil { - logrus.Errorf("Unable to create output file: %v", err) - return err + logrus.Errorf("Unable to create goutube object %s: %v", videoUrl, err) } - defer fp.Close() - - // write audio/video file to output - reader := bufio.NewReader(tmp) - writer := lame.NewEncoder(fp) - defer writer.Close() - - writer.SetBrate(int(bitrate)) - writer.SetQuality(1) - reader.WriteTo(writer) - return nil -} - -// encodeVideoStream consumes video data stream and -// encodeVideoStream encodes the video in flv -func encodeVideoStream(file, path, surl string) error { - data, err := downloadVideoStream(surl) + file := removeWhiteSpace(res.Info.Title) + fixExtension(format) + videoStream, err := res.Download(context.TODO(), format) if err != nil { - log.Printf("Http.Get\nerror: %s\nURL: %s\n", err, surl) - return err + logrus.Errorf("Unable to download %s stream: %v", format, err) } // Create output file @@ -135,137 +73,8 @@ func encodeVideoStream(file, path, surl string) error { } defer fp.Close() - //saving downloaded file. - if _, err = fp.Write(data); err != nil { - logrus.Errorf("Unable to encode video stream: %s `->` %v", surl, err) - return err - } - return nil -} - -// downloadVideoStream downloads video streams from youtube -// downloadVideoStream returns the *http.Reponse body -func downloadVideoStream(url string) ([]byte, error) { - resp, err := http.Get(url) - if err != nil { - logrus.Errorf("Unable to fetch Data stream from URL(%s)\n: %v", url, err) - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - logrus.Errorf("Video Download error with status: '%v'", resp.StatusCode) - return nil, errors.New("Non 200 status code received") - } - - output, _ := ioutil.ReadAll(resp.Body) - - return output, nil -} - -// getVideoId extracts the video id string from youtube url -// getVideoId returns a video id string to calling function -func getVideoId(url string) (string, error) { - if len(url) < 15 { - return url, nil - } else { - if !strings.Contains(url, "youtube.com") { - return "", errors.New("Invalid Youtube URL") - } - - s := strings.Split(url, "?v=")[1] - if len(s) == 0 { - return s, errors.New("Empty string") - } - - return s, nil - } -} - -// decodeStream accept Values and decodes them individually -// decodeStream returns the final RawVideoStream object -func decodeStream(values url.Values, streams *RawVideoStream, rawstream []stream) error { - streams.Author = values.Get("author") - streams.Title = values.Get("title") - streamMap := values.Get("url_encoded_fmt_stream_map") - - // read and decode streams - streamsList := strings.Split(string(streamMap), ",") - for streamPos, streamRaw := range streamsList { - streamQry, err := url.ParseQuery(streamRaw) - if err != nil { - logrus.Infof("Error occured during stream decoding %d: %s\n", streamPos, err) - continue - } - var sig string - sig = streamQry.Get("sig") - rawstream = append(rawstream, stream{ - "quality": streamQry.Get("quality"), - "type": streamQry.Get("type"), - "url": streamQry.Get("url"), - "sig": sig, - "title": values.Get("title"), - "author": values.Get("author"), - }) - logrus.Infof("Stream found: quality '%s', format '%s'", streamQry.Get("quality"), streamQry.Get("type")) - } - - streams.URLEncodedFmtStreamMap = rawstream - return nil -} - -// decodeVideoStream processes downloaded video stream and -// decodeVideoStream calls helper functions and writes the -// output in the required format -func decodeVideoStream(videoId, path, format string, bitrate uint) error { - var decStreams []stream //decoded video streams - rawVideo := new(RawVideoStream) // raw video stream - - // Get video data - rawVideo.VideoId = videoId - rawVideo.VideoInfo = streamApiUrl + videoId - - data, err := downloadVideoStream(rawVideo.VideoInfo) - if err != nil { - logrus.Errorf("Unable to get video stream: %v", err) - return err - } - - parsedResp, err := url.ParseQuery(string(data)) - if err != nil { - logrus.Errorf("Error parsing video byte stream: %v", err) - return err - } - - status, ok := parsedResp["status"] - if !ok { - return errors.New("No response from server") - } - - reason, _ := parsedResp["reason"] - if status[0] == "fail" { - return errors.New(fmt.Sprintf("'fail' response with reason: %s", reason)) - } else if status[0] != "ok" { - return errors.New(fmt.Sprintf("'non-success' response with reason: %s", reason)) - } - - if err := decodeStream(parsedResp, rawVideo, decStreams); err != nil { - return errors.New("Unable to decode raw video streams") - } - - file := removeWhiteSpace(rawVideo.Title) + fixExtension(format) - surl := decStreams[0]["url"] + "&signature" + decStreams[0]["sig"] - - logrus.Infof("Downloading data to file: %s", file) - if strings.Contains(file, "mp3") { - if err := encodeAudioStream(file, path, surl, bitrate); err != nil { - logrus.Errorf("Unable to encode %s: %v", format, err) - } - } else { - if err := encodeVideoStream(file, path, surl); err != nil { - logrus.Errorf("Unable to encode %s: %v", format, err) - } - } + io.Copy(fp, videoStream) + videoStream.Close() return nil } diff --git a/download_test.go b/download_test.go index efbac2d..2fcd431 100644 --- a/download_test.go +++ b/download_test.go @@ -22,44 +22,17 @@ func TestApi(t *testing.T) { // path := "test" for i, table := range tables { - ID, _ := getVideoId(table.url) - if ID != table.id { - t.Errorf("videoId(%d): expected %q, actual %q", i, table.id, ID) + err := decodeVideoStream(table.url, "~/Downloads", "mp3") + if err != nil { + t.Errorf("videoId(%d): expected %q, actual %q", i, table.id, err) } - - // if ID != "" { - // if err := getVideoStream("mp3", ID, path, 192); err != nil { - // t.Errorf("videoStream(%d): expected %v, actual %v", i, nil, err) - // } - // } - } -} - -func TestGetVideoId(t *testing.T) { - urls := []string{"https://www.youtube.com/watch?v=HpNluHOAJFA"} - - url, err := getVideoId(urls[0]) - if err != nil { - t.Errorf("videoId: expected %q, actual %q", "HpNluHOAJFA", url) } } func BenchmarkVideoId(b *testing.B) { for n := 0; n < b.N; n++ { - getVideoId(tables[0].url) + if err := decodeVideoStream(tables[0].url, "~/Downloads", "mp3"); err != nil { + b.Errorf("Error downloading video: %v", err) + } } } - -// func BenchmarkApivideoStream(b *testing.B) { -// for n := 0; n < b.N; n++ { -// getVideoStream("mp3", tables[0].id, "~/Downloads", 192) -// } -// } - -/*func BenchmarkApiConvertVideo(b *testing.B) { - path := "~/Downloads/" - for n := 0; n < b.N; n++ { - file := path + tables[0].id + ".mp3" - convertVideo(file, 123, tables[0].id, vid) - } -}*/ diff --git a/go.mod b/go.mod index b154248..667e73a 100644 --- a/go.mod +++ b/go.mod @@ -4,11 +4,12 @@ go 1.12 require ( github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect - github.com/kr/pretty v0.2.0 // indirect github.com/sirupsen/logrus v1.4.2 + github.com/stretchr/objx v0.2.0 // indirect github.com/stretchr/testify v1.4.0 // indirect - github.com/viert/go-lame v0.0.0-20190822173615-801f1be8d24f - golang.org/x/sys v0.0.0-20200103143344-a1369afcdac7 // indirect + github.com/wader/goutubedl v0.0.0-20200115162246-9eae90476a5d + github.com/wader/osleaktest v0.0.0-20191111175233-f643b0fed071 // indirect + golang.org/x/sys v0.0.0-20200117145432-59e60aa80a0c // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect gopkg.in/yaml.v2 v2.2.7 // indirect ) diff --git a/go.sum b/go.sum index c742466..0b8537c 100644 --- a/go.sum +++ b/go.sum @@ -1,33 +1,25 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/viert/go-lame v0.0.0-20190822173615-801f1be8d24f h1:Nyjl5q+O34OmyQ8q/xUibfORaLDhBpmT8OU2nPUJQOU= -github.com/viert/go-lame v0.0.0-20190822173615-801f1be8d24f/go.mod h1:EqTcYM7y4JlSfeTI47pmNu3EZQuCuLQefsQyg1Imlz8= +github.com/wader/goutubedl v0.0.0-20200115162246-9eae90476a5d h1:+noQWJMxTu1ruNLG1KcA7tz0kHR7QYUBu+88Y2+w930= +github.com/wader/goutubedl v0.0.0-20200115162246-9eae90476a5d/go.mod h1:TEOvzRw4YvTeOdSvGyoZD9KbKK4xbx0EJoSqj4v6yAs= +github.com/wader/osleaktest v0.0.0-20190723190525-c53af4cfc4a3/go.mod h1:XD6emOFPHVzb0+qQpiNOdPL2XZ0SRUM0N5JHuq6OmXo= +github.com/wader/osleaktest v0.0.0-20191111175233-f643b0fed071/go.mod h1:XD6emOFPHVzb0+qQpiNOdPL2XZ0SRUM0N5JHuq6OmXo= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200103143344-a1369afcdac7 h1:/W9OPMnnpmFXHYkcp2rQsbFUbRlRzfECQjmAFiOyHE8= -golang.org/x/sys v0.0.0-20200103143344-a1369afcdac7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200117145432-59e60aa80a0c h1:gUYreENmqtjZb2brVfUas1sC6UivSY8XwKwPo8tloLs= +golang.org/x/sys v0.0.0-20200117145432-59e60aa80a0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo= gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/main.go b/main.go index a506e78..c582368 100644 --- a/main.go +++ b/main.go @@ -90,22 +90,18 @@ func parseUrls(urls string) []string { func beginDownload(urls []string) { if len(urls) < 2 { - if vId, err := getVideoId(urls[0]); err != nil { - logrus.Errorf("Error fetching videoId: %v", err) - } else { - if err := decodeVideoStream(vId, path, format, bitrate); err != nil { - logrus.Errorf("Unable to beginDownload: %v", err) - } + if err := decodeVideoStream(urls[0], path, format); err != nil { + logrus.Errorf("Unable to beginDownload: %v", err) } } else { - if err := concurrentDownload(MAXDOWNLOADS, format, path, bitrate, urls); err != nil { + if err := concurrentDownload(MAXDOWNLOADS, format, path, urls); err != nil { logrus.Errorf("Unable to concurrently download videos: %v with errors => %v", urls, err) } } } //DownloadStreams download a batch of elements asynchronously -func concurrentDownload(maxOperations int, format, outputPath string, bitrate uint, urls []string) <-chan error { +func concurrentDownload(maxOperations int, format, outputPath string, urls []string) <-chan error { var wg sync.WaitGroup wg.Add(len(urls)) @@ -114,12 +110,7 @@ func concurrentDownload(maxOperations int, format, outputPath string, bitrate ui for _, url := range urls { go func(url string) { defer wg.Done() - - if videoId, err := getVideoId(url); err != nil { - ch <- err - } else { - ch <- decodeVideoStream(videoId, path, format, bitrate) - } + ch <- decodeVideoStream(url, path, format) }(url) }