From 071dec87d27b838beacefde6232e27acb40a8be4 Mon Sep 17 00:00:00 2001 From: Nyah Check Date: Sun, 5 Jan 2020 19:14:36 -0800 Subject: [PATCH] Add download sub module --- concurrent_download.go | 33 --- concurrent_download_test.go | 1 - download.go | 281 ++++++++++++++++++++ video_extractor_test.go => download_test.go | 2 +- main.go | 13 +- video_converter.go | 106 -------- video_converter_test.go | 1 - video_extractor.go | 170 ------------ 8 files changed, 289 insertions(+), 318 deletions(-) delete mode 100644 concurrent_download.go delete mode 100644 concurrent_download_test.go create mode 100644 download.go rename video_extractor_test.go => download_test.go (97%) delete mode 100644 video_converter.go delete mode 100644 video_converter_test.go delete mode 100644 video_extractor.go diff --git a/concurrent_download.go b/concurrent_download.go deleted file mode 100644 index d5b0058..0000000 --- a/concurrent_download.go +++ /dev/null @@ -1,33 +0,0 @@ -/** - * download video files concurrently - */ -package main - -import "sync" - -//DownloadStreams download a batch of elements asynchronously -func downloadStreams(maxOperations int, format, outputPath string, bitrate uint, urls []string) <-chan error { - - var wg sync.WaitGroup - wg.Add(len(urls)) - - ch := make(chan error, maxOperations) - for _, url := range urls { - go func(url string) { - defer wg.Done() - - if ID, err := getVideoId(url); err != nil { - ch <- err - } else { - ch <- getVideoStream(format, ID, outputPath, bitrate) - } - }(url) - } - - go func() { - wg.Wait() - close(ch) - }() - - return ch -} diff --git a/concurrent_download_test.go b/concurrent_download_test.go deleted file mode 100644 index 06ab7d0..0000000 --- a/concurrent_download_test.go +++ /dev/null @@ -1 +0,0 @@ -package main diff --git a/download.go b/download.go new file mode 100644 index 0000000..47aebe8 --- /dev/null +++ b/download.go @@ -0,0 +1,281 @@ +package main + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "net/url" + "os" + "os/user" + "path/filepath" + "strings" + "unicode" + + "github.com/sirupsen/logrus" + "github.com/viert/go-lame" +) + +const ( + audioBitRate = 123 + + streamApiUrl = "https://youtube.com/get_video_info?video_id=" +) + +type stream map[string]string + +type RawVideoStream struct { + VideoId string + VideoInfo string + Title string `json:"title"` + Author string `json:"author"` + URLEncodedFmtStreamMap []stream `json:"url_encoded_fmt_stream_map"` + Status string `json:"status"` +} + +// removeWhiteSpace removes white spaces from string +// removeWhiteSpace returns a filename without whitespaces +func removeWhiteSpace(str string) string { + return strings.Map(func(r rune) rune { + if unicode.IsSpace(r) { + return -1 + } + return r + }, str) +} + +// fixExtension is a helper function that +// fixes file the extension +func fixExtension(str string) string { + if strings.Contains(str, "mp3") { + str = ".mp3" + } else { + format = ".flv" + } + + return str +} + +// decodeStream accept Values and decodes them individually +// decodeStream returns the final RawVideoStream object +func decodeStream(values Values, streams *RawVideoStream, rawstream *stream) error { + streams.Author = values["author"][0] + streams.Title = values["title"][0] + streamMap, ok := values["url_encoded_fmt_stream_map"] + if !ok { + return errors.New("Error reading encoded stream map") + } + + // read and decode streams + streamsList := strings.Split(string(StreamMap[0]), ",") + for streamPos, streamRaw := range streamsList { + streamQry, err := url.ParseQuery(streamRaw) + if err != nil { + logrus.Infof("Error occured during stream decoding %d: %s\n", streamPos, err) + continue + } + var sig string + if _, exist := streamQry["sig"]; exist { + sig = streamQry["sig"][0] + } + + rawstream = append(rawstream, stream{ + "quality": streamQry["quality"][0], + "type": streamQry["type"][0], + "url": streamQry["url"][0], + "sig": sig, + "title": output["title"][0], + "author": output["author"][0], + }) + logrus.Infof("Stream found: quality '%s', format '%s'", streamQry["quality"][0], streamQry["type"][0]) + } + streams.URLEncodedFmtStreamMap = rawstream +} + +// encodeAudioStream consumes a raw data stream and +// encodeAudioStream encodes the data stream in mp3 +func encodeAudioStream(file, path, surl string, bitrate uint) error { + resp, err := downloadVideoStream(surl) + if err != nil { + log.Printf("Http.Get\nerror: %s\nURL: %s\n", err, surl) + return err + } + defer resp.Body.Close() + + tmp, _ := os.OpenFile("_temp_", os.O_CREATE, 0755) + defer tmp.Close() + if _, err := io.Copy(resp.Body, tmp); err != nil { + logrus.Errorf("Failed to read response body: %v", err) + return err + } + + // Create output file + currentDirectory, err := user.Current() + if err != nil { + logrus.Errorf("Error getting current user directory: %v", err) + return err + } + + outputDirectory := currentDirectory.HomeDir + "Downloads" + path + outputFile := filepath.Join(outputDirectory, file) + if err != os.MkdirAll(filepath.Dir(outputFile), 0775); err != nil { + logrus.Errorf("Unable to create output directory: %v", err) + } + + fp, err := os.OpenFile(outputFile, os.O_CREATE, 0755) + if err != nil { + logrus.Errorf("Unable to create output file: %v", err) + return err + } + defer fp.Close() + + // write audio/video file to output + reader := bufio.NewReader(bytes.NewReader(rawData)) + writer := lame.NewEncoder(fp) + defer writer.Close() + + writer.setBrate(int(bitrate)) + writer.SetQuality(1) + reader.WriteTo(writer) + + return nil +} + +// encodeVideoStream consumes video data stream and +// encodeVideoStream encodes the video in flv +func encodeVideoStream(file, path, surl string) error { + resp, err := downloadVideoStream(surl) + if err != nil { + log.Printf("Http.Get\nerror: %s\nURL: %s\n", err, surl) + return err + } + defer resp.Body.Close() + + // Create output file + currentDirectory, err := user.Current() + if err != nil { + logrus.Errorf("Error getting current user directory: %v", err) + return err + } + + outputDirectory := currentDirectory.HomeDir + "Downloads" + path + outputFile := filepath.Join(outputDirectory, file) + if err != os.MkdirAll(filepath.Dir(outputFile), 0775); err != nil { + logrus.Errorf("Unable to create output directory: %v", err) + } + + fp, err := os.OpenFile(outputFile, os.O_CREATE, 0755) + if err != nil { + logrus.Errorf("Unable to create output file: %v", err) + return err + } + defer fp.Close() + + //saving downloaded file. + if _, err = io.Copy(fp, resp.Body); err != nil { + logrus.Errorf("Unable to encode video stream: %s `->` %v", surl, err) + return err + } + return nil +} + +// downloadVideoStream downloads video streams from youtube +// downloadVideoStream returns the *http.Reponse body +func downloadVideoStream(url string) (*http.Response, error) { + resp, err := http.Get(url) + if err != nil { + logrus.Errorf("Unable to fetch Data stream from URL(%s)\n: %v", url, err) + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + logrus.Errorf("Video Download error with status: '%v'", resp.StatusCode) + return nil, errors.New("Non 200 status code received") + } + + return resp, nil +} + +// getVideoId extracts the video id string from youtube url +// getVideoId returns a video id string to calling function +func getVideoId(url string) (string, error) { + if !strings.Contains(url, "youtube.com") { + return "", errors.New("Invalid Youtube URL") + } + + s := strings.Split(url, "?v=") + s = strings.Split(s[1], "&") + if len(s[0]) == 0 { + return s[0], errors.New("Empty string") + } + + return s[0], nil +} + +// decodeVideoStream processes downloaded video stream and +// decodeVideoStream calls helper functions and writes the +// output in the required format +func decodeVideoStream(videoId, path, format string, bitrate uint) error { + var decStreams []stream //decoded video streams + rawVideo := new(RawVideoStream) // raw video stream + + // Get video data + rawVideo.VideoId = videoId + rawVideo.VideoInfo = streamApiUrl + videoId + + videoStream, err := downloadVideoStream(rawVideo.VideoInfo) + if err != nil { + logrus.Errorf("Unable to get video stream: %v", err) + return err + } + + tempfile, _ := os.OpenFile("_temp_download", os.O_CREATE, 0755) + defer tempfile.Close() + if _, err := io.Copy(videoStream.Body, tempfile); err != nil { + logrus.Errorf("Failed to read response body: %v", err) + return err + } + + parsedResp, err := url.ParseQuery(string(tempfile)) + if err != nil { + logrus.Errorf("Error parsing video byte stream: %v", err) + return err + } + + status, ok := parsedResp["status"] + if !ok { + return error.New("No response from server") + } + + reason, _ := parsedResp["reason"] + if status[0] == "fail" { + return errors.New(fmt.Sprint("'fail' response with reason: %v", reason)) + } else if status[0] != "ok" { + return errors.New(fmt.Sprint("'non-success' response with reason: %v", reason)) + } + + if err := decodeStream(parsedResp, &rawVideo, &decStreams); err != nil { + return error.New("Unable to decode raw video streams") + } + + file := removeWhiteSpace(rawVideo.Title + fixExtension(format)) + surl := decStreams[0]["url"] + "&signature" + decStreams[0]["sig"] + logrus.Infof("Downloading data to file: %s", file) + + if String.Contains(file, "mp3") { + if err := encodeAudioStream(file, path, surl, bitrate); err != nil { + logrus.Errorf("Unable to encode %s: %v", format, err) + } + } else { + if err := encodeVideoStream(file, path, surl); err != nil { + logrus.Errorf("Unable to encode %s: %v", format, err) + } + } + + return nil +} diff --git a/video_extractor_test.go b/download_test.go similarity index 97% rename from video_extractor_test.go rename to download_test.go index fa1f27f..63863a4 100644 --- a/video_extractor_test.go +++ b/download_test.go @@ -38,7 +38,7 @@ func TestApi(t *testing.T) { func TestVideoId(t *testing.T) { urls := []string{"https://www.youtube.com/watch?v=HpNluHOAJFA&list=RDHpNluHOAJFA"} - url, err := getVideoId(urls[0]) + url, err := getVideoId(urls) if err != nil { t.Log(err) } diff --git a/main.go b/main.go index 3db9405..a99fefd 100644 --- a/main.go +++ b/main.go @@ -88,12 +88,13 @@ func main() { } func beginDownload(urls []string) { - ch := downloadStreams(defaultMaxDownloads, format, path, bitrate, urls) - for err := range ch { - //Extract Video data and decode - if err != nil { - logrus.Errorf("Error decoding Video stream: %v", err) - } + vId, err := getVideoId(urls[0]) + if err != nil { + logrus.Errorf("Error getting videoId: %v", err) + } + + if err := getVideoStream(format, vId, path, bitrate); err != nil { + logrus.Errorf("Error downloading video stream: %v", err) } } diff --git a/video_converter.go b/video_converter.go deleted file mode 100644 index 092767a..0000000 --- a/video_converter.go +++ /dev/null @@ -1,106 +0,0 @@ -/** - * Converts decoded video data to mp3, webm, mp4 or flv - * TODO: rework to use Go ffmpeg bindings - */ -package main - -import ( - "bufio" - "bytes" - "errors" - "io" - "io/ioutil" - "log" - "net/http" - "os" - "os/user" - "path/filepath" - - "github.com/sirupsen/logrus" - "github.com/viert/go-lame" -) - -//Downloads decoded audio stream -func convertVideo(file, path string, bitrate uint, url string) error { - resp, err := http.Get(url) - if err != nil { - log.Printf("Http.Get\nerror: %s\nURL: %s\n", err, url) - return err - } - defer resp.Body.Close() - - data, e := ioutil.ReadAll(resp.Body) - if e != nil { - logrus.Errorf("Error reading video data: %v", e) - } - - curDir, er := user.Current() - if er != nil { - return er - } - - homeDir := curDir.HomeDir - dir := homeDir + "/Downloads/youtube-dl/" + path - fp := filepath.Join(dir, file) - if err := os.MkdirAll(filepath.Dir(fp), 0775); err != nil { - return err - } - - os.Remove(fp) //delete if file exists. - out, err := os.Create(fp) - if err != nil { - return err - } - defer out.Close() - r := bytes.NewReader(data) - reader := bufio.NewReader(r) - audioWriter := lame.NewEncoder(out) - audioWriter.SetBrate(int(bitrate)) - audioWriter.SetQuality(1) - - // IMPORTANT! - // audioWriter.initParams() - reader.WriteTo(audioWriter) - - return nil -} - -//Downloads decoded video stream. -func downloadVideo(path, file, url string) error { - resp, err := http.Get(url) - if err != nil { - log.Printf("Http.Get\nerror: %s\nURL: %s\n", err, url) - return err - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - log.Printf("Reading Output: status code: '%v'", resp.StatusCode) - return errors.New("Non 200 status code received") - } - - curDir, er := user.Current() - if er != nil { - return er - } - homeDir := curDir.HomeDir - dir := homeDir + "/Downloads/youtube-dl/" + path - fp := filepath.Join(dir, file) - err = os.MkdirAll(filepath.Dir(fp), 0775) - if err != nil { - return err - } - os.Remove(fp) //delete if file exists - out, err := os.Create(fp) - if err != nil { - return err - } - - //saving downloaded file. - _, err = io.Copy(out, resp.Body) - if err != nil { - log.Println("Download Error: ", err) - return err - } - return nil -} diff --git a/video_converter_test.go b/video_converter_test.go deleted file mode 100644 index 06ab7d0..0000000 --- a/video_converter_test.go +++ /dev/null @@ -1 +0,0 @@ -package main diff --git a/video_extractor.go b/video_extractor.go deleted file mode 100644 index d21f969..0000000 --- a/video_extractor.go +++ /dev/null @@ -1,170 +0,0 @@ -/** - * process download requests with youtube api and extract video stream - */ -package main - -import ( - "errors" - "fmt" - "io/ioutil" - "net/http" - "net/url" - "reflect" - "strings" - "unicode" - - "github.com/sirupsen/logrus" -) - -//const variables -const ( - audioBitRate = 123 //default audio bit rate. - - //Video extractor - streamApiUrl = "https://youtube.com/get_video_info?video_id=" -) - -type stream map[string]string - -//Youtube Downloader Data file. -type RawVideoData struct { - Title string `json:"title"` - Author string `json:"author"` - Status string `json:"status"` - URLEncodedFmtStreamMap []stream `json:"url_encoded_fmt_stream_map"` - VideoId string - VideoInfo string -} - -//gets the Video ID from youtube url -func getVideoId(url string) (string, error) { - if !strings.Contains(url, "youtube.com") { - return "", errors.New("Invalid Youtube link") - } - s := strings.Split(url, "?v=") - s = strings.Split(s[1], "&") - if len(s[0]) == 0 { - return s[0], errors.New("Empty string") - } - - return s[0], nil -} - -//Gets Video Info, Decode Video Info from a Video ID. -func getVideoStream(format, id, path string, bitrate uint) (err error) { - - video := new(RawVideoData) //raw video data - var streams []stream //decoded video data - - //Get Video Data stream - video.VideoId = id - videoUrl := streamApiUrl + id - video.VideoInfo = videoUrl - resp, er := http.Get(videoUrl) - if er != nil { - logrus.Errorf("Error in GET request: %v", er) - } - - defer resp.Body.Close() - out, e := ioutil.ReadAll(resp.Body) - if e != nil { - logrus.Errorf("Error reading video data: %v", e) - } - - output, er := url.ParseQuery(string(out)) - if e != nil { - logrus.Errorf("Error parsing video byte stream: %v", e) - return nil - } - - status, ok := output["status"] - if !ok { - err = fmt.Errorf("No response status in server") - return err - } - if status[0] == "fail" { - reason, ok := output["reason"] - if ok { - err = fmt.Errorf("'fail' response status found in the server, reason: '%s'", reason[0]) - } else { - err = errors.New(fmt.Sprint("'fail' response status found in the server, no reason given")) - } - return err - } - if status[0] != "ok" { - err = fmt.Errorf("non-success response status found in the server (status: '%s')", status) - return err - } - - // read the streams map - reflect.TypeOf(output) - video.Author = output["author"][0] - video.Title = output["title"][0] - StreamMap, ok := output["url_encoded_fmt_stream_map"] - if !ok { - err = fmt.Errorf("Error reading encoded stream map.") - return err - } - - // read and decode streams. - streamsList := strings.Split(string(StreamMap[0]), ",") - for streamPos, streamRaw := range streamsList { - streamQry, err := url.ParseQuery(streamRaw) - if err != nil { - logrus.Infof("An error occured while decoding one of the video's streams: stream %d: %s\n", streamPos, err) - continue - } - var sig string - if _, exist := streamQry["sig"]; exist { - sig = streamQry["sig"][0] - } - - streams = append(streams, stream{ - "quality": streamQry["quality"][0], - "type": streamQry["type"][0], - "url": streamQry["url"][0], - "sig": sig, - "title": output["title"][0], - "author": output["author"][0], - }) - logrus.Infof("Stream found: quality '%s', format '%s'", streamQry["quality"][0], streamQry["type"][0]) - } - - video.URLEncodedFmtStreamMap = streams - //Download Video stream to file - if format == "mp3" || format == ".mp3" { - format = ".mp3" - } else { - format = ".flv" - } - - //create output file name and set path properly. - file := video.Title + format - file = spaceMap(file) - vstream := streams[0] - url := vstream["url"] + "&signature" + vstream["sig"] - logrus.Infof("Downloading file to %s", file) - if format == ".mp3" { - err = convertVideo(file, path, bitrate, url) - if err != nil { - logrus.Errorf("Error downloading audio: %v", err) - } - - } else { - if err := downloadVideo(path, file, url); err != nil { - logrus.Errorf("Error downloading video: %v", err) - } - } - - return nil -} - -//remove whitespaces in filename -func spaceMap(str string) string { - return strings.Map(func(r rune) rune { - if unicode.IsSpace(r) { - return -1 - } - return r - }, str) -}