From 98719f1cc02a2dd907036011be5b1415e9d8c76a Mon Sep 17 00:00:00 2001 From: Noel Yahan Date: Wed, 20 Jun 2018 09:46:06 +0530 Subject: [PATCH] :bug: fix download youtube video with deciper signatures --- api/gonyvido.go | 2 +- domain/video.go | 23 +++++- youtube/signature.go | 190 +++++++++++++++++++++++++++++++++++++++++++ youtube/youtube.go | 47 +++++++---- 4 files changed, 241 insertions(+), 21 deletions(-) create mode 100644 youtube/signature.go diff --git a/api/gonyvido.go b/api/gonyvido.go index d8236b6..1199bb9 100644 --- a/api/gonyvido.go +++ b/api/gonyvido.go @@ -36,6 +36,6 @@ func getFilteredVideo(url, vtype string) *domain.Video { return &video } } - v := domain.NewVideo("", "", "", "", "") + v := domain.NewVideo("", "", "", "", "", "") return &v } diff --git a/domain/video.go b/domain/video.go index 3a392c4..b8cce4a 100644 --- a/domain/video.go +++ b/domain/video.go @@ -18,6 +18,8 @@ type Video struct { videoType string url string + playerScript string + meta map[string]interface{} videoLength int downloadedLength int downloadPro chan float64 @@ -29,6 +31,18 @@ func (v *Video) getUrl() string { return v.url } +func (v *Video) SetUrl(url string) { + v.url = url +} + +func (v *Video) GetMeta() map[string]interface{} { + return v.meta +} + +func (v *Video) GetPlayerScript() string { + return v.playerScript +} + func (v Video) getExt() string { ext := s.Split(v.videoType, ";")[0] ext = s.Split(ext, "/")[1] @@ -68,13 +82,15 @@ func (v *Video) SetSavePath(savePath string) *Video { return v } -func NewVideo(t, a, q, vt, url string) Video { +func NewVideo(t, a, q, vt, url, script string) Video { return Video{ t, a, q, vt, url, + script, + make(map[string]interface{}), 0, 0, make(chan float64), @@ -118,6 +134,8 @@ func (v *Video) Write(b []byte) (n int, err error) { } func (v *Video) Download() *Video { + //log.Panic(v.getUrl()) + // this has to have a public chan to notify the download is done go func() { resp, err := http.Get(v.getUrl()) @@ -161,7 +179,7 @@ func (v *Video) ToMP3() { log.Fatal("ffmpeg not found") } fmt.Println(`Converting: ` + v.GetTitle() + ` to mp3`) - cmd := exec.Command(ffmpeg, "-y", "-loglevel", "quiet", "-i", v.getSavePath() + mp4, "-b:a", "320K", "-vn", v.getSavePath() + mp3) + cmd := exec.Command(ffmpeg, "-y", "-loglevel", "quiet", "-i", v.getSavePath()+mp4, "-b:a", "320K", "-vn", v.getSavePath()+mp3) cmd.Stdin = os.Stdin cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr @@ -171,7 +189,6 @@ func (v *Video) ToMP3() { fmt.Println(`Finished: ` + v.GetTitle() + `.mp3`) } - func removeFile(file string) { err := os.Remove(file) if err != nil { diff --git a/youtube/signature.go b/youtube/signature.go new file mode 100644 index 0000000..77e7b10 --- /dev/null +++ b/youtube/signature.go @@ -0,0 +1,190 @@ +package youtube + +// source: rylio/ytdl project +import ( + "fmt" + "io/ioutil" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" +) + +func getDownloadURL(meta map[string]interface{}, htmlPlayerFile string) (*url.URL, error) { + var sig string + if s, ok := meta["s"]; ok { + tokens, err := getSigTokens(htmlPlayerFile) + if err != nil { + return nil, err + } + sig = decipherTokens(tokens, s.(string)) + } else { + if s, ok := meta["sig"]; ok { + sig = s.(string) + } + } + var urlString string + if s, ok := meta["url"]; ok { + urlString = s.(string) + } else if s, ok := meta["stream"]; ok { + if c, ok := meta["conn"]; ok { + urlString = c.(string) + if urlString[len(urlString)-1] != '/' { + urlString += "/" + } + } + urlString += s.(string) + } else { + return nil, fmt.Errorf("Couldn't extract url from format") + } + urlString, err := url.QueryUnescape(urlString) + if err != nil { + return nil, err + } + u, err := url.Parse(urlString) + if err != nil { + return nil, err + } + query := u.Query() + query.Set("ratebypass", "yes") + if len(sig) > 0 { + query.Set("signature", sig) + } + u.RawQuery = query.Encode() + return u, nil +} + +func decipherTokens(tokens []string, sig string) string { + var pos int + sigSplit := strings.Split(sig, "") + for i, l := 0, len(tokens); i < l; i++ { + tok := tokens[i] + if len(tok) > 1 { + pos, _ = strconv.Atoi(string(tok[1:])) + pos = ^^pos + } + switch string(tok[0]) { + case "r": + reverseStringSlice(sigSplit) + case "w": + s := sigSplit[0] + sigSplit[0] = sigSplit[pos] + sigSplit[pos] = s + case "s": + sigSplit = sigSplit[pos:] + case "p": + sigSplit = sigSplit[pos:] + } + } + return strings.Join(sigSplit, "") +} + +const ( + jsvarStr = "[a-zA-Z_\\$][a-zA-Z_0-9]*" + reverseStr = ":function\\(a\\)\\{" + + "(?:return )?a\\.reverse\\(\\)" + + "\\}" + sliceStr = ":function\\(a,b\\)\\{" + + "return a\\.slice\\(b\\)" + + "\\}" + spliceStr = ":function\\(a,b\\)\\{" + + "a\\.splice\\(0,b\\)" + + "\\}" + swapStr = ":function\\(a,b\\)\\{" + + "var c=a\\[0\\];a\\[0\\]=a\\[b%a\\.length\\];a\\[b(?:%a\\.length)?\\]=c(?:;return a)?" + + "\\}" +) + +var actionsObjRegexp = regexp.MustCompile(fmt.Sprintf( + "var (%s)=\\{((?:(?:%s%s|%s%s|%s%s|%s%s),?\\n?)+)\\};", jsvarStr, jsvarStr, reverseStr, jsvarStr, sliceStr, jsvarStr, spliceStr, jsvarStr, swapStr)) + +var actionsFuncRegexp = regexp.MustCompile(fmt.Sprintf( + "function(?: %s)?\\(a\\)\\{"+ + "a=a\\.split\\(\"\"\\);\\s*"+ + "((?:(?:a=)?%s\\.%s\\(a,\\d+\\);)+)"+ + "return a\\.join\\(\"\"\\)"+ + "\\}", jsvarStr, jsvarStr, jsvarStr)) + +var reverseRegexp = regexp.MustCompile(fmt.Sprintf( + "(?m)(?:^|,)(%s)%s", jsvarStr, reverseStr)) +var sliceRegexp = regexp.MustCompile(fmt.Sprintf( + "(?m)(?:^|,)(%s)%s", jsvarStr, sliceStr)) +var spliceRegexp = regexp.MustCompile(fmt.Sprintf( + "(?m)(?:^|,)(%s)%s", jsvarStr, spliceStr)) +var swapRegexp = regexp.MustCompile(fmt.Sprintf( + "(?m)(?:^|,)(%s)%s", jsvarStr, swapStr)) + +func getSigTokens(htmlPlayerFile string) ([]string, error) { + u, _ := url.Parse(youtubeBaseURL) + p, err := url.Parse(htmlPlayerFile) + if err != nil { + return nil, err + } + resp, err := http.Get(u.ResolveReference(p).String()) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + return nil, fmt.Errorf("Error fetching signature tokens, status code %d", resp.StatusCode) + } + body, err := ioutil.ReadAll(resp.Body) + bodyString := string(body) + if err != nil { + return nil, err + } + + objResult := actionsObjRegexp.FindStringSubmatch(bodyString) + funcResult := actionsFuncRegexp.FindStringSubmatch(bodyString) + + if len(objResult) < 3 || len(funcResult) < 2 { + return nil, fmt.Errorf("Error parsing signature tokens") + } + obj := strings.Replace(objResult[1], "$", "\\$", -1) + objBody := strings.Replace(objResult[2], "$", "\\$", -1) + funcBody := strings.Replace(funcResult[1], "$", "\\$", -1) + + var reverseKey, sliceKey, spliceKey, swapKey string + var result []string + + if result = reverseRegexp.FindStringSubmatch(objBody); len(result) > 1 { + reverseKey = strings.Replace(result[1], "$", "\\$", -1) + } + if result = sliceRegexp.FindStringSubmatch(objBody); len(result) > 1 { + sliceKey = strings.Replace(result[1], "$", "\\$", -1) + } + if result = spliceRegexp.FindStringSubmatch(objBody); len(result) > 1 { + spliceKey = strings.Replace(result[1], "$", "\\$", -1) + } + if result = swapRegexp.FindStringSubmatch(objBody); len(result) > 1 { + swapKey = strings.Replace(result[1], "$", "\\$", -1) + } + + keys := []string{reverseKey, sliceKey, spliceKey, swapKey} + regex, err := regexp.Compile(fmt.Sprintf("(?:a=)?%s\\.(%s)\\(a,(\\d+)\\)", obj, strings.Join(keys, "|"))) + if err != nil { + return nil, err + } + results := regex.FindAllStringSubmatch(funcBody, -1) + var tokens []string + for _, s := range results { + switch s[1] { + case swapKey: + tokens = append(tokens, "w"+s[2]) + case reverseKey: + tokens = append(tokens, "r") + case sliceKey: + tokens = append(tokens, "s"+s[2]) + case spliceKey: + tokens = append(tokens, "p"+s[2]) + } + } + return tokens, nil +} + +func reverseStringSlice(s []string) { + for i, j := 0, len(s)-1; i < len(s)/2; i, j = i+1, j-1 { + s[i], s[j] = s[j], s[i] + } +} diff --git a/youtube/youtube.go b/youtube/youtube.go index d2ef6e8..6bc16be 100644 --- a/youtube/youtube.go +++ b/youtube/youtube.go @@ -15,6 +15,7 @@ const ( YoutubeVideoInfoApi = "http://youtube.com/get_video_info?video_id=" YoutubeWatchUrl = "https://www.youtube.com/watch?v" YoutubeShortUrl = "youtu.be" + youtubeBaseURL = "https://www.youtube.com/" ) @@ -33,19 +34,19 @@ func GetYoutubeVideos(url string) ([]domain.Video, error) { if s.Contains(videoId, "&") { videoId = s.Split(videoId, "&")[0] } - infoApiUrl := YoutubeVideoInfoApi + videoId + //infoApiUrl := YoutubeVideoInfoApi + videoId defaultApiUrl := YoutubeWatchUrl + "=" + videoId - videos, err := getVideoInfo(infoApiUrl) - if err != nil { - videos, err = getVideoInfo(defaultApiUrl) + //videos, err := getVideoInfo(infoApiUrl) + //if err != nil { + videos, err := getVideoInfo(defaultApiUrl) if err != nil { fmt.Println("Sorry this video is cannot be download.", err) return nil, err } return videos, nil - } - return videos, nil + //} + //return videos, nil } func getVideoInfo(url string) ([]domain.Video, error) { @@ -63,43 +64,48 @@ func getVideoInfo(url string) ([]domain.Video, error) { } // Stratergy pattern - var title, author, streamMapStr string + var title, author, streamMapStr, playerScript string if s.Contains(url, YoutubeVideoInfoApi) { title, author, streamMapStr, err = extractFromInfoApi(string(body)) } else { - title, author, streamMapStr, err = extractFromDefault(string(body)) + title, author, streamMapStr, playerScript, err = extractFromDefault(string(body)) } if err != nil { return nil, err } - return constructVideos(title, author, streamMapStr) + return constructVideos(title, author, streamMapStr, playerScript) } -func extractFromDefault(strBody string) (string, string, string, error) { +func extractFromDefault(strBody string) (string, string, string, string, error) { streamMapFilter := "url_encoded_fmt_stream_map\":" streamMapRegexp, err := regexp.Compile(streamMapFilter + "(.*?)\",") if err != nil { - return "", "", "", err + return "", "", "", "", err } titleRegexp, err := regexp.Compile("(.*?)<\\/title>") if err != nil { - return "", "", "", err + return "", "", "", "", err } authorRegexp, err := regexp.Compile("author\":\"(.*?)\",") if err != nil { - return "", "", "", err + return "", "", "", "", err + } + playerScriptRegexp, err := regexp.Compile(`src="(.*?)base.js`) + if err != nil { + return "", "", "", "", err } title := titleRegexp.FindString(strBody) author := authorRegexp.FindString(strBody) title = title[len("<title>") : len(title)-len("")] author = author[len("author:\"")+1 : len(author)-2] + playerScript := s.Replace(playerScriptRegexp.FindString(strBody), `src="`, "", -1) streamMapStr := streamMapRegexp.FindString(strBody) streamMapStr = streamMapStr[len(streamMapFilter)+1 : len(streamMapStr)-2] streamMapStr = s.Replace(streamMapStr, "\\u0026", "&", -1) - return title, author, streamMapStr, err + return title, author, streamMapStr, playerScript, err } func extractFromInfoApi(strBody string) (string, string, string, error) { @@ -121,7 +127,7 @@ func extractFromInfoApi(strBody string) (string, string, string, error) { return title, author, streamMapStr, nil } -func constructVideos(title, author, streamMapStr string) ([]domain.Video, error) { +func constructVideos(title, author, streamMapStr, playerScript string) ([]domain.Video, error) { streamList := s.Split(streamMapStr, ",") videos := make([]domain.Video, 0) for _, streamItem := range streamList { @@ -129,13 +135,20 @@ func constructVideos(title, author, streamMapStr string) ([]domain.Video, error) if err != nil { return nil, err } - videos = append(videos, domain.NewVideo( + v := domain.NewVideo( title, author, stream["quality"][0], stream["type"][0], stream["url"][0], - )) + playerScript, + ) + for k, val := range stream { + v.GetMeta()[k] = val[0] + } + url, _ := getDownloadURL(v.GetMeta(), v.GetPlayerScript()) + v.SetUrl(url.String()) + videos = append(videos, v) } return videos, nil }