Skip to content

Commit

Permalink
🐛 fix download youtube video with deciper signatures
Browse files Browse the repository at this point in the history
  • Loading branch information
noelyahan committed Jun 20, 2018
1 parent ec5b38c commit 98719f1
Show file tree
Hide file tree
Showing 4 changed files with 241 additions and 21 deletions.
2 changes: 1 addition & 1 deletion api/gonyvido.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,6 @@ func getFilteredVideo(url, vtype string) *domain.Video {
return &video
}
}
v := domain.NewVideo("", "", "", "", "")
v := domain.NewVideo("", "", "", "", "", "")
return &v
}
23 changes: 20 additions & 3 deletions domain/video.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ type Video struct {
videoType string
url string

playerScript string
meta map[string]interface{}
videoLength int
downloadedLength int
downloadPro chan float64
Expand All @@ -29,6 +31,18 @@ func (v *Video) getUrl() string {
return v.url
}

func (v *Video) SetUrl(url string) {
v.url = url
}

func (v *Video) GetMeta() map[string]interface{} {
return v.meta
}

func (v *Video) GetPlayerScript() string {
return v.playerScript
}

func (v Video) getExt() string {
ext := s.Split(v.videoType, ";")[0]
ext = s.Split(ext, "/")[1]
Expand Down Expand Up @@ -68,13 +82,15 @@ func (v *Video) SetSavePath(savePath string) *Video {
return v
}

func NewVideo(t, a, q, vt, url string) Video {
func NewVideo(t, a, q, vt, url, script string) Video {
return Video{
t,
a,
q,
vt,
url,
script,
make(map[string]interface{}),
0,
0,
make(chan float64),
Expand Down Expand Up @@ -118,6 +134,8 @@ func (v *Video) Write(b []byte) (n int, err error) {
}

func (v *Video) Download() *Video {
//log.Panic(v.getUrl())

// this has to have a public chan to notify the download is done
go func() {
resp, err := http.Get(v.getUrl())
Expand Down Expand Up @@ -161,7 +179,7 @@ func (v *Video) ToMP3() {
log.Fatal("ffmpeg not found")
}
fmt.Println(`Converting: ` + v.GetTitle() + ` to mp3`)
cmd := exec.Command(ffmpeg, "-y", "-loglevel", "quiet", "-i", v.getSavePath() + mp4, "-b:a", "320K", "-vn", v.getSavePath() + mp3)
cmd := exec.Command(ffmpeg, "-y", "-loglevel", "quiet", "-i", v.getSavePath()+mp4, "-b:a", "320K", "-vn", v.getSavePath()+mp3)
cmd.Stdin = os.Stdin
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
Expand All @@ -171,7 +189,6 @@ func (v *Video) ToMP3() {
fmt.Println(`Finished: ` + v.GetTitle() + `.mp3`)
}


func removeFile(file string) {
err := os.Remove(file)
if err != nil {
Expand Down
190 changes: 190 additions & 0 deletions youtube/signature.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,190 @@
package youtube

// source: rylio/ytdl project
import (
"fmt"
"io/ioutil"
"net/http"
"net/url"
"regexp"
"strconv"
"strings"
)

func getDownloadURL(meta map[string]interface{}, htmlPlayerFile string) (*url.URL, error) {
var sig string
if s, ok := meta["s"]; ok {
tokens, err := getSigTokens(htmlPlayerFile)
if err != nil {
return nil, err
}
sig = decipherTokens(tokens, s.(string))
} else {
if s, ok := meta["sig"]; ok {
sig = s.(string)
}
}
var urlString string
if s, ok := meta["url"]; ok {
urlString = s.(string)
} else if s, ok := meta["stream"]; ok {
if c, ok := meta["conn"]; ok {
urlString = c.(string)
if urlString[len(urlString)-1] != '/' {
urlString += "/"
}
}
urlString += s.(string)
} else {
return nil, fmt.Errorf("Couldn't extract url from format")
}
urlString, err := url.QueryUnescape(urlString)
if err != nil {
return nil, err
}
u, err := url.Parse(urlString)
if err != nil {
return nil, err
}
query := u.Query()
query.Set("ratebypass", "yes")
if len(sig) > 0 {
query.Set("signature", sig)
}
u.RawQuery = query.Encode()
return u, nil
}

func decipherTokens(tokens []string, sig string) string {
var pos int
sigSplit := strings.Split(sig, "")
for i, l := 0, len(tokens); i < l; i++ {
tok := tokens[i]
if len(tok) > 1 {
pos, _ = strconv.Atoi(string(tok[1:]))
pos = ^^pos
}
switch string(tok[0]) {
case "r":
reverseStringSlice(sigSplit)
case "w":
s := sigSplit[0]
sigSplit[0] = sigSplit[pos]
sigSplit[pos] = s
case "s":
sigSplit = sigSplit[pos:]
case "p":
sigSplit = sigSplit[pos:]
}
}
return strings.Join(sigSplit, "")
}

const (
jsvarStr = "[a-zA-Z_\\$][a-zA-Z_0-9]*"
reverseStr = ":function\\(a\\)\\{" +
"(?:return )?a\\.reverse\\(\\)" +
"\\}"
sliceStr = ":function\\(a,b\\)\\{" +
"return a\\.slice\\(b\\)" +
"\\}"
spliceStr = ":function\\(a,b\\)\\{" +
"a\\.splice\\(0,b\\)" +
"\\}"
swapStr = ":function\\(a,b\\)\\{" +
"var c=a\\[0\\];a\\[0\\]=a\\[b%a\\.length\\];a\\[b(?:%a\\.length)?\\]=c(?:;return a)?" +
"\\}"
)

var actionsObjRegexp = regexp.MustCompile(fmt.Sprintf(
"var (%s)=\\{((?:(?:%s%s|%s%s|%s%s|%s%s),?\\n?)+)\\};", jsvarStr, jsvarStr, reverseStr, jsvarStr, sliceStr, jsvarStr, spliceStr, jsvarStr, swapStr))

var actionsFuncRegexp = regexp.MustCompile(fmt.Sprintf(
"function(?: %s)?\\(a\\)\\{"+
"a=a\\.split\\(\"\"\\);\\s*"+
"((?:(?:a=)?%s\\.%s\\(a,\\d+\\);)+)"+
"return a\\.join\\(\"\"\\)"+
"\\}", jsvarStr, jsvarStr, jsvarStr))

var reverseRegexp = regexp.MustCompile(fmt.Sprintf(
"(?m)(?:^|,)(%s)%s", jsvarStr, reverseStr))
var sliceRegexp = regexp.MustCompile(fmt.Sprintf(
"(?m)(?:^|,)(%s)%s", jsvarStr, sliceStr))
var spliceRegexp = regexp.MustCompile(fmt.Sprintf(
"(?m)(?:^|,)(%s)%s", jsvarStr, spliceStr))
var swapRegexp = regexp.MustCompile(fmt.Sprintf(
"(?m)(?:^|,)(%s)%s", jsvarStr, swapStr))

func getSigTokens(htmlPlayerFile string) ([]string, error) {
u, _ := url.Parse(youtubeBaseURL)
p, err := url.Parse(htmlPlayerFile)
if err != nil {
return nil, err
}
resp, err := http.Get(u.ResolveReference(p).String())
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return nil, fmt.Errorf("Error fetching signature tokens, status code %d", resp.StatusCode)
}
body, err := ioutil.ReadAll(resp.Body)
bodyString := string(body)
if err != nil {
return nil, err
}

objResult := actionsObjRegexp.FindStringSubmatch(bodyString)
funcResult := actionsFuncRegexp.FindStringSubmatch(bodyString)

if len(objResult) < 3 || len(funcResult) < 2 {
return nil, fmt.Errorf("Error parsing signature tokens")
}
obj := strings.Replace(objResult[1], "$", "\\$", -1)
objBody := strings.Replace(objResult[2], "$", "\\$", -1)
funcBody := strings.Replace(funcResult[1], "$", "\\$", -1)

var reverseKey, sliceKey, spliceKey, swapKey string
var result []string

if result = reverseRegexp.FindStringSubmatch(objBody); len(result) > 1 {
reverseKey = strings.Replace(result[1], "$", "\\$", -1)
}
if result = sliceRegexp.FindStringSubmatch(objBody); len(result) > 1 {
sliceKey = strings.Replace(result[1], "$", "\\$", -1)
}
if result = spliceRegexp.FindStringSubmatch(objBody); len(result) > 1 {
spliceKey = strings.Replace(result[1], "$", "\\$", -1)
}
if result = swapRegexp.FindStringSubmatch(objBody); len(result) > 1 {
swapKey = strings.Replace(result[1], "$", "\\$", -1)
}

keys := []string{reverseKey, sliceKey, spliceKey, swapKey}
regex, err := regexp.Compile(fmt.Sprintf("(?:a=)?%s\\.(%s)\\(a,(\\d+)\\)", obj, strings.Join(keys, "|")))
if err != nil {
return nil, err
}
results := regex.FindAllStringSubmatch(funcBody, -1)
var tokens []string
for _, s := range results {
switch s[1] {
case swapKey:
tokens = append(tokens, "w"+s[2])
case reverseKey:
tokens = append(tokens, "r")
case sliceKey:
tokens = append(tokens, "s"+s[2])
case spliceKey:
tokens = append(tokens, "p"+s[2])
}
}
return tokens, nil
}

func reverseStringSlice(s []string) {
for i, j := 0, len(s)-1; i < len(s)/2; i, j = i+1, j-1 {
s[i], s[j] = s[j], s[i]
}
}
47 changes: 30 additions & 17 deletions youtube/youtube.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ const (
YoutubeVideoInfoApi = "http://youtube.com/get_video_info?video_id="
YoutubeWatchUrl = "https://www.youtube.com/watch?v"
YoutubeShortUrl = "youtu.be"
youtubeBaseURL = "https://www.youtube.com/"
)


Expand All @@ -33,19 +34,19 @@ func GetYoutubeVideos(url string) ([]domain.Video, error) {
if s.Contains(videoId, "&") {
videoId = s.Split(videoId, "&")[0]
}
infoApiUrl := YoutubeVideoInfoApi + videoId
//infoApiUrl := YoutubeVideoInfoApi + videoId
defaultApiUrl := YoutubeWatchUrl + "=" + videoId

videos, err := getVideoInfo(infoApiUrl)
if err != nil {
videos, err = getVideoInfo(defaultApiUrl)
//videos, err := getVideoInfo(infoApiUrl)
//if err != nil {
videos, err := getVideoInfo(defaultApiUrl)
if err != nil {
fmt.Println("Sorry this video is cannot be download.", err)
return nil, err
}
return videos, nil
}
return videos, nil
//}
//return videos, nil
}

func getVideoInfo(url string) ([]domain.Video, error) {
Expand All @@ -63,43 +64,48 @@ func getVideoInfo(url string) ([]domain.Video, error) {
}

// Stratergy pattern
var title, author, streamMapStr string
var title, author, streamMapStr, playerScript string
if s.Contains(url, YoutubeVideoInfoApi) {
title, author, streamMapStr, err = extractFromInfoApi(string(body))
} else {
title, author, streamMapStr, err = extractFromDefault(string(body))
title, author, streamMapStr, playerScript, err = extractFromDefault(string(body))
}
if err != nil {
return nil, err
}

return constructVideos(title, author, streamMapStr)
return constructVideos(title, author, streamMapStr, playerScript)
}

func extractFromDefault(strBody string) (string, string, string, error) {
func extractFromDefault(strBody string) (string, string, string, string, error) {
streamMapFilter := "url_encoded_fmt_stream_map\":"
streamMapRegexp, err := regexp.Compile(streamMapFilter + "(.*?)\",")
if err != nil {
return "", "", "", err
return "", "", "", "", err
}
titleRegexp, err := regexp.Compile("<title>(.*?)<\\/title>")
if err != nil {
return "", "", "", err
return "", "", "", "", err
}
authorRegexp, err := regexp.Compile("author\":\"(.*?)\",")
if err != nil {
return "", "", "", err
return "", "", "", "", err
}
playerScriptRegexp, err := regexp.Compile(`src="(.*?)base.js`)
if err != nil {
return "", "", "", "", err
}
title := titleRegexp.FindString(strBody)
author := authorRegexp.FindString(strBody)
title = title[len("<title>") : len(title)-len("</title>")]
author = author[len("author:\"")+1 : len(author)-2]
playerScript := s.Replace(playerScriptRegexp.FindString(strBody), `src="`, "", -1)

streamMapStr := streamMapRegexp.FindString(strBody)
streamMapStr = streamMapStr[len(streamMapFilter)+1 : len(streamMapStr)-2]
streamMapStr = s.Replace(streamMapStr, "\\u0026", "&", -1)

return title, author, streamMapStr, err
return title, author, streamMapStr, playerScript, err
}

func extractFromInfoApi(strBody string) (string, string, string, error) {
Expand All @@ -121,21 +127,28 @@ func extractFromInfoApi(strBody string) (string, string, string, error) {
return title, author, streamMapStr, nil
}

func constructVideos(title, author, streamMapStr string) ([]domain.Video, error) {
func constructVideos(title, author, streamMapStr, playerScript string) ([]domain.Video, error) {
streamList := s.Split(streamMapStr, ",")
videos := make([]domain.Video, 0)
for _, streamItem := range streamList {
stream, err := url.ParseQuery(streamItem)
if err != nil {
return nil, err
}
videos = append(videos, domain.NewVideo(
v := domain.NewVideo(
title,
author,
stream["quality"][0],
stream["type"][0],
stream["url"][0],
))
playerScript,
)
for k, val := range stream {
v.GetMeta()[k] = val[0]
}
url, _ := getDownloadURL(v.GetMeta(), v.GetPlayerScript())
v.SetUrl(url.String())
videos = append(videos, v)
}
return videos, nil
}

0 comments on commit 98719f1

Please sign in to comment.