Compare commits

...
4 Commits
Author SHA1 Message Date
Niko Storni 89fe1e99d6 improve ipv6 validation
fix audio merge issue
2019-07-12 01:28:00 +02:00
Niko Storni ec81af0857 fix download filters 2019-07-12 01:04:45 +02:00
Niko Storni 51cb7f5e34 improve youtube antithrottle
fix logging
2019-07-11 19:14:15 +02:00
Niko Storni 8ad2acc936 remove original downloader
add locking to IP selection to avoid concurrent access
2019-07-11 17:32:46 +02:00
5 changed files with 76 additions and 101 deletions

No files matched your search

+1
View File
@@ -5,6 +5,7 @@ require (
github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61
github.com/PuerkitoBio/goquery v1.5.0 // indirect
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf
github.com/aws/aws-sdk-go v1.17.3
github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 // indirect
github.com/go-ini/ini v1.42.0 // indirect
+1
View File
@@ -25,6 +25,7 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM=
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf h1:eg0MeVzsP1G42dRafH3vf+al2vQIJU0YHX+1Tw87oco=
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
github.com/aws/aws-sdk-go v1.16.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.17.3 h1:KBXxg7Jh0TxE5zmpNB2DwKmJeDUqh0O6jhy25TuYOmc=
-1
View File
@@ -199,7 +199,6 @@ func (s *SyncManager) Start() error {
"failure uploading wallet",
"the channel in the wallet is different than the channel in the database",
"this channel does not belong to this wallet!",
"Invalid status code: 429",
"HTTP Error 429",
}
if util.SubstringInSlice(err.Error(), fatalErrors) {
+1 -2
View File
@@ -656,7 +656,6 @@ func (s *Sync) startWorker(workerNum int) {
"Cannot publish using channel",
"cannot concatenate 'str' and 'NoneType' objects",
"more than 90% of the space has been used.",
"Invalid status code: 429",
"HTTP Error 429",
}
if util.SubstringInSlice(err.Error(), fatalErrors) || s.StopOnError {
@@ -679,7 +678,7 @@ func (s *Sync) startWorker(workerNum int) {
"Watch this video on YouTube.",
"have blocked it on copyright grounds",
"the video must be republished as we can't get the right size",
"Invalid status code: 403",
"HTTP Error 403",
}
if util.SubstringInSlice(err.Error(), errorsNoRetry) {
log.Println("This error should not be retried at all")
+73 -98
View File
@@ -24,8 +24,8 @@ import (
"github.com/lbryio/ytsync/thumbs"
"github.com/ChannelMeter/iso8601duration"
"github.com/asaskevich/govalidator"
"github.com/aws/aws-sdk-go/aws"
"github.com/nikooo777/ytdl"
"github.com/shopspring/decimal"
log "github.com/sirupsen/logrus"
"google.golang.org/api/youtube/v3"
@@ -176,8 +176,11 @@ func (v *YoutubeVideo) getAbbrevDescription() string {
var ipPool []string
var IPIndex int
var ipMutex sync.Mutex
func getNextIP() (string, error) {
ipMutex.Lock()
defer ipMutex.Unlock()
if len(ipPool) < 1 {
IPIndex = 0
addrs, err := net.InterfaceAddrs()
@@ -187,7 +190,7 @@ func getNextIP() (string, error) {
for _, address := range addrs {
if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
if ipnet.IP.To16() != nil && len(ipnet.IP) == net.IPv6len {
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) {
ipPool = append(ipPool, ipnet.IP.String())
}
}
@@ -202,40 +205,7 @@ func getNextIP() (string, error) {
return nextIP, nil
}
func (v *YoutubeVideo) fallbackDownload() error {
sourceAddress, err := getNextIP()
if err != nil {
return errors.Err(err)
}
cmd := exec.Command("youtube-dl",
"--no-progress",
"-fbestvideo[ext=mp4,height<=1080,filesize<2000M]+best[ext=mp4,height<=1080,filesize<2000M]",
"-o"+strings.TrimRight(v.getFullPath(), ".mp4"),
"--merge-output-format",
"mp4",
"-6",
"--source-address",
sourceAddress,
"https://www.youtube.com/watch?v="+v.ID())
log.Printf("Running command and waiting for it to finish...")
output, err := cmd.CombinedOutput()
log.Debugln(string(output))
if err != nil {
log.Printf("Command finished with error: %v", errors.Err(string(output)))
_ = v.delete()
return errors.Err(err)
}
fi, err := os.Stat(v.getFullPath())
if err != nil {
return errors.Err(err)
}
videoSize := fi.Size()
v.size = &videoSize
return nil
}
func (v *YoutubeVideo) download() error {
func (v *YoutubeVideo) download(youtubeAntiThrottle bool) error {
videoPath := v.getFullPath()
err := os.Mkdir(v.videoDir(), 0750)
@@ -251,71 +221,74 @@ func (v *YoutubeVideo) download() error {
return nil
}
videoUrl := "https://www.youtube.com/watch?v=" + v.id
videoInfo, err := ytdl.GetVideoInfo(videoUrl)
useIPv4Env := os.Getenv("USE_IPV4")
ytdlArgs := []string{
"--no-progress",
"--max-filesize",
fmt.Sprintf("%dM", v.maxVideoSize),
"-fbestvideo[ext=mp4][height<=1080]+bestaudio[ext!=webm]",
"-o" + strings.TrimRight(v.getFullPath(), ".mp4"),
"--merge-output-format",
"mp4",
}
useIPv4 := useIPv4Env != ""
if !useIPv4 || youtubeAntiThrottle {
sourceAddress, err := getNextIP()
if err != nil {
return errors.Err(err)
}
log.Infof("using IPv6: %s", sourceAddress)
ytdlArgs = append(ytdlArgs,
"-6",
"--source-address",
sourceAddress,
)
} else {
log.Infoln("using IPv4")
ytdlArgs = append(ytdlArgs, "-4")
}
ytdlArgs = append(ytdlArgs, "https://www.youtube.com/watch?v="+v.ID())
cmd := exec.Command("youtube-dl", ytdlArgs...)
log.Printf("Running command and waiting for it to finish...")
stderr, err := cmd.StderrPipe()
if err != nil {
return errors.Err(err)
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return errors.Err(err)
}
codec := []string{"H.264"}
ext := []string{"mp4"}
//Filter requires a [] interface{}
codecFilter := make([]interface{}, len(codec))
for i, v := range codec {
codecFilter[i] = v
if err := cmd.Start(); err != nil {
return errors.Err(err)
}
//Filter requires a [] interface{}
extFilter := make([]interface{}, len(ext))
for i, v := range ext {
extFilter[i] = v
errorLog, _ := ioutil.ReadAll(stderr)
outLog, _ := ioutil.ReadAll(stdout)
if err = cmd.Wait(); err != nil {
if strings.Contains(err.Error(), "exit status 1") {
return errors.Err(string(errorLog))
}
return errors.Err(err)
}
formats := videoInfo.Formats.Filter(ytdl.FormatVideoEncodingKey, codecFilter).Filter(ytdl.FormatExtensionKey, extFilter)
if len(formats) == 0 {
return errors.Err("no compatible format available for this video")
}
maxRetryAttempts := 5
isLengthLimitSet := v.maxVideoLength > 0.01
if isLengthLimitSet && videoInfo.Duration.Hours() > v.maxVideoLength {
return errors.Err("video is too long to process")
}
log.Debugln(string(outLog))
for i := 0; i < len(formats) && i < maxRetryAttempts; i++ {
formatIndex := i
if i == maxRetryAttempts-1 {
formatIndex = len(formats) - 1
}
var downloadedFile *os.File
downloadedFile, err = os.Create(videoPath)
if err != nil {
return errors.Err(err)
}
err = videoInfo.Download(formats[formatIndex], downloadedFile)
_ = downloadedFile.Close()
if err != nil {
//delete the video and ignore the error
_ = v.delete()
return errors.Err(err.Error())
}
fi, err := os.Stat(v.getFullPath())
if err != nil {
return errors.Err(err)
}
videoSize := fi.Size()
v.size = &videoSize
isVideoSizeLimitSet := v.maxVideoSize > 0
if isVideoSizeLimitSet && videoSize > v.maxVideoSize {
//delete the video and ignore the error
_ = v.delete()
err = errors.Err("file is too big and there is no other format available")
} else {
break
}
if string(errorLog) != "" {
log.Printf("Command finished with error: %v", errors.Err(string(errorLog)))
_ = v.delete()
return errors.Err(string(errorLog))
}
return errors.Err(err)
fi, err := os.Stat(v.getFullPath())
if err != nil {
return errors.Err(err)
}
videoSize := fi.Size()
v.size = &videoSize
return nil
}
func (v *YoutubeVideo) videoDir() string {
@@ -418,7 +391,7 @@ type SyncParams struct {
}
func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo, reprocess bool, walletLock *sync.RWMutex) (*SyncSummary, error) {
v.maxVideoSize = int64(params.MaxVideoSize) * 1024 * 1024
v.maxVideoSize = int64(params.MaxVideoSize)
v.maxVideoLength = params.MaxVideoLength
v.lbryChannelID = params.ChannelID
v.walletLock = walletLock
@@ -430,13 +403,15 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV
}
func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
err := v.download()
err := v.download(false)
if err != nil {
log.Errorf("standard downloader failed: %s. Trying fallback downloader\n", err.Error())
fallBackErr := v.fallbackDownload()
if fallBackErr != nil {
log.Errorf("fallback downloader failed: %s\n", fallBackErr.Error())
return nil, errors.Prefix("download error", err) //return original error
if strings.Contains(err.Error(), "HTTP Error 429") {
err = v.download(true)
if err != nil {
return nil, errors.Prefix("download error", err)
}
} else {
return nil, errors.Prefix("download error", err)
}
}
log.Debugln("Downloaded " + v.id)