Compare commits

...
8 Commits
Author SHA1 Message Date
Niko Storni bab01a6354 prefer IPv4 over IPv6
youtube sucks and they ban /64 subnet blocks on IPv6
2019-07-12 01:54:34 +02:00
Niko Storni 89fe1e99d6 improve ipv6 validation
fix audio merge issue
2019-07-12 01:28:00 +02:00
Niko Storni ec81af0857 fix download filters 2019-07-12 01:04:45 +02:00
Niko Storni 51cb7f5e34 improve youtube antithrottle
fix logging
2019-07-11 19:14:15 +02:00
Niko Storni 8ad2acc936 remove original downloader
add locking to IP selection to avoid concurrent access
2019-07-11 17:32:46 +02:00
Niko Storni b8caf3f82b add IPv6 source pooling to bypass youtube throttling 2019-07-11 16:22:58 +02:00
Niko Storni b34d328cba halt ytsync when youtube throttles us 2019-07-10 22:47:01 +02:00
Niko Storni a0dcb2216d close db handles 2019-07-10 22:23:57 +02:00
6 changed files with 112 additions and 89 deletions

No files matched your search

+7
View File
@@ -63,6 +63,13 @@ func reflectBlobs() error {
if err != nil {
return errors.Err(err)
}
defer func() {
err := dbHandle.CloseDB()
if err != nil {
log.Errorf("failed to close db handle: %s", err.Error())
}
}()
st := store.NewDBBackedS3Store(
store.NewS3BlobStore(config.AwsID, config.AwsSecret, config.BucketRegion, config.BucketName),
dbHandle)
+2 -1
View File
@@ -5,6 +5,7 @@ require (
github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61
github.com/PuerkitoBio/goquery v1.5.0 // indirect
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf
github.com/aws/aws-sdk-go v1.17.3
github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 // indirect
github.com/go-ini/ini v1.42.0 // indirect
@@ -18,7 +19,7 @@ require (
github.com/kr/pretty v0.1.0 // indirect
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c
github.com/lbryio/lbry.go v1.0.15
github.com/lbryio/reflector.go v1.0.6-0.20190710190215-191671b3cf71
github.com/lbryio/reflector.go v1.0.6-0.20190710201919-7c7ed9da72ce
github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2 // indirect
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936
github.com/nikooo777/ytdl v0.0.0-20190215151411-9c7832eaf457
+3 -2
View File
@@ -25,6 +25,7 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM=
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf h1:eg0MeVzsP1G42dRafH3vf+al2vQIJU0YHX+1Tw87oco=
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
github.com/aws/aws-sdk-go v1.16.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.17.3 h1:KBXxg7Jh0TxE5zmpNB2DwKmJeDUqh0O6jhy25TuYOmc=
@@ -197,8 +198,8 @@ github.com/lbryio/lbry.go v1.0.15/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002 h1:urfYK5ElpUrAv90auPLldoVC60LwiGAcY0OE6HJB9KI=
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4=
github.com/lbryio/reflector.go v1.0.6-0.20190710190215-191671b3cf71 h1:mH+Xa6K441w+QJ6u3GVZ+bmlJtm1W7SFhGE6cL6CCbQ=
github.com/lbryio/reflector.go v1.0.6-0.20190710190215-191671b3cf71/go.mod h1:7Y3YYeAKS6egH2WzwfU8f6+uNGjVHfLzKvwn+Nv3VMY=
github.com/lbryio/reflector.go v1.0.6-0.20190710201919-7c7ed9da72ce h1:ixsKssly++rkh0DWxNDrZyiRp00NieKZlndkqtpgj/c=
github.com/lbryio/reflector.go v1.0.6-0.20190710201919-7c7ed9da72ce/go.mod h1:7Y3YYeAKS6egH2WzwfU8f6+uNGjVHfLzKvwn+Nv3VMY=
github.com/lbryio/types v0.0.0-20181001180206-594241d24e00/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c h1:m3O7561xBQ00lfUVayW4c6SnpVbUDQtPUwGcGYSUYQA=
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
+1
View File
@@ -199,6 +199,7 @@ func (s *SyncManager) Start() error {
"failure uploading wallet",
"the channel in the wallet is different than the channel in the database",
"this channel does not belong to this wallet!",
"HTTP Error 429",
}
if util.SubstringInSlice(err.Error(), fatalErrors) {
return errors.Prefix("@Nikooo777 this requires manual intervention! Exiting...", err)
+2 -1
View File
@@ -656,6 +656,7 @@ func (s *Sync) startWorker(workerNum int) {
"Cannot publish using channel",
"cannot concatenate 'str' and 'NoneType' objects",
"more than 90% of the space has been used.",
"HTTP Error 429",
}
if util.SubstringInSlice(err.Error(), fatalErrors) || s.StopOnError {
s.grp.Stop()
@@ -677,7 +678,7 @@ func (s *Sync) startWorker(workerNum int) {
"Watch this video on YouTube.",
"have blocked it on copyright grounds",
"the video must be republished as we can't get the right size",
"Invalid status code: 403",
"HTTP Error 403",
}
if util.SubstringInSlice(err.Error(), errorsNoRetry) {
log.Println("This error should not be retried at all")
+97 -85
View File
@@ -4,6 +4,7 @@ import (
"fmt"
"io/ioutil"
"math"
"net"
"os"
"os/exec"
"path/filepath"
@@ -23,8 +24,8 @@ import (
"github.com/lbryio/ytsync/thumbs"
"github.com/ChannelMeter/iso8601duration"
"github.com/asaskevich/govalidator"
"github.com/aws/aws-sdk-go/aws"
"github.com/nikooo777/ytdl"
"github.com/shopspring/decimal"
log "github.com/sirupsen/logrus"
"google.golang.org/api/youtube/v3"
@@ -173,33 +174,38 @@ func (v *YoutubeVideo) getAbbrevDescription() string {
return strings.Join(strings.Split(description, "\n")[:maxLines], "\n") + "\n..." + additionalDescription
}
func (v *YoutubeVideo) fallbackDownload() error {
cmd := exec.Command("youtube-dl",
"--no-progress",
"-fbestvideo[ext=mp4,height<=1080,filesize<2000M]+best[ext=mp4,height<=1080,filesize<2000M]",
"-o"+strings.TrimRight(v.getFullPath(), ".mp4"),
"--merge-output-format",
"mp4",
"https://www.youtube.com/watch?v="+v.ID())
var ipPool []string
var IPIndex int
var ipMutex sync.Mutex
log.Printf("Running command and waiting for it to finish...")
output, err := cmd.CombinedOutput()
log.Debugln(string(output))
if err != nil {
log.Printf("Command finished with error: %v", errors.Err(string(output)))
_ = v.delete()
return errors.Err(err)
func getNextIP() (string, error) {
ipMutex.Lock()
defer ipMutex.Unlock()
if len(ipPool) < 1 {
IPIndex = 0
addrs, err := net.InterfaceAddrs()
if err != nil {
return "", errors.Err(err)
}
for _, address := range addrs {
if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) {
ipPool = append(ipPool, ipnet.IP.String())
}
}
}
}
fi, err := os.Stat(v.getFullPath())
if err != nil {
return errors.Err(err)
nextIP := ipPool[IPIndex]
if IPIndex == len(ipPool)-1 {
IPIndex = 0
} else {
IPIndex++
}
videoSize := fi.Size()
v.size = &videoSize
return nil
return nextIP, nil
}
func (v *YoutubeVideo) download() error {
func (v *YoutubeVideo) download(useIPv6 bool) error {
videoPath := v.getFullPath()
err := os.Mkdir(v.videoDir(), 0750)
@@ -215,71 +221,72 @@ func (v *YoutubeVideo) download() error {
return nil
}
videoUrl := "https://www.youtube.com/watch?v=" + v.id
videoInfo, err := ytdl.GetVideoInfo(videoUrl)
ytdlArgs := []string{
"--no-progress",
"--max-filesize",
fmt.Sprintf("%dM", v.maxVideoSize),
"-fbestvideo[ext=mp4][height<=1080]+bestaudio[ext!=webm]",
"-o" + strings.TrimRight(v.getFullPath(), ".mp4"),
"--merge-output-format",
"mp4",
}
if useIPv6 {
sourceAddress, err := getNextIP()
if err != nil {
return errors.Err(err)
}
log.Infof("using IPv6: %s", sourceAddress)
ytdlArgs = append(ytdlArgs,
"-6",
"--source-address",
sourceAddress,
)
} else {
log.Infoln("using IPv4")
ytdlArgs = append(ytdlArgs, "-4")
}
ytdlArgs = append(ytdlArgs, "https://www.youtube.com/watch?v="+v.ID())
cmd := exec.Command("youtube-dl", ytdlArgs...)
log.Printf("Running command and waiting for it to finish...")
stderr, err := cmd.StderrPipe()
if err != nil {
return errors.Err(err)
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return errors.Err(err)
}
codec := []string{"H.264"}
ext := []string{"mp4"}
//Filter requires a [] interface{}
codecFilter := make([]interface{}, len(codec))
for i, v := range codec {
codecFilter[i] = v
if err := cmd.Start(); err != nil {
return errors.Err(err)
}
//Filter requires a [] interface{}
extFilter := make([]interface{}, len(ext))
for i, v := range ext {
extFilter[i] = v
errorLog, _ := ioutil.ReadAll(stderr)
outLog, _ := ioutil.ReadAll(stdout)
if err = cmd.Wait(); err != nil {
if strings.Contains(err.Error(), "exit status 1") {
return errors.Err(string(errorLog))
}
return errors.Err(err)
}
formats := videoInfo.Formats.Filter(ytdl.FormatVideoEncodingKey, codecFilter).Filter(ytdl.FormatExtensionKey, extFilter)
if len(formats) == 0 {
return errors.Err("no compatible format available for this video")
}
maxRetryAttempts := 5
isLengthLimitSet := v.maxVideoLength > 0.01
if isLengthLimitSet && videoInfo.Duration.Hours() > v.maxVideoLength {
return errors.Err("video is too long to process")
}
log.Debugln(string(outLog))
for i := 0; i < len(formats) && i < maxRetryAttempts; i++ {
formatIndex := i
if i == maxRetryAttempts-1 {
formatIndex = len(formats) - 1
}
var downloadedFile *os.File
downloadedFile, err = os.Create(videoPath)
if err != nil {
return errors.Err(err)
}
err = videoInfo.Download(formats[formatIndex], downloadedFile)
_ = downloadedFile.Close()
if err != nil {
//delete the video and ignore the error
_ = v.delete()
return errors.Err(err.Error())
}
fi, err := os.Stat(v.getFullPath())
if err != nil {
return errors.Err(err)
}
videoSize := fi.Size()
v.size = &videoSize
isVideoSizeLimitSet := v.maxVideoSize > 0
if isVideoSizeLimitSet && videoSize > v.maxVideoSize {
//delete the video and ignore the error
_ = v.delete()
err = errors.Err("file is too big and there is no other format available")
} else {
break
}
if string(errorLog) != "" {
log.Printf("Command finished with error: %v", errors.Err(string(errorLog)))
_ = v.delete()
return errors.Err(string(errorLog))
}
return errors.Err(err)
fi, err := os.Stat(v.getFullPath())
if err != nil {
return errors.Err(err)
}
videoSize := fi.Size()
v.size = &videoSize
return nil
}
func (v *YoutubeVideo) videoDir() string {
@@ -382,7 +389,7 @@ type SyncParams struct {
}
func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo, reprocess bool, walletLock *sync.RWMutex) (*SyncSummary, error) {
v.maxVideoSize = int64(params.MaxVideoSize) * 1024 * 1024
v.maxVideoSize = int64(params.MaxVideoSize)
v.maxVideoLength = params.MaxVideoLength
v.lbryChannelID = params.ChannelID
v.walletLock = walletLock
@@ -393,14 +400,19 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV
return v.downloadAndPublish(daemon, params)
}
var isThrottled bool
func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
err := v.download()
err := v.download(isThrottled)
if err != nil {
log.Errorf("standard downloader failed: %s. Trying fallback downloader\n", err.Error())
fallBackErr := v.fallbackDownload()
if fallBackErr != nil {
log.Errorf("fallback downloader failed: %s\n", fallBackErr.Error())
return nil, errors.Prefix("download error", err) //return original error
if strings.Contains(err.Error(), "HTTP Error 429") && !isThrottled {
isThrottled = true
err = v.download(isThrottled)
if err != nil {
return nil, errors.Prefix("download error", err)
}
} else {
return nil, errors.Prefix("download error", err)
}
}
log.Debugln("Downloaded " + v.id)