Compare commits

...
15 Commits
Author SHA1 Message Date
Niko Storni 1331e5769f actually use the functions I create... 2019-07-13 11:33:40 +02:00
Niko Storni 43f745ae20 improve ip shuffling (prevent ips in use from being dealt) 2019-07-12 23:54:48 +02:00
Niko Storni 8c2a8262e6 fix bugs 2019-07-12 23:20:01 +02:00
Niko Storni 6f486717da fix stop pattern
fix noretry string
fix youtube-dl param
2019-07-12 22:58:34 +02:00
Niko Storni a3fcd67611 better handle interruptions by user
refactor IP throttling in its own package
2019-07-12 21:32:49 +02:00
Niko Storni ae1ffb60c5 add ip throttling and better pooling
fix nil pointer dereference
2019-07-12 20:42:44 +02:00
Niko Storni fa24d83ae9 add max length filter 2019-07-12 02:39:24 +02:00
Niko Storni 2f11d99a97 stop on rare bug/failure 2019-07-12 02:21:27 +02:00
Niko Storni bab01a6354 prefer IPv4 over IPv6
youtube sucks and they ban /64 subnet blocks on IPv6
2019-07-12 01:54:34 +02:00
Niko Storni 89fe1e99d6 improve ipv6 validation
fix audio merge issue
2019-07-12 01:28:00 +02:00
Niko Storni ec81af0857 fix download filters 2019-07-12 01:04:45 +02:00
Niko Storni 51cb7f5e34 improve youtube antithrottle
fix logging
2019-07-11 19:14:15 +02:00
Niko Storni 8ad2acc936 remove original downloader
add locking to IP selection to avoid concurrent access
2019-07-11 17:32:46 +02:00
Niko Storni b8caf3f82b add IPv6 source pooling to bypass youtube throttling 2019-07-11 16:22:58 +02:00
Niko Storni b34d328cba halt ytsync when youtube throttles us 2019-07-10 22:47:01 +02:00
7 changed files with 240 additions and 99 deletions

No files matched your search

+1
View File
@@ -5,6 +5,7 @@ require (
github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61
github.com/PuerkitoBio/goquery v1.5.0 // indirect
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf
github.com/aws/aws-sdk-go v1.17.3
github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 // indirect
github.com/go-ini/ini v1.42.0 // indirect
+1
View File
@@ -25,6 +25,7 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM=
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf h1:eg0MeVzsP1G42dRafH3vf+al2vQIJU0YHX+1Tw87oco=
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
github.com/aws/aws-sdk-go v1.16.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.17.3 h1:KBXxg7Jh0TxE5zmpNB2DwKmJeDUqh0O6jhy25TuYOmc=
+119
View File
@@ -0,0 +1,119 @@
package ipManager
import (
"github.com/asaskevich/govalidator"
log "github.com/sirupsen/logrus"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/stop"
"net"
"sync"
"time"
)
const IPCooldownPeriod = 20 * time.Second
const unbanTimeout = 3 * time.Hour
var ipv6Pool []string
var ipv4Pool []string
var throttledIPs map[string]bool
var ipInUse map[string]bool
var ipLastUsed map[string]time.Time
var ipMutex sync.Mutex
var stopper = stop.New()
func GetNextIP(ipv6 bool) (string, error) {
ipMutex.Lock()
defer ipMutex.Unlock()
if len(ipv4Pool) < 1 || len(ipv6Pool) < 1 {
throttledIPs = make(map[string]bool)
ipInUse = make(map[string]bool)
ipLastUsed = make(map[string]time.Time)
addrs, err := net.InterfaceAddrs()
if err != nil {
return "", errors.Err(err)
}
for _, address := range addrs {
if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) {
ipv6Pool = append(ipv6Pool, ipnet.IP.String())
ipLastUsed[ipnet.IP.String()] = time.Now().Add(-IPCooldownPeriod)
} else if ipnet.IP.To4() != nil && govalidator.IsIPv4(ipnet.IP.String()) {
ipv4Pool = append(ipv4Pool, ipnet.IP.String())
ipLastUsed[ipnet.IP.String()] = time.Now().Add(-IPCooldownPeriod)
}
}
}
}
nextIP := ""
if ipv6 {
nextIP = getLeastUsedIP(ipv6Pool)
} else {
nextIP = getLeastUsedIP(ipv4Pool)
}
if nextIP == "" {
return "throttled", errors.Err("all IPs are throttled")
}
lastUse := ipLastUsed[nextIP]
if time.Since(lastUse) < IPCooldownPeriod {
log.Debugf("The IP %s is too hot, waiting for %.1f seconds before continuing", nextIP, (IPCooldownPeriod - time.Since(lastUse)).Seconds())
time.Sleep(IPCooldownPeriod - time.Since(lastUse))
}
ipInUse[nextIP] = true
return nextIP, nil
}
func ReleaseIP(ip string) {
ipMutex.Lock()
defer ipMutex.Unlock()
ipLastUsed[ip] = time.Now()
ipInUse[ip] = false
}
func getLeastUsedIP(ipPool []string) string {
nextIP := ""
veryLastUse := time.Now()
for _, ip := range ipPool {
isThrottled := throttledIPs[ip]
if isThrottled {
continue
}
inUse := ipInUse[ip]
if inUse {
continue
}
lastUse := ipLastUsed[ip]
if lastUse.Before(veryLastUse) {
nextIP = ip
veryLastUse = lastUse
}
}
return nextIP
}
func SetIpThrottled(ip string, stopGrp *stop.Group) {
ipMutex.Lock()
defer ipMutex.Unlock()
isThrottled := throttledIPs[ip]
if isThrottled {
return
}
throttledIPs[ip] = true
log.Printf("%s set to throttled", ip)
stopper.Add(1)
go func() {
defer stopper.Done()
unbanTimer := time.NewTimer(unbanTimeout)
select {
case <-unbanTimer.C:
throttledIPs[ip] = false
log.Printf("%s set back to not throttled", ip)
case <-stopGrp.Ch():
unbanTimer.Stop()
}
}()
}
+1
View File
@@ -199,6 +199,7 @@ func (s *SyncManager) Start() error {
"failure uploading wallet",
"the channel in the wallet is different than the channel in the database",
"this channel does not belong to this wallet!",
"HTTP Error 429",
}
if util.SubstringInSlice(err.Error(), fatalErrors) {
return errors.Prefix("@Nikooo777 this requires manual intervention! Exiting...", err)
+10 -4
View File
@@ -649,6 +649,9 @@ func (s *Sync) startWorker(workerNum int) {
if err != nil {
logMsg := fmt.Sprintf("error processing video %s: %s", v.ID(), err.Error())
log.Errorln(logMsg)
if strings.Contains(strings.ToLower(err.Error()), "interrupted by user") {
return
}
fatalErrors := []string{
":5279: read: connection reset by peer",
"no space left on device",
@@ -656,13 +659,15 @@ func (s *Sync) startWorker(workerNum int) {
"Cannot publish using channel",
"cannot concatenate 'str' and 'NoneType' objects",
"more than 90% of the space has been used.",
"HTTP Error 429",
"Couldn't find private key for id",
}
if util.SubstringInSlice(err.Error(), fatalErrors) || s.StopOnError {
s.grp.Stop()
} else if s.MaxTries > 1 {
errorsNoRetry := []string{
"non 200 status code received",
" reason: 'This video contains content from",
"This video contains content from",
"dont know which claim to update",
"uploader has not made this video available in your country",
"download error: AccessDenied: Access Denied",
@@ -677,7 +682,7 @@ func (s *Sync) startWorker(workerNum int) {
"Watch this video on YouTube.",
"have blocked it on copyright grounds",
"the video must be republished as we can't get the right size",
"Invalid status code: 403",
"HTTP Error 403",
}
if util.SubstringInSlice(err.Error(), errorsNoRetry) {
log.Println("This error should not be retried at all")
@@ -809,7 +814,7 @@ func (s *Sync) enqueueYoutubeVideos() error {
return errors.Prefix("error getting videos info", err)
}
for _, item := range videosListResponse.Items {
videos = append(videos, sources.NewYoutubeVideo(s.videoDirectory, item, playlistMap[item.Id].Position, s.Manager.GetS3AWSConfig()))
videos = append(videos, sources.NewYoutubeVideo(s.videoDirectory, item, playlistMap[item.Id].Position, s.Manager.GetS3AWSConfig(), s.grp))
}
log.Infof("Got info for %d videos from youtube API", len(videos))
@@ -825,7 +830,7 @@ func (s *Sync) enqueueYoutubeVideos() error {
}
_, ok := playlistMap[k]
if !ok {
videos = append(videos, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig()))
videos = append(videos, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig(), s.grp))
}
}
@@ -880,6 +885,7 @@ func (s *Sync) processVideo(v video) (err error) {
"Unable to extract signature tokens",
"the video is too big to sync, skipping for now",
"video is too long to process",
"This video contains content from",
"no compatible format available for this video",
"Watch this video on YouTube.",
"have blocked it on copyright grounds",
+5 -2
View File
@@ -54,7 +54,7 @@ func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeC
Data []YoutubeChannel `json:"data"`
}
endpoint := a.ApiURL + "/yt/jobs"
res, _ := http.PostForm(endpoint, url.Values{
res, err := http.PostForm(endpoint, url.Values{
"auth_token": {a.ApiToken},
"sync_status": {status},
"min_videos": {strconv.Itoa(1)},
@@ -63,10 +63,13 @@ func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeC
"sync_server": {a.HostName},
"channel_id": {cp.YoutubeChannelID},
})
if err != nil {
return nil, errors.Err(err)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
var response apiJobsResponse
err := json.Unmarshal(body, &response)
err = json.Unmarshal(body, &response)
if err != nil {
return nil, err
}
+103 -93
View File
@@ -15,8 +15,10 @@ import (
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/jsonrpc"
"github.com/lbryio/lbry.go/extras/stop"
"github.com/lbryio/lbry.go/extras/util"
"github.com/lbryio/ytsync/ipManager"
"github.com/lbryio/ytsync/namer"
"github.com/lbryio/ytsync/sdk"
"github.com/lbryio/ytsync/tagsManager"
@@ -24,7 +26,6 @@ import (
"github.com/ChannelMeter/iso8601duration"
"github.com/aws/aws-sdk-go/aws"
"github.com/nikooo777/ytdl"
"github.com/shopspring/decimal"
log "github.com/sirupsen/logrus"
"google.golang.org/api/youtube/v3"
@@ -48,10 +49,9 @@ type YoutubeVideo struct {
lbryChannelID string
mocked bool
walletLock *sync.RWMutex
stopGroup *stop.Group
}
const reflectorURL = "http://blobs.lbry.io/"
var youtubeCategories = map[string]string{
"1": "film & animation",
"2": "autos & vehicles",
@@ -87,7 +87,7 @@ var youtubeCategories = map[string]string{
"44": "trailers",
}
func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64, awsConfig aws.Config) *YoutubeVideo {
func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group) *YoutubeVideo {
publishedAt, _ := time.Parse(time.RFC3339Nano, videoData.Snippet.PublishedAt) // ignore parse errors
return &YoutubeVideo{
id: videoData.Id,
@@ -100,9 +100,10 @@ func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPositio
awsConfig: awsConfig,
mocked: false,
youtubeChannelID: videoData.Snippet.ChannelId,
stopGroup: stopGroup,
}
}
func NewMockedVideo(directory string, videoID string, youtubeChannelID string, awsConfig aws.Config) *YoutubeVideo {
func NewMockedVideo(directory string, videoID string, youtubeChannelID string, awsConfig aws.Config, stopGroup *stop.Group) *YoutubeVideo {
return &YoutubeVideo{
id: videoID,
playlistPosition: 0,
@@ -110,6 +111,7 @@ func NewMockedVideo(directory string, videoID string, youtubeChannelID string, a
awsConfig: awsConfig,
mocked: true,
youtubeChannelID: youtubeChannelID,
stopGroup: stopGroup,
}
}
@@ -173,33 +175,7 @@ func (v *YoutubeVideo) getAbbrevDescription() string {
return strings.Join(strings.Split(description, "\n")[:maxLines], "\n") + "\n..." + additionalDescription
}
func (v *YoutubeVideo) fallbackDownload() error {
cmd := exec.Command("youtube-dl",
"--no-progress",
"-fbestvideo[ext=mp4,height<=1080,filesize<2000M]+best[ext=mp4,height<=1080,filesize<2000M]",
"-o"+strings.TrimRight(v.getFullPath(), ".mp4"),
"--merge-output-format",
"mp4",
"https://www.youtube.com/watch?v="+v.ID())
log.Printf("Running command and waiting for it to finish...")
output, err := cmd.CombinedOutput()
log.Debugln(string(output))
if err != nil {
log.Printf("Command finished with error: %v", errors.Err(string(output)))
_ = v.delete()
return errors.Err(err)
}
fi, err := os.Stat(v.getFullPath())
if err != nil {
return errors.Err(err)
}
videoSize := fi.Size()
v.size = &videoSize
return nil
}
func (v *YoutubeVideo) download() error {
func (v *YoutubeVideo) download(useIPv6 bool) error {
videoPath := v.getFullPath()
err := os.Mkdir(v.videoDir(), 0750)
@@ -215,71 +191,100 @@ func (v *YoutubeVideo) download() error {
return nil
}
videoUrl := "https://www.youtube.com/watch?v=" + v.id
videoInfo, err := ytdl.GetVideoInfo(videoUrl)
ytdlArgs := []string{
"--no-progress",
"--max-filesize",
fmt.Sprintf("%dM", v.maxVideoSize),
"--match-filter",
fmt.Sprintf("duration <= %d", int(math.Round(v.maxVideoLength*3600))),
"-fbestvideo[ext=mp4][height<=1080]+bestaudio[ext!=webm]",
"-o" + strings.TrimSuffix(v.getFullPath(), ".mp4"),
"--merge-output-format",
"mp4",
}
sourceAddress, err := ipManager.GetNextIP(useIPv6)
if err != nil {
if sourceAddress == "throttled" {
for {
select {
case <-v.stopGroup.Ch():
return errors.Err("interrupted by user")
default:
time.Sleep(20 * time.Second)
sourceAddress, err = ipManager.GetNextIP(useIPv6)
if err == nil {
break
}
}
}
} else {
return errors.Err(err)
}
}
defer ipManager.ReleaseIP(sourceAddress)
if useIPv6 {
log.Infof("using IPv6: %s", sourceAddress)
ytdlArgs = append(ytdlArgs,
"-6",
"--source-address",
sourceAddress,
)
} else {
log.Infof("using IPv4: %s", sourceAddress)
ytdlArgs = append(ytdlArgs,
"-4",
"--source-address",
sourceAddress,
)
}
ytdlArgs = append(ytdlArgs, "https://www.youtube.com/watch?v="+v.ID())
cmd := exec.Command("youtube-dl", ytdlArgs...)
log.Printf("Running command and waiting for it to finish...")
stderr, err := cmd.StderrPipe()
if err != nil {
return errors.Err(err)
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return errors.Err(err)
}
codec := []string{"H.264"}
ext := []string{"mp4"}
//Filter requires a [] interface{}
codecFilter := make([]interface{}, len(codec))
for i, v := range codec {
codecFilter[i] = v
if err := cmd.Start(); err != nil {
return errors.Err(err)
}
//Filter requires a [] interface{}
extFilter := make([]interface{}, len(ext))
for i, v := range ext {
extFilter[i] = v
}
errorLog, _ := ioutil.ReadAll(stderr)
outLog, _ := ioutil.ReadAll(stdout)
formats := videoInfo.Formats.Filter(ytdl.FormatVideoEncodingKey, codecFilter).Filter(ytdl.FormatExtensionKey, extFilter)
if len(formats) == 0 {
return errors.Err("no compatible format available for this video")
if err = cmd.Wait(); err != nil {
if strings.Contains(err.Error(), "exit status 1") {
if strings.Contains(string(errorLog), "HTTP Error 429") {
ipManager.SetIpThrottled(sourceAddress, v.stopGroup)
}
return errors.Err(string(errorLog))
}
return errors.Err(err)
}
maxRetryAttempts := 5
isLengthLimitSet := v.maxVideoLength > 0.01
if isLengthLimitSet && videoInfo.Duration.Hours() > v.maxVideoLength {
log.Debugln(string(outLog))
if strings.Contains(string(outLog), "does not pass filter duration") {
_ = v.delete()
return errors.Err("video is too long to process")
}
for i := 0; i < len(formats) && i < maxRetryAttempts; i++ {
formatIndex := i
if i == maxRetryAttempts-1 {
formatIndex = len(formats) - 1
}
var downloadedFile *os.File
downloadedFile, err = os.Create(videoPath)
if err != nil {
return errors.Err(err)
}
err = videoInfo.Download(formats[formatIndex], downloadedFile)
_ = downloadedFile.Close()
if err != nil {
//delete the video and ignore the error
_ = v.delete()
return errors.Err(err.Error())
}
fi, err := os.Stat(v.getFullPath())
if err != nil {
return errors.Err(err)
}
videoSize := fi.Size()
v.size = &videoSize
isVideoSizeLimitSet := v.maxVideoSize > 0
if isVideoSizeLimitSet && videoSize > v.maxVideoSize {
//delete the video and ignore the error
_ = v.delete()
err = errors.Err("file is too big and there is no other format available")
} else {
break
}
if string(errorLog) != "" {
log.Printf("Command finished with error: %v", errors.Err(string(errorLog)))
_ = v.delete()
return errors.Err(string(errorLog))
}
return errors.Err(err)
fi, err := os.Stat(v.getFullPath())
if err != nil {
return errors.Err(err)
}
videoSize := fi.Size()
v.size = &videoSize
return nil
}
func (v *YoutubeVideo) videoDir() string {
@@ -382,7 +387,7 @@ type SyncParams struct {
}
func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo, reprocess bool, walletLock *sync.RWMutex) (*SyncSummary, error) {
v.maxVideoSize = int64(params.MaxVideoSize) * 1024 * 1024
v.maxVideoSize = int64(params.MaxVideoSize)
v.maxVideoLength = params.MaxVideoLength
v.lbryChannelID = params.ChannelID
v.walletLock = walletLock
@@ -393,14 +398,19 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV
return v.downloadAndPublish(daemon, params)
}
var isThrottled bool
func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
err := v.download()
err := v.download(isThrottled)
if err != nil {
log.Errorf("standard downloader failed: %s. Trying fallback downloader\n", err.Error())
fallBackErr := v.fallbackDownload()
if fallBackErr != nil {
log.Errorf("fallback downloader failed: %s\n", fallBackErr.Error())
return nil, errors.Prefix("download error", err) //return original error
if strings.Contains(err.Error(), "HTTP Error 429") && !isThrottled {
isThrottled = true
err = v.download(isThrottled)
if err != nil {
return nil, errors.Prefix("download error", err)
}
} else {
return nil, errors.Prefix("download error", err)
}
}
log.Debugln("Downloaded " + v.id)