Compare commits

...
11 Commits
Author SHA1 Message Date
Niko Storni df08d42d9b improve ip re-selection 2019-07-13 12:15:19 +02:00
Niko Storni 1b55033bb8 don't retry big videos 2019-07-13 11:40:50 +02:00
Niko Storni 1331e5769f actually use the functions I create... 2019-07-13 11:33:40 +02:00
Niko Storni 43f745ae20 improve ip shuffling (prevent ips in use from being dealt) 2019-07-12 23:54:48 +02:00
Niko Storni 8c2a8262e6 fix bugs 2019-07-12 23:20:01 +02:00
Niko Storni 6f486717da fix stop pattern
fix noretry string
fix youtube-dl param
2019-07-12 22:58:34 +02:00
Niko Storni a3fcd67611 better handle interruptions by user
refactor IP throttling in its own package
2019-07-12 21:32:49 +02:00
Niko Storni ae1ffb60c5 add ip throttling and better pooling
fix nil pointer dereference
2019-07-12 20:42:44 +02:00
Niko Storni fa24d83ae9 add max length filter 2019-07-12 02:39:24 +02:00
Niko Storni 2f11d99a97 stop on rare bug/failure 2019-07-12 02:21:27 +02:00
Niko Storni bab01a6354 prefer IPv4 over IPv6
youtube sucks and they ban /64 subnet blocks on IPv6
2019-07-12 01:54:34 +02:00
4 changed files with 186 additions and 55 deletions

No files matched your search

+119
View File
@@ -0,0 +1,119 @@
package ipManager
import (
"github.com/asaskevich/govalidator"
log "github.com/sirupsen/logrus"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/stop"
"net"
"sync"
"time"
)
const IPCooldownPeriod = 20 * time.Second
const unbanTimeout = 3 * time.Hour
var ipv6Pool []string
var ipv4Pool []string
var throttledIPs map[string]bool
var ipInUse map[string]bool
var ipLastUsed map[string]time.Time
var ipMutex sync.Mutex
var stopper = stop.New()
func GetNextIP(ipv6 bool) (string, error) {
ipMutex.Lock()
defer ipMutex.Unlock()
if len(ipv4Pool) < 1 || len(ipv6Pool) < 1 {
throttledIPs = make(map[string]bool)
ipInUse = make(map[string]bool)
ipLastUsed = make(map[string]time.Time)
addrs, err := net.InterfaceAddrs()
if err != nil {
return "", errors.Err(err)
}
for _, address := range addrs {
if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) {
ipv6Pool = append(ipv6Pool, ipnet.IP.String())
ipLastUsed[ipnet.IP.String()] = time.Now().Add(-IPCooldownPeriod)
} else if ipnet.IP.To4() != nil && govalidator.IsIPv4(ipnet.IP.String()) {
ipv4Pool = append(ipv4Pool, ipnet.IP.String())
ipLastUsed[ipnet.IP.String()] = time.Now().Add(-IPCooldownPeriod)
}
}
}
}
nextIP := ""
if ipv6 {
nextIP = getLeastUsedIP(ipv6Pool)
} else {
nextIP = getLeastUsedIP(ipv4Pool)
}
if nextIP == "" {
return "throttled", errors.Err("all IPs are throttled")
}
lastUse := ipLastUsed[nextIP]
if time.Since(lastUse) < IPCooldownPeriod {
log.Debugf("The IP %s is too hot, waiting for %.1f seconds before continuing", nextIP, (IPCooldownPeriod - time.Since(lastUse)).Seconds())
time.Sleep(IPCooldownPeriod - time.Since(lastUse))
}
ipInUse[nextIP] = true
return nextIP, nil
}
func ReleaseIP(ip string) {
ipMutex.Lock()
defer ipMutex.Unlock()
ipLastUsed[ip] = time.Now()
ipInUse[ip] = false
}
func getLeastUsedIP(ipPool []string) string {
nextIP := ""
veryLastUse := time.Now()
for _, ip := range ipPool {
isThrottled := throttledIPs[ip]
if isThrottled {
continue
}
inUse := ipInUse[ip]
if inUse {
continue
}
lastUse := ipLastUsed[ip]
if lastUse.Before(veryLastUse) {
nextIP = ip
veryLastUse = lastUse
}
}
return nextIP
}
func SetIpThrottled(ip string, stopGrp *stop.Group) {
ipMutex.Lock()
defer ipMutex.Unlock()
isThrottled := throttledIPs[ip]
if isThrottled {
return
}
throttledIPs[ip] = true
log.Printf("%s set to throttled", ip)
stopper.Add(1)
go func() {
defer stopper.Done()
unbanTimer := time.NewTimer(unbanTimeout)
select {
case <-unbanTimer.C:
throttledIPs[ip] = false
log.Printf("%s set back to not throttled", ip)
case <-stopGrp.Ch():
unbanTimer.Stop()
}
}()
}
+8 -3
View File
@@ -649,6 +649,9 @@ func (s *Sync) startWorker(workerNum int) {
if err != nil {
logMsg := fmt.Sprintf("error processing video %s: %s", v.ID(), err.Error())
log.Errorln(logMsg)
if strings.Contains(strings.ToLower(err.Error()), "interrupted by user") {
return
}
fatalErrors := []string{
":5279: read: connection reset by peer",
"no space left on device",
@@ -657,13 +660,14 @@ func (s *Sync) startWorker(workerNum int) {
"cannot concatenate 'str' and 'NoneType' objects",
"more than 90% of the space has been used.",
"HTTP Error 429",
"Couldn't find private key for id",
}
if util.SubstringInSlice(err.Error(), fatalErrors) || s.StopOnError {
s.grp.Stop()
} else if s.MaxTries > 1 {
errorsNoRetry := []string{
"non 200 status code received",
" reason: 'This video contains content from",
"This video contains content from",
"dont know which claim to update",
"uploader has not made this video available in your country",
"download error: AccessDenied: Access Denied",
@@ -810,7 +814,7 @@ func (s *Sync) enqueueYoutubeVideos() error {
return errors.Prefix("error getting videos info", err)
}
for _, item := range videosListResponse.Items {
videos = append(videos, sources.NewYoutubeVideo(s.videoDirectory, item, playlistMap[item.Id].Position, s.Manager.GetS3AWSConfig()))
videos = append(videos, sources.NewYoutubeVideo(s.videoDirectory, item, playlistMap[item.Id].Position, s.Manager.GetS3AWSConfig(), s.grp))
}
log.Infof("Got info for %d videos from youtube API", len(videos))
@@ -826,7 +830,7 @@ func (s *Sync) enqueueYoutubeVideos() error {
}
_, ok := playlistMap[k]
if !ok {
videos = append(videos, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig()))
videos = append(videos, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig(), s.grp))
}
}
@@ -881,6 +885,7 @@ func (s *Sync) processVideo(v video) (err error) {
"Unable to extract signature tokens",
"the video is too big to sync, skipping for now",
"video is too long to process",
"This video contains content from",
"no compatible format available for this video",
"Watch this video on YouTube.",
"have blocked it on copyright grounds",
+5 -2
View File
@@ -54,7 +54,7 @@ func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeC
Data []YoutubeChannel `json:"data"`
}
endpoint := a.ApiURL + "/yt/jobs"
res, _ := http.PostForm(endpoint, url.Values{
res, err := http.PostForm(endpoint, url.Values{
"auth_token": {a.ApiToken},
"sync_status": {status},
"min_videos": {strconv.Itoa(1)},
@@ -63,10 +63,13 @@ func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeC
"sync_server": {a.HostName},
"channel_id": {cp.YoutubeChannelID},
})
if err != nil {
return nil, errors.Err(err)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
var response apiJobsResponse
err := json.Unmarshal(body, &response)
err = json.Unmarshal(body, &response)
if err != nil {
return nil, err
}
+54 -50
View File
@@ -4,7 +4,6 @@ import (
"fmt"
"io/ioutil"
"math"
"net"
"os"
"os/exec"
"path/filepath"
@@ -16,15 +15,16 @@ import (
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/jsonrpc"
"github.com/lbryio/lbry.go/extras/stop"
"github.com/lbryio/lbry.go/extras/util"
"github.com/lbryio/ytsync/ipManager"
"github.com/lbryio/ytsync/namer"
"github.com/lbryio/ytsync/sdk"
"github.com/lbryio/ytsync/tagsManager"
"github.com/lbryio/ytsync/thumbs"
"github.com/ChannelMeter/iso8601duration"
"github.com/asaskevich/govalidator"
"github.com/aws/aws-sdk-go/aws"
"github.com/shopspring/decimal"
log "github.com/sirupsen/logrus"
@@ -49,10 +49,9 @@ type YoutubeVideo struct {
lbryChannelID string
mocked bool
walletLock *sync.RWMutex
stopGroup *stop.Group
}
const reflectorURL = "http://blobs.lbry.io/"
var youtubeCategories = map[string]string{
"1": "film & animation",
"2": "autos & vehicles",
@@ -88,7 +87,7 @@ var youtubeCategories = map[string]string{
"44": "trailers",
}
func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64, awsConfig aws.Config) *YoutubeVideo {
func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group) *YoutubeVideo {
publishedAt, _ := time.Parse(time.RFC3339Nano, videoData.Snippet.PublishedAt) // ignore parse errors
return &YoutubeVideo{
id: videoData.Id,
@@ -101,9 +100,10 @@ func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPositio
awsConfig: awsConfig,
mocked: false,
youtubeChannelID: videoData.Snippet.ChannelId,
stopGroup: stopGroup,
}
}
func NewMockedVideo(directory string, videoID string, youtubeChannelID string, awsConfig aws.Config) *YoutubeVideo {
func NewMockedVideo(directory string, videoID string, youtubeChannelID string, awsConfig aws.Config, stopGroup *stop.Group) *YoutubeVideo {
return &YoutubeVideo{
id: videoID,
playlistPosition: 0,
@@ -111,6 +111,7 @@ func NewMockedVideo(directory string, videoID string, youtubeChannelID string, a
awsConfig: awsConfig,
mocked: true,
youtubeChannelID: youtubeChannelID,
stopGroup: stopGroup,
}
}
@@ -174,38 +175,7 @@ func (v *YoutubeVideo) getAbbrevDescription() string {
return strings.Join(strings.Split(description, "\n")[:maxLines], "\n") + "\n..." + additionalDescription
}
var ipPool []string
var IPIndex int
var ipMutex sync.Mutex
func getNextIP() (string, error) {
ipMutex.Lock()
defer ipMutex.Unlock()
if len(ipPool) < 1 {
IPIndex = 0
addrs, err := net.InterfaceAddrs()
if err != nil {
return "", errors.Err(err)
}
for _, address := range addrs {
if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) {
ipPool = append(ipPool, ipnet.IP.String())
}
}
}
}
nextIP := ipPool[IPIndex]
if IPIndex == len(ipPool)-1 {
IPIndex = 0
} else {
IPIndex++
}
return nextIP, nil
}
func (v *YoutubeVideo) download(youtubeAntiThrottle bool) error {
func (v *YoutubeVideo) download(useIPv6 bool) error {
videoPath := v.getFullPath()
err := os.Mkdir(v.videoDir(), 0750)
@@ -221,22 +191,39 @@ func (v *YoutubeVideo) download(youtubeAntiThrottle bool) error {
return nil
}
useIPv4Env := os.Getenv("USE_IPV4")
ytdlArgs := []string{
"--no-progress",
"--max-filesize",
fmt.Sprintf("%dM", v.maxVideoSize),
"--match-filter",
fmt.Sprintf("duration <= %d", int(math.Round(v.maxVideoLength*3600))),
"-fbestvideo[ext=mp4][height<=1080]+bestaudio[ext!=webm]",
"-o" + strings.TrimRight(v.getFullPath(), ".mp4"),
"-o" + strings.TrimSuffix(v.getFullPath(), ".mp4"),
"--merge-output-format",
"mp4",
}
useIPv4 := useIPv4Env != ""
if !useIPv4 || youtubeAntiThrottle {
sourceAddress, err := getNextIP()
if err != nil {
sourceAddress, err := ipManager.GetNextIP(useIPv6)
if err != nil {
if sourceAddress == "throttled" {
for {
select {
case <-v.stopGroup.Ch():
return errors.Err("interrupted by user")
default:
}
time.Sleep(ipManager.IPCooldownPeriod)
sourceAddress, err = ipManager.GetNextIP(useIPv6)
if err == nil {
break
}
}
} else {
return errors.Err(err)
}
}
defer ipManager.ReleaseIP(sourceAddress)
if useIPv6 {
log.Infof("using IPv6: %s", sourceAddress)
ytdlArgs = append(ytdlArgs,
"-6",
@@ -244,8 +231,12 @@ func (v *YoutubeVideo) download(youtubeAntiThrottle bool) error {
sourceAddress,
)
} else {
log.Infoln("using IPv4")
ytdlArgs = append(ytdlArgs, "-4")
log.Infof("using IPv4: %s", sourceAddress)
ytdlArgs = append(ytdlArgs,
"-4",
"--source-address",
sourceAddress,
)
}
ytdlArgs = append(ytdlArgs, "https://www.youtube.com/watch?v="+v.ID())
cmd := exec.Command("youtube-dl", ytdlArgs...)
@@ -270,13 +261,23 @@ func (v *YoutubeVideo) download(youtubeAntiThrottle bool) error {
if err = cmd.Wait(); err != nil {
if strings.Contains(err.Error(), "exit status 1") {
if strings.Contains(string(errorLog), "HTTP Error 429") {
ipManager.SetIpThrottled(sourceAddress, v.stopGroup)
}
return errors.Err(string(errorLog))
}
return errors.Err(err)
}
log.Debugln(string(outLog))
if strings.Contains(string(outLog), "does not pass filter duration") {
_ = v.delete()
return errors.Err("video is too long to process")
}
if strings.Contains(string(outLog), "File is larger than max-filesize") {
_ = v.delete()
return errors.Err("the video is too big to sync, skipping for now")
}
if string(errorLog) != "" {
log.Printf("Command finished with error: %v", errors.Err(string(errorLog)))
_ = v.delete()
@@ -402,11 +403,14 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV
return v.downloadAndPublish(daemon, params)
}
var isThrottled bool
func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
err := v.download(false)
err := v.download(isThrottled)
if err != nil {
if strings.Contains(err.Error(), "HTTP Error 429") {
err = v.download(true)
if strings.Contains(err.Error(), "HTTP Error 429") && !isThrottled {
isThrottled = true
err = v.download(isThrottled)
if err != nil {
return nil, errors.Prefix("download error", err)
}