Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1b55033bb8 | ||
|
|
1331e5769f | ||
|
|
43f745ae20 | ||
|
|
8c2a8262e6 | ||
|
|
6f486717da | ||
|
|
a3fcd67611 | ||
|
|
ae1ffb60c5 | ||
|
|
fa24d83ae9 | ||
|
|
2f11d99a97 |
No files matched your search
@@ -0,0 +1,119 @@
|
||||
package ipManager
|
||||
|
||||
import (
|
||||
"github.com/asaskevich/govalidator"
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
"github.com/lbryio/lbry.go/extras/stop"
|
||||
|
||||
"net"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const IPCooldownPeriod = 20 * time.Second
|
||||
const unbanTimeout = 3 * time.Hour
|
||||
|
||||
var ipv6Pool []string
|
||||
var ipv4Pool []string
|
||||
var throttledIPs map[string]bool
|
||||
var ipInUse map[string]bool
|
||||
var ipLastUsed map[string]time.Time
|
||||
var ipMutex sync.Mutex
|
||||
var stopper = stop.New()
|
||||
|
||||
func GetNextIP(ipv6 bool) (string, error) {
|
||||
ipMutex.Lock()
|
||||
defer ipMutex.Unlock()
|
||||
if len(ipv4Pool) < 1 || len(ipv6Pool) < 1 {
|
||||
throttledIPs = make(map[string]bool)
|
||||
ipInUse = make(map[string]bool)
|
||||
ipLastUsed = make(map[string]time.Time)
|
||||
addrs, err := net.InterfaceAddrs()
|
||||
if err != nil {
|
||||
return "", errors.Err(err)
|
||||
}
|
||||
|
||||
for _, address := range addrs {
|
||||
if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
|
||||
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) {
|
||||
ipv6Pool = append(ipv6Pool, ipnet.IP.String())
|
||||
ipLastUsed[ipnet.IP.String()] = time.Now().Add(-IPCooldownPeriod)
|
||||
} else if ipnet.IP.To4() != nil && govalidator.IsIPv4(ipnet.IP.String()) {
|
||||
ipv4Pool = append(ipv4Pool, ipnet.IP.String())
|
||||
ipLastUsed[ipnet.IP.String()] = time.Now().Add(-IPCooldownPeriod)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
nextIP := ""
|
||||
if ipv6 {
|
||||
nextIP = getLeastUsedIP(ipv6Pool)
|
||||
} else {
|
||||
nextIP = getLeastUsedIP(ipv4Pool)
|
||||
}
|
||||
if nextIP == "" {
|
||||
return "throttled", errors.Err("all IPs are throttled")
|
||||
}
|
||||
lastUse := ipLastUsed[nextIP]
|
||||
if time.Since(lastUse) < IPCooldownPeriod {
|
||||
log.Debugf("The IP %s is too hot, waiting for %.1f seconds before continuing", nextIP, (IPCooldownPeriod - time.Since(lastUse)).Seconds())
|
||||
time.Sleep(IPCooldownPeriod - time.Since(lastUse))
|
||||
}
|
||||
|
||||
ipInUse[nextIP] = true
|
||||
return nextIP, nil
|
||||
}
|
||||
|
||||
func ReleaseIP(ip string) {
|
||||
ipMutex.Lock()
|
||||
defer ipMutex.Unlock()
|
||||
ipLastUsed[ip] = time.Now()
|
||||
ipInUse[ip] = false
|
||||
}
|
||||
|
||||
func getLeastUsedIP(ipPool []string) string {
|
||||
nextIP := ""
|
||||
veryLastUse := time.Now()
|
||||
for _, ip := range ipPool {
|
||||
isThrottled := throttledIPs[ip]
|
||||
if isThrottled {
|
||||
continue
|
||||
}
|
||||
inUse := ipInUse[ip]
|
||||
if inUse {
|
||||
continue
|
||||
}
|
||||
lastUse := ipLastUsed[ip]
|
||||
if lastUse.Before(veryLastUse) {
|
||||
nextIP = ip
|
||||
veryLastUse = lastUse
|
||||
}
|
||||
}
|
||||
return nextIP
|
||||
}
|
||||
|
||||
func SetIpThrottled(ip string, stopGrp *stop.Group) {
|
||||
ipMutex.Lock()
|
||||
defer ipMutex.Unlock()
|
||||
isThrottled := throttledIPs[ip]
|
||||
if isThrottled {
|
||||
return
|
||||
}
|
||||
throttledIPs[ip] = true
|
||||
log.Printf("%s set to throttled", ip)
|
||||
|
||||
stopper.Add(1)
|
||||
go func() {
|
||||
defer stopper.Done()
|
||||
unbanTimer := time.NewTimer(unbanTimeout)
|
||||
select {
|
||||
case <-unbanTimer.C:
|
||||
throttledIPs[ip] = false
|
||||
log.Printf("%s set back to not throttled", ip)
|
||||
case <-stopGrp.Ch():
|
||||
unbanTimer.Stop()
|
||||
}
|
||||
}()
|
||||
}
|
||||
+8
-3
@@ -649,6 +649,9 @@ func (s *Sync) startWorker(workerNum int) {
|
||||
if err != nil {
|
||||
logMsg := fmt.Sprintf("error processing video %s: %s", v.ID(), err.Error())
|
||||
log.Errorln(logMsg)
|
||||
if strings.Contains(strings.ToLower(err.Error()), "interrupted by user") {
|
||||
return
|
||||
}
|
||||
fatalErrors := []string{
|
||||
":5279: read: connection reset by peer",
|
||||
"no space left on device",
|
||||
@@ -657,13 +660,14 @@ func (s *Sync) startWorker(workerNum int) {
|
||||
"cannot concatenate 'str' and 'NoneType' objects",
|
||||
"more than 90% of the space has been used.",
|
||||
"HTTP Error 429",
|
||||
"Couldn't find private key for id",
|
||||
}
|
||||
if util.SubstringInSlice(err.Error(), fatalErrors) || s.StopOnError {
|
||||
s.grp.Stop()
|
||||
} else if s.MaxTries > 1 {
|
||||
errorsNoRetry := []string{
|
||||
"non 200 status code received",
|
||||
" reason: 'This video contains content from",
|
||||
"This video contains content from",
|
||||
"dont know which claim to update",
|
||||
"uploader has not made this video available in your country",
|
||||
"download error: AccessDenied: Access Denied",
|
||||
@@ -810,7 +814,7 @@ func (s *Sync) enqueueYoutubeVideos() error {
|
||||
return errors.Prefix("error getting videos info", err)
|
||||
}
|
||||
for _, item := range videosListResponse.Items {
|
||||
videos = append(videos, sources.NewYoutubeVideo(s.videoDirectory, item, playlistMap[item.Id].Position, s.Manager.GetS3AWSConfig()))
|
||||
videos = append(videos, sources.NewYoutubeVideo(s.videoDirectory, item, playlistMap[item.Id].Position, s.Manager.GetS3AWSConfig(), s.grp))
|
||||
}
|
||||
|
||||
log.Infof("Got info for %d videos from youtube API", len(videos))
|
||||
@@ -826,7 +830,7 @@ func (s *Sync) enqueueYoutubeVideos() error {
|
||||
}
|
||||
_, ok := playlistMap[k]
|
||||
if !ok {
|
||||
videos = append(videos, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig()))
|
||||
videos = append(videos, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig(), s.grp))
|
||||
}
|
||||
|
||||
}
|
||||
@@ -881,6 +885,7 @@ func (s *Sync) processVideo(v video) (err error) {
|
||||
"Unable to extract signature tokens",
|
||||
"the video is too big to sync, skipping for now",
|
||||
"video is too long to process",
|
||||
"This video contains content from",
|
||||
"no compatible format available for this video",
|
||||
"Watch this video on YouTube.",
|
||||
"have blocked it on copyright grounds",
|
||||
|
||||
+5
-2
@@ -54,7 +54,7 @@ func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeC
|
||||
Data []YoutubeChannel `json:"data"`
|
||||
}
|
||||
endpoint := a.ApiURL + "/yt/jobs"
|
||||
res, _ := http.PostForm(endpoint, url.Values{
|
||||
res, err := http.PostForm(endpoint, url.Values{
|
||||
"auth_token": {a.ApiToken},
|
||||
"sync_status": {status},
|
||||
"min_videos": {strconv.Itoa(1)},
|
||||
@@ -63,10 +63,13 @@ func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeC
|
||||
"sync_server": {a.HostName},
|
||||
"channel_id": {cp.YoutubeChannelID},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
var response apiJobsResponse
|
||||
err := json.Unmarshal(body, &response)
|
||||
err = json.Unmarshal(body, &response)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
+46
-44
@@ -4,7 +4,6 @@ import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
@@ -16,15 +15,16 @@ import (
|
||||
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
"github.com/lbryio/lbry.go/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/extras/stop"
|
||||
"github.com/lbryio/lbry.go/extras/util"
|
||||
|
||||
"github.com/lbryio/ytsync/ipManager"
|
||||
"github.com/lbryio/ytsync/namer"
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
"github.com/lbryio/ytsync/tagsManager"
|
||||
"github.com/lbryio/ytsync/thumbs"
|
||||
|
||||
"github.com/ChannelMeter/iso8601duration"
|
||||
"github.com/asaskevich/govalidator"
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/shopspring/decimal"
|
||||
log "github.com/sirupsen/logrus"
|
||||
@@ -49,10 +49,9 @@ type YoutubeVideo struct {
|
||||
lbryChannelID string
|
||||
mocked bool
|
||||
walletLock *sync.RWMutex
|
||||
stopGroup *stop.Group
|
||||
}
|
||||
|
||||
const reflectorURL = "http://blobs.lbry.io/"
|
||||
|
||||
var youtubeCategories = map[string]string{
|
||||
"1": "film & animation",
|
||||
"2": "autos & vehicles",
|
||||
@@ -88,7 +87,7 @@ var youtubeCategories = map[string]string{
|
||||
"44": "trailers",
|
||||
}
|
||||
|
||||
func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64, awsConfig aws.Config) *YoutubeVideo {
|
||||
func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group) *YoutubeVideo {
|
||||
publishedAt, _ := time.Parse(time.RFC3339Nano, videoData.Snippet.PublishedAt) // ignore parse errors
|
||||
return &YoutubeVideo{
|
||||
id: videoData.Id,
|
||||
@@ -101,9 +100,10 @@ func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPositio
|
||||
awsConfig: awsConfig,
|
||||
mocked: false,
|
||||
youtubeChannelID: videoData.Snippet.ChannelId,
|
||||
stopGroup: stopGroup,
|
||||
}
|
||||
}
|
||||
func NewMockedVideo(directory string, videoID string, youtubeChannelID string, awsConfig aws.Config) *YoutubeVideo {
|
||||
func NewMockedVideo(directory string, videoID string, youtubeChannelID string, awsConfig aws.Config, stopGroup *stop.Group) *YoutubeVideo {
|
||||
return &YoutubeVideo{
|
||||
id: videoID,
|
||||
playlistPosition: 0,
|
||||
@@ -111,6 +111,7 @@ func NewMockedVideo(directory string, videoID string, youtubeChannelID string, a
|
||||
awsConfig: awsConfig,
|
||||
mocked: true,
|
||||
youtubeChannelID: youtubeChannelID,
|
||||
stopGroup: stopGroup,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -174,37 +175,6 @@ func (v *YoutubeVideo) getAbbrevDescription() string {
|
||||
return strings.Join(strings.Split(description, "\n")[:maxLines], "\n") + "\n..." + additionalDescription
|
||||
}
|
||||
|
||||
var ipPool []string
|
||||
var IPIndex int
|
||||
var ipMutex sync.Mutex
|
||||
|
||||
func getNextIP() (string, error) {
|
||||
ipMutex.Lock()
|
||||
defer ipMutex.Unlock()
|
||||
if len(ipPool) < 1 {
|
||||
IPIndex = 0
|
||||
addrs, err := net.InterfaceAddrs()
|
||||
if err != nil {
|
||||
return "", errors.Err(err)
|
||||
}
|
||||
|
||||
for _, address := range addrs {
|
||||
if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
|
||||
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) {
|
||||
ipPool = append(ipPool, ipnet.IP.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
nextIP := ipPool[IPIndex]
|
||||
if IPIndex == len(ipPool)-1 {
|
||||
IPIndex = 0
|
||||
} else {
|
||||
IPIndex++
|
||||
}
|
||||
return nextIP, nil
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) download(useIPv6 bool) error {
|
||||
videoPath := v.getFullPath()
|
||||
|
||||
@@ -225,16 +195,34 @@ func (v *YoutubeVideo) download(useIPv6 bool) error {
|
||||
"--no-progress",
|
||||
"--max-filesize",
|
||||
fmt.Sprintf("%dM", v.maxVideoSize),
|
||||
"--match-filter",
|
||||
fmt.Sprintf("duration <= %d", int(math.Round(v.maxVideoLength*3600))),
|
||||
"-fbestvideo[ext=mp4][height<=1080]+bestaudio[ext!=webm]",
|
||||
"-o" + strings.TrimRight(v.getFullPath(), ".mp4"),
|
||||
"-o" + strings.TrimSuffix(v.getFullPath(), ".mp4"),
|
||||
"--merge-output-format",
|
||||
"mp4",
|
||||
}
|
||||
if useIPv6 {
|
||||
sourceAddress, err := getNextIP()
|
||||
if err != nil {
|
||||
sourceAddress, err := ipManager.GetNextIP(useIPv6)
|
||||
if err != nil {
|
||||
if sourceAddress == "throttled" {
|
||||
for {
|
||||
select {
|
||||
case <-v.stopGroup.Ch():
|
||||
return errors.Err("interrupted by user")
|
||||
default:
|
||||
time.Sleep(20 * time.Second)
|
||||
sourceAddress, err = ipManager.GetNextIP(useIPv6)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return errors.Err(err)
|
||||
}
|
||||
}
|
||||
defer ipManager.ReleaseIP(sourceAddress)
|
||||
if useIPv6 {
|
||||
log.Infof("using IPv6: %s", sourceAddress)
|
||||
ytdlArgs = append(ytdlArgs,
|
||||
"-6",
|
||||
@@ -242,8 +230,12 @@ func (v *YoutubeVideo) download(useIPv6 bool) error {
|
||||
sourceAddress,
|
||||
)
|
||||
} else {
|
||||
log.Infoln("using IPv4")
|
||||
ytdlArgs = append(ytdlArgs, "-4")
|
||||
log.Infof("using IPv4: %s", sourceAddress)
|
||||
ytdlArgs = append(ytdlArgs,
|
||||
"-4",
|
||||
"--source-address",
|
||||
sourceAddress,
|
||||
)
|
||||
}
|
||||
ytdlArgs = append(ytdlArgs, "https://www.youtube.com/watch?v="+v.ID())
|
||||
cmd := exec.Command("youtube-dl", ytdlArgs...)
|
||||
@@ -268,13 +260,23 @@ func (v *YoutubeVideo) download(useIPv6 bool) error {
|
||||
|
||||
if err = cmd.Wait(); err != nil {
|
||||
if strings.Contains(err.Error(), "exit status 1") {
|
||||
if strings.Contains(string(errorLog), "HTTP Error 429") {
|
||||
ipManager.SetIpThrottled(sourceAddress, v.stopGroup)
|
||||
}
|
||||
return errors.Err(string(errorLog))
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
log.Debugln(string(outLog))
|
||||
|
||||
if strings.Contains(string(outLog), "does not pass filter duration") {
|
||||
_ = v.delete()
|
||||
return errors.Err("video is too long to process")
|
||||
}
|
||||
if strings.Contains(string(outLog), "File is larger than max-filesize") {
|
||||
_ = v.delete()
|
||||
return errors.Err("the video is too big to sync, skipping for now")
|
||||
}
|
||||
if string(errorLog) != "" {
|
||||
log.Printf("Command finished with error: %v", errors.Err(string(errorLog)))
|
||||
_ = v.delete()
|
||||
|
||||
Reference in new issue
Block a user