Compare commits

...
4 Commits
Author SHA1 Message Date
Niko Storni 54d04dcf2c add IP debugging
improve process halting
rename improper language code for hebrew
2019-12-18 18:22:15 +01:00
Niko Storni 65c4f99c1c make sure abandons spend from default account only 2019-12-16 04:53:15 +01:00
Niko Storni 68c0fd9ed7 add new throttling detection 2019-12-16 00:29:56 +01:00
Niko Storni 2bde06e4b9 fixes to throttling 2019-12-14 14:58:04 +01:00
6 changed files with 178 additions and 37 deletions

No files matched your search

+111
View File
@@ -0,0 +1,111 @@
From 30380338ba9af01696c94b61f0597131638eaec1 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Mon, 16 Dec 2019 00:13:36 +0100
Subject: [PATCH] lbry-patch
---
youtube_dl/extractor/youtube.py | 45 +++++++++++++++++++++++++--------
1 file changed, 35 insertions(+), 10 deletions(-)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index b913d07a6..cd66a5b01 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -10,6 +10,7 @@ import random
import re
import time
import traceback
+import subprocess
from .common import InfoExtractor, SearchInfoExtractor
from ..jsinterp import JSInterpreter
@@ -536,6 +537,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
_GEO_BYPASS = False
+ _WGET_429_RATE_LIMIT = 8191
+ _WGET_BINARY = "wget"
+
IE_NAME = 'youtube'
_TESTS = [
{
@@ -1254,6 +1258,17 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
""" Return a string representation of a signature """
return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
+ def _rate_limit_download(self, url, video_id, note=None):
+ if note is None:
+ self.report_download_webpage(video_id)
+ elif note is not False:
+ if video_id is None:
+ self.to_screen('%s' % (note,))
+ else:
+ self.to_screen('%s: %s' % (video_id, note))
+ source_address = self._downloader.params.get('source_address')
+ return subprocess.run([self._WGET_BINARY, '-q', '--limit-rate', str(self._WGET_429_RATE_LIMIT), '--bind-address', source_address, '-O', '-', url], check=True, stdout=subprocess.PIPE).stdout.decode(encoding='UTF-8')
+
def _extract_signature_function(self, video_id, player_url, example_sig):
id_m = re.match(
r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player(?:-new)?|(?:/[a-z]{2,3}_[A-Z]{2})?/base)?\.(?P<ext>[a-z]+)$',
@@ -1678,7 +1693,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
# Get video webpage
url = proto + '://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1&bpctr=9999999999' % video_id
- video_webpage = self._download_webpage(url, video_id)
+ video_webpage = self._rate_limit_download(url, video_id)
# Attempt to extract SWF player URL
mobj = re.search(r'swfConfig.*?"(https?:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
@@ -1736,10 +1751,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
r'"sts"\s*:\s*(\d+)', embed_webpage, 'sts', default=''),
})
video_info_url = proto + '://www.youtube.com/get_video_info?' + data
- video_info_webpage = self._download_webpage(
+ video_info_webpage = self._rate_limit_download(
video_info_url, video_id,
- note='Refetching age-gated info webpage',
- errnote='unable to download video info webpage')
+ note='Refetching age-gated info webpage')
video_info = compat_parse_qs(video_info_webpage)
pl_response = video_info.get('player_response', [None])[0]
player_response = extract_player_response(pl_response, video_id)
@@ -1777,7 +1791,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
# The general idea is to take a union of itags of both DASH manifests (for example
# video with such 'manifest behavior' see https://github.com/ytdl-org/youtube-dl/issues/6093)
self.report_video_info_webpage_download(video_id)
- for el in ('embedded', 'detailpage', 'vevo', ''):
+ for el in ('', 'embedded', 'detailpage', 'vevo'):
query = {
'video_id': video_id,
'ps': 'default',
@@ -1789,11 +1803,22 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
query['el'] = el
if sts:
query['sts'] = sts
- video_info_webpage = self._download_webpage(
- '%s://www.youtube.com/get_video_info' % proto,
- video_id, note=False,
- errnote='unable to download video info webpage',
- fatal=False, query=query)
+
+ if el == '':
+ base_url = 'https://youtube.com/get_video_info?video_id={}'.format(video_id)
+ else:
+ base_url = 'https://youtube.com/get_video_info'
+
+ for q in query:
+ if q is None or q is "":
+ continue
+ if query[q] is None or query[q] is "":
+ continue
+
+ base_url = base_url + "?{}={}".format(q, query[q])
+
+ video_info_webpage = self._rate_limit_download(base_url, video_id)
+
if not video_info_webpage:
continue
get_video_info = compat_parse_qs(video_info_webpage)
--
2.17.1
+42 -17
View File
@@ -13,26 +13,28 @@ import (
log "github.com/sirupsen/logrus"
)
const IPCooldownPeriod = 35 * time.Second
const IPCooldownPeriod = 20 * time.Second
const unbanTimeout = 3 * time.Hour
var stopper = stop.New()
type IPPool struct {
ips []throttledIP
lock *sync.Mutex
ips []throttledIP
lock *sync.RWMutex
stopGrp *stop.Group
}
type throttledIP struct {
IP string
LastUse time.Time
Throttled bool
InUse bool
IP string
UsedForVideo string
LastUse time.Time
Throttled bool
InUse bool
}
var ipPoolInstance *IPPool
func GetIPPool() (*IPPool, error) {
func GetIPPool(stopGrp *stop.Group) (*IPPool, error) {
if ipPoolInstance != nil {
return ipPoolInstance, nil
}
@@ -59,9 +61,25 @@ func GetIPPool() (*IPPool, error) {
}
}
ipPoolInstance = &IPPool{
ips: pool,
lock: &sync.Mutex{},
ips: pool,
lock: &sync.RWMutex{},
stopGrp: stopGrp,
}
ticker := time.NewTicker(10 * time.Second)
go func() {
for {
select {
case <-stopGrp.Ch():
return
case <-ticker.C:
ipPoolInstance.lock.RLock()
for _, ip := range ipPoolInstance.ips {
log.Debugf("IP: %s\tInUse: %t\tVideoID: %s\tThrottled: %t\tLastUse: %.1f", ip.IP, ip.InUse, ip.UsedForVideo, ip.Throttled, time.Since(ip.LastUse).Seconds())
}
ipPoolInstance.lock.RUnlock()
}
}
}()
return ipPoolInstance, nil
}
@@ -102,7 +120,7 @@ func (i *IPPool) ReleaseIP(ip string) {
}
}
func (i *IPPool) SetThrottled(ip string, stopGrp *stop.Group) {
func (i *IPPool) SetThrottled(ip string) {
i.lock.Lock()
defer i.lock.Unlock()
var tIP *throttledIP
@@ -129,7 +147,7 @@ func (i *IPPool) SetThrottled(ip string, stopGrp *stop.Group) {
tIP.Throttled = false
i.lock.Unlock()
util.SendInfoToSlack("%s set back to not throttled", ip)
case <-stopGrp.Ch():
case <-i.stopGrp.Ch():
unbanTimer.Stop()
}
}(tIP)
@@ -138,8 +156,9 @@ func (i *IPPool) SetThrottled(ip string, stopGrp *stop.Group) {
var ErrAllInUse = errors.Base("all IPs are in use, try again")
var ErrAllThrottled = errors.Base("all IPs are throttled")
var ErrResourceLock = errors.Base("error getting next ip, did you forget to lock on the resource?")
var ErrInterruptedByUser = errors.Base("interrupted by user")
func (i *IPPool) nextIP() (*throttledIP, error) {
func (i *IPPool) nextIP(forVideo string) (*throttledIP, error) {
i.lock.Lock()
defer i.lock.Unlock()
@@ -165,18 +184,24 @@ func (i *IPPool) nextIP() (*throttledIP, error) {
return nil, errors.Err(ErrResourceLock)
}
nextIP.InUse = true
nextIP.UsedForVideo = forVideo
return nextIP, nil
}
return nil, errors.Err(ErrAllThrottled)
}
func (i *IPPool) GetIP() (string, error) {
func (i *IPPool) GetIP(forVideo string) (string, error) {
for {
ip, err := i.nextIP()
ip, err := i.nextIP(forVideo)
if err != nil {
if errors.Is(err, ErrAllInUse) {
time.Sleep(5 * time.Second)
continue
select {
case <-i.stopGrp.Ch():
return "", errors.Err(ErrInterruptedByUser)
default:
time.Sleep(5 * time.Second)
continue
}
} else if errors.Is(err, ErrAllThrottled) {
return "throttled", err
}
+3
View File
@@ -399,6 +399,9 @@ func (s *Sync) ensureChannelOwnership() error {
var languages []string = nil
if channelInfo.DefaultLanguage != "" {
if channelInfo.DefaultLanguage == "iw" {
channelInfo.DefaultLanguage = "he"
}
languages = []string{channelInfo.DefaultLanguage}
}
var locations []jsonrpc.Location = nil
+2 -2
View File
@@ -96,11 +96,11 @@ func abandonSupports(s *Sync) (float64, error) {
if !more {
return
} else {
summary, err := s.daemon.SupportAbandon(&claimID, nil, nil, nil, nil)
summary, err := s.daemon.SupportAbandon(&claimID, nil, nil, nil, &defaultAccount)
if err != nil {
if strings.Contains(err.Error(), "Client.Timeout exceeded while awaiting headers") {
log.Errorf("Support abandon for %s timed out, retrying...", claimID)
summary, err = s.daemon.SupportAbandon(&claimID, nil, nil, nil, nil)
summary, err = s.daemon.SupportAbandon(&claimID, nil, nil, nil, &defaultAccount)
if err != nil {
//TODO GUESS HOW MUCH LBC WAS RELEASED THAT WE DON'T KNOW ABOUT, because screw you SDK
abandonRspChan <- abandonResponse{
+3 -1
View File
@@ -872,6 +872,8 @@ func (s *Sync) startWorker(workerNum int) {
"giving up after 0 fragment retries",
"Sorry about that",
"This video is not available",
"requested format not available",
"interrupted by user",
}
if util.SubstringInSlice(err.Error(), errorsNoRetry) {
log.Println("This error should not be retried at all")
@@ -974,7 +976,7 @@ func (s *Sync) enqueueYoutubeVideos() error {
}
var videos []video
ipPool, err := ip_manager.GetIPPool()
ipPool, err := ip_manager.GetIPPool(s.grp)
if err != nil {
return err
}
+17 -17
View File
@@ -178,7 +178,7 @@ func (v *YoutubeVideo) getAbbrevDescription() string {
return description + "\n..." + additionalDescription
}
func (v *YoutubeVideo) download(useIPv6 bool) error {
func (v *YoutubeVideo) download() error {
videoPath := v.getFullPath()
err := os.Mkdir(v.videoDir(), 0777)
@@ -224,7 +224,7 @@ func (v *YoutubeVideo) download(useIPv6 bool) error {
)
}
sourceAddress, err := v.pool.GetIP()
sourceAddress, err := v.pool.GetIP(v.id)
if err != nil {
if sourceAddress == "throttled" {
for {
@@ -235,7 +235,7 @@ func (v *YoutubeVideo) download(useIPv6 bool) error {
}
time.Sleep(ip_manager.IPCooldownPeriod)
sourceAddress, err = v.pool.GetIP()
sourceAddress, err = v.pool.GetIP(v.id)
if err == nil {
break
} else if !errors.Is(err, ip_manager.ErrAllThrottled) {
@@ -277,11 +277,11 @@ runcmd:
if err = cmd.Wait(); err != nil {
if strings.Contains(err.Error(), "exit status 1") {
if strings.Contains(string(errorLog), "HTTP Error 429") {
v.pool.SetThrottled(sourceAddress, v.stopGroup)
if strings.Contains(string(errorLog), "HTTP Error 429") || strings.Contains(string(errorLog), "returned non-zero exit status 8") {
v.pool.SetThrottled(sourceAddress)
} else if strings.Contains(string(errorLog), "giving up after 0 fragment retries") && qualityIndex < len(qualities)-1 {
qualityIndex++
goto runcmd
goto runcmd //this bypasses the yt throttling IP redistribution... TODO: don't
}
return errors.Err(string(errorLog))
}
@@ -426,21 +426,18 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV
return v.downloadAndPublish(daemon, params)
}
var isThrottled bool
func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
err := v.download(isThrottled)
if err != nil {
if strings.Contains(err.Error(), "HTTP Error 429") && !isThrottled {
isThrottled = true
err = v.download(isThrottled)
if err != nil {
return nil, errors.Prefix("download error", err)
}
} else {
var err error
for {
err = v.download()
if err != nil && strings.Contains(err.Error(), "HTTP Error 429") {
continue
} else if err != nil {
return nil, errors.Prefix("download error", err)
}
break
}
log.Debugln("Downloaded " + v.id)
err = v.triggerThumbnailSave()
@@ -462,6 +459,9 @@ func (v *YoutubeVideo) getMetadata() (languages []string, locations []jsonrpc.Lo
tags = nil
if !v.mocked {
if v.youtubeInfo.Snippet.DefaultLanguage != "" {
if v.youtubeInfo.Snippet.DefaultLanguage == "iw" {
v.youtubeInfo.Snippet.DefaultLanguage = "he"
}
languages = []string{v.youtubeInfo.Snippet.DefaultLanguage}
}