Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
68c0fd9ed7 | ||
|
|
2bde06e4b9 | ||
|
|
5df6db6e96 | ||
|
|
62dfdc1adb | ||
|
|
fe55304184 | ||
|
|
bb76500f04 |
No files matched your search
@@ -0,0 +1,111 @@
|
||||
From 30380338ba9af01696c94b61f0597131638eaec1 Mon Sep 17 00:00:00 2001
|
||||
From: Niko Storni <niko@lbry.io>
|
||||
Date: Mon, 16 Dec 2019 00:13:36 +0100
|
||||
Subject: [PATCH] lbry-patch
|
||||
|
||||
---
|
||||
youtube_dl/extractor/youtube.py | 45 +++++++++++++++++++++++++--------
|
||||
1 file changed, 35 insertions(+), 10 deletions(-)
|
||||
|
||||
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
|
||||
index b913d07a6..cd66a5b01 100644
|
||||
--- a/youtube_dl/extractor/youtube.py
|
||||
+++ b/youtube_dl/extractor/youtube.py
|
||||
@@ -10,6 +10,7 @@ import random
|
||||
import re
|
||||
import time
|
||||
import traceback
|
||||
+import subprocess
|
||||
|
||||
from .common import InfoExtractor, SearchInfoExtractor
|
||||
from ..jsinterp import JSInterpreter
|
||||
@@ -536,6 +537,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
|
||||
_GEO_BYPASS = False
|
||||
|
||||
+ _WGET_429_RATE_LIMIT = 8191
|
||||
+ _WGET_BINARY = "wget"
|
||||
+
|
||||
IE_NAME = 'youtube'
|
||||
_TESTS = [
|
||||
{
|
||||
@@ -1254,6 +1258,17 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
""" Return a string representation of a signature """
|
||||
return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
|
||||
|
||||
+ def _rate_limit_download(self, url, video_id, note=None):
|
||||
+ if note is None:
|
||||
+ self.report_download_webpage(video_id)
|
||||
+ elif note is not False:
|
||||
+ if video_id is None:
|
||||
+ self.to_screen('%s' % (note,))
|
||||
+ else:
|
||||
+ self.to_screen('%s: %s' % (video_id, note))
|
||||
+ source_address = self._downloader.params.get('source_address')
|
||||
+ return subprocess.run([self._WGET_BINARY, '-q', '--limit-rate', str(self._WGET_429_RATE_LIMIT), '--bind-address', source_address, '-O', '-', url], check=True, stdout=subprocess.PIPE).stdout.decode(encoding='UTF-8')
|
||||
+
|
||||
def _extract_signature_function(self, video_id, player_url, example_sig):
|
||||
id_m = re.match(
|
||||
r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player(?:-new)?|(?:/[a-z]{2,3}_[A-Z]{2})?/base)?\.(?P<ext>[a-z]+)$',
|
||||
@@ -1678,7 +1693,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
|
||||
# Get video webpage
|
||||
url = proto + '://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1&bpctr=9999999999' % video_id
|
||||
- video_webpage = self._download_webpage(url, video_id)
|
||||
+ video_webpage = self._rate_limit_download(url, video_id)
|
||||
|
||||
# Attempt to extract SWF player URL
|
||||
mobj = re.search(r'swfConfig.*?"(https?:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
|
||||
@@ -1736,10 +1751,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
r'"sts"\s*:\s*(\d+)', embed_webpage, 'sts', default=''),
|
||||
})
|
||||
video_info_url = proto + '://www.youtube.com/get_video_info?' + data
|
||||
- video_info_webpage = self._download_webpage(
|
||||
+ video_info_webpage = self._rate_limit_download(
|
||||
video_info_url, video_id,
|
||||
- note='Refetching age-gated info webpage',
|
||||
- errnote='unable to download video info webpage')
|
||||
+ note='Refetching age-gated info webpage')
|
||||
video_info = compat_parse_qs(video_info_webpage)
|
||||
pl_response = video_info.get('player_response', [None])[0]
|
||||
player_response = extract_player_response(pl_response, video_id)
|
||||
@@ -1777,7 +1791,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
# The general idea is to take a union of itags of both DASH manifests (for example
|
||||
# video with such 'manifest behavior' see https://github.com/ytdl-org/youtube-dl/issues/6093)
|
||||
self.report_video_info_webpage_download(video_id)
|
||||
- for el in ('embedded', 'detailpage', 'vevo', ''):
|
||||
+ for el in ('', 'embedded', 'detailpage', 'vevo'):
|
||||
query = {
|
||||
'video_id': video_id,
|
||||
'ps': 'default',
|
||||
@@ -1789,11 +1803,22 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
query['el'] = el
|
||||
if sts:
|
||||
query['sts'] = sts
|
||||
- video_info_webpage = self._download_webpage(
|
||||
- '%s://www.youtube.com/get_video_info' % proto,
|
||||
- video_id, note=False,
|
||||
- errnote='unable to download video info webpage',
|
||||
- fatal=False, query=query)
|
||||
+
|
||||
+ if el == '':
|
||||
+ base_url = 'https://youtube.com/get_video_info?video_id={}'.format(video_id)
|
||||
+ else:
|
||||
+ base_url = 'https://youtube.com/get_video_info'
|
||||
+
|
||||
+ for q in query:
|
||||
+ if q is None or q is "":
|
||||
+ continue
|
||||
+ if query[q] is None or query[q] is "":
|
||||
+ continue
|
||||
+
|
||||
+ base_url = base_url + "?{}={}".format(q, query[q])
|
||||
+
|
||||
+ video_info_webpage = self._rate_limit_download(base_url, video_id)
|
||||
+
|
||||
if not video_info_webpage:
|
||||
continue
|
||||
get_video_info = compat_parse_qs(video_info_webpage)
|
||||
--
|
||||
2.17.1
|
||||
|
||||
@@ -87,7 +87,7 @@ services:
|
||||
## Internal APIs ##
|
||||
###################
|
||||
internalapis:
|
||||
image: lbry/internal-apis:master #expand_publish
|
||||
image: lbry/internal-apis:master
|
||||
restart: "no"
|
||||
ports:
|
||||
- "15400:8080"
|
||||
@@ -104,7 +104,7 @@ services:
|
||||
## Chainquery ##
|
||||
################
|
||||
chainquery:
|
||||
image: lbry/chainquery:master
|
||||
image: lbry/chainquery:v2.0.8
|
||||
restart: "no"
|
||||
ports:
|
||||
- 6300:6300
|
||||
|
||||
+2
-2
@@ -56,8 +56,8 @@ echo "successfully started..."
|
||||
./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 #Force channel intended...just in case. This channel lines up with the api container
|
||||
status=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM youtube_data WHERE id=1')
|
||||
videoStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM synced_video WHERE id=1')
|
||||
videoClaimID1=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT claim_id FROM synced_video WHERE id=1')
|
||||
videoClaimID2=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT claim_id FROM synced_video WHERE id=2')
|
||||
videoClaimID1=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT publish.claim_id FROM synced_video INNER JOIN publish ON publish.id = synced_video.publish_id WHERE synced_video.id=1')
|
||||
videoClaimID2=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT publish.claim_id FROM synced_video INNER JOIN publish ON publish.id = synced_video.publish_id WHERE synced_video.id=2')
|
||||
videoClaimAddress1=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT claim_address FROM claim WHERE id=2')
|
||||
videoClaimAddress2=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT claim_address FROM claim WHERE id=3')
|
||||
# Create Supports for published claim
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const IPCooldownPeriod = 35 * time.Second
|
||||
const IPCooldownPeriod = 20 * time.Second
|
||||
const unbanTimeout = 3 * time.Hour
|
||||
|
||||
var stopper = stop.New()
|
||||
@@ -47,13 +47,13 @@ func GetIPPool() (*IPPool, error) {
|
||||
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) && !ipv6Added {
|
||||
pool = append(pool, throttledIP{
|
||||
IP: ipnet.IP.String(),
|
||||
LastUse: time.Time{},
|
||||
LastUse: time.Now().Add(-5 * time.Minute),
|
||||
})
|
||||
ipv6Added = true
|
||||
} else if ipnet.IP.To4() != nil && govalidator.IsIPv4(ipnet.IP.String()) {
|
||||
pool = append(pool, throttledIP{
|
||||
IP: ipnet.IP.String(),
|
||||
LastUse: time.Time{},
|
||||
LastUse: time.Now().Add(-5 * time.Minute),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -144,7 +144,7 @@ func (i *IPPool) nextIP() (*throttledIP, error) {
|
||||
defer i.lock.Unlock()
|
||||
|
||||
sort.Slice(i.ips, func(j, k int) bool {
|
||||
return i.ips[j].LastUse.Before(i.ips[j].LastUse)
|
||||
return i.ips[j].LastUse.Before(i.ips[k].LastUse)
|
||||
})
|
||||
|
||||
if !AllThrottled(i.ips) {
|
||||
|
||||
@@ -9,14 +9,28 @@ func TestAll(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ip, err := pool.GetIP()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log(ip)
|
||||
pool.ReleaseIP(ip)
|
||||
ip2, err := pool.GetIP()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ip == ip2 && len(pool.ips) > 1 {
|
||||
t.Fatalf("the same IP was returned twice! %s, %s", ip, ip2)
|
||||
}
|
||||
t.Log(ip2)
|
||||
pool.ReleaseIP(ip2)
|
||||
|
||||
for range pool.ips {
|
||||
ip, err := pool.GetIP()
|
||||
_, err = pool.GetIP()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log(ip)
|
||||
}
|
||||
|
||||
next, err := pool.nextIP()
|
||||
if err != nil {
|
||||
t.Logf("%s", err.Error())
|
||||
|
||||
+1
-1
@@ -192,7 +192,7 @@ func transferVideos(s *Sync) error {
|
||||
|
||||
var stream *jsonrpc.Claim = nil
|
||||
for _, c := range streams.Items {
|
||||
if c.ClaimID != video.ClaimID {
|
||||
if c.ClaimID != video.ClaimID || (c.SigningChannel != nil && c.SigningChannel.ClaimID != s.lbryChannelID) {
|
||||
continue
|
||||
}
|
||||
stream = &c
|
||||
|
||||
+17
-6
@@ -467,7 +467,7 @@ var thumbnailHosts = []string{
|
||||
thumbs.ThumbnailEndpoint,
|
||||
}
|
||||
|
||||
func isYtsyncClaim(c jsonrpc.Claim) bool {
|
||||
func isYtsyncClaim(c jsonrpc.Claim, expectedChannelID string) bool {
|
||||
if !util.InSlice(c.Type, []string{"claim", "update"}) || c.Value.GetStream() == nil {
|
||||
return false
|
||||
}
|
||||
@@ -475,7 +475,12 @@ func isYtsyncClaim(c jsonrpc.Claim) bool {
|
||||
//most likely a claim created outside of ytsync, ignore!
|
||||
return false
|
||||
}
|
||||
|
||||
if c.SigningChannel == nil {
|
||||
return false
|
||||
}
|
||||
if c.SigningChannel.ClaimID != expectedChannelID {
|
||||
return false
|
||||
}
|
||||
for _, th := range thumbnailHosts {
|
||||
if strings.Contains(c.Value.GetThumbnail().GetUrl(), th) {
|
||||
return true
|
||||
@@ -489,7 +494,7 @@ func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) {
|
||||
abandonedClaims := false
|
||||
videoIDs := make(map[string]jsonrpc.Claim)
|
||||
for _, c := range claims {
|
||||
if !isYtsyncClaim(c) {
|
||||
if !isYtsyncClaim(c, s.lbryChannelID) {
|
||||
continue
|
||||
}
|
||||
tn := c.Value.GetThumbnail().GetUrl()
|
||||
@@ -536,7 +541,7 @@ type ytsyncClaim struct {
|
||||
func (s *Sync) mapFromClaims(claims []jsonrpc.Claim) map[string]ytsyncClaim {
|
||||
videoIDMap := make(map[string]ytsyncClaim, len(claims))
|
||||
for _, c := range claims {
|
||||
if !isYtsyncClaim(c) {
|
||||
if !isYtsyncClaim(c, s.lbryChannelID) {
|
||||
continue
|
||||
}
|
||||
tn := c.Value.GetThumbnail().GetUrl()
|
||||
@@ -675,8 +680,13 @@ func (s *Sync) getClaims(defaultOnly bool) ([]jsonrpc.Claim, error) {
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("cannot list claims", err)
|
||||
}
|
||||
|
||||
return claims.Items, nil
|
||||
items := make([]jsonrpc.Claim, 0, len(claims.Items))
|
||||
for _, c := range claims.Items {
|
||||
if c.SigningChannel != nil && c.SigningChannel.ClaimID == s.lbryChannelID {
|
||||
items = append(items, c)
|
||||
}
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (s *Sync) checkIntegrity() error {
|
||||
@@ -862,6 +872,7 @@ func (s *Sync) startWorker(workerNum int) {
|
||||
"giving up after 0 fragment retries",
|
||||
"Sorry about that",
|
||||
"This video is not available",
|
||||
"requested format not available",
|
||||
}
|
||||
if util.SubstringInSlice(err.Error(), errorsNoRetry) {
|
||||
log.Println("This error should not be retried at all")
|
||||
|
||||
+14
-15
@@ -178,7 +178,7 @@ func (v *YoutubeVideo) getAbbrevDescription() string {
|
||||
return description + "\n..." + additionalDescription
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) download(useIPv6 bool) error {
|
||||
func (v *YoutubeVideo) download() error {
|
||||
videoPath := v.getFullPath()
|
||||
|
||||
err := os.Mkdir(v.videoDir(), 0777)
|
||||
@@ -236,8 +236,10 @@ func (v *YoutubeVideo) download(useIPv6 bool) error {
|
||||
|
||||
time.Sleep(ip_manager.IPCooldownPeriod)
|
||||
sourceAddress, err = v.pool.GetIP()
|
||||
if err == nil { //TODO: This is possibly not 100% right, but it works so I'm not touching it...
|
||||
if err == nil {
|
||||
break
|
||||
} else if !errors.Is(err, ip_manager.ErrAllThrottled) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
} else {
|
||||
@@ -275,11 +277,11 @@ runcmd:
|
||||
|
||||
if err = cmd.Wait(); err != nil {
|
||||
if strings.Contains(err.Error(), "exit status 1") {
|
||||
if strings.Contains(string(errorLog), "HTTP Error 429") {
|
||||
if strings.Contains(string(errorLog), "HTTP Error 429") || strings.Contains(string(errorLog), "returned non-zero exit status 8") {
|
||||
v.pool.SetThrottled(sourceAddress, v.stopGroup)
|
||||
} else if strings.Contains(string(errorLog), "giving up after 0 fragment retries") && qualityIndex < len(qualities)-1 {
|
||||
qualityIndex++
|
||||
goto runcmd
|
||||
goto runcmd //this bypasses the yt throttling IP redistribution... TODO: don't
|
||||
}
|
||||
return errors.Err(string(errorLog))
|
||||
}
|
||||
@@ -424,21 +426,18 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV
|
||||
return v.downloadAndPublish(daemon, params)
|
||||
}
|
||||
|
||||
var isThrottled bool
|
||||
|
||||
func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
|
||||
err := v.download(isThrottled)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "HTTP Error 429") && !isThrottled {
|
||||
isThrottled = true
|
||||
err = v.download(isThrottled)
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("download error", err)
|
||||
}
|
||||
} else {
|
||||
var err error
|
||||
for {
|
||||
err = v.download()
|
||||
if err != nil && strings.Contains(err.Error(), "HTTP Error 429") {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return nil, errors.Prefix("download error", err)
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
log.Debugln("Downloaded " + v.id)
|
||||
|
||||
err = v.triggerThumbnailSave()
|
||||
|
||||
Reference in new issue
Block a user