Compare commits

...
15 Commits
Author SHA1 Message Date
Niko Storni 68c0fd9ed7 add new throttling detection 2019-12-16 00:29:56 +01:00
Niko Storni 2bde06e4b9 fixes to throttling 2019-12-14 14:58:04 +01:00
Niko Storni 5df6db6e96 fix bug with non-related publishes 2019-12-14 06:43:01 +01:00
Niko Storni 62dfdc1adb fix possible resource leak 2019-12-14 05:49:25 +01:00
Niko Storni fe55304184 fix anti-throttling
fix duplicates with multi-channels
2019-12-13 18:20:41 +01:00
Mark Beamer Jr bb76500f04 Update e2e test for publish table migration in internal-apis 2019-12-12 22:28:19 -05:00
Niko Storni 4e2efec66e disable logging for reflection 2019-12-11 13:13:30 +01:00
Mark 8d650e032c Merge pull request #49 from lbryio/merge_e2e
Merge chainquery and lbry databases
2019-12-10 20:01:26 -05:00
Mark Beamer Jr f5d97957e7 Fixed the e2e test. It wasn't checking the right claims and videos. Looks like a copy paste error. 2019-12-10 18:43:44 -05:00
Niko Storni 36f539ef5d change ip throttling management 2019-12-10 23:02:56 +01:00
Mark Beamer Jr 77461f71af Merge chainquery and lbry databases to account for claim table being required by internal-apis due to replication changes. 2019-12-10 15:45:29 -05:00
Niko Storni caca13de61 increase anti-throttle timeout 2019-12-09 18:09:31 +01:00
Niko Storni 7bdc117ba0 upgrade ytsync to latest SDK
upgrade e2e tests
2019-12-08 11:49:27 -05:00
Niko Storni fc575dac80 improve logging
fix ipv6 switching
2019-12-08 10:31:15 -05:00
Niko Storni d520ff2c68 change cache control for thumbnails 2019-11-11 15:11:34 -05:00
18 changed files with 479 additions and 238 deletions

No files matched your search

+111
View File
@@ -0,0 +1,111 @@
From 30380338ba9af01696c94b61f0597131638eaec1 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Mon, 16 Dec 2019 00:13:36 +0100
Subject: [PATCH] lbry-patch
---
youtube_dl/extractor/youtube.py | 45 +++++++++++++++++++++++++--------
1 file changed, 35 insertions(+), 10 deletions(-)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index b913d07a6..cd66a5b01 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -10,6 +10,7 @@ import random
import re
import time
import traceback
+import subprocess
from .common import InfoExtractor, SearchInfoExtractor
from ..jsinterp import JSInterpreter
@@ -536,6 +537,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
_GEO_BYPASS = False
+ _WGET_429_RATE_LIMIT = 8191
+ _WGET_BINARY = "wget"
+
IE_NAME = 'youtube'
_TESTS = [
{
@@ -1254,6 +1258,17 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
""" Return a string representation of a signature """
return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
+ def _rate_limit_download(self, url, video_id, note=None):
+ if note is None:
+ self.report_download_webpage(video_id)
+ elif note is not False:
+ if video_id is None:
+ self.to_screen('%s' % (note,))
+ else:
+ self.to_screen('%s: %s' % (video_id, note))
+ source_address = self._downloader.params.get('source_address')
+ return subprocess.run([self._WGET_BINARY, '-q', '--limit-rate', str(self._WGET_429_RATE_LIMIT), '--bind-address', source_address, '-O', '-', url], check=True, stdout=subprocess.PIPE).stdout.decode(encoding='UTF-8')
+
def _extract_signature_function(self, video_id, player_url, example_sig):
id_m = re.match(
r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player(?:-new)?|(?:/[a-z]{2,3}_[A-Z]{2})?/base)?\.(?P<ext>[a-z]+)$',
@@ -1678,7 +1693,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
# Get video webpage
url = proto + '://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1&bpctr=9999999999' % video_id
- video_webpage = self._download_webpage(url, video_id)
+ video_webpage = self._rate_limit_download(url, video_id)
# Attempt to extract SWF player URL
mobj = re.search(r'swfConfig.*?"(https?:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
@@ -1736,10 +1751,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
r'"sts"\s*:\s*(\d+)', embed_webpage, 'sts', default=''),
})
video_info_url = proto + '://www.youtube.com/get_video_info?' + data
- video_info_webpage = self._download_webpage(
+ video_info_webpage = self._rate_limit_download(
video_info_url, video_id,
- note='Refetching age-gated info webpage',
- errnote='unable to download video info webpage')
+ note='Refetching age-gated info webpage')
video_info = compat_parse_qs(video_info_webpage)
pl_response = video_info.get('player_response', [None])[0]
player_response = extract_player_response(pl_response, video_id)
@@ -1777,7 +1791,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
# The general idea is to take a union of itags of both DASH manifests (for example
# video with such 'manifest behavior' see https://github.com/ytdl-org/youtube-dl/issues/6093)
self.report_video_info_webpage_download(video_id)
- for el in ('embedded', 'detailpage', 'vevo', ''):
+ for el in ('', 'embedded', 'detailpage', 'vevo'):
query = {
'video_id': video_id,
'ps': 'default',
@@ -1789,11 +1803,22 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
query['el'] = el
if sts:
query['sts'] = sts
- video_info_webpage = self._download_webpage(
- '%s://www.youtube.com/get_video_info' % proto,
- video_id, note=False,
- errnote='unable to download video info webpage',
- fatal=False, query=query)
+
+ if el == '':
+ base_url = 'https://youtube.com/get_video_info?video_id={}'.format(video_id)
+ else:
+ base_url = 'https://youtube.com/get_video_info'
+
+ for q in query:
+ if q is None or q is "":
+ continue
+ if query[q] is None or query[q] is "":
+ continue
+
+ base_url = base_url + "?{}={}".format(q, query[q])
+
+ video_info_webpage = self._rate_limit_download(base_url, video_id)
+
if not video_info_webpage:
continue
get_video_info = compat_parse_qs(video_info_webpage)
--
2.17.1
+3 -1
View File
@@ -12,6 +12,7 @@ import (
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/store"
"github.com/sirupsen/logrus"
"github.com/lbryio/ytsync/util"
)
@@ -53,7 +54,8 @@ func reflectBlobs() error {
if running {
return errors.Prefix("cannot reflect blobs as the daemon is running", err)
}
logrus.SetLevel(logrus.InfoLevel)
defer logrus.SetLevel(logrus.DebugLevel)
ex, err := os.Executable()
if err != nil {
return errors.Err(err)
+2 -2
View File
@@ -13,12 +13,12 @@ lbrycrdurl="rpc://lbry:lbry@lbrycrd:29245"
#MySQL DSN is required for chainquery to store information.
#DEFAULT: "lbry:lbry@tcp(localhost:3306)/chainquery"
#SUGGESTED: "lbry:lbry@unix(/var/run/mysqld/mysqld.sock)/chainquery"
mysqldsn="lbry:lbry@tcp(mysqlCQ:3306)/chainquery"
mysqldsn="lbry:lbry@tcp(mysql:3306)/chainquery"
#API MySQL DSN is required for chainquery to expose a SQL query service
#DEFAULT: "lbry:lbry@tcp(localhost:3306)/chainquery"
#SUGGESTED: "lbry:lbry@unix(/var/run/mysqld/mysqld.sock)/chainquery"
apimysqldsn="lbry:lbry@tcp(mysqlCQ:3306)/chainquery"
apimysqldsn="lbry:lbry@tcp(mysql:3306)/chainquery"
#API Host and Port is required for the API Server to bind and listen on.
#DEFAULT: "0.0.0.0:6300"
+2 -5
View File
@@ -1,12 +1,9 @@
#blockchain_name: lbrycrd_main
#blockchain_name: lbrycrd_testnet
blockchain_name: lbrycrd_regtest
lbryum_servers:
# - spv1.lbry.com:50001 #Production Wallet Server
- walletserver:50001
- walletserver:50001
reflect_streams: false
save_blobs: true
save_files: false
reflect_streams: false #for the love of god, don't upload regtest streams to reflector!
share_usage_data: false
tcp_port: 3333
udp_port: 4444
+10 -28
View File
@@ -4,7 +4,7 @@ services:
## Lbrycrd ##
#############
lbrycrd:
image: lbry/lbrycrd:v0.17.2.1
image: lbry/lbrycrd:v0.17.3.2-deprecatedrpc
restart: "no"
ports:
- "15201:29246"
@@ -21,7 +21,7 @@ services:
## Wallet Server ##
###################
walletserver:
image: lbry/wallet-server:v0.42.2
image: lbry/wallet-server:v0.48.1
restart: always
environment:
- DB_DIRECTORY=/database
@@ -47,7 +47,7 @@ services:
## Lbrynet ##
#############
lbrynet:
image: lbry/lbrynet:v0.42.2
image: lbry/lbrynet:v0.48.1
restart: always
ports:
- "15100:5279"
@@ -80,6 +80,9 @@ services:
- MYSQL_USER=lbry
- MYSQL_PASSWORD=lbry
- MYSQL_LOG_CONSOLE=true
volumes:
- "./init.sql:/docker-entrypoint-initdb.d/init.sql"
- "./chainquery/docker/my.cnf:/etc/mysql/conf.d/chainquery-optimizations.cnf"
###################
## Internal APIs ##
###################
@@ -96,40 +99,19 @@ services:
environment:
- MYSQL_DSN=lbry:lbry@tcp(mysql:3306)/lbry
- LBRYCRD_CONNECT=rpc://lbry:lbry@lbrycrd:29245
- MYSQL_USER=lbry
- MYSQL_PASS=lbry
- MYSQL_DATABASE=lbry
entrypoint: wait-for-it -t 0 mysql:3306 -- wait-for-it -t 0 lbrycrd:29245 -- ./latest serve
######################
## MySQL Chainquery ##
######################
mysqlCQ:
image: mysql/mysql-server:5.7.27
restart: "no"
ports:
- "15600:3306"
expose:
- "3306"
environment:
- MYSQL_ALLOW_EMPTY_PASSWORD=true
- MYSQL_DATABASE=chainquery
- MYSQL_USER=lbry
- MYSQL_PASSWORD=lbry
- MYSQL_LOG_CONSOLE=true
volumes:
- "./chainquery/docker/my.cnf:/etc/mysql/conf.d/chainquery-optimizations.cnf"
entrypoint: wait-for-it -t 0 chainquery:6300 -- wait-for-it -t 0 lbrycrd:29245 -- ./latest serve
################
## Chainquery ##
################
chainquery:
image: lbry/chainquery:master
image: lbry/chainquery:v2.0.8
restart: "no"
ports:
- 6300:6300
depends_on:
- lbrycrd
- mysqlCQ
- mysql
## TODO: Uncomment this in a docker-compose.override.yml to allow for external configurations.
volumes:
- ./chainqueryconfig.toml:/etc/lbry/chainqueryconfig.toml
entrypoint: wait-for-it -t 0 lbrycrd:29245 -- wait-for-it -t 0 mysqlCQ:3306 -- start
entrypoint: wait-for-it -t 0 lbrycrd:29245 -- wait-for-it -t 0 mysql:3306 -- start
+5 -5
View File
@@ -56,10 +56,10 @@ echo "successfully started..."
./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 #Force channel intended...just in case. This channel lines up with the api container
status=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM youtube_data WHERE id=1')
videoStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM synced_video WHERE id=1')
videoClaimID1=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT claim_id FROM synced_video WHERE id=1')
videoClaimID2=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT claim_id FROM synced_video WHERE id=1')
videoClaimAddress1=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15600 -e 'SELECT claim_address FROM claim WHERE id=2')
videoClaimAddress2=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15600 -e 'SELECT claim_address FROM claim WHERE id=2')
videoClaimID1=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT publish.claim_id FROM synced_video INNER JOIN publish ON publish.id = synced_video.publish_id WHERE synced_video.id=1')
videoClaimID2=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT publish.claim_id FROM synced_video INNER JOIN publish ON publish.id = synced_video.publish_id WHERE synced_video.id=2')
videoClaimAddress1=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT claim_address FROM claim WHERE id=2')
videoClaimAddress2=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT claim_address FROM claim WHERE id=3')
# Create Supports for published claim
./supporty/supporty @BeamerTest "${videoClaimID1}" "${videoClaimAddress1}" lbrycrd_regtest 1.0
./supporty/supporty @BeamerTest "${videoClaimID2}" "${videoClaimAddress2}" lbrycrd_regtest 2.0
@@ -75,7 +75,7 @@ curl -i -H 'Accept: application/json' -H 'Content-Type: application/json' 'http:
# Check that the channel and the video are marked as transferred and that all supports are spent
channelTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT distinct transfer_state FROM youtube_data')
videoTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT distinct transferred FROM synced_video')
nrUnspentSupports=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15600 -e 'SELECT COUNT(*) FROM chainquery.support INNER JOIN output ON output.transaction_hash = support.transaction_hash_id AND output.vout = support.vout WHERE output.is_spent = 0')
nrUnspentSupports=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT COUNT(*) FROM chainquery.support INNER JOIN output ON output.transaction_hash = support.transaction_hash_id AND output.vout = support.vout WHERE output.is_spent = 0')
if [[ $status != "synced" || $videoStatus != "published" || $channelTransferStatus != "2" || $videoTransferStatus != "1" || $nrUnspentSupports != "1" ]]; then
echo "~~!!!~~~FAILED~~~!!!~~"
echo "Channel Status: $status"
+3
View File
@@ -0,0 +1,3 @@
CREATE DATABASE IF NOT EXISTS chainquery;
GRANT ALL PRIVILEGES ON chainquery.* TO 'lbry'@'%';
FLUSH PRIVILEGES;
+15 -13
View File
@@ -11,9 +11,9 @@ function override_config_option() {
local var=$1 option=$2 config=$3
if [[ -v $var ]]; then
# Remove the existing config option:
sed -i "/^$option\W*=/d" $config
sed -i "/^$option\W*=/d" "$config"
# Add the value from the environment:
echo "$option=${!var}" >> $config
echo "$option=${!var}" >> "$config"
fi
}
@@ -38,15 +38,17 @@ function set_config() {
else
echo "Creating a fresh config file from environment variables."
## Set config params
echo "port=${PORT=9246}" > $CONFIG_PATH
echo "rpcuser=${RPC_USER=lbry}" >> $CONFIG_PATH
echo "rpcpassword=${RPC_PASSWORD=lbry}" >> $CONFIG_PATH
echo "rpcallowip=${RPC_ALLOW_IP=127.0.0.1/24}" >> $CONFIG_PATH
echo "rpcport=${RPC_PORT=9245}" >> $CONFIG_PATH
echo "rpcbind=${RPC_BIND=0.0.0.0}" >> $CONFIG_PATH
echo "deprecatedrpc=accounts" >> $CONFIG_PATH
echo "deprecatedrpc=validateaddress" >> $CONFIG_PATH
echo "deprecatedrpc=signrawtransaction" >> $CONFIG_PATH
{
echo "port=${PORT=9246}"
echo "rpcuser=${RPC_USER=lbry}"
echo "rpcpassword=${RPC_PASSWORD=lbry}"
echo "rpcallowip=${RPC_ALLOW_IP=127.0.0.1/24}"
echo "rpcport=${RPC_PORT=9245}"
echo "rpcbind=${RPC_BIND=0.0.0.0}"
echo "deprecatedrpc=accounts"
echo "deprecatedrpc=validateaddress"
echo "deprecatedrpc=signrawtransaction"
} >> $CONFIG_PATH
fi
echo "Config: "
cat $CONFIG_PATH
@@ -79,7 +81,7 @@ case $RUN_MODE in
regtest )
## Set config params
## TODO: Make this more automagic in the future.
mkdir -p `dirname $CONFIG_PATH`
mkdir -p "$(dirname $CONFIG_PATH)"
echo "rpcuser=lbry" > $CONFIG_PATH
echo "rpcpassword=lbry" >> $CONFIG_PATH
echo "rpcport=29245" >> $CONFIG_PATH
@@ -101,7 +103,7 @@ case $RUN_MODE in
testnet )
## Set config params
## TODO: Make this more automagic in the future.
mkdir -p `dirname $CONFIG_PATH`
mkdir -p "$(dirname $CONFIG_PATH)"
echo "rpcuser=lbry" > $CONFIG_PATH
echo "rpcpassword=lbry" >> $CONFIG_PATH
echo "rpcport=29245" >> $CONFIG_PATH
+1 -1
View File
@@ -8,7 +8,7 @@ services:
## Lbrynet ##
#############
lbrynet:
image: lbry/lbrynet:v0.38.6
image: lbry/lbrynet:v0.48.1
restart: "no"
networks:
lbry-network:
+3 -2
View File
@@ -13,13 +13,12 @@ require (
github.com/docker/go-units v0.4.0 // indirect
github.com/go-sql-driver/mysql v1.4.1 // indirect
github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc // indirect
github.com/google/btree v1.0.0 // indirect
github.com/hashicorp/go-immutable-radix v1.1.0 // indirect
github.com/hashicorp/go-sockaddr v1.0.2 // indirect
github.com/hashicorp/golang-lru v0.5.3 // indirect
github.com/hashicorp/memberlist v0.1.5 // indirect
github.com/hashicorp/serf v0.8.5 // indirect
github.com/lbryio/lbry.go/v2 v2.3.3-0.20191010144133-fa9bab156a4e
github.com/lbryio/lbry.go/v2 v2.4.1-0.20191206194319-06764c3d007e
github.com/lbryio/reflector.go v1.0.6-0.20190828131602-ce3d4403dbc6
github.com/miekg/dns v1.1.22 // indirect
github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b
@@ -34,3 +33,5 @@ require (
google.golang.org/api v0.11.0
google.golang.org/appengine v1.6.5 // indirect
)
go 1.13
+2 -2
View File
@@ -213,8 +213,8 @@ github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpU
github.com/lbryio/lbry.go v0.0.0-20190828131228-f3a1fbdd5303/go.mod h1:qR+Ui0hYhemIU4fXqM3d1P9eiaRFlof777VJgV7KJ8w=
github.com/lbryio/lbry.go v1.1.2 h1:Dyxc+glT/rVWJwHfIf7vjlPYYbjzrQz5ARmJd5Hp69c=
github.com/lbryio/lbry.go v1.1.2/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
github.com/lbryio/lbry.go/v2 v2.3.3-0.20191010144133-fa9bab156a4e h1:B1ekb7lXFuMol/tUdwTajOe56ab38drBJmr0X8IoZYE=
github.com/lbryio/lbry.go/v2 v2.3.3-0.20191010144133-fa9bab156a4e/go.mod h1:Fm4jBlDyDauHIBC5OABuqcBWKBpcbRgPk5K0eG2mhSs=
github.com/lbryio/lbry.go/v2 v2.4.1-0.20191206194319-06764c3d007e h1:wN6r46L8EjgC2JyOImagQXaQvyF8Bd2QPjfXqv7oDAg=
github.com/lbryio/lbry.go/v2 v2.4.1-0.20191206194319-06764c3d007e/go.mod h1:gbs/+PVII96AqsRgf+hY/F7sJkb/ciiX2yy3j+NApd0=
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46 h1:LemfR+rMxhf7nnOrzy2HqS7Me7SZ5gEwOcNFzKC8ySQ=
github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
+155 -86
View File
@@ -1,122 +1,191 @@
package ip_manager
import (
"github.com/asaskevich/govalidator"
"github.com/lbryio/ytsync/util"
log "github.com/sirupsen/logrus"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
"net"
"sort"
"sync"
"time"
"github.com/asaskevich/govalidator"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/ytsync/util"
log "github.com/sirupsen/logrus"
)
const IPCooldownPeriod = 25 * time.Second
const IPCooldownPeriod = 20 * time.Second
const unbanTimeout = 3 * time.Hour
var ipv6Pool []string
var ipv4Pool []string
var throttledIPs map[string]bool
var ipInUse map[string]bool
var ipLastUsed map[string]time.Time
var ipMutex sync.Mutex
var stopper = stop.New()
func GetNextIP(ipv6 bool) (string, error) {
ipMutex.Lock()
defer ipMutex.Unlock()
if len(ipv4Pool) < 1 || len(ipv6Pool) < 1 {
throttledIPs = make(map[string]bool)
ipInUse = make(map[string]bool)
ipLastUsed = make(map[string]time.Time)
addrs, err := net.InterfaceAddrs()
if err != nil {
return "", errors.Err(err)
}
type IPPool struct {
ips []throttledIP
lock *sync.Mutex
}
for _, address := range addrs {
if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) {
ipv6Pool = append(ipv6Pool, ipnet.IP.String())
ipLastUsed[ipnet.IP.String()] = time.Now().Add(-IPCooldownPeriod)
} else if ipnet.IP.To4() != nil && govalidator.IsIPv4(ipnet.IP.String()) {
ipv4Pool = append(ipv4Pool, ipnet.IP.String())
ipLastUsed[ipnet.IP.String()] = time.Now().Add(-IPCooldownPeriod)
}
type throttledIP struct {
IP string
LastUse time.Time
Throttled bool
InUse bool
}
var ipPoolInstance *IPPool
func GetIPPool() (*IPPool, error) {
if ipPoolInstance != nil {
return ipPoolInstance, nil
}
addrs, err := net.InterfaceAddrs()
if err != nil {
return nil, errors.Err(err)
}
var pool []throttledIP
ipv6Added := false
for _, address := range addrs {
if ipnet, ok := address.(*net.IPNet); ok && ipnet.IP.IsGlobalUnicast() {
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) && !ipv6Added {
pool = append(pool, throttledIP{
IP: ipnet.IP.String(),
LastUse: time.Now().Add(-5 * time.Minute),
})
ipv6Added = true
} else if ipnet.IP.To4() != nil && govalidator.IsIPv4(ipnet.IP.String()) {
pool = append(pool, throttledIP{
IP: ipnet.IP.String(),
LastUse: time.Now().Add(-5 * time.Minute),
})
}
}
}
nextIP := ""
if ipv6 {
nextIP = getLeastUsedIP(ipv6Pool)
} else {
nextIP = getLeastUsedIP(ipv4Pool)
ipPoolInstance = &IPPool{
ips: pool,
lock: &sync.Mutex{},
}
if nextIP == "" {
return "throttled", errors.Err("all IPs are throttled")
}
lastUse := ipLastUsed[nextIP]
if time.Since(lastUse) < IPCooldownPeriod {
log.Debugf("The IP %s is too hot, waiting for %.1f seconds before continuing", nextIP, (IPCooldownPeriod - time.Since(lastUse)).Seconds())
time.Sleep(IPCooldownPeriod - time.Since(lastUse))
}
ipInUse[nextIP] = true
return nextIP, nil
return ipPoolInstance, nil
}
func ReleaseIP(ip string) {
ipMutex.Lock()
defer ipMutex.Unlock()
ipLastUsed[ip] = time.Now()
ipInUse[ip] = false
}
func getLeastUsedIP(ipPool []string) string {
nextIP := ""
veryLastUse := time.Now()
for _, ip := range ipPool {
isThrottled := throttledIPs[ip]
if isThrottled {
continue
}
inUse := ipInUse[ip]
if inUse {
continue
}
lastUse := ipLastUsed[ip]
if lastUse.Before(veryLastUse) {
nextIP = ip
veryLastUse = lastUse
// AllThrottled checks whether the IPs provided are all throttled.
// returns false if at least one IP is not throttled
// Not thread safe, should use locking when called
func AllThrottled(ips []throttledIP) bool {
for _, i := range ips {
if !i.Throttled {
return false
}
}
return nextIP
return true
}
func SetIpThrottled(ip string, stopGrp *stop.Group) {
ipMutex.Lock()
isThrottled := throttledIPs[ip]
if isThrottled {
return
// AllInUse checks whether the IPs provided are all currently in use.
// returns false if at least one IP is not in use AND is not throttled
// Not thread safe, should use locking when called
func AllInUse(ips []throttledIP) bool {
for _, i := range ips {
if !i.InUse && !i.Throttled {
return false
}
}
return true
}
func (i *IPPool) ReleaseIP(ip string) {
i.lock.Lock()
defer i.lock.Unlock()
for j, _ := range i.ips {
localIP := &i.ips[j]
if localIP.IP == ip {
localIP.InUse = false
localIP.LastUse = time.Now()
break
}
}
}
func (i *IPPool) SetThrottled(ip string, stopGrp *stop.Group) {
i.lock.Lock()
defer i.lock.Unlock()
var tIP *throttledIP
for j, _ := range i.ips {
localIP := &i.ips[j]
if localIP.IP == ip {
if localIP.Throttled {
return
}
localIP.Throttled = true
tIP = localIP
break
}
}
throttledIPs[ip] = true
ipMutex.Unlock()
util.SendErrorToSlack("%s set to throttled", ip)
stopper.Add(1)
go func() {
go func(tIP *throttledIP) {
defer stopper.Done()
unbanTimer := time.NewTimer(unbanTimeout)
select {
case <-unbanTimer.C:
ipMutex.Lock()
throttledIPs[ip] = false
ipMutex.Unlock()
i.lock.Lock()
tIP.Throttled = false
i.lock.Unlock()
util.SendInfoToSlack("%s set back to not throttled", ip)
case <-stopGrp.Ch():
unbanTimer.Stop()
}
}()
}(tIP)
}
var ErrAllInUse = errors.Base("all IPs are in use, try again")
var ErrAllThrottled = errors.Base("all IPs are throttled")
var ErrResourceLock = errors.Base("error getting next ip, did you forget to lock on the resource?")
func (i *IPPool) nextIP() (*throttledIP, error) {
i.lock.Lock()
defer i.lock.Unlock()
sort.Slice(i.ips, func(j, k int) bool {
return i.ips[j].LastUse.Before(i.ips[k].LastUse)
})
if !AllThrottled(i.ips) {
if AllInUse(i.ips) {
return nil, errors.Err(ErrAllInUse)
}
var nextIP *throttledIP
for j, _ := range i.ips {
ip := &i.ips[j]
if ip.InUse || ip.Throttled {
continue
}
nextIP = ip
break
}
if nextIP == nil {
return nil, errors.Err(ErrResourceLock)
}
nextIP.InUse = true
return nextIP, nil
}
return nil, errors.Err(ErrAllThrottled)
}
func (i *IPPool) GetIP() (string, error) {
for {
ip, err := i.nextIP()
if err != nil {
if errors.Is(err, ErrAllInUse) {
time.Sleep(5 * time.Second)
continue
} else if errors.Is(err, ErrAllThrottled) {
return "throttled", err
}
return "", err
}
if time.Since(ip.LastUse) < IPCooldownPeriod {
log.Debugf("The IP %s is too hot, waiting for %.1f seconds before continuing", ip.IP, (IPCooldownPeriod - time.Since(ip.LastUse)).Seconds())
time.Sleep(IPCooldownPeriod - time.Since(ip.LastUse))
}
return ip.IP, nil
}
}
+40
View File
@@ -0,0 +1,40 @@
package ip_manager
import (
"testing"
)
func TestAll(t *testing.T) {
pool, err := GetIPPool()
if err != nil {
t.Fatal(err)
}
ip, err := pool.GetIP()
if err != nil {
t.Fatal(err)
}
t.Log(ip)
pool.ReleaseIP(ip)
ip2, err := pool.GetIP()
if err != nil {
t.Fatal(err)
}
if ip == ip2 && len(pool.ips) > 1 {
t.Fatalf("the same IP was returned twice! %s, %s", ip, ip2)
}
t.Log(ip2)
pool.ReleaseIP(ip2)
for range pool.ips {
_, err = pool.GetIP()
if err != nil {
t.Fatal(err)
}
}
next, err := pool.nextIP()
if err != nil {
t.Logf("%s", err.Error())
} else {
t.Fatal(next)
}
}
+48 -26
View File
@@ -22,14 +22,21 @@ import (
)
func (s *Sync) enableAddressReuse() error {
accountsResponse, err := s.daemon.AccountList()
accountsResponse, err := s.daemon.AccountList(1, 50)
if err != nil {
return errors.Err(err)
}
accounts := accountsResponse.LBCMainnet
accounts := make([]jsonrpc.Account, 0, len(accountsResponse.Items))
ledger := "lbc_mainnet"
if logUtils.IsRegTest() {
accounts = accountsResponse.LBCRegtest
ledger = "lbc_regtest"
}
for _, a := range accountsResponse.Items {
if *a.Ledger == ledger {
accounts = append(accounts, a)
}
}
for _, a := range accounts {
_, err = s.daemon.AccountSet(a.ID, jsonrpc.AccountSettings{
ChangeMaxUses: util.PtrToInt(1000),
@@ -121,13 +128,13 @@ func (s *Sync) walletSetup() error {
}
}
claimAddress, err := s.daemon.AddressList(nil, nil)
claimAddress, err := s.daemon.AddressList(nil, nil, 1, 20)
if err != nil {
return err
} else if claimAddress == nil {
return errors.Err("could not get unused address")
return errors.Err("could not get an address")
}
s.claimAddress = string((*claimAddress)[0].Address)
s.claimAddress = string(claimAddress.Items[0].Address)
if s.claimAddress == "" {
return errors.Err("found blank claim address")
}
@@ -145,20 +152,23 @@ func (s *Sync) walletSetup() error {
func (s *Sync) getDefaultAccount() (string, error) {
if s.defaultAccountID == "" {
accounts, err := s.daemon.AccountList()
accountsResponse, err := s.daemon.AccountList(1, 50)
if err != nil {
return "", errors.Err(err)
}
accountsNet := (*accounts).LBCMainnet
ledger := "lbc_mainnet"
if logUtils.IsRegTest() {
accountsNet = (*accounts).LBCRegtest
ledger = "lbc_regtest"
}
for _, account := range accountsNet {
if account.IsDefault {
s.defaultAccountID = account.ID
break
for _, a := range accountsResponse.Items {
if *a.Ledger == ledger {
if a.IsDefault {
s.defaultAccountID = a.ID
break
}
}
}
if s.defaultAccountID == "" {
return "", errors.Err("No default account found")
}
@@ -172,7 +182,7 @@ func (s *Sync) ensureEnoughUTXOs() error {
return err
}
utxolist, err := s.daemon.UTXOList(&defaultAccount)
utxolist, err := s.daemon.UTXOList(&defaultAccount, 1, 10000)
if err != nil {
return err
} else if utxolist == nil {
@@ -184,7 +194,7 @@ func (s *Sync) ensureEnoughUTXOs() error {
count := 0
confirmedCount := 0
for _, utxo := range *utxolist {
for _, utxo := range utxolist.Items {
amount, _ := strconv.ParseFloat(utxo.Amount, 64)
if utxo.IsMine && utxo.Type == "payment" && amount > 0.001 {
if utxo.Confirmations > 0 {
@@ -240,21 +250,11 @@ func (s *Sync) ensureEnoughUTXOs() error {
}
func (s *Sync) waitForNewBlock() error {
if logUtils.IsRegTest() && logUtils.IsUsingDocker() {
lbrycrd, err := logUtils.GetLbrycrdClient(s.LbrycrdString)
if err != nil {
return errors.Prefix("error getting lbrycrd client: ", err)
}
txs, err := lbrycrd.Generate(1)
for _, tx := range txs {
log.Info("Generated tx: ", tx.String())
}
}
log.Printf("regtest: %t, docker: %t", logUtils.IsRegTest(), logUtils.IsUsingDocker())
status, err := s.daemon.Status()
if err != nil {
return err
}
for status.Wallet.Blocks == 0 || status.Wallet.BlocksBehind != 0 {
time.Sleep(5 * time.Second)
status, err = s.daemon.Status()
@@ -262,11 +262,18 @@ func (s *Sync) waitForNewBlock() error {
return err
}
}
currentBlock := status.Wallet.Blocks
for i := 0; status.Wallet.Blocks <= currentBlock; i++ {
if i%3 == 0 {
log.Printf("Waiting for new block (%d)...", currentBlock+1)
}
if logUtils.IsRegTest() && logUtils.IsUsingDocker() {
err = s.GenerateRegtestBlock()
if err != nil {
return err
}
}
time.Sleep(10 * time.Second)
status, err = s.daemon.Status()
if err != nil {
@@ -276,6 +283,21 @@ func (s *Sync) waitForNewBlock() error {
return nil
}
func (s *Sync) GenerateRegtestBlock() error {
lbrycrd, err := logUtils.GetLbrycrdClient(s.LbrycrdString)
if err != nil {
return errors.Prefix("error getting lbrycrd client: ", err)
}
txs, err := lbrycrd.Generate(1)
if err != nil {
return errors.Prefix("error generating new block: ", err)
}
for _, tx := range txs {
log.Info("Generated tx: ", tx.String())
}
return nil
}
func (s *Sync) ensureChannelOwnership() error {
if s.LbryChannelName == "" {
return errors.Err("no channel name set")
+5 -5
View File
@@ -24,14 +24,14 @@ func waitConfirmations(s *Sync) error {
allConfirmed := false
waiting:
for !allConfirmed {
utxolist, err := s.daemon.UTXOList(&defaultAccount)
utxolist, err := s.daemon.UTXOList(&defaultAccount, 1, 10000)
if err != nil {
return err
} else if utxolist == nil {
return errors.Err("no response")
}
for _, utxo := range *utxolist {
for _, utxo := range utxolist.Items {
if utxo.Confirmations <= 0 {
err = s.waitForNewBlock()
if err != nil {
@@ -177,7 +177,7 @@ func transferVideos(s *Sync) error {
if err != nil {
return err
}
streams, err := s.daemon.StreamList(&account)
streams, err := s.daemon.StreamList(&account, 1, 30000)
if err != nil {
return errors.Err(err)
}
@@ -191,8 +191,8 @@ func transferVideos(s *Sync) error {
}
var stream *jsonrpc.Claim = nil
for _, c := range *streams {
if c.ClaimID != video.ClaimID {
for _, c := range streams.Items {
if c.ClaimID != video.ClaimID || (c.SigningChannel != nil && c.SigningChannel.ClaimID != s.lbryChannelID) {
continue
}
stream = &c
+38 -20
View File
@@ -14,6 +14,7 @@ import (
"syscall"
"time"
"github.com/lbryio/ytsync/ip_manager"
"github.com/lbryio/ytsync/namer"
"github.com/lbryio/ytsync/sdk"
"github.com/lbryio/ytsync/sources"
@@ -333,7 +334,7 @@ func (s *Sync) processTransfers() (e error) {
return errors.Prefix(fmt.Sprintf("%.6f LBCs were abandoned before failing", supportAmount), err)
}
if supportAmount > 0 {
logUtils.SendInfoToSlack("%.6f LBCs were abandoned and should be used as support", supportAmount)
logUtils.SendInfoToSlack("(%s) %.6f LBCs were abandoned and should be used as support", s.YoutubeChannelID, supportAmount)
}
err = transferVideos(s)
if err != nil {
@@ -354,7 +355,7 @@ func (s *Sync) processTransfers() (e error) {
return err
}
isTip := true
summary, err := s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount), &isTip, nil, []string{defaultAccount})
summary, err := s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount), &isTip, nil, []string{defaultAccount}, nil)
if err != nil {
return errors.Err(err)
}
@@ -466,7 +467,7 @@ var thumbnailHosts = []string{
thumbs.ThumbnailEndpoint,
}
func isYtsyncClaim(c jsonrpc.Claim) bool {
func isYtsyncClaim(c jsonrpc.Claim, expectedChannelID string) bool {
if !util.InSlice(c.Type, []string{"claim", "update"}) || c.Value.GetStream() == nil {
return false
}
@@ -474,7 +475,12 @@ func isYtsyncClaim(c jsonrpc.Claim) bool {
//most likely a claim created outside of ytsync, ignore!
return false
}
if c.SigningChannel == nil {
return false
}
if c.SigningChannel.ClaimID != expectedChannelID {
return false
}
for _, th := range thumbnailHosts {
if strings.Contains(c.Value.GetThumbnail().GetUrl(), th) {
return true
@@ -488,7 +494,7 @@ func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) {
abandonedClaims := false
videoIDs := make(map[string]jsonrpc.Claim)
for _, c := range claims {
if !isYtsyncClaim(c) {
if !isYtsyncClaim(c, s.lbryChannelID) {
continue
}
tn := c.Value.GetThumbnail().GetUrl()
@@ -535,7 +541,7 @@ type ytsyncClaim struct {
func (s *Sync) mapFromClaims(claims []jsonrpc.Claim) map[string]ytsyncClaim {
videoIDMap := make(map[string]ytsyncClaim, len(claims))
for _, c := range claims {
if !isYtsyncClaim(c) {
if !isYtsyncClaim(c, s.lbryChannelID) {
continue
}
tn := c.Value.GetThumbnail().GetUrl()
@@ -670,12 +676,17 @@ func (s *Sync) getClaims(defaultOnly bool) ([]jsonrpc.Claim, error) {
}
account = &a
}
claims, err := s.daemon.StreamList(account)
claims, err := s.daemon.StreamList(account, 1, 30000)
if err != nil {
return nil, errors.Prefix("cannot list claims", err)
}
return *claims, nil
items := make([]jsonrpc.Claim, 0, len(claims.Items))
for _, c := range claims.Items {
if c.SigningChannel != nil && c.SigningChannel.ClaimID == s.lbryChannelID {
items = append(items, c)
}
}
return items, nil
}
func (s *Sync) checkIntegrity() error {
@@ -861,6 +872,7 @@ func (s *Sync) startWorker(workerNum int) {
"giving up after 0 fragment retries",
"Sorry about that",
"This video is not available",
"requested format not available",
}
if util.SubstringInSlice(err.Error(), errorsNoRetry) {
log.Println("This error should not be retried at all")
@@ -963,6 +975,10 @@ func (s *Sync) enqueueYoutubeVideos() error {
}
var videos []video
ipPool, err := ip_manager.GetIPPool()
if err != nil {
return err
}
playlistMap := make(map[string]*youtube.PlaylistItemSnippet, 50)
nextPageToken := ""
for {
@@ -1000,7 +1016,7 @@ func (s *Sync) enqueueYoutubeVideos() error {
return errors.Prefix("error getting videos info", err)
}
for _, item := range videosListResponse.Items {
videos = append(videos, sources.NewYoutubeVideo(s.videoDirectory, item, playlistMap[item.Id].Position, s.Manager.GetS3AWSConfig(), s.grp))
videos = append(videos, sources.NewYoutubeVideo(s.videoDirectory, item, playlistMap[item.Id].Position, s.Manager.GetS3AWSConfig(), s.grp, ipPool))
}
log.Infof("Got info for %d videos from youtube API", len(videos))
@@ -1016,7 +1032,7 @@ func (s *Sync) enqueueYoutubeVideos() error {
}
_, ok := playlistMap[k]
if !ok {
videos = append(videos, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig(), s.grp))
videos = append(videos, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig(), s.grp, ipPool))
}
}
@@ -1134,17 +1150,19 @@ func (s *Sync) processVideo(v video) (err error) {
func (s *Sync) importPublicKey() error {
if s.publicKey != "" {
accountsResponse, err := s.daemon.AccountList()
accountsResponse, err := s.daemon.AccountList(1, 50)
if err != nil {
return errors.Err(err)
}
accounts := accountsResponse.LBCMainnet
ledger := "lbc_mainnet"
if logUtils.IsRegTest() {
accounts = accountsResponse.LBCRegtest
ledger = "lbc_regtest"
}
for _, a := range accounts {
if a.PublicKey == s.publicKey {
return nil
for _, a := range accountsResponse.Items {
if *a.Ledger == ledger {
if a.PublicKey == s.publicKey {
return nil
}
}
}
log.Infof("Could not find public key %s in the wallet. Importing it...", s.publicKey)
@@ -1172,12 +1190,12 @@ func (s *Sync) getUnsentSupports() (float64, error) {
if err != nil {
return 0, errors.Err(err)
}
transactionList, err := s.daemon.TransactionList(&defaultAccount)
transactionList, err := s.daemon.TransactionList(&defaultAccount, 1, 90000)
if err != nil {
return 0, errors.Err(err)
}
sentSupports := 0.0
for _, t := range *transactionList {
for _, t := range transactionList.Items {
if len(t.SupportInfo) == 0 {
continue
}
@@ -1192,7 +1210,7 @@ func (s *Sync) getUnsentSupports() (float64, error) {
}
}
if balanceAmount > 10 && sentSupports < 1 {
logUtils.SendErrorToSlack("this channel has quite some LBCs in it (%.2f) and %.2f LBC in sent tips, it's likely that the tips weren't actually sent or the wallet has unnecessary extra credits in it", balanceAmount, sentSupports)
logUtils.SendErrorToSlack("(%s) this channel has quite some LBCs in it (%.2f) and %.2f LBC in sent tips, it's likely that the tips weren't actually sent or the wallet has unnecessary extra credits in it", s.YoutubeChannelID, balanceAmount, sentSupports)
return balanceAmount - 10, nil
}
}
+30 -37
View File
@@ -50,6 +50,7 @@ type YoutubeVideo struct {
mocked bool
walletLock *sync.RWMutex
stopGroup *stop.Group
pool *ip_manager.IPPool
}
var youtubeCategories = map[string]string{
@@ -87,7 +88,7 @@ var youtubeCategories = map[string]string{
"44": "trailers",
}
func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group) *YoutubeVideo {
func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo {
publishedAt, _ := time.Parse(time.RFC3339Nano, videoData.Snippet.PublishedAt) // ignore parse errors
return &YoutubeVideo{
id: videoData.Id,
@@ -101,9 +102,10 @@ func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPositio
mocked: false,
youtubeChannelID: videoData.Snippet.ChannelId,
stopGroup: stopGroup,
pool: pool,
}
}
func NewMockedVideo(directory string, videoID string, youtubeChannelID string, awsConfig aws.Config, stopGroup *stop.Group) *YoutubeVideo {
func NewMockedVideo(directory string, videoID string, youtubeChannelID string, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo {
return &YoutubeVideo{
id: videoID,
playlistPosition: 0,
@@ -112,6 +114,7 @@ func NewMockedVideo(directory string, videoID string, youtubeChannelID string, a
mocked: true,
youtubeChannelID: youtubeChannelID,
stopGroup: stopGroup,
pool: pool,
}
}
@@ -175,7 +178,7 @@ func (v *YoutubeVideo) getAbbrevDescription() string {
return description + "\n..." + additionalDescription
}
func (v *YoutubeVideo) download(useIPv6 bool) error {
func (v *YoutubeVideo) download() error {
videoPath := v.getFullPath()
err := os.Mkdir(v.videoDir(), 0777)
@@ -220,7 +223,8 @@ func (v *YoutubeVideo) download(useIPv6 bool) error {
fmt.Sprintf("duration <= %d", int(math.Round(v.maxVideoLength*3600))),
)
}
sourceAddress, err := ip_manager.GetNextIP(useIPv6)
sourceAddress, err := v.pool.GetIP()
if err != nil {
if sourceAddress == "throttled" {
for {
@@ -231,32 +235,24 @@ func (v *YoutubeVideo) download(useIPv6 bool) error {
}
time.Sleep(ip_manager.IPCooldownPeriod)
sourceAddress, err = ip_manager.GetNextIP(useIPv6)
sourceAddress, err = v.pool.GetIP()
if err == nil {
break
} else if !errors.Is(err, ip_manager.ErrAllThrottled) {
return err
}
}
} else {
return errors.Err(err)
}
}
defer ip_manager.ReleaseIP(sourceAddress)
if useIPv6 {
log.Infof("using IPv6: %s", sourceAddress)
ytdlArgs = append(ytdlArgs,
"-6",
"--source-address",
sourceAddress,
)
} else {
log.Infof("using IPv4: %s", sourceAddress)
ytdlArgs = append(ytdlArgs,
"-4",
"--source-address",
sourceAddress,
)
}
ytdlArgs = append(ytdlArgs, "https://www.youtube.com/watch?v="+v.ID())
defer v.pool.ReleaseIP(sourceAddress)
ytdlArgs = append(ytdlArgs,
"--source-address",
sourceAddress,
"https://www.youtube.com/watch?v="+v.ID(),
)
runcmd:
argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][height<="+qualities[qualityIndex]+"]+bestaudio[ext!=webm]")
cmd := exec.Command("youtube-dl", argsWithFilters...)
@@ -281,11 +277,11 @@ runcmd:
if err = cmd.Wait(); err != nil {
if strings.Contains(err.Error(), "exit status 1") {
if strings.Contains(string(errorLog), "HTTP Error 429") {
ip_manager.SetIpThrottled(sourceAddress, v.stopGroup)
if strings.Contains(string(errorLog), "HTTP Error 429") || strings.Contains(string(errorLog), "returned non-zero exit status 8") {
v.pool.SetThrottled(sourceAddress, v.stopGroup)
} else if strings.Contains(string(errorLog), "giving up after 0 fragment retries") && qualityIndex < len(qualities)-1 {
qualityIndex++
goto runcmd
goto runcmd //this bypasses the yt throttling IP redistribution... TODO: don't
}
return errors.Err(string(errorLog))
}
@@ -430,21 +426,18 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV
return v.downloadAndPublish(daemon, params)
}
var isThrottled bool
func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
err := v.download(isThrottled)
if err != nil {
if strings.Contains(err.Error(), "HTTP Error 429") && !isThrottled {
isThrottled = true
err = v.download(isThrottled)
if err != nil {
return nil, errors.Prefix("download error", err)
}
} else {
var err error
for {
err = v.download()
if err != nil && strings.Contains(err.Error(), "HTTP Error 429") {
continue
} else if err != nil {
return nil, errors.Prefix("download error", err)
}
break
}
log.Debugln("Downloaded " + v.id)
err = v.triggerThumbnailSave()
@@ -489,7 +482,7 @@ func (v *YoutubeVideo) getMetadata() (languages []string, locations []jsonrpc.Lo
}
func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo) (*SyncSummary, error) {
c, err := daemon.ClaimSearch(nil, &existingVideoData.ClaimID, nil, nil)
c, err := daemon.ClaimSearch(nil, &existingVideoData.ClaimID, nil, nil, 1, 20)
if err != nil {
return nil, errors.Err(err)
}
+6 -5
View File
@@ -61,11 +61,12 @@ func (u *thumbnailUploader) uploadThumbnail() error {
uploader := s3manager.NewUploader(s3Session)
_, err = uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String("thumbnails.lbry.com"),
Key: key,
Body: thumb,
ACL: aws.String("public-read"),
ContentType: aws.String("image/jpeg"),
Bucket: aws.String("thumbnails.lbry.com"),
Key: key,
Body: thumb,
ACL: aws.String("public-read"),
ContentType: aws.String("image/jpeg"),
CacheControl: aws.String("public, max-age=2592000"),
})
u.mirroredUrl = ThumbnailEndpoint + u.name
return errors.Err(err)