Compare commits

..
1 Commits
Author SHA1 Message Date
Alex Grintsvayg 8b32381b23 grin 2019-10-16 15:00:11 -04:00
23 changed files with 359 additions and 643 deletions

No files matched your search

-111
View File
@@ -1,111 +0,0 @@
From 30380338ba9af01696c94b61f0597131638eaec1 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Mon, 16 Dec 2019 00:13:36 +0100
Subject: [PATCH] lbry-patch
---
youtube_dl/extractor/youtube.py | 45 +++++++++++++++++++++++++--------
1 file changed, 35 insertions(+), 10 deletions(-)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index b913d07a6..cd66a5b01 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -10,6 +10,7 @@ import random
import re
import time
import traceback
+import subprocess
from .common import InfoExtractor, SearchInfoExtractor
from ..jsinterp import JSInterpreter
@@ -536,6 +537,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
_GEO_BYPASS = False
+ _WGET_429_RATE_LIMIT = 8191
+ _WGET_BINARY = "wget"
+
IE_NAME = 'youtube'
_TESTS = [
{
@@ -1254,6 +1258,17 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
""" Return a string representation of a signature """
return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
+ def _rate_limit_download(self, url, video_id, note=None):
+ if note is None:
+ self.report_download_webpage(video_id)
+ elif note is not False:
+ if video_id is None:
+ self.to_screen('%s' % (note,))
+ else:
+ self.to_screen('%s: %s' % (video_id, note))
+ source_address = self._downloader.params.get('source_address')
+ return subprocess.run([self._WGET_BINARY, '-q', '--limit-rate', str(self._WGET_429_RATE_LIMIT), '--bind-address', source_address, '-O', '-', url], check=True, stdout=subprocess.PIPE).stdout.decode(encoding='UTF-8')
+
def _extract_signature_function(self, video_id, player_url, example_sig):
id_m = re.match(
r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player(?:-new)?|(?:/[a-z]{2,3}_[A-Z]{2})?/base)?\.(?P<ext>[a-z]+)$',
@@ -1678,7 +1693,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
# Get video webpage
url = proto + '://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1&bpctr=9999999999' % video_id
- video_webpage = self._download_webpage(url, video_id)
+ video_webpage = self._rate_limit_download(url, video_id)
# Attempt to extract SWF player URL
mobj = re.search(r'swfConfig.*?"(https?:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
@@ -1736,10 +1751,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
r'"sts"\s*:\s*(\d+)', embed_webpage, 'sts', default=''),
})
video_info_url = proto + '://www.youtube.com/get_video_info?' + data
- video_info_webpage = self._download_webpage(
+ video_info_webpage = self._rate_limit_download(
video_info_url, video_id,
- note='Refetching age-gated info webpage',
- errnote='unable to download video info webpage')
+ note='Refetching age-gated info webpage')
video_info = compat_parse_qs(video_info_webpage)
pl_response = video_info.get('player_response', [None])[0]
player_response = extract_player_response(pl_response, video_id)
@@ -1777,7 +1791,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
# The general idea is to take a union of itags of both DASH manifests (for example
# video with such 'manifest behavior' see https://github.com/ytdl-org/youtube-dl/issues/6093)
self.report_video_info_webpage_download(video_id)
- for el in ('embedded', 'detailpage', 'vevo', ''):
+ for el in ('', 'embedded', 'detailpage', 'vevo'):
query = {
'video_id': video_id,
'ps': 'default',
@@ -1789,11 +1803,22 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
query['el'] = el
if sts:
query['sts'] = sts
- video_info_webpage = self._download_webpage(
- '%s://www.youtube.com/get_video_info' % proto,
- video_id, note=False,
- errnote='unable to download video info webpage',
- fatal=False, query=query)
+
+ if el == '':
+ base_url = 'https://youtube.com/get_video_info?video_id={}'.format(video_id)
+ else:
+ base_url = 'https://youtube.com/get_video_info'
+
+ for q in query:
+ if q is None or q is "":
+ continue
+ if query[q] is None or query[q] is "":
+ continue
+
+ base_url = base_url + "?{}={}".format(q, query[q])
+
+ video_info_webpage = self._rate_limit_download(base_url, video_id)
+
if not video_info_webpage:
continue
get_video_info = compat_parse_qs(video_info_webpage)
--
2.17.1
+1 -3
View File
@@ -12,7 +12,6 @@ import (
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/store"
"github.com/sirupsen/logrus"
"github.com/lbryio/ytsync/util"
)
@@ -54,8 +53,7 @@ func reflectBlobs() error {
if running {
return errors.Prefix("cannot reflect blobs as the daemon is running", err)
}
logrus.SetLevel(logrus.InfoLevel)
defer logrus.SetLevel(logrus.DebugLevel)
ex, err := os.Executable()
if err != nil {
return errors.Err(err)
+2 -2
View File
@@ -13,12 +13,12 @@ lbrycrdurl="rpc://lbry:lbry@lbrycrd:29245"
#MySQL DSN is required for chainquery to store information.
#DEFAULT: "lbry:lbry@tcp(localhost:3306)/chainquery"
#SUGGESTED: "lbry:lbry@unix(/var/run/mysqld/mysqld.sock)/chainquery"
mysqldsn="lbry:lbry@tcp(mysql:3306)/chainquery"
mysqldsn="lbry:lbry@tcp(mysqlCQ:3306)/chainquery"
#API MySQL DSN is required for chainquery to expose a SQL query service
#DEFAULT: "lbry:lbry@tcp(localhost:3306)/chainquery"
#SUGGESTED: "lbry:lbry@unix(/var/run/mysqld/mysqld.sock)/chainquery"
apimysqldsn="lbry:lbry@tcp(mysql:3306)/chainquery"
apimysqldsn="lbry:lbry@tcp(mysqlCQ:3306)/chainquery"
#API Host and Port is required for the API Server to bind and listen on.
#DEFAULT: "0.0.0.0:6300"
+5 -2
View File
@@ -1,9 +1,12 @@
#blockchain_name: lbrycrd_main
#blockchain_name: lbrycrd_testnet
blockchain_name: lbrycrd_regtest
lbryum_servers:
- walletserver:50001
reflect_streams: false
# - spv1.lbry.com:50001 #Production Wallet Server
- walletserver:50001
save_blobs: true
save_files: false
reflect_streams: false #for the love of god, don't upload regtest streams to reflector!
share_usage_data: false
tcp_port: 3333
udp_port: 4444
+28 -10
View File
@@ -4,7 +4,7 @@ services:
## Lbrycrd ##
#############
lbrycrd:
image: lbry/lbrycrd:v0.17.3.2-deprecatedrpc
image: lbry/lbrycrd:v0.17.2.1
restart: "no"
ports:
- "15201:29246"
@@ -21,7 +21,7 @@ services:
## Wallet Server ##
###################
walletserver:
image: lbry/wallet-server:v0.48.1
image: lbry/wallet-server:v0.42.2
restart: always
environment:
- DB_DIRECTORY=/database
@@ -47,7 +47,7 @@ services:
## Lbrynet ##
#############
lbrynet:
image: lbry/lbrynet:v0.48.1
image: lbry/lbrynet:v0.42.2
restart: always
ports:
- "15100:5279"
@@ -80,9 +80,6 @@ services:
- MYSQL_USER=lbry
- MYSQL_PASSWORD=lbry
- MYSQL_LOG_CONSOLE=true
volumes:
- "./init.sql:/docker-entrypoint-initdb.d/init.sql"
- "./chainquery/docker/my.cnf:/etc/mysql/conf.d/chainquery-optimizations.cnf"
###################
## Internal APIs ##
###################
@@ -99,19 +96,40 @@ services:
environment:
- MYSQL_DSN=lbry:lbry@tcp(mysql:3306)/lbry
- LBRYCRD_CONNECT=rpc://lbry:lbry@lbrycrd:29245
entrypoint: wait-for-it -t 0 chainquery:6300 -- wait-for-it -t 0 lbrycrd:29245 -- ./latest serve
- MYSQL_USER=lbry
- MYSQL_PASS=lbry
- MYSQL_DATABASE=lbry
entrypoint: wait-for-it -t 0 mysql:3306 -- wait-for-it -t 0 lbrycrd:29245 -- ./latest serve
######################
## MySQL Chainquery ##
######################
mysqlCQ:
image: mysql/mysql-server:5.7.27
restart: "no"
ports:
- "15600:3306"
expose:
- "3306"
environment:
- MYSQL_ALLOW_EMPTY_PASSWORD=true
- MYSQL_DATABASE=chainquery
- MYSQL_USER=lbry
- MYSQL_PASSWORD=lbry
- MYSQL_LOG_CONSOLE=true
volumes:
- "./chainquery/docker/my.cnf:/etc/mysql/conf.d/chainquery-optimizations.cnf"
################
## Chainquery ##
################
chainquery:
image: lbry/chainquery:v2.0.8
image: lbry/chainquery:master
restart: "no"
ports:
- 6300:6300
depends_on:
- lbrycrd
- mysql
- mysqlCQ
## TODO: Uncomment this in a docker-compose.override.yml to allow for external configurations.
volumes:
- ./chainqueryconfig.toml:/etc/lbry/chainqueryconfig.toml
entrypoint: wait-for-it -t 0 lbrycrd:29245 -- wait-for-it -t 0 mysql:3306 -- start
entrypoint: wait-for-it -t 0 lbrycrd:29245 -- wait-for-it -t 0 mysqlCQ:3306 -- start
+5 -5
View File
@@ -56,10 +56,10 @@ echo "successfully started..."
./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 #Force channel intended...just in case. This channel lines up with the api container
status=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM youtube_data WHERE id=1')
videoStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM synced_video WHERE id=1')
videoClaimID1=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT publish.claim_id FROM synced_video INNER JOIN publish ON publish.id = synced_video.publish_id WHERE synced_video.id=1')
videoClaimID2=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT publish.claim_id FROM synced_video INNER JOIN publish ON publish.id = synced_video.publish_id WHERE synced_video.id=2')
videoClaimAddress1=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT claim_address FROM claim WHERE id=2')
videoClaimAddress2=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT claim_address FROM claim WHERE id=3')
videoClaimID1=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT claim_id FROM synced_video WHERE id=1')
videoClaimID2=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT claim_id FROM synced_video WHERE id=1')
videoClaimAddress1=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15600 -e 'SELECT claim_address FROM claim WHERE id=2')
videoClaimAddress2=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15600 -e 'SELECT claim_address FROM claim WHERE id=2')
# Create Supports for published claim
./supporty/supporty @BeamerTest "${videoClaimID1}" "${videoClaimAddress1}" lbrycrd_regtest 1.0
./supporty/supporty @BeamerTest "${videoClaimID2}" "${videoClaimAddress2}" lbrycrd_regtest 2.0
@@ -75,7 +75,7 @@ curl -i -H 'Accept: application/json' -H 'Content-Type: application/json' 'http:
# Check that the channel and the video are marked as transferred and that all supports are spent
channelTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT distinct transfer_state FROM youtube_data')
videoTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT distinct transferred FROM synced_video')
nrUnspentSupports=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT COUNT(*) FROM chainquery.support INNER JOIN output ON output.transaction_hash = support.transaction_hash_id AND output.vout = support.vout WHERE output.is_spent = 0')
nrUnspentSupports=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15600 -e 'SELECT COUNT(*) FROM chainquery.support INNER JOIN output ON output.transaction_hash = support.transaction_hash_id AND output.vout = support.vout WHERE output.is_spent = 0')
if [[ $status != "synced" || $videoStatus != "published" || $channelTransferStatus != "2" || $videoTransferStatus != "1" || $nrUnspentSupports != "1" ]]; then
echo "~~!!!~~~FAILED~~~!!!~~"
echo "Channel Status: $status"
-3
View File
@@ -1,3 +0,0 @@
CREATE DATABASE IF NOT EXISTS chainquery;
GRANT ALL PRIVILEGES ON chainquery.* TO 'lbry'@'%';
FLUSH PRIVILEGES;
+13 -15
View File
@@ -11,9 +11,9 @@ function override_config_option() {
local var=$1 option=$2 config=$3
if [[ -v $var ]]; then
# Remove the existing config option:
sed -i "/^$option\W*=/d" "$config"
sed -i "/^$option\W*=/d" $config
# Add the value from the environment:
echo "$option=${!var}" >> "$config"
echo "$option=${!var}" >> $config
fi
}
@@ -38,17 +38,15 @@ function set_config() {
else
echo "Creating a fresh config file from environment variables."
## Set config params
{
echo "port=${PORT=9246}"
echo "rpcuser=${RPC_USER=lbry}"
echo "rpcpassword=${RPC_PASSWORD=lbry}"
echo "rpcallowip=${RPC_ALLOW_IP=127.0.0.1/24}"
echo "rpcport=${RPC_PORT=9245}"
echo "rpcbind=${RPC_BIND=0.0.0.0}"
echo "deprecatedrpc=accounts"
echo "deprecatedrpc=validateaddress"
echo "deprecatedrpc=signrawtransaction"
} >> $CONFIG_PATH
echo "port=${PORT=9246}" > $CONFIG_PATH
echo "rpcuser=${RPC_USER=lbry}" >> $CONFIG_PATH
echo "rpcpassword=${RPC_PASSWORD=lbry}" >> $CONFIG_PATH
echo "rpcallowip=${RPC_ALLOW_IP=127.0.0.1/24}" >> $CONFIG_PATH
echo "rpcport=${RPC_PORT=9245}" >> $CONFIG_PATH
echo "rpcbind=${RPC_BIND=0.0.0.0}" >> $CONFIG_PATH
echo "deprecatedrpc=accounts" >> $CONFIG_PATH
echo "deprecatedrpc=validateaddress" >> $CONFIG_PATH
echo "deprecatedrpc=signrawtransaction" >> $CONFIG_PATH
fi
echo "Config: "
cat $CONFIG_PATH
@@ -81,7 +79,7 @@ case $RUN_MODE in
regtest )
## Set config params
## TODO: Make this more automagic in the future.
mkdir -p "$(dirname $CONFIG_PATH)"
mkdir -p `dirname $CONFIG_PATH`
echo "rpcuser=lbry" > $CONFIG_PATH
echo "rpcpassword=lbry" >> $CONFIG_PATH
echo "rpcport=29245" >> $CONFIG_PATH
@@ -103,7 +101,7 @@ case $RUN_MODE in
testnet )
## Set config params
## TODO: Make this more automagic in the future.
mkdir -p "$(dirname $CONFIG_PATH)"
mkdir -p `dirname $CONFIG_PATH`
echo "rpcuser=lbry" > $CONFIG_PATH
echo "rpcpassword=lbry" >> $CONFIG_PATH
echo "rpcport=29245" >> $CONFIG_PATH
+1 -1
View File
@@ -8,7 +8,7 @@ services:
## Lbrynet ##
#############
lbrynet:
image: lbry/lbrynet:v0.48.1
image: lbry/lbrynet:v0.38.6
restart: "no"
networks:
lbry-network:
+1 -1
View File
@@ -18,7 +18,7 @@ require (
github.com/hashicorp/golang-lru v0.5.3 // indirect
github.com/hashicorp/memberlist v0.1.5 // indirect
github.com/hashicorp/serf v0.8.5 // indirect
github.com/lbryio/lbry.go/v2 v2.4.1-0.20191206194319-06764c3d007e
github.com/lbryio/lbry.go/v2 v2.3.3-0.20191010144133-fa9bab156a4e
github.com/lbryio/reflector.go v1.0.6-0.20190828131602-ce3d4403dbc6
github.com/miekg/dns v1.1.22 // indirect
github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b
+2 -2
View File
@@ -213,8 +213,8 @@ github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpU
github.com/lbryio/lbry.go v0.0.0-20190828131228-f3a1fbdd5303/go.mod h1:qR+Ui0hYhemIU4fXqM3d1P9eiaRFlof777VJgV7KJ8w=
github.com/lbryio/lbry.go v1.1.2 h1:Dyxc+glT/rVWJwHfIf7vjlPYYbjzrQz5ARmJd5Hp69c=
github.com/lbryio/lbry.go v1.1.2/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
github.com/lbryio/lbry.go/v2 v2.4.1-0.20191206194319-06764c3d007e h1:wN6r46L8EjgC2JyOImagQXaQvyF8Bd2QPjfXqv7oDAg=
github.com/lbryio/lbry.go/v2 v2.4.1-0.20191206194319-06764c3d007e/go.mod h1:gbs/+PVII96AqsRgf+hY/F7sJkb/ciiX2yy3j+NApd0=
github.com/lbryio/lbry.go/v2 v2.3.3-0.20191010144133-fa9bab156a4e h1:B1ekb7lXFuMol/tUdwTajOe56ab38drBJmr0X8IoZYE=
github.com/lbryio/lbry.go/v2 v2.3.3-0.20191010144133-fa9bab156a4e/go.mod h1:Fm4jBlDyDauHIBC5OABuqcBWKBpcbRgPk5K0eG2mhSs=
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46 h1:LemfR+rMxhf7nnOrzy2HqS7Me7SZ5gEwOcNFzKC8ySQ=
github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
+122
View File
@@ -0,0 +1,122 @@
package ipManager
import (
"github.com/asaskevich/govalidator"
"github.com/lbryio/ytsync/util"
log "github.com/sirupsen/logrus"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
"net"
"sync"
"time"
)
const IPCooldownPeriod = 25 * time.Second
const unbanTimeout = 3 * time.Hour
var ipv6Pool []string
var ipv4Pool []string
var throttledIPs map[string]bool
var ipInUse map[string]bool
var ipLastUsed map[string]time.Time
var ipMutex sync.Mutex
var stopper = stop.New()
func GetNextIP(ipv6 bool) (string, error) {
ipMutex.Lock()
defer ipMutex.Unlock()
if len(ipv4Pool) < 1 || len(ipv6Pool) < 1 {
throttledIPs = make(map[string]bool)
ipInUse = make(map[string]bool)
ipLastUsed = make(map[string]time.Time)
addrs, err := net.InterfaceAddrs()
if err != nil {
return "", errors.Err(err)
}
for _, address := range addrs {
if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) {
ipv6Pool = append(ipv6Pool, ipnet.IP.String())
ipLastUsed[ipnet.IP.String()] = time.Now().Add(-IPCooldownPeriod)
} else if ipnet.IP.To4() != nil && govalidator.IsIPv4(ipnet.IP.String()) {
ipv4Pool = append(ipv4Pool, ipnet.IP.String())
ipLastUsed[ipnet.IP.String()] = time.Now().Add(-IPCooldownPeriod)
}
}
}
}
nextIP := ""
if ipv6 {
nextIP = getLeastUsedIP(ipv6Pool)
} else {
nextIP = getLeastUsedIP(ipv4Pool)
}
if nextIP == "" {
return "throttled", errors.Err("all IPs are throttled")
}
lastUse := ipLastUsed[nextIP]
if time.Since(lastUse) < IPCooldownPeriod {
log.Debugf("The IP %s is too hot, waiting for %.1f seconds before continuing", nextIP, (IPCooldownPeriod - time.Since(lastUse)).Seconds())
time.Sleep(IPCooldownPeriod - time.Since(lastUse))
}
ipInUse[nextIP] = true
return nextIP, nil
}
func ReleaseIP(ip string) {
ipMutex.Lock()
defer ipMutex.Unlock()
ipLastUsed[ip] = time.Now()
ipInUse[ip] = false
}
func getLeastUsedIP(ipPool []string) string {
nextIP := ""
veryLastUse := time.Now()
for _, ip := range ipPool {
isThrottled := throttledIPs[ip]
if isThrottled {
continue
}
inUse := ipInUse[ip]
if inUse {
continue
}
lastUse := ipLastUsed[ip]
if lastUse.Before(veryLastUse) {
nextIP = ip
veryLastUse = lastUse
}
}
return nextIP
}
func SetIpThrottled(ip string, stopGrp *stop.Group) {
ipMutex.Lock()
isThrottled := throttledIPs[ip]
if isThrottled {
return
}
throttledIPs[ip] = true
ipMutex.Unlock()
util.SendErrorToSlack("%s set to throttled", ip)
stopper.Add(1)
go func() {
defer stopper.Done()
unbanTimer := time.NewTimer(unbanTimeout)
select {
case <-unbanTimer.C:
ipMutex.Lock()
throttledIPs[ip] = false
ipMutex.Unlock()
util.SendInfoToSlack("%s set back to not throttled", ip)
case <-stopGrp.Ch():
unbanTimer.Stop()
}
}()
}
-216
View File
@@ -1,216 +0,0 @@
package ip_manager
import (
"net"
"sort"
"sync"
"time"
"github.com/asaskevich/govalidator"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/ytsync/util"
log "github.com/sirupsen/logrus"
)
const IPCooldownPeriod = 20 * time.Second
const unbanTimeout = 3 * time.Hour
var stopper = stop.New()
type IPPool struct {
ips []throttledIP
lock *sync.RWMutex
stopGrp *stop.Group
}
type throttledIP struct {
IP string
UsedForVideo string
LastUse time.Time
Throttled bool
InUse bool
}
var ipPoolInstance *IPPool
func GetIPPool(stopGrp *stop.Group) (*IPPool, error) {
if ipPoolInstance != nil {
return ipPoolInstance, nil
}
addrs, err := net.InterfaceAddrs()
if err != nil {
return nil, errors.Err(err)
}
var pool []throttledIP
ipv6Added := false
for _, address := range addrs {
if ipnet, ok := address.(*net.IPNet); ok && ipnet.IP.IsGlobalUnicast() {
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) && !ipv6Added {
pool = append(pool, throttledIP{
IP: ipnet.IP.String(),
LastUse: time.Now().Add(-5 * time.Minute),
})
ipv6Added = true
} else if ipnet.IP.To4() != nil && govalidator.IsIPv4(ipnet.IP.String()) {
pool = append(pool, throttledIP{
IP: ipnet.IP.String(),
LastUse: time.Now().Add(-5 * time.Minute),
})
}
}
}
ipPoolInstance = &IPPool{
ips: pool,
lock: &sync.RWMutex{},
stopGrp: stopGrp,
}
ticker := time.NewTicker(10 * time.Second)
go func() {
for {
select {
case <-stopGrp.Ch():
return
case <-ticker.C:
ipPoolInstance.lock.RLock()
for _, ip := range ipPoolInstance.ips {
log.Debugf("IP: %s\tInUse: %t\tVideoID: %s\tThrottled: %t\tLastUse: %.1f", ip.IP, ip.InUse, ip.UsedForVideo, ip.Throttled, time.Since(ip.LastUse).Seconds())
}
ipPoolInstance.lock.RUnlock()
}
}
}()
return ipPoolInstance, nil
}
// AllThrottled checks whether the IPs provided are all throttled.
// returns false if at least one IP is not throttled
// Not thread safe, should use locking when called
func AllThrottled(ips []throttledIP) bool {
for _, i := range ips {
if !i.Throttled {
return false
}
}
return true
}
// AllInUse checks whether the IPs provided are all currently in use.
// returns false if at least one IP is not in use AND is not throttled
// Not thread safe, should use locking when called
func AllInUse(ips []throttledIP) bool {
for _, i := range ips {
if !i.InUse && !i.Throttled {
return false
}
}
return true
}
func (i *IPPool) ReleaseIP(ip string) {
i.lock.Lock()
defer i.lock.Unlock()
for j, _ := range i.ips {
localIP := &i.ips[j]
if localIP.IP == ip {
localIP.InUse = false
localIP.LastUse = time.Now()
break
}
}
}
func (i *IPPool) SetThrottled(ip string) {
i.lock.Lock()
defer i.lock.Unlock()
var tIP *throttledIP
for j, _ := range i.ips {
localIP := &i.ips[j]
if localIP.IP == ip {
if localIP.Throttled {
return
}
localIP.Throttled = true
tIP = localIP
break
}
}
util.SendErrorToSlack("%s set to throttled", ip)
stopper.Add(1)
go func(tIP *throttledIP) {
defer stopper.Done()
unbanTimer := time.NewTimer(unbanTimeout)
select {
case <-unbanTimer.C:
i.lock.Lock()
tIP.Throttled = false
i.lock.Unlock()
util.SendInfoToSlack("%s set back to not throttled", ip)
case <-i.stopGrp.Ch():
unbanTimer.Stop()
}
}(tIP)
}
var ErrAllInUse = errors.Base("all IPs are in use, try again")
var ErrAllThrottled = errors.Base("all IPs are throttled")
var ErrResourceLock = errors.Base("error getting next ip, did you forget to lock on the resource?")
var ErrInterruptedByUser = errors.Base("interrupted by user")
func (i *IPPool) nextIP(forVideo string) (*throttledIP, error) {
i.lock.Lock()
defer i.lock.Unlock()
sort.Slice(i.ips, func(j, k int) bool {
return i.ips[j].LastUse.Before(i.ips[k].LastUse)
})
if !AllThrottled(i.ips) {
if AllInUse(i.ips) {
return nil, errors.Err(ErrAllInUse)
}
var nextIP *throttledIP
for j, _ := range i.ips {
ip := &i.ips[j]
if ip.InUse || ip.Throttled {
continue
}
nextIP = ip
break
}
if nextIP == nil {
return nil, errors.Err(ErrResourceLock)
}
nextIP.InUse = true
nextIP.UsedForVideo = forVideo
return nextIP, nil
}
return nil, errors.Err(ErrAllThrottled)
}
func (i *IPPool) GetIP(forVideo string) (string, error) {
for {
ip, err := i.nextIP(forVideo)
if err != nil {
if errors.Is(err, ErrAllInUse) {
select {
case <-i.stopGrp.Ch():
return "", errors.Err(ErrInterruptedByUser)
default:
time.Sleep(5 * time.Second)
continue
}
} else if errors.Is(err, ErrAllThrottled) {
return "throttled", err
}
return "", err
}
if time.Since(ip.LastUse) < IPCooldownPeriod {
log.Debugf("The IP %s is too hot, waiting for %.1f seconds before continuing", ip.IP, (IPCooldownPeriod - time.Since(ip.LastUse)).Seconds())
time.Sleep(IPCooldownPeriod - time.Since(ip.LastUse))
}
return ip.IP, nil
}
}
-40
View File
@@ -1,40 +0,0 @@
package ip_manager
import (
"testing"
)
func TestAll(t *testing.T) {
pool, err := GetIPPool()
if err != nil {
t.Fatal(err)
}
ip, err := pool.GetIP()
if err != nil {
t.Fatal(err)
}
t.Log(ip)
pool.ReleaseIP(ip)
ip2, err := pool.GetIP()
if err != nil {
t.Fatal(err)
}
if ip == ip2 && len(pool.ips) > 1 {
t.Fatalf("the same IP was returned twice! %s, %s", ip, ip2)
}
t.Log(ip2)
pool.ReleaseIP(ip2)
for range pool.ips {
_, err = pool.GetIP()
if err != nil {
t.Fatal(err)
}
}
next, err := pool.nextIP()
if err != nil {
t.Logf("%s", err.Error())
} else {
t.Fatal(next)
}
}
+1
View File
@@ -92,6 +92,7 @@ const (
TransferStateNotTouched = iota
TransferStatePending
TransferStateComplete
TransferStateFailed = -1
)
func (s *SyncManager) Start() error {
+29 -55
View File
@@ -10,10 +10,9 @@ import (
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
"github.com/lbryio/lbry.go/v2/extras/util"
logUtils "github.com/lbryio/ytsync/util"
"github.com/lbryio/ytsync/tags_manager"
"github.com/lbryio/ytsync/tagsManager"
"github.com/lbryio/ytsync/thumbs"
logUtils "github.com/lbryio/ytsync/util"
"github.com/shopspring/decimal"
log "github.com/sirupsen/logrus"
@@ -22,21 +21,14 @@ import (
)
func (s *Sync) enableAddressReuse() error {
accountsResponse, err := s.daemon.AccountList(1, 50)
accountsResponse, err := s.daemon.AccountList()
if err != nil {
return errors.Err(err)
}
accounts := make([]jsonrpc.Account, 0, len(accountsResponse.Items))
ledger := "lbc_mainnet"
accounts := accountsResponse.LBCMainnet
if logUtils.IsRegTest() {
ledger = "lbc_regtest"
accounts = accountsResponse.LBCRegtest
}
for _, a := range accountsResponse.Items {
if *a.Ledger == ledger {
accounts = append(accounts, a)
}
}
for _, a := range accounts {
_, err = s.daemon.AccountSet(a.ID, jsonrpc.AccountSettings{
ChangeMaxUses: util.PtrToInt(1000),
@@ -128,13 +120,13 @@ func (s *Sync) walletSetup() error {
}
}
claimAddress, err := s.daemon.AddressList(nil, nil, 1, 20)
claimAddress, err := s.daemon.AddressList(nil, nil)
if err != nil {
return err
} else if claimAddress == nil {
return errors.Err("could not get an address")
return errors.Err("could not get unused address")
}
s.claimAddress = string(claimAddress.Items[0].Address)
s.claimAddress = string((*claimAddress)[0].Address)
if s.claimAddress == "" {
return errors.Err("found blank claim address")
}
@@ -152,23 +144,20 @@ func (s *Sync) walletSetup() error {
func (s *Sync) getDefaultAccount() (string, error) {
if s.defaultAccountID == "" {
accountsResponse, err := s.daemon.AccountList(1, 50)
accounts, err := s.daemon.AccountList()
if err != nil {
return "", errors.Err(err)
}
ledger := "lbc_mainnet"
accountsNet := (*accounts).LBCMainnet
if logUtils.IsRegTest() {
ledger = "lbc_regtest"
accountsNet = (*accounts).LBCRegtest
}
for _, a := range accountsResponse.Items {
if *a.Ledger == ledger {
if a.IsDefault {
s.defaultAccountID = a.ID
break
}
for _, account := range accountsNet {
if account.IsDefault {
s.defaultAccountID = account.ID
break
}
}
if s.defaultAccountID == "" {
return "", errors.Err("No default account found")
}
@@ -182,7 +171,7 @@ func (s *Sync) ensureEnoughUTXOs() error {
return err
}
utxolist, err := s.daemon.UTXOList(&defaultAccount, 1, 10000)
utxolist, err := s.daemon.UTXOList(&defaultAccount)
if err != nil {
return err
} else if utxolist == nil {
@@ -194,7 +183,7 @@ func (s *Sync) ensureEnoughUTXOs() error {
count := 0
confirmedCount := 0
for _, utxo := range utxolist.Items {
for _, utxo := range *utxolist {
amount, _ := strconv.ParseFloat(utxo.Amount, 64)
if utxo.IsMine && utxo.Type == "payment" && amount > 0.001 {
if utxo.Confirmations > 0 {
@@ -250,11 +239,21 @@ func (s *Sync) ensureEnoughUTXOs() error {
}
func (s *Sync) waitForNewBlock() error {
log.Printf("regtest: %t, docker: %t", logUtils.IsRegTest(), logUtils.IsUsingDocker())
if logUtils.IsRegTest() && logUtils.IsUsingDocker() {
lbrycrd, err := logUtils.GetLbrycrdClient(s.LbrycrdString)
if err != nil {
return errors.Prefix("error getting lbrycrd client: ", err)
}
txs, err := lbrycrd.Generate(1)
for _, tx := range txs {
log.Info("Generated tx: ", tx.String())
}
}
status, err := s.daemon.Status()
if err != nil {
return err
}
for status.Wallet.Blocks == 0 || status.Wallet.BlocksBehind != 0 {
time.Sleep(5 * time.Second)
status, err = s.daemon.Status()
@@ -262,18 +261,11 @@ func (s *Sync) waitForNewBlock() error {
return err
}
}
currentBlock := status.Wallet.Blocks
for i := 0; status.Wallet.Blocks <= currentBlock; i++ {
if i%3 == 0 {
log.Printf("Waiting for new block (%d)...", currentBlock+1)
}
if logUtils.IsRegTest() && logUtils.IsUsingDocker() {
err = s.GenerateRegtestBlock()
if err != nil {
return err
}
}
time.Sleep(10 * time.Second)
status, err = s.daemon.Status()
if err != nil {
@@ -283,21 +275,6 @@ func (s *Sync) waitForNewBlock() error {
return nil
}
func (s *Sync) GenerateRegtestBlock() error {
lbrycrd, err := logUtils.GetLbrycrdClient(s.LbrycrdString)
if err != nil {
return errors.Prefix("error getting lbrycrd client: ", err)
}
txs, err := lbrycrd.Generate(1)
if err != nil {
return errors.Prefix("error generating new block: ", err)
}
for _, tx := range txs {
log.Info("Generated tx: ", tx.String())
}
return nil
}
func (s *Sync) ensureChannelOwnership() error {
if s.LbryChannelName == "" {
return errors.Err("no channel name set")
@@ -399,9 +376,6 @@ func (s *Sync) ensureChannelOwnership() error {
var languages []string = nil
if channelInfo.DefaultLanguage != "" {
if channelInfo.DefaultLanguage == "iw" {
channelInfo.DefaultLanguage = "he"
}
languages = []string{channelInfo.DefaultLanguage}
}
var locations []jsonrpc.Location = nil
@@ -412,7 +386,7 @@ func (s *Sync) ensureChannelOwnership() error {
claimCreateOptions := jsonrpc.ClaimCreateOptions{
Title: &channelInfo.Title,
Description: &channelInfo.Description,
Tags: tags_manager.GetTagsForChannel(s.YoutubeChannelID),
Tags: tagsManager.GetTagsForChannel(s.YoutubeChannelID),
Languages: languages,
Locations: locations,
ThumbnailURL: &thumbnailURL,
+63 -67
View File
@@ -6,6 +6,8 @@ import (
"strings"
"time"
"github.com/uber-go/atomic"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
"github.com/lbryio/lbry.go/v2/extras/stop"
@@ -24,14 +26,14 @@ func waitConfirmations(s *Sync) error {
allConfirmed := false
waiting:
for !allConfirmed {
utxolist, err := s.daemon.UTXOList(&defaultAccount, 1, 10000)
utxolist, err := s.daemon.UTXOList(&defaultAccount)
if err != nil {
return err
} else if utxolist == nil {
return errors.Err("no response")
}
for _, utxo := range utxolist.Items {
for _, utxo := range *utxolist {
if utxo.Confirmations <= 0 {
err = s.waitForNewBlock()
if err != nil {
@@ -45,13 +47,13 @@ waiting:
return nil
}
type abandonResponse struct {
ClaimID string
Error error
Amount float64
}
func abandonSupports(s *Sync) (float64, error) {
type abandonResponse struct {
ClaimID string
Error error
Amount float64
}
totalPages := uint64(1)
var allSupports []jsonrpc.Claim
defaultAccount, err := s.getDefaultAccount()
@@ -66,23 +68,25 @@ func abandonSupports(s *Sync) (float64, error) {
allSupports = append(allSupports, (*supports).Items...)
totalPages = (*supports).TotalPages
}
producerWG := &stop.Group{}
claimIDChan := make(chan string, len(allSupports))
abandonRspChan := make(chan abandonResponse, len(allSupports))
claimIDChan := make(chan string)
abandonRspChan := make(chan abandonResponse)
collectorChan := make(chan bool)
alreadyAbandoned := make(map[string]bool, len(allSupports))
producerWG.Add(1)
totalAbandoned := atomic.NewFloat64(0)
go func() {
defer producerWG.Done()
for _, support := range allSupports {
_, ok := alreadyAbandoned[support.ClaimID]
if ok {
for r := range abandonRspChan {
if r.Error != nil {
log.Errorf("Failed abandoning supports for %s: %s", r.ClaimID, r.Error.Error())
continue
}
alreadyAbandoned[support.ClaimID] = true
claimIDChan <- support.ClaimID
totalAbandoned.Add(r.Amount)
}
close(collectorChan)
}()
consumerWG := &stop.Group{}
//TODO: remove this once the SDK team fixes their RPC bugs....
s.daemon.SetRPCTimeout(30 * time.Second)
@@ -95,72 +99,64 @@ func abandonSupports(s *Sync) (float64, error) {
claimID, more := <-claimIDChan
if !more {
return
} else {
summary, err := s.daemon.SupportAbandon(&claimID, nil, nil, nil, &defaultAccount)
if err != nil {
if strings.Contains(err.Error(), "Client.Timeout exceeded while awaiting headers") {
log.Errorf("Support abandon for %s timed out, retrying...", claimID)
summary, err = s.daemon.SupportAbandon(&claimID, nil, nil, nil, &defaultAccount)
if err != nil {
//TODO GUESS HOW MUCH LBC WAS RELEASED THAT WE DON'T KNOW ABOUT, because screw you SDK
abandonRspChan <- abandonResponse{
ClaimID: claimID,
Error: err,
Amount: 0, // this is likely wrong, but oh well... there is literally nothing I can do about it
}
continue
}
} else {
}
summary, err := s.daemon.SupportAbandon(&claimID, nil, nil, nil, nil)
if err != nil {
if strings.Contains(err.Error(), "Client.Timeout exceeded while awaiting headers") {
log.Errorf("Support abandon for %s timed out, retrying...", claimID)
summary, err = s.daemon.SupportAbandon(&claimID, nil, nil, nil, nil)
if err != nil {
//TODO GUESS HOW MUCH LBC WAS RELEASED THAT WE DON'T KNOW ABOUT, because screw you SDK
abandonRspChan <- abandonResponse{
ClaimID: claimID,
Error: err,
Amount: 0,
Amount: 0, // this is likely wrong, but oh well... there is literally nothing I can do about it
}
continue
}
}
if len(summary.Outputs) < 1 {
abandonRspChan <- abandonResponse{
ClaimID: claimID,
Error: errors.Err("error abandoning supports: no outputs while abandoning %s", claimID),
Amount: 0,
}
} else {
abandonRspChan <- abandonResponse{ClaimID: claimID, Error: err}
continue
}
outputAmount, err := strconv.ParseFloat(summary.Outputs[0].Amount, 64)
if err != nil {
abandonRspChan <- abandonResponse{
ClaimID: claimID,
Error: errors.Err(err),
Amount: 0,
}
continue
}
log.Infof("Abandoned supports of %.4f LBC for claim %s", outputAmount, claimID)
}
if len(summary.Outputs) < 1 {
abandonRspChan <- abandonResponse{
ClaimID: claimID,
Error: nil,
Amount: outputAmount,
Error: errors.Err("abandoning supports: no outputs for %s", claimID),
}
continue
}
outputAmount, err := strconv.ParseFloat(summary.Outputs[0].Amount, 64)
if err != nil {
abandonRspChan <- abandonResponse{ClaimID: claimID, Error: errors.Err(err)}
continue
}
log.Infof("Abandoned supports of %.4f LBC for claim %s", outputAmount, claimID)
abandonRspChan <- abandonResponse{ClaimID: claimID, Amount: outputAmount}
continue
}
}()
}
producerWG.Wait()
close(claimIDChan)
consumerWG.Wait()
close(abandonRspChan)
totalAbandoned := 0.0
for r := range abandonRspChan {
if r.Error != nil {
log.Errorf("Failed abandoning supports for %s: %s", r.ClaimID, r.Error.Error())
for _, support := range allSupports {
_, ok := alreadyAbandoned[support.ClaimID]
if ok {
continue
}
totalAbandoned += r.Amount
alreadyAbandoned[support.ClaimID] = true
claimIDChan <- support.ClaimID
}
return totalAbandoned, nil
close(claimIDChan)
consumerWG.Wait()
close(abandonRspChan)
<-collectorChan
return totalAbandoned.Load(), nil
}
type updateInfo struct {
@@ -177,7 +173,7 @@ func transferVideos(s *Sync) error {
if err != nil {
return err
}
streams, err := s.daemon.StreamList(&account, 1, 30000)
streams, err := s.daemon.StreamList(&account)
if err != nil {
return errors.Err(err)
}
@@ -191,8 +187,8 @@ func transferVideos(s *Sync) error {
}
var stream *jsonrpc.Claim = nil
for _, c := range streams.Items {
if c.ClaimID != video.ClaimID || (c.SigningChannel != nil && c.SigningChannel.ClaimID != s.lbryChannelID) {
for _, c := range *streams {
if c.ClaimID != video.ClaimID {
continue
}
stream = &c
+29 -58
View File
@@ -14,7 +14,6 @@ import (
"syscall"
"time"
"github.com/lbryio/ytsync/ip_manager"
"github.com/lbryio/ytsync/namer"
"github.com/lbryio/ytsync/sdk"
"github.com/lbryio/ytsync/sources"
@@ -334,7 +333,7 @@ func (s *Sync) processTransfers() (e error) {
return errors.Prefix(fmt.Sprintf("%.6f LBCs were abandoned before failing", supportAmount), err)
}
if supportAmount > 0 {
logUtils.SendInfoToSlack("(%s) %.6f LBCs were abandoned and should be used as support", s.YoutubeChannelID, supportAmount)
logUtils.SendInfoToSlack("%.6f LBCs were abandoned and should be used as support", supportAmount)
}
err = transferVideos(s)
if err != nil {
@@ -355,7 +354,7 @@ func (s *Sync) processTransfers() (e error) {
return err
}
isTip := true
summary, err := s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount), &isTip, nil, []string{defaultAccount}, nil)
summary, err := s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount), &isTip, nil, []string{defaultAccount})
if err != nil {
return errors.Err(err)
}
@@ -385,7 +384,9 @@ func (s *Sync) setChannelTerminationStatus(e *error) {
var transferState *int
if s.shouldTransfer() {
if *e == nil {
if *e != nil {
transferState = util.PtrToInt(TransferStateFailed)
} else {
transferState = util.PtrToInt(TransferStateComplete)
}
}
@@ -467,7 +468,7 @@ var thumbnailHosts = []string{
thumbs.ThumbnailEndpoint,
}
func isYtsyncClaim(c jsonrpc.Claim, expectedChannelID string) bool {
func isYtsyncClaim(c jsonrpc.Claim) bool {
if !util.InSlice(c.Type, []string{"claim", "update"}) || c.Value.GetStream() == nil {
return false
}
@@ -475,12 +476,7 @@ func isYtsyncClaim(c jsonrpc.Claim, expectedChannelID string) bool {
//most likely a claim created outside of ytsync, ignore!
return false
}
if c.SigningChannel == nil {
return false
}
if c.SigningChannel.ClaimID != expectedChannelID {
return false
}
for _, th := range thumbnailHosts {
if strings.Contains(c.Value.GetThumbnail().GetUrl(), th) {
return true
@@ -494,7 +490,7 @@ func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) {
abandonedClaims := false
videoIDs := make(map[string]jsonrpc.Claim)
for _, c := range claims {
if !isYtsyncClaim(c, s.lbryChannelID) {
if !isYtsyncClaim(c) {
continue
}
tn := c.Value.GetThumbnail().GetUrl()
@@ -541,7 +537,7 @@ type ytsyncClaim struct {
func (s *Sync) mapFromClaims(claims []jsonrpc.Claim) map[string]ytsyncClaim {
videoIDMap := make(map[string]ytsyncClaim, len(claims))
for _, c := range claims {
if !isYtsyncClaim(c, s.lbryChannelID) {
if !isYtsyncClaim(c) {
continue
}
tn := c.Value.GetThumbnail().GetUrl()
@@ -638,17 +634,8 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
if sv.Transferred {
_, ok := allClaimsInfo[vID]
if !ok && sv.Published {
searchResponse, err := s.daemon.ClaimSearch(nil, &sv.ClaimID, nil, nil, 1, 20)
if err != nil {
log.Error(err.Error())
continue
}
if len(searchResponse.Claims) == 0 {
log.Debugf("%s: was transferred but appears abandoned! we should ignore this - claimID: %s", vID, sv.ClaimID)
continue //TODO: we should flag these on the db
} else {
return count, fixed, 0, errors.Err("%s: isn't our control but is on the database and on the blockchain. wtf is up? ClaimID: %s", vID, sv.ClaimID)
}
log.Warnf("%s: claims to be published and transferred but wasn't found in the list of claims", vID)
idsToRemove = append(idsToRemove, vID)
}
continue
}
@@ -685,17 +672,12 @@ func (s *Sync) getClaims(defaultOnly bool) ([]jsonrpc.Claim, error) {
}
account = &a
}
claims, err := s.daemon.StreamList(account, 1, 30000)
claims, err := s.daemon.StreamList(account)
if err != nil {
return nil, errors.Prefix("cannot list claims", err)
}
items := make([]jsonrpc.Claim, 0, len(claims.Items))
for _, c := range claims.Items {
if c.SigningChannel != nil && c.SigningChannel.ClaimID == s.lbryChannelID {
items = append(items, c)
}
}
return items, nil
return *claims, nil
}
func (s *Sync) checkIntegrity() error {
@@ -842,9 +824,9 @@ func (s *Sync) startWorker(workerNum int) {
err := s.processVideo(v)
if err != nil {
logMsg := fmt.Sprintf("error processing video %s: %s", v.ID(), err.Error())
logMsg := fmt.Sprintf("error processing video %s: %s", v.ID(), errors.FullTrace(err))
log.Errorln(logMsg)
if strings.Contains(strings.ToLower(err.Error()), "interrupted by user") {
if errors.Is(err, sources.ErrInterruptedByUser) {
return
}
fatalErrors := []string{
@@ -881,16 +863,12 @@ func (s *Sync) startWorker(workerNum int) {
"giving up after 0 fragment retries",
"Sorry about that",
"This video is not available",
"requested format not available",
"interrupted by user",
}
if util.SubstringInSlice(err.Error(), errorsNoRetry) {
log.Println("This error should not be retried at all")
} else if tryCount < s.MaxTries {
if util.SubstringInSlice(err.Error(), []string{
"txn-mempool-conflict",
"too-long-mempool-chain",
}) {
if strings.Contains(err.Error(), "txn-mempool-conflict") ||
strings.Contains(err.Error(), "too-long-mempool-chain") {
log.Println("waiting for a block before retrying")
err := s.waitForNewBlock()
if err != nil {
@@ -902,7 +880,6 @@ func (s *Sync) startWorker(workerNum int) {
"Not enough funds to cover this transaction",
"failed: Not enough funds",
"Error in daemon: Insufficient funds, please deposit additional LBC",
"Missing inputs",
}) {
log.Println("checking funds and UTXOs before retrying...")
err := s.walletSetup()
@@ -988,10 +965,6 @@ func (s *Sync) enqueueYoutubeVideos() error {
}
var videos []video
ipPool, err := ip_manager.GetIPPool(s.grp)
if err != nil {
return err
}
playlistMap := make(map[string]*youtube.PlaylistItemSnippet, 50)
nextPageToken := ""
for {
@@ -1029,7 +1002,7 @@ func (s *Sync) enqueueYoutubeVideos() error {
return errors.Prefix("error getting videos info", err)
}
for _, item := range videosListResponse.Items {
videos = append(videos, sources.NewYoutubeVideo(s.videoDirectory, item, playlistMap[item.Id].Position, s.Manager.GetS3AWSConfig(), s.grp, ipPool))
videos = append(videos, sources.NewYoutubeVideo(s.videoDirectory, item, playlistMap[item.Id].Position, s.Manager.GetS3AWSConfig(), s.grp))
}
log.Infof("Got info for %d videos from youtube API", len(videos))
@@ -1045,7 +1018,7 @@ func (s *Sync) enqueueYoutubeVideos() error {
}
_, ok := playlistMap[k]
if !ok {
videos = append(videos, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig(), s.grp, ipPool))
videos = append(videos, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig(), s.grp))
}
}
@@ -1163,19 +1136,17 @@ func (s *Sync) processVideo(v video) (err error) {
func (s *Sync) importPublicKey() error {
if s.publicKey != "" {
accountsResponse, err := s.daemon.AccountList(1, 50)
accountsResponse, err := s.daemon.AccountList()
if err != nil {
return errors.Err(err)
}
ledger := "lbc_mainnet"
accounts := accountsResponse.LBCMainnet
if logUtils.IsRegTest() {
ledger = "lbc_regtest"
accounts = accountsResponse.LBCRegtest
}
for _, a := range accountsResponse.Items {
if *a.Ledger == ledger {
if a.PublicKey == s.publicKey {
return nil
}
for _, a := range accounts {
if a.PublicKey == s.publicKey {
return nil
}
}
log.Infof("Could not find public key %s in the wallet. Importing it...", s.publicKey)
@@ -1203,12 +1174,12 @@ func (s *Sync) getUnsentSupports() (float64, error) {
if err != nil {
return 0, errors.Err(err)
}
transactionList, err := s.daemon.TransactionList(&defaultAccount, 1, 90000)
transactionList, err := s.daemon.TransactionList(&defaultAccount)
if err != nil {
return 0, errors.Err(err)
}
sentSupports := 0.0
for _, t := range transactionList.Items {
for _, t := range *transactionList {
if len(t.SupportInfo) == 0 {
continue
}
@@ -1223,7 +1194,7 @@ func (s *Sync) getUnsentSupports() (float64, error) {
}
}
if balanceAmount > 10 && sentSupports < 1 {
logUtils.SendErrorToSlack("(%s) this channel has quite some LBCs in it (%.2f) and %.2f LBC in sent tips, it's likely that the tips weren't actually sent or the wallet has unnecessary extra credits in it", s.YoutubeChannelID, balanceAmount, sentSupports)
logUtils.SendErrorToSlack("this channel has quite some LBCs in it (%.2f) and %.2f LBC in sent tips, it's likely that the tips weren't actually sent or the wallet has unnecessary extra credits in it", balanceAmount, sentSupports)
return balanceAmount - 10, nil
}
}
+6 -6
View File
@@ -86,7 +86,7 @@ func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeC
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode != http.StatusOK {
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
log.Debugln(string(body))
log.Debugln(body)
time.Sleep(30 * time.Second)
return a.FetchChannels(status, cp)
}
@@ -185,9 +185,9 @@ func (a *APIConfig) SetChannelStatus(channelID string, status string, failureRea
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode >= http.StatusInternalServerError {
if res.StatusCode != http.StatusOK {
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
log.Debugln(string(body))
log.Debugln(body)
time.Sleep(30 * time.Second)
return a.SetChannelStatus(channelID, status, failureReason, transferState)
}
@@ -232,7 +232,7 @@ func (a *APIConfig) SetChannelClaimID(channelID string, channelClaimID string) e
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode != http.StatusOK {
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
log.Debugln(string(body))
log.Debugln(body)
time.Sleep(30 * time.Second)
return a.SetChannelClaimID(channelID, channelClaimID)
}
@@ -271,7 +271,7 @@ func (a *APIConfig) DeleteVideos(videos []string) error {
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode != http.StatusOK {
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
log.Debugln(string(body))
log.Debugln(body)
time.Sleep(30 * time.Second)
return a.DeleteVideos(videos)
}
@@ -344,7 +344,7 @@ func (a *APIConfig) MarkVideoStatus(status VideoStatus) error {
body, _ := ioutil.ReadAll(res.Body)
if res.StatusCode != http.StatusOK {
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
log.Debugln(string(body))
log.Debugln(body)
time.Sleep(30 * time.Second)
return a.MarkVideoStatus(status)
}
+44 -38
View File
@@ -18,10 +18,10 @@ import (
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/lbryio/ytsync/ip_manager"
"github.com/lbryio/ytsync/ipManager"
"github.com/lbryio/ytsync/namer"
"github.com/lbryio/ytsync/sdk"
"github.com/lbryio/ytsync/tags_manager"
"github.com/lbryio/ytsync/tagsManager"
"github.com/lbryio/ytsync/thumbs"
duration "github.com/ChannelMeter/iso8601duration"
@@ -31,6 +31,8 @@ import (
"google.golang.org/api/youtube/v3"
)
var ErrInterruptedByUser = errors.Base("interrupted by user")
type YoutubeVideo struct {
id string
title string
@@ -50,7 +52,6 @@ type YoutubeVideo struct {
mocked bool
walletLock *sync.RWMutex
stopGroup *stop.Group
pool *ip_manager.IPPool
}
var youtubeCategories = map[string]string{
@@ -88,7 +89,7 @@ var youtubeCategories = map[string]string{
"44": "trailers",
}
func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo {
func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group) *YoutubeVideo {
publishedAt, _ := time.Parse(time.RFC3339Nano, videoData.Snippet.PublishedAt) // ignore parse errors
return &YoutubeVideo{
id: videoData.Id,
@@ -102,10 +103,9 @@ func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPositio
mocked: false,
youtubeChannelID: videoData.Snippet.ChannelId,
stopGroup: stopGroup,
pool: pool,
}
}
func NewMockedVideo(directory string, videoID string, youtubeChannelID string, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo {
func NewMockedVideo(directory string, videoID string, youtubeChannelID string, awsConfig aws.Config, stopGroup *stop.Group) *YoutubeVideo {
return &YoutubeVideo{
id: videoID,
playlistPosition: 0,
@@ -114,7 +114,6 @@ func NewMockedVideo(directory string, videoID string, youtubeChannelID string, a
mocked: true,
youtubeChannelID: youtubeChannelID,
stopGroup: stopGroup,
pool: pool,
}
}
@@ -178,7 +177,7 @@ func (v *YoutubeVideo) getAbbrevDescription() string {
return description + "\n..." + additionalDescription
}
func (v *YoutubeVideo) download() error {
func (v *YoutubeVideo) download(useIPv6 bool) error {
videoPath := v.getFullPath()
err := os.Mkdir(v.videoDir(), 0777)
@@ -223,36 +222,43 @@ func (v *YoutubeVideo) download() error {
fmt.Sprintf("duration <= %d", int(math.Round(v.maxVideoLength*3600))),
)
}
sourceAddress, err := v.pool.GetIP(v.id)
sourceAddress, err := ipManager.GetNextIP(useIPv6)
if err != nil {
if sourceAddress == "throttled" {
for {
select {
case <-v.stopGroup.Ch():
return errors.Err("interrupted by user")
return errors.Err(ErrInterruptedByUser)
default:
}
time.Sleep(ip_manager.IPCooldownPeriod)
sourceAddress, err = v.pool.GetIP(v.id)
time.Sleep(ipManager.IPCooldownPeriod)
sourceAddress, err = ipManager.GetNextIP(useIPv6)
if err == nil {
break
} else if !errors.Is(err, ip_manager.ErrAllThrottled) {
return err
}
}
} else {
return errors.Err(err)
}
}
defer v.pool.ReleaseIP(sourceAddress)
ytdlArgs = append(ytdlArgs,
"--source-address",
sourceAddress,
"https://www.youtube.com/watch?v="+v.ID(),
)
defer ipManager.ReleaseIP(sourceAddress)
if useIPv6 {
log.Infof("using IPv6: %s", sourceAddress)
ytdlArgs = append(ytdlArgs,
"-6",
"--source-address",
sourceAddress,
)
} else {
log.Infof("using IPv4: %s", sourceAddress)
ytdlArgs = append(ytdlArgs,
"-4",
"--source-address",
sourceAddress,
)
}
ytdlArgs = append(ytdlArgs, "https://www.youtube.com/watch?v="+v.ID())
runcmd:
argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][height<="+qualities[qualityIndex]+"]+bestaudio[ext!=webm]")
cmd := exec.Command("youtube-dl", argsWithFilters...)
@@ -277,11 +283,11 @@ runcmd:
if err = cmd.Wait(); err != nil {
if strings.Contains(err.Error(), "exit status 1") {
if strings.Contains(string(errorLog), "HTTP Error 429") || strings.Contains(string(errorLog), "returned non-zero exit status 8") {
v.pool.SetThrottled(sourceAddress)
if strings.Contains(string(errorLog), "HTTP Error 429") {
ipManager.SetIpThrottled(sourceAddress, v.stopGroup)
} else if strings.Contains(string(errorLog), "giving up after 0 fragment retries") && qualityIndex < len(qualities)-1 {
qualityIndex++
goto runcmd //this bypasses the yt throttling IP redistribution... TODO: don't
goto runcmd
}
return errors.Err(string(errorLog))
}
@@ -426,18 +432,21 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV
return v.downloadAndPublish(daemon, params)
}
var isThrottled bool
func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
var err error
for {
err = v.download()
if err != nil && strings.Contains(err.Error(), "HTTP Error 429") {
continue
} else if err != nil {
err := v.download(isThrottled)
if err != nil {
if strings.Contains(err.Error(), "HTTP Error 429") && !isThrottled {
isThrottled = true
err = v.download(isThrottled)
if err != nil {
return nil, errors.Prefix("download error", err)
}
} else {
return nil, errors.Prefix("download error", err)
}
break
}
log.Debugln("Downloaded " + v.id)
err = v.triggerThumbnailSave()
@@ -459,9 +468,6 @@ func (v *YoutubeVideo) getMetadata() (languages []string, locations []jsonrpc.Lo
tags = nil
if !v.mocked {
if v.youtubeInfo.Snippet.DefaultLanguage != "" {
if v.youtubeInfo.Snippet.DefaultLanguage == "iw" {
v.youtubeInfo.Snippet.DefaultLanguage = "he"
}
languages = []string{v.youtubeInfo.Snippet.DefaultLanguage}
}
@@ -473,7 +479,7 @@ func (v *YoutubeVideo) getMetadata() (languages []string, locations []jsonrpc.Lo
}
tags = v.youtubeInfo.Snippet.Tags
}
tags, err := tags_manager.SanitizeTags(tags, v.youtubeChannelID)
tags, err := tagsManager.SanitizeTags(tags, v.youtubeChannelID)
if err != nil {
log.Errorln(err.Error())
}
@@ -485,7 +491,7 @@ func (v *YoutubeVideo) getMetadata() (languages []string, locations []jsonrpc.Lo
}
func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo) (*SyncSummary, error) {
c, err := daemon.ClaimSearch(nil, &existingVideoData.ClaimID, nil, nil, 1, 20)
c, err := daemon.ClaimSearch(nil, &existingVideoData.ClaimID, nil, nil)
if err != nil {
return nil, errors.Err(err)
}
@@ -1,4 +1,4 @@
package tags_manager
package tagsManager
import (
"regexp"
@@ -1,4 +1,4 @@
package tags_manager
package tagsManager
import (
"fmt"
+5 -6
View File
@@ -61,12 +61,11 @@ func (u *thumbnailUploader) uploadThumbnail() error {
uploader := s3manager.NewUploader(s3Session)
_, err = uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String("thumbnails.lbry.com"),
Key: key,
Body: thumb,
ACL: aws.String("public-read"),
ContentType: aws.String("image/jpeg"),
CacheControl: aws.String("public, max-age=2592000"),
Bucket: aws.String("thumbnails.lbry.com"),
Key: key,
Body: thumb,
ACL: aws.String("public-read"),
ContentType: aws.String("image/jpeg"),
})
u.mirroredUrl = ThumbnailEndpoint + u.name
return errors.Err(err)