Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
62a1a0eed5 | ||
|
|
93f9185c71 | ||
|
|
ef62242f42 | ||
|
|
4a356e86da | ||
|
|
ac95ca57af | ||
|
|
c28d5a716b | ||
|
|
59471f9c26 | ||
|
|
8028c6621c | ||
|
|
8abf6ad255 | ||
|
|
1f9a645c57 | ||
|
|
a5657aaf46 | ||
|
|
b6861dae9b | ||
|
|
cff147bb95 | ||
|
|
54d04dcf2c | ||
|
|
65c4f99c1c | ||
|
|
68c0fd9ed7 | ||
|
|
2bde06e4b9 | ||
|
|
5df6db6e96 | ||
|
|
62dfdc1adb | ||
|
|
fe55304184 | ||
|
|
bb76500f04 | ||
|
|
4e2efec66e | ||
|
|
8d650e032c | ||
|
|
f5d97957e7 | ||
|
|
36f539ef5d | ||
|
|
77461f71af | ||
|
|
caca13de61 |
No files matched your search
+8
-8
@@ -9,12 +9,13 @@ builds:
|
||||
- amd64
|
||||
ldflags:
|
||||
- -X "{{ .Env.IMPORTPATH }}/meta.semVersion={{ .Tag }}" -X "{{ .Env.IMPORTPATH }}/meta.version={{ .Env.VERSIONSHORT }}" -X "{{ .Env.IMPORTPATH }}/meta.versionLong={{ .Env.VERSIONLONG }}" -X "{{ .Env.IMPORTPATH }}/meta.commitMsg={{ .Env.COMMITMSG }}"
|
||||
archive:
|
||||
name_template: '{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}'
|
||||
replacements:
|
||||
linux: Linux
|
||||
amd64: x86_64
|
||||
format: zip
|
||||
archives:
|
||||
- id: zip
|
||||
name_template: '{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}'
|
||||
replacements:
|
||||
linux: Linux
|
||||
amd64: x86_64
|
||||
format: zip
|
||||
checksum:
|
||||
name_template: 'checksums.txt'
|
||||
snapshot:
|
||||
@@ -24,5 +25,4 @@ changelog:
|
||||
filters:
|
||||
exclude:
|
||||
- '^docs:'
|
||||
- '^test:'
|
||||
|
||||
- '^test:'
|
||||
@@ -0,0 +1,111 @@
|
||||
From 30380338ba9af01696c94b61f0597131638eaec1 Mon Sep 17 00:00:00 2001
|
||||
From: Niko Storni <niko@lbry.io>
|
||||
Date: Mon, 16 Dec 2019 00:13:36 +0100
|
||||
Subject: [PATCH] lbry-patch
|
||||
|
||||
---
|
||||
youtube_dl/extractor/youtube.py | 45 +++++++++++++++++++++++++--------
|
||||
1 file changed, 35 insertions(+), 10 deletions(-)
|
||||
|
||||
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
|
||||
index b913d07a6..cd66a5b01 100644
|
||||
--- a/youtube_dl/extractor/youtube.py
|
||||
+++ b/youtube_dl/extractor/youtube.py
|
||||
@@ -10,6 +10,7 @@ import random
|
||||
import re
|
||||
import time
|
||||
import traceback
|
||||
+import subprocess
|
||||
|
||||
from .common import InfoExtractor, SearchInfoExtractor
|
||||
from ..jsinterp import JSInterpreter
|
||||
@@ -536,6 +537,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
|
||||
_GEO_BYPASS = False
|
||||
|
||||
+ _WGET_429_RATE_LIMIT = 8191
|
||||
+ _WGET_BINARY = "wget"
|
||||
+
|
||||
IE_NAME = 'youtube'
|
||||
_TESTS = [
|
||||
{
|
||||
@@ -1254,6 +1258,17 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
""" Return a string representation of a signature """
|
||||
return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
|
||||
|
||||
+ def _rate_limit_download(self, url, video_id, note=None):
|
||||
+ if note is None:
|
||||
+ self.report_download_webpage(video_id)
|
||||
+ elif note is not False:
|
||||
+ if video_id is None:
|
||||
+ self.to_screen('%s' % (note,))
|
||||
+ else:
|
||||
+ self.to_screen('%s: %s' % (video_id, note))
|
||||
+ source_address = self._downloader.params.get('source_address')
|
||||
+ return subprocess.run([self._WGET_BINARY, '-q', '--limit-rate', str(self._WGET_429_RATE_LIMIT), '--bind-address', source_address, '-O', '-', url], check=True, stdout=subprocess.PIPE).stdout.decode(encoding='UTF-8')
|
||||
+
|
||||
def _extract_signature_function(self, video_id, player_url, example_sig):
|
||||
id_m = re.match(
|
||||
r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player(?:-new)?|(?:/[a-z]{2,3}_[A-Z]{2})?/base)?\.(?P<ext>[a-z]+)$',
|
||||
@@ -1678,7 +1693,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
|
||||
# Get video webpage
|
||||
url = proto + '://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1&bpctr=9999999999' % video_id
|
||||
- video_webpage = self._download_webpage(url, video_id)
|
||||
+ video_webpage = self._rate_limit_download(url, video_id)
|
||||
|
||||
# Attempt to extract SWF player URL
|
||||
mobj = re.search(r'swfConfig.*?"(https?:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
|
||||
@@ -1736,10 +1751,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
r'"sts"\s*:\s*(\d+)', embed_webpage, 'sts', default=''),
|
||||
})
|
||||
video_info_url = proto + '://www.youtube.com/get_video_info?' + data
|
||||
- video_info_webpage = self._download_webpage(
|
||||
+ video_info_webpage = self._rate_limit_download(
|
||||
video_info_url, video_id,
|
||||
- note='Refetching age-gated info webpage',
|
||||
- errnote='unable to download video info webpage')
|
||||
+ note='Refetching age-gated info webpage')
|
||||
video_info = compat_parse_qs(video_info_webpage)
|
||||
pl_response = video_info.get('player_response', [None])[0]
|
||||
player_response = extract_player_response(pl_response, video_id)
|
||||
@@ -1777,7 +1791,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
# The general idea is to take a union of itags of both DASH manifests (for example
|
||||
# video with such 'manifest behavior' see https://github.com/ytdl-org/youtube-dl/issues/6093)
|
||||
self.report_video_info_webpage_download(video_id)
|
||||
- for el in ('embedded', 'detailpage', 'vevo', ''):
|
||||
+ for el in ('', 'embedded', 'detailpage', 'vevo'):
|
||||
query = {
|
||||
'video_id': video_id,
|
||||
'ps': 'default',
|
||||
@@ -1789,11 +1803,22 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
query['el'] = el
|
||||
if sts:
|
||||
query['sts'] = sts
|
||||
- video_info_webpage = self._download_webpage(
|
||||
- '%s://www.youtube.com/get_video_info' % proto,
|
||||
- video_id, note=False,
|
||||
- errnote='unable to download video info webpage',
|
||||
- fatal=False, query=query)
|
||||
+
|
||||
+ if el == '':
|
||||
+ base_url = 'https://youtube.com/get_video_info?video_id={}'.format(video_id)
|
||||
+ else:
|
||||
+ base_url = 'https://youtube.com/get_video_info'
|
||||
+
|
||||
+ for q in query:
|
||||
+ if q is None or q is "":
|
||||
+ continue
|
||||
+ if query[q] is None or query[q] is "":
|
||||
+ continue
|
||||
+
|
||||
+ base_url = base_url + "?{}={}".format(q, query[q])
|
||||
+
|
||||
+ video_info_webpage = self._rate_limit_download(base_url, video_id)
|
||||
+
|
||||
if not video_info_webpage:
|
||||
continue
|
||||
get_video_info = compat_parse_qs(video_info_webpage)
|
||||
--
|
||||
2.17.1
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/lbryio/reflector.go/db"
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/lbryio/ytsync/util"
|
||||
)
|
||||
@@ -53,7 +54,8 @@ func reflectBlobs() error {
|
||||
if running {
|
||||
return errors.Prefix("cannot reflect blobs as the daemon is running", err)
|
||||
}
|
||||
|
||||
logrus.SetLevel(logrus.InfoLevel)
|
||||
defer logrus.SetLevel(logrus.DebugLevel)
|
||||
ex, err := os.Executable()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
|
||||
@@ -13,12 +13,12 @@ lbrycrdurl="rpc://lbry:lbry@lbrycrd:29245"
|
||||
#MySQL DSN is required for chainquery to store information.
|
||||
#DEFAULT: "lbry:lbry@tcp(localhost:3306)/chainquery"
|
||||
#SUGGESTED: "lbry:lbry@unix(/var/run/mysqld/mysqld.sock)/chainquery"
|
||||
mysqldsn="lbry:lbry@tcp(mysqlCQ:3306)/chainquery"
|
||||
mysqldsn="lbry:lbry@tcp(mysql:3306)/chainquery"
|
||||
|
||||
#API MySQL DSN is required for chainquery to expose a SQL query service
|
||||
#DEFAULT: "lbry:lbry@tcp(localhost:3306)/chainquery"
|
||||
#SUGGESTED: "lbry:lbry@unix(/var/run/mysqld/mysqld.sock)/chainquery"
|
||||
apimysqldsn="lbry:lbry@tcp(mysqlCQ:3306)/chainquery"
|
||||
apimysqldsn="lbry:lbry@tcp(mysql:3306)/chainquery"
|
||||
|
||||
#API Host and Port is required for the API Server to bind and listen on.
|
||||
#DEFAULT: "0.0.0.0:6300"
|
||||
|
||||
+8
-26
@@ -21,7 +21,7 @@ services:
|
||||
## Wallet Server ##
|
||||
###################
|
||||
walletserver:
|
||||
image: lbry/wallet-server:v0.48.1
|
||||
image: lbry/wallet-server:v0.50.1
|
||||
restart: always
|
||||
environment:
|
||||
- DB_DIRECTORY=/database
|
||||
@@ -80,6 +80,9 @@ services:
|
||||
- MYSQL_USER=lbry
|
||||
- MYSQL_PASSWORD=lbry
|
||||
- MYSQL_LOG_CONSOLE=true
|
||||
volumes:
|
||||
- "./init.sql:/docker-entrypoint-initdb.d/init.sql"
|
||||
- "./chainquery/docker/my.cnf:/etc/mysql/conf.d/chainquery-optimizations.cnf"
|
||||
###################
|
||||
## Internal APIs ##
|
||||
###################
|
||||
@@ -96,40 +99,19 @@ services:
|
||||
environment:
|
||||
- MYSQL_DSN=lbry:lbry@tcp(mysql:3306)/lbry
|
||||
- LBRYCRD_CONNECT=rpc://lbry:lbry@lbrycrd:29245
|
||||
- MYSQL_USER=lbry
|
||||
- MYSQL_PASS=lbry
|
||||
- MYSQL_DATABASE=lbry
|
||||
entrypoint: wait-for-it -t 0 mysql:3306 -- wait-for-it -t 0 lbrycrd:29245 -- ./latest serve
|
||||
######################
|
||||
## MySQL Chainquery ##
|
||||
######################
|
||||
mysqlCQ:
|
||||
image: mysql/mysql-server:5.7.27
|
||||
restart: "no"
|
||||
ports:
|
||||
- "15600:3306"
|
||||
expose:
|
||||
- "3306"
|
||||
environment:
|
||||
- MYSQL_ALLOW_EMPTY_PASSWORD=true
|
||||
- MYSQL_DATABASE=chainquery
|
||||
- MYSQL_USER=lbry
|
||||
- MYSQL_PASSWORD=lbry
|
||||
- MYSQL_LOG_CONSOLE=true
|
||||
volumes:
|
||||
- "./chainquery/docker/my.cnf:/etc/mysql/conf.d/chainquery-optimizations.cnf"
|
||||
entrypoint: wait-for-it -t 0 chainquery:6300 -- wait-for-it -t 0 lbrycrd:29245 -- ./latest serve
|
||||
################
|
||||
## Chainquery ##
|
||||
################
|
||||
chainquery:
|
||||
image: lbry/chainquery:master
|
||||
image: lbry/chainquery:v2.0.8
|
||||
restart: "no"
|
||||
ports:
|
||||
- 6300:6300
|
||||
depends_on:
|
||||
- lbrycrd
|
||||
- mysqlCQ
|
||||
- mysql
|
||||
## TODO: Uncomment this in a docker-compose.override.yml to allow for external configurations.
|
||||
volumes:
|
||||
- ./chainqueryconfig.toml:/etc/lbry/chainqueryconfig.toml
|
||||
entrypoint: wait-for-it -t 0 lbrycrd:29245 -- wait-for-it -t 0 mysqlCQ:3306 -- start
|
||||
entrypoint: wait-for-it -t 0 lbrycrd:29245 -- wait-for-it -t 0 mysql:3306 -- start
|
||||
+8
-8
@@ -53,29 +53,29 @@ echo "successfully started..."
|
||||
./data_setup.sh
|
||||
|
||||
# Execute the sync test!
|
||||
./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 #Force channel intended...just in case. This channel lines up with the api container
|
||||
./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 --quick #Force channel intended...just in case. This channel lines up with the api container
|
||||
status=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM youtube_data WHERE id=1')
|
||||
videoStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM synced_video WHERE id=1')
|
||||
videoClaimID1=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT claim_id FROM synced_video WHERE id=1')
|
||||
videoClaimID2=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT claim_id FROM synced_video WHERE id=1')
|
||||
videoClaimAddress1=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15600 -e 'SELECT claim_address FROM claim WHERE id=2')
|
||||
videoClaimAddress2=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15600 -e 'SELECT claim_address FROM claim WHERE id=2')
|
||||
videoClaimID1=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT publish.claim_id FROM synced_video INNER JOIN publish ON publish.id = synced_video.publish_id WHERE synced_video.id=1')
|
||||
videoClaimID2=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT publish.claim_id FROM synced_video INNER JOIN publish ON publish.id = synced_video.publish_id WHERE synced_video.id=2')
|
||||
videoClaimAddress1=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT claim_address FROM claim WHERE id=2')
|
||||
videoClaimAddress2=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT claim_address FROM claim WHERE id=3')
|
||||
# Create Supports for published claim
|
||||
./supporty/supporty @BeamerTest "${videoClaimID1}" "${videoClaimAddress1}" lbrycrd_regtest 1.0
|
||||
./supporty/supporty @BeamerTest "${videoClaimID2}" "${videoClaimAddress2}" lbrycrd_regtest 2.0
|
||||
./supporty/supporty @BeamerTest "${videoClaimID1}" "${videoClaimAddress1}" lbrycrd_regtest 3.0
|
||||
./supporty/supporty @BeamerTest "${videoClaimID2}" "${videoClaimAddress2}" lbrycrd_regtest 3.0
|
||||
curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"generate","params":[1]}' -H 'content-type:text/plain;' --user lbry:lbry http://localhost:15200
|
||||
# Reset status for tranfer test
|
||||
# Reset status for transfer test
|
||||
mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e "UPDATE youtube_data SET status = 'queued' WHERE id = 1"
|
||||
# Trigger transfer api
|
||||
curl -i -H 'Accept: application/json' -H 'Content-Type: application/json' 'http://localhost:15400/yt/transfer?auth_token=youtubertoken&address=n4eYeXAYmHo4YRUDEfsEhucy8y5LKRMcHg&public_key=tpubDA9GDAntyJu4hD3wU7175p7CuV6DWbYXfyb2HedBA3yuBp9HZ4n3QE4Ex6RHCSiEuVp2nKAL1Lzf2ZLo9ApaFgNaJjG6Xo1wB3iEeVbrDZp'
|
||||
# Execute the transfer test!
|
||||
./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 #Force channel intended...just in case. This channel lines up with the api container
|
||||
./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 --quick #Force channel intended...just in case. This channel lines up with the api container
|
||||
# Check that the channel and the video are marked as transferred and that all supports are spent
|
||||
channelTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT distinct transfer_state FROM youtube_data')
|
||||
videoTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT distinct transferred FROM synced_video')
|
||||
nrUnspentSupports=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15600 -e 'SELECT COUNT(*) FROM chainquery.support INNER JOIN output ON output.transaction_hash = support.transaction_hash_id AND output.vout = support.vout WHERE output.is_spent = 0')
|
||||
nrUnspentSupports=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT COUNT(*) FROM chainquery.support INNER JOIN output ON output.transaction_hash = support.transaction_hash_id AND output.vout = support.vout WHERE output.is_spent = 0')
|
||||
if [[ $status != "synced" || $videoStatus != "published" || $channelTransferStatus != "2" || $videoTransferStatus != "1" || $nrUnspentSupports != "1" ]]; then
|
||||
echo "~~!!!~~~FAILED~~~!!!~~"
|
||||
echo "Channel Status: $status"
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
CREATE DATABASE IF NOT EXISTS chainquery;
|
||||
GRANT ALL PRIVILEGES ON chainquery.* TO 'lbry'@'%';
|
||||
FLUSH PRIVILEGES;
|
||||
+192
-84
@@ -1,122 +1,230 @@
|
||||
package ip_manager
|
||||
|
||||
import (
|
||||
"github.com/asaskevich/govalidator"
|
||||
"github.com/lbryio/ytsync/util"
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
|
||||
"net"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/asaskevich/govalidator"
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/ytsync/util"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const IPCooldownPeriod = 25 * time.Second
|
||||
const IPCooldownPeriod = 20 * time.Second
|
||||
const unbanTimeout = 3 * time.Hour
|
||||
|
||||
var ipv6Pool []string
|
||||
var ipv4Pool []string
|
||||
var throttledIPs map[string]bool
|
||||
var ipInUse map[string]bool
|
||||
var ipLastUsed map[string]time.Time
|
||||
var ipMutex sync.Mutex
|
||||
var stopper = stop.New()
|
||||
|
||||
func GetNextIP(ipv6 bool) (string, error) {
|
||||
ipMutex.Lock()
|
||||
defer ipMutex.Unlock()
|
||||
if len(ipv4Pool) < 1 || len(ipv6Pool) < 1 {
|
||||
throttledIPs = make(map[string]bool)
|
||||
ipInUse = make(map[string]bool)
|
||||
ipLastUsed = make(map[string]time.Time)
|
||||
addrs, err := net.InterfaceAddrs()
|
||||
if err != nil {
|
||||
return "", errors.Err(err)
|
||||
}
|
||||
type IPPool struct {
|
||||
ips []throttledIP
|
||||
lock *sync.RWMutex
|
||||
stopGrp *stop.Group
|
||||
}
|
||||
|
||||
for _, address := range addrs {
|
||||
if ipnet, ok := address.(*net.IPNet); ok && ipnet.IP.IsGlobalUnicast() {
|
||||
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) {
|
||||
ipv6Pool = append(ipv6Pool, ipnet.IP.String())
|
||||
ipLastUsed[ipnet.IP.String()] = time.Now().Add(-IPCooldownPeriod)
|
||||
} else if ipnet.IP.To4() != nil && govalidator.IsIPv4(ipnet.IP.String()) {
|
||||
ipv4Pool = append(ipv4Pool, ipnet.IP.String())
|
||||
ipLastUsed[ipnet.IP.String()] = time.Now().Add(-IPCooldownPeriod)
|
||||
}
|
||||
type throttledIP struct {
|
||||
IP string
|
||||
UsedForVideo string
|
||||
LastUse time.Time
|
||||
Throttled bool
|
||||
InUse bool
|
||||
}
|
||||
|
||||
var ipPoolInstance *IPPool
|
||||
|
||||
func GetIPPool(stopGrp *stop.Group) (*IPPool, error) {
|
||||
if ipPoolInstance != nil {
|
||||
return ipPoolInstance, nil
|
||||
}
|
||||
addrs, err := net.InterfaceAddrs()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
var pool []throttledIP
|
||||
ipv6Added := false
|
||||
for _, address := range addrs {
|
||||
if ipnet, ok := address.(*net.IPNet); ok && ipnet.IP.IsGlobalUnicast() {
|
||||
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) && !ipv6Added {
|
||||
pool = append(pool, throttledIP{
|
||||
IP: ipnet.IP.String(),
|
||||
LastUse: time.Now().Add(-5 * time.Minute),
|
||||
})
|
||||
ipv6Added = true
|
||||
} else if ipnet.IP.To4() != nil && govalidator.IsIPv4(ipnet.IP.String()) {
|
||||
pool = append(pool, throttledIP{
|
||||
IP: ipnet.IP.String(),
|
||||
LastUse: time.Now().Add(-5 * time.Minute),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
nextIP := ""
|
||||
if ipv6 {
|
||||
nextIP = getLeastUsedIP(ipv6Pool)
|
||||
} else {
|
||||
nextIP = getLeastUsedIP(ipv4Pool)
|
||||
ipPoolInstance = &IPPool{
|
||||
ips: pool,
|
||||
lock: &sync.RWMutex{},
|
||||
stopGrp: stopGrp,
|
||||
}
|
||||
if nextIP == "" {
|
||||
return "throttled", errors.Err("all IPs are throttled")
|
||||
}
|
||||
lastUse := ipLastUsed[nextIP]
|
||||
if time.Since(lastUse) < IPCooldownPeriod {
|
||||
log.Debugf("The IP %s is too hot, waiting for %.1f seconds before continuing", nextIP, (IPCooldownPeriod - time.Since(lastUse)).Seconds())
|
||||
time.Sleep(IPCooldownPeriod - time.Since(lastUse))
|
||||
}
|
||||
|
||||
ipInUse[nextIP] = true
|
||||
return nextIP, nil
|
||||
ticker := time.NewTicker(10 * time.Second)
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-stopGrp.Ch():
|
||||
return
|
||||
case <-ticker.C:
|
||||
ipPoolInstance.lock.RLock()
|
||||
for _, ip := range ipPoolInstance.ips {
|
||||
log.Debugf("IP: %s\tInUse: %t\tVideoID: %s\tThrottled: %t\tLastUse: %.1f", ip.IP, ip.InUse, ip.UsedForVideo, ip.Throttled, time.Since(ip.LastUse).Seconds())
|
||||
}
|
||||
ipPoolInstance.lock.RUnlock()
|
||||
}
|
||||
}
|
||||
}()
|
||||
return ipPoolInstance, nil
|
||||
}
|
||||
|
||||
func ReleaseIP(ip string) {
|
||||
ipMutex.Lock()
|
||||
defer ipMutex.Unlock()
|
||||
ipLastUsed[ip] = time.Now()
|
||||
ipInUse[ip] = false
|
||||
// AllThrottled checks whether the IPs provided are all throttled.
|
||||
// returns false if at least one IP is not throttled
|
||||
// Not thread safe, should use locking when called
|
||||
func AllThrottled(ips []throttledIP) bool {
|
||||
for _, i := range ips {
|
||||
if !i.Throttled {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func getLeastUsedIP(ipPool []string) string {
|
||||
nextIP := ""
|
||||
veryLastUse := time.Now()
|
||||
for _, ip := range ipPool {
|
||||
isThrottled := throttledIPs[ip]
|
||||
if isThrottled {
|
||||
// AllInUse checks whether the IPs provided are all currently in use.
|
||||
// returns false if at least one IP is not in use AND is not throttled
|
||||
// Not thread safe, should use locking when called
|
||||
func AllInUse(ips []throttledIP) bool {
|
||||
for _, i := range ips {
|
||||
if !i.InUse && !i.Throttled {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (i *IPPool) ReleaseIP(ip string) {
|
||||
i.lock.Lock()
|
||||
defer i.lock.Unlock()
|
||||
for j, _ := range i.ips {
|
||||
localIP := &i.ips[j]
|
||||
if localIP.IP == ip {
|
||||
localIP.InUse = false
|
||||
localIP.LastUse = time.Now()
|
||||
return
|
||||
}
|
||||
}
|
||||
util.SendErrorToSlack("something went wrong while releasing the IP %s as we reached the end of the function", ip)
|
||||
}
|
||||
|
||||
func (i *IPPool) ReleaseAll() {
|
||||
i.lock.Lock()
|
||||
defer i.lock.Unlock()
|
||||
for j, _ := range i.ips {
|
||||
if i.ips[j].Throttled {
|
||||
continue
|
||||
}
|
||||
inUse := ipInUse[ip]
|
||||
if inUse {
|
||||
continue
|
||||
}
|
||||
lastUse := ipLastUsed[ip]
|
||||
if lastUse.Before(veryLastUse) {
|
||||
nextIP = ip
|
||||
veryLastUse = lastUse
|
||||
}
|
||||
localIP := &i.ips[j]
|
||||
localIP.InUse = false
|
||||
localIP.LastUse = time.Now()
|
||||
}
|
||||
return nextIP
|
||||
}
|
||||
|
||||
func SetIpThrottled(ip string, stopGrp *stop.Group) {
|
||||
ipMutex.Lock()
|
||||
isThrottled := throttledIPs[ip]
|
||||
if isThrottled {
|
||||
return
|
||||
func (i *IPPool) SetThrottled(ip string) {
|
||||
i.lock.Lock()
|
||||
defer i.lock.Unlock()
|
||||
var tIP *throttledIP
|
||||
for j, _ := range i.ips {
|
||||
localIP := &i.ips[j]
|
||||
if localIP.IP == ip {
|
||||
if localIP.Throttled {
|
||||
return
|
||||
}
|
||||
localIP.Throttled = true
|
||||
tIP = localIP
|
||||
break
|
||||
}
|
||||
}
|
||||
throttledIPs[ip] = true
|
||||
ipMutex.Unlock()
|
||||
util.SendErrorToSlack("%s set to throttled", ip)
|
||||
|
||||
stopper.Add(1)
|
||||
go func() {
|
||||
go func(tIP *throttledIP) {
|
||||
defer stopper.Done()
|
||||
unbanTimer := time.NewTimer(unbanTimeout)
|
||||
select {
|
||||
case <-unbanTimer.C:
|
||||
ipMutex.Lock()
|
||||
throttledIPs[ip] = false
|
||||
ipMutex.Unlock()
|
||||
i.lock.Lock()
|
||||
tIP.Throttled = false
|
||||
i.lock.Unlock()
|
||||
util.SendInfoToSlack("%s set back to not throttled", ip)
|
||||
case <-stopGrp.Ch():
|
||||
case <-i.stopGrp.Ch():
|
||||
unbanTimer.Stop()
|
||||
}
|
||||
}()
|
||||
}(tIP)
|
||||
}
|
||||
|
||||
var ErrAllInUse = errors.Base("all IPs are in use, try again")
|
||||
var ErrAllThrottled = errors.Base("all IPs are throttled")
|
||||
var ErrResourceLock = errors.Base("error getting next ip, did you forget to lock on the resource?")
|
||||
var ErrInterruptedByUser = errors.Base("interrupted by user")
|
||||
|
||||
func (i *IPPool) nextIP(forVideo string) (*throttledIP, error) {
|
||||
i.lock.Lock()
|
||||
defer i.lock.Unlock()
|
||||
|
||||
sort.Slice(i.ips, func(j, k int) bool {
|
||||
return i.ips[j].LastUse.Before(i.ips[k].LastUse)
|
||||
})
|
||||
|
||||
if !AllThrottled(i.ips) {
|
||||
if AllInUse(i.ips) {
|
||||
return nil, errors.Err(ErrAllInUse)
|
||||
}
|
||||
|
||||
var nextIP *throttledIP
|
||||
for j, _ := range i.ips {
|
||||
ip := &i.ips[j]
|
||||
if ip.InUse || ip.Throttled {
|
||||
continue
|
||||
}
|
||||
nextIP = ip
|
||||
break
|
||||
}
|
||||
if nextIP == nil {
|
||||
return nil, errors.Err(ErrResourceLock)
|
||||
}
|
||||
nextIP.InUse = true
|
||||
nextIP.UsedForVideo = forVideo
|
||||
return nextIP, nil
|
||||
}
|
||||
return nil, errors.Err(ErrAllThrottled)
|
||||
}
|
||||
|
||||
func (i *IPPool) GetIP(forVideo string) (string, error) {
|
||||
for {
|
||||
ip, err := i.nextIP(forVideo)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrAllInUse) {
|
||||
select {
|
||||
case <-i.stopGrp.Ch():
|
||||
return "", errors.Err(ErrInterruptedByUser)
|
||||
default:
|
||||
time.Sleep(5 * time.Second)
|
||||
continue
|
||||
}
|
||||
} else if errors.Is(err, ErrAllThrottled) {
|
||||
return "throttled", err
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
if time.Since(ip.LastUse) < IPCooldownPeriod {
|
||||
log.Debugf("The IP %s is too hot, waiting for %.1f seconds before continuing", ip.IP, (IPCooldownPeriod - time.Since(ip.LastUse)).Seconds())
|
||||
time.Sleep(IPCooldownPeriod - time.Since(ip.LastUse))
|
||||
}
|
||||
return ip.IP, nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package ip_manager
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAll(t *testing.T) {
|
||||
pool, err := GetIPPool()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ip, err := pool.GetIP()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log(ip)
|
||||
pool.ReleaseIP(ip)
|
||||
ip2, err := pool.GetIP()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ip == ip2 && len(pool.ips) > 1 {
|
||||
t.Fatalf("the same IP was returned twice! %s, %s", ip, ip2)
|
||||
}
|
||||
t.Log(ip2)
|
||||
pool.ReleaseIP(ip2)
|
||||
|
||||
for range pool.ips {
|
||||
_, err = pool.GetIP()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
next, err := pool.nextIP()
|
||||
if err != nil {
|
||||
t.Logf("%s", err.Error())
|
||||
} else {
|
||||
t.Fatal(next)
|
||||
}
|
||||
}
|
||||
@@ -56,6 +56,7 @@ func main() {
|
||||
cmd.Flags().BoolVar(&flags.RemoveDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published")
|
||||
cmd.Flags().BoolVar(&flags.UpgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version")
|
||||
cmd.Flags().BoolVar(&flags.DisableTransfers, "no-transfers", false, "Skips the transferring process of videos, channels and supports")
|
||||
cmd.Flags().BoolVar(&flags.QuickSync, "quick", false, "Look up only the last 50 videos from youtube")
|
||||
cmd.Flags().StringVar(&syncStatus, "status", "", "Specify which queue to pull from. Overrides --update")
|
||||
cmd.Flags().StringVar(&channelID, "channelID", "", "If specified, only this channel will be synced.")
|
||||
cmd.Flags().Int64Var(&syncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)")
|
||||
|
||||
+10
-2
@@ -7,6 +7,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/ytsync/blobs_reflector"
|
||||
"github.com/lbryio/ytsync/ip_manager"
|
||||
"github.com/lbryio/ytsync/namer"
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
logUtils "github.com/lbryio/ytsync/util"
|
||||
@@ -92,6 +93,7 @@ const (
|
||||
TransferStateNotTouched = iota
|
||||
TransferStatePending
|
||||
TransferStateComplete
|
||||
TransferStateManual
|
||||
)
|
||||
|
||||
func (s *SyncManager) Start() error {
|
||||
@@ -147,7 +149,6 @@ func (s *SyncManager) Start() error {
|
||||
shouldInterruptLoop = true
|
||||
} else {
|
||||
var queuesToSync []string
|
||||
//TODO: implement scrambling to avoid starvation of queues
|
||||
if s.syncStatus != "" {
|
||||
queuesToSync = append(queuesToSync, s.syncStatus)
|
||||
} else if s.SyncFlags.SyncUpdate {
|
||||
@@ -159,7 +160,7 @@ func (s *SyncManager) Start() error {
|
||||
for _, q := range queuesToSync {
|
||||
//temporary override for sync-until to give tom the time to review the channels
|
||||
if q == StatusQueued {
|
||||
s.syncProperties.SyncUntil = time.Now().AddDate(0, 0, -1).Unix()
|
||||
s.syncProperties.SyncUntil = time.Now().Add(-8 * time.Hour).Unix()
|
||||
}
|
||||
channels, err := s.apiConfig.FetchChannels(q, s.syncProperties)
|
||||
if err != nil {
|
||||
@@ -201,6 +202,12 @@ func (s *SyncManager) Start() error {
|
||||
shouldNotCount := false
|
||||
logUtils.SendInfoToSlack("Syncing %s (%s) to LBRY! total processed channels since startup: %d", sync.LbryChannelName, sync.YoutubeChannelID, syncCount+1)
|
||||
err := sync.FullCycle()
|
||||
//TODO: THIS IS A TEMPORARY WORK AROUND FOR THE STUPID IP LOCKUP BUG
|
||||
ipPool, _ := ip_manager.GetIPPool(sync.grp)
|
||||
if ipPool != nil {
|
||||
ipPool.ReleaseAll()
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
fatalErrors := []string{
|
||||
"default_wallet already exists",
|
||||
@@ -211,6 +218,7 @@ func (s *SyncManager) Start() error {
|
||||
"the channel in the wallet is different than the channel in the database",
|
||||
"this channel does not belong to this wallet!",
|
||||
"You already have a stream claim published under the name",
|
||||
"Daily Limit Exceeded",
|
||||
}
|
||||
if util.SubstringInSlice(err.Error(), fatalErrors) {
|
||||
return errors.Prefix("@Nikooo777 this requires manual intervention! Exiting...", err)
|
||||
|
||||
+8
-1
@@ -399,6 +399,9 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
|
||||
var languages []string = nil
|
||||
if channelInfo.DefaultLanguage != "" {
|
||||
if channelInfo.DefaultLanguage == "iw" {
|
||||
channelInfo.DefaultLanguage = "he"
|
||||
}
|
||||
languages = []string{channelInfo.DefaultLanguage}
|
||||
}
|
||||
var locations []jsonrpc.Location = nil
|
||||
@@ -445,7 +448,11 @@ func (s *Sync) addCredits(amountToAdd float64) error {
|
||||
return err
|
||||
}
|
||||
|
||||
addressResp, err := s.daemon.AddressUnused(nil)
|
||||
defaultAccount, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
addressResp, err := s.daemon.AddressUnused(&defaultAccount)
|
||||
if err != nil {
|
||||
return err
|
||||
} else if addressResp == nil {
|
||||
|
||||
+9
-4
@@ -96,11 +96,11 @@ func abandonSupports(s *Sync) (float64, error) {
|
||||
if !more {
|
||||
return
|
||||
} else {
|
||||
summary, err := s.daemon.SupportAbandon(&claimID, nil, nil, nil, nil)
|
||||
summary, err := s.daemon.SupportAbandon(&claimID, nil, nil, nil, &defaultAccount)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "Client.Timeout exceeded while awaiting headers") {
|
||||
log.Errorf("Support abandon for %s timed out, retrying...", claimID)
|
||||
summary, err = s.daemon.SupportAbandon(&claimID, nil, nil, nil, nil)
|
||||
summary, err = s.daemon.SupportAbandon(&claimID, nil, nil, nil, &defaultAccount)
|
||||
if err != nil {
|
||||
//TODO GUESS HOW MUCH LBC WAS RELEASED THAT WE DON'T KNOW ABOUT, because screw you SDK
|
||||
abandonRspChan <- abandonResponse{
|
||||
@@ -192,7 +192,7 @@ func transferVideos(s *Sync) error {
|
||||
|
||||
var stream *jsonrpc.Claim = nil
|
||||
for _, c := range streams.Items {
|
||||
if c.ClaimID != video.ClaimID {
|
||||
if c.ClaimID != video.ClaimID || (c.SigningChannel != nil && c.SigningChannel.ClaimID != s.lbryChannelID) {
|
||||
continue
|
||||
}
|
||||
stream = &c
|
||||
@@ -204,7 +204,12 @@ func transferVideos(s *Sync) error {
|
||||
|
||||
streamUpdateOptions := jsonrpc.StreamUpdateOptions{
|
||||
StreamCreateOptions: &jsonrpc.StreamCreateOptions{
|
||||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{ClaimAddress: &s.clientPublishAddress},
|
||||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
|
||||
ClaimAddress: &s.clientPublishAddress,
|
||||
FundingAccountIDs: []string{
|
||||
account,
|
||||
},
|
||||
},
|
||||
},
|
||||
Bid: util.PtrToString("0.005"), // Todo - Dont hardcode
|
||||
}
|
||||
|
||||
+69
-19
@@ -14,6 +14,7 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/ytsync/ip_manager"
|
||||
"github.com/lbryio/ytsync/namer"
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
"github.com/lbryio/ytsync/sources"
|
||||
@@ -245,6 +246,8 @@ func (s *Sync) setExceptions() {
|
||||
}
|
||||
}
|
||||
|
||||
var stopGroup = stop.New()
|
||||
|
||||
func (s *Sync) FullCycle() (e error) {
|
||||
if os.Getenv("HOME") == "" {
|
||||
return errors.Err("no $HOME env var found")
|
||||
@@ -257,7 +260,7 @@ func (s *Sync) FullCycle() (e error) {
|
||||
|
||||
s.syncedVideosMux = &sync.RWMutex{}
|
||||
s.walletMux = &sync.RWMutex{}
|
||||
s.grp = stop.New()
|
||||
s.grp = stopGroup
|
||||
s.queue = make(chan video)
|
||||
interruptChan := make(chan os.Signal, 1)
|
||||
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
|
||||
@@ -356,7 +359,18 @@ func (s *Sync) processTransfers() (e error) {
|
||||
isTip := true
|
||||
summary, err := s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount), &isTip, nil, []string{defaultAccount}, nil)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
if strings.Contains(err.Error(), "tx-size") { //TODO: this is a silly workaround and should be written in an recursive function
|
||||
summary, err = s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount/2.0), &isTip, nil, []string{defaultAccount}, nil)
|
||||
if err != nil {
|
||||
return errors.Prefix(fmt.Sprintf("something went wrong while tipping the channel for %.6f LBCs", supportAmount), err)
|
||||
}
|
||||
summary, err = s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount/2.0), &isTip, nil, []string{defaultAccount}, nil)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
} else {
|
||||
return errors.Err(err)
|
||||
}
|
||||
}
|
||||
if len(summary.Outputs) < 1 {
|
||||
return errors.Err("something went wrong while tipping the channel for %.6f LBCs", supportAmount)
|
||||
@@ -466,7 +480,7 @@ var thumbnailHosts = []string{
|
||||
thumbs.ThumbnailEndpoint,
|
||||
}
|
||||
|
||||
func isYtsyncClaim(c jsonrpc.Claim) bool {
|
||||
func isYtsyncClaim(c jsonrpc.Claim, expectedChannelID string) bool {
|
||||
if !util.InSlice(c.Type, []string{"claim", "update"}) || c.Value.GetStream() == nil {
|
||||
return false
|
||||
}
|
||||
@@ -474,7 +488,12 @@ func isYtsyncClaim(c jsonrpc.Claim) bool {
|
||||
//most likely a claim created outside of ytsync, ignore!
|
||||
return false
|
||||
}
|
||||
|
||||
if c.SigningChannel == nil {
|
||||
return false
|
||||
}
|
||||
if c.SigningChannel.ClaimID != expectedChannelID {
|
||||
return false
|
||||
}
|
||||
for _, th := range thumbnailHosts {
|
||||
if strings.Contains(c.Value.GetThumbnail().GetUrl(), th) {
|
||||
return true
|
||||
@@ -488,7 +507,7 @@ func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) {
|
||||
abandonedClaims := false
|
||||
videoIDs := make(map[string]jsonrpc.Claim)
|
||||
for _, c := range claims {
|
||||
if !isYtsyncClaim(c) {
|
||||
if !isYtsyncClaim(c, s.lbryChannelID) {
|
||||
continue
|
||||
}
|
||||
tn := c.Value.GetThumbnail().GetUrl()
|
||||
@@ -535,7 +554,7 @@ type ytsyncClaim struct {
|
||||
func (s *Sync) mapFromClaims(claims []jsonrpc.Claim) map[string]ytsyncClaim {
|
||||
videoIDMap := make(map[string]ytsyncClaim, len(claims))
|
||||
for _, c := range claims {
|
||||
if !isYtsyncClaim(c) {
|
||||
if !isYtsyncClaim(c, s.lbryChannelID) {
|
||||
continue
|
||||
}
|
||||
tn := c.Value.GetThumbnail().GetUrl()
|
||||
@@ -632,8 +651,18 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
|
||||
if sv.Transferred {
|
||||
_, ok := allClaimsInfo[vID]
|
||||
if !ok && sv.Published {
|
||||
log.Warnf("%s: claims to be published and transferred but wasn't found in the list of claims", vID)
|
||||
idsToRemove = append(idsToRemove, vID)
|
||||
searchResponse, err := s.daemon.ClaimSearch(nil, &sv.ClaimID, nil, nil, 1, 20)
|
||||
if err != nil {
|
||||
log.Error(err.Error())
|
||||
continue
|
||||
}
|
||||
if len(searchResponse.Claims) == 0 {
|
||||
log.Debugf("%s: was transferred but appears abandoned! we should ignore this - claimID: %s", vID, sv.ClaimID)
|
||||
continue //TODO: we should flag these on the db
|
||||
} else {
|
||||
log.Debugf("%s: was transferred adn was then edited! we should ignore this - claimID: %s", vID, sv.ClaimID)
|
||||
//return count, fixed, 0, errors.Err("%s: isn't our control but is on the database and on the blockchain. wtf is up? ClaimID: %s", vID, sv.ClaimID)
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
@@ -674,8 +703,13 @@ func (s *Sync) getClaims(defaultOnly bool) ([]jsonrpc.Claim, error) {
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("cannot list claims", err)
|
||||
}
|
||||
|
||||
return claims.Items, nil
|
||||
items := make([]jsonrpc.Claim, 0, len(claims.Items))
|
||||
for _, c := range claims.Items {
|
||||
if c.SigningChannel != nil && c.SigningChannel.ClaimID == s.lbryChannelID {
|
||||
items = append(items, c)
|
||||
}
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (s *Sync) checkIntegrity() error {
|
||||
@@ -757,7 +791,7 @@ func (s *Sync) doSync() error {
|
||||
return err
|
||||
}
|
||||
|
||||
if s.transferState != TransferStateComplete {
|
||||
if s.transferState < TransferStateComplete {
|
||||
cert, err := s.daemon.ChannelExport(s.lbryChannelID, nil, nil)
|
||||
if err != nil {
|
||||
return errors.Prefix("error getting channel cert", err)
|
||||
@@ -861,12 +895,18 @@ func (s *Sync) startWorker(workerNum int) {
|
||||
"giving up after 0 fragment retries",
|
||||
"Sorry about that",
|
||||
"This video is not available",
|
||||
"requested format not available",
|
||||
"interrupted by user",
|
||||
"Sign in to confirm your age",
|
||||
"This video is unavailable",
|
||||
}
|
||||
if util.SubstringInSlice(err.Error(), errorsNoRetry) {
|
||||
log.Println("This error should not be retried at all")
|
||||
} else if tryCount < s.MaxTries {
|
||||
if strings.Contains(err.Error(), "txn-mempool-conflict") ||
|
||||
strings.Contains(err.Error(), "too-long-mempool-chain") {
|
||||
if util.SubstringInSlice(err.Error(), []string{
|
||||
"txn-mempool-conflict",
|
||||
"too-long-mempool-chain",
|
||||
}) {
|
||||
log.Println("waiting for a block before retrying")
|
||||
err := s.waitForNewBlock()
|
||||
if err != nil {
|
||||
@@ -878,6 +918,7 @@ func (s *Sync) startWorker(workerNum int) {
|
||||
"Not enough funds to cover this transaction",
|
||||
"failed: Not enough funds",
|
||||
"Error in daemon: Insufficient funds, please deposit additional LBC",
|
||||
"Missing inputs",
|
||||
}) {
|
||||
log.Println("checking funds and UTXOs before retrying...")
|
||||
err := s.walletSetup()
|
||||
@@ -963,6 +1004,11 @@ func (s *Sync) enqueueYoutubeVideos() error {
|
||||
}
|
||||
|
||||
var videos []video
|
||||
ipPool, err := ip_manager.GetIPPool(s.grp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
playlistMap := make(map[string]*youtube.PlaylistItemSnippet, 50)
|
||||
nextPageToken := ""
|
||||
for {
|
||||
@@ -985,7 +1031,6 @@ func (s *Sync) enqueueYoutubeVideos() error {
|
||||
}
|
||||
return errors.Err("playlist items not found")
|
||||
}
|
||||
//playlistMap := make(map[string]*youtube.PlaylistItemSnippet, 50)
|
||||
videoIDs := make([]string, 50)
|
||||
for i, item := range playlistResponse.Items {
|
||||
// normally we'd send the video into the channel here, but youtube api doesn't have sorting
|
||||
@@ -1000,13 +1045,13 @@ func (s *Sync) enqueueYoutubeVideos() error {
|
||||
return errors.Prefix("error getting videos info", err)
|
||||
}
|
||||
for _, item := range videosListResponse.Items {
|
||||
videos = append(videos, sources.NewYoutubeVideo(s.videoDirectory, item, playlistMap[item.Id].Position, s.Manager.GetS3AWSConfig(), s.grp))
|
||||
videos = append(videos, sources.NewYoutubeVideo(s.videoDirectory, item, playlistMap[item.Id].Position, s.Manager.GetS3AWSConfig(), s.grp, ipPool))
|
||||
}
|
||||
|
||||
log.Infof("Got info for %d videos from youtube API", len(videos))
|
||||
|
||||
nextPageToken = playlistResponse.NextPageToken
|
||||
if nextPageToken == "" {
|
||||
if nextPageToken == "" || s.Manager.SyncFlags.QuickSync || len(videos) >= s.Manager.videosLimit {
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -1016,12 +1061,11 @@ func (s *Sync) enqueueYoutubeVideos() error {
|
||||
}
|
||||
_, ok := playlistMap[k]
|
||||
if !ok {
|
||||
videos = append(videos, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig(), s.grp))
|
||||
videos = append(videos, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig(), s.grp, ipPool))
|
||||
}
|
||||
|
||||
}
|
||||
sort.Sort(byPublishedAt(videos))
|
||||
//or sort.Sort(sort.Reverse(byPlaylistPosition(videos)))
|
||||
|
||||
Enqueue:
|
||||
for _, v := range videos {
|
||||
@@ -1044,7 +1088,7 @@ Enqueue:
|
||||
func (s *Sync) processVideo(v video) (err error) {
|
||||
defer func() {
|
||||
if p := recover(); p != nil {
|
||||
log.Printf("stack: %s", debug.Stack())
|
||||
logUtils.SendErrorToSlack("Video processing panic! %s", debug.Stack())
|
||||
var ok bool
|
||||
err, ok = p.(error)
|
||||
if !ok {
|
||||
@@ -1076,6 +1120,7 @@ func (s *Sync) processVideo(v video) (err error) {
|
||||
"Watch this video on YouTube.",
|
||||
"have blocked it on copyright grounds",
|
||||
"giving up after 0 fragment retries",
|
||||
"Sign in to confirm your age",
|
||||
}
|
||||
if ok && !sv.Published && util.SubstringInSlice(sv.FailureReason, neverRetryFailures) {
|
||||
log.Println(v.ID() + " can't ever be published")
|
||||
@@ -1099,6 +1144,10 @@ func (s *Sync) processVideo(v video) (err error) {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
da, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sp := sources.SyncParams{
|
||||
ClaimAddress: s.claimAddress,
|
||||
Amount: publishAmount,
|
||||
@@ -1107,6 +1156,7 @@ func (s *Sync) processVideo(v video) (err error) {
|
||||
Namer: s.namer,
|
||||
MaxVideoLength: s.Manager.maxVideoLength,
|
||||
Fee: s.Fee,
|
||||
DefaultAccount: da,
|
||||
}
|
||||
|
||||
summary, err := v.Sync(s.daemon, sp, &sv, videoRequiresUpgrade, s.walletMux)
|
||||
|
||||
@@ -45,6 +45,7 @@ type SyncFlags struct {
|
||||
RemoveDBUnpublished bool
|
||||
UpgradeMetadata bool
|
||||
DisableTransfers bool
|
||||
QuickSync bool
|
||||
}
|
||||
|
||||
type Fee struct {
|
||||
|
||||
+103
-98
@@ -50,6 +50,7 @@ type YoutubeVideo struct {
|
||||
mocked bool
|
||||
walletLock *sync.RWMutex
|
||||
stopGroup *stop.Group
|
||||
pool *ip_manager.IPPool
|
||||
}
|
||||
|
||||
var youtubeCategories = map[string]string{
|
||||
@@ -87,7 +88,7 @@ var youtubeCategories = map[string]string{
|
||||
"44": "trailers",
|
||||
}
|
||||
|
||||
func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group) *YoutubeVideo {
|
||||
func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo {
|
||||
publishedAt, _ := time.Parse(time.RFC3339Nano, videoData.Snippet.PublishedAt) // ignore parse errors
|
||||
return &YoutubeVideo{
|
||||
id: videoData.Id,
|
||||
@@ -101,9 +102,10 @@ func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPositio
|
||||
mocked: false,
|
||||
youtubeChannelID: videoData.Snippet.ChannelId,
|
||||
stopGroup: stopGroup,
|
||||
pool: pool,
|
||||
}
|
||||
}
|
||||
func NewMockedVideo(directory string, videoID string, youtubeChannelID string, awsConfig aws.Config, stopGroup *stop.Group) *YoutubeVideo {
|
||||
func NewMockedVideo(directory string, videoID string, youtubeChannelID string, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo {
|
||||
return &YoutubeVideo{
|
||||
id: videoID,
|
||||
playlistPosition: 0,
|
||||
@@ -112,6 +114,7 @@ func NewMockedVideo(directory string, videoID string, youtubeChannelID string, a
|
||||
mocked: true,
|
||||
youtubeChannelID: youtubeChannelID,
|
||||
stopGroup: stopGroup,
|
||||
pool: pool,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -175,7 +178,7 @@ func (v *YoutubeVideo) getAbbrevDescription() string {
|
||||
return description + "\n..." + additionalDescription
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) download(useIPv6 bool) error {
|
||||
func (v *YoutubeVideo) download() error {
|
||||
videoPath := v.getFullPath()
|
||||
|
||||
err := os.Mkdir(v.videoDir(), 0777)
|
||||
@@ -196,7 +199,6 @@ func (v *YoutubeVideo) download(useIPv6 bool) error {
|
||||
"480",
|
||||
"320",
|
||||
}
|
||||
qualityIndex := 0
|
||||
ytdlArgs := []string{
|
||||
"--no-progress",
|
||||
"-o" + strings.TrimSuffix(v.getFullPath(), ".mp4"),
|
||||
@@ -207,6 +209,10 @@ func (v *YoutubeVideo) download(useIPv6 bool) error {
|
||||
"--abort-on-unavailable-fragment",
|
||||
"--fragment-retries",
|
||||
"0",
|
||||
"--user-agent",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36",
|
||||
"--cookies",
|
||||
"cookies.txt",
|
||||
}
|
||||
if v.maxVideoSize > 0 {
|
||||
ytdlArgs = append(ytdlArgs,
|
||||
@@ -220,102 +226,95 @@ func (v *YoutubeVideo) download(useIPv6 bool) error {
|
||||
fmt.Sprintf("duration <= %d", int(math.Round(v.maxVideoLength*3600))),
|
||||
)
|
||||
}
|
||||
sourceAddress, err := ip_manager.GetNextIP(useIPv6)
|
||||
if err != nil {
|
||||
if sourceAddress == "throttled" {
|
||||
for {
|
||||
|
||||
var sourceAddress string
|
||||
for {
|
||||
sourceAddress, err = v.pool.GetIP(v.id)
|
||||
if err != nil {
|
||||
if errors.Is(err, ip_manager.ErrAllThrottled) {
|
||||
select {
|
||||
case <-v.stopGroup.Ch():
|
||||
return errors.Err("interrupted by user")
|
||||
default:
|
||||
time.Sleep(ip_manager.IPCooldownPeriod)
|
||||
continue
|
||||
}
|
||||
|
||||
time.Sleep(ip_manager.IPCooldownPeriod)
|
||||
sourceAddress, err = ip_manager.GetNextIP(useIPv6)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
}
|
||||
break
|
||||
}
|
||||
defer v.pool.ReleaseIP(sourceAddress)
|
||||
|
||||
ytdlArgs = append(ytdlArgs,
|
||||
"--source-address",
|
||||
sourceAddress,
|
||||
"https://www.youtube.com/watch?v="+v.ID(),
|
||||
)
|
||||
|
||||
for i, quality := range qualities {
|
||||
argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][height<="+quality+"]+bestaudio[ext!=webm]")
|
||||
cmd := exec.Command("youtube-dl", argsWithFilters...)
|
||||
log.Printf("Running command youtube-dl %s", strings.Join(argsWithFilters, " "))
|
||||
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
}
|
||||
defer ip_manager.ReleaseIP(sourceAddress)
|
||||
if useIPv6 {
|
||||
log.Infof("using IPv6: %s", sourceAddress)
|
||||
ytdlArgs = append(ytdlArgs,
|
||||
"-6",
|
||||
"--source-address",
|
||||
sourceAddress,
|
||||
)
|
||||
} else {
|
||||
log.Infof("using IPv4: %s", sourceAddress)
|
||||
ytdlArgs = append(ytdlArgs,
|
||||
"-4",
|
||||
"--source-address",
|
||||
sourceAddress,
|
||||
)
|
||||
}
|
||||
ytdlArgs = append(ytdlArgs, "https://www.youtube.com/watch?v="+v.ID())
|
||||
runcmd:
|
||||
argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][height<="+qualities[qualityIndex]+"]+bestaudio[ext!=webm]")
|
||||
cmd := exec.Command("youtube-dl", argsWithFilters...)
|
||||
|
||||
log.Printf("Running command youtube-dl %s", strings.Join(argsWithFilters, " "))
|
||||
if err := cmd.Start(); err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
errorLog, _ := ioutil.ReadAll(stderr)
|
||||
outLog, _ := ioutil.ReadAll(stdout)
|
||||
|
||||
if err = cmd.Wait(); err != nil {
|
||||
if strings.Contains(err.Error(), "exit status 1") {
|
||||
if strings.Contains(string(errorLog), "HTTP Error 429") {
|
||||
ip_manager.SetIpThrottled(sourceAddress, v.stopGroup)
|
||||
} else if strings.Contains(string(errorLog), "giving up after 0 fragment retries") && qualityIndex < len(qualities)-1 {
|
||||
qualityIndex++
|
||||
goto runcmd
|
||||
errorLog, _ := ioutil.ReadAll(stderr)
|
||||
outLog, _ := ioutil.ReadAll(stdout)
|
||||
err = cmd.Wait()
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "exit status 1") {
|
||||
if strings.Contains(string(errorLog), "HTTP Error 429") || strings.Contains(string(errorLog), "returned non-zero exit status 8") {
|
||||
v.pool.SetThrottled(sourceAddress)
|
||||
} else if strings.Contains(string(errorLog), "giving up after 0 fragment retries") {
|
||||
if i == (len(qualities) - 1) {
|
||||
return errors.Err(string(errorLog))
|
||||
}
|
||||
continue //this bypasses the yt throttling IP redistribution... TODO: don't
|
||||
}
|
||||
return errors.Err(string(errorLog))
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
log.Debugln(string(outLog))
|
||||
|
||||
if strings.Contains(string(outLog), "does not pass filter duration") {
|
||||
_ = v.delete("does not pass filter duration")
|
||||
return errors.Err("video is too long to process")
|
||||
}
|
||||
if strings.Contains(string(outLog), "File is larger than max-filesize") {
|
||||
_ = v.delete("File is larger than max-filesize")
|
||||
return errors.Err("the video is too big to sync, skipping for now")
|
||||
}
|
||||
if string(errorLog) != "" {
|
||||
log.Printf("Command finished with error: %v", errors.Err(string(errorLog)))
|
||||
_ = v.delete("due to error")
|
||||
return errors.Err(string(errorLog))
|
||||
}
|
||||
return errors.Err(err)
|
||||
fi, err := os.Stat(v.getFullPath())
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
err = os.Chmod(v.getFullPath(), 0777)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
videoSize := fi.Size()
|
||||
v.size = &videoSize
|
||||
break
|
||||
}
|
||||
log.Debugln(string(outLog))
|
||||
|
||||
if strings.Contains(string(outLog), "does not pass filter duration") {
|
||||
_ = v.delete("does not pass filter duration")
|
||||
return errors.Err("video is too long to process")
|
||||
}
|
||||
if strings.Contains(string(outLog), "File is larger than max-filesize") {
|
||||
_ = v.delete("File is larger than max-filesize")
|
||||
return errors.Err("the video is too big to sync, skipping for now")
|
||||
}
|
||||
if string(errorLog) != "" {
|
||||
log.Printf("Command finished with error: %v", errors.Err(string(errorLog)))
|
||||
_ = v.delete("due to error")
|
||||
return errors.Err(string(errorLog))
|
||||
}
|
||||
fi, err := os.Stat(v.getFullPath())
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
err = os.Chmod(v.getFullPath(), 0777)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
videoSize := fi.Size()
|
||||
v.size = &videoSize
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -368,7 +367,6 @@ func (v *YoutubeVideo) triggerThumbnailSave() (err error) {
|
||||
|
||||
func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
|
||||
languages, locations, tags := v.getMetadata()
|
||||
|
||||
var fee *jsonrpc.Fee
|
||||
if params.Fee != nil {
|
||||
feeAmount, err := decimal.NewFromString(params.Fee.Amount)
|
||||
@@ -391,6 +389,9 @@ func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, params SyncParams) (*Sync
|
||||
ThumbnailURL: &v.thumbnailURL,
|
||||
Tags: tags,
|
||||
Locations: locations,
|
||||
FundingAccountIDs: []string{
|
||||
params.DefaultAccount,
|
||||
},
|
||||
},
|
||||
Fee: fee,
|
||||
License: util.PtrToString("Copyrighted (contact publisher)"),
|
||||
@@ -416,6 +417,7 @@ type SyncParams struct {
|
||||
Namer *namer.Namer
|
||||
MaxVideoLength float64
|
||||
Fee *sdk.Fee
|
||||
DefaultAccount string
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo, reprocess bool, walletLock *sync.RWMutex) (*SyncSummary, error) {
|
||||
@@ -430,21 +432,18 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV
|
||||
return v.downloadAndPublish(daemon, params)
|
||||
}
|
||||
|
||||
var isThrottled bool
|
||||
|
||||
func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
|
||||
err := v.download(isThrottled)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "HTTP Error 429") && !isThrottled {
|
||||
isThrottled = true
|
||||
err = v.download(isThrottled)
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("download error", err)
|
||||
}
|
||||
} else {
|
||||
var err error
|
||||
for {
|
||||
err = v.download()
|
||||
if err != nil && strings.Contains(err.Error(), "HTTP Error 429") {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return nil, errors.Prefix("download error", err)
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
log.Debugln("Downloaded " + v.id)
|
||||
|
||||
err = v.triggerThumbnailSave()
|
||||
@@ -466,6 +465,9 @@ func (v *YoutubeVideo) getMetadata() (languages []string, locations []jsonrpc.Lo
|
||||
tags = nil
|
||||
if !v.mocked {
|
||||
if v.youtubeInfo.Snippet.DefaultLanguage != "" {
|
||||
if v.youtubeInfo.Snippet.DefaultLanguage == "iw" {
|
||||
v.youtubeInfo.Snippet.DefaultLanguage = "he"
|
||||
}
|
||||
languages = []string{v.youtubeInfo.Snippet.DefaultLanguage}
|
||||
}
|
||||
|
||||
@@ -549,6 +551,9 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
|
||||
ThumbnailURL: &thumbnailURL,
|
||||
Languages: languages,
|
||||
Locations: locations,
|
||||
FundingAccountIDs: []string{
|
||||
params.DefaultAccount,
|
||||
},
|
||||
},
|
||||
Author: util.PtrToString(""),
|
||||
License: util.PtrToString("Copyrighted (contact publisher)"),
|
||||
|
||||
Reference in new issue
Block a user