Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8b32381b23 |
No files matched your search
+8
-8
@@ -9,13 +9,12 @@ builds:
|
||||
- amd64
|
||||
ldflags:
|
||||
- -X "{{ .Env.IMPORTPATH }}/meta.semVersion={{ .Tag }}" -X "{{ .Env.IMPORTPATH }}/meta.version={{ .Env.VERSIONSHORT }}" -X "{{ .Env.IMPORTPATH }}/meta.versionLong={{ .Env.VERSIONLONG }}" -X "{{ .Env.IMPORTPATH }}/meta.commitMsg={{ .Env.COMMITMSG }}"
|
||||
archives:
|
||||
- id: zip
|
||||
name_template: '{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}'
|
||||
replacements:
|
||||
linux: Linux
|
||||
amd64: x86_64
|
||||
format: zip
|
||||
archive:
|
||||
name_template: '{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}'
|
||||
replacements:
|
||||
linux: Linux
|
||||
amd64: x86_64
|
||||
format: zip
|
||||
checksum:
|
||||
name_template: 'checksums.txt'
|
||||
snapshot:
|
||||
@@ -25,4 +24,5 @@ changelog:
|
||||
filters:
|
||||
exclude:
|
||||
- '^docs:'
|
||||
- '^test:'
|
||||
- '^test:'
|
||||
|
||||
@@ -1,111 +0,0 @@
|
||||
From 30380338ba9af01696c94b61f0597131638eaec1 Mon Sep 17 00:00:00 2001
|
||||
From: Niko Storni <niko@lbry.io>
|
||||
Date: Mon, 16 Dec 2019 00:13:36 +0100
|
||||
Subject: [PATCH] lbry-patch
|
||||
|
||||
---
|
||||
youtube_dl/extractor/youtube.py | 45 +++++++++++++++++++++++++--------
|
||||
1 file changed, 35 insertions(+), 10 deletions(-)
|
||||
|
||||
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
|
||||
index b913d07a6..cd66a5b01 100644
|
||||
--- a/youtube_dl/extractor/youtube.py
|
||||
+++ b/youtube_dl/extractor/youtube.py
|
||||
@@ -10,6 +10,7 @@ import random
|
||||
import re
|
||||
import time
|
||||
import traceback
|
||||
+import subprocess
|
||||
|
||||
from .common import InfoExtractor, SearchInfoExtractor
|
||||
from ..jsinterp import JSInterpreter
|
||||
@@ -536,6 +537,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
|
||||
_GEO_BYPASS = False
|
||||
|
||||
+ _WGET_429_RATE_LIMIT = 8191
|
||||
+ _WGET_BINARY = "wget"
|
||||
+
|
||||
IE_NAME = 'youtube'
|
||||
_TESTS = [
|
||||
{
|
||||
@@ -1254,6 +1258,17 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
""" Return a string representation of a signature """
|
||||
return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
|
||||
|
||||
+ def _rate_limit_download(self, url, video_id, note=None):
|
||||
+ if note is None:
|
||||
+ self.report_download_webpage(video_id)
|
||||
+ elif note is not False:
|
||||
+ if video_id is None:
|
||||
+ self.to_screen('%s' % (note,))
|
||||
+ else:
|
||||
+ self.to_screen('%s: %s' % (video_id, note))
|
||||
+ source_address = self._downloader.params.get('source_address')
|
||||
+ return subprocess.run([self._WGET_BINARY, '-q', '--limit-rate', str(self._WGET_429_RATE_LIMIT), '--bind-address', source_address, '-O', '-', url], check=True, stdout=subprocess.PIPE).stdout.decode(encoding='UTF-8')
|
||||
+
|
||||
def _extract_signature_function(self, video_id, player_url, example_sig):
|
||||
id_m = re.match(
|
||||
r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player(?:-new)?|(?:/[a-z]{2,3}_[A-Z]{2})?/base)?\.(?P<ext>[a-z]+)$',
|
||||
@@ -1678,7 +1693,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
|
||||
# Get video webpage
|
||||
url = proto + '://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1&bpctr=9999999999' % video_id
|
||||
- video_webpage = self._download_webpage(url, video_id)
|
||||
+ video_webpage = self._rate_limit_download(url, video_id)
|
||||
|
||||
# Attempt to extract SWF player URL
|
||||
mobj = re.search(r'swfConfig.*?"(https?:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
|
||||
@@ -1736,10 +1751,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
r'"sts"\s*:\s*(\d+)', embed_webpage, 'sts', default=''),
|
||||
})
|
||||
video_info_url = proto + '://www.youtube.com/get_video_info?' + data
|
||||
- video_info_webpage = self._download_webpage(
|
||||
+ video_info_webpage = self._rate_limit_download(
|
||||
video_info_url, video_id,
|
||||
- note='Refetching age-gated info webpage',
|
||||
- errnote='unable to download video info webpage')
|
||||
+ note='Refetching age-gated info webpage')
|
||||
video_info = compat_parse_qs(video_info_webpage)
|
||||
pl_response = video_info.get('player_response', [None])[0]
|
||||
player_response = extract_player_response(pl_response, video_id)
|
||||
@@ -1777,7 +1791,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
# The general idea is to take a union of itags of both DASH manifests (for example
|
||||
# video with such 'manifest behavior' see https://github.com/ytdl-org/youtube-dl/issues/6093)
|
||||
self.report_video_info_webpage_download(video_id)
|
||||
- for el in ('embedded', 'detailpage', 'vevo', ''):
|
||||
+ for el in ('', 'embedded', 'detailpage', 'vevo'):
|
||||
query = {
|
||||
'video_id': video_id,
|
||||
'ps': 'default',
|
||||
@@ -1789,11 +1803,22 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
query['el'] = el
|
||||
if sts:
|
||||
query['sts'] = sts
|
||||
- video_info_webpage = self._download_webpage(
|
||||
- '%s://www.youtube.com/get_video_info' % proto,
|
||||
- video_id, note=False,
|
||||
- errnote='unable to download video info webpage',
|
||||
- fatal=False, query=query)
|
||||
+
|
||||
+ if el == '':
|
||||
+ base_url = 'https://youtube.com/get_video_info?video_id={}'.format(video_id)
|
||||
+ else:
|
||||
+ base_url = 'https://youtube.com/get_video_info'
|
||||
+
|
||||
+ for q in query:
|
||||
+ if q is None or q is "":
|
||||
+ continue
|
||||
+ if query[q] is None or query[q] is "":
|
||||
+ continue
|
||||
+
|
||||
+ base_url = base_url + "?{}={}".format(q, query[q])
|
||||
+
|
||||
+ video_info_webpage = self._rate_limit_download(base_url, video_id)
|
||||
+
|
||||
if not video_info_webpage:
|
||||
continue
|
||||
get_video_info = compat_parse_qs(video_info_webpage)
|
||||
--
|
||||
2.17.1
|
||||
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
"github.com/lbryio/reflector.go/db"
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/lbryio/ytsync/util"
|
||||
)
|
||||
@@ -54,8 +53,7 @@ func reflectBlobs() error {
|
||||
if running {
|
||||
return errors.Prefix("cannot reflect blobs as the daemon is running", err)
|
||||
}
|
||||
logrus.SetLevel(logrus.InfoLevel)
|
||||
defer logrus.SetLevel(logrus.DebugLevel)
|
||||
|
||||
ex, err := os.Executable()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
|
||||
@@ -13,12 +13,12 @@ lbrycrdurl="rpc://lbry:lbry@lbrycrd:29245"
|
||||
#MySQL DSN is required for chainquery to store information.
|
||||
#DEFAULT: "lbry:lbry@tcp(localhost:3306)/chainquery"
|
||||
#SUGGESTED: "lbry:lbry@unix(/var/run/mysqld/mysqld.sock)/chainquery"
|
||||
mysqldsn="lbry:lbry@tcp(mysql:3306)/chainquery"
|
||||
mysqldsn="lbry:lbry@tcp(mysqlCQ:3306)/chainquery"
|
||||
|
||||
#API MySQL DSN is required for chainquery to expose a SQL query service
|
||||
#DEFAULT: "lbry:lbry@tcp(localhost:3306)/chainquery"
|
||||
#SUGGESTED: "lbry:lbry@unix(/var/run/mysqld/mysqld.sock)/chainquery"
|
||||
apimysqldsn="lbry:lbry@tcp(mysql:3306)/chainquery"
|
||||
apimysqldsn="lbry:lbry@tcp(mysqlCQ:3306)/chainquery"
|
||||
|
||||
#API Host and Port is required for the API Server to bind and listen on.
|
||||
#DEFAULT: "0.0.0.0:6300"
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
#blockchain_name: lbrycrd_main
|
||||
#blockchain_name: lbrycrd_testnet
|
||||
blockchain_name: lbrycrd_regtest
|
||||
lbryum_servers:
|
||||
- walletserver:50001
|
||||
reflect_streams: false
|
||||
# - spv1.lbry.com:50001 #Production Wallet Server
|
||||
- walletserver:50001
|
||||
save_blobs: true
|
||||
save_files: false
|
||||
reflect_streams: false #for the love of god, don't upload regtest streams to reflector!
|
||||
share_usage_data: false
|
||||
tcp_port: 3333
|
||||
udp_port: 4444
|
||||
|
||||
+28
-10
@@ -4,7 +4,7 @@ services:
|
||||
## Lbrycrd ##
|
||||
#############
|
||||
lbrycrd:
|
||||
image: lbry/lbrycrd:v0.17.3.2-deprecatedrpc
|
||||
image: lbry/lbrycrd:v0.17.2.1
|
||||
restart: "no"
|
||||
ports:
|
||||
- "15201:29246"
|
||||
@@ -21,7 +21,7 @@ services:
|
||||
## Wallet Server ##
|
||||
###################
|
||||
walletserver:
|
||||
image: lbry/wallet-server:v0.51.2
|
||||
image: lbry/wallet-server:v0.42.2
|
||||
restart: always
|
||||
environment:
|
||||
- DB_DIRECTORY=/database
|
||||
@@ -47,7 +47,7 @@ services:
|
||||
## Lbrynet ##
|
||||
#############
|
||||
lbrynet:
|
||||
image: lbry/lbrynet:v0.51.2
|
||||
image: lbry/lbrynet:v0.42.2
|
||||
restart: always
|
||||
ports:
|
||||
- "15100:5279"
|
||||
@@ -80,9 +80,6 @@ services:
|
||||
- MYSQL_USER=lbry
|
||||
- MYSQL_PASSWORD=lbry
|
||||
- MYSQL_LOG_CONSOLE=true
|
||||
volumes:
|
||||
- "./init.sql:/docker-entrypoint-initdb.d/init.sql"
|
||||
- "./chainquery/docker/my.cnf:/etc/mysql/conf.d/chainquery-optimizations.cnf"
|
||||
###################
|
||||
## Internal APIs ##
|
||||
###################
|
||||
@@ -99,19 +96,40 @@ services:
|
||||
environment:
|
||||
- MYSQL_DSN=lbry:lbry@tcp(mysql:3306)/lbry
|
||||
- LBRYCRD_CONNECT=rpc://lbry:lbry@lbrycrd:29245
|
||||
entrypoint: wait-for-it -t 0 chainquery:6300 -- wait-for-it -t 0 lbrycrd:29245 -- ./latest serve
|
||||
- MYSQL_USER=lbry
|
||||
- MYSQL_PASS=lbry
|
||||
- MYSQL_DATABASE=lbry
|
||||
entrypoint: wait-for-it -t 0 mysql:3306 -- wait-for-it -t 0 lbrycrd:29245 -- ./latest serve
|
||||
######################
|
||||
## MySQL Chainquery ##
|
||||
######################
|
||||
mysqlCQ:
|
||||
image: mysql/mysql-server:5.7.27
|
||||
restart: "no"
|
||||
ports:
|
||||
- "15600:3306"
|
||||
expose:
|
||||
- "3306"
|
||||
environment:
|
||||
- MYSQL_ALLOW_EMPTY_PASSWORD=true
|
||||
- MYSQL_DATABASE=chainquery
|
||||
- MYSQL_USER=lbry
|
||||
- MYSQL_PASSWORD=lbry
|
||||
- MYSQL_LOG_CONSOLE=true
|
||||
volumes:
|
||||
- "./chainquery/docker/my.cnf:/etc/mysql/conf.d/chainquery-optimizations.cnf"
|
||||
################
|
||||
## Chainquery ##
|
||||
################
|
||||
chainquery:
|
||||
image: lbry/chainquery:v2.0.8
|
||||
image: lbry/chainquery:master
|
||||
restart: "no"
|
||||
ports:
|
||||
- 6300:6300
|
||||
depends_on:
|
||||
- lbrycrd
|
||||
- mysql
|
||||
- mysqlCQ
|
||||
## TODO: Uncomment this in a docker-compose.override.yml to allow for external configurations.
|
||||
volumes:
|
||||
- ./chainqueryconfig.toml:/etc/lbry/chainqueryconfig.toml
|
||||
entrypoint: wait-for-it -t 0 lbrycrd:29245 -- wait-for-it -t 0 mysql:3306 -- start
|
||||
entrypoint: wait-for-it -t 0 lbrycrd:29245 -- wait-for-it -t 0 mysqlCQ:3306 -- start
|
||||
+8
-8
@@ -53,29 +53,29 @@ echo "successfully started..."
|
||||
./data_setup.sh
|
||||
|
||||
# Execute the sync test!
|
||||
./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 --quick #Force channel intended...just in case. This channel lines up with the api container
|
||||
./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 #Force channel intended...just in case. This channel lines up with the api container
|
||||
status=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM youtube_data WHERE id=1')
|
||||
videoStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM synced_video WHERE id=1')
|
||||
videoClaimID1=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT publish.claim_id FROM synced_video INNER JOIN publish ON publish.id = synced_video.publish_id WHERE synced_video.id=1')
|
||||
videoClaimID2=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT publish.claim_id FROM synced_video INNER JOIN publish ON publish.id = synced_video.publish_id WHERE synced_video.id=2')
|
||||
videoClaimAddress1=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT claim_address FROM claim WHERE id=2')
|
||||
videoClaimAddress2=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT claim_address FROM claim WHERE id=3')
|
||||
videoClaimID1=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT claim_id FROM synced_video WHERE id=1')
|
||||
videoClaimID2=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT claim_id FROM synced_video WHERE id=1')
|
||||
videoClaimAddress1=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15600 -e 'SELECT claim_address FROM claim WHERE id=2')
|
||||
videoClaimAddress2=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15600 -e 'SELECT claim_address FROM claim WHERE id=2')
|
||||
# Create Supports for published claim
|
||||
./supporty/supporty @BeamerTest "${videoClaimID1}" "${videoClaimAddress1}" lbrycrd_regtest 1.0
|
||||
./supporty/supporty @BeamerTest "${videoClaimID2}" "${videoClaimAddress2}" lbrycrd_regtest 2.0
|
||||
./supporty/supporty @BeamerTest "${videoClaimID1}" "${videoClaimAddress1}" lbrycrd_regtest 3.0
|
||||
./supporty/supporty @BeamerTest "${videoClaimID2}" "${videoClaimAddress2}" lbrycrd_regtest 3.0
|
||||
curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"generate","params":[1]}' -H 'content-type:text/plain;' --user lbry:lbry http://localhost:15200
|
||||
# Reset status for transfer test
|
||||
# Reset status for tranfer test
|
||||
mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e "UPDATE youtube_data SET status = 'queued' WHERE id = 1"
|
||||
# Trigger transfer api
|
||||
curl -i -H 'Accept: application/json' -H 'Content-Type: application/json' 'http://localhost:15400/yt/transfer?auth_token=youtubertoken&address=n4eYeXAYmHo4YRUDEfsEhucy8y5LKRMcHg&public_key=tpubDA9GDAntyJu4hD3wU7175p7CuV6DWbYXfyb2HedBA3yuBp9HZ4n3QE4Ex6RHCSiEuVp2nKAL1Lzf2ZLo9ApaFgNaJjG6Xo1wB3iEeVbrDZp'
|
||||
# Execute the transfer test!
|
||||
./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 --quick #Force channel intended...just in case. This channel lines up with the api container
|
||||
./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 #Force channel intended...just in case. This channel lines up with the api container
|
||||
# Check that the channel and the video are marked as transferred and that all supports are spent
|
||||
channelTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT distinct transfer_state FROM youtube_data')
|
||||
videoTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT distinct transferred FROM synced_video')
|
||||
nrUnspentSupports=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT COUNT(*) FROM chainquery.support INNER JOIN output ON output.transaction_hash = support.transaction_hash_id AND output.vout = support.vout WHERE output.is_spent = 0')
|
||||
nrUnspentSupports=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15600 -e 'SELECT COUNT(*) FROM chainquery.support INNER JOIN output ON output.transaction_hash = support.transaction_hash_id AND output.vout = support.vout WHERE output.is_spent = 0')
|
||||
if [[ $status != "synced" || $videoStatus != "published" || $channelTransferStatus != "2" || $videoTransferStatus != "1" || $nrUnspentSupports != "1" ]]; then
|
||||
echo "~~!!!~~~FAILED~~~!!!~~"
|
||||
echo "Channel Status: $status"
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
CREATE DATABASE IF NOT EXISTS chainquery;
|
||||
GRANT ALL PRIVILEGES ON chainquery.* TO 'lbry'@'%';
|
||||
FLUSH PRIVILEGES;
|
||||
+13
-15
@@ -11,9 +11,9 @@ function override_config_option() {
|
||||
local var=$1 option=$2 config=$3
|
||||
if [[ -v $var ]]; then
|
||||
# Remove the existing config option:
|
||||
sed -i "/^$option\W*=/d" "$config"
|
||||
sed -i "/^$option\W*=/d" $config
|
||||
# Add the value from the environment:
|
||||
echo "$option=${!var}" >> "$config"
|
||||
echo "$option=${!var}" >> $config
|
||||
fi
|
||||
}
|
||||
|
||||
@@ -38,17 +38,15 @@ function set_config() {
|
||||
else
|
||||
echo "Creating a fresh config file from environment variables."
|
||||
## Set config params
|
||||
{
|
||||
echo "port=${PORT=9246}"
|
||||
echo "rpcuser=${RPC_USER=lbry}"
|
||||
echo "rpcpassword=${RPC_PASSWORD=lbry}"
|
||||
echo "rpcallowip=${RPC_ALLOW_IP=127.0.0.1/24}"
|
||||
echo "rpcport=${RPC_PORT=9245}"
|
||||
echo "rpcbind=${RPC_BIND=0.0.0.0}"
|
||||
echo "deprecatedrpc=accounts"
|
||||
echo "deprecatedrpc=validateaddress"
|
||||
echo "deprecatedrpc=signrawtransaction"
|
||||
} >> $CONFIG_PATH
|
||||
echo "port=${PORT=9246}" > $CONFIG_PATH
|
||||
echo "rpcuser=${RPC_USER=lbry}" >> $CONFIG_PATH
|
||||
echo "rpcpassword=${RPC_PASSWORD=lbry}" >> $CONFIG_PATH
|
||||
echo "rpcallowip=${RPC_ALLOW_IP=127.0.0.1/24}" >> $CONFIG_PATH
|
||||
echo "rpcport=${RPC_PORT=9245}" >> $CONFIG_PATH
|
||||
echo "rpcbind=${RPC_BIND=0.0.0.0}" >> $CONFIG_PATH
|
||||
echo "deprecatedrpc=accounts" >> $CONFIG_PATH
|
||||
echo "deprecatedrpc=validateaddress" >> $CONFIG_PATH
|
||||
echo "deprecatedrpc=signrawtransaction" >> $CONFIG_PATH
|
||||
fi
|
||||
echo "Config: "
|
||||
cat $CONFIG_PATH
|
||||
@@ -81,7 +79,7 @@ case $RUN_MODE in
|
||||
regtest )
|
||||
## Set config params
|
||||
## TODO: Make this more automagic in the future.
|
||||
mkdir -p "$(dirname $CONFIG_PATH)"
|
||||
mkdir -p `dirname $CONFIG_PATH`
|
||||
echo "rpcuser=lbry" > $CONFIG_PATH
|
||||
echo "rpcpassword=lbry" >> $CONFIG_PATH
|
||||
echo "rpcport=29245" >> $CONFIG_PATH
|
||||
@@ -103,7 +101,7 @@ case $RUN_MODE in
|
||||
testnet )
|
||||
## Set config params
|
||||
## TODO: Make this more automagic in the future.
|
||||
mkdir -p "$(dirname $CONFIG_PATH)"
|
||||
mkdir -p `dirname $CONFIG_PATH`
|
||||
echo "rpcuser=lbry" > $CONFIG_PATH
|
||||
echo "rpcpassword=lbry" >> $CONFIG_PATH
|
||||
echo "rpcport=29245" >> $CONFIG_PATH
|
||||
|
||||
@@ -8,7 +8,7 @@ services:
|
||||
## Lbrynet ##
|
||||
#############
|
||||
lbrynet:
|
||||
image: lbry/lbrynet:v0.51.2
|
||||
image: lbry/lbrynet:v0.38.6
|
||||
restart: "no"
|
||||
networks:
|
||||
lbry-network:
|
||||
|
||||
@@ -18,7 +18,7 @@ require (
|
||||
github.com/hashicorp/golang-lru v0.5.3 // indirect
|
||||
github.com/hashicorp/memberlist v0.1.5 // indirect
|
||||
github.com/hashicorp/serf v0.8.5 // indirect
|
||||
github.com/lbryio/lbry.go/v2 v2.4.1-0.20191206194319-06764c3d007e
|
||||
github.com/lbryio/lbry.go/v2 v2.3.3-0.20191010144133-fa9bab156a4e
|
||||
github.com/lbryio/reflector.go v1.0.6-0.20190828131602-ce3d4403dbc6
|
||||
github.com/miekg/dns v1.1.22 // indirect
|
||||
github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b
|
||||
|
||||
@@ -213,8 +213,8 @@ github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpU
|
||||
github.com/lbryio/lbry.go v0.0.0-20190828131228-f3a1fbdd5303/go.mod h1:qR+Ui0hYhemIU4fXqM3d1P9eiaRFlof777VJgV7KJ8w=
|
||||
github.com/lbryio/lbry.go v1.1.2 h1:Dyxc+glT/rVWJwHfIf7vjlPYYbjzrQz5ARmJd5Hp69c=
|
||||
github.com/lbryio/lbry.go v1.1.2/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
|
||||
github.com/lbryio/lbry.go/v2 v2.4.1-0.20191206194319-06764c3d007e h1:wN6r46L8EjgC2JyOImagQXaQvyF8Bd2QPjfXqv7oDAg=
|
||||
github.com/lbryio/lbry.go/v2 v2.4.1-0.20191206194319-06764c3d007e/go.mod h1:gbs/+PVII96AqsRgf+hY/F7sJkb/ciiX2yy3j+NApd0=
|
||||
github.com/lbryio/lbry.go/v2 v2.3.3-0.20191010144133-fa9bab156a4e h1:B1ekb7lXFuMol/tUdwTajOe56ab38drBJmr0X8IoZYE=
|
||||
github.com/lbryio/lbry.go/v2 v2.3.3-0.20191010144133-fa9bab156a4e/go.mod h1:Fm4jBlDyDauHIBC5OABuqcBWKBpcbRgPk5K0eG2mhSs=
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46 h1:LemfR+rMxhf7nnOrzy2HqS7Me7SZ5gEwOcNFzKC8ySQ=
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
package ipManager
|
||||
|
||||
import (
|
||||
"github.com/asaskevich/govalidator"
|
||||
"github.com/lbryio/ytsync/util"
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
|
||||
"net"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const IPCooldownPeriod = 25 * time.Second
|
||||
const unbanTimeout = 3 * time.Hour
|
||||
|
||||
var ipv6Pool []string
|
||||
var ipv4Pool []string
|
||||
var throttledIPs map[string]bool
|
||||
var ipInUse map[string]bool
|
||||
var ipLastUsed map[string]time.Time
|
||||
var ipMutex sync.Mutex
|
||||
var stopper = stop.New()
|
||||
|
||||
func GetNextIP(ipv6 bool) (string, error) {
|
||||
ipMutex.Lock()
|
||||
defer ipMutex.Unlock()
|
||||
if len(ipv4Pool) < 1 || len(ipv6Pool) < 1 {
|
||||
throttledIPs = make(map[string]bool)
|
||||
ipInUse = make(map[string]bool)
|
||||
ipLastUsed = make(map[string]time.Time)
|
||||
addrs, err := net.InterfaceAddrs()
|
||||
if err != nil {
|
||||
return "", errors.Err(err)
|
||||
}
|
||||
|
||||
for _, address := range addrs {
|
||||
if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
|
||||
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) {
|
||||
ipv6Pool = append(ipv6Pool, ipnet.IP.String())
|
||||
ipLastUsed[ipnet.IP.String()] = time.Now().Add(-IPCooldownPeriod)
|
||||
} else if ipnet.IP.To4() != nil && govalidator.IsIPv4(ipnet.IP.String()) {
|
||||
ipv4Pool = append(ipv4Pool, ipnet.IP.String())
|
||||
ipLastUsed[ipnet.IP.String()] = time.Now().Add(-IPCooldownPeriod)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
nextIP := ""
|
||||
if ipv6 {
|
||||
nextIP = getLeastUsedIP(ipv6Pool)
|
||||
} else {
|
||||
nextIP = getLeastUsedIP(ipv4Pool)
|
||||
}
|
||||
if nextIP == "" {
|
||||
return "throttled", errors.Err("all IPs are throttled")
|
||||
}
|
||||
lastUse := ipLastUsed[nextIP]
|
||||
if time.Since(lastUse) < IPCooldownPeriod {
|
||||
log.Debugf("The IP %s is too hot, waiting for %.1f seconds before continuing", nextIP, (IPCooldownPeriod - time.Since(lastUse)).Seconds())
|
||||
time.Sleep(IPCooldownPeriod - time.Since(lastUse))
|
||||
}
|
||||
|
||||
ipInUse[nextIP] = true
|
||||
return nextIP, nil
|
||||
}
|
||||
|
||||
func ReleaseIP(ip string) {
|
||||
ipMutex.Lock()
|
||||
defer ipMutex.Unlock()
|
||||
ipLastUsed[ip] = time.Now()
|
||||
ipInUse[ip] = false
|
||||
}
|
||||
|
||||
func getLeastUsedIP(ipPool []string) string {
|
||||
nextIP := ""
|
||||
veryLastUse := time.Now()
|
||||
for _, ip := range ipPool {
|
||||
isThrottled := throttledIPs[ip]
|
||||
if isThrottled {
|
||||
continue
|
||||
}
|
||||
inUse := ipInUse[ip]
|
||||
if inUse {
|
||||
continue
|
||||
}
|
||||
lastUse := ipLastUsed[ip]
|
||||
if lastUse.Before(veryLastUse) {
|
||||
nextIP = ip
|
||||
veryLastUse = lastUse
|
||||
}
|
||||
}
|
||||
return nextIP
|
||||
}
|
||||
|
||||
func SetIpThrottled(ip string, stopGrp *stop.Group) {
|
||||
ipMutex.Lock()
|
||||
isThrottled := throttledIPs[ip]
|
||||
if isThrottled {
|
||||
return
|
||||
}
|
||||
throttledIPs[ip] = true
|
||||
ipMutex.Unlock()
|
||||
util.SendErrorToSlack("%s set to throttled", ip)
|
||||
|
||||
stopper.Add(1)
|
||||
go func() {
|
||||
defer stopper.Done()
|
||||
unbanTimer := time.NewTimer(unbanTimeout)
|
||||
select {
|
||||
case <-unbanTimer.C:
|
||||
ipMutex.Lock()
|
||||
throttledIPs[ip] = false
|
||||
ipMutex.Unlock()
|
||||
util.SendInfoToSlack("%s set back to not throttled", ip)
|
||||
case <-stopGrp.Ch():
|
||||
unbanTimer.Stop()
|
||||
}
|
||||
}()
|
||||
}
|
||||
@@ -1,230 +0,0 @@
|
||||
package ip_manager
|
||||
|
||||
import (
|
||||
"net"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/asaskevich/govalidator"
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/ytsync/util"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const IPCooldownPeriod = 20 * time.Second
|
||||
const unbanTimeout = 3 * time.Hour
|
||||
|
||||
var stopper = stop.New()
|
||||
|
||||
type IPPool struct {
|
||||
ips []throttledIP
|
||||
lock *sync.RWMutex
|
||||
stopGrp *stop.Group
|
||||
}
|
||||
|
||||
type throttledIP struct {
|
||||
IP string
|
||||
UsedForVideo string
|
||||
LastUse time.Time
|
||||
Throttled bool
|
||||
InUse bool
|
||||
}
|
||||
|
||||
var ipPoolInstance *IPPool
|
||||
|
||||
func GetIPPool(stopGrp *stop.Group) (*IPPool, error) {
|
||||
if ipPoolInstance != nil {
|
||||
return ipPoolInstance, nil
|
||||
}
|
||||
addrs, err := net.InterfaceAddrs()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
var pool []throttledIP
|
||||
ipv6Added := false
|
||||
for _, address := range addrs {
|
||||
if ipnet, ok := address.(*net.IPNet); ok && ipnet.IP.IsGlobalUnicast() {
|
||||
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) && !ipv6Added {
|
||||
pool = append(pool, throttledIP{
|
||||
IP: ipnet.IP.String(),
|
||||
LastUse: time.Now().Add(-5 * time.Minute),
|
||||
})
|
||||
ipv6Added = true
|
||||
} else if ipnet.IP.To4() != nil && govalidator.IsIPv4(ipnet.IP.String()) {
|
||||
pool = append(pool, throttledIP{
|
||||
IP: ipnet.IP.String(),
|
||||
LastUse: time.Now().Add(-5 * time.Minute),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
ipPoolInstance = &IPPool{
|
||||
ips: pool,
|
||||
lock: &sync.RWMutex{},
|
||||
stopGrp: stopGrp,
|
||||
}
|
||||
ticker := time.NewTicker(10 * time.Second)
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-stopGrp.Ch():
|
||||
return
|
||||
case <-ticker.C:
|
||||
ipPoolInstance.lock.RLock()
|
||||
for _, ip := range ipPoolInstance.ips {
|
||||
log.Debugf("IP: %s\tInUse: %t\tVideoID: %s\tThrottled: %t\tLastUse: %.1f", ip.IP, ip.InUse, ip.UsedForVideo, ip.Throttled, time.Since(ip.LastUse).Seconds())
|
||||
}
|
||||
ipPoolInstance.lock.RUnlock()
|
||||
}
|
||||
}
|
||||
}()
|
||||
return ipPoolInstance, nil
|
||||
}
|
||||
|
||||
// AllThrottled checks whether the IPs provided are all throttled.
|
||||
// returns false if at least one IP is not throttled
|
||||
// Not thread safe, should use locking when called
|
||||
func AllThrottled(ips []throttledIP) bool {
|
||||
for _, i := range ips {
|
||||
if !i.Throttled {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// AllInUse checks whether the IPs provided are all currently in use.
|
||||
// returns false if at least one IP is not in use AND is not throttled
|
||||
// Not thread safe, should use locking when called
|
||||
func AllInUse(ips []throttledIP) bool {
|
||||
for _, i := range ips {
|
||||
if !i.InUse && !i.Throttled {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (i *IPPool) ReleaseIP(ip string) {
|
||||
i.lock.Lock()
|
||||
defer i.lock.Unlock()
|
||||
for j, _ := range i.ips {
|
||||
localIP := &i.ips[j]
|
||||
if localIP.IP == ip {
|
||||
localIP.InUse = false
|
||||
localIP.LastUse = time.Now()
|
||||
return
|
||||
}
|
||||
}
|
||||
util.SendErrorToSlack("something went wrong while releasing the IP %s as we reached the end of the function", ip)
|
||||
}
|
||||
|
||||
func (i *IPPool) ReleaseAll() {
|
||||
i.lock.Lock()
|
||||
defer i.lock.Unlock()
|
||||
for j, _ := range i.ips {
|
||||
if i.ips[j].Throttled {
|
||||
continue
|
||||
}
|
||||
localIP := &i.ips[j]
|
||||
localIP.InUse = false
|
||||
localIP.LastUse = time.Now()
|
||||
}
|
||||
}
|
||||
|
||||
func (i *IPPool) SetThrottled(ip string) {
|
||||
i.lock.Lock()
|
||||
defer i.lock.Unlock()
|
||||
var tIP *throttledIP
|
||||
for j, _ := range i.ips {
|
||||
localIP := &i.ips[j]
|
||||
if localIP.IP == ip {
|
||||
if localIP.Throttled {
|
||||
return
|
||||
}
|
||||
localIP.Throttled = true
|
||||
tIP = localIP
|
||||
break
|
||||
}
|
||||
}
|
||||
util.SendErrorToSlack("%s set to throttled", ip)
|
||||
|
||||
stopper.Add(1)
|
||||
go func(tIP *throttledIP) {
|
||||
defer stopper.Done()
|
||||
unbanTimer := time.NewTimer(unbanTimeout)
|
||||
select {
|
||||
case <-unbanTimer.C:
|
||||
i.lock.Lock()
|
||||
tIP.Throttled = false
|
||||
i.lock.Unlock()
|
||||
util.SendInfoToSlack("%s set back to not throttled", ip)
|
||||
case <-i.stopGrp.Ch():
|
||||
unbanTimer.Stop()
|
||||
}
|
||||
}(tIP)
|
||||
}
|
||||
|
||||
var ErrAllInUse = errors.Base("all IPs are in use, try again")
|
||||
var ErrAllThrottled = errors.Base("all IPs are throttled")
|
||||
var ErrResourceLock = errors.Base("error getting next ip, did you forget to lock on the resource?")
|
||||
var ErrInterruptedByUser = errors.Base("interrupted by user")
|
||||
|
||||
func (i *IPPool) nextIP(forVideo string) (*throttledIP, error) {
|
||||
i.lock.Lock()
|
||||
defer i.lock.Unlock()
|
||||
|
||||
sort.Slice(i.ips, func(j, k int) bool {
|
||||
return i.ips[j].LastUse.Before(i.ips[k].LastUse)
|
||||
})
|
||||
|
||||
if !AllThrottled(i.ips) {
|
||||
if AllInUse(i.ips) {
|
||||
return nil, errors.Err(ErrAllInUse)
|
||||
}
|
||||
|
||||
var nextIP *throttledIP
|
||||
for j, _ := range i.ips {
|
||||
ip := &i.ips[j]
|
||||
if ip.InUse || ip.Throttled {
|
||||
continue
|
||||
}
|
||||
nextIP = ip
|
||||
break
|
||||
}
|
||||
if nextIP == nil {
|
||||
return nil, errors.Err(ErrResourceLock)
|
||||
}
|
||||
nextIP.InUse = true
|
||||
nextIP.UsedForVideo = forVideo
|
||||
return nextIP, nil
|
||||
}
|
||||
return nil, errors.Err(ErrAllThrottled)
|
||||
}
|
||||
|
||||
func (i *IPPool) GetIP(forVideo string) (string, error) {
|
||||
for {
|
||||
ip, err := i.nextIP(forVideo)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrAllInUse) {
|
||||
select {
|
||||
case <-i.stopGrp.Ch():
|
||||
return "", errors.Err(ErrInterruptedByUser)
|
||||
default:
|
||||
time.Sleep(5 * time.Second)
|
||||
continue
|
||||
}
|
||||
} else if errors.Is(err, ErrAllThrottled) {
|
||||
return "throttled", err
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
if time.Since(ip.LastUse) < IPCooldownPeriod {
|
||||
log.Debugf("The IP %s is too hot, waiting for %.1f seconds before continuing", ip.IP, (IPCooldownPeriod - time.Since(ip.LastUse)).Seconds())
|
||||
time.Sleep(IPCooldownPeriod - time.Since(ip.LastUse))
|
||||
}
|
||||
return ip.IP, nil
|
||||
}
|
||||
}
|
||||
@@ -1,40 +0,0 @@
|
||||
package ip_manager
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAll(t *testing.T) {
|
||||
pool, err := GetIPPool()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ip, err := pool.GetIP()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log(ip)
|
||||
pool.ReleaseIP(ip)
|
||||
ip2, err := pool.GetIP()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ip == ip2 && len(pool.ips) > 1 {
|
||||
t.Fatalf("the same IP was returned twice! %s, %s", ip, ip2)
|
||||
}
|
||||
t.Log(ip2)
|
||||
pool.ReleaseIP(ip2)
|
||||
|
||||
for range pool.ips {
|
||||
_, err = pool.GetIP()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
next, err := pool.nextIP()
|
||||
if err != nil {
|
||||
t.Logf("%s", err.Error())
|
||||
} else {
|
||||
t.Fatal(next)
|
||||
}
|
||||
}
|
||||
@@ -56,7 +56,6 @@ func main() {
|
||||
cmd.Flags().BoolVar(&flags.RemoveDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published")
|
||||
cmd.Flags().BoolVar(&flags.UpgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version")
|
||||
cmd.Flags().BoolVar(&flags.DisableTransfers, "no-transfers", false, "Skips the transferring process of videos, channels and supports")
|
||||
cmd.Flags().BoolVar(&flags.QuickSync, "quick", false, "Look up only the last 50 videos from youtube")
|
||||
cmd.Flags().StringVar(&syncStatus, "status", "", "Specify which queue to pull from. Overrides --update")
|
||||
cmd.Flags().StringVar(&channelID, "channelID", "", "If specified, only this channel will be synced.")
|
||||
cmd.Flags().Int64Var(&syncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)")
|
||||
|
||||
+3
-10
@@ -7,7 +7,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/ytsync/blobs_reflector"
|
||||
"github.com/lbryio/ytsync/ip_manager"
|
||||
"github.com/lbryio/ytsync/namer"
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
logUtils "github.com/lbryio/ytsync/util"
|
||||
@@ -93,7 +92,7 @@ const (
|
||||
TransferStateNotTouched = iota
|
||||
TransferStatePending
|
||||
TransferStateComplete
|
||||
TransferStateManual
|
||||
TransferStateFailed = -1
|
||||
)
|
||||
|
||||
func (s *SyncManager) Start() error {
|
||||
@@ -149,6 +148,7 @@ func (s *SyncManager) Start() error {
|
||||
shouldInterruptLoop = true
|
||||
} else {
|
||||
var queuesToSync []string
|
||||
//TODO: implement scrambling to avoid starvation of queues
|
||||
if s.syncStatus != "" {
|
||||
queuesToSync = append(queuesToSync, s.syncStatus)
|
||||
} else if s.SyncFlags.SyncUpdate {
|
||||
@@ -160,7 +160,7 @@ func (s *SyncManager) Start() error {
|
||||
for _, q := range queuesToSync {
|
||||
//temporary override for sync-until to give tom the time to review the channels
|
||||
if q == StatusQueued {
|
||||
s.syncProperties.SyncUntil = time.Now().Add(-8 * time.Hour).Unix()
|
||||
s.syncProperties.SyncUntil = time.Now().AddDate(0, 0, -1).Unix()
|
||||
}
|
||||
channels, err := s.apiConfig.FetchChannels(q, s.syncProperties)
|
||||
if err != nil {
|
||||
@@ -202,12 +202,6 @@ func (s *SyncManager) Start() error {
|
||||
shouldNotCount := false
|
||||
logUtils.SendInfoToSlack("Syncing %s (%s) to LBRY! total processed channels since startup: %d", sync.LbryChannelName, sync.YoutubeChannelID, syncCount+1)
|
||||
err := sync.FullCycle()
|
||||
//TODO: THIS IS A TEMPORARY WORK AROUND FOR THE STUPID IP LOCKUP BUG
|
||||
ipPool, _ := ip_manager.GetIPPool(sync.grp)
|
||||
if ipPool != nil {
|
||||
ipPool.ReleaseAll()
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
fatalErrors := []string{
|
||||
"default_wallet already exists",
|
||||
@@ -218,7 +212,6 @@ func (s *SyncManager) Start() error {
|
||||
"the channel in the wallet is different than the channel in the database",
|
||||
"this channel does not belong to this wallet!",
|
||||
"You already have a stream claim published under the name",
|
||||
"Daily Limit Exceeded",
|
||||
}
|
||||
if util.SubstringInSlice(err.Error(), fatalErrors) {
|
||||
return errors.Prefix("@Nikooo777 this requires manual intervention! Exiting...", err)
|
||||
|
||||
+33
-62
@@ -10,10 +10,9 @@ import (
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
logUtils "github.com/lbryio/ytsync/util"
|
||||
|
||||
"github.com/lbryio/ytsync/tags_manager"
|
||||
"github.com/lbryio/ytsync/tagsManager"
|
||||
"github.com/lbryio/ytsync/thumbs"
|
||||
logUtils "github.com/lbryio/ytsync/util"
|
||||
|
||||
"github.com/shopspring/decimal"
|
||||
log "github.com/sirupsen/logrus"
|
||||
@@ -22,21 +21,14 @@ import (
|
||||
)
|
||||
|
||||
func (s *Sync) enableAddressReuse() error {
|
||||
accountsResponse, err := s.daemon.AccountList(1, 50)
|
||||
accountsResponse, err := s.daemon.AccountList()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
accounts := make([]jsonrpc.Account, 0, len(accountsResponse.Items))
|
||||
ledger := "lbc_mainnet"
|
||||
accounts := accountsResponse.LBCMainnet
|
||||
if logUtils.IsRegTest() {
|
||||
ledger = "lbc_regtest"
|
||||
accounts = accountsResponse.LBCRegtest
|
||||
}
|
||||
for _, a := range accountsResponse.Items {
|
||||
if *a.Ledger == ledger {
|
||||
accounts = append(accounts, a)
|
||||
}
|
||||
}
|
||||
|
||||
for _, a := range accounts {
|
||||
_, err = s.daemon.AccountSet(a.ID, jsonrpc.AccountSettings{
|
||||
ChangeMaxUses: util.PtrToInt(1000),
|
||||
@@ -128,13 +120,13 @@ func (s *Sync) walletSetup() error {
|
||||
}
|
||||
}
|
||||
|
||||
claimAddress, err := s.daemon.AddressList(nil, nil, 1, 20)
|
||||
claimAddress, err := s.daemon.AddressList(nil, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
} else if claimAddress == nil {
|
||||
return errors.Err("could not get an address")
|
||||
return errors.Err("could not get unused address")
|
||||
}
|
||||
s.claimAddress = string(claimAddress.Items[0].Address)
|
||||
s.claimAddress = string((*claimAddress)[0].Address)
|
||||
if s.claimAddress == "" {
|
||||
return errors.Err("found blank claim address")
|
||||
}
|
||||
@@ -152,23 +144,20 @@ func (s *Sync) walletSetup() error {
|
||||
|
||||
func (s *Sync) getDefaultAccount() (string, error) {
|
||||
if s.defaultAccountID == "" {
|
||||
accountsResponse, err := s.daemon.AccountList(1, 50)
|
||||
accounts, err := s.daemon.AccountList()
|
||||
if err != nil {
|
||||
return "", errors.Err(err)
|
||||
}
|
||||
ledger := "lbc_mainnet"
|
||||
accountsNet := (*accounts).LBCMainnet
|
||||
if logUtils.IsRegTest() {
|
||||
ledger = "lbc_regtest"
|
||||
accountsNet = (*accounts).LBCRegtest
|
||||
}
|
||||
for _, a := range accountsResponse.Items {
|
||||
if *a.Ledger == ledger {
|
||||
if a.IsDefault {
|
||||
s.defaultAccountID = a.ID
|
||||
break
|
||||
}
|
||||
for _, account := range accountsNet {
|
||||
if account.IsDefault {
|
||||
s.defaultAccountID = account.ID
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if s.defaultAccountID == "" {
|
||||
return "", errors.Err("No default account found")
|
||||
}
|
||||
@@ -182,7 +171,7 @@ func (s *Sync) ensureEnoughUTXOs() error {
|
||||
return err
|
||||
}
|
||||
|
||||
utxolist, err := s.daemon.UTXOList(&defaultAccount, 1, 10000)
|
||||
utxolist, err := s.daemon.UTXOList(&defaultAccount)
|
||||
if err != nil {
|
||||
return err
|
||||
} else if utxolist == nil {
|
||||
@@ -194,7 +183,7 @@ func (s *Sync) ensureEnoughUTXOs() error {
|
||||
count := 0
|
||||
confirmedCount := 0
|
||||
|
||||
for _, utxo := range utxolist.Items {
|
||||
for _, utxo := range *utxolist {
|
||||
amount, _ := strconv.ParseFloat(utxo.Amount, 64)
|
||||
if utxo.IsMine && utxo.Type == "payment" && amount > 0.001 {
|
||||
if utxo.Confirmations > 0 {
|
||||
@@ -250,11 +239,21 @@ func (s *Sync) ensureEnoughUTXOs() error {
|
||||
}
|
||||
|
||||
func (s *Sync) waitForNewBlock() error {
|
||||
log.Printf("regtest: %t, docker: %t", logUtils.IsRegTest(), logUtils.IsUsingDocker())
|
||||
if logUtils.IsRegTest() && logUtils.IsUsingDocker() {
|
||||
lbrycrd, err := logUtils.GetLbrycrdClient(s.LbrycrdString)
|
||||
if err != nil {
|
||||
return errors.Prefix("error getting lbrycrd client: ", err)
|
||||
}
|
||||
txs, err := lbrycrd.Generate(1)
|
||||
for _, tx := range txs {
|
||||
log.Info("Generated tx: ", tx.String())
|
||||
}
|
||||
}
|
||||
status, err := s.daemon.Status()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for status.Wallet.Blocks == 0 || status.Wallet.BlocksBehind != 0 {
|
||||
time.Sleep(5 * time.Second)
|
||||
status, err = s.daemon.Status()
|
||||
@@ -262,18 +261,11 @@ func (s *Sync) waitForNewBlock() error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
currentBlock := status.Wallet.Blocks
|
||||
for i := 0; status.Wallet.Blocks <= currentBlock; i++ {
|
||||
if i%3 == 0 {
|
||||
log.Printf("Waiting for new block (%d)...", currentBlock+1)
|
||||
}
|
||||
if logUtils.IsRegTest() && logUtils.IsUsingDocker() {
|
||||
err = s.GenerateRegtestBlock()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
time.Sleep(10 * time.Second)
|
||||
status, err = s.daemon.Status()
|
||||
if err != nil {
|
||||
@@ -283,26 +275,14 @@ func (s *Sync) waitForNewBlock() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Sync) GenerateRegtestBlock() error {
|
||||
lbrycrd, err := logUtils.GetLbrycrdClient(s.LbrycrdString)
|
||||
if err != nil {
|
||||
return errors.Prefix("error getting lbrycrd client: ", err)
|
||||
}
|
||||
txs, err := lbrycrd.Generate(1)
|
||||
if err != nil {
|
||||
return errors.Prefix("error generating new block: ", err)
|
||||
}
|
||||
for _, tx := range txs {
|
||||
log.Info("Generated tx: ", tx.String())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Sync) ensureChannelOwnership() error {
|
||||
if s.LbryChannelName == "" {
|
||||
return errors.Err("no channel name set")
|
||||
}
|
||||
|
||||
if s.transferState == TransferStateComplete {
|
||||
return nil
|
||||
}
|
||||
channels, err := s.daemon.ChannelList(nil, 1, 50, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -329,8 +309,6 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
if channelToUse == nil {
|
||||
return errors.Err("this wallet has channels but not a single one is ours! Expected claim_id: %s (%s)", s.lbryChannelID, s.LbryChannelName)
|
||||
}
|
||||
} else if s.transferState == TransferStateComplete {
|
||||
return errors.Err("the channel was transferred but appears to have been abandoned!")
|
||||
}
|
||||
|
||||
channelUsesOldMetadata := false
|
||||
@@ -398,9 +376,6 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
|
||||
var languages []string = nil
|
||||
if channelInfo.DefaultLanguage != "" {
|
||||
if channelInfo.DefaultLanguage == "iw" {
|
||||
channelInfo.DefaultLanguage = "he"
|
||||
}
|
||||
languages = []string{channelInfo.DefaultLanguage}
|
||||
}
|
||||
var locations []jsonrpc.Location = nil
|
||||
@@ -411,7 +386,7 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
claimCreateOptions := jsonrpc.ClaimCreateOptions{
|
||||
Title: &channelInfo.Title,
|
||||
Description: &channelInfo.Description,
|
||||
Tags: tags_manager.GetTagsForChannel(s.YoutubeChannelID),
|
||||
Tags: tagsManager.GetTagsForChannel(s.YoutubeChannelID),
|
||||
Languages: languages,
|
||||
Locations: locations,
|
||||
ThumbnailURL: &thumbnailURL,
|
||||
@@ -447,11 +422,7 @@ func (s *Sync) addCredits(amountToAdd float64) error {
|
||||
return err
|
||||
}
|
||||
|
||||
defaultAccount, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
addressResp, err := s.daemon.AddressUnused(&defaultAccount)
|
||||
addressResp, err := s.daemon.AddressUnused(nil)
|
||||
if err != nil {
|
||||
return err
|
||||
} else if addressResp == nil {
|
||||
|
||||
+64
-73
@@ -6,6 +6,8 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/uber-go/atomic"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
@@ -24,14 +26,14 @@ func waitConfirmations(s *Sync) error {
|
||||
allConfirmed := false
|
||||
waiting:
|
||||
for !allConfirmed {
|
||||
utxolist, err := s.daemon.UTXOList(&defaultAccount, 1, 10000)
|
||||
utxolist, err := s.daemon.UTXOList(&defaultAccount)
|
||||
if err != nil {
|
||||
return err
|
||||
} else if utxolist == nil {
|
||||
return errors.Err("no response")
|
||||
}
|
||||
|
||||
for _, utxo := range utxolist.Items {
|
||||
for _, utxo := range *utxolist {
|
||||
if utxo.Confirmations <= 0 {
|
||||
err = s.waitForNewBlock()
|
||||
if err != nil {
|
||||
@@ -45,13 +47,13 @@ waiting:
|
||||
return nil
|
||||
}
|
||||
|
||||
type abandonResponse struct {
|
||||
ClaimID string
|
||||
Error error
|
||||
Amount float64
|
||||
}
|
||||
|
||||
func abandonSupports(s *Sync) (float64, error) {
|
||||
type abandonResponse struct {
|
||||
ClaimID string
|
||||
Error error
|
||||
Amount float64
|
||||
}
|
||||
|
||||
totalPages := uint64(1)
|
||||
var allSupports []jsonrpc.Claim
|
||||
defaultAccount, err := s.getDefaultAccount()
|
||||
@@ -66,23 +68,25 @@ func abandonSupports(s *Sync) (float64, error) {
|
||||
allSupports = append(allSupports, (*supports).Items...)
|
||||
totalPages = (*supports).TotalPages
|
||||
}
|
||||
producerWG := &stop.Group{}
|
||||
|
||||
claimIDChan := make(chan string, len(allSupports))
|
||||
abandonRspChan := make(chan abandonResponse, len(allSupports))
|
||||
claimIDChan := make(chan string)
|
||||
abandonRspChan := make(chan abandonResponse)
|
||||
collectorChan := make(chan bool)
|
||||
|
||||
alreadyAbandoned := make(map[string]bool, len(allSupports))
|
||||
producerWG.Add(1)
|
||||
totalAbandoned := atomic.NewFloat64(0)
|
||||
|
||||
go func() {
|
||||
defer producerWG.Done()
|
||||
for _, support := range allSupports {
|
||||
_, ok := alreadyAbandoned[support.ClaimID]
|
||||
if ok {
|
||||
for r := range abandonRspChan {
|
||||
if r.Error != nil {
|
||||
log.Errorf("Failed abandoning supports for %s: %s", r.ClaimID, r.Error.Error())
|
||||
continue
|
||||
}
|
||||
alreadyAbandoned[support.ClaimID] = true
|
||||
claimIDChan <- support.ClaimID
|
||||
totalAbandoned.Add(r.Amount)
|
||||
}
|
||||
close(collectorChan)
|
||||
}()
|
||||
|
||||
consumerWG := &stop.Group{}
|
||||
//TODO: remove this once the SDK team fixes their RPC bugs....
|
||||
s.daemon.SetRPCTimeout(30 * time.Second)
|
||||
@@ -95,72 +99,64 @@ func abandonSupports(s *Sync) (float64, error) {
|
||||
claimID, more := <-claimIDChan
|
||||
if !more {
|
||||
return
|
||||
} else {
|
||||
summary, err := s.daemon.SupportAbandon(&claimID, nil, nil, nil, &defaultAccount)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "Client.Timeout exceeded while awaiting headers") {
|
||||
log.Errorf("Support abandon for %s timed out, retrying...", claimID)
|
||||
summary, err = s.daemon.SupportAbandon(&claimID, nil, nil, nil, &defaultAccount)
|
||||
if err != nil {
|
||||
//TODO GUESS HOW MUCH LBC WAS RELEASED THAT WE DON'T KNOW ABOUT, because screw you SDK
|
||||
abandonRspChan <- abandonResponse{
|
||||
ClaimID: claimID,
|
||||
Error: err,
|
||||
Amount: 0, // this is likely wrong, but oh well... there is literally nothing I can do about it
|
||||
}
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
}
|
||||
|
||||
summary, err := s.daemon.SupportAbandon(&claimID, nil, nil, nil, nil)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "Client.Timeout exceeded while awaiting headers") {
|
||||
log.Errorf("Support abandon for %s timed out, retrying...", claimID)
|
||||
summary, err = s.daemon.SupportAbandon(&claimID, nil, nil, nil, nil)
|
||||
if err != nil {
|
||||
//TODO GUESS HOW MUCH LBC WAS RELEASED THAT WE DON'T KNOW ABOUT, because screw you SDK
|
||||
abandonRspChan <- abandonResponse{
|
||||
ClaimID: claimID,
|
||||
Error: err,
|
||||
Amount: 0,
|
||||
Amount: 0, // this is likely wrong, but oh well... there is literally nothing I can do about it
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
if len(summary.Outputs) < 1 {
|
||||
abandonRspChan <- abandonResponse{
|
||||
ClaimID: claimID,
|
||||
Error: errors.Err("error abandoning supports: no outputs while abandoning %s", claimID),
|
||||
Amount: 0,
|
||||
}
|
||||
} else {
|
||||
abandonRspChan <- abandonResponse{ClaimID: claimID, Error: err}
|
||||
continue
|
||||
}
|
||||
outputAmount, err := strconv.ParseFloat(summary.Outputs[0].Amount, 64)
|
||||
if err != nil {
|
||||
abandonRspChan <- abandonResponse{
|
||||
ClaimID: claimID,
|
||||
Error: errors.Err(err),
|
||||
Amount: 0,
|
||||
}
|
||||
continue
|
||||
}
|
||||
log.Infof("Abandoned supports of %.4f LBC for claim %s", outputAmount, claimID)
|
||||
}
|
||||
|
||||
if len(summary.Outputs) < 1 {
|
||||
abandonRspChan <- abandonResponse{
|
||||
ClaimID: claimID,
|
||||
Error: nil,
|
||||
Amount: outputAmount,
|
||||
Error: errors.Err("abandoning supports: no outputs for %s", claimID),
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
outputAmount, err := strconv.ParseFloat(summary.Outputs[0].Amount, 64)
|
||||
if err != nil {
|
||||
abandonRspChan <- abandonResponse{ClaimID: claimID, Error: errors.Err(err)}
|
||||
continue
|
||||
}
|
||||
|
||||
log.Infof("Abandoned supports of %.4f LBC for claim %s", outputAmount, claimID)
|
||||
abandonRspChan <- abandonResponse{ClaimID: claimID, Amount: outputAmount}
|
||||
continue
|
||||
}
|
||||
}()
|
||||
}
|
||||
producerWG.Wait()
|
||||
close(claimIDChan)
|
||||
consumerWG.Wait()
|
||||
close(abandonRspChan)
|
||||
|
||||
totalAbandoned := 0.0
|
||||
for r := range abandonRspChan {
|
||||
if r.Error != nil {
|
||||
log.Errorf("Failed abandoning supports for %s: %s", r.ClaimID, r.Error.Error())
|
||||
for _, support := range allSupports {
|
||||
_, ok := alreadyAbandoned[support.ClaimID]
|
||||
if ok {
|
||||
continue
|
||||
}
|
||||
totalAbandoned += r.Amount
|
||||
alreadyAbandoned[support.ClaimID] = true
|
||||
claimIDChan <- support.ClaimID
|
||||
}
|
||||
return totalAbandoned, nil
|
||||
close(claimIDChan)
|
||||
|
||||
consumerWG.Wait()
|
||||
close(abandonRspChan)
|
||||
<-collectorChan
|
||||
|
||||
return totalAbandoned.Load(), nil
|
||||
}
|
||||
|
||||
type updateInfo struct {
|
||||
@@ -177,7 +173,7 @@ func transferVideos(s *Sync) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
streams, err := s.daemon.StreamList(&account, 1, 30000)
|
||||
streams, err := s.daemon.StreamList(&account)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
@@ -191,8 +187,8 @@ func transferVideos(s *Sync) error {
|
||||
}
|
||||
|
||||
var stream *jsonrpc.Claim = nil
|
||||
for _, c := range streams.Items {
|
||||
if c.ClaimID != video.ClaimID || (c.SigningChannel != nil && c.SigningChannel.ClaimID != s.lbryChannelID) {
|
||||
for _, c := range *streams {
|
||||
if c.ClaimID != video.ClaimID {
|
||||
continue
|
||||
}
|
||||
stream = &c
|
||||
@@ -204,12 +200,7 @@ func transferVideos(s *Sync) error {
|
||||
|
||||
streamUpdateOptions := jsonrpc.StreamUpdateOptions{
|
||||
StreamCreateOptions: &jsonrpc.StreamCreateOptions{
|
||||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
|
||||
ClaimAddress: &s.clientPublishAddress,
|
||||
FundingAccountIDs: []string{
|
||||
account,
|
||||
},
|
||||
},
|
||||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{ClaimAddress: &s.clientPublishAddress},
|
||||
},
|
||||
Bid: util.PtrToString("0.005"), // Todo - Dont hardcode
|
||||
}
|
||||
|
||||
+36
-86
@@ -14,7 +14,6 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/ytsync/ip_manager"
|
||||
"github.com/lbryio/ytsync/namer"
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
"github.com/lbryio/ytsync/sources"
|
||||
@@ -246,8 +245,6 @@ func (s *Sync) setExceptions() {
|
||||
}
|
||||
}
|
||||
|
||||
var stopGroup = stop.New()
|
||||
|
||||
func (s *Sync) FullCycle() (e error) {
|
||||
if os.Getenv("HOME") == "" {
|
||||
return errors.Err("no $HOME env var found")
|
||||
@@ -260,7 +257,7 @@ func (s *Sync) FullCycle() (e error) {
|
||||
|
||||
s.syncedVideosMux = &sync.RWMutex{}
|
||||
s.walletMux = &sync.RWMutex{}
|
||||
s.grp = stopGroup
|
||||
s.grp = stop.New()
|
||||
s.queue = make(chan video)
|
||||
interruptChan := make(chan os.Signal, 1)
|
||||
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
|
||||
@@ -336,7 +333,7 @@ func (s *Sync) processTransfers() (e error) {
|
||||
return errors.Prefix(fmt.Sprintf("%.6f LBCs were abandoned before failing", supportAmount), err)
|
||||
}
|
||||
if supportAmount > 0 {
|
||||
logUtils.SendInfoToSlack("(%s) %.6f LBCs were abandoned and should be used as support", s.YoutubeChannelID, supportAmount)
|
||||
logUtils.SendInfoToSlack("%.6f LBCs were abandoned and should be used as support", supportAmount)
|
||||
}
|
||||
err = transferVideos(s)
|
||||
if err != nil {
|
||||
@@ -357,20 +354,9 @@ func (s *Sync) processTransfers() (e error) {
|
||||
return err
|
||||
}
|
||||
isTip := true
|
||||
summary, err := s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount), &isTip, nil, []string{defaultAccount}, nil)
|
||||
summary, err := s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount), &isTip, nil, []string{defaultAccount})
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "tx-size") { //TODO: this is a silly workaround and should be written in an recursive function
|
||||
summary, err = s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount/2.0), &isTip, nil, []string{defaultAccount}, nil)
|
||||
if err != nil {
|
||||
return errors.Prefix(fmt.Sprintf("something went wrong while tipping the channel for %.6f LBCs", supportAmount), err)
|
||||
}
|
||||
summary, err = s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount/2.0), &isTip, nil, []string{defaultAccount}, nil)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
} else {
|
||||
return errors.Err(err)
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
if len(summary.Outputs) < 1 {
|
||||
return errors.Err("something went wrong while tipping the channel for %.6f LBCs", supportAmount)
|
||||
@@ -398,7 +384,9 @@ func (s *Sync) setChannelTerminationStatus(e *error) {
|
||||
var transferState *int
|
||||
|
||||
if s.shouldTransfer() {
|
||||
if *e == nil {
|
||||
if *e != nil {
|
||||
transferState = util.PtrToInt(TransferStateFailed)
|
||||
} else {
|
||||
transferState = util.PtrToInt(TransferStateComplete)
|
||||
}
|
||||
}
|
||||
@@ -480,7 +468,7 @@ var thumbnailHosts = []string{
|
||||
thumbs.ThumbnailEndpoint,
|
||||
}
|
||||
|
||||
func isYtsyncClaim(c jsonrpc.Claim, expectedChannelID string) bool {
|
||||
func isYtsyncClaim(c jsonrpc.Claim) bool {
|
||||
if !util.InSlice(c.Type, []string{"claim", "update"}) || c.Value.GetStream() == nil {
|
||||
return false
|
||||
}
|
||||
@@ -488,12 +476,7 @@ func isYtsyncClaim(c jsonrpc.Claim, expectedChannelID string) bool {
|
||||
//most likely a claim created outside of ytsync, ignore!
|
||||
return false
|
||||
}
|
||||
if c.SigningChannel == nil {
|
||||
return false
|
||||
}
|
||||
if c.SigningChannel.ClaimID != expectedChannelID {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, th := range thumbnailHosts {
|
||||
if strings.Contains(c.Value.GetThumbnail().GetUrl(), th) {
|
||||
return true
|
||||
@@ -507,7 +490,7 @@ func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) {
|
||||
abandonedClaims := false
|
||||
videoIDs := make(map[string]jsonrpc.Claim)
|
||||
for _, c := range claims {
|
||||
if !isYtsyncClaim(c, s.lbryChannelID) {
|
||||
if !isYtsyncClaim(c) {
|
||||
continue
|
||||
}
|
||||
tn := c.Value.GetThumbnail().GetUrl()
|
||||
@@ -554,7 +537,7 @@ type ytsyncClaim struct {
|
||||
func (s *Sync) mapFromClaims(claims []jsonrpc.Claim) map[string]ytsyncClaim {
|
||||
videoIDMap := make(map[string]ytsyncClaim, len(claims))
|
||||
for _, c := range claims {
|
||||
if !isYtsyncClaim(c, s.lbryChannelID) {
|
||||
if !isYtsyncClaim(c) {
|
||||
continue
|
||||
}
|
||||
tn := c.Value.GetThumbnail().GetUrl()
|
||||
@@ -651,18 +634,8 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
|
||||
if sv.Transferred {
|
||||
_, ok := allClaimsInfo[vID]
|
||||
if !ok && sv.Published {
|
||||
searchResponse, err := s.daemon.ClaimSearch(nil, &sv.ClaimID, nil, nil, 1, 20)
|
||||
if err != nil {
|
||||
log.Error(err.Error())
|
||||
continue
|
||||
}
|
||||
if len(searchResponse.Claims) == 0 {
|
||||
log.Debugf("%s: was transferred but appears abandoned! we should ignore this - claimID: %s", vID, sv.ClaimID)
|
||||
continue //TODO: we should flag these on the db
|
||||
} else {
|
||||
log.Debugf("%s: was transferred and was then edited! we should ignore this - claimID: %s", vID, sv.ClaimID)
|
||||
//return count, fixed, 0, errors.Err("%s: isn't our control but is on the database and on the blockchain. wtf is up? ClaimID: %s", vID, sv.ClaimID)
|
||||
}
|
||||
log.Warnf("%s: claims to be published and transferred but wasn't found in the list of claims", vID)
|
||||
idsToRemove = append(idsToRemove, vID)
|
||||
}
|
||||
continue
|
||||
}
|
||||
@@ -699,17 +672,12 @@ func (s *Sync) getClaims(defaultOnly bool) ([]jsonrpc.Claim, error) {
|
||||
}
|
||||
account = &a
|
||||
}
|
||||
claims, err := s.daemon.StreamList(account, 1, 30000)
|
||||
claims, err := s.daemon.StreamList(account)
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("cannot list claims", err)
|
||||
}
|
||||
items := make([]jsonrpc.Claim, 0, len(claims.Items))
|
||||
for _, c := range claims.Items {
|
||||
if c.SigningChannel != nil && c.SigningChannel.ClaimID == s.lbryChannelID {
|
||||
items = append(items, c)
|
||||
}
|
||||
}
|
||||
return items, nil
|
||||
|
||||
return *claims, nil
|
||||
}
|
||||
|
||||
func (s *Sync) checkIntegrity() error {
|
||||
@@ -791,7 +759,7 @@ func (s *Sync) doSync() error {
|
||||
return err
|
||||
}
|
||||
|
||||
if s.transferState < TransferStateComplete {
|
||||
if s.transferState != TransferStateComplete {
|
||||
cert, err := s.daemon.ChannelExport(s.lbryChannelID, nil, nil)
|
||||
if err != nil {
|
||||
return errors.Prefix("error getting channel cert", err)
|
||||
@@ -856,9 +824,9 @@ func (s *Sync) startWorker(workerNum int) {
|
||||
err := s.processVideo(v)
|
||||
|
||||
if err != nil {
|
||||
logMsg := fmt.Sprintf("error processing video %s: %s", v.ID(), err.Error())
|
||||
logMsg := fmt.Sprintf("error processing video %s: %s", v.ID(), errors.FullTrace(err))
|
||||
log.Errorln(logMsg)
|
||||
if strings.Contains(strings.ToLower(err.Error()), "interrupted by user") {
|
||||
if errors.Is(err, sources.ErrInterruptedByUser) {
|
||||
return
|
||||
}
|
||||
fatalErrors := []string{
|
||||
@@ -895,18 +863,12 @@ func (s *Sync) startWorker(workerNum int) {
|
||||
"giving up after 0 fragment retries",
|
||||
"Sorry about that",
|
||||
"This video is not available",
|
||||
"requested format not available",
|
||||
"interrupted by user",
|
||||
"Sign in to confirm your age",
|
||||
"This video is unavailable",
|
||||
}
|
||||
if util.SubstringInSlice(err.Error(), errorsNoRetry) {
|
||||
log.Println("This error should not be retried at all")
|
||||
} else if tryCount < s.MaxTries {
|
||||
if util.SubstringInSlice(err.Error(), []string{
|
||||
"txn-mempool-conflict",
|
||||
"too-long-mempool-chain",
|
||||
}) {
|
||||
if strings.Contains(err.Error(), "txn-mempool-conflict") ||
|
||||
strings.Contains(err.Error(), "too-long-mempool-chain") {
|
||||
log.Println("waiting for a block before retrying")
|
||||
err := s.waitForNewBlock()
|
||||
if err != nil {
|
||||
@@ -918,7 +880,6 @@ func (s *Sync) startWorker(workerNum int) {
|
||||
"Not enough funds to cover this transaction",
|
||||
"failed: Not enough funds",
|
||||
"Error in daemon: Insufficient funds, please deposit additional LBC",
|
||||
"Missing inputs",
|
||||
}) {
|
||||
log.Println("checking funds and UTXOs before retrying...")
|
||||
err := s.walletSetup()
|
||||
@@ -1004,11 +965,6 @@ func (s *Sync) enqueueYoutubeVideos() error {
|
||||
}
|
||||
|
||||
var videos []video
|
||||
ipPool, err := ip_manager.GetIPPool(s.grp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
playlistMap := make(map[string]*youtube.PlaylistItemSnippet, 50)
|
||||
nextPageToken := ""
|
||||
for {
|
||||
@@ -1031,6 +987,7 @@ func (s *Sync) enqueueYoutubeVideos() error {
|
||||
}
|
||||
return errors.Err("playlist items not found")
|
||||
}
|
||||
//playlistMap := make(map[string]*youtube.PlaylistItemSnippet, 50)
|
||||
videoIDs := make([]string, 50)
|
||||
for i, item := range playlistResponse.Items {
|
||||
// normally we'd send the video into the channel here, but youtube api doesn't have sorting
|
||||
@@ -1045,13 +1002,13 @@ func (s *Sync) enqueueYoutubeVideos() error {
|
||||
return errors.Prefix("error getting videos info", err)
|
||||
}
|
||||
for _, item := range videosListResponse.Items {
|
||||
videos = append(videos, sources.NewYoutubeVideo(s.videoDirectory, item, playlistMap[item.Id].Position, s.Manager.GetS3AWSConfig(), s.grp, ipPool))
|
||||
videos = append(videos, sources.NewYoutubeVideo(s.videoDirectory, item, playlistMap[item.Id].Position, s.Manager.GetS3AWSConfig(), s.grp))
|
||||
}
|
||||
|
||||
log.Infof("Got info for %d videos from youtube API", len(videos))
|
||||
|
||||
nextPageToken = playlistResponse.NextPageToken
|
||||
if nextPageToken == "" || s.Manager.SyncFlags.QuickSync || len(videos) >= s.Manager.videosLimit {
|
||||
if nextPageToken == "" {
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -1061,11 +1018,12 @@ func (s *Sync) enqueueYoutubeVideos() error {
|
||||
}
|
||||
_, ok := playlistMap[k]
|
||||
if !ok {
|
||||
videos = append(videos, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig(), s.grp, ipPool))
|
||||
videos = append(videos, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig(), s.grp))
|
||||
}
|
||||
|
||||
}
|
||||
sort.Sort(byPublishedAt(videos))
|
||||
//or sort.Sort(sort.Reverse(byPlaylistPosition(videos)))
|
||||
|
||||
Enqueue:
|
||||
for _, v := range videos {
|
||||
@@ -1088,7 +1046,7 @@ Enqueue:
|
||||
func (s *Sync) processVideo(v video) (err error) {
|
||||
defer func() {
|
||||
if p := recover(); p != nil {
|
||||
logUtils.SendErrorToSlack("Video processing panic! %s", debug.Stack())
|
||||
log.Printf("stack: %s", debug.Stack())
|
||||
var ok bool
|
||||
err, ok = p.(error)
|
||||
if !ok {
|
||||
@@ -1120,7 +1078,6 @@ func (s *Sync) processVideo(v video) (err error) {
|
||||
"Watch this video on YouTube.",
|
||||
"have blocked it on copyright grounds",
|
||||
"giving up after 0 fragment retries",
|
||||
"Sign in to confirm your age",
|
||||
}
|
||||
if ok && !sv.Published && util.SubstringInSlice(sv.FailureReason, neverRetryFailures) {
|
||||
log.Println(v.ID() + " can't ever be published")
|
||||
@@ -1144,10 +1101,6 @@ func (s *Sync) processVideo(v video) (err error) {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
da, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sp := sources.SyncParams{
|
||||
ClaimAddress: s.claimAddress,
|
||||
Amount: publishAmount,
|
||||
@@ -1156,7 +1109,6 @@ func (s *Sync) processVideo(v video) (err error) {
|
||||
Namer: s.namer,
|
||||
MaxVideoLength: s.Manager.maxVideoLength,
|
||||
Fee: s.Fee,
|
||||
DefaultAccount: da,
|
||||
}
|
||||
|
||||
summary, err := v.Sync(s.daemon, sp, &sv, videoRequiresUpgrade, s.walletMux)
|
||||
@@ -1184,19 +1136,17 @@ func (s *Sync) processVideo(v video) (err error) {
|
||||
|
||||
func (s *Sync) importPublicKey() error {
|
||||
if s.publicKey != "" {
|
||||
accountsResponse, err := s.daemon.AccountList(1, 50)
|
||||
accountsResponse, err := s.daemon.AccountList()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
ledger := "lbc_mainnet"
|
||||
accounts := accountsResponse.LBCMainnet
|
||||
if logUtils.IsRegTest() {
|
||||
ledger = "lbc_regtest"
|
||||
accounts = accountsResponse.LBCRegtest
|
||||
}
|
||||
for _, a := range accountsResponse.Items {
|
||||
if *a.Ledger == ledger {
|
||||
if a.PublicKey == s.publicKey {
|
||||
return nil
|
||||
}
|
||||
for _, a := range accounts {
|
||||
if a.PublicKey == s.publicKey {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
log.Infof("Could not find public key %s in the wallet. Importing it...", s.publicKey)
|
||||
@@ -1224,12 +1174,12 @@ func (s *Sync) getUnsentSupports() (float64, error) {
|
||||
if err != nil {
|
||||
return 0, errors.Err(err)
|
||||
}
|
||||
transactionList, err := s.daemon.TransactionList(&defaultAccount, 1, 90000)
|
||||
transactionList, err := s.daemon.TransactionList(&defaultAccount)
|
||||
if err != nil {
|
||||
return 0, errors.Err(err)
|
||||
}
|
||||
sentSupports := 0.0
|
||||
for _, t := range transactionList.Items {
|
||||
for _, t := range *transactionList {
|
||||
if len(t.SupportInfo) == 0 {
|
||||
continue
|
||||
}
|
||||
@@ -1244,7 +1194,7 @@ func (s *Sync) getUnsentSupports() (float64, error) {
|
||||
}
|
||||
}
|
||||
if balanceAmount > 10 && sentSupports < 1 {
|
||||
logUtils.SendErrorToSlack("(%s) this channel has quite some LBCs in it (%.2f) and %.2f LBC in sent tips, it's likely that the tips weren't actually sent or the wallet has unnecessary extra credits in it", s.YoutubeChannelID, balanceAmount, sentSupports)
|
||||
logUtils.SendErrorToSlack("this channel has quite some LBCs in it (%.2f) and %.2f LBC in sent tips, it's likely that the tips weren't actually sent or the wallet has unnecessary extra credits in it", balanceAmount, sentSupports)
|
||||
return balanceAmount - 10, nil
|
||||
}
|
||||
}
|
||||
|
||||
+6
-7
@@ -45,7 +45,6 @@ type SyncFlags struct {
|
||||
RemoveDBUnpublished bool
|
||||
UpgradeMetadata bool
|
||||
DisableTransfers bool
|
||||
QuickSync bool
|
||||
}
|
||||
|
||||
type Fee struct {
|
||||
@@ -87,7 +86,7 @@ func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeC
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
if res.StatusCode != http.StatusOK {
|
||||
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
|
||||
log.Debugln(string(body))
|
||||
log.Debugln(body)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.FetchChannels(status, cp)
|
||||
}
|
||||
@@ -186,9 +185,9 @@ func (a *APIConfig) SetChannelStatus(channelID string, status string, failureRea
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
if res.StatusCode >= http.StatusInternalServerError {
|
||||
if res.StatusCode != http.StatusOK {
|
||||
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
|
||||
log.Debugln(string(body))
|
||||
log.Debugln(body)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.SetChannelStatus(channelID, status, failureReason, transferState)
|
||||
}
|
||||
@@ -233,7 +232,7 @@ func (a *APIConfig) SetChannelClaimID(channelID string, channelClaimID string) e
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
if res.StatusCode != http.StatusOK {
|
||||
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
|
||||
log.Debugln(string(body))
|
||||
log.Debugln(body)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.SetChannelClaimID(channelID, channelClaimID)
|
||||
}
|
||||
@@ -272,7 +271,7 @@ func (a *APIConfig) DeleteVideos(videos []string) error {
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
if res.StatusCode != http.StatusOK {
|
||||
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
|
||||
log.Debugln(string(body))
|
||||
log.Debugln(body)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.DeleteVideos(videos)
|
||||
}
|
||||
@@ -345,7 +344,7 @@ func (a *APIConfig) MarkVideoStatus(status VideoStatus) error {
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
if res.StatusCode != http.StatusOK {
|
||||
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
|
||||
log.Debugln(string(body))
|
||||
log.Debugln(body)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.MarkVideoStatus(status)
|
||||
}
|
||||
|
||||
+102
-105
@@ -18,10 +18,10 @@ import (
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
|
||||
"github.com/lbryio/ytsync/ip_manager"
|
||||
"github.com/lbryio/ytsync/ipManager"
|
||||
"github.com/lbryio/ytsync/namer"
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
"github.com/lbryio/ytsync/tags_manager"
|
||||
"github.com/lbryio/ytsync/tagsManager"
|
||||
"github.com/lbryio/ytsync/thumbs"
|
||||
|
||||
duration "github.com/ChannelMeter/iso8601duration"
|
||||
@@ -31,6 +31,8 @@ import (
|
||||
"google.golang.org/api/youtube/v3"
|
||||
)
|
||||
|
||||
var ErrInterruptedByUser = errors.Base("interrupted by user")
|
||||
|
||||
type YoutubeVideo struct {
|
||||
id string
|
||||
title string
|
||||
@@ -50,7 +52,6 @@ type YoutubeVideo struct {
|
||||
mocked bool
|
||||
walletLock *sync.RWMutex
|
||||
stopGroup *stop.Group
|
||||
pool *ip_manager.IPPool
|
||||
}
|
||||
|
||||
var youtubeCategories = map[string]string{
|
||||
@@ -88,7 +89,7 @@ var youtubeCategories = map[string]string{
|
||||
"44": "trailers",
|
||||
}
|
||||
|
||||
func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo {
|
||||
func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group) *YoutubeVideo {
|
||||
publishedAt, _ := time.Parse(time.RFC3339Nano, videoData.Snippet.PublishedAt) // ignore parse errors
|
||||
return &YoutubeVideo{
|
||||
id: videoData.Id,
|
||||
@@ -102,10 +103,9 @@ func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPositio
|
||||
mocked: false,
|
||||
youtubeChannelID: videoData.Snippet.ChannelId,
|
||||
stopGroup: stopGroup,
|
||||
pool: pool,
|
||||
}
|
||||
}
|
||||
func NewMockedVideo(directory string, videoID string, youtubeChannelID string, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo {
|
||||
func NewMockedVideo(directory string, videoID string, youtubeChannelID string, awsConfig aws.Config, stopGroup *stop.Group) *YoutubeVideo {
|
||||
return &YoutubeVideo{
|
||||
id: videoID,
|
||||
playlistPosition: 0,
|
||||
@@ -114,7 +114,6 @@ func NewMockedVideo(directory string, videoID string, youtubeChannelID string, a
|
||||
mocked: true,
|
||||
youtubeChannelID: youtubeChannelID,
|
||||
stopGroup: stopGroup,
|
||||
pool: pool,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -178,7 +177,7 @@ func (v *YoutubeVideo) getAbbrevDescription() string {
|
||||
return description + "\n..." + additionalDescription
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) download() error {
|
||||
func (v *YoutubeVideo) download(useIPv6 bool) error {
|
||||
videoPath := v.getFullPath()
|
||||
|
||||
err := os.Mkdir(v.videoDir(), 0777)
|
||||
@@ -199,6 +198,7 @@ func (v *YoutubeVideo) download() error {
|
||||
"480",
|
||||
"320",
|
||||
}
|
||||
qualityIndex := 0
|
||||
ytdlArgs := []string{
|
||||
"--no-progress",
|
||||
"-o" + strings.TrimSuffix(v.getFullPath(), ".mp4"),
|
||||
@@ -209,10 +209,6 @@ func (v *YoutubeVideo) download() error {
|
||||
"--abort-on-unavailable-fragment",
|
||||
"--fragment-retries",
|
||||
"0",
|
||||
"--user-agent",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36",
|
||||
"--cookies",
|
||||
"cookies.txt",
|
||||
}
|
||||
if v.maxVideoSize > 0 {
|
||||
ytdlArgs = append(ytdlArgs,
|
||||
@@ -226,95 +222,102 @@ func (v *YoutubeVideo) download() error {
|
||||
fmt.Sprintf("duration <= %d", int(math.Round(v.maxVideoLength*3600))),
|
||||
)
|
||||
}
|
||||
|
||||
var sourceAddress string
|
||||
for {
|
||||
sourceAddress, err = v.pool.GetIP(v.id)
|
||||
if err != nil {
|
||||
if errors.Is(err, ip_manager.ErrAllThrottled) {
|
||||
sourceAddress, err := ipManager.GetNextIP(useIPv6)
|
||||
if err != nil {
|
||||
if sourceAddress == "throttled" {
|
||||
for {
|
||||
select {
|
||||
case <-v.stopGroup.Ch():
|
||||
return errors.Err("interrupted by user")
|
||||
return errors.Err(ErrInterruptedByUser)
|
||||
default:
|
||||
time.Sleep(ip_manager.IPCooldownPeriod)
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
return err
|
||||
|
||||
time.Sleep(ipManager.IPCooldownPeriod)
|
||||
sourceAddress, err = ipManager.GetNextIP(useIPv6)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return errors.Err(err)
|
||||
}
|
||||
break
|
||||
}
|
||||
defer v.pool.ReleaseIP(sourceAddress)
|
||||
defer ipManager.ReleaseIP(sourceAddress)
|
||||
if useIPv6 {
|
||||
log.Infof("using IPv6: %s", sourceAddress)
|
||||
ytdlArgs = append(ytdlArgs,
|
||||
"-6",
|
||||
"--source-address",
|
||||
sourceAddress,
|
||||
)
|
||||
} else {
|
||||
log.Infof("using IPv4: %s", sourceAddress)
|
||||
ytdlArgs = append(ytdlArgs,
|
||||
"-4",
|
||||
"--source-address",
|
||||
sourceAddress,
|
||||
)
|
||||
}
|
||||
ytdlArgs = append(ytdlArgs, "https://www.youtube.com/watch?v="+v.ID())
|
||||
runcmd:
|
||||
argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][height<="+qualities[qualityIndex]+"]+bestaudio[ext!=webm]")
|
||||
cmd := exec.Command("youtube-dl", argsWithFilters...)
|
||||
|
||||
ytdlArgs = append(ytdlArgs,
|
||||
"--source-address",
|
||||
sourceAddress,
|
||||
"https://www.youtube.com/watch?v="+v.ID(),
|
||||
)
|
||||
log.Printf("Running command youtube-dl %s", strings.Join(argsWithFilters, " "))
|
||||
|
||||
for i, quality := range qualities {
|
||||
argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][height<="+quality+"]+bestaudio[ext!=webm]")
|
||||
cmd := exec.Command("youtube-dl", argsWithFilters...)
|
||||
log.Printf("Running command youtube-dl %s", strings.Join(argsWithFilters, " "))
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
if err := cmd.Start(); err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
errorLog, _ := ioutil.ReadAll(stderr)
|
||||
outLog, _ := ioutil.ReadAll(stdout)
|
||||
|
||||
errorLog, _ := ioutil.ReadAll(stderr)
|
||||
outLog, _ := ioutil.ReadAll(stdout)
|
||||
err = cmd.Wait()
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "exit status 1") {
|
||||
if strings.Contains(string(errorLog), "HTTP Error 429") || strings.Contains(string(errorLog), "returned non-zero exit status 8") {
|
||||
v.pool.SetThrottled(sourceAddress)
|
||||
} else if strings.Contains(string(errorLog), "giving up after 0 fragment retries") {
|
||||
if i == (len(qualities) - 1) {
|
||||
return errors.Err(string(errorLog))
|
||||
}
|
||||
continue //this bypasses the yt throttling IP redistribution... TODO: don't
|
||||
}
|
||||
return errors.Err(string(errorLog))
|
||||
if err = cmd.Wait(); err != nil {
|
||||
if strings.Contains(err.Error(), "exit status 1") {
|
||||
if strings.Contains(string(errorLog), "HTTP Error 429") {
|
||||
ipManager.SetIpThrottled(sourceAddress, v.stopGroup)
|
||||
} else if strings.Contains(string(errorLog), "giving up after 0 fragment retries") && qualityIndex < len(qualities)-1 {
|
||||
qualityIndex++
|
||||
goto runcmd
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
log.Debugln(string(outLog))
|
||||
|
||||
if strings.Contains(string(outLog), "does not pass filter duration") {
|
||||
_ = v.delete("does not pass filter duration")
|
||||
return errors.Err("video is too long to process")
|
||||
}
|
||||
if strings.Contains(string(outLog), "File is larger than max-filesize") {
|
||||
_ = v.delete("File is larger than max-filesize")
|
||||
return errors.Err("the video is too big to sync, skipping for now")
|
||||
}
|
||||
if string(errorLog) != "" {
|
||||
log.Printf("Command finished with error: %v", errors.Err(string(errorLog)))
|
||||
_ = v.delete("due to error")
|
||||
return errors.Err(string(errorLog))
|
||||
}
|
||||
fi, err := os.Stat(v.getFullPath())
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
err = os.Chmod(v.getFullPath(), 0777)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
videoSize := fi.Size()
|
||||
v.size = &videoSize
|
||||
break
|
||||
return errors.Err(err)
|
||||
}
|
||||
log.Debugln(string(outLog))
|
||||
|
||||
if strings.Contains(string(outLog), "does not pass filter duration") {
|
||||
_ = v.delete("does not pass filter duration")
|
||||
return errors.Err("video is too long to process")
|
||||
}
|
||||
if strings.Contains(string(outLog), "File is larger than max-filesize") {
|
||||
_ = v.delete("File is larger than max-filesize")
|
||||
return errors.Err("the video is too big to sync, skipping for now")
|
||||
}
|
||||
if string(errorLog) != "" {
|
||||
log.Printf("Command finished with error: %v", errors.Err(string(errorLog)))
|
||||
_ = v.delete("due to error")
|
||||
return errors.Err(string(errorLog))
|
||||
}
|
||||
fi, err := os.Stat(v.getFullPath())
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
err = os.Chmod(v.getFullPath(), 0777)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
videoSize := fi.Size()
|
||||
v.size = &videoSize
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -367,6 +370,7 @@ func (v *YoutubeVideo) triggerThumbnailSave() (err error) {
|
||||
|
||||
func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
|
||||
languages, locations, tags := v.getMetadata()
|
||||
|
||||
var fee *jsonrpc.Fee
|
||||
if params.Fee != nil {
|
||||
feeAmount, err := decimal.NewFromString(params.Fee.Amount)
|
||||
@@ -389,9 +393,6 @@ func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, params SyncParams) (*Sync
|
||||
ThumbnailURL: &v.thumbnailURL,
|
||||
Tags: tags,
|
||||
Locations: locations,
|
||||
FundingAccountIDs: []string{
|
||||
params.DefaultAccount,
|
||||
},
|
||||
},
|
||||
Fee: fee,
|
||||
License: util.PtrToString("Copyrighted (contact publisher)"),
|
||||
@@ -417,7 +418,6 @@ type SyncParams struct {
|
||||
Namer *namer.Namer
|
||||
MaxVideoLength float64
|
||||
Fee *sdk.Fee
|
||||
DefaultAccount string
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo, reprocess bool, walletLock *sync.RWMutex) (*SyncSummary, error) {
|
||||
@@ -432,18 +432,21 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV
|
||||
return v.downloadAndPublish(daemon, params)
|
||||
}
|
||||
|
||||
var isThrottled bool
|
||||
|
||||
func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
|
||||
var err error
|
||||
for {
|
||||
err = v.download()
|
||||
if err != nil && strings.Contains(err.Error(), "HTTP Error 429") {
|
||||
continue
|
||||
} else if err != nil {
|
||||
err := v.download(isThrottled)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "HTTP Error 429") && !isThrottled {
|
||||
isThrottled = true
|
||||
err = v.download(isThrottled)
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("download error", err)
|
||||
}
|
||||
} else {
|
||||
return nil, errors.Prefix("download error", err)
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
log.Debugln("Downloaded " + v.id)
|
||||
|
||||
err = v.triggerThumbnailSave()
|
||||
@@ -465,9 +468,6 @@ func (v *YoutubeVideo) getMetadata() (languages []string, locations []jsonrpc.Lo
|
||||
tags = nil
|
||||
if !v.mocked {
|
||||
if v.youtubeInfo.Snippet.DefaultLanguage != "" {
|
||||
if v.youtubeInfo.Snippet.DefaultLanguage == "iw" {
|
||||
v.youtubeInfo.Snippet.DefaultLanguage = "he"
|
||||
}
|
||||
languages = []string{v.youtubeInfo.Snippet.DefaultLanguage}
|
||||
}
|
||||
|
||||
@@ -479,7 +479,7 @@ func (v *YoutubeVideo) getMetadata() (languages []string, locations []jsonrpc.Lo
|
||||
}
|
||||
tags = v.youtubeInfo.Snippet.Tags
|
||||
}
|
||||
tags, err := tags_manager.SanitizeTags(tags, v.youtubeChannelID)
|
||||
tags, err := tagsManager.SanitizeTags(tags, v.youtubeChannelID)
|
||||
if err != nil {
|
||||
log.Errorln(err.Error())
|
||||
}
|
||||
@@ -491,7 +491,7 @@ func (v *YoutubeVideo) getMetadata() (languages []string, locations []jsonrpc.Lo
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo) (*SyncSummary, error) {
|
||||
c, err := daemon.ClaimSearch(nil, &existingVideoData.ClaimID, nil, nil, 1, 20)
|
||||
c, err := daemon.ClaimSearch(nil, &existingVideoData.ClaimID, nil, nil)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
@@ -551,9 +551,6 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
|
||||
ThumbnailURL: &thumbnailURL,
|
||||
Languages: languages,
|
||||
Locations: locations,
|
||||
FundingAccountIDs: []string{
|
||||
params.DefaultAccount,
|
||||
},
|
||||
},
|
||||
Author: util.PtrToString(""),
|
||||
License: util.PtrToString("Copyrighted (contact publisher)"),
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package tags_manager
|
||||
package tagsManager
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
@@ -1,4 +1,4 @@
|
||||
package tags_manager
|
||||
package tagsManager
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
+5
-6
@@ -61,12 +61,11 @@ func (u *thumbnailUploader) uploadThumbnail() error {
|
||||
uploader := s3manager.NewUploader(s3Session)
|
||||
|
||||
_, err = uploader.Upload(&s3manager.UploadInput{
|
||||
Bucket: aws.String("thumbnails.lbry.com"),
|
||||
Key: key,
|
||||
Body: thumb,
|
||||
ACL: aws.String("public-read"),
|
||||
ContentType: aws.String("image/jpeg"),
|
||||
CacheControl: aws.String("public, max-age=2592000"),
|
||||
Bucket: aws.String("thumbnails.lbry.com"),
|
||||
Key: key,
|
||||
Body: thumb,
|
||||
ACL: aws.String("public-read"),
|
||||
ContentType: aws.String("image/jpeg"),
|
||||
})
|
||||
u.mirroredUrl = ThumbnailEndpoint + u.name
|
||||
return errors.Err(err)
|
||||
|
||||
Reference in new issue
Block a user