Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3eb74cc8b5 | ||
|
|
9bfb1656fe | ||
|
|
5d40d24804 | ||
|
|
79d3aadda8 | ||
|
|
eba59a0806 | ||
|
|
3fd4909cd1 | ||
|
|
b59372c1d8 | ||
|
|
c9e713e9df | ||
|
|
bc72d2f1c4 | ||
|
|
df539e5d01 | ||
|
|
1b89104101 | ||
|
|
3dc30c3b98 | ||
|
|
259df8d257 | ||
|
|
5460224cd6 | ||
|
|
29479a73a7 | ||
|
|
625f339709 | ||
|
|
d6fcfd888a | ||
|
|
5f7d9d4940 | ||
|
|
826fcb78ea | ||
|
|
45e87c3007 | ||
|
|
cdee0d0de7 | ||
|
|
e30f21dc85 | ||
|
|
df88c7e952 | ||
|
|
62a1a0eed5 | ||
|
|
93f9185c71 | ||
|
|
ef62242f42 | ||
|
|
4a356e86da | ||
|
|
ac95ca57af | ||
|
|
c28d5a716b | ||
|
|
59471f9c26 | ||
|
|
8028c6621c | ||
|
|
8abf6ad255 | ||
|
|
1f9a645c57 | ||
|
|
a5657aaf46 | ||
|
|
b6861dae9b | ||
|
|
cff147bb95 | ||
|
|
54d04dcf2c | ||
|
|
65c4f99c1c | ||
|
|
68c0fd9ed7 | ||
|
|
2bde06e4b9 | ||
|
|
5df6db6e96 | ||
|
|
62dfdc1adb | ||
|
|
fe55304184 | ||
|
|
bb76500f04 | ||
|
|
4e2efec66e | ||
|
|
8d650e032c | ||
|
|
f5d97957e7 | ||
|
|
77461f71af |
No files matched your search
+8
-8
@@ -9,12 +9,13 @@ builds:
|
||||
- amd64
|
||||
ldflags:
|
||||
- -X "{{ .Env.IMPORTPATH }}/meta.semVersion={{ .Tag }}" -X "{{ .Env.IMPORTPATH }}/meta.version={{ .Env.VERSIONSHORT }}" -X "{{ .Env.IMPORTPATH }}/meta.versionLong={{ .Env.VERSIONLONG }}" -X "{{ .Env.IMPORTPATH }}/meta.commitMsg={{ .Env.COMMITMSG }}"
|
||||
archive:
|
||||
name_template: '{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}'
|
||||
replacements:
|
||||
linux: Linux
|
||||
amd64: x86_64
|
||||
format: zip
|
||||
archives:
|
||||
- id: zip
|
||||
name_template: '{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}'
|
||||
replacements:
|
||||
linux: Linux
|
||||
amd64: x86_64
|
||||
format: zip
|
||||
checksum:
|
||||
name_template: 'checksums.txt'
|
||||
snapshot:
|
||||
@@ -24,5 +25,4 @@ changelog:
|
||||
filters:
|
||||
exclude:
|
||||
- '^docs:'
|
||||
- '^test:'
|
||||
|
||||
- '^test:'
|
||||
@@ -0,0 +1,111 @@
|
||||
From 30380338ba9af01696c94b61f0597131638eaec1 Mon Sep 17 00:00:00 2001
|
||||
From: Niko Storni <niko@lbry.io>
|
||||
Date: Mon, 16 Dec 2019 00:13:36 +0100
|
||||
Subject: [PATCH] lbry-patch
|
||||
|
||||
---
|
||||
youtube_dl/extractor/youtube.py | 45 +++++++++++++++++++++++++--------
|
||||
1 file changed, 35 insertions(+), 10 deletions(-)
|
||||
|
||||
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
|
||||
index b913d07a6..cd66a5b01 100644
|
||||
--- a/youtube_dl/extractor/youtube.py
|
||||
+++ b/youtube_dl/extractor/youtube.py
|
||||
@@ -10,6 +10,7 @@ import random
|
||||
import re
|
||||
import time
|
||||
import traceback
|
||||
+import subprocess
|
||||
|
||||
from .common import InfoExtractor, SearchInfoExtractor
|
||||
from ..jsinterp import JSInterpreter
|
||||
@@ -536,6 +537,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
|
||||
_GEO_BYPASS = False
|
||||
|
||||
+ _WGET_429_RATE_LIMIT = 8191
|
||||
+ _WGET_BINARY = "wget"
|
||||
+
|
||||
IE_NAME = 'youtube'
|
||||
_TESTS = [
|
||||
{
|
||||
@@ -1254,6 +1258,17 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
""" Return a string representation of a signature """
|
||||
return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
|
||||
|
||||
+ def _rate_limit_download(self, url, video_id, note=None):
|
||||
+ if note is None:
|
||||
+ self.report_download_webpage(video_id)
|
||||
+ elif note is not False:
|
||||
+ if video_id is None:
|
||||
+ self.to_screen('%s' % (note,))
|
||||
+ else:
|
||||
+ self.to_screen('%s: %s' % (video_id, note))
|
||||
+ source_address = self._downloader.params.get('source_address')
|
||||
+ return subprocess.run([self._WGET_BINARY, '-q', '--limit-rate', str(self._WGET_429_RATE_LIMIT), '--bind-address', source_address, '-O', '-', url], check=True, stdout=subprocess.PIPE).stdout.decode(encoding='UTF-8')
|
||||
+
|
||||
def _extract_signature_function(self, video_id, player_url, example_sig):
|
||||
id_m = re.match(
|
||||
r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player(?:-new)?|(?:/[a-z]{2,3}_[A-Z]{2})?/base)?\.(?P<ext>[a-z]+)$',
|
||||
@@ -1678,7 +1693,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
|
||||
# Get video webpage
|
||||
url = proto + '://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1&bpctr=9999999999' % video_id
|
||||
- video_webpage = self._download_webpage(url, video_id)
|
||||
+ video_webpage = self._rate_limit_download(url, video_id)
|
||||
|
||||
# Attempt to extract SWF player URL
|
||||
mobj = re.search(r'swfConfig.*?"(https?:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
|
||||
@@ -1736,10 +1751,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
r'"sts"\s*:\s*(\d+)', embed_webpage, 'sts', default=''),
|
||||
})
|
||||
video_info_url = proto + '://www.youtube.com/get_video_info?' + data
|
||||
- video_info_webpage = self._download_webpage(
|
||||
+ video_info_webpage = self._rate_limit_download(
|
||||
video_info_url, video_id,
|
||||
- note='Refetching age-gated info webpage',
|
||||
- errnote='unable to download video info webpage')
|
||||
+ note='Refetching age-gated info webpage')
|
||||
video_info = compat_parse_qs(video_info_webpage)
|
||||
pl_response = video_info.get('player_response', [None])[0]
|
||||
player_response = extract_player_response(pl_response, video_id)
|
||||
@@ -1777,7 +1791,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
# The general idea is to take a union of itags of both DASH manifests (for example
|
||||
# video with such 'manifest behavior' see https://github.com/ytdl-org/youtube-dl/issues/6093)
|
||||
self.report_video_info_webpage_download(video_id)
|
||||
- for el in ('embedded', 'detailpage', 'vevo', ''):
|
||||
+ for el in ('', 'embedded', 'detailpage', 'vevo'):
|
||||
query = {
|
||||
'video_id': video_id,
|
||||
'ps': 'default',
|
||||
@@ -1789,11 +1803,22 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
query['el'] = el
|
||||
if sts:
|
||||
query['sts'] = sts
|
||||
- video_info_webpage = self._download_webpage(
|
||||
- '%s://www.youtube.com/get_video_info' % proto,
|
||||
- video_id, note=False,
|
||||
- errnote='unable to download video info webpage',
|
||||
- fatal=False, query=query)
|
||||
+
|
||||
+ if el == '':
|
||||
+ base_url = 'https://youtube.com/get_video_info?video_id={}'.format(video_id)
|
||||
+ else:
|
||||
+ base_url = 'https://youtube.com/get_video_info'
|
||||
+
|
||||
+ for q in query:
|
||||
+ if q is None or q is "":
|
||||
+ continue
|
||||
+ if query[q] is None or query[q] is "":
|
||||
+ continue
|
||||
+
|
||||
+ base_url = base_url + "?{}={}".format(q, query[q])
|
||||
+
|
||||
+ video_info_webpage = self._rate_limit_download(base_url, video_id)
|
||||
+
|
||||
if not video_info_webpage:
|
||||
continue
|
||||
get_video_info = compat_parse_qs(video_info_webpage)
|
||||
--
|
||||
2.17.1
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2017-2019 LBRY Inc.
|
||||
Copyright (c) 2017-2020 LBRY Inc.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the
|
||||
"Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish,
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/lbryio/reflector.go/db"
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/lbryio/ytsync/util"
|
||||
)
|
||||
@@ -53,7 +54,8 @@ func reflectBlobs() error {
|
||||
if running {
|
||||
return errors.Prefix("cannot reflect blobs as the daemon is running", err)
|
||||
}
|
||||
|
||||
logrus.SetLevel(logrus.InfoLevel)
|
||||
defer logrus.SetLevel(logrus.DebugLevel)
|
||||
ex, err := os.Executable()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
|
||||
@@ -13,12 +13,12 @@ lbrycrdurl="rpc://lbry:lbry@lbrycrd:29245"
|
||||
#MySQL DSN is required for chainquery to store information.
|
||||
#DEFAULT: "lbry:lbry@tcp(localhost:3306)/chainquery"
|
||||
#SUGGESTED: "lbry:lbry@unix(/var/run/mysqld/mysqld.sock)/chainquery"
|
||||
mysqldsn="lbry:lbry@tcp(mysqlCQ:3306)/chainquery"
|
||||
mysqldsn="lbry:lbry@tcp(mysql:3306)/chainquery"
|
||||
|
||||
#API MySQL DSN is required for chainquery to expose a SQL query service
|
||||
#DEFAULT: "lbry:lbry@tcp(localhost:3306)/chainquery"
|
||||
#SUGGESTED: "lbry:lbry@unix(/var/run/mysqld/mysqld.sock)/chainquery"
|
||||
apimysqldsn="lbry:lbry@tcp(mysqlCQ:3306)/chainquery"
|
||||
apimysqldsn="lbry:lbry@tcp(mysql:3306)/chainquery"
|
||||
|
||||
#API Host and Port is required for the API Server to bind and listen on.
|
||||
#DEFAULT: "0.0.0.0:6300"
|
||||
|
||||
+9
-27
@@ -21,7 +21,7 @@ services:
|
||||
## Wallet Server ##
|
||||
###################
|
||||
walletserver:
|
||||
image: lbry/wallet-server:v0.48.1
|
||||
image: lbry/wallet-server:v0.67.0
|
||||
restart: always
|
||||
environment:
|
||||
- DB_DIRECTORY=/database
|
||||
@@ -47,7 +47,7 @@ services:
|
||||
## Lbrynet ##
|
||||
#############
|
||||
lbrynet:
|
||||
image: lbry/lbrynet:v0.48.1
|
||||
image: lbry/lbrynet:v0.67.0
|
||||
restart: always
|
||||
ports:
|
||||
- "15100:5279"
|
||||
@@ -80,6 +80,9 @@ services:
|
||||
- MYSQL_USER=lbry
|
||||
- MYSQL_PASSWORD=lbry
|
||||
- MYSQL_LOG_CONSOLE=true
|
||||
volumes:
|
||||
- "./init.sql:/docker-entrypoint-initdb.d/init.sql"
|
||||
- "./chainquery/docker/my.cnf:/etc/mysql/conf.d/chainquery-optimizations.cnf"
|
||||
###################
|
||||
## Internal APIs ##
|
||||
###################
|
||||
@@ -96,40 +99,19 @@ services:
|
||||
environment:
|
||||
- MYSQL_DSN=lbry:lbry@tcp(mysql:3306)/lbry
|
||||
- LBRYCRD_CONNECT=rpc://lbry:lbry@lbrycrd:29245
|
||||
- MYSQL_USER=lbry
|
||||
- MYSQL_PASS=lbry
|
||||
- MYSQL_DATABASE=lbry
|
||||
entrypoint: wait-for-it -t 0 mysql:3306 -- wait-for-it -t 0 lbrycrd:29245 -- ./latest serve
|
||||
######################
|
||||
## MySQL Chainquery ##
|
||||
######################
|
||||
mysqlCQ:
|
||||
image: mysql/mysql-server:5.7.27
|
||||
restart: "no"
|
||||
ports:
|
||||
- "15600:3306"
|
||||
expose:
|
||||
- "3306"
|
||||
environment:
|
||||
- MYSQL_ALLOW_EMPTY_PASSWORD=true
|
||||
- MYSQL_DATABASE=chainquery
|
||||
- MYSQL_USER=lbry
|
||||
- MYSQL_PASSWORD=lbry
|
||||
- MYSQL_LOG_CONSOLE=true
|
||||
volumes:
|
||||
- "./chainquery/docker/my.cnf:/etc/mysql/conf.d/chainquery-optimizations.cnf"
|
||||
entrypoint: wait-for-it -t 0 chainquery:6300 -- wait-for-it -t 0 lbrycrd:29245 -- ./latest serve
|
||||
################
|
||||
## Chainquery ##
|
||||
################
|
||||
chainquery:
|
||||
image: lbry/chainquery:master
|
||||
image: lbry/chainquery:v2.0.8
|
||||
restart: "no"
|
||||
ports:
|
||||
- 6300:6300
|
||||
depends_on:
|
||||
- lbrycrd
|
||||
- mysqlCQ
|
||||
- mysql
|
||||
## TODO: Uncomment this in a docker-compose.override.yml to allow for external configurations.
|
||||
volumes:
|
||||
- ./chainqueryconfig.toml:/etc/lbry/chainqueryconfig.toml
|
||||
entrypoint: wait-for-it -t 0 lbrycrd:29245 -- wait-for-it -t 0 mysqlCQ:3306 -- start
|
||||
entrypoint: wait-for-it -t 0 lbrycrd:29245 -- wait-for-it -t 0 mysql:3306 -- start
|
||||
+9
-8
@@ -28,6 +28,7 @@ export BLOBS_DIRECTORY="$(pwd)/e2e/blobsfiles"
|
||||
export LBRYNET_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbrynet/"
|
||||
export LBRYNET_WALLETS_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbryum"
|
||||
export TMP_DIR="/var/tmp"
|
||||
export CHAINNAME="lbrycrd_regtest"
|
||||
export UID
|
||||
|
||||
cd ./e2e
|
||||
@@ -53,29 +54,29 @@ echo "successfully started..."
|
||||
./data_setup.sh
|
||||
|
||||
# Execute the sync test!
|
||||
./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 #Force channel intended...just in case. This channel lines up with the api container
|
||||
./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 --quick #Force channel intended...just in case. This channel lines up with the api container
|
||||
status=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM youtube_data WHERE id=1')
|
||||
videoStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM synced_video WHERE id=1')
|
||||
videoClaimID1=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT claim_id FROM synced_video WHERE id=1')
|
||||
videoClaimID2=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT claim_id FROM synced_video WHERE id=1')
|
||||
videoClaimAddress1=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15600 -e 'SELECT claim_address FROM claim WHERE id=2')
|
||||
videoClaimAddress2=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15600 -e 'SELECT claim_address FROM claim WHERE id=2')
|
||||
videoClaimID1=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT publish.claim_id FROM synced_video INNER JOIN publish ON publish.id = synced_video.publish_id WHERE synced_video.id=1')
|
||||
videoClaimID2=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT publish.claim_id FROM synced_video INNER JOIN publish ON publish.id = synced_video.publish_id WHERE synced_video.id=2')
|
||||
videoClaimAddress1=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT claim_address FROM claim WHERE id=2')
|
||||
videoClaimAddress2=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT claim_address FROM claim WHERE id=3')
|
||||
# Create Supports for published claim
|
||||
./supporty/supporty @BeamerTest "${videoClaimID1}" "${videoClaimAddress1}" lbrycrd_regtest 1.0
|
||||
./supporty/supporty @BeamerTest "${videoClaimID2}" "${videoClaimAddress2}" lbrycrd_regtest 2.0
|
||||
./supporty/supporty @BeamerTest "${videoClaimID1}" "${videoClaimAddress1}" lbrycrd_regtest 3.0
|
||||
./supporty/supporty @BeamerTest "${videoClaimID2}" "${videoClaimAddress2}" lbrycrd_regtest 3.0
|
||||
curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"generate","params":[1]}' -H 'content-type:text/plain;' --user lbry:lbry http://localhost:15200
|
||||
# Reset status for tranfer test
|
||||
# Reset status for transfer test
|
||||
mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e "UPDATE youtube_data SET status = 'queued' WHERE id = 1"
|
||||
# Trigger transfer api
|
||||
curl -i -H 'Accept: application/json' -H 'Content-Type: application/json' 'http://localhost:15400/yt/transfer?auth_token=youtubertoken&address=n4eYeXAYmHo4YRUDEfsEhucy8y5LKRMcHg&public_key=tpubDA9GDAntyJu4hD3wU7175p7CuV6DWbYXfyb2HedBA3yuBp9HZ4n3QE4Ex6RHCSiEuVp2nKAL1Lzf2ZLo9ApaFgNaJjG6Xo1wB3iEeVbrDZp'
|
||||
# Execute the transfer test!
|
||||
./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 #Force channel intended...just in case. This channel lines up with the api container
|
||||
./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 --quick #Force channel intended...just in case. This channel lines up with the api container
|
||||
# Check that the channel and the video are marked as transferred and that all supports are spent
|
||||
channelTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT distinct transfer_state FROM youtube_data')
|
||||
videoTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT distinct transferred FROM synced_video')
|
||||
nrUnspentSupports=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15600 -e 'SELECT COUNT(*) FROM chainquery.support INNER JOIN output ON output.transaction_hash = support.transaction_hash_id AND output.vout = support.vout WHERE output.is_spent = 0')
|
||||
nrUnspentSupports=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT COUNT(*) FROM chainquery.support INNER JOIN output ON output.transaction_hash = support.transaction_hash_id AND output.vout = support.vout WHERE output.is_spent = 0')
|
||||
if [[ $status != "synced" || $videoStatus != "published" || $channelTransferStatus != "2" || $videoTransferStatus != "1" || $nrUnspentSupports != "1" ]]; then
|
||||
echo "~~!!!~~~FAILED~~~!!!~~"
|
||||
echo "Channel Status: $status"
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
CREATE DATABASE IF NOT EXISTS chainquery;
|
||||
GRANT ALL PRIVILEGES ON chainquery.* TO 'lbry'@'%';
|
||||
FLUSH PRIVILEGES;
|
||||
@@ -8,7 +8,7 @@ services:
|
||||
## Lbrynet ##
|
||||
#############
|
||||
lbrynet:
|
||||
image: lbry/lbrynet:v0.48.1
|
||||
image: lbry/lbrynet:v0.51.2
|
||||
restart: "no"
|
||||
networks:
|
||||
lbry-network:
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
module github.com/lbryio/ytsync
|
||||
|
||||
replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.46.3 // indirect
|
||||
github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61
|
||||
@@ -18,7 +20,7 @@ require (
|
||||
github.com/hashicorp/golang-lru v0.5.3 // indirect
|
||||
github.com/hashicorp/memberlist v0.1.5 // indirect
|
||||
github.com/hashicorp/serf v0.8.5 // indirect
|
||||
github.com/lbryio/lbry.go/v2 v2.4.1-0.20191206194319-06764c3d007e
|
||||
github.com/lbryio/lbry.go/v2 v2.5.3
|
||||
github.com/lbryio/reflector.go v1.0.6-0.20190828131602-ce3d4403dbc6
|
||||
github.com/miekg/dns v1.1.22 // indirect
|
||||
github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg=
|
||||
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
|
||||
cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
|
||||
@@ -17,7 +16,6 @@ github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61/go.mo
|
||||
github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
|
||||
github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU=
|
||||
github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
|
||||
github.com/aead/siphash v1.0.1 h1:FwHfE/T45KPKYuuSAKyyvE+oPWcaQ+CUmFW0bPlM+kg=
|
||||
github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
|
||||
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
|
||||
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
|
||||
@@ -25,7 +23,6 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV
|
||||
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM=
|
||||
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg=
|
||||
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
|
||||
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf h1:eg0MeVzsP1G42dRafH3vf+al2vQIJU0YHX+1Tw87oco=
|
||||
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
|
||||
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA=
|
||||
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
|
||||
@@ -34,17 +31,14 @@ github.com/aws/aws-sdk-go v1.25.9 h1:WtVzerf5wSgPwlTTwl+ktCq/0GCS5MI9ZlLIcjsTr+Q
|
||||
github.com/aws/aws-sdk-go v1.25.9/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
|
||||
github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32 h1:qkOC5Gd33k54tobS36cXdAzJbeHaduLtnLQQwNoIi78=
|
||||
github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8=
|
||||
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo=
|
||||
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
|
||||
github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803 h1:j3AgPKKZtZStM2nyhrDSLSYgT7YHrZKdSkq1OYeLjvM=
|
||||
github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
|
||||
github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d h1:yJzD/yFppdVCf6ApMkVy8cUxV0XrxdP9rVf6D87/Mng=
|
||||
github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
|
||||
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd h1:R/opQEbFEy9JGkIguV40SvRY1uliPX8ifOvi6ICsFCw=
|
||||
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg=
|
||||
github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd h1:qdGvebPBDuYDPGi1WCPjy1tGyMpmDK8IEapSsszn7HE=
|
||||
github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY=
|
||||
github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723 h1:ZA/jbKoGcVAnER6pCHPEkGdZOV7U1oLUedErBHCUMs0=
|
||||
github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
|
||||
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3SkEwmHoWBmX1DNXhXZqlTpq6s4tyJGc=
|
||||
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
|
||||
@@ -82,7 +76,6 @@ github.com/go-ini/ini v1.48.0 h1:TvO60hO/2xgaaTWp2P0wUe4CFxwdMzfbkv3+343Xzqw=
|
||||
github.com/go-ini/ini v1.48.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
|
||||
github.com/go-ozzo/ozzo-validation v3.6.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
|
||||
github.com/go-sql-driver/mysql v0.0.0-20180719071942-99ff426eb706 h1:P3NPKb7qq581SeMCB+dU1SuCX1kQh8VoQ/4HmT2ftQY=
|
||||
github.com/go-sql-driver/mysql v0.0.0-20180719071942-99ff426eb706/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
||||
github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
|
||||
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
||||
@@ -96,15 +89,12 @@ github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb
|
||||
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
|
||||
github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
|
||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
|
||||
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw=
|
||||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
|
||||
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
|
||||
@@ -117,29 +107,23 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+
|
||||
github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f h1:TyqzGm2z1h3AGhjOoRYyeLcW4WlW81MDQkWa+rx/000=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8=
|
||||
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
|
||||
github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk=
|
||||
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
|
||||
github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
|
||||
github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
|
||||
github.com/gorilla/rpc v1.1.0 h1:marKfvVP0Gpd/jHlVBKCQ8RAoUPdX7K1Nuh6l1BNh7A=
|
||||
github.com/gorilla/rpc v1.1.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ=
|
||||
github.com/gorilla/rpc v1.2.0 h1:WvvdC2lNeT1SP32zrIce5l0ECBfbAlmrmSBsuc57wfk=
|
||||
github.com/gorilla/rpc v1.2.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ=
|
||||
github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
|
||||
github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
|
||||
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
|
||||
github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM=
|
||||
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
|
||||
github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0=
|
||||
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
|
||||
github.com/hashicorp/go-immutable-radix v1.1.0 h1:vN9wG1D6KG6YHRTWr8512cxGOVgTMEfgEdSj/hr8MPc=
|
||||
github.com/hashicorp/go-immutable-radix v1.1.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
|
||||
@@ -149,7 +133,6 @@ github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iP
|
||||
github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
|
||||
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
|
||||
github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
|
||||
github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs=
|
||||
github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
|
||||
github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc=
|
||||
github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A=
|
||||
@@ -158,7 +141,6 @@ github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b
|
||||
github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE=
|
||||
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
|
||||
github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk=
|
||||
@@ -167,11 +149,9 @@ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T
|
||||
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
|
||||
github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
|
||||
github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
|
||||
github.com/hashicorp/memberlist v0.1.4 h1:gkyML/r71w3FL8gUi74Vk76avkj/9lYAY9lvg0OcoGs=
|
||||
github.com/hashicorp/memberlist v0.1.4/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
|
||||
github.com/hashicorp/memberlist v0.1.5 h1:AYBsgJOW9gab/toO5tEB8lWetVgDKZycqkebJ8xxpqM=
|
||||
github.com/hashicorp/memberlist v0.1.5/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
|
||||
github.com/hashicorp/serf v0.8.2 h1:YZ7UKsJv+hKjqGVUUbtE3HNj79Eln2oQ75tniF6iPt0=
|
||||
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
|
||||
github.com/hashicorp/serf v0.8.5 h1:ZynDUIQiA8usmRgPdGPHFdPnb1wgGI9tK3mO9hcAJjc=
|
||||
github.com/hashicorp/serf v0.8.5/go.mod h1:UpNcs7fFbpKIyZaUuSW6EPiH+eZC7OuyFD+wc1oal+k=
|
||||
@@ -181,7 +161,6 @@ github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf h1:WfD7V
|
||||
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h1:hyb9oH7vZsitZCiBt0ZvifOrB+qc8PS5IiilCIb87rg=
|
||||
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
|
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89 h1:12K8AlpT0/6QUXSfV0yi4Q0jkbq8NDtIKFtF61AoqV0=
|
||||
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
|
||||
@@ -189,17 +168,13 @@ github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22 h1:jKUP9TQ0c7X3
|
||||
github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22/go.mod h1:u0Jo4f2dNlTJeeOywkM6bLwxq6gC3pZ9rEFHn3AhTdk=
|
||||
github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07 h1:+kBG/8rjCa6vxJZbUjAiE4MQmBEBYc8nLEb51frnvBY=
|
||||
github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07/go.mod h1:j1kV/8f3jowErEq4XyeypkCdvg5EeHkf0YCKCcq5Ybo=
|
||||
github.com/jrick/logrotate v1.0.0 h1:lQ1bL/n9mBNeIXoTUoYRlK4dHuNJVofX9oWqBtPnSzI=
|
||||
github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE=
|
||||
github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
|
||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23 h1:FOOIBWrEkLgmlgGfMuZT83xIwfPDxEI2OHu6xUmJMFE=
|
||||
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
@@ -213,21 +188,20 @@ github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpU
|
||||
github.com/lbryio/lbry.go v0.0.0-20190828131228-f3a1fbdd5303/go.mod h1:qR+Ui0hYhemIU4fXqM3d1P9eiaRFlof777VJgV7KJ8w=
|
||||
github.com/lbryio/lbry.go v1.1.2 h1:Dyxc+glT/rVWJwHfIf7vjlPYYbjzrQz5ARmJd5Hp69c=
|
||||
github.com/lbryio/lbry.go v1.1.2/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
|
||||
github.com/lbryio/lbry.go/v2 v2.4.1-0.20191206194319-06764c3d007e h1:wN6r46L8EjgC2JyOImagQXaQvyF8Bd2QPjfXqv7oDAg=
|
||||
github.com/lbryio/lbry.go/v2 v2.4.1-0.20191206194319-06764c3d007e/go.mod h1:gbs/+PVII96AqsRgf+hY/F7sJkb/ciiX2yy3j+NApd0=
|
||||
github.com/lbryio/lbry.go/v2 v2.5.3 h1:Qw4APDuVZDkwjQ49usy4p/ty/KfuQr18ld6GiREYlog=
|
||||
github.com/lbryio/lbry.go/v2 v2.5.3/go.mod h1:LgFKEpZzJE72DVgSXXfg+2IGOhrC9Lzj6eLma18iNz8=
|
||||
github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19 h1:/zWD8dVIl7bV1TdJWqPqy9tpqixzX2Qxgit48h3hQcY=
|
||||
github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46 h1:LemfR+rMxhf7nnOrzy2HqS7Me7SZ5gEwOcNFzKC8ySQ=
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
|
||||
github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4=
|
||||
github.com/lbryio/reflector.go v1.0.6-0.20190828131602-ce3d4403dbc6 h1:8k12lI18EEgMHkgGrgtRvJGl2Bq1vDSEbE4s+QaQ5Qc=
|
||||
github.com/lbryio/reflector.go v1.0.6-0.20190828131602-ce3d4403dbc6/go.mod h1:Q1Cnuv5iLsB2rS4Vr0o0EJr4Gs/EgDXFH2V4WtoM5o8=
|
||||
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c h1:m3O7561xBQ00lfUVayW4c6SnpVbUDQtPUwGcGYSUYQA=
|
||||
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
|
||||
github.com/lbryio/types v0.0.0-20191009145016-1bb8107e04f8 h1:jSNW/rK6DQsz7Zh+iv1zR384PeQdHt0gS4hKY17tkuM=
|
||||
github.com/lbryio/types v0.0.0-20191009145016-1bb8107e04f8/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
|
||||
github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5 h1:AsEBgzv3DhuYHI/GiQh2HxvTP71HCCE9E/tzGUzGdtU=
|
||||
github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0=
|
||||
github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6 h1:iOAVXzZyXtW408TMYejlUPo6BIn92HmOacWtIfNyYns=
|
||||
github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg=
|
||||
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 h1:mG83tLXWSRdcXMWfkoumVwhcCbf3jHF9QKv/m37BkM0=
|
||||
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCWffGOaDcjkw1iB7W9DVLp6GXmfcJY/7YZCWPA4=
|
||||
@@ -235,7 +209,6 @@ github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP
|
||||
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
|
||||
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA=
|
||||
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
|
||||
github.com/miekg/dns v1.1.22 h1:Jm64b3bO9kP43ddLjL2EY3Io6bmy1qGb9Xxz6TqS6rc=
|
||||
github.com/miekg/dns v1.1.22/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
|
||||
@@ -250,16 +223,13 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F
|
||||
github.com/mitchellh/mapstructure v0.0.0-20180511142126-bb74f1db0675/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
|
||||
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/nlopes/slack v0.5.0 h1:NbIae8Kd0NpqaEI3iUrsuS0KbcEDhzhc939jLW5fNm0=
|
||||
github.com/nlopes/slack v0.5.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM=
|
||||
github.com/nlopes/slack v0.6.0 h1:jt0jxVQGhssx1Ib7naAOZEZcGdtIhTzkP0nopK0AsRA=
|
||||
github.com/nlopes/slack v0.6.0/go.mod h1:JzQ9m3PMAqcpeCam7UaHSuBuupz7CmpjehYMayT6YOk=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
|
||||
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.10.2 h1:uqH7bpe+ERSiDa34FDOF7RikN6RzXgduUF8yarlZp94=
|
||||
github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
|
||||
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
|
||||
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
@@ -269,7 +239,6 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI
|
||||
github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
|
||||
github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
||||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||
github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6 h1:2bae6N0SZjgzk+Zg8mzTsfmpwHXY9VBNp9UdjhaElA0=
|
||||
github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
|
||||
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc=
|
||||
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
|
||||
@@ -289,7 +258,6 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb
|
||||
github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
|
||||
github.com/sebdah/goldie v0.0.0-20180424091453-8784dd1ab561 h1:IY+sDBJR/wRtsxq+626xJnt4Tw7/ROA9cDIR8MMhWyg=
|
||||
github.com/sebdah/goldie v0.0.0-20180424091453-8784dd1ab561/go.mod h1:lvjGftC8oe7XPtyrOidaMi0rp5B9+XY/ZRUynGnuaxQ=
|
||||
github.com/sebdah/goldie v0.0.0-20190531093107-d313ffb52c77 h1:Msb6XRY62jQOueNNlB5LGin1rljK7c49NLniGwJG2bg=
|
||||
github.com/sebdah/goldie v0.0.0-20190531093107-d313ffb52c77/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4=
|
||||
@@ -300,15 +268,12 @@ github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5 h1:Gojs/hac/DoY
|
||||
github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k=
|
||||
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
|
||||
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w=
|
||||
github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
|
||||
github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c h1:Ho+uVpkel/udgjbwB5Lktg9BtvJSh2DT0Hi6LPSyI2w=
|
||||
github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
|
||||
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8=
|
||||
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||
@@ -320,7 +285,6 @@ github.com/spf13/cobra v0.0.0-20180722215644-7c4570c3ebeb/go.mod h1:1l0Ry5zgKvJa
|
||||
github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
|
||||
github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
|
||||
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
|
||||
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
|
||||
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
|
||||
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
@@ -328,12 +292,10 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
|
||||
github.com/uber-go/atomic v1.3.2 h1:Azu9lPBWRNKzYXSIwRfgRuDuS0YKsK4NFhiQv98gkxo=
|
||||
github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
|
||||
github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o=
|
||||
github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
|
||||
@@ -345,7 +307,6 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
|
||||
go.opencensus.io v0.22.1 h1:8dP3SGL7MPB94crU3bEPplMPe83FI4EouesJUeFHv50=
|
||||
go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA=
|
||||
go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4=
|
||||
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
|
||||
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
@@ -354,7 +315,6 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf
|
||||
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c h1:Vj5n4GlwjmQteupaxJ9+0FNOmBrHfq7vN4btdGoDZgI=
|
||||
golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
@@ -388,7 +348,6 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c h1:uOCk1iQW6Vc18bnC13MfzScl+wdKBmM9Y9kU7Z83/lw=
|
||||
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
@@ -396,7 +355,6 @@ golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLL
|
||||
golang.org/x/net v0.0.0-20191009170851-d66e71096ffb h1:TR699M2v0qoKTOHxeLgp6zPqaQNs74f01a/ob9W0qko=
|
||||
golang.org/x/net v0.0.0-20191009170851-d66e71096ffb/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
@@ -404,7 +362,6 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
@@ -421,7 +378,6 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190520201301-c432e742b0af/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5 h1:sM3evRHxE/1RuMe1FYAL3j7C7fUfIjkbE+NiDAYUF8U=
|
||||
golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
@@ -433,7 +389,6 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 h1:xQwXv67TxFo9nC1GJFyab5eq/5B590r6RlnL/G8Sz7w=
|
||||
@@ -463,7 +418,6 @@ google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEn
|
||||
google.golang.org/api v0.11.0 h1:n/qM3q0/rV2F0pox7o0CvNhlPvZAo7pLbef122cbLJ0=
|
||||
google.golang.org/api v0.11.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
|
||||
@@ -472,7 +426,6 @@ google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19 h1:Lj2SnHtxkRGJDqnGaSjo+CCdIieEnwVazbOXILwQemk=
|
||||
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
@@ -495,7 +448,6 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/ini.v1 v1.41.0 h1:Ka3ViY6gNYSKiVy71zXBEqKplnV35ImDLVG+8uoIklE=
|
||||
gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/ini.v1 v1.48.0 h1:URjZc+8ugRY5mL5uUeQH/a63JcHwdX9xZaWvmNWD7z8=
|
||||
gopkg.in/ini.v1 v1.48.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
@@ -504,7 +456,6 @@ gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79/go.mod h1:gWkaRU7CoX
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
|
||||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
|
||||
+60
-21
@@ -13,26 +13,28 @@ import (
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const IPCooldownPeriod = 35 * time.Second
|
||||
const IPCooldownPeriod = 20 * time.Second
|
||||
const unbanTimeout = 3 * time.Hour
|
||||
|
||||
var stopper = stop.New()
|
||||
|
||||
type IPPool struct {
|
||||
ips []throttledIP
|
||||
lock *sync.Mutex
|
||||
ips []throttledIP
|
||||
lock *sync.RWMutex
|
||||
stopGrp *stop.Group
|
||||
}
|
||||
|
||||
type throttledIP struct {
|
||||
IP string
|
||||
LastUse time.Time
|
||||
Throttled bool
|
||||
InUse bool
|
||||
IP string
|
||||
UsedForVideo string
|
||||
LastUse time.Time
|
||||
Throttled bool
|
||||
InUse bool
|
||||
}
|
||||
|
||||
var ipPoolInstance *IPPool
|
||||
|
||||
func GetIPPool() (*IPPool, error) {
|
||||
func GetIPPool(stopGrp *stop.Group) (*IPPool, error) {
|
||||
if ipPoolInstance != nil {
|
||||
return ipPoolInstance, nil
|
||||
}
|
||||
@@ -47,21 +49,37 @@ func GetIPPool() (*IPPool, error) {
|
||||
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) && !ipv6Added {
|
||||
pool = append(pool, throttledIP{
|
||||
IP: ipnet.IP.String(),
|
||||
LastUse: time.Time{},
|
||||
LastUse: time.Now().Add(-5 * time.Minute),
|
||||
})
|
||||
ipv6Added = true
|
||||
} else if ipnet.IP.To4() != nil && govalidator.IsIPv4(ipnet.IP.String()) {
|
||||
pool = append(pool, throttledIP{
|
||||
IP: ipnet.IP.String(),
|
||||
LastUse: time.Time{},
|
||||
LastUse: time.Now().Add(-5 * time.Minute),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
ipPoolInstance = &IPPool{
|
||||
ips: pool,
|
||||
lock: &sync.Mutex{},
|
||||
ips: pool,
|
||||
lock: &sync.RWMutex{},
|
||||
stopGrp: stopGrp,
|
||||
}
|
||||
ticker := time.NewTicker(10 * time.Second)
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-stopGrp.Ch():
|
||||
return
|
||||
case <-ticker.C:
|
||||
ipPoolInstance.lock.RLock()
|
||||
for _, ip := range ipPoolInstance.ips {
|
||||
log.Debugf("IP: %s\tInUse: %t\tVideoID: %s\tThrottled: %t\tLastUse: %.1f", ip.IP, ip.InUse, ip.UsedForVideo, ip.Throttled, time.Since(ip.LastUse).Seconds())
|
||||
}
|
||||
ipPoolInstance.lock.RUnlock()
|
||||
}
|
||||
}
|
||||
}()
|
||||
return ipPoolInstance, nil
|
||||
}
|
||||
|
||||
@@ -97,12 +115,26 @@ func (i *IPPool) ReleaseIP(ip string) {
|
||||
if localIP.IP == ip {
|
||||
localIP.InUse = false
|
||||
localIP.LastUse = time.Now()
|
||||
break
|
||||
return
|
||||
}
|
||||
}
|
||||
util.SendErrorToSlack("something went wrong while releasing the IP %s as we reached the end of the function", ip)
|
||||
}
|
||||
|
||||
func (i *IPPool) ReleaseAll() {
|
||||
i.lock.Lock()
|
||||
defer i.lock.Unlock()
|
||||
for j, _ := range i.ips {
|
||||
if i.ips[j].Throttled {
|
||||
continue
|
||||
}
|
||||
localIP := &i.ips[j]
|
||||
localIP.InUse = false
|
||||
localIP.LastUse = time.Now()
|
||||
}
|
||||
}
|
||||
|
||||
func (i *IPPool) SetThrottled(ip string, stopGrp *stop.Group) {
|
||||
func (i *IPPool) SetThrottled(ip string) {
|
||||
i.lock.Lock()
|
||||
defer i.lock.Unlock()
|
||||
var tIP *throttledIP
|
||||
@@ -129,7 +161,7 @@ func (i *IPPool) SetThrottled(ip string, stopGrp *stop.Group) {
|
||||
tIP.Throttled = false
|
||||
i.lock.Unlock()
|
||||
util.SendInfoToSlack("%s set back to not throttled", ip)
|
||||
case <-stopGrp.Ch():
|
||||
case <-i.stopGrp.Ch():
|
||||
unbanTimer.Stop()
|
||||
}
|
||||
}(tIP)
|
||||
@@ -138,13 +170,14 @@ func (i *IPPool) SetThrottled(ip string, stopGrp *stop.Group) {
|
||||
var ErrAllInUse = errors.Base("all IPs are in use, try again")
|
||||
var ErrAllThrottled = errors.Base("all IPs are throttled")
|
||||
var ErrResourceLock = errors.Base("error getting next ip, did you forget to lock on the resource?")
|
||||
var ErrInterruptedByUser = errors.Base("interrupted by user")
|
||||
|
||||
func (i *IPPool) nextIP() (*throttledIP, error) {
|
||||
func (i *IPPool) nextIP(forVideo string) (*throttledIP, error) {
|
||||
i.lock.Lock()
|
||||
defer i.lock.Unlock()
|
||||
|
||||
sort.Slice(i.ips, func(j, k int) bool {
|
||||
return i.ips[j].LastUse.Before(i.ips[j].LastUse)
|
||||
return i.ips[j].LastUse.Before(i.ips[k].LastUse)
|
||||
})
|
||||
|
||||
if !AllThrottled(i.ips) {
|
||||
@@ -165,18 +198,24 @@ func (i *IPPool) nextIP() (*throttledIP, error) {
|
||||
return nil, errors.Err(ErrResourceLock)
|
||||
}
|
||||
nextIP.InUse = true
|
||||
nextIP.UsedForVideo = forVideo
|
||||
return nextIP, nil
|
||||
}
|
||||
return nil, errors.Err(ErrAllThrottled)
|
||||
}
|
||||
|
||||
func (i *IPPool) GetIP() (string, error) {
|
||||
func (i *IPPool) GetIP(forVideo string) (string, error) {
|
||||
for {
|
||||
ip, err := i.nextIP()
|
||||
ip, err := i.nextIP(forVideo)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrAllInUse) {
|
||||
time.Sleep(5 * time.Second)
|
||||
continue
|
||||
select {
|
||||
case <-i.stopGrp.Ch():
|
||||
return "", errors.Err(ErrInterruptedByUser)
|
||||
default:
|
||||
time.Sleep(5 * time.Second)
|
||||
continue
|
||||
}
|
||||
} else if errors.Is(err, ErrAllThrottled) {
|
||||
return "throttled", err
|
||||
}
|
||||
|
||||
@@ -9,14 +9,28 @@ func TestAll(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ip, err := pool.GetIP()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log(ip)
|
||||
pool.ReleaseIP(ip)
|
||||
ip2, err := pool.GetIP()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ip == ip2 && len(pool.ips) > 1 {
|
||||
t.Fatalf("the same IP was returned twice! %s, %s", ip, ip2)
|
||||
}
|
||||
t.Log(ip2)
|
||||
pool.ReleaseIP(ip2)
|
||||
|
||||
for range pool.ips {
|
||||
ip, err := pool.GetIP()
|
||||
_, err = pool.GetIP()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log(ip)
|
||||
}
|
||||
|
||||
next, err := pool.nextIP()
|
||||
if err != nil {
|
||||
t.Logf("%s", err.Error())
|
||||
|
||||
@@ -56,6 +56,7 @@ func main() {
|
||||
cmd.Flags().BoolVar(&flags.RemoveDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published")
|
||||
cmd.Flags().BoolVar(&flags.UpgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version")
|
||||
cmd.Flags().BoolVar(&flags.DisableTransfers, "no-transfers", false, "Skips the transferring process of videos, channels and supports")
|
||||
cmd.Flags().BoolVar(&flags.QuickSync, "quick", false, "Look up only the last 50 videos from youtube")
|
||||
cmd.Flags().StringVar(&syncStatus, "status", "", "Specify which queue to pull from. Overrides --update")
|
||||
cmd.Flags().StringVar(&channelID, "channelID", "", "If specified, only this channel will be synced.")
|
||||
cmd.Flags().Int64Var(&syncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)")
|
||||
|
||||
+16
-2
@@ -7,6 +7,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/ytsync/blobs_reflector"
|
||||
"github.com/lbryio/ytsync/ip_manager"
|
||||
"github.com/lbryio/ytsync/namer"
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
logUtils "github.com/lbryio/ytsync/util"
|
||||
@@ -92,6 +93,7 @@ const (
|
||||
TransferStateNotTouched = iota
|
||||
TransferStatePending
|
||||
TransferStateComplete
|
||||
TransferStateManual
|
||||
)
|
||||
|
||||
func (s *SyncManager) Start() error {
|
||||
@@ -133,6 +135,7 @@ func (s *SyncManager) Start() error {
|
||||
ConcurrentVideos: s.concurrentVideos,
|
||||
Refill: s.refill,
|
||||
Manager: s,
|
||||
MaxVideoLength: s.maxVideoLength,
|
||||
LbrycrdString: s.lbrycrdString,
|
||||
AwsS3ID: s.awsS3ID,
|
||||
AwsS3Secret: s.awsS3Secret,
|
||||
@@ -147,7 +150,6 @@ func (s *SyncManager) Start() error {
|
||||
shouldInterruptLoop = true
|
||||
} else {
|
||||
var queuesToSync []string
|
||||
//TODO: implement scrambling to avoid starvation of queues
|
||||
if s.syncStatus != "" {
|
||||
queuesToSync = append(queuesToSync, s.syncStatus)
|
||||
} else if s.SyncFlags.SyncUpdate {
|
||||
@@ -159,7 +161,7 @@ func (s *SyncManager) Start() error {
|
||||
for _, q := range queuesToSync {
|
||||
//temporary override for sync-until to give tom the time to review the channels
|
||||
if q == StatusQueued {
|
||||
s.syncProperties.SyncUntil = time.Now().AddDate(0, 0, -1).Unix()
|
||||
s.syncProperties.SyncUntil = time.Now().Add(-8 * time.Hour).Unix()
|
||||
}
|
||||
channels, err := s.apiConfig.FetchChannels(q, s.syncProperties)
|
||||
if err != nil {
|
||||
@@ -167,6 +169,10 @@ func (s *SyncManager) Start() error {
|
||||
}
|
||||
for i, c := range channels {
|
||||
log.Infof("There are %d channels in the \"%s\" queue", len(channels)-i, q)
|
||||
maxVideoLength := s.maxVideoLength
|
||||
if c.TotalSubscribers < 1000 {
|
||||
maxVideoLength = 1.0
|
||||
}
|
||||
syncs = append(syncs, Sync{
|
||||
APIConfig: s.apiConfig,
|
||||
YoutubeChannelID: c.ChannelId,
|
||||
@@ -174,6 +180,7 @@ func (s *SyncManager) Start() error {
|
||||
lbryChannelID: c.ChannelClaimID,
|
||||
MaxTries: s.maxTries,
|
||||
ConcurrentVideos: s.concurrentVideos,
|
||||
MaxVideoLength: maxVideoLength,
|
||||
Refill: s.refill,
|
||||
Manager: s,
|
||||
LbrycrdString: s.lbrycrdString,
|
||||
@@ -201,6 +208,12 @@ func (s *SyncManager) Start() error {
|
||||
shouldNotCount := false
|
||||
logUtils.SendInfoToSlack("Syncing %s (%s) to LBRY! total processed channels since startup: %d", sync.LbryChannelName, sync.YoutubeChannelID, syncCount+1)
|
||||
err := sync.FullCycle()
|
||||
//TODO: THIS IS A TEMPORARY WORK AROUND FOR THE STUPID IP LOCKUP BUG
|
||||
ipPool, _ := ip_manager.GetIPPool(sync.grp)
|
||||
if ipPool != nil {
|
||||
ipPool.ReleaseAll()
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
fatalErrors := []string{
|
||||
"default_wallet already exists",
|
||||
@@ -211,6 +224,7 @@ func (s *SyncManager) Start() error {
|
||||
"the channel in the wallet is different than the channel in the database",
|
||||
"this channel does not belong to this wallet!",
|
||||
"You already have a stream claim published under the name",
|
||||
"Daily Limit Exceeded",
|
||||
}
|
||||
if util.SubstringInSlice(err.Error(), fatalErrors) {
|
||||
return errors.Prefix("@Nikooo777 this requires manual intervention! Exiting...", err)
|
||||
|
||||
+30
-14
@@ -196,7 +196,7 @@ func (s *Sync) ensureEnoughUTXOs() error {
|
||||
|
||||
for _, utxo := range utxolist.Items {
|
||||
amount, _ := strconv.ParseFloat(utxo.Amount, 64)
|
||||
if utxo.IsMine && utxo.Type == "payment" && amount > 0.001 {
|
||||
if utxo.IsMyOutput && utxo.Type == "payment" && amount > 0.001 {
|
||||
if utxo.Confirmations > 0 {
|
||||
confirmedCount++
|
||||
}
|
||||
@@ -222,6 +222,9 @@ func (s *Sync) ensureEnoughUTXOs() error {
|
||||
if desiredUTXOCount > maxUTXOs {
|
||||
desiredUTXOCount = maxUTXOs
|
||||
}
|
||||
if desiredUTXOCount < uint64(confirmedCount) {
|
||||
return nil
|
||||
}
|
||||
availableBalance, _ := balance.Available.Float64()
|
||||
log.Infof("Splitting balance of %.3f evenly between %d UTXOs", availableBalance, desiredUTXOCount)
|
||||
|
||||
@@ -303,9 +306,6 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
return errors.Err("no channel name set")
|
||||
}
|
||||
|
||||
if s.transferState == TransferStateComplete {
|
||||
return nil
|
||||
}
|
||||
channels, err := s.daemon.ChannelList(nil, 1, 50, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -332,6 +332,10 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
if channelToUse == nil {
|
||||
return errors.Err("this wallet has channels but not a single one is ours! Expected claim_id: %s (%s)", s.lbryChannelID, s.LbryChannelName)
|
||||
}
|
||||
} else if s.transferState == TransferStateComplete {
|
||||
return errors.Err("the channel was transferred but appears to have been abandoned!")
|
||||
} else if s.lbryChannelID != "" {
|
||||
return errors.Err("the database has a channel recorded (%s) but nothing was found in our control", s.lbryChannelID)
|
||||
}
|
||||
|
||||
channelUsesOldMetadata := false
|
||||
@@ -399,6 +403,9 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
|
||||
var languages []string = nil
|
||||
if channelInfo.DefaultLanguage != "" {
|
||||
if channelInfo.DefaultLanguage == "iw" {
|
||||
channelInfo.DefaultLanguage = "he"
|
||||
}
|
||||
languages = []string{channelInfo.DefaultLanguage}
|
||||
}
|
||||
var locations []jsonrpc.Location = nil
|
||||
@@ -415,15 +422,20 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
ThumbnailURL: &thumbnailURL,
|
||||
}
|
||||
if channelUsesOldMetadata {
|
||||
c, err = s.daemon.ChannelUpdate(s.lbryChannelID, jsonrpc.ChannelUpdateOptions{
|
||||
ClearTags: util.PtrToBool(true),
|
||||
ClearLocations: util.PtrToBool(true),
|
||||
ClearLanguages: util.PtrToBool(true),
|
||||
ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
|
||||
ClaimCreateOptions: claimCreateOptions,
|
||||
CoverURL: bannerURL,
|
||||
},
|
||||
})
|
||||
if s.transferState <= 1 {
|
||||
c, err = s.daemon.ChannelUpdate(s.lbryChannelID, jsonrpc.ChannelUpdateOptions{
|
||||
ClearTags: util.PtrToBool(true),
|
||||
ClearLocations: util.PtrToBool(true),
|
||||
ClearLanguages: util.PtrToBool(true),
|
||||
ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
|
||||
ClaimCreateOptions: claimCreateOptions,
|
||||
CoverURL: bannerURL,
|
||||
},
|
||||
})
|
||||
} else {
|
||||
logUtils.SendInfoToSlack("%s (%s) has a channel with old metadata but isn't in our control anymore. Ignoring", s.LbryChannelName, s.lbryChannelID)
|
||||
return nil
|
||||
}
|
||||
} else {
|
||||
c, err = s.daemon.ChannelCreate(s.LbryChannelName, channelBidAmount, jsonrpc.ChannelCreateOptions{
|
||||
ClaimCreateOptions: claimCreateOptions,
|
||||
@@ -445,7 +457,11 @@ func (s *Sync) addCredits(amountToAdd float64) error {
|
||||
return err
|
||||
}
|
||||
|
||||
addressResp, err := s.daemon.AddressUnused(nil)
|
||||
defaultAccount, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
addressResp, err := s.daemon.AddressUnused(&defaultAccount)
|
||||
if err != nil {
|
||||
return err
|
||||
} else if addressResp == nil {
|
||||
|
||||
+24
-6
@@ -91,16 +91,17 @@ func abandonSupports(s *Sync) (float64, error) {
|
||||
consumerWG.Add(1)
|
||||
go func() {
|
||||
defer consumerWG.Done()
|
||||
outer:
|
||||
for {
|
||||
claimID, more := <-claimIDChan
|
||||
if !more {
|
||||
return
|
||||
} else {
|
||||
summary, err := s.daemon.SupportAbandon(&claimID, nil, nil, nil, nil)
|
||||
summary, err := s.daemon.TxoSpend(util.PtrToString("support"), &claimID, nil, nil, nil, &defaultAccount)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "Client.Timeout exceeded while awaiting headers") {
|
||||
log.Errorf("Support abandon for %s timed out, retrying...", claimID)
|
||||
summary, err = s.daemon.SupportAbandon(&claimID, nil, nil, nil, nil)
|
||||
summary, err = s.daemon.TxoSpend(util.PtrToString("support"), &claimID, nil, nil, nil, &defaultAccount)
|
||||
if err != nil {
|
||||
//TODO GUESS HOW MUCH LBC WAS RELEASED THAT WE DON'T KNOW ABOUT, because screw you SDK
|
||||
abandonRspChan <- abandonResponse{
|
||||
@@ -119,7 +120,7 @@ func abandonSupports(s *Sync) (float64, error) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if len(summary.Outputs) < 1 {
|
||||
if summary == nil || len(*summary) < 1 {
|
||||
abandonRspChan <- abandonResponse{
|
||||
ClaimID: claimID,
|
||||
Error: errors.Err("error abandoning supports: no outputs while abandoning %s", claimID),
|
||||
@@ -127,7 +128,19 @@ func abandonSupports(s *Sync) (float64, error) {
|
||||
}
|
||||
continue
|
||||
}
|
||||
outputAmount, err := strconv.ParseFloat(summary.Outputs[0].Amount, 64)
|
||||
var outputAmount float64
|
||||
for _, tx := range *summary {
|
||||
amount, err := strconv.ParseFloat(tx.Outputs[0].Amount, 64)
|
||||
if err != nil {
|
||||
abandonRspChan <- abandonResponse{
|
||||
ClaimID: claimID,
|
||||
Error: errors.Err(err),
|
||||
Amount: 0,
|
||||
}
|
||||
continue outer
|
||||
}
|
||||
outputAmount += amount
|
||||
}
|
||||
if err != nil {
|
||||
abandonRspChan <- abandonResponse{
|
||||
ClaimID: claimID,
|
||||
@@ -192,7 +205,7 @@ func transferVideos(s *Sync) error {
|
||||
|
||||
var stream *jsonrpc.Claim = nil
|
||||
for _, c := range streams.Items {
|
||||
if c.ClaimID != video.ClaimID {
|
||||
if c.ClaimID != video.ClaimID || (c.SigningChannel != nil && c.SigningChannel.ClaimID != s.lbryChannelID) {
|
||||
continue
|
||||
}
|
||||
stream = &c
|
||||
@@ -204,7 +217,12 @@ func transferVideos(s *Sync) error {
|
||||
|
||||
streamUpdateOptions := jsonrpc.StreamUpdateOptions{
|
||||
StreamCreateOptions: &jsonrpc.StreamCreateOptions{
|
||||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{ClaimAddress: &s.clientPublishAddress},
|
||||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
|
||||
ClaimAddress: &s.clientPublishAddress,
|
||||
FundingAccountIDs: []string{
|
||||
account,
|
||||
},
|
||||
},
|
||||
},
|
||||
Bid: util.PtrToString("0.005"), // Todo - Dont hardcode
|
||||
}
|
||||
|
||||
+76
-20
@@ -91,6 +91,7 @@ type Sync struct {
|
||||
clientPublishAddress string
|
||||
publicKey string
|
||||
defaultAccountID string
|
||||
MaxVideoLength float64
|
||||
}
|
||||
|
||||
func (s *Sync) AppendSyncedVideo(videoID string, published bool, failureReason string, claimName string, claimID string, metadataVersion int8, size int64) {
|
||||
@@ -246,6 +247,8 @@ func (s *Sync) setExceptions() {
|
||||
}
|
||||
}
|
||||
|
||||
var stopGroup = stop.New()
|
||||
|
||||
func (s *Sync) FullCycle() (e error) {
|
||||
if os.Getenv("HOME") == "" {
|
||||
return errors.Err("no $HOME env var found")
|
||||
@@ -258,7 +261,7 @@ func (s *Sync) FullCycle() (e error) {
|
||||
|
||||
s.syncedVideosMux = &sync.RWMutex{}
|
||||
s.walletMux = &sync.RWMutex{}
|
||||
s.grp = stop.New()
|
||||
s.grp = stopGroup
|
||||
s.queue = make(chan video)
|
||||
interruptChan := make(chan os.Signal, 1)
|
||||
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
|
||||
@@ -357,7 +360,18 @@ func (s *Sync) processTransfers() (e error) {
|
||||
isTip := true
|
||||
summary, err := s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount), &isTip, nil, []string{defaultAccount}, nil)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
if strings.Contains(err.Error(), "tx-size") { //TODO: this is a silly workaround and should be written in an recursive function
|
||||
summary, err = s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount/2.0), &isTip, nil, []string{defaultAccount}, nil)
|
||||
if err != nil {
|
||||
return errors.Prefix(fmt.Sprintf("something went wrong while tipping the channel for %.6f LBCs", supportAmount), err)
|
||||
}
|
||||
summary, err = s.daemon.SupportCreate(s.lbryChannelID, fmt.Sprintf("%.6f", supportAmount/2.0), &isTip, nil, []string{defaultAccount}, nil)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
} else {
|
||||
return errors.Err(err)
|
||||
}
|
||||
}
|
||||
if len(summary.Outputs) < 1 {
|
||||
return errors.Err("something went wrong while tipping the channel for %.6f LBCs", supportAmount)
|
||||
@@ -467,7 +481,7 @@ var thumbnailHosts = []string{
|
||||
thumbs.ThumbnailEndpoint,
|
||||
}
|
||||
|
||||
func isYtsyncClaim(c jsonrpc.Claim) bool {
|
||||
func isYtsyncClaim(c jsonrpc.Claim, expectedChannelID string) bool {
|
||||
if !util.InSlice(c.Type, []string{"claim", "update"}) || c.Value.GetStream() == nil {
|
||||
return false
|
||||
}
|
||||
@@ -475,7 +489,12 @@ func isYtsyncClaim(c jsonrpc.Claim) bool {
|
||||
//most likely a claim created outside of ytsync, ignore!
|
||||
return false
|
||||
}
|
||||
|
||||
if c.SigningChannel == nil {
|
||||
return false
|
||||
}
|
||||
if c.SigningChannel.ClaimID != expectedChannelID {
|
||||
return false
|
||||
}
|
||||
for _, th := range thumbnailHosts {
|
||||
if strings.Contains(c.Value.GetThumbnail().GetUrl(), th) {
|
||||
return true
|
||||
@@ -489,7 +508,7 @@ func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) {
|
||||
abandonedClaims := false
|
||||
videoIDs := make(map[string]jsonrpc.Claim)
|
||||
for _, c := range claims {
|
||||
if !isYtsyncClaim(c) {
|
||||
if !isYtsyncClaim(c, s.lbryChannelID) {
|
||||
continue
|
||||
}
|
||||
tn := c.Value.GetThumbnail().GetUrl()
|
||||
@@ -536,7 +555,7 @@ type ytsyncClaim struct {
|
||||
func (s *Sync) mapFromClaims(claims []jsonrpc.Claim) map[string]ytsyncClaim {
|
||||
videoIDMap := make(map[string]ytsyncClaim, len(claims))
|
||||
for _, c := range claims {
|
||||
if !isYtsyncClaim(c) {
|
||||
if !isYtsyncClaim(c, s.lbryChannelID) {
|
||||
continue
|
||||
}
|
||||
tn := c.Value.GetThumbnail().GetUrl()
|
||||
@@ -633,8 +652,18 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim, ownClaims []jsonrpc.Claim)
|
||||
if sv.Transferred {
|
||||
_, ok := allClaimsInfo[vID]
|
||||
if !ok && sv.Published {
|
||||
log.Warnf("%s: claims to be published and transferred but wasn't found in the list of claims", vID)
|
||||
idsToRemove = append(idsToRemove, vID)
|
||||
searchResponse, err := s.daemon.ClaimSearch(nil, &sv.ClaimID, nil, nil, 1, 20)
|
||||
if err != nil {
|
||||
log.Error(err.Error())
|
||||
continue
|
||||
}
|
||||
if len(searchResponse.Claims) == 0 {
|
||||
log.Debugf("%s: was transferred but appears abandoned! we should ignore this - claimID: %s", vID, sv.ClaimID)
|
||||
continue //TODO: we should flag these on the db
|
||||
} else {
|
||||
log.Debugf("%s: was transferred and was then edited! we should ignore this - claimID: %s", vID, sv.ClaimID)
|
||||
//return count, fixed, 0, errors.Err("%s: isn't our control but is on the database and on the blockchain. wtf is up? ClaimID: %s", vID, sv.ClaimID)
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
@@ -675,8 +704,13 @@ func (s *Sync) getClaims(defaultOnly bool) ([]jsonrpc.Claim, error) {
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("cannot list claims", err)
|
||||
}
|
||||
|
||||
return claims.Items, nil
|
||||
items := make([]jsonrpc.Claim, 0, len(claims.Items))
|
||||
for _, c := range claims.Items {
|
||||
if c.SigningChannel != nil && c.SigningChannel.ClaimID == s.lbryChannelID {
|
||||
items = append(items, c)
|
||||
}
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (s *Sync) checkIntegrity() error {
|
||||
@@ -748,6 +782,10 @@ func (s *Sync) doSync() error {
|
||||
if err != nil {
|
||||
return errors.Prefix("could not import the transferee public key", err)
|
||||
}
|
||||
_, err = s.daemon.UTXORelease(nil)
|
||||
if err != nil {
|
||||
return errors.Prefix("could not run uxo_release", err)
|
||||
}
|
||||
err = s.walletSetup()
|
||||
if err != nil {
|
||||
return errors.Prefix("Initial wallet setup failed! Manual Intervention is required.", err)
|
||||
@@ -758,7 +796,7 @@ func (s *Sync) doSync() error {
|
||||
return err
|
||||
}
|
||||
|
||||
if s.transferState != TransferStateComplete {
|
||||
if s.transferState < TransferStateComplete {
|
||||
cert, err := s.daemon.ChannelExport(s.lbryChannelID, nil, nil)
|
||||
if err != nil {
|
||||
return errors.Prefix("error getting channel cert", err)
|
||||
@@ -862,12 +900,18 @@ func (s *Sync) startWorker(workerNum int) {
|
||||
"giving up after 0 fragment retries",
|
||||
"Sorry about that",
|
||||
"This video is not available",
|
||||
"requested format not available",
|
||||
"interrupted by user",
|
||||
"Sign in to confirm your age",
|
||||
"This video is unavailable",
|
||||
}
|
||||
if util.SubstringInSlice(err.Error(), errorsNoRetry) {
|
||||
log.Println("This error should not be retried at all")
|
||||
} else if tryCount < s.MaxTries {
|
||||
if strings.Contains(err.Error(), "txn-mempool-conflict") ||
|
||||
strings.Contains(err.Error(), "too-long-mempool-chain") {
|
||||
if util.SubstringInSlice(err.Error(), []string{
|
||||
"txn-mempool-conflict",
|
||||
"too-long-mempool-chain",
|
||||
}) {
|
||||
log.Println("waiting for a block before retrying")
|
||||
err := s.waitForNewBlock()
|
||||
if err != nil {
|
||||
@@ -879,6 +923,7 @@ func (s *Sync) startWorker(workerNum int) {
|
||||
"Not enough funds to cover this transaction",
|
||||
"failed: Not enough funds",
|
||||
"Error in daemon: Insufficient funds, please deposit additional LBC",
|
||||
"Missing inputs",
|
||||
}) {
|
||||
log.Println("checking funds and UTXOs before retrying...")
|
||||
err := s.walletSetup()
|
||||
@@ -935,6 +980,8 @@ func (s *Sync) startWorker(workerNum int) {
|
||||
}
|
||||
}
|
||||
|
||||
var mostRecentlyFailedChannel string
|
||||
|
||||
func (s *Sync) enqueueYoutubeVideos() error {
|
||||
client := &http.Client{
|
||||
Transport: &transport.APIKey{Key: s.APIConfig.YoutubeAPIKey},
|
||||
@@ -964,10 +1011,11 @@ func (s *Sync) enqueueYoutubeVideos() error {
|
||||
}
|
||||
|
||||
var videos []video
|
||||
ipPool, err := ip_manager.GetIPPool()
|
||||
ipPool, err := ip_manager.GetIPPool(s.grp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
playlistMap := make(map[string]*youtube.PlaylistItemSnippet, 50)
|
||||
nextPageToken := ""
|
||||
for {
|
||||
@@ -988,9 +1036,12 @@ func (s *Sync) enqueueYoutubeVideos() error {
|
||||
if youtubeIsLying {
|
||||
break
|
||||
}
|
||||
return errors.Err("playlist items not found")
|
||||
if s.YoutubeChannelID == mostRecentlyFailedChannel {
|
||||
return errors.Err("playlist items not found")
|
||||
}
|
||||
mostRecentlyFailedChannel = s.YoutubeChannelID
|
||||
break //return errors.Err("playlist items not found") //TODO: will this work?
|
||||
}
|
||||
//playlistMap := make(map[string]*youtube.PlaylistItemSnippet, 50)
|
||||
videoIDs := make([]string, 50)
|
||||
for i, item := range playlistResponse.Items {
|
||||
// normally we'd send the video into the channel here, but youtube api doesn't have sorting
|
||||
@@ -1011,7 +1062,7 @@ func (s *Sync) enqueueYoutubeVideos() error {
|
||||
log.Infof("Got info for %d videos from youtube API", len(videos))
|
||||
|
||||
nextPageToken = playlistResponse.NextPageToken
|
||||
if nextPageToken == "" {
|
||||
if nextPageToken == "" || s.Manager.SyncFlags.QuickSync || len(videos) >= s.Manager.videosLimit {
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -1026,7 +1077,6 @@ func (s *Sync) enqueueYoutubeVideos() error {
|
||||
|
||||
}
|
||||
sort.Sort(byPublishedAt(videos))
|
||||
//or sort.Sort(sort.Reverse(byPlaylistPosition(videos)))
|
||||
|
||||
Enqueue:
|
||||
for _, v := range videos {
|
||||
@@ -1049,7 +1099,7 @@ Enqueue:
|
||||
func (s *Sync) processVideo(v video) (err error) {
|
||||
defer func() {
|
||||
if p := recover(); p != nil {
|
||||
log.Printf("stack: %s", debug.Stack())
|
||||
logUtils.SendErrorToSlack("Video processing panic! %s", debug.Stack())
|
||||
var ok bool
|
||||
err, ok = p.(error)
|
||||
if !ok {
|
||||
@@ -1081,6 +1131,7 @@ func (s *Sync) processVideo(v video) (err error) {
|
||||
"Watch this video on YouTube.",
|
||||
"have blocked it on copyright grounds",
|
||||
"giving up after 0 fragment retries",
|
||||
"Sign in to confirm your age",
|
||||
}
|
||||
if ok && !sv.Published && util.SubstringInSlice(sv.FailureReason, neverRetryFailures) {
|
||||
log.Println(v.ID() + " can't ever be published")
|
||||
@@ -1104,14 +1155,19 @@ func (s *Sync) processVideo(v video) (err error) {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
da, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sp := sources.SyncParams{
|
||||
ClaimAddress: s.claimAddress,
|
||||
Amount: publishAmount,
|
||||
ChannelID: s.lbryChannelID,
|
||||
MaxVideoSize: s.Manager.maxVideoSize,
|
||||
Namer: s.namer,
|
||||
MaxVideoLength: s.Manager.maxVideoLength,
|
||||
MaxVideoLength: s.MaxVideoLength,
|
||||
Fee: s.Fee,
|
||||
DefaultAccount: da,
|
||||
}
|
||||
|
||||
summary, err := v.Sync(s.daemon, sp, &sv, videoRequiresUpgrade, s.walletMux)
|
||||
|
||||
@@ -45,6 +45,7 @@ type SyncFlags struct {
|
||||
RemoveDBUnpublished bool
|
||||
UpgradeMetadata bool
|
||||
DisableTransfers bool
|
||||
QuickSync bool
|
||||
}
|
||||
|
||||
type Fee struct {
|
||||
@@ -55,6 +56,7 @@ type Fee struct {
|
||||
type YoutubeChannel struct {
|
||||
ChannelId string `json:"channel_id"`
|
||||
TotalVideos uint `json:"total_videos"`
|
||||
TotalSubscribers uint `json:"total_subscribers"`
|
||||
DesiredChannelName string `json:"desired_channel_name"`
|
||||
Fee *Fee `json:"fee"`
|
||||
ChannelClaimID string `json:"channel_claim_id"`
|
||||
|
||||
+96
-77
@@ -178,7 +178,7 @@ func (v *YoutubeVideo) getAbbrevDescription() string {
|
||||
return description + "\n..." + additionalDescription
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) download(useIPv6 bool) error {
|
||||
func (v *YoutubeVideo) download() error {
|
||||
videoPath := v.getFullPath()
|
||||
|
||||
err := os.Mkdir(v.videoDir(), 0777)
|
||||
@@ -199,7 +199,6 @@ func (v *YoutubeVideo) download(useIPv6 bool) error {
|
||||
"480",
|
||||
"320",
|
||||
}
|
||||
qualityIndex := 0
|
||||
ytdlArgs := []string{
|
||||
"--no-progress",
|
||||
"-o" + strings.TrimSuffix(v.getFullPath(), ".mp4"),
|
||||
@@ -210,6 +209,10 @@ func (v *YoutubeVideo) download(useIPv6 bool) error {
|
||||
"--abort-on-unavailable-fragment",
|
||||
"--fragment-retries",
|
||||
"0",
|
||||
"--user-agent",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36",
|
||||
"--cookies",
|
||||
"cookies.txt",
|
||||
}
|
||||
if v.maxVideoSize > 0 {
|
||||
ytdlArgs = append(ytdlArgs,
|
||||
@@ -224,92 +227,94 @@ func (v *YoutubeVideo) download(useIPv6 bool) error {
|
||||
)
|
||||
}
|
||||
|
||||
sourceAddress, err := v.pool.GetIP()
|
||||
if err != nil {
|
||||
if sourceAddress == "throttled" {
|
||||
for {
|
||||
var sourceAddress string
|
||||
for {
|
||||
sourceAddress, err = v.pool.GetIP(v.id)
|
||||
if err != nil {
|
||||
if errors.Is(err, ip_manager.ErrAllThrottled) {
|
||||
select {
|
||||
case <-v.stopGroup.Ch():
|
||||
return errors.Err("interrupted by user")
|
||||
default:
|
||||
time.Sleep(ip_manager.IPCooldownPeriod)
|
||||
continue
|
||||
}
|
||||
|
||||
time.Sleep(ip_manager.IPCooldownPeriod)
|
||||
sourceAddress, err = v.pool.GetIP()
|
||||
if err == nil { //TODO: This is possibly not 100% right, but it works so I'm not touching it...
|
||||
break
|
||||
}
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
return errors.Err(err)
|
||||
}
|
||||
break
|
||||
}
|
||||
defer v.pool.ReleaseIP(sourceAddress)
|
||||
|
||||
ytdlArgs = append(ytdlArgs,
|
||||
"--source-address",
|
||||
sourceAddress,
|
||||
"https://www.youtube.com/watch?v="+v.ID(),
|
||||
)
|
||||
|
||||
runcmd:
|
||||
argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][height<="+qualities[qualityIndex]+"]+bestaudio[ext!=webm]")
|
||||
cmd := exec.Command("youtube-dl", argsWithFilters...)
|
||||
for i, quality := range qualities {
|
||||
argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][height<="+quality+"]+bestaudio[ext!=webm]")
|
||||
cmd := exec.Command("youtube-dl", argsWithFilters...)
|
||||
log.Printf("Running command youtube-dl %s", strings.Join(argsWithFilters, " "))
|
||||
|
||||
log.Printf("Running command youtube-dl %s", strings.Join(argsWithFilters, " "))
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
if err := cmd.Start(); err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
errorLog, _ := ioutil.ReadAll(stderr)
|
||||
outLog, _ := ioutil.ReadAll(stdout)
|
||||
|
||||
if err = cmd.Wait(); err != nil {
|
||||
if strings.Contains(err.Error(), "exit status 1") {
|
||||
if strings.Contains(string(errorLog), "HTTP Error 429") {
|
||||
v.pool.SetThrottled(sourceAddress, v.stopGroup)
|
||||
} else if strings.Contains(string(errorLog), "giving up after 0 fragment retries") && qualityIndex < len(qualities)-1 {
|
||||
qualityIndex++
|
||||
goto runcmd
|
||||
errorLog, _ := ioutil.ReadAll(stderr)
|
||||
outLog, _ := ioutil.ReadAll(stdout)
|
||||
err = cmd.Wait()
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "exit status 1") {
|
||||
if strings.Contains(string(errorLog), "HTTP Error 429") || strings.Contains(string(errorLog), "returned non-zero exit status 8") {
|
||||
v.pool.SetThrottled(sourceAddress)
|
||||
} else if strings.Contains(string(errorLog), "giving up after 0 fragment retries") {
|
||||
if i == (len(qualities) - 1) {
|
||||
return errors.Err(string(errorLog))
|
||||
}
|
||||
continue //this bypasses the yt throttling IP redistribution... TODO: don't
|
||||
}
|
||||
return errors.Err(string(errorLog))
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
log.Debugln(string(outLog))
|
||||
|
||||
if strings.Contains(string(outLog), "does not pass filter duration") {
|
||||
_ = v.delete("does not pass filter duration")
|
||||
return errors.Err("video is too long to process")
|
||||
}
|
||||
if strings.Contains(string(outLog), "File is larger than max-filesize") {
|
||||
_ = v.delete("File is larger than max-filesize")
|
||||
return errors.Err("the video is too big to sync, skipping for now")
|
||||
}
|
||||
if string(errorLog) != "" {
|
||||
log.Printf("Command finished with error: %v", errors.Err(string(errorLog)))
|
||||
_ = v.delete("due to error")
|
||||
return errors.Err(string(errorLog))
|
||||
}
|
||||
return errors.Err(err)
|
||||
fi, err := os.Stat(v.getFullPath())
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
err = os.Chmod(v.getFullPath(), 0777)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
videoSize := fi.Size()
|
||||
v.size = &videoSize
|
||||
break
|
||||
}
|
||||
log.Debugln(string(outLog))
|
||||
|
||||
if strings.Contains(string(outLog), "does not pass filter duration") {
|
||||
_ = v.delete("does not pass filter duration")
|
||||
return errors.Err("video is too long to process")
|
||||
}
|
||||
if strings.Contains(string(outLog), "File is larger than max-filesize") {
|
||||
_ = v.delete("File is larger than max-filesize")
|
||||
return errors.Err("the video is too big to sync, skipping for now")
|
||||
}
|
||||
if string(errorLog) != "" {
|
||||
log.Printf("Command finished with error: %v", errors.Err(string(errorLog)))
|
||||
_ = v.delete("due to error")
|
||||
return errors.Err(string(errorLog))
|
||||
}
|
||||
fi, err := os.Stat(v.getFullPath())
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
err = os.Chmod(v.getFullPath(), 0777)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
videoSize := fi.Size()
|
||||
v.size = &videoSize
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -362,7 +367,6 @@ func (v *YoutubeVideo) triggerThumbnailSave() (err error) {
|
||||
|
||||
func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
|
||||
languages, locations, tags := v.getMetadata()
|
||||
|
||||
var fee *jsonrpc.Fee
|
||||
if params.Fee != nil {
|
||||
feeAmount, err := decimal.NewFromString(params.Fee.Amount)
|
||||
@@ -385,6 +389,9 @@ func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, params SyncParams) (*Sync
|
||||
ThumbnailURL: &v.thumbnailURL,
|
||||
Tags: tags,
|
||||
Locations: locations,
|
||||
FundingAccountIDs: []string{
|
||||
params.DefaultAccount,
|
||||
},
|
||||
},
|
||||
Fee: fee,
|
||||
License: util.PtrToString("Copyrighted (contact publisher)"),
|
||||
@@ -410,6 +417,7 @@ type SyncParams struct {
|
||||
Namer *namer.Namer
|
||||
MaxVideoLength float64
|
||||
Fee *sdk.Fee
|
||||
DefaultAccount string
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo, reprocess bool, walletLock *sync.RWMutex) (*SyncSummary, error) {
|
||||
@@ -424,21 +432,26 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV
|
||||
return v.downloadAndPublish(daemon, params)
|
||||
}
|
||||
|
||||
var isThrottled bool
|
||||
|
||||
func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
|
||||
err := v.download(isThrottled)
|
||||
var err error
|
||||
videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "HTTP Error 429") && !isThrottled {
|
||||
isThrottled = true
|
||||
err = v.download(isThrottled)
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("download error", err)
|
||||
}
|
||||
} else {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
if videoDuration.ToDuration() > time.Duration(v.maxVideoLength*60)*time.Minute {
|
||||
log.Infof("%s is %s long and the limit is %s", v.id, videoDuration.ToDuration().String(), (time.Duration(v.maxVideoLength*60) * time.Minute).String())
|
||||
return nil, errors.Err("video is too long to process")
|
||||
}
|
||||
for {
|
||||
err = v.download()
|
||||
if err != nil && strings.Contains(err.Error(), "HTTP Error 429") {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return nil, errors.Prefix("download error", err)
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
log.Debugln("Downloaded " + v.id)
|
||||
|
||||
err = v.triggerThumbnailSave()
|
||||
@@ -460,6 +473,9 @@ func (v *YoutubeVideo) getMetadata() (languages []string, locations []jsonrpc.Lo
|
||||
tags = nil
|
||||
if !v.mocked {
|
||||
if v.youtubeInfo.Snippet.DefaultLanguage != "" {
|
||||
if v.youtubeInfo.Snippet.DefaultLanguage == "iw" {
|
||||
v.youtubeInfo.Snippet.DefaultLanguage = "he"
|
||||
}
|
||||
languages = []string{v.youtubeInfo.Snippet.DefaultLanguage}
|
||||
}
|
||||
|
||||
@@ -543,6 +559,9 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
|
||||
ThumbnailURL: &thumbnailURL,
|
||||
Languages: languages,
|
||||
Locations: locations,
|
||||
FundingAccountIDs: []string{
|
||||
params.DefaultAccount,
|
||||
},
|
||||
},
|
||||
Author: util.PtrToString(""),
|
||||
License: util.PtrToString("Copyrighted (contact publisher)"),
|
||||
|
||||
+48
-2
@@ -49,6 +49,19 @@ func GetLBRYNetDir() string {
|
||||
return lbrynetDir
|
||||
}
|
||||
|
||||
func GetLbryumDir() string {
|
||||
lbryumDir := os.Getenv("LBRYNET_WALLETS_DIR")
|
||||
if lbryumDir == "" {
|
||||
usr, err := user.Current()
|
||||
if err != nil {
|
||||
log.Errorln(err.Error())
|
||||
return ""
|
||||
}
|
||||
return usr.HomeDir + "/.lbryum/"
|
||||
}
|
||||
return lbryumDir + "/"
|
||||
}
|
||||
|
||||
const ALL = true
|
||||
const ONLINE = false
|
||||
|
||||
@@ -94,15 +107,20 @@ func IsRegTest() bool {
|
||||
}
|
||||
|
||||
func GetLbrycrdClient(lbrycrdString string) (*lbrycrd.Client, error) {
|
||||
chainName := os.Getenv("CHAINNAME")
|
||||
chainParams, ok := lbrycrd.ChainParamsMap[chainName]
|
||||
if !ok {
|
||||
chainParams = lbrycrd.MainNetParams
|
||||
}
|
||||
var lbrycrdd *lbrycrd.Client
|
||||
var err error
|
||||
if lbrycrdString == "" {
|
||||
lbrycrdd, err = lbrycrd.NewWithDefaultURL()
|
||||
lbrycrdd, err = lbrycrd.NewWithDefaultURL(&chainParams)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
lbrycrdd, err = lbrycrd.New(lbrycrdString)
|
||||
lbrycrdd, err = lbrycrd.New(lbrycrdString, &chainParams)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -220,6 +238,34 @@ func CleanupLbrynet() error {
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
lbryumDir := GetLbryumDir()
|
||||
ledger := "lbc_mainnet"
|
||||
if IsRegTest() {
|
||||
ledger = "lbc_regtest"
|
||||
}
|
||||
lbryumDir = lbryumDir + ledger
|
||||
|
||||
db, err := os.Stat(lbryumDir + "/blockchain.db")
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
dbSizeLimit := int64(1 * 1024 * 1024 * 1024)
|
||||
if db.Size() > dbSizeLimit {
|
||||
files, err := filepath.Glob(lbryumDir + "/blockchain.db*")
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
for _, f := range files {
|
||||
err = os.Remove(f)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user