Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f17110ab7f | ||
|
|
768743a200 | ||
|
|
7c652b22a1 | ||
|
|
a0fb4e579e | ||
|
|
519e1e4648 | ||
|
|
087f20c133 | ||
|
|
21e6603a26 | ||
|
|
ca41f5052e | ||
|
|
d739a98beb | ||
|
|
7c7ceed333 | ||
|
|
198473b62b | ||
|
|
d53d0a1d52 | ||
|
|
55577201a4 | ||
|
|
cacd21f840 | ||
|
|
9799b0a732 | ||
|
|
e985a60761 | ||
|
|
4b4cee9fcc | ||
|
|
acca80dc70 | ||
|
|
053c618458 | ||
|
|
2cf3526c74 | ||
|
|
d7e194cb5c | ||
|
|
9a8b1922fe | ||
|
|
12d627bd35 | ||
|
|
9397a40c9f | ||
|
|
6a4093f3f8 | ||
|
|
bac2969295 | ||
|
|
5dfd8dee1b | ||
|
|
06c43c3f71 | ||
|
|
881d86799b | ||
|
|
f7e4366a67 | ||
|
|
5b5f7e1c68 | ||
|
|
fecf67118c | ||
|
|
beade71aa6 | ||
|
|
ca8ff505d4 | ||
|
|
f5f12e1560 | ||
|
|
c4693c221f | ||
|
|
692c796770 | ||
|
|
de798ade36 | ||
|
|
a93c2c4538 | ||
|
|
53e0b7c47b | ||
|
|
070dda8739 | ||
|
|
f773569920 | ||
|
|
ad6fa4d725 | ||
|
|
d4ca71a89d | ||
|
|
c53cf4c1b3 | ||
|
|
ad5a30da9e | ||
|
|
ecda80b02d | ||
|
|
24cf937e14 | ||
|
|
0b002c8228 | ||
|
|
a56166ee51 | ||
|
|
ddca850c17 | ||
|
|
31ce612e2e | ||
|
|
eb8900c66a | ||
|
|
67da4142d5 | ||
|
|
fb0e567caf | ||
|
|
7e83b17b69 | ||
|
|
7c02c5b92d | ||
|
|
2a33f44317 | ||
|
|
b7037900f8 | ||
|
|
5d230a6b54 | ||
|
|
f0280b51b4 | ||
|
|
775e4881cb | ||
|
|
ec9f46f552 | ||
|
|
6c6e93cefc | ||
|
|
eab9bcf1ff | ||
|
|
751bc84ce5 | ||
|
|
41fd9f6844 | ||
|
|
7f1906d58b | ||
|
|
5be3551abe | ||
|
|
fc18151d77 | ||
|
|
13543b20e9 | ||
|
|
4eba7200d8 | ||
|
|
cd11e82676 | ||
|
|
c7c220ecd3 | ||
|
|
4d56339756 | ||
|
|
140353097e | ||
|
|
a1caea4a28 | ||
|
|
b1e10e7b09 | ||
|
|
ad27425471 | ||
|
|
81e9378b9d | ||
|
|
64040ea67a | ||
|
|
d99e200178 | ||
|
|
357aebbcce | ||
|
|
0eef62b5fd | ||
|
|
a3dd3dc626 | ||
|
|
f942bf8025 | ||
|
|
a05864404d | ||
|
|
70ad891dfa | ||
|
|
8e61cde0a0 | ||
|
|
8600077caa | ||
|
|
e7fdd21bac | ||
|
|
0bb6b6d833 | ||
|
|
b59ef28267 | ||
|
|
e926a2c1f6 | ||
|
|
1369ed0b48 | ||
|
|
f39fc11697 | ||
|
|
bffc0823be | ||
|
|
edbb22fcf0 | ||
|
|
843303301a | ||
|
|
be7fd7ddd8 | ||
|
|
bfcfe5f36b | ||
|
|
4c4da93ef3 | ||
|
|
c2b5878daa | ||
|
|
749d5095c9 | ||
|
|
ba77b61ae0 | ||
|
|
07915ce697 | ||
|
|
4b2221ce5e | ||
|
|
1d5b69bfe6 | ||
|
|
4d915587e0 | ||
|
|
83f2c28c20 | ||
|
|
47e467057d | ||
|
|
1ae32d638b | ||
|
|
7907ee3579 | ||
|
|
414ed1c130 | ||
|
|
3bb17b09d3 | ||
|
|
046f46267e | ||
|
|
adc3ffe194 | ||
|
|
76301d40ba | ||
|
|
963c450730 | ||
|
|
6e7bb994d0 | ||
|
|
95890cbabf | ||
|
|
69c2a91ec7 | ||
|
|
fbe09a692c | ||
|
|
0ccc6e7979 | ||
|
|
3eb74cc8b5 | ||
|
|
9bfb1656fe | ||
|
|
5d40d24804 | ||
|
|
79d3aadda8 | ||
|
|
eba59a0806 | ||
|
|
3fd4909cd1 | ||
|
|
b59372c1d8 | ||
|
|
c9e713e9df | ||
|
|
bc72d2f1c4 | ||
|
|
df539e5d01 | ||
|
|
1b89104101 | ||
|
|
3dc30c3b98 | ||
|
|
259df8d257 | ||
|
|
5460224cd6 | ||
|
|
29479a73a7 | ||
|
|
625f339709 | ||
|
|
d6fcfd888a | ||
|
|
5f7d9d4940 | ||
|
|
826fcb78ea | ||
|
|
45e87c3007 | ||
|
|
cdee0d0de7 | ||
|
|
e30f21dc85 | ||
|
|
df88c7e952 | ||
|
|
62a1a0eed5 | ||
|
|
93f9185c71 | ||
|
|
ef62242f42 | ||
|
|
4a356e86da | ||
|
|
ac95ca57af | ||
|
|
c28d5a716b | ||
|
|
59471f9c26 | ||
|
|
8028c6621c | ||
|
|
8abf6ad255 | ||
|
|
1f9a645c57 | ||
|
|
a5657aaf46 | ||
|
|
b6861dae9b | ||
|
|
cff147bb95 | ||
|
|
54d04dcf2c | ||
|
|
65c4f99c1c | ||
|
|
68c0fd9ed7 | ||
|
|
2bde06e4b9 | ||
|
|
5df6db6e96 | ||
|
|
62dfdc1adb | ||
|
|
fe55304184 | ||
|
|
bb76500f04 | ||
|
|
4e2efec66e | ||
|
|
8d650e032c | ||
|
|
f5d97957e7 | ||
|
|
36f539ef5d | ||
|
|
77461f71af | ||
|
|
caca13de61 | ||
|
|
7bdc117ba0 | ||
|
|
fc575dac80 | ||
|
|
d520ff2c68 | ||
|
|
25e49f8ff2 | ||
|
|
77d0efcf3c | ||
|
|
291a105269 | ||
|
|
682117a030 | ||
|
|
6d2e4aa6e6 | ||
|
|
77749f05ba | ||
|
|
10a855a8ba | ||
|
|
a9e76149e8 | ||
|
|
dd4aebdba4 | ||
|
|
b409d36de0 | ||
|
|
30af4a0136 | ||
|
|
a3b660fab5 | ||
|
|
65daeec216 | ||
|
|
ad1704fb39 | ||
|
|
44e6cb5ddc | ||
|
|
5171acc007 | ||
|
|
946314da94 | ||
|
|
5961327adf | ||
|
|
d2bbb36312 | ||
|
|
c7272e988a | ||
|
|
5b9846bcca | ||
|
|
8d849b8f06 | ||
|
|
c41c368f34 | ||
|
|
7ce4c4a2ce | ||
|
|
0d473cc4f6 | ||
|
|
2f701a887a | ||
|
|
8d56ef852f | ||
|
|
849ff11bfd | ||
|
|
45982f30c5 | ||
|
|
07413b499a | ||
|
|
92d8ef75f8 | ||
|
|
1b08bb0e61 | ||
|
|
bdd55c9965 | ||
|
|
14668c339e | ||
|
|
7d38aa7b29 | ||
|
|
3c3ea2138e | ||
|
|
7b23235f83 | ||
|
|
79ae630bc4 | ||
|
|
0d8db26e96 | ||
|
|
bacb91e82a | ||
|
|
577411cf50 | ||
|
|
e98061f2eb | ||
|
|
4548c41082 | ||
|
|
f7c80c2e5d | ||
|
|
2d6e53be32 | ||
|
|
419434db28 | ||
|
|
cd6ddaaa85 | ||
|
|
4d16a1d6e8 | ||
|
|
1fb1077a94 | ||
|
|
2c9c999c71 | ||
|
|
c754562f96 | ||
|
|
7a7de03c0f | ||
|
|
1b9ed266e0 | ||
|
|
eb2f6273e4 | ||
|
|
45cea808ed | ||
|
|
9a5b6b4e56 | ||
|
|
d63aba568d | ||
|
|
4cc0b71279 | ||
|
|
b8d66cb8c0 | ||
|
|
eff76d22eb | ||
|
|
8ebdf33213 | ||
|
|
81616c1d96 | ||
|
|
b477d18a16 | ||
|
|
5cb7f72ef7 | ||
|
|
1b1d7f5ec0 | ||
|
|
adeabc97f8 | ||
|
|
d73c6945dd | ||
|
|
7a76a4b79d | ||
|
|
cd57e1ab08 | ||
|
|
b90e367e82 | ||
|
|
e477759395 | ||
|
|
6438048f40 | ||
|
|
73937a541b | ||
|
|
517c2c56a3 | ||
|
|
c84e1e6fb9 | ||
|
|
f26712b022 | ||
|
|
5e56ac0516 | ||
|
|
0308da2e4c | ||
|
|
df63b91f61 | ||
|
|
03161a5a02 | ||
|
|
6dc2c39ee6 | ||
|
|
87211e802d | ||
|
|
3ba0c97634 | ||
|
|
9d65fce6bc | ||
|
|
d5ed4ce753 | ||
|
|
0aedf80386 | ||
|
|
c193dae95a | ||
|
|
0c47510785 | ||
|
|
8f556a86c8 | ||
|
|
64ab2490fe | ||
|
|
1760935edf | ||
|
|
60f2585f33 | ||
|
|
df08d42d9b | ||
|
|
1b55033bb8 | ||
|
|
1331e5769f | ||
|
|
43f745ae20 | ||
|
|
8c2a8262e6 | ||
|
|
6f486717da | ||
|
|
a3fcd67611 | ||
|
|
ae1ffb60c5 | ||
|
|
fa24d83ae9 | ||
|
|
2f11d99a97 | ||
|
|
bab01a6354 | ||
|
|
89fe1e99d6 | ||
|
|
ec81af0857 | ||
|
|
51cb7f5e34 | ||
|
|
8ad2acc936 |
No files matched your search
+6
-1
@@ -1 +1,6 @@
|
||||
bin/
|
||||
bin/
|
||||
e2e/persist
|
||||
e2e/supporty/supporty
|
||||
.env
|
||||
blobsfiles
|
||||
ytsync_docker
|
||||
+8
-8
@@ -9,12 +9,13 @@ builds:
|
||||
- amd64
|
||||
ldflags:
|
||||
- -X "{{ .Env.IMPORTPATH }}/meta.semVersion={{ .Tag }}" -X "{{ .Env.IMPORTPATH }}/meta.version={{ .Env.VERSIONSHORT }}" -X "{{ .Env.IMPORTPATH }}/meta.versionLong={{ .Env.VERSIONLONG }}" -X "{{ .Env.IMPORTPATH }}/meta.commitMsg={{ .Env.COMMITMSG }}"
|
||||
archive:
|
||||
name_template: '{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}'
|
||||
replacements:
|
||||
linux: Linux
|
||||
amd64: x86_64
|
||||
format: zip
|
||||
archives:
|
||||
- id: zip
|
||||
name_template: '{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}'
|
||||
replacements:
|
||||
linux: Linux
|
||||
amd64: x86_64
|
||||
format: zip
|
||||
checksum:
|
||||
name_template: 'checksums.txt'
|
||||
snapshot:
|
||||
@@ -24,5 +25,4 @@ changelog:
|
||||
filters:
|
||||
exclude:
|
||||
- '^docs:'
|
||||
- '^test:'
|
||||
|
||||
- '^test:'
|
||||
+31
-6
@@ -1,15 +1,40 @@
|
||||
os: linux
|
||||
dist: xenial
|
||||
dist: bionic
|
||||
language: go
|
||||
go_import_path: github.com/lbryio/ytsync
|
||||
go:
|
||||
- 1.11.5
|
||||
- master
|
||||
- 1.16.3
|
||||
|
||||
install: true
|
||||
|
||||
cache:
|
||||
directories:
|
||||
- $HOME/.cache/go-build
|
||||
- $HOME/gopath/pkg/mod
|
||||
|
||||
services:
|
||||
- docker
|
||||
|
||||
addons:
|
||||
apt:
|
||||
update: true
|
||||
packages:
|
||||
- ffmpeg
|
||||
- tree
|
||||
- python3-pip
|
||||
|
||||
before_script:
|
||||
- sudo pip3 install -U yt-dlp
|
||||
- sudo add-apt-repository -y ppa:savoury1/ffmpeg4
|
||||
|
||||
env:
|
||||
global:
|
||||
- GO111MODULE=on
|
||||
#GITHUB_TOKEN
|
||||
- secure: "Ps3KocRP5xnM3/uA99CeYhDTVxRIuW7fGyrtqBeRWZW0cXzeA4XCTKxqcFbrPUPw67XkrBVgE58JDdWoQEJ7tm67PjMm/ltp5Evhx/QAJDh+YSofXyGDVpG1mrTZFI66R3NVVJLkSGALMkuWWXvfYZeU//AworJbyRoaIK/CVt5OP23i5N4tdd5UXc5dfLuYqnKRynyMmCkz9c3yEIQMXoPhG2hx7l7L2BeMJvcKmVhkSN7nQayjnrbUXGm/IRqrb88lvkyBevN5E3IB2V5IKEieIPZjbD/N0IfcnAt89Z96tgDhtIbx3ZvXm92lsvHA8buqQpG9d2AmSi6GKs64lQcnGeM5o0wER2JHWl1OSa1Nr/UAo5Xb/PM65Yt3yZE8AuMKHBmbfDSBzdkTXx58AeDzFUd3kMXD/fFjeQQWyXFlOss3ygH9SObl827Txmz9OJqZaxabs5Q3AP6m3EjKjz7zfLfrgpcxJM2WBiU1bN0ZxUgZkImy/CHk5gCZ7vhcnaLiDO4HZnzY/aRJwKYQPE5i0O2nHpIfovqkc0DFBA7U/7Cjin7e1E0UZvF3meLOxMqkfc6X7QTxqQpt2Tej6jlpdxw4CTLwGUhGkAw9IAPkUB3L0EbZ1/ksGhNvGDvUeSTq8hYdMAPmA+k9jS6653V4SQ+qBMy5++tbr5AeZQI="
|
||||
|
||||
script:
|
||||
#- ./e2e/e2e.sh # Hold until we can resolve the /var/tmp issue - talk to beamer/niko
|
||||
- make
|
||||
|
||||
deploy:
|
||||
provider: script
|
||||
skip_cleanup: true
|
||||
@@ -17,4 +42,4 @@ deploy:
|
||||
file: bin/ytsync
|
||||
on:
|
||||
repo: lbryio/ytsync
|
||||
tags: true
|
||||
tags: true
|
||||
@@ -0,0 +1,111 @@
|
||||
From 30380338ba9af01696c94b61f0597131638eaec1 Mon Sep 17 00:00:00 2001
|
||||
From: Niko Storni <niko@lbry.io>
|
||||
Date: Mon, 16 Dec 2019 00:13:36 +0100
|
||||
Subject: [PATCH] lbry-patch
|
||||
|
||||
---
|
||||
youtube_dl/extractor/youtube.py | 45 +++++++++++++++++++++++++--------
|
||||
1 file changed, 35 insertions(+), 10 deletions(-)
|
||||
|
||||
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
|
||||
index b913d07a6..cd66a5b01 100644
|
||||
--- a/youtube_dl/extractor/youtube.py
|
||||
+++ b/youtube_dl/extractor/youtube.py
|
||||
@@ -10,6 +10,7 @@ import random
|
||||
import re
|
||||
import time
|
||||
import traceback
|
||||
+import subprocess
|
||||
|
||||
from .common import InfoExtractor, SearchInfoExtractor
|
||||
from ..jsinterp import JSInterpreter
|
||||
@@ -536,6 +537,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
|
||||
_GEO_BYPASS = False
|
||||
|
||||
+ _WGET_429_RATE_LIMIT = 8191
|
||||
+ _WGET_BINARY = "wget"
|
||||
+
|
||||
IE_NAME = 'youtube'
|
||||
_TESTS = [
|
||||
{
|
||||
@@ -1254,6 +1258,17 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
""" Return a string representation of a signature """
|
||||
return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
|
||||
|
||||
+ def _rate_limit_download(self, url, video_id, note=None):
|
||||
+ if note is None:
|
||||
+ self.report_download_webpage(video_id)
|
||||
+ elif note is not False:
|
||||
+ if video_id is None:
|
||||
+ self.to_screen('%s' % (note,))
|
||||
+ else:
|
||||
+ self.to_screen('%s: %s' % (video_id, note))
|
||||
+ source_address = self._downloader.params.get('source_address')
|
||||
+ return subprocess.run([self._WGET_BINARY, '-q', '--limit-rate', str(self._WGET_429_RATE_LIMIT), '--bind-address', source_address, '-O', '-', url], check=True, stdout=subprocess.PIPE).stdout.decode(encoding='UTF-8')
|
||||
+
|
||||
def _extract_signature_function(self, video_id, player_url, example_sig):
|
||||
id_m = re.match(
|
||||
r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player(?:-new)?|(?:/[a-z]{2,3}_[A-Z]{2})?/base)?\.(?P<ext>[a-z]+)$',
|
||||
@@ -1678,7 +1693,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
|
||||
# Get video webpage
|
||||
url = proto + '://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1&bpctr=9999999999' % video_id
|
||||
- video_webpage = self._download_webpage(url, video_id)
|
||||
+ video_webpage = self._rate_limit_download(url, video_id)
|
||||
|
||||
# Attempt to extract SWF player URL
|
||||
mobj = re.search(r'swfConfig.*?"(https?:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
|
||||
@@ -1736,10 +1751,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
r'"sts"\s*:\s*(\d+)', embed_webpage, 'sts', default=''),
|
||||
})
|
||||
video_info_url = proto + '://www.youtube.com/get_video_info?' + data
|
||||
- video_info_webpage = self._download_webpage(
|
||||
+ video_info_webpage = self._rate_limit_download(
|
||||
video_info_url, video_id,
|
||||
- note='Refetching age-gated info webpage',
|
||||
- errnote='unable to download video info webpage')
|
||||
+ note='Refetching age-gated info webpage')
|
||||
video_info = compat_parse_qs(video_info_webpage)
|
||||
pl_response = video_info.get('player_response', [None])[0]
|
||||
player_response = extract_player_response(pl_response, video_id)
|
||||
@@ -1777,7 +1791,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
# The general idea is to take a union of itags of both DASH manifests (for example
|
||||
# video with such 'manifest behavior' see https://github.com/ytdl-org/youtube-dl/issues/6093)
|
||||
self.report_video_info_webpage_download(video_id)
|
||||
- for el in ('embedded', 'detailpage', 'vevo', ''):
|
||||
+ for el in ('', 'embedded', 'detailpage', 'vevo'):
|
||||
query = {
|
||||
'video_id': video_id,
|
||||
'ps': 'default',
|
||||
@@ -1789,11 +1803,22 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
query['el'] = el
|
||||
if sts:
|
||||
query['sts'] = sts
|
||||
- video_info_webpage = self._download_webpage(
|
||||
- '%s://www.youtube.com/get_video_info' % proto,
|
||||
- video_id, note=False,
|
||||
- errnote='unable to download video info webpage',
|
||||
- fatal=False, query=query)
|
||||
+
|
||||
+ if el == '':
|
||||
+ base_url = 'https://youtube.com/get_video_info?video_id={}'.format(video_id)
|
||||
+ else:
|
||||
+ base_url = 'https://youtube.com/get_video_info'
|
||||
+
|
||||
+ for q in query:
|
||||
+ if q is None or q is "":
|
||||
+ continue
|
||||
+ if query[q] is None or query[q] is "":
|
||||
+ continue
|
||||
+
|
||||
+ base_url = base_url + "?{}={}".format(q, query[q])
|
||||
+
|
||||
+ video_info_webpage = self._rate_limit_download(base_url, video_id)
|
||||
+
|
||||
if not video_info_webpage:
|
||||
continue
|
||||
get_video_info = compat_parse_qs(video_info_webpage)
|
||||
--
|
||||
2.17.1
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2017-2020 LBRY Inc.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the
|
||||
"Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish,
|
||||
distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the
|
||||
following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
@@ -13,7 +13,7 @@ LDFLAGS = -ldflags "-X ${IMPORT_PATH}/meta.Version=${VERSION} -X ${IMPORT_PATH}/
|
||||
|
||||
|
||||
build:
|
||||
mkdir -p ${BIN_DIR} && CGO_ENABLED=0 go build ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/${BINARY} main.go
|
||||
mkdir -p ${BIN_DIR} && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/${BINARY} main.go
|
||||
|
||||
clean:
|
||||
if [ -f ${BIN_DIR}/${BINARY} ]; then rm ${BIN_DIR}/${BINARY}; fi
|
||||
|
||||
+25
-78
@@ -2,26 +2,29 @@ package blobs_reflector
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
"github.com/lbryio/reflector.go/cmd"
|
||||
"github.com/lbryio/reflector.go/db"
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
"github.com/mitchellh/go-ps"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/user"
|
||||
"path/filepath"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/reflector.go/cmd"
|
||||
"github.com/lbryio/reflector.go/db"
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/lbryio/ytsync/v5/util"
|
||||
)
|
||||
|
||||
var dbHandle *db.SQL
|
||||
|
||||
func ReflectAndClean() error {
|
||||
err := reflectBlobs()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return cleanupLbrynet()
|
||||
return util.CleanupLbrynet()
|
||||
}
|
||||
|
||||
func loadConfig(path string) (cmd.Config, error) {
|
||||
@@ -40,16 +43,20 @@ func loadConfig(path string) (cmd.Config, error) {
|
||||
}
|
||||
|
||||
func reflectBlobs() error {
|
||||
if util.IsBlobReflectionOff() {
|
||||
return nil
|
||||
}
|
||||
logrus.Infoln("reflecting blobs...")
|
||||
//make sure lbrynet is off
|
||||
running, err := isLbrynetRunning()
|
||||
running, err := util.IsLbrynetRunning()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if running {
|
||||
return errors.Prefix("cannot reflect blobs as the daemon is running", err)
|
||||
}
|
||||
|
||||
dbHandle := new(db.SQL)
|
||||
logrus.SetLevel(logrus.InfoLevel)
|
||||
defer logrus.SetLevel(logrus.DebugLevel)
|
||||
ex, err := os.Executable()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
@@ -59,23 +66,17 @@ func reflectBlobs() error {
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
err = dbHandle.Connect(config.DBConn)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer func() {
|
||||
err := dbHandle.CloseDB()
|
||||
if dbHandle == nil {
|
||||
dbHandle = new(db.SQL)
|
||||
err = dbHandle.Connect(config.DBConn)
|
||||
if err != nil {
|
||||
log.Errorf("failed to close db handle: %s", err.Error())
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
}()
|
||||
st := store.NewDBBackedS3Store(
|
||||
store.NewS3BlobStore(config.AwsID, config.AwsSecret, config.BucketRegion, config.BucketName),
|
||||
dbHandle)
|
||||
}
|
||||
st := store.NewDBBackedStore(store.NewS3Store(config.AwsID, config.AwsSecret, config.BucketRegion, config.BucketName), dbHandle, false)
|
||||
|
||||
uploadWorkers := 10
|
||||
uploader := reflector.NewUploader(dbHandle, st, uploadWorkers, false)
|
||||
uploader := reflector.NewUploader(dbHandle, st, uploadWorkers, false, false)
|
||||
usr, err := user.Current()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
@@ -90,57 +91,3 @@ func reflectBlobs() error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func cleanupLbrynet() error {
|
||||
//make sure lbrynet is off
|
||||
running, err := isLbrynetRunning()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if running {
|
||||
return errors.Prefix("cannot cleanup lbrynet as the daemon is running", err)
|
||||
}
|
||||
usr, err := user.Current()
|
||||
if err != nil {
|
||||
log.Errorln(err.Error())
|
||||
return errors.Err(err)
|
||||
}
|
||||
lbrynetDir := usr.HomeDir + "/.lbrynet/"
|
||||
files, err := filepath.Glob(lbrynetDir + "lbrynet.sqlite*")
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
for _, f := range files {
|
||||
err = os.Remove(f)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
}
|
||||
blobsDir := lbrynetDir + "/blobfiles/"
|
||||
err = os.RemoveAll(blobsDir)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
err = os.Mkdir(blobsDir, 0755)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isLbrynetRunning() (bool, error) {
|
||||
processes, err := ps.Processes()
|
||||
if err != nil {
|
||||
return true, errors.Err(err)
|
||||
}
|
||||
var daemonProcessId = -1
|
||||
for _, p := range processes {
|
||||
if p.Executable() == "lbrynet" {
|
||||
daemonProcessId = p.Pid()
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
running := daemonProcessId != -1
|
||||
return running, nil
|
||||
}
|
||||
@@ -0,0 +1,391 @@
|
||||
package downloader
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/lbryio/ytsync/v5/downloader/ytdl"
|
||||
"github.com/lbryio/ytsync/v5/ip_manager"
|
||||
"github.com/lbryio/ytsync/v5/sdk"
|
||||
util2 "github.com/lbryio/ytsync/v5/util"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
|
||||
args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName + "/videos", "--get-id", "--flat-playlist", "--cookies", "cookies.txt"}
|
||||
ids, err := run(channelName, args, stopChan, pool, true)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
videoIDs := make([]string, 0, maxVideos)
|
||||
for i, v := range ids {
|
||||
logrus.Debugf("%d - video id %s", i, v)
|
||||
if i >= maxVideos {
|
||||
break
|
||||
}
|
||||
videoIDs = append(videoIDs, v)
|
||||
}
|
||||
return videoIDs, nil
|
||||
}
|
||||
|
||||
const releaseTimeFormat = "2006-01-02, 15:04:05 (MST)"
|
||||
|
||||
func GetVideoInformation(config *sdk.APIConfig, videoID string, stopChan stop.Chan, ip *net.TCPAddr, pool *ip_manager.IPPool) (*ytdl.YtdlVideo, error) {
|
||||
args := []string{
|
||||
"--skip-download",
|
||||
"--write-info-json",
|
||||
fmt.Sprintf("https://www.youtube.com/watch?v=%s", videoID),
|
||||
"--cookies",
|
||||
"cookies.txt",
|
||||
"-o",
|
||||
path.Join(util2.GetVideoMetadataDir(), videoID),
|
||||
}
|
||||
_, err := run(videoID, args, stopChan, pool, false)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
f, err := os.Open(path.Join(util2.GetVideoMetadataDir(), videoID+".info.json"))
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
// defer the closing of our jsonFile so that we can parse it later on
|
||||
defer f.Close()
|
||||
// read our opened jsonFile as a byte array.
|
||||
byteValue, _ := ioutil.ReadAll(f)
|
||||
|
||||
var video *ytdl.YtdlVideo
|
||||
err = json.Unmarshal(byteValue, &video)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
// now get an accurate time
|
||||
const maxTries = 5
|
||||
tries := 0
|
||||
GetTime:
|
||||
tries++
|
||||
t, err := getUploadTime(config, videoID, ip, video.UploadDate)
|
||||
if err != nil {
|
||||
//slack(":warning: Upload time error: %v", err)
|
||||
if tries <= maxTries && (errors.Is(err, errNotScraped) || errors.Is(err, errUploadTimeEmpty) || errors.Is(err, errStatusParse) || errors.Is(err, errConnectionIssue)) {
|
||||
err := triggerScrape(videoID, ip)
|
||||
if err == nil {
|
||||
time.Sleep(2 * time.Second) // let them scrape it
|
||||
goto GetTime
|
||||
} else {
|
||||
//slack("triggering scrape returned error: %v", err)
|
||||
}
|
||||
} else if !errors.Is(err, errNotScraped) && !errors.Is(err, errUploadTimeEmpty) {
|
||||
//slack(":warning: Error while trying to get accurate upload time for %s: %v", videoID, err)
|
||||
if t == "" {
|
||||
return nil, errors.Err(err)
|
||||
} else {
|
||||
t = "" //TODO: get rid of the other piece below?
|
||||
}
|
||||
}
|
||||
// do fallback below
|
||||
}
|
||||
//slack("After all that, upload time for %s is %s", videoID, t)
|
||||
|
||||
if t != "" {
|
||||
parsed, err := time.Parse("2006-01-02, 15:04:05 (MST)", t) // this will probably be UTC, but Go's timezone parsing is fucked up. it ignores the timezone in the date
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
//slack(":exclamation: Got an accurate time for %s", videoID)
|
||||
video.UploadDateForReal = parsed
|
||||
} else { //TODO: this is the piece that isn't needed!
|
||||
slack(":warning: Could not get accurate time for %s. Falling back to time from upload ytdl: %s.", videoID, video.UploadDate)
|
||||
// fall back to UploadDate from youtube-dl
|
||||
video.UploadDateForReal, err = time.Parse("20060102", video.UploadDate)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return video, nil
|
||||
}
|
||||
|
||||
var errNotScraped = errors.Base("not yet scraped by caa.iti.gr")
|
||||
var errUploadTimeEmpty = errors.Base("upload time is empty")
|
||||
var errStatusParse = errors.Base("could not parse status, got number, need string")
|
||||
var errConnectionIssue = errors.Base("there was a connection issue with the api")
|
||||
|
||||
func slack(format string, a ...interface{}) {
|
||||
fmt.Printf(format+"\n", a...)
|
||||
util.SendToSlack(format, a...)
|
||||
}
|
||||
|
||||
func triggerScrape(videoID string, ip *net.TCPAddr) error {
|
||||
//slack("Triggering scrape for %s", videoID)
|
||||
u, err := url.Parse("https://caa.iti.gr/verify_videoV3")
|
||||
q := u.Query()
|
||||
q.Set("twtimeline", "0")
|
||||
q.Set("url", "https://www.youtube.com/watch?v="+videoID)
|
||||
u.RawQuery = q.Encode()
|
||||
//slack("GET %s", u.String())
|
||||
|
||||
client := getClient(ip)
|
||||
req, err := http.NewRequest(http.MethodGet, u.String(), nil)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
req.Header.Set("User-Agent", ChromeUA)
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
var response struct {
|
||||
Message string `json:"message"`
|
||||
Status string `json:"status"`
|
||||
VideoURL string `json:"video_url"`
|
||||
}
|
||||
err = json.NewDecoder(res.Body).Decode(&response)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "cannot unmarshal number") {
|
||||
return errors.Err(errStatusParse)
|
||||
}
|
||||
if strings.Contains(err.Error(), "no route to host") {
|
||||
return errors.Err(errConnectionIssue)
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
switch response.Status {
|
||||
case "removed_video":
|
||||
return errors.Err("video previously removed from service")
|
||||
case "no_video":
|
||||
return errors.Err("they say 'video cannot be found'. wtf?")
|
||||
default:
|
||||
spew.Dump(response)
|
||||
}
|
||||
|
||||
return nil
|
||||
//https://caa.iti.gr/caa/api/v4/videos/reports/h-tuxHS5lSM
|
||||
}
|
||||
|
||||
func getUploadTime(config *sdk.APIConfig, videoID string, ip *net.TCPAddr, uploadDate string) (string, error) {
|
||||
//slack("Getting upload time for %s", videoID)
|
||||
release, err := config.GetReleasedDate(videoID)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
ytdlUploadDate, err := time.Parse("20060102", uploadDate)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
if release != nil {
|
||||
//const sqlTimeFormat = "2006-01-02 15:04:05"
|
||||
sqlTime, err := time.ParseInLocation(time.RFC3339, release.ReleaseTime, time.UTC)
|
||||
if err == nil {
|
||||
hoursDiff := math.Abs(sqlTime.Sub(ytdlUploadDate).Hours())
|
||||
if hoursDiff > 48 {
|
||||
logrus.Infof("upload day from APIs differs from the ytdl one by more than 2 days.")
|
||||
} else {
|
||||
return sqlTime.Format(releaseTimeFormat), nil
|
||||
}
|
||||
} else {
|
||||
logrus.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
if time.Now().AddDate(0, 0, -3).After(ytdlUploadDate) {
|
||||
return ytdlUploadDate.Format(releaseTimeFormat), nil
|
||||
}
|
||||
client := getClient(ip)
|
||||
req, err := http.NewRequest(http.MethodGet, "https://caa.iti.gr/get_verificationV3?url=https://www.youtube.com/watch?v="+videoID, nil)
|
||||
if err != nil {
|
||||
return ytdlUploadDate.Format(releaseTimeFormat), errors.Err(err)
|
||||
}
|
||||
req.Header.Set("User-Agent", ChromeUA)
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return ytdlUploadDate.Format(releaseTimeFormat), errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
var uploadTime struct {
|
||||
Time string `json:"video_upload_time"`
|
||||
Message string `json:"message"`
|
||||
Status string `json:"status"`
|
||||
}
|
||||
err = json.NewDecoder(res.Body).Decode(&uploadTime)
|
||||
if err != nil {
|
||||
return ytdlUploadDate.Format(releaseTimeFormat), errors.Err(err)
|
||||
}
|
||||
|
||||
if uploadTime.Status == "ERROR1" {
|
||||
return ytdlUploadDate.Format(releaseTimeFormat), errNotScraped
|
||||
}
|
||||
|
||||
if uploadTime.Status == "" && strings.HasPrefix(uploadTime.Message, "CANNOT_RETRIEVE_REPORT_FOR_VIDEO_") {
|
||||
return ytdlUploadDate.Format(releaseTimeFormat), errors.Err("cannot retrieve report for video")
|
||||
}
|
||||
|
||||
if uploadTime.Time == "" {
|
||||
return ytdlUploadDate.Format(releaseTimeFormat), errUploadTimeEmpty
|
||||
}
|
||||
|
||||
return uploadTime.Time, nil
|
||||
}
|
||||
|
||||
func getClient(ip *net.TCPAddr) *http.Client {
|
||||
if ip == nil {
|
||||
return http.DefaultClient
|
||||
}
|
||||
|
||||
return &http.Client{
|
||||
Transport: &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
DialContext: (&net.Dialer{
|
||||
LocalAddr: ip,
|
||||
Timeout: 30 * time.Second,
|
||||
KeepAlive: 30 * time.Second,
|
||||
}).DialContext,
|
||||
MaxIdleConns: 100,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
TLSHandshakeTimeout: 10 * time.Second,
|
||||
ExpectContinueTimeout: 1 * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
GoogleBotUA = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
|
||||
ChromeUA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36"
|
||||
maxAttempts = 3
|
||||
extractionError = "YouTube said: Unable to extract video data"
|
||||
throttledError = "HTTP Error 429"
|
||||
AlternateThrottledError = "returned non-zero exit status 8"
|
||||
youtubeDlError = "exit status 1"
|
||||
)
|
||||
|
||||
func run(use string, args []string, stopChan stop.Chan, pool *ip_manager.IPPool, dlc bool) ([]string, error) {
|
||||
var useragent []string
|
||||
var lastError error
|
||||
for attempts := 0; attempts < maxAttempts; attempts++ {
|
||||
sourceAddress, err := getIPFromPool(use, stopChan, pool)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
argsForCommand := append(args, "--source-address", sourceAddress)
|
||||
argsForCommand = append(argsForCommand, useragent...)
|
||||
binary := "yt-dlp"
|
||||
if dlc {
|
||||
binary = "yt-dlp"
|
||||
}
|
||||
cmd := exec.Command(binary, argsForCommand...)
|
||||
|
||||
res, err := runCmd(cmd, stopChan)
|
||||
pool.ReleaseIP(sourceAddress)
|
||||
if err == nil {
|
||||
return res, nil
|
||||
}
|
||||
lastError = err
|
||||
if strings.Contains(err.Error(), youtubeDlError) {
|
||||
if strings.Contains(err.Error(), extractionError) {
|
||||
logrus.Warnf("known extraction error: %s", errors.FullTrace(err))
|
||||
useragent = nextUA(useragent)
|
||||
}
|
||||
if strings.Contains(err.Error(), throttledError) || strings.Contains(err.Error(), AlternateThrottledError) {
|
||||
pool.SetThrottled(sourceAddress)
|
||||
//we don't want throttle errors to count toward the max retries
|
||||
attempts--
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil, lastError
|
||||
}
|
||||
|
||||
func nextUA(current []string) []string {
|
||||
if len(current) == 0 {
|
||||
return []string{"--user-agent", GoogleBotUA}
|
||||
}
|
||||
return []string{"--user-agent", ChromeUA}
|
||||
}
|
||||
|
||||
func runCmd(cmd *exec.Cmd, stopChan stop.Chan) ([]string, error) {
|
||||
logrus.Infof("running yt-dlp cmd: %s", strings.Join(cmd.Args, " "))
|
||||
var err error
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
err = cmd.Start()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
outLog, err := ioutil.ReadAll(stdout)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
errorLog, err := ioutil.ReadAll(stderr)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- cmd.Wait()
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-stopChan:
|
||||
err := cmd.Process.Kill()
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("failed to kill command after stopper cancellation", err)
|
||||
}
|
||||
return nil, errors.Err("interrupted by user")
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("yt-dlp "+strings.Join(cmd.Args, " ")+" ["+string(errorLog)+"]", err)
|
||||
}
|
||||
return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil
|
||||
}
|
||||
}
|
||||
|
||||
func getIPFromPool(use string, stopChan stop.Chan, pool *ip_manager.IPPool) (sourceAddress string, err error) {
|
||||
for {
|
||||
sourceAddress, err = pool.GetIP(use)
|
||||
if err != nil {
|
||||
if errors.Is(err, ip_manager.ErrAllThrottled) {
|
||||
select {
|
||||
case <-stopChan:
|
||||
return "", errors.Err("interrupted by user")
|
||||
|
||||
default:
|
||||
time.Sleep(ip_manager.IPCooldownPeriod)
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package downloader
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/lbryio/ytsync/v5/sdk"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestGetPlaylistVideoIDs(t *testing.T) {
|
||||
videoIDs, err := GetPlaylistVideoIDs("UCJ0-OtVpF0wOKEqT2Z1HEtA", 50, nil, nil)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
for _, id := range videoIDs {
|
||||
println(id)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetVideoInformation(t *testing.T) {
|
||||
video, err := GetVideoInformation(nil, "zj7pXM9gE5M", nil, nil, nil)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
if video != nil {
|
||||
logrus.Info(video.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func Test_getUploadTime(t *testing.T) {
|
||||
configs := sdk.APIConfig{
|
||||
YoutubeAPIKey: "",
|
||||
ApiURL: "https://api.lbry.com",
|
||||
ApiToken: "Ht4NETrL5oWKyAaZkuSV68BKhtXkiLh5",
|
||||
HostName: "test",
|
||||
}
|
||||
got, err := getUploadTime(&configs, "kDGOHNpRjzc", nil, "20060102")
|
||||
assert.NoError(t, err)
|
||||
t.Log(got)
|
||||
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package ytdl
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
type YtdlVideo struct {
|
||||
UploadDate string `json:"upload_date"`
|
||||
UploadDateForReal time.Time // you need to manually set this since the value in the API doesn't include the time
|
||||
Extractor string `json:"extractor"`
|
||||
Series interface{} `json:"series"`
|
||||
Format string `json:"format"`
|
||||
Vbr interface{} `json:"vbr"`
|
||||
Chapters interface{} `json:"chapters"`
|
||||
Height int `json:"height"`
|
||||
LikeCount interface{} `json:"like_count"`
|
||||
Duration int `json:"duration"`
|
||||
Fulltitle string `json:"fulltitle"`
|
||||
PlaylistIndex interface{} `json:"playlist_index"`
|
||||
Album interface{} `json:"album"`
|
||||
ViewCount int `json:"view_count"`
|
||||
Playlist interface{} `json:"playlist"`
|
||||
Title string `json:"title"`
|
||||
Filename string `json:"_filename"`
|
||||
Creator interface{} `json:"creator"`
|
||||
Ext string `json:"ext"`
|
||||
ID string `json:"id"`
|
||||
DislikeCount interface{} `json:"dislike_count"`
|
||||
AverageRating float64 `json:"average_rating"`
|
||||
Abr float64 `json:"abr"`
|
||||
UploaderURL string `json:"uploader_url"`
|
||||
Categories []string `json:"categories"`
|
||||
Fps float64 `json:"fps"`
|
||||
StretchedRatio interface{} `json:"stretched_ratio"`
|
||||
SeasonNumber interface{} `json:"season_number"`
|
||||
Annotations interface{} `json:"annotations"`
|
||||
WebpageURLBasename string `json:"webpage_url_basename"`
|
||||
Acodec string `json:"acodec"`
|
||||
DisplayID string `json:"display_id"`
|
||||
//RequestedFormats []RequestedFormat `json:"requested_formats"`
|
||||
//AutomaticCaptions struct{} `json:"automatic_captions"`
|
||||
Description string `json:"description"`
|
||||
Tags []string `json:"tags"`
|
||||
Track interface{} `json:"track"`
|
||||
RequestedSubtitles interface{} `json:"requested_subtitles"`
|
||||
StartTime interface{} `json:"start_time"`
|
||||
Uploader string `json:"uploader"`
|
||||
ExtractorKey string `json:"extractor_key"`
|
||||
FormatID string `json:"format_id"`
|
||||
EpisodeNumber interface{} `json:"episode_number"`
|
||||
UploaderID string `json:"uploader_id"`
|
||||
//Subtitles struct{} `json:"subtitles"`
|
||||
ReleaseYear interface{} `json:"release_year"`
|
||||
Thumbnails []Thumbnail `json:"thumbnails"`
|
||||
License interface{} `json:"license"`
|
||||
Artist interface{} `json:"artist"`
|
||||
AgeLimit int `json:"age_limit"`
|
||||
ReleaseDate interface{} `json:"release_date"`
|
||||
AltTitle interface{} `json:"alt_title"`
|
||||
Thumbnail string `json:"thumbnail"`
|
||||
ChannelID string `json:"channel_id"`
|
||||
IsLive interface{} `json:"is_live"`
|
||||
Width int `json:"width"`
|
||||
EndTime interface{} `json:"end_time"`
|
||||
WebpageURL string `json:"webpage_url"`
|
||||
Formats []Format `json:"formats"`
|
||||
ChannelURL string `json:"channel_url"`
|
||||
Resolution interface{} `json:"resolution"`
|
||||
Vcodec string `json:"vcodec"`
|
||||
}
|
||||
|
||||
type RequestedFormat struct {
|
||||
Asr interface{} `json:"asr"`
|
||||
Tbr float64 `json:"tbr"`
|
||||
Container string `json:"container"`
|
||||
Language interface{} `json:"language"`
|
||||
Format string `json:"format"`
|
||||
URL string `json:"url"`
|
||||
Vcodec string `json:"vcodec"`
|
||||
FormatNote string `json:"format_note"`
|
||||
Height int `json:"height"`
|
||||
Width int `json:"width"`
|
||||
Ext string `json:"ext"`
|
||||
FragmentBaseURL string `json:"fragment_base_url"`
|
||||
Filesize interface{} `json:"filesize"`
|
||||
Fps float64 `json:"fps"`
|
||||
ManifestURL string `json:"manifest_url"`
|
||||
Protocol string `json:"protocol"`
|
||||
FormatID string `json:"format_id"`
|
||||
HTTPHeaders struct {
|
||||
AcceptCharset string `json:"Accept-Charset"`
|
||||
AcceptLanguage string `json:"Accept-Language"`
|
||||
AcceptEncoding string `json:"Accept-Encoding"`
|
||||
Accept string `json:"Accept"`
|
||||
UserAgent string `json:"User-Agent"`
|
||||
} `json:"http_headers"`
|
||||
Fragments []struct {
|
||||
Path string `json:"path"`
|
||||
Duration float64 `json:"duration,omitempty"`
|
||||
} `json:"fragments"`
|
||||
Acodec string `json:"acodec"`
|
||||
Abr int `json:"abr,omitempty"`
|
||||
}
|
||||
|
||||
type Format struct {
|
||||
Asr int `json:"asr"`
|
||||
Filesize int `json:"filesize"`
|
||||
FormatID string `json:"format_id"`
|
||||
FormatNote string `json:"format_note"`
|
||||
Fps interface{} `json:"fps"`
|
||||
Height interface{} `json:"height"`
|
||||
Quality int `json:"quality"`
|
||||
Tbr float64 `json:"tbr"`
|
||||
URL string `json:"url"`
|
||||
Width interface{} `json:"width"`
|
||||
Ext string `json:"ext"`
|
||||
Vcodec string `json:"vcodec"`
|
||||
Acodec string `json:"acodec"`
|
||||
Abr float64 `json:"abr,omitempty"`
|
||||
DownloaderOptions struct {
|
||||
HTTPChunkSize int `json:"http_chunk_size"`
|
||||
} `json:"downloader_options,omitempty"`
|
||||
Container string `json:"container,omitempty"`
|
||||
Format string `json:"format"`
|
||||
Protocol string `json:"protocol"`
|
||||
HTTPHeaders struct {
|
||||
UserAgent string `json:"User-Agent"`
|
||||
AcceptCharset string `json:"Accept-Charset"`
|
||||
Accept string `json:"Accept"`
|
||||
AcceptEncoding string `json:"Accept-Encoding"`
|
||||
AcceptLanguage string `json:"Accept-Language"`
|
||||
} `json:"http_headers"`
|
||||
Vbr float64 `json:"vbr,omitempty"`
|
||||
}
|
||||
|
||||
type Thumbnail struct {
|
||||
URL string `json:"url"`
|
||||
Width int `json:"width"`
|
||||
Resolution string `json:"resolution"`
|
||||
ID string `json:"id"`
|
||||
Height int `json:"height"`
|
||||
}
|
||||
|
||||
type HTTPHeaders struct {
|
||||
AcceptCharset string `json:"Accept-Charset"`
|
||||
AcceptLanguage string `json:"Accept-Language"`
|
||||
AcceptEncoding string `json:"Accept-Encoding"`
|
||||
Accept string `json:"Accept"`
|
||||
UserAgent string `json:"User-Agent"`
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
version: '3.4'
|
||||
|
||||
services:
|
||||
###########
|
||||
## MYSQL ##
|
||||
###########
|
||||
mysql:
|
||||
image: mysql:5.7.23
|
||||
restart: "no"
|
||||
ports:
|
||||
- 3306:3306
|
||||
volumes:
|
||||
- "../persist/chainquery/db:/var/lib/mysql"
|
||||
## This one may need to be tweaked based on where you run this docker-compose from.
|
||||
- "../stuff/my.cnf:/etc/mysql/conf.d/chainquery-optimizations.cnf"
|
||||
################
|
||||
## Chainquery ##
|
||||
################
|
||||
chainquery:
|
||||
image: lbry/chainquery:v1.8.1
|
||||
restart: "no"
|
||||
ports:
|
||||
- 6300:6300
|
||||
depends_on:
|
||||
- mysql
|
||||
## TODO: Uncomment this in a docker-compose.override.yml to allow for external configurations.
|
||||
volumes:
|
||||
- "../persist/chainquery/config/chainqueryconfig.toml:/etc/chainquery/chainqueryconfig.toml"
|
||||
@@ -0,0 +1,33 @@
|
||||
## Get the latest source and extract it for the app container.
|
||||
## Design choices, two RUN layers intended to keep builds faster, the zipped
|
||||
FROM ubuntu:18.04 as prep
|
||||
LABEL MAINTAINER="leopere [at] nixc [dot] us"
|
||||
RUN apt-get update && \
|
||||
apt-get -y install unzip curl telnet wait-for-it && \
|
||||
apt-get autoclean -y && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
WORKDIR /
|
||||
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
|
||||
COPY ./start.sh start
|
||||
COPY ./healthcheck.sh healthcheck
|
||||
ARG VERSION="master"
|
||||
RUN curl -s -o /chainquery http://build.lbry.io/chainquery/branch-"${VERSION}"/chainquery && \
|
||||
chmod +x /chainquery
|
||||
|
||||
|
||||
FROM ubuntu:18.04 as app
|
||||
RUN apt-get update && \
|
||||
apt-get -y install telnet wait-for-it && \
|
||||
apt-get autoclean -y && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
ARG VERSION="master"
|
||||
ADD https://raw.githubusercontent.com/lbryio/chainquery/"${VERSION}"/config/default/chainqueryconfig.toml /etc/lbry/chainqueryconfig.toml.orig
|
||||
RUN adduser chainquery --gecos GECOS --shell /bin/bash --disabled-password --home /home/chainquery && \
|
||||
chown -R chainquery:chainquery /etc/lbry
|
||||
COPY --from=prep ./healthcheck /chainquery /start /usr/bin/
|
||||
HEALTHCHECK --interval=1m --timeout=30s \
|
||||
CMD healthcheck
|
||||
EXPOSE 6300
|
||||
USER chainquery
|
||||
STOPSIGNAL SIGINT
|
||||
CMD ["start"]
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/bin/bash
|
||||
if [ $# -eq 0 ]
|
||||
then
|
||||
echo "No docker tag argument supplied. Use './build.sh <tag>'"
|
||||
exit 1
|
||||
fi
|
||||
docker build --no-cache --build-arg VERSION=$1 --tag lbry/chainquery:$1 .
|
||||
docker push lbry/chainquery:$1
|
||||
Executable
+2
@@ -0,0 +1,2 @@
|
||||
#!/usr/bin/env bash
|
||||
curl --fail http://localhost:6300/api/status || exit 1
|
||||
@@ -0,0 +1,9 @@
|
||||
# Default Homebrew MySQL server config
|
||||
[mysqld]
|
||||
# Only allow connections from localhost
|
||||
innodb_log_file_size=5G
|
||||
key_buffer_size=1G
|
||||
innodb_flush_log_at_trx_commit = 0
|
||||
innodb_autoinc_lock_mode=2
|
||||
innodb_buffer_pool_size=1G
|
||||
innodb_log_buffer_size=1G
|
||||
Executable
+51
@@ -0,0 +1,51 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
## Config setup
|
||||
|
||||
## Setup Values
|
||||
DEBUGMODE=$(echo "debugmode=$DEBUGMODE")
|
||||
LBRYCRDURL=$(echo "lbrycrdurl=\"rpc://$RPC_USER:$RPC_PASSWORD@10.5.1.2:9245\"")
|
||||
MYSQLDSN=$(echo "mysqldsn=\"$MYSQL_USER:$MYSQL_PASSWORD@tcp($MYSQL_SERVER:3306)/$MYSQL_DATABASE\"")
|
||||
APIMYSQLDSN=$(echo "apimysqldsn=\"$MYSQL_USER:$MYSQL_PASSWORD@tcp($MYSQL_SERVER:3306)/$MYSQL_DATABASE\"")
|
||||
|
||||
## Setup Defaults
|
||||
DEBUGMODE_DEFAULT='#DEFAULT-debugmode=false'
|
||||
LBRYCRDURL_DEFAULT='#DEFAULT-lbrycrdurl="rpc://lbry:lbry@localhost:9245"'
|
||||
MYSQLDSN_DEFAULT='#DEFAULT-mysqldsn="lbry:lbry@tcp(localhost:3306)/chainquery"'
|
||||
APIMYSQLDSN_DEFAULT='#DEFAULT-apimysqldsn="lbry:lbry@tcp(localhost:3306)/chainquery"'
|
||||
|
||||
## Add setup value variable name to this list to get processed on container start
|
||||
CONFIG_SETTINGS=(
|
||||
DEBUGMODE
|
||||
LBRYCRDURL
|
||||
MYSQLDSN
|
||||
APIMYSQLDSN
|
||||
)
|
||||
|
||||
function set_configs() {
|
||||
## Set configs on container start if not already set.
|
||||
for i in "${!CONFIG_SETTINGS[@]}"; do
|
||||
## Indirect references http://tldp.org/LDP/abs/html/ivr.html
|
||||
eval FROM_STRING=\$"${CONFIG_SETTINGS[$i]}_DEFAULT"
|
||||
eval TO_STRING=\$${CONFIG_SETTINGS[$i]}
|
||||
## TODO: Add a bit more magic to make sure that you're only configuring things if not set by config mounts.
|
||||
sed -i "s~$FROM_STRING~"$TO_STRING"~g" /etc/lbry/chainqueryconfig.toml
|
||||
done
|
||||
echo "Reading config for debugging."
|
||||
cat /etc/lbry/chainqueryconfig.toml
|
||||
}
|
||||
|
||||
if [[ ! -f /etc/lbry/chainqueryconfig.toml ]]; then
|
||||
echo "[INFO]: Did not find chainqueryconfig.toml"
|
||||
echo " Installing default and configuring with provided environment variables if any."
|
||||
## Install fresh copy of config file.
|
||||
echo "cp -v /etc/lbry/chainqueryconfig.toml.orig /etc/lbry/chainqueryconfig.toml"
|
||||
cp -v /etc/lbry/chainqueryconfig.toml.orig /etc/lbry/chainqueryconfig.toml
|
||||
ls -lAh /etc/lbry/
|
||||
set_configs
|
||||
else
|
||||
echo "[INFO]: Found a copy of chainqueryconfig.toml in /etc/lbry"
|
||||
fi
|
||||
|
||||
## For now keeping this simple. Potentially eventually add all command args as envvars for the Dockerfile or use safe way to add args via docker-compose.yml
|
||||
chainquery serve --configpath "/etc/lbry/"
|
||||
@@ -0,0 +1,99 @@
|
||||
#Debug mode outputs specific information to the console
|
||||
#DEFAULT: false
|
||||
#debugmode=
|
||||
|
||||
#DebugQueryMode outputs SQL Boiler queries to the console.
|
||||
#DEFAULT: false
|
||||
#debugquerymode=
|
||||
|
||||
#LBRYcrd URL is required for chainquery to query the blockchain
|
||||
#DEFAULT: "rpc://lbry:lbry@localhost:9245"
|
||||
lbrycrdurl="rpc://lbry:lbry@lbrycrd:29245"
|
||||
|
||||
#MySQL DSN is required for chainquery to store information.
|
||||
#DEFAULT: "lbry:lbry@tcp(localhost:3306)/chainquery"
|
||||
#SUGGESTED: "lbry:lbry@unix(/var/run/mysqld/mysqld.sock)/chainquery"
|
||||
mysqldsn="lbry:lbry@tcp(mysql:3306)/chainquery"
|
||||
|
||||
#API MySQL DSN is required for chainquery to expose a SQL query service
|
||||
#DEFAULT: "lbry:lbry@tcp(localhost:3306)/chainquery"
|
||||
#SUGGESTED: "lbry:lbry@unix(/var/run/mysqld/mysqld.sock)/chainquery"
|
||||
apimysqldsn="lbry:lbry@tcp(mysql:3306)/chainquery"
|
||||
|
||||
#API Host and Port is required for the API Server to bind and listen on.
|
||||
#DEFAULT: "0.0.0.0:6300"
|
||||
#apihostport=
|
||||
|
||||
#Profile mode enables and disables the reporting of a profile for chainquery
|
||||
#DEFAULT: false
|
||||
#profilemode=
|
||||
|
||||
#Daemon mode tells chainquery how hard it should work catch up processing the blockchain
|
||||
#deamonmode=0 #BeastMode it continuously process block after block until caughtup.
|
||||
#daemonmode=1 #SlowAndSteadyMode it will process block with a frequency of 1 block every 100ms
|
||||
#daemonmode=2 #DelayMode it will process a block with a configured delay frequency (set via 'processingdelay')
|
||||
#daemonmode=3 #DaemonMode it will process a block every iteration of the daemon.
|
||||
#DEFAULT: 0
|
||||
#deamonmode=
|
||||
|
||||
#Default client timeout is for communication with the api of chainquery
|
||||
#DEFAULT: 20 #Measured in seconds
|
||||
#defaultclienttimeout=
|
||||
|
||||
#Processing delay is used to determine how frequently chainquery should process a block
|
||||
# It is only used if Daemon mode is set to delay mode
|
||||
#DEFAULT: 100 #Measured in milliseconds
|
||||
#processingdelay=
|
||||
|
||||
#Daemon delay is the frequency at which chainquery checks for work to do.
|
||||
#DEFAULT: 1 #Measured in seconds
|
||||
#daemondelay=
|
||||
|
||||
#Profiling options - will output the time take for certain opertions related to the below category
|
||||
#DEFAULT: false (for all 3 params)
|
||||
#daemonprofile=
|
||||
#lbrycrdprofile=
|
||||
#mysqlprofile=
|
||||
|
||||
#Slack Hook URL allows slack integration. All logging info level and above is posted to a slack channel.
|
||||
#DEFAULT: ""
|
||||
#slackhookurl=
|
||||
|
||||
#Slack Channel is the channel that you want the messages to appear. Works together with the hook url.
|
||||
#DEFAULT: ""
|
||||
#slackchannel=
|
||||
|
||||
#Slack Log Level tells chainquery what level of logging will be sent to the slack channel. It will log all levels below
|
||||
# it as well. Panic=0,Fatal=1,Error=2,Warning=3,Info=4,Debug=5
|
||||
#DEFAULT: 0
|
||||
#slackloglevel=
|
||||
|
||||
#The command that should be executed to trigger a self update of the software. For linux, for example, `<yourscript>.sh`
|
||||
#DEFAULT: ""
|
||||
#autoupdatecommand=
|
||||
|
||||
#Twilio service of chainquery to send specifically important information to key users of the Chainquery install.
|
||||
#DEFAULT:
|
||||
##twiliosid=""
|
||||
##twilioauthtoken=""
|
||||
##smsrecipients=["",""]
|
||||
##smsfromphonenumber=""
|
||||
#twiliosid=
|
||||
#twilioauthtoken=
|
||||
#smsrecipients=
|
||||
#smsfromphonenumber=
|
||||
|
||||
#API Keys - Disallowed by default unless keys are entered.
|
||||
#DEFAULT: []
|
||||
#apikeys=
|
||||
|
||||
#Max Failures - Specifies the number of failures that can happen in processing a transaction. This is for parallel
|
||||
#transaction processing which puts a transaction to the back of the processing queue if it fails. It can fail say if its
|
||||
#source output to spend is not already processed.
|
||||
#DEFAULT: 1000
|
||||
#maxfailures=
|
||||
|
||||
#Block Chain Name - Specifies the chain params for parsing blocks, transactions, claims, and addresses. valid choices are
|
||||
#lbrycrd_main, lbrycrd_testnet, and lbrycrd_regtest.
|
||||
#DEFAULT: "lbrycrd_main"
|
||||
blockchainname="lbrycrd_regtest"
|
||||
@@ -0,0 +1,10 @@
|
||||
blockchain_name: lbrycrd_regtest
|
||||
lbryum_servers:
|
||||
- walletserver:50001
|
||||
reflect_streams: false
|
||||
save_blobs: true
|
||||
save_files: false
|
||||
share_usage_data: false
|
||||
tcp_port: 3333
|
||||
udp_port: 4444
|
||||
use_upnp: false
|
||||
Executable
+24
@@ -0,0 +1,24 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
#Add a ytsync user
|
||||
ADDYTSYNCUSER='INSERT INTO user (given_name) VALUE("ytsync user")'
|
||||
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCUSER"
|
||||
#Insert an auth token for the user to be used by ytsync
|
||||
ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(1,"ytsyntoken")'
|
||||
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN"
|
||||
#Give priveledges to ytsync user
|
||||
ASSIGNGROOP='INSERT INTO user_groop (user_id, groop_id) VALUE( 1,3)'
|
||||
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ASSIGNGROOP"
|
||||
|
||||
#Add youtuber to sync
|
||||
ADDYTSYNCER='INSERT INTO user (given_name) VALUE("youtuber")'
|
||||
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCER"
|
||||
#Insert an auth token for the youtuber to be used by ytsync
|
||||
ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(2,"youtubertoken")'
|
||||
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN"
|
||||
#Add their youtube channel to be synced
|
||||
ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers,should_sync,redeemable,total_views,reviewed,last_uploaded_video,length_limit,size_limit,reward_amount,reward_expiration)
|
||||
VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','$1','$2','СтопХам','queued','2019-08-01 00:00:00','sync',1000,1000,1,1,10000,1,'$3',60,2048,0,'2019-08-01 00:00:00')"
|
||||
mysql -u lbry -plbry -D lbry -h "127.0.0.1" --default-character-set=utf8 -P 15500 -e "$ADDYTCHANNEL"
|
||||
@@ -0,0 +1,140 @@
|
||||
version: "3.5"
|
||||
services:
|
||||
#############
|
||||
## Lbrycrd ##
|
||||
#############
|
||||
lbrycrd:
|
||||
image: lbry/lbrycrd:v0.17.3.2-deprecatedrpc
|
||||
restart: "no"
|
||||
ports:
|
||||
- "15201:29246"
|
||||
- "15200:29245"
|
||||
expose:
|
||||
- "29246"
|
||||
- "29245"
|
||||
## host volumes for persistent data such as wallet private keys.
|
||||
volumes:
|
||||
- "./persist:/data"
|
||||
environment:
|
||||
- RUN_MODE=regtest
|
||||
###################
|
||||
## Wallet Server ##
|
||||
###################
|
||||
walletserver:
|
||||
image: lbry/wallet-server:latest-release
|
||||
restart: always
|
||||
environment:
|
||||
- DB_DIRECTORY=/database
|
||||
- MAX_SEND=1000000000000000000000
|
||||
- DAEMON_URL=http://lbry:lbry@lbrycrd:29245
|
||||
- MAX_SUBS=1000000000000
|
||||
- BANDWIDTH_LIMIT=80000000000
|
||||
- SESSION_TIMEOUT=10000000000000000000000000
|
||||
- TCP_PORT=50001
|
||||
- ELASTIC_HOST=es01
|
||||
ports:
|
||||
- "15300:50001"
|
||||
expose:
|
||||
- "50001"
|
||||
depends_on:
|
||||
- lbrycrd
|
||||
- es01
|
||||
ulimits:
|
||||
nofile:
|
||||
soft: 90000
|
||||
hard: 90000
|
||||
#command: lbry.wallet.server.coin.LBC
|
||||
command: lbry.wallet.server.coin.LBCRegTest
|
||||
#############
|
||||
## elasticsearch ##
|
||||
#############
|
||||
es01:
|
||||
image: docker.elastic.co/elasticsearch/elasticsearch:7.11.0
|
||||
container_name: es01
|
||||
environment:
|
||||
- node.name=es01
|
||||
- discovery.type=single-node
|
||||
- indices.query.bool.max_clause_count=8196
|
||||
- bootstrap.memory_lock=true
|
||||
- "ES_JAVA_OPTS=-Xms4g -Xmx4g"
|
||||
ulimits:
|
||||
memlock:
|
||||
soft: -1
|
||||
hard: -1
|
||||
ports:
|
||||
- "9200:9200"
|
||||
expose:
|
||||
- "9200"
|
||||
#############
|
||||
## Lbrynet ##
|
||||
#############
|
||||
lbrynet:
|
||||
image: lbry/lbrynet:v0.99.0
|
||||
restart: always
|
||||
ports:
|
||||
- "15100:5279"
|
||||
- "15101:5280"
|
||||
expose:
|
||||
- "5279"
|
||||
- "5280"
|
||||
depends_on:
|
||||
- walletserver
|
||||
environment:
|
||||
- LBRY_STREAMING_SERVER=0.0.0.0:5280
|
||||
volumes:
|
||||
- "./persist/.lbrynet:/home/lbrynet"
|
||||
- ".:/etc/lbry" #Put your daemon_settings.yml here
|
||||
# /private/var/tmp for OSX and /var/tmp for Linux
|
||||
- "${LOCAL_TMP_DIR}"
|
||||
###########
|
||||
## MySQL ##
|
||||
###########
|
||||
mysql:
|
||||
image: mysql/mysql-server:5.7.33
|
||||
restart: "no"
|
||||
ports:
|
||||
- "15500:3306"
|
||||
expose:
|
||||
- "3306"
|
||||
environment:
|
||||
- MYSQL_ALLOW_EMPTY_PASSWORD=true
|
||||
- MYSQL_DATABASE=lbry
|
||||
- MYSQL_USER=lbry
|
||||
- MYSQL_PASSWORD=lbry
|
||||
- MYSQL_LOG_CONSOLE=true
|
||||
volumes:
|
||||
- "./init.sql:/docker-entrypoint-initdb.d/init.sql"
|
||||
- "./chainquery/docker/my.cnf:/etc/mysql/conf.d/chainquery-optimizations.cnf"
|
||||
###################
|
||||
## Internal APIs ##
|
||||
###################
|
||||
internalapis:
|
||||
image: lbry/internal-apis:master
|
||||
restart: "no"
|
||||
ports:
|
||||
- "15400:8080"
|
||||
expose:
|
||||
- "8080"
|
||||
depends_on:
|
||||
- mysql
|
||||
- lbrycrd
|
||||
environment:
|
||||
- MYSQL_DSN=lbry:lbry@tcp(mysql:3306)/lbry
|
||||
- LBRYCRD_CONNECT=rpc://lbry:lbry@lbrycrd:29245
|
||||
- REPLICA_DSN=lbry:lbry@tcp(mysql:3306)/lbry
|
||||
entrypoint: wait-for-it -t 0 chainquery:6300 -- wait-for-it -t 0 lbrycrd:29245 -- ./latest serve
|
||||
################
|
||||
## Chainquery ##
|
||||
################
|
||||
chainquery:
|
||||
image: lbry/chainquery:master
|
||||
restart: "no"
|
||||
ports:
|
||||
- 6300:6300
|
||||
depends_on:
|
||||
- lbrycrd
|
||||
- mysql
|
||||
## TODO: Uncomment this in a docker-compose.override.yml to allow for external configurations.
|
||||
volumes:
|
||||
- ./chainqueryconfig.toml:/etc/lbry/chainqueryconfig.toml
|
||||
entrypoint: wait-for-it -t 0 lbrycrd:29245 -- wait-for-it -t 0 mysql:3306 -- start
|
||||
Executable
+99
@@ -0,0 +1,99 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
#Always compile ytsync
|
||||
make
|
||||
#Always compile supporty
|
||||
cd e2e/supporty && make && cd ../..
|
||||
|
||||
#OVERRIDE this in your .env file if running from mac. Check docker-compose.yml for details
|
||||
export LOCAL_TMP_DIR="/var/tmp:/var/tmp"
|
||||
|
||||
#Private Variables Set in local installations: SLACK_TOKEN,YOUTUBE_API_KEY,AWS_S3_ID,AWS_S3_SECRET,AWS_S3_REGION,AWS_S3_BUCKET
|
||||
touch -a .env && set -o allexport; source ./.env; set +o allexport
|
||||
echo "LOCAL_TMP_DIR=$LOCAL_TMP_DIR"
|
||||
# Compose settings - docker only
|
||||
export SLACK_CHANNEL="ytsync-travis"
|
||||
export LBRY_API_TOKEN="ytsyntoken"
|
||||
export LBRY_WEB_API="http://localhost:15400"
|
||||
export LBRYNET_ADDRESS="http://localhost:15100"
|
||||
export LBRYCRD_STRING="tcp://lbry:lbry@localhost:15200"
|
||||
export LBRYNET_USE_DOCKER=true
|
||||
export REFLECT_BLOBS=false
|
||||
export CLEAN_ON_STARTUP=true
|
||||
export REGTEST=true
|
||||
# Local settings
|
||||
export BLOBS_DIRECTORY="$(pwd)/e2e/blobsfiles"
|
||||
export LBRYNET_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbrynet/"
|
||||
export LBRYUM_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbryum"
|
||||
export TMP_DIR="/var/tmp"
|
||||
export CHAINNAME="lbrycrd_regtest"
|
||||
export UID
|
||||
|
||||
cd ./e2e
|
||||
docker-compose stop
|
||||
docker-compose rm -f
|
||||
echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
|
||||
docker-compose pull
|
||||
if [[ -d persist ]]; then rm -rf persist; fi
|
||||
mkdir -m 0777 -p ./persist
|
||||
mkdir -m 777 -p ./persist/.walletserver
|
||||
mkdir -m 777 -p ./persist/.lbrynet
|
||||
#sudo chown -Rv 999:999 ./persist/.walletserver
|
||||
#sudo chown -Rv 1000:1000 ./persist/.lbrynet
|
||||
docker-compose up -d
|
||||
printf 'waiting for internal apis'
|
||||
until curl --output /dev/null --silent --head --fail http://localhost:15400; do
|
||||
printf '.'
|
||||
sleep 1
|
||||
done
|
||||
echo "successfully started..."
|
||||
|
||||
channelToSync="UCGyoEsIRjmnmzrsB67DhrOA"
|
||||
channelName=@Alaminemoh11"$(date +%s)"
|
||||
latestVideoID="ejWF7Jjdgmc"
|
||||
|
||||
#Data Setup for test
|
||||
./data_setup.sh "$channelName" "$channelToSync" "$latestVideoID"
|
||||
|
||||
# Execute the sync test!
|
||||
./../bin/ytsync --channelID "$channelToSync" --videos-limit 2 --concurrent-jobs 4 --quick #Force channel intended...just in case. This channel lines up with the api container
|
||||
status=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM youtube_data WHERE id=1')
|
||||
videoStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM synced_video WHERE id=1')
|
||||
videoClaimID1=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT publish.claim_id FROM synced_video INNER JOIN publish ON publish.id = synced_video.publish_id WHERE synced_video.id=1')
|
||||
videoClaimID2=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT publish.claim_id FROM synced_video INNER JOIN publish ON publish.id = synced_video.publish_id WHERE synced_video.id=2')
|
||||
videoClaimAddress1=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT claim_address FROM claim WHERE id=2')
|
||||
videoClaimAddress2=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT claim_address FROM claim WHERE id=3')
|
||||
# Create Supports for published claim
|
||||
./supporty/supporty "$channelName" "${videoClaimID1}" "${videoClaimAddress1}" lbrycrd_regtest 1.0
|
||||
./supporty/supporty "$channelName" "${videoClaimID2}" "${videoClaimAddress2}" lbrycrd_regtest 2.0
|
||||
./supporty/supporty "$channelName" "${videoClaimID2}" "${videoClaimAddress2}" lbrycrd_regtest 3.0
|
||||
./supporty/supporty "$channelName" "${videoClaimID1}" "${videoClaimAddress1}" lbrycrd_regtest 3.0
|
||||
curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"generate","params":[1]}' -H 'content-type:text/plain;' --user lbry:lbry http://localhost:15200
|
||||
# Reset status for transfer test
|
||||
mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e "UPDATE youtube_data SET status = 'queued' WHERE id = 1"
|
||||
# Trigger transfer api
|
||||
curl -i -H 'Accept: application/json' -H 'Content-Type: application/json' 'http://localhost:15400/yt/transfer?auth_token=youtubertoken&address=n4eYeXAYmHo4YRUDEfsEhucy8y5LKRMcHg&public_key=tpubDA9GDAntyJu4hD3wU7175p7CuV6DWbYXfyb2HedBA3yuBp9HZ4n3QE4Ex6RHCSiEuVp2nKAL1Lzf2ZLo9ApaFgNaJjG6Xo1wB3iEeVbrDZp'
|
||||
# Execute the transfer test!
|
||||
./../bin/ytsync --channelID $channelToSync --videos-limit 2 --concurrent-jobs 4 --quick #Force channel intended...just in case. This channel lines up with the api container
|
||||
# Check that the channel and the video are marked as transferred and that all supports are spent
|
||||
channelTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT distinct transfer_state FROM youtube_data')
|
||||
videoTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT distinct transferred FROM synced_video')
|
||||
nrUnspentSupports=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT COUNT(*) FROM chainquery.support INNER JOIN output ON output.transaction_hash = support.transaction_hash_id AND output.vout = support.vout WHERE output.is_spent = 0')
|
||||
if [[ $status != "synced" || $videoStatus != "published" || $channelTransferStatus != "2" || $videoTransferStatus != "1" || $nrUnspentSupports != "1" ]]; then
|
||||
echo "~~!!!~~~FAILED~~~!!!~~"
|
||||
echo "Channel Status: $status"
|
||||
echo "Video Status: $videoStatus"
|
||||
echo "Channel Transfer Status: $channelTransferStatus"
|
||||
echo "Video Transfer Status: $videoTransferStatus"
|
||||
echo "Nr Unspent Supports: $nrUnspentSupports"
|
||||
#docker-compose logs --tail="all" lbrycrd
|
||||
#docker-compose logs --tail="all" walletserver
|
||||
#docker-compose logs --tail="all" lbrynet
|
||||
#docker-compose logs --tail="all" internalapis
|
||||
exit 1;
|
||||
else
|
||||
echo "SUCCESSSSSSSSSSSSS!"
|
||||
fi;
|
||||
docker-compose down
|
||||
@@ -0,0 +1,3 @@
|
||||
CREATE DATABASE IF NOT EXISTS chainquery;
|
||||
GRANT ALL PRIVILEGES ON chainquery.* TO 'lbry'@'%';
|
||||
FLUSH PRIVILEGES;
|
||||
@@ -0,0 +1,26 @@
|
||||
version: "3"
|
||||
networks:
|
||||
lbry-network:
|
||||
external: true
|
||||
|
||||
services:
|
||||
#############
|
||||
## Lbrycrd ##
|
||||
#############
|
||||
lbrycrd:
|
||||
image: lbry/lbrycrd:v0.12.4.1
|
||||
restart: always
|
||||
networks:
|
||||
lbry-network:
|
||||
ipv4_address: 10.6.1.1
|
||||
ports:
|
||||
- "15201:29246"
|
||||
- "15200:29245"
|
||||
expose:
|
||||
- "29246"
|
||||
- "29245"
|
||||
## host volumes for persistent data such as wallet private keys.
|
||||
volumes:
|
||||
- "../persist/data:/data"
|
||||
environment:
|
||||
- RUN_MODE=regtest
|
||||
@@ -0,0 +1,37 @@
|
||||
FROM ubuntu:18.04 as prep
|
||||
LABEL MAINTAINER="leopere [at] nixc [dot] us"
|
||||
## TODO: Implement version pinning. `apt-get install curl=<version>`
|
||||
RUN apt-get update && \
|
||||
apt-get -y install unzip curl build-essential && \
|
||||
apt-get autoclean -y && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
WORKDIR /
|
||||
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
|
||||
COPY ./start.sh start
|
||||
COPY ./healthcheck.sh healthcheck
|
||||
COPY ./advance_blocks.sh advance
|
||||
COPY ./fix-permissions.c fix-permissions.c
|
||||
|
||||
## Add lbrycrd - Change the version below to create an image for a different tag/version
|
||||
ARG VERSION="v0.12.4.1"
|
||||
RUN URL=$(curl -s https://api.github.com/repos/lbryio/lbrycrd/releases/$(if [ "${VERSION}" = 'latest' ]; then echo "latest"; else echo "tags/${VERSION}"; fi) | grep browser_download_url | grep lbrycrd-linux.zip | cut -d'"' -f4) && echo $URL && curl -L -o /lbrycrd-linux.zip $URL
|
||||
|
||||
RUN unzip ./lbrycrd-linux.zip && \
|
||||
gcc fix-permissions.c -o fix-permissions && \
|
||||
chmod +x ./lbrycrdd ./lbrycrd-cli ./lbrycrd-tx ./start ./healthcheck ./fix-permissions ./advance
|
||||
|
||||
FROM ubuntu:18.04 as app
|
||||
COPY --from=prep /lbrycrdd /lbrycrd-cli /lbrycrd-tx /start /healthcheck /fix-permissions /advance /usr/bin/
|
||||
RUN addgroup --gid 1000 lbrycrd && \
|
||||
adduser lbrycrd --uid 1000 --gid 1000 --gecos GECOS --shell /bin/bash --disabled-password --home /data && \
|
||||
mkdir /etc/lbry && \
|
||||
chown lbrycrd /etc/lbry && \
|
||||
chmod a+s /usr/bin/fix-permissions
|
||||
VOLUME ["/data"]
|
||||
WORKDIR /data
|
||||
## TODO: Implement healthcheck.
|
||||
# HEALTHCHECK ["healthcheck"]
|
||||
EXPOSE 9246 9245 11337 29245
|
||||
|
||||
USER lbrycrd
|
||||
CMD ["start"]
|
||||
@@ -0,0 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
while true; do
|
||||
lbrycrd-cli -conf=/etc/lbry/lbrycrd.conf generate 100 >> /tmp/output.log
|
||||
sleep 2
|
||||
done
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/bin/bash
|
||||
if [ $# -eq 0 ]
|
||||
then
|
||||
echo "No docker tag argument supplied. Use './build.sh <tag>'"
|
||||
exit 1
|
||||
fi
|
||||
docker build --build-arg VERSION=$1 --tag lbry/lbrycrd:$1 .
|
||||
docker push lbry/lbrycrd:$1
|
||||
@@ -0,0 +1,9 @@
|
||||
#include <unistd.h>
|
||||
int main() {
|
||||
// This program needs to run with setuid == root
|
||||
// This needs to be in a compiled language because you cannot setuid bash scripts
|
||||
setuid(0);
|
||||
execle("/bin/bash", "bash", "-c",
|
||||
"/bin/chown -R lbrycrd:lbrycrd /data && /bin/chmod -R 755 /data/",
|
||||
(char*) NULL, (char*) NULL);
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
## TODO: Implement a healthcheck for lbrycrd.
|
||||
curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"getinfo","params":[]}' -H 'content-type:text/plain;' http://$RPC_USER:$RPC_PASSWORD@127.0.0.1:9246
|
||||
## OR
|
||||
lbrycrd-cli getinfo
|
||||
@@ -0,0 +1,127 @@
|
||||
#!/usr/bin/env bash
|
||||
CONFIG_PATH=/etc/lbry/lbrycrd.conf
|
||||
|
||||
function override_config_option() {
|
||||
# Remove existing config line from a config file
|
||||
# and replace with environment fed value.
|
||||
# Does nothing if the variable does not exist.
|
||||
# var Name of ENV variable
|
||||
# option Name of config option
|
||||
# config Path of config file
|
||||
local var=$1 option=$2 config=$3
|
||||
if [[ -v $var ]]; then
|
||||
# Remove the existing config option:
|
||||
sed -i "/^$option\W*=/d" "$config"
|
||||
# Add the value from the environment:
|
||||
echo "$option=${!var}" >> "$config"
|
||||
fi
|
||||
}
|
||||
|
||||
function set_config() {
|
||||
if [ -d "$CONFIG_PATH" ]; then
|
||||
echo "$CONFIG_PATH is a directory when it should be a file."
|
||||
exit 1
|
||||
elif [ -f "$CONFIG_PATH" ]; then
|
||||
echo "Merging the mounted config file with environment variables."
|
||||
local MERGED_CONFIG=/tmp/lbrycrd_merged.conf
|
||||
cat $CONFIG_PATH > $MERGED_CONFIG
|
||||
echo "" >> $MERGED_CONFIG
|
||||
override_config_option PORT port $MERGED_CONFIG
|
||||
override_config_option RPC_USER rpcuser $MERGED_CONFIG
|
||||
override_config_option RPC_PASSWORD rpcpassword $MERGED_CONFIG
|
||||
override_config_option RPC_ALLOW_IP rpcallowip $MERGED_CONFIG
|
||||
override_config_option RPC_PORT rpcport $MERGED_CONFIG
|
||||
override_config_option RPC_BIND rpcbind $MERGED_CONFIG
|
||||
# Make the new merged config file the new CONFIG_PATH
|
||||
# This ensures that the original file the user mounted remains unmodified
|
||||
CONFIG_PATH=$MERGED_CONFIG
|
||||
else
|
||||
echo "Creating a fresh config file from environment variables."
|
||||
## Set config params
|
||||
{
|
||||
echo "port=${PORT=9246}"
|
||||
echo "rpcuser=${RPC_USER=lbry}"
|
||||
echo "rpcpassword=${RPC_PASSWORD=lbry}"
|
||||
echo "rpcallowip=${RPC_ALLOW_IP=127.0.0.1/24}"
|
||||
echo "rpcport=${RPC_PORT=9245}"
|
||||
echo "rpcbind=${RPC_BIND=0.0.0.0}"
|
||||
echo "deprecatedrpc=accounts"
|
||||
echo "deprecatedrpc=validateaddress"
|
||||
echo "deprecatedrpc=signrawtransaction"
|
||||
} >> $CONFIG_PATH
|
||||
fi
|
||||
echo "Config: "
|
||||
cat $CONFIG_PATH
|
||||
}
|
||||
|
||||
## Ensure perms are correct prior to running main binary
|
||||
/usr/bin/fix-permissions
|
||||
|
||||
## You can optionally specify a run mode if you want to use lbry defined presets for compatibility.
|
||||
case $RUN_MODE in
|
||||
default )
|
||||
set_config
|
||||
lbrycrdd -server -conf=$CONFIG_PATH -printtoconsole
|
||||
;;
|
||||
## If it's a first run you need to do a full index including all transactions
|
||||
## tx index creates an index of every single transaction in the block history if
|
||||
## not specified it will only create an index for transactions that are related to the wallet or have unspent outputs.
|
||||
## This is generally specific to chainquery.
|
||||
reindex )
|
||||
## Apply this RUN_MODE in the case you need to update a dataset. NOTE: you do not need to use `RUN_MODE reindex` for more than one complete run.
|
||||
set_config
|
||||
lbrycrdd -server -txindex -reindex -conf=$CONFIG_PATH -printtoconsole
|
||||
;;
|
||||
chainquery )
|
||||
## If your only goal is to run Chainquery against this instance of lbrycrd and you're starting a
|
||||
## fresh local dataset use this run mode.
|
||||
set_config
|
||||
lbrycrdd -server -txindex -conf=$CONFIG_PATH -printtoconsole
|
||||
;;
|
||||
regtest )
|
||||
## Set config params
|
||||
## TODO: Make this more automagic in the future.
|
||||
mkdir -p "$(dirname $CONFIG_PATH)"
|
||||
echo "rpcuser=lbry" > $CONFIG_PATH
|
||||
echo "rpcpassword=lbry" >> $CONFIG_PATH
|
||||
echo "rpcport=29245" >> $CONFIG_PATH
|
||||
echo "rpcbind=0.0.0.0" >> $CONFIG_PATH
|
||||
echo "rpcallowip=0.0.0.0/0" >> $CONFIG_PATH
|
||||
echo "regtest=1" >> $CONFIG_PATH
|
||||
echo "txindex=1" >> $CONFIG_PATH
|
||||
echo "server=1" >> $CONFIG_PATH
|
||||
echo "printtoconsole=1" >> $CONFIG_PATH
|
||||
echo "deprecatedrpc=accounts" >> $CONFIG_PATH
|
||||
echo "deprecatedrpc=validateaddress" >> $CONFIG_PATH
|
||||
echo "deprecatedrpc=signrawtransaction" >> $CONFIG_PATH
|
||||
echo "vbparams=segwit:0:999999999999" >> $CONFIG_PATH
|
||||
echo "addresstype=legacy" >> $CONFIG_PATH
|
||||
|
||||
#nohup advance &>/dev/null &
|
||||
lbrycrdd -conf=$CONFIG_PATH $1
|
||||
;;
|
||||
testnet )
|
||||
## Set config params
|
||||
## TODO: Make this more automagic in the future.
|
||||
mkdir -p "$(dirname $CONFIG_PATH)"
|
||||
echo "rpcuser=lbry" > $CONFIG_PATH
|
||||
echo "rpcpassword=lbry" >> $CONFIG_PATH
|
||||
echo "rpcport=29245" >> $CONFIG_PATH
|
||||
echo "rpcbind=0.0.0.0" >> $CONFIG_PATH
|
||||
echo "rpcallowip=0.0.0.0/0" >> $CONFIG_PATH
|
||||
echo "testnet=1" >> $CONFIG_PATH
|
||||
echo "txindex=1" >> $CONFIG_PATH
|
||||
echo "server=1" >> $CONFIG_PATH
|
||||
echo "printtoconsole=1" >> $CONFIG_PATH
|
||||
echo "deprecatedrpc=accounts" >> $CONFIG_PATH
|
||||
echo "deprecatedrpc=validateaddress" >> $CONFIG_PATH
|
||||
echo "deprecatedrpc=signrawtransaction" >> $CONFIG_PATH
|
||||
|
||||
#nohup advance &>/dev/null &
|
||||
lbrycrdd -conf=$CONFIG_PATH $1
|
||||
;;
|
||||
* )
|
||||
echo "Error, you must define a RUN_MODE environment variable."
|
||||
echo "Available options are testnet, regtest, chainquery, default, and reindex"
|
||||
;;
|
||||
esac
|
||||
@@ -0,0 +1,23 @@
|
||||
version: "3"
|
||||
networks:
|
||||
lbry-network:
|
||||
external: true
|
||||
|
||||
services:
|
||||
#############
|
||||
## Lbrynet ##
|
||||
#############
|
||||
lbrynet:
|
||||
image: lbry/lbrynet:v0.99.0
|
||||
restart: "no"
|
||||
networks:
|
||||
lbry-network:
|
||||
ipv4_address: 10.6.1.3
|
||||
ports:
|
||||
- "15100:5279"
|
||||
- "15101:5280"
|
||||
environment:
|
||||
- LBRY_STREAMING_SERVER=0.0.0.0:5280
|
||||
volumes:
|
||||
- "../persist/data/.lbrynet:/home/lbrynet"
|
||||
- "./settings:/etc/lbry" #Put your daemon_settings.yml here
|
||||
@@ -0,0 +1,27 @@
|
||||
## This base image is for running the latest lbrynet-daemon release.
|
||||
FROM ubuntu:18.04 as prep
|
||||
LABEL MAINTAINER="leopere [at] nixc [dot] us"
|
||||
RUN apt-get update && apt-get -y install unzip curl telnet wait-for-it
|
||||
|
||||
## Add lbrynet
|
||||
ARG VERSION="latest"
|
||||
RUN URL=$(curl -s https://api.github.com/repos/lbryio/lbry-sdk/releases/$(if [ "${VERSION}" = 'latest' ]; then echo "latest"; else echo "tags/${VERSION}"; fi) | grep browser_download_url | grep lbrynet-linux.zip | cut -d'"' -f4) && echo $URL && curl -L -o /lbrynet.linux.zip $URL
|
||||
|
||||
COPY start.sh /usr/bin/start
|
||||
COPY checkmount.sh /usr/bin/checkmount
|
||||
RUN unzip /lbrynet.linux.zip -d /lbrynet/ && \
|
||||
mv /lbrynet/lbrynet /usr/bin && \
|
||||
chmod a+x /usr/bin/checkmount /usr/bin/start /usr/bin/lbrynet
|
||||
|
||||
FROM ubuntu:18.04 as app
|
||||
COPY --from=prep /usr/bin/start /usr/bin/checkmount /usr/bin/lbrynet /usr/bin/
|
||||
RUN adduser lbrynet --gecos GECOS --shell /bin/bash --disabled-password --home /home/lbrynet
|
||||
## Daemon port [Intended for internal use]
|
||||
## LBRYNET talks to peers on port 3333 [Intended for external use] this port is used to discover other lbrynet daemons with blobs.
|
||||
## Expose 5566 Reflector port to listen on
|
||||
## Expose 5279 Port the daemon API will listen on
|
||||
## the lbryumx aka Wallet port [Intended for internal use]
|
||||
#EXPOSE 4444 3333 5566 5279 50001
|
||||
USER lbrynet
|
||||
ENTRYPOINT ["/usr/bin/checkmount"]
|
||||
CMD ["start"]
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/bin/bash
|
||||
if [ $# -eq 0 ]
|
||||
then
|
||||
echo "No docker tag argument supplied. Use './build.sh <tag>'"
|
||||
exit 1
|
||||
fi
|
||||
docker build --build-arg VERSION=$1 --tag lbry/lbrynet:$1 .
|
||||
docker push lbry/lbrynet:$1
|
||||
@@ -0,0 +1,12 @@
|
||||
#!/bin/bash
|
||||
|
||||
## TODO: Make a bit more aware of the run mode of this appliance in case there is ever a test mode enabled in the start.sh
|
||||
mountpoint=/home/lbrynet
|
||||
|
||||
if ! grep -qs ".* $mountpoint " /proc/mounts; then
|
||||
echo "$mountpoint not mounted, refusing to run."
|
||||
## TODO: We should have documentation that this error references directly with a URL as to why it won't run without a volume.
|
||||
exit 1
|
||||
else
|
||||
bash -c "$*"
|
||||
fi
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/bin/bash
|
||||
lbrynet start \
|
||||
--api="${API_BIND_IP:-0.0.0.0}":"${API_PORT:-5279}" \
|
||||
--config="${CONFIG_PATH:-/etc/lbry/daemon_settings.yml}"
|
||||
@@ -0,0 +1,13 @@
|
||||
#blockchain_name: lbrycrd_main
|
||||
#blockchain_name: lbrycrd_testnet
|
||||
blockchain_name: lbrycrd_regtest
|
||||
lbryum_servers:
|
||||
# - spv1.lbry.com:50001 #Production Wallet Server
|
||||
- walletserver:50001
|
||||
save_blobs: true
|
||||
save_files: false
|
||||
reflect_streams: false #for the love of god, don't upload regtest streams to reflector!
|
||||
share_usage_data: false
|
||||
tcp_port: 3333
|
||||
udp_port: 4444
|
||||
use_upnp: true
|
||||
@@ -0,0 +1,12 @@
|
||||
BINARY=supporty
|
||||
|
||||
DIR = $(shell cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)
|
||||
BIN_DIR = ${DIR}
|
||||
|
||||
.PHONY: build clean test lint
|
||||
.DEFAULT_GOAL: build
|
||||
|
||||
|
||||
build:
|
||||
mkdir -p ${BIN_DIR} && CGO_ENABLED=0 go build -asmflags -trimpath=${DIR} -o ${BIN_DIR}/${BINARY} supporty.go
|
||||
chmod +x ${BIN_DIR}/${BINARY}
|
||||
@@ -0,0 +1,43 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/lbryio/ytsync/v5/util"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if len(os.Args) != 6 {
|
||||
logrus.Info(strings.Join(os.Args, ","))
|
||||
logrus.Fatal("Not enough arguments: name, claimID, address, blockchainName, claimAmount")
|
||||
}
|
||||
println("Supporty!")
|
||||
lbrycrd, err := util.GetLbrycrdClient(os.Getenv("LBRYCRD_STRING"))
|
||||
if err != nil {
|
||||
logrus.Fatal(err)
|
||||
}
|
||||
if lbrycrd == nil {
|
||||
logrus.Fatal("Lbrycrd Client is nil")
|
||||
}
|
||||
amount, err := strconv.ParseFloat(os.Args[5], 64)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
name := os.Args[1]
|
||||
claimid := os.Args[2]
|
||||
claimAddress := os.Args[3]
|
||||
blockChainName := os.Args[4]
|
||||
logrus.Infof("Supporting %s[%s] with %.2f LBC on chain %s at address %s", name, claimid, amount, blockChainName, claimAddress)
|
||||
hash, err := lbrycrd.SupportClaim(name, claimid, claimAddress, blockChainName, amount)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
if hash == nil {
|
||||
logrus.Fatal("Tx not created!")
|
||||
}
|
||||
logrus.Info("Tx: ", hash.String())
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
version: "3"
|
||||
networks:
|
||||
lbry-network:
|
||||
external: true
|
||||
|
||||
services:
|
||||
###################
|
||||
## Wallet Server ##
|
||||
###################
|
||||
walletserver:
|
||||
image: lbry/wallet-server:v0.73.1
|
||||
restart: always
|
||||
networks:
|
||||
lbry-network:
|
||||
ipv4_address: 10.6.1.2
|
||||
volumes:
|
||||
- "../persist/data/.walletserver/database:/database"
|
||||
environment:
|
||||
- DB_DIRECTORY=/database
|
||||
- MAX_SEND=1000000000000000000000
|
||||
- DAEMON_URL=http://lbry:lbry@lbrycrd:29245/
|
||||
- MAX_SUBS=1000000000000
|
||||
- BANDWIDTH_LIMIT=80000000000
|
||||
- SESSION_TIMEOUT=10000000000000000000000000
|
||||
- TCP_PORT=50001
|
||||
#network_mode: host
|
||||
#network_mode: bridge
|
||||
ports:
|
||||
- "50001:50001"
|
||||
expose:
|
||||
- "50001"
|
||||
ulimits:
|
||||
nofile: 90000
|
||||
# command: lbry.wallet.server.coin.LBC
|
||||
command: lbry.wallet.server.coin.LBCRegTest
|
||||
@@ -1,47 +1,42 @@
|
||||
module github.com/lbryio/ytsync
|
||||
module github.com/lbryio/ytsync/v5
|
||||
|
||||
replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19
|
||||
|
||||
//replace github.com/lbryio/lbry.go/v2 => /home/niko/go/src/github.com/lbryio/lbry.go/
|
||||
//replace github.com/lbryio/reflector.go => /home/niko/go/src/github.com/lbryio/reflector.go/
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.37.4 // indirect
|
||||
github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61
|
||||
github.com/PuerkitoBio/goquery v1.5.0 // indirect
|
||||
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
|
||||
github.com/aws/aws-sdk-go v1.17.3
|
||||
github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 // indirect
|
||||
github.com/go-ini/ini v1.42.0 // indirect
|
||||
github.com/go-sql-driver/mysql v1.4.1 // indirect
|
||||
github.com/gopherjs/gopherjs v0.0.0-20190411002643-bd77b112433e // indirect
|
||||
github.com/hashicorp/go-msgpack v0.5.5 // indirect
|
||||
github.com/hashicorp/golang-lru v0.5.1 // indirect
|
||||
github.com/hashicorp/memberlist v0.1.4 // indirect
|
||||
github.com/hashicorp/serf v0.8.2 // indirect
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
|
||||
github.com/kr/pretty v0.1.0 // indirect
|
||||
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c
|
||||
github.com/lbryio/lbry.go v1.0.15
|
||||
github.com/lbryio/reflector.go v1.0.6-0.20190710201919-7c7ed9da72ce
|
||||
github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2 // indirect
|
||||
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936
|
||||
github.com/nikooo777/ytdl v0.0.0-20190215151411-9c7832eaf457
|
||||
github.com/onsi/ginkgo v1.8.0 // indirect
|
||||
github.com/onsi/gomega v1.5.0 // indirect
|
||||
github.com/Microsoft/go-winio v0.4.14 // indirect
|
||||
github.com/abadojack/whatlanggo v1.0.1
|
||||
github.com/asaskevich/govalidator v0.0.0-20200819183940-29e1ff8eb0bb
|
||||
github.com/aws/aws-sdk-go v1.25.9
|
||||
github.com/davecgh/go-spew v1.1.1
|
||||
github.com/docker/distribution v2.7.1+incompatible // indirect
|
||||
github.com/docker/docker v1.13.1
|
||||
github.com/docker/go-connections v0.4.0 // indirect
|
||||
github.com/docker/go-units v0.4.0 // indirect
|
||||
github.com/hashicorp/go-immutable-radix v1.1.0 // indirect
|
||||
github.com/hashicorp/go-sockaddr v1.0.2 // indirect
|
||||
github.com/hashicorp/memberlist v0.1.5 // indirect
|
||||
github.com/hashicorp/serf v0.8.5 // indirect
|
||||
github.com/kr/pretty v0.2.1 // indirect
|
||||
github.com/lbryio/lbry.go/v2 v2.7.2-0.20210412222918-ed51ece75c3d
|
||||
github.com/lbryio/reflector.go v1.1.3-0.20210412225256-4392c9724262
|
||||
github.com/miekg/dns v1.1.22 // indirect
|
||||
github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b
|
||||
github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
|
||||
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 // indirect
|
||||
github.com/prometheus/common v0.3.0
|
||||
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24
|
||||
github.com/sirupsen/logrus v1.4.1
|
||||
github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3 // indirect
|
||||
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect
|
||||
github.com/spf13/cobra v0.0.0-20190109003409-7547e83b2d85
|
||||
github.com/spf13/pflag v1.0.3 // indirect
|
||||
github.com/ybbus/jsonrpc v2.1.2+incompatible // indirect
|
||||
go.opencensus.io v0.20.2 // indirect
|
||||
golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480 // indirect
|
||||
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c // indirect
|
||||
golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a // indirect
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58 // indirect
|
||||
golang.org/x/text v0.3.2 // indirect
|
||||
google.golang.org/api v0.3.2
|
||||
google.golang.org/appengine v1.5.0 // indirect
|
||||
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7 // indirect
|
||||
google.golang.org/grpc v1.20.0 // indirect
|
||||
gopkg.in/ini.v1 v1.42.0 // indirect
|
||||
github.com/prometheus/client_golang v0.9.3
|
||||
github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5
|
||||
github.com/sirupsen/logrus v1.4.2
|
||||
github.com/spf13/cobra v0.0.5
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
github.com/stretchr/testify v1.7.0
|
||||
github.com/vbauerster/mpb/v7 v7.0.2
|
||||
google.golang.org/appengine v1.6.5 // indirect
|
||||
gopkg.in/ini.v1 v1.60.2 // indirect
|
||||
gopkg.in/vansante/go-ffprobe.v2 v2.0.2
|
||||
gotest.tools v2.2.0+incompatible
|
||||
)
|
||||
|
||||
go 1.13
|
||||
@@ -1,431 +1,759 @@
|
||||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.37.4 h1:glPeL3BQJsbF6aIIYfZizMwc5LTYz250bDMjttbBGAU=
|
||||
cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7hw=
|
||||
cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo=
|
||||
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
|
||||
cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
|
||||
cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
|
||||
cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
|
||||
cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
|
||||
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
|
||||
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
|
||||
cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
|
||||
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
|
||||
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
|
||||
dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU=
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU=
|
||||
dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4=
|
||||
dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU=
|
||||
git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
|
||||
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 h1:N5Vqww5QISEHsWHOWDEx4PzdIay3Cg0Jp7zItq2ZAro=
|
||||
github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61/go.mod h1:GnKXcK+7DYNy/8w2Ex//Uql4IgfaU82Cd5rWKb7ah00=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
|
||||
github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP7EJk=
|
||||
github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg=
|
||||
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
|
||||
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
|
||||
github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU=
|
||||
github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/StackExchange/wmi v0.0.0-20170410192909-ea383cf3ba6e/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
|
||||
github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
|
||||
github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
|
||||
github.com/abadojack/whatlanggo v1.0.1 h1:19N6YogDnf71CTHm3Mp2qhYfkRdyvbgwWdd2EPxJRG4=
|
||||
github.com/abadojack/whatlanggo v1.0.1/go.mod h1:66WiQbSbJBIlOZMsvbKe5m6pzQovxCH9B/K8tQB2uoc=
|
||||
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
|
||||
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
|
||||
github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
|
||||
github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
|
||||
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
|
||||
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
|
||||
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
|
||||
github.com/armon/go-metrics v0.0.0-20180713145231-3c58d8115a78 h1:mdRSArcFLfW0VoL34LZAKSz6LkkK4jFxVx2xYavACMg=
|
||||
github.com/armon/go-metrics v0.0.0-20180713145231-3c58d8115a78/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
|
||||
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
|
||||
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
|
||||
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM=
|
||||
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg=
|
||||
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
|
||||
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
|
||||
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
|
||||
github.com/asaskevich/govalidator v0.0.0-20200819183940-29e1ff8eb0bb h1:kvlW1qyM1aU3xeyeIVTU2jx5fSvjKpsU3aXvuaCMg3Q=
|
||||
github.com/asaskevich/govalidator v0.0.0-20200819183940-29e1ff8eb0bb/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
|
||||
github.com/aws/aws-sdk-go v1.16.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
||||
github.com/aws/aws-sdk-go v1.17.3 h1:KBXxg7Jh0TxE5zmpNB2DwKmJeDUqh0O6jhy25TuYOmc=
|
||||
github.com/aws/aws-sdk-go v1.17.3/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
||||
github.com/aws/aws-sdk-go v1.25.9 h1:WtVzerf5wSgPwlTTwl+ktCq/0GCS5MI9ZlLIcjsTr+Q=
|
||||
github.com/aws/aws-sdk-go v1.25.9/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||
github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
|
||||
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
|
||||
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
|
||||
github.com/btcsuite/btcd v0.0.0-20180531025944-86fed781132a/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ=
|
||||
github.com/btcsuite/btcd v0.0.0-20190109040709-5bda5314ca95/go.mod h1:d3C0AkH6BRcvO8T0UEPu53cnw4IbV63x1bEjildYhO0=
|
||||
github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32 h1:qkOC5Gd33k54tobS36cXdAzJbeHaduLtnLQQwNoIi78=
|
||||
github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8=
|
||||
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
|
||||
github.com/bparli/lfuda-go v0.3.1 h1:nO9Szo627RC8/z+R+MMPBItNwHCOonchmpjQuQi8jVY=
|
||||
github.com/bparli/lfuda-go v0.3.1/go.mod h1:BR5a9lwlqRqnPhU3F5ojFK3VhTKg8iFVtJJKgZBQhAo=
|
||||
github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
|
||||
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo=
|
||||
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
|
||||
github.com/btcsuite/btcutil v0.0.0-20180524032703-d4cc87b86016/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
|
||||
github.com/btcsuite/btcutil v0.0.0-20180706230648-ab6388e0c60a/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
|
||||
github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803 h1:j3AgPKKZtZStM2nyhrDSLSYgT7YHrZKdSkq1OYeLjvM=
|
||||
github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
|
||||
github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d h1:yJzD/yFppdVCf6ApMkVy8cUxV0XrxdP9rVf6D87/Mng=
|
||||
github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
|
||||
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd h1:R/opQEbFEy9JGkIguV40SvRY1uliPX8ifOvi6ICsFCw=
|
||||
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg=
|
||||
github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY=
|
||||
github.com/btcsuite/goleveldb v1.0.0/go.mod h1:QiK9vBlgftBg6rWQIj6wFzbPfRjiykIEhBH4obrXJ/I=
|
||||
github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
|
||||
github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
|
||||
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3SkEwmHoWBmX1DNXhXZqlTpq6s4tyJGc=
|
||||
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
|
||||
github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
|
||||
github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 h1:o64h9XF42kVEUuhuer2ehqrlX8rZmvQSU0+Vpj1rF6Q=
|
||||
github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61/go.mod h1:Rp8e0DCtEKwXFOC6JPJQVTz8tuGoGvw6Xfexggh/ed0=
|
||||
github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
|
||||
github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2 h1:t8KYCwSKsOEZBFELI4Pn/phbp38iJ1RRAkDFNin1aak=
|
||||
github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M=
|
||||
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
||||
github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
|
||||
github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
|
||||
github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
|
||||
github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
|
||||
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
|
||||
github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
|
||||
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
|
||||
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
|
||||
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
|
||||
github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
|
||||
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
|
||||
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
|
||||
github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
|
||||
github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
|
||||
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
|
||||
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
|
||||
github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g=
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
|
||||
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
|
||||
github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
|
||||
github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
|
||||
github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo=
|
||||
github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
|
||||
github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
|
||||
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
|
||||
github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
|
||||
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
|
||||
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
|
||||
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
|
||||
github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
|
||||
github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
|
||||
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
|
||||
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
|
||||
github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
|
||||
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
|
||||
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
|
||||
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
|
||||
github.com/go-errors/errors v1.1.1 h1:ljK/pL5ltg3qoN+OtN6yCv9HWSfMwxSx90GJCZQxYNg=
|
||||
github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-ini/ini v1.38.2/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-ini/ini v1.41.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-ini/ini v1.42.0 h1:TWr1wGj35+UiWHlBA8er89seFXxzwFn11spilrrj+38=
|
||||
github.com/go-ini/ini v1.42.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-ini/ini v1.48.0 h1:TvO60hO/2xgaaTWp2P0wUe4CFxwdMzfbkv3+343Xzqw=
|
||||
github.com/go-ini/ini v1.48.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
|
||||
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
|
||||
github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
|
||||
github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
|
||||
github.com/go-sql-driver/mysql v0.0.0-20180719071942-99ff426eb706/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
||||
github.com/go-ozzo/ozzo-validation v3.6.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
|
||||
github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
|
||||
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
|
||||
github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE=
|
||||
github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
|
||||
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
|
||||
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
|
||||
github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
|
||||
github.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g=
|
||||
github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
|
||||
github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
|
||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw=
|
||||
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
|
||||
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
|
||||
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
|
||||
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
|
||||
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
|
||||
github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
|
||||
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
|
||||
github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
|
||||
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
|
||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
|
||||
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
|
||||
github.com/google/gops v0.3.7/go.mod h1:bj0cwMmX1X4XIJFTjR99R5sCxNssNJ8HebFNvoQlmgY=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
|
||||
github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20190411002643-bd77b112433e h1:XWcjeEtTFTOVA9Fs1w7n2XBftk5ib4oZrhzWk0B+3eA=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20190411002643-bd77b112433e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f h1:TyqzGm2z1h3AGhjOoRYyeLcW4WlW81MDQkWa+rx/000=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
|
||||
github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk=
|
||||
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
|
||||
github.com/gorilla/rpc v1.1.0 h1:marKfvVP0Gpd/jHlVBKCQ8RAoUPdX7K1Nuh6l1BNh7A=
|
||||
github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
|
||||
github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc=
|
||||
github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
|
||||
github.com/gorilla/rpc v1.1.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ=
|
||||
github.com/gorilla/rpc v1.2.0 h1:WvvdC2lNeT1SP32zrIce5l0ECBfbAlmrmSBsuc57wfk=
|
||||
github.com/gorilla/rpc v1.2.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ=
|
||||
github.com/gorilla/schema v1.0.2/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU=
|
||||
github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
|
||||
github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
|
||||
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
|
||||
github.com/hashicorp/errwrap v0.0.0-20180715044906-d6c0cd880357 h1:Rem2+U35z1QtPQc6r+WolF7yXiefXqDKyk+lN2pE164=
|
||||
github.com/hashicorp/errwrap v0.0.0-20180715044906-d6c0cd880357/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
|
||||
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
|
||||
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
|
||||
github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
|
||||
github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
|
||||
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
|
||||
github.com/hashicorp/go-immutable-radix v0.0.0-20180129170900-7f3cd4390caa h1:0nA8i+6Rwqaq9xlpmVxxTwk6rxiEhX+E6Wh4vPNHiS8=
|
||||
github.com/hashicorp/go-immutable-radix v0.0.0-20180129170900-7f3cd4390caa/go.mod h1:6ij3Z20p+OhOkCSrA0gImAWoHYQRGbnlcuk6XYTiaRw=
|
||||
github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0=
|
||||
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
|
||||
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
|
||||
github.com/hashicorp/go-msgpack v0.0.0-20150518234257-fa3f63826f7c h1:BTAbnbegUIMB6xmQCwWE8yRzbA4XSpnZY5hvRJC188I=
|
||||
github.com/hashicorp/go-msgpack v0.0.0-20150518234257-fa3f63826f7c/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
|
||||
github.com/hashicorp/go-immutable-radix v1.1.0 h1:vN9wG1D6KG6YHRTWr8512cxGOVgTMEfgEdSj/hr8MPc=
|
||||
github.com/hashicorp/go-immutable-radix v1.1.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
|
||||
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
|
||||
github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI=
|
||||
github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
|
||||
github.com/hashicorp/go-multierror v0.0.0-20180717150148-3d5d8f294aa0 h1:j30noezaCfvNLcdMYSvHLv81DxYRSt1grlpseG67vhU=
|
||||
github.com/hashicorp/go-multierror v0.0.0-20180717150148-3d5d8f294aa0/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=
|
||||
github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
|
||||
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
|
||||
github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
|
||||
github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86 h1:7YOlAIO2YWnJZkQp7B5eFykaIY7C9JndqAFQyVV5BhM=
|
||||
github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
|
||||
github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs=
|
||||
github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
|
||||
github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
|
||||
github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc=
|
||||
github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A=
|
||||
github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
|
||||
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE=
|
||||
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
|
||||
github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
|
||||
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
|
||||
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
|
||||
github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
|
||||
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
|
||||
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
|
||||
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
|
||||
github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
|
||||
github.com/hashicorp/memberlist v0.1.0 h1:qSsCiC0WYD39lbSitKNt40e30uorm2Ss/d4JGU1hzH8=
|
||||
github.com/hashicorp/memberlist v0.1.0/go.mod h1:ncdBp14cuox2iFOq3kDiquKU6fqsTBc3W6JvZwjxxsE=
|
||||
github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
|
||||
github.com/hashicorp/memberlist v0.1.4 h1:gkyML/r71w3FL8gUi74Vk76avkj/9lYAY9lvg0OcoGs=
|
||||
github.com/hashicorp/memberlist v0.1.4/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
|
||||
github.com/hashicorp/serf v0.0.0-20180530155958-984a73625de3 h1:NUr1hG6WO9sI1x8ofSimmpqfJ+rEHiHP/PLEA33rcfQ=
|
||||
github.com/hashicorp/serf v0.0.0-20180530155958-984a73625de3/go.mod h1:h/Ru6tmZazX7WO/GDmwdpS975F019L4t5ng5IgwbNrE=
|
||||
github.com/hashicorp/serf v0.8.2 h1:YZ7UKsJv+hKjqGVUUbtE3HNj79Eln2oQ75tniF6iPt0=
|
||||
github.com/hashicorp/memberlist v0.1.5 h1:AYBsgJOW9gab/toO5tEB8lWetVgDKZycqkebJ8xxpqM=
|
||||
github.com/hashicorp/memberlist v0.1.5/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
|
||||
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
|
||||
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
|
||||
github.com/hashicorp/serf v0.8.5 h1:ZynDUIQiA8usmRgPdGPHFdPnb1wgGI9tK3mO9hcAJjc=
|
||||
github.com/hashicorp/serf v0.8.5/go.mod h1:UpNcs7fFbpKIyZaUuSW6EPiH+eZC7OuyFD+wc1oal+k=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf h1:WfD7VjIE6z8dIvMsI4/s+1qr5EL+zoIGev1BQj1eoJ8=
|
||||
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h1:hyb9oH7vZsitZCiBt0ZvifOrB+qc8PS5IiilCIb87rg=
|
||||
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
|
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
|
||||
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
|
||||
github.com/jmoiron/sqlx v0.0.0-20170430194603-d9bd385d68c0/go.mod h1:IiEW3SEiiErVyFdH8NTuWjSifiEQKUoyK3LNqr2kCHU=
|
||||
github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22 h1:jKUP9TQ0c7X3w6+IPyMit07RE42MtTWNd77sN2cHngQ=
|
||||
github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22/go.mod h1:u0Jo4f2dNlTJeeOywkM6bLwxq6gC3pZ9rEFHn3AhTdk=
|
||||
github.com/johntdyer/slackrus v0.0.0-20170926115001-3992f319fd0a/go.mod h1:j1kV/8f3jowErEq4XyeypkCdvg5EeHkf0YCKCcq5Ybo=
|
||||
github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07 h1:+kBG/8rjCa6vxJZbUjAiE4MQmBEBYc8nLEb51frnvBY=
|
||||
github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07/go.mod h1:j1kV/8f3jowErEq4XyeypkCdvg5EeHkf0YCKCcq5Ybo=
|
||||
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
|
||||
github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
|
||||
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
github.com/jteeuwen/go-bindata v3.0.7+incompatible/go.mod h1:JVvhzYOiGBnFSYRyV00iY8q7/0PThjIYav1p9h5dmKs=
|
||||
github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
|
||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
|
||||
github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
|
||||
github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw=
|
||||
github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
|
||||
github.com/keybase/go-ps v0.0.0-20161005175911-668c8856d999/go.mod h1:hY+WOq6m2FpbvyrI93sMaypsttvaIL5nhVR92dTMUcQ=
|
||||
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
|
||||
github.com/kkdai/bstream v0.0.0-20181106074824-b3251f7901ec/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M=
|
||||
github.com/lbryio/chainquery v1.9.0 h1:NfBZ3eKYwD3PqXU/vt+2tF3ox3WUWoW4J5YdEQ0rxw0=
|
||||
github.com/lbryio/chainquery v1.9.0/go.mod h1:7G8l7jNtANS1I7fQOvtzbiHsv6qKVmN4codXHc3C4kk=
|
||||
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8=
|
||||
github.com/lbryio/lbry.go v0.0.0-20190109223729-30c312501602/go.mod h1:YEuFJD/oHNra6BFy+NfuvS84Wg6RMWJFGtiCCCc6MmQ=
|
||||
github.com/lbryio/lbry.go v1.0.15 h1:g4g9cDDUsobmWgTA+1jEIb5k2fRCP0/NvPOMXduP8xY=
|
||||
github.com/lbryio/lbry.go v1.0.15/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002 h1:urfYK5ElpUrAv90auPLldoVC60LwiGAcY0OE6HJB9KI=
|
||||
github.com/lbryio/lbry.go v1.1.1-0.20190825202001-8fa28d3d656f/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
|
||||
github.com/lbryio/lbry.go v1.1.2 h1:Dyxc+glT/rVWJwHfIf7vjlPYYbjzrQz5ARmJd5Hp69c=
|
||||
github.com/lbryio/lbry.go v1.1.2/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
|
||||
github.com/lbryio/lbry.go/v2 v2.7.2-0.20210412222918-ed51ece75c3d h1:VUaOZ3cbCe7gfpycN/srCOk6U2bBS9NZHEz9RiRxd4E=
|
||||
github.com/lbryio/lbry.go/v2 v2.7.2-0.20210412222918-ed51ece75c3d/go.mod h1:I1q8W9fwU+t0IWNiprPgE1SorWQwcO6ser0nzP3L5Pk=
|
||||
github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19 h1:/zWD8dVIl7bV1TdJWqPqy9tpqixzX2Qxgit48h3hQcY=
|
||||
github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
|
||||
github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4=
|
||||
github.com/lbryio/reflector.go v1.0.6-0.20190710201919-7c7ed9da72ce h1:ixsKssly++rkh0DWxNDrZyiRp00NieKZlndkqtpgj/c=
|
||||
github.com/lbryio/reflector.go v1.0.6-0.20190710201919-7c7ed9da72ce/go.mod h1:7Y3YYeAKS6egH2WzwfU8f6+uNGjVHfLzKvwn+Nv3VMY=
|
||||
github.com/lbryio/types v0.0.0-20181001180206-594241d24e00/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
|
||||
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c h1:m3O7561xBQ00lfUVayW4c6SnpVbUDQtPUwGcGYSUYQA=
|
||||
github.com/lbryio/reflector.go v1.1.3-0.20210412225256-4392c9724262 h1:eC5o6fJT9k+RcE0aDwKyPaE4GttUC9fa7uUNNN6mzZg=
|
||||
github.com/lbryio/reflector.go v1.1.3-0.20210412225256-4392c9724262/go.mod h1:4zX+MQt3s9Hi1j01idC+hPWrmcokDSp8VDTIuo9Eodg=
|
||||
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
|
||||
github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5 h1:AsEBgzv3DhuYHI/GiQh2HxvTP71HCCE9E/tzGUzGdtU=
|
||||
github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386 h1:JOQkGpeCM9FWkEHRx+kRPqySPCXElNW1em1++7tVS4M=
|
||||
github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
|
||||
github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
github.com/lucas-clemente/quic-go v0.20.1 h1:hb5m76V8QS/8Nw/suHvXqo3BMHAozvIkcnzpJdpanSk=
|
||||
github.com/lucas-clemente/quic-go v0.20.1/go.mod h1:fZq/HUDIM+mW6X6wtzORjC0E/WDBMKe5Hf9bgjISwLk=
|
||||
github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
|
||||
github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0=
|
||||
github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg=
|
||||
github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2 h1:CTzpAplpuXay3gDDqbErv/OSV6SoKZ9UTsGxLKfxB8M=
|
||||
github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg=
|
||||
github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
|
||||
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 h1:mG83tLXWSRdcXMWfkoumVwhcCbf3jHF9QKv/m37BkM0=
|
||||
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCWffGOaDcjkw1iB7W9DVLp6GXmfcJY/7YZCWPA4=
|
||||
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
|
||||
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
|
||||
github.com/marten-seemann/qpack v0.2.1 h1:jvTsT/HpCn2UZJdP+UUB53FfUUgeOyG5K1ns0OJOGVs=
|
||||
github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc=
|
||||
github.com/marten-seemann/qtls-go1-15 v0.1.4 h1:RehYMOyRW8hPVEja1KBVsFVNSm35Jj9Mvs5yNoZZ28A=
|
||||
github.com/marten-seemann/qtls-go1-15 v0.1.4/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I=
|
||||
github.com/marten-seemann/qtls-go1-16 v0.1.3 h1:XEZ1xGorVy9u+lJq+WXNE+hiqRYLNvJGYmwfwKQN2gU=
|
||||
github.com/marten-seemann/qtls-go1-16 v0.1.3/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk=
|
||||
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
|
||||
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
|
||||
github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
github.com/miekg/dns v1.0.8 h1:Zi8HNpze3NeRWH1PQV6O71YcvJRQ6j0lORO6DAEmAAI=
|
||||
github.com/miekg/dns v1.0.8/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
|
||||
github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA=
|
||||
github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
|
||||
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
|
||||
github.com/miekg/dns v1.1.22 h1:Jm64b3bO9kP43ddLjL2EY3Io6bmy1qGb9Xxz6TqS6rc=
|
||||
github.com/miekg/dns v1.1.22/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
|
||||
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
|
||||
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936 h1:kw1v0NlnN+GZcU8Ma8CLF2Zzgjfx95gs3/GN3vYAPpo=
|
||||
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk=
|
||||
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b h1:9+ke9YJ9KGWw5ANXK6ozjoK47uI3uNbXv4YVINBnGm8=
|
||||
github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk=
|
||||
github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
|
||||
github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
|
||||
github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
|
||||
github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
|
||||
github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/mitchellh/mapstructure v0.0.0-20180511142126-bb74f1db0675/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
|
||||
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/nikooo777/ytdl v0.0.0-20190215151411-9c7832eaf457 h1:S6VzSeZkOv8pKevvKF1isVp9HF294LxVsYw5JrV0FdQ=
|
||||
github.com/nikooo777/ytdl v0.0.0-20190215151411-9c7832eaf457/go.mod h1:Z/EmaCtMz3DxZsLaTnbHjqjZrOH4MHQBvvxtMObRGAY=
|
||||
github.com/nlopes/slack v0.2.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM=
|
||||
github.com/nlopes/slack v0.4.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM=
|
||||
github.com/nlopes/slack v0.5.0 h1:NbIae8Kd0NpqaEI3iUrsuS0KbcEDhzhc939jLW5fNm0=
|
||||
github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
|
||||
github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
|
||||
github.com/nlopes/slack v0.5.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM=
|
||||
github.com/nlopes/slack v0.6.0 h1:jt0jxVQGhssx1Ib7naAOZEZcGdtIhTzkP0nopK0AsRA=
|
||||
github.com/nlopes/slack v0.6.0/go.mod h1:JzQ9m3PMAqcpeCam7UaHSuBuupz7CmpjehYMayT6YOk=
|
||||
github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
|
||||
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
|
||||
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
|
||||
github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
|
||||
github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
|
||||
github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA=
|
||||
github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
|
||||
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
|
||||
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
|
||||
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
|
||||
github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
|
||||
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
|
||||
github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=
|
||||
github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
|
||||
github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
|
||||
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
||||
github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
|
||||
github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
||||
github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
|
||||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||
github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
|
||||
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc=
|
||||
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
|
||||
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/profile v1.3.0/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
|
||||
github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
|
||||
github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
|
||||
github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
|
||||
github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8=
|
||||
github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
|
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
||||
github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
||||
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
|
||||
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
|
||||
github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
|
||||
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
|
||||
github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
|
||||
github.com/prometheus/common v0.3.0 h1:taZ4h8Tkxv2kNyoSctBvfXEHmBmxrwmIidZTIaHons4=
|
||||
github.com/prometheus/common v0.3.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
|
||||
github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM=
|
||||
github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
|
||||
github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
||||
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY=
|
||||
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
|
||||
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
|
||||
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rubenv/sql-migrate v0.0.0-20170330050058-38004e7a77f2/go.mod h1:WS0rl9eEliYI8DPnr3TOwz4439pay+qNgzJoVya/DmY=
|
||||
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
|
||||
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
|
||||
github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
|
||||
github.com/sebdah/goldie v0.0.0-20180424091453-8784dd1ab561 h1:IY+sDBJR/wRtsxq+626xJnt4Tw7/ROA9cDIR8MMhWyg=
|
||||
github.com/sebdah/goldie v0.0.0-20180424091453-8784dd1ab561/go.mod h1:lvjGftC8oe7XPtyrOidaMi0rp5B9+XY/ZRUynGnuaxQ=
|
||||
github.com/sebdah/goldie v0.0.0-20190531093107-d313ffb52c77 h1:Msb6XRY62jQOueNNlB5LGin1rljK7c49NLniGwJG2bg=
|
||||
github.com/sebdah/goldie v0.0.0-20190531093107-d313ffb52c77/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4=
|
||||
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/sfreiberg/gotwilio v0.0.0-20180612161623-8fb7259ba8bf/go.mod h1:60PiR0SAnAcYSiwrXB6BaxeqHdXMf172toCosHfV+Yk=
|
||||
github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
|
||||
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
|
||||
github.com/shopspring/decimal v0.0.0-20180607144847-19e3cb6c2930/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
|
||||
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 h1:pntxY8Ary0t43dCZ5dqY4YTJCObLY1kIXl0uzMv+7DE=
|
||||
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
|
||||
github.com/sirupsen/logrus v0.0.0-20180523074243-ea8897e79973/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
|
||||
github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5 h1:Gojs/hac/DoYEM7WEICT45+hNWczIeuL5D21e5/HPAw=
|
||||
github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
|
||||
github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY=
|
||||
github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM=
|
||||
github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0=
|
||||
github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
|
||||
github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
|
||||
github.com/shurcooL/gofontwoff v0.0.0-20180329035133-29b52fc0a18d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw=
|
||||
github.com/shurcooL/gopherjslib v0.0.0-20160914041154-feb6d3990c2c/go.mod h1:8d3azKNyqcHP1GaQE/c6dDgjkgSx2BZ4IoEi4F1reUI=
|
||||
github.com/shurcooL/highlight_diff v0.0.0-20170515013008-09bb4053de1b/go.mod h1:ZpfEhSmds4ytuByIcDnOLkTHGUI6KNqRNPDLHDk+mUU=
|
||||
github.com/shurcooL/highlight_go v0.0.0-20181028180052-98c3abbbae20/go.mod h1:UDKB5a1T23gOMUJrI+uSuH0VRDStOiUVSjBTRDVBVag=
|
||||
github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg=
|
||||
github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw=
|
||||
github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y=
|
||||
github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg=
|
||||
github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q=
|
||||
github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ=
|
||||
github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I=
|
||||
github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b5uSkrEVM1jQUspwbixRBhaIjIzL2xazXp6kntxYle0=
|
||||
github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ=
|
||||
github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk=
|
||||
github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
|
||||
github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4=
|
||||
github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k=
|
||||
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
|
||||
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3 h1:hBSHahWMEgzwRyS6dRpxY0XyjZsHyQ61s084wo5PJe0=
|
||||
github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w=
|
||||
github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
|
||||
github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
|
||||
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs=
|
||||
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||
github.com/spf13/cast v1.2.0 h1:HHl1DSRbEQN2i8tJmtS6ViPyHx35+p51amrdsiTCrkg=
|
||||
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
|
||||
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
|
||||
github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE=
|
||||
github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA=
|
||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/spf13/afero v1.1.1/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
|
||||
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
|
||||
github.com/spf13/afero v1.4.1 h1:asw9sl74539yqavKaglDM5hFpdJVK0Y5Dr/JOgQ89nQ=
|
||||
github.com/spf13/afero v1.4.1/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
|
||||
github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
|
||||
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
|
||||
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
|
||||
github.com/spf13/cobra v0.0.0-20180722215644-7c4570c3ebeb/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
|
||||
github.com/spf13/cobra v0.0.0-20190109003409-7547e83b2d85 h1:UQHWkFUuJBy5rWN1DxosG/efssLu7u0fXXSTC2HHKfQ=
|
||||
github.com/spf13/cobra v0.0.0-20190109003409-7547e83b2d85/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
|
||||
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
|
||||
github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
|
||||
github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
|
||||
github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
|
||||
github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
|
||||
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
|
||||
github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
|
||||
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
|
||||
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
|
||||
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
|
||||
github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk=
|
||||
github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
|
||||
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
|
||||
github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
|
||||
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
|
||||
github.com/uber-go/atomic v1.3.2 h1:Azu9lPBWRNKzYXSIwRfgRuDuS0YKsK4NFhiQv98gkxo=
|
||||
github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
|
||||
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
|
||||
github.com/vbauerster/mpb/v7 v7.0.2 h1:eN6AD/ytv1nqCO7Dm8MO0/pGMKmMyH/WMnTJhAUuc/w=
|
||||
github.com/vbauerster/mpb/v7 v7.0.2/go.mod h1:Mnq3gESXJ9eQhccbGZDggJ1faTCrmaA4iN57fUloRGE=
|
||||
github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
|
||||
github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
|
||||
github.com/volatiletech/inflect v0.0.0-20170731032912-e7201282ae8d h1:gI4/tqP6lCY5k6Sg+4k9qSoBXmPwG+xXgMpK7jivD4M=
|
||||
github.com/volatiletech/inflect v0.0.0-20170731032912-e7201282ae8d/go.mod h1:jspfvgf53t5NLUT4o9L1IX0kIBNKamGq1tWc/MgWK9Q=
|
||||
github.com/volatiletech/null v8.0.0+incompatible h1:7wP8m5d/gZ6kW/9GnrLtMCRre2dlEnaQ9Km5OXlK4zg=
|
||||
github.com/volatiletech/null v8.0.0+incompatible/go.mod h1:0wD98JzdqB+rLyZ70fN05VDbXbafIb0KU0MdVhCzmOQ=
|
||||
github.com/volatiletech/sqlboiler v3.4.0+incompatible h1:saQ6WxZ9wEJp33q3w/DHs7an7SYi1H7Yzf4/moxCbJU=
|
||||
github.com/volatiletech/sqlboiler v3.4.0+incompatible/go.mod h1:jLfDkkHWPbS2cWRLkyC20vQWaIQsASEY7gM7zSo11Yw=
|
||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
|
||||
github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
|
||||
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
|
||||
github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d h1:tQo6hjclyv3RHUgZOl6iWb2Y44A/sN9bf9LAYfuioEg=
|
||||
github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE=
|
||||
github.com/ybbus/jsonrpc v2.1.2+incompatible h1:V4mkE9qhbDQ92/MLMIhlhMSbz8jNXdagC3xBR5NDwaQ=
|
||||
github.com/ybbus/jsonrpc v2.1.2+incompatible/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE=
|
||||
go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
|
||||
go.opencensus.io v0.20.2 h1:NAfh7zF0/3/HqtMvJNZ/RFrSlCE6ZTlHmKfhL/Dm1Jk=
|
||||
go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
|
||||
go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4=
|
||||
github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0=
|
||||
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
|
||||
go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
|
||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
|
||||
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
go.uber.org/atomic v1.5.1 h1:rsqfU5vBkVknbhUGbAUwQKR2H4ItV8tjJ+6kJX4cxHM=
|
||||
go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
|
||||
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
|
||||
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
|
||||
go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
|
||||
golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw=
|
||||
golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480 h1:O5YqonU5IWby+w98jVUG9h7zlCWCcH4RHyPVReBmhzk=
|
||||
golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
|
||||
golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
|
||||
golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
|
||||
golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20181217174547-8f45f776aaf1/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE=
|
||||
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
|
||||
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
|
||||
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
|
||||
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
|
||||
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
|
||||
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c h1:uOCk1iQW6Vc18bnC13MfzScl+wdKBmM9Y9kU7Z83/lw=
|
||||
golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20191009170851-d66e71096ffb/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
|
||||
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a h1:tImsplftrFpALCYumobsd0K86vlAs/eXGFms2txfJfA=
|
||||
golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI=
|
||||
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20171017063910-8dbc5d05d6ed/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190520201301-c432e742b0af h1:NXfmMfXz6JqGfG3ikSxcz2N93j6DgScr19Oo2uwFu88=
|
||||
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190520201301-c432e742b0af/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201231184435-2d18734c6014/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210603125802-9665404d3644 h1:CA1DEQ4NdKphKeL70tvsWNdT5oFh1lOjihRcEDROi0I=
|
||||
golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=
|
||||
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 h1:xQwXv67TxFo9nC1GJFyab5eq/5B590r6RlnL/G8Sz7w=
|
||||
golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190109165630-d30e00c24034/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
|
||||
google.golang.org/api v0.3.2 h1:iTp+3yyl/KOtxa/d1/JUE0GGSoR6FuW5udver22iwpw=
|
||||
google.golang.org/api v0.3.2/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
|
||||
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4 h1:Toz2IK7k8rbltAXwNAxKcn9OzqyNfMUhUNjz3sL0NMk=
|
||||
golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
|
||||
google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
|
||||
google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y=
|
||||
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
|
||||
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
|
||||
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
|
||||
google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
|
||||
google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
|
||||
google.golang.org/appengine v1.6.2/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
|
||||
google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
|
||||
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190108161440-ae2f86662275/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
|
||||
google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
|
||||
google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7 h1:ZUjXAXmrAyrmmCPHgCA/vChHcpsX27MZ3yBonD/z1KE=
|
||||
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||
google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
|
||||
google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
|
||||
google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
|
||||
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.0 h1:DlsSIrgEBuZAUFJcta2B5i/lzeHHbnfkNFAfFXLVFYQ=
|
||||
google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=
|
||||
gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
|
||||
google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
|
||||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
|
||||
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
|
||||
google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
|
||||
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
|
||||
gopkg.in/gorp.v1 v1.7.1/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw=
|
||||
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
|
||||
gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk=
|
||||
gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/ini.v1 v1.48.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/ini.v1 v1.60.2 h1:7i8mqModL63zqi8nQn8Q3+0zvSCZy1AxhBgthKfi4WU=
|
||||
gopkg.in/ini.v1 v1.60.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 h1:FpCr9V8wuOei4BAen+93HtVJ+XSi+KPbaPKm0Vj5R64=
|
||||
gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79/go.mod h1:gWkaRU7CoXpezCBWfWjm3999QqS+1pYPXGbqQCTMzo8=
|
||||
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/vansante/go-ffprobe.v2 v2.0.2 h1:DdxSfFnlqeawPIVbIQEI6LR6OQHQNR7tNgWb2mWuC4w=
|
||||
gopkg.in/vansante/go-ffprobe.v2 v2.0.2/go.mod h1:qF0AlAjk7Nqzqf3y333Ly+KxN3cKF2JqA3JT5ZheUGE=
|
||||
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
|
||||
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
|
||||
gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
|
||||
grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o=
|
||||
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190109154334-5bcec433c8ea/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
|
||||
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
|
||||
rsc.io/goversion v1.0.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo=
|
||||
sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck=
|
||||
sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
|
||||
@@ -0,0 +1,227 @@
|
||||
package ip_manager
|
||||
|
||||
import (
|
||||
"net"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/asaskevich/govalidator"
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/ytsync/v5/util"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const IPCooldownPeriod = 20 * time.Second
|
||||
const unbanTimeout = 48 * time.Hour
|
||||
|
||||
var stopper = stop.New()
|
||||
|
||||
type IPPool struct {
|
||||
ips []throttledIP
|
||||
lock *sync.RWMutex
|
||||
stopGrp *stop.Group
|
||||
}
|
||||
|
||||
type throttledIP struct {
|
||||
IP string
|
||||
UsedForVideo string
|
||||
LastUse time.Time
|
||||
Throttled bool
|
||||
InUse bool
|
||||
}
|
||||
|
||||
var ipPoolInstance *IPPool
|
||||
|
||||
func GetIPPool(stopGrp *stop.Group) (*IPPool, error) {
|
||||
if ipPoolInstance != nil {
|
||||
return ipPoolInstance, nil
|
||||
}
|
||||
addrs, err := net.InterfaceAddrs()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
var pool []throttledIP
|
||||
for _, address := range addrs {
|
||||
if ipnet, ok := address.(*net.IPNet); ok && ipnet.IP.IsGlobalUnicast() {
|
||||
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) {
|
||||
pool = append(pool, throttledIP{
|
||||
IP: ipnet.IP.String(),
|
||||
LastUse: time.Now().Add(-5 * time.Minute),
|
||||
})
|
||||
} else if ipnet.IP.To4() != nil && govalidator.IsIPv4(ipnet.IP.String()) {
|
||||
pool = append(pool, throttledIP{
|
||||
IP: ipnet.IP.String(),
|
||||
LastUse: time.Now().Add(-5 * time.Minute),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
ipPoolInstance = &IPPool{
|
||||
ips: pool,
|
||||
lock: &sync.RWMutex{},
|
||||
stopGrp: stopGrp,
|
||||
}
|
||||
//ticker := time.NewTicker(10 * time.Second)
|
||||
//go func() {
|
||||
// for {
|
||||
// select {
|
||||
// case <-stopGrp.Ch():
|
||||
// return
|
||||
// case <-ticker.C:
|
||||
// ipPoolInstance.lock.RLock()
|
||||
// for _, ip := range ipPoolInstance.ips {
|
||||
// log.Debugf("IP: %s\tInUse: %t\tVideoID: %s\tThrottled: %t\tLastUse: %.1f", ip.IP, ip.InUse, ip.UsedForVideo, ip.Throttled, time.Since(ip.LastUse).Seconds())
|
||||
// }
|
||||
// ipPoolInstance.lock.RUnlock()
|
||||
// }
|
||||
// }
|
||||
//}()
|
||||
return ipPoolInstance, nil
|
||||
}
|
||||
|
||||
// AllThrottled checks whether the IPs provided are all throttled.
|
||||
// returns false if at least one IP is not throttled
|
||||
// Not thread safe, should use locking when called
|
||||
func AllThrottled(ips []throttledIP) bool {
|
||||
for _, i := range ips {
|
||||
if !i.Throttled {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// AllInUse checks whether the IPs provided are all currently in use.
|
||||
// returns false if at least one IP is not in use AND is not throttled
|
||||
// Not thread safe, should use locking when called
|
||||
func AllInUse(ips []throttledIP) bool {
|
||||
for _, i := range ips {
|
||||
if !i.InUse && !i.Throttled {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (i *IPPool) ReleaseIP(ip string) {
|
||||
i.lock.Lock()
|
||||
defer i.lock.Unlock()
|
||||
for j := range i.ips {
|
||||
localIP := &i.ips[j]
|
||||
if localIP.IP == ip {
|
||||
localIP.InUse = false
|
||||
localIP.LastUse = time.Now()
|
||||
return
|
||||
}
|
||||
}
|
||||
util.SendErrorToSlack("something went wrong while releasing the IP %s as we reached the end of the function", ip)
|
||||
}
|
||||
|
||||
func (i *IPPool) ReleaseAll() {
|
||||
i.lock.Lock()
|
||||
defer i.lock.Unlock()
|
||||
for j := range i.ips {
|
||||
if i.ips[j].Throttled {
|
||||
continue
|
||||
}
|
||||
localIP := &i.ips[j]
|
||||
localIP.InUse = false
|
||||
}
|
||||
}
|
||||
|
||||
func (i *IPPool) SetThrottled(ip string) {
|
||||
i.lock.Lock()
|
||||
defer i.lock.Unlock()
|
||||
var tIP *throttledIP
|
||||
for j, _ := range i.ips {
|
||||
localIP := &i.ips[j]
|
||||
if localIP.IP == ip {
|
||||
if localIP.Throttled {
|
||||
return
|
||||
}
|
||||
localIP.Throttled = true
|
||||
tIP = localIP
|
||||
break
|
||||
}
|
||||
}
|
||||
util.SendErrorToSlack("%s set to throttled", ip)
|
||||
|
||||
stopper.Add(1)
|
||||
go func(tIP *throttledIP) {
|
||||
defer stopper.Done()
|
||||
unbanTimer := time.NewTimer(unbanTimeout)
|
||||
select {
|
||||
case <-unbanTimer.C:
|
||||
i.lock.Lock()
|
||||
tIP.Throttled = false
|
||||
i.lock.Unlock()
|
||||
util.SendInfoToSlack("%s set back to not throttled", ip)
|
||||
case <-i.stopGrp.Ch():
|
||||
unbanTimer.Stop()
|
||||
}
|
||||
}(tIP)
|
||||
}
|
||||
|
||||
var ErrAllInUse = errors.Base("all IPs are in use, try again")
|
||||
var ErrAllThrottled = errors.Base("all IPs are throttled")
|
||||
var ErrResourceLock = errors.Base("error getting next ip, did you forget to lock on the resource?")
|
||||
var ErrInterruptedByUser = errors.Base("interrupted by user")
|
||||
|
||||
func (i *IPPool) nextIP(forVideo string) (*throttledIP, error) {
|
||||
i.lock.Lock()
|
||||
defer i.lock.Unlock()
|
||||
|
||||
sort.Slice(i.ips, func(j, k int) bool {
|
||||
return i.ips[j].LastUse.Before(i.ips[k].LastUse)
|
||||
})
|
||||
|
||||
if !AllThrottled(i.ips) {
|
||||
if AllInUse(i.ips) {
|
||||
return nil, errors.Err(ErrAllInUse)
|
||||
}
|
||||
|
||||
var nextIP *throttledIP
|
||||
for j := range i.ips {
|
||||
ip := &i.ips[j]
|
||||
if ip.InUse || ip.Throttled {
|
||||
continue
|
||||
}
|
||||
nextIP = ip
|
||||
break
|
||||
}
|
||||
if nextIP == nil {
|
||||
return nil, errors.Err(ErrResourceLock)
|
||||
}
|
||||
nextIP.InUse = true
|
||||
nextIP.UsedForVideo = forVideo
|
||||
return nextIP, nil
|
||||
}
|
||||
return nil, errors.Err(ErrAllThrottled)
|
||||
}
|
||||
|
||||
func (i *IPPool) GetIP(forVideo string) (string, error) {
|
||||
for {
|
||||
ip, err := i.nextIP(forVideo)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrAllInUse) {
|
||||
select {
|
||||
case <-i.stopGrp.Ch():
|
||||
return "", errors.Err(ErrInterruptedByUser)
|
||||
default:
|
||||
time.Sleep(5 * time.Second)
|
||||
continue
|
||||
}
|
||||
} else if errors.Is(err, ErrAllThrottled) {
|
||||
return "throttled", err
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
if time.Since(ip.LastUse) < IPCooldownPeriod {
|
||||
log.Debugf("The IP %s is too hot, waiting for %.1f seconds before continuing", ip.IP, (IPCooldownPeriod - time.Since(ip.LastUse)).Seconds())
|
||||
time.Sleep(IPCooldownPeriod - time.Since(ip.LastUse))
|
||||
}
|
||||
return ip.IP, nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package ip_manager
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAll(t *testing.T) {
|
||||
pool, err := GetIPPool()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ip, err := pool.GetIP()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log(ip)
|
||||
pool.ReleaseIP(ip)
|
||||
ip2, err := pool.GetIP()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ip == ip2 && len(pool.ips) > 1 {
|
||||
t.Fatalf("the same IP was returned twice! %s, %s", ip, ip2)
|
||||
}
|
||||
t.Log(ip2)
|
||||
pool.ReleaseIP(ip2)
|
||||
|
||||
for range pool.ips {
|
||||
_, err = pool.GetIP()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
next, err := pool.nextIP()
|
||||
if err != nil {
|
||||
t.Logf("%s", err.Error())
|
||||
} else {
|
||||
t.Fatal(next)
|
||||
}
|
||||
}
|
||||
@@ -3,16 +3,20 @@ package main
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/user"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/extras/util"
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
"github.com/lbryio/ytsync/v5/manager"
|
||||
"github.com/lbryio/ytsync/v5/sdk"
|
||||
"github.com/lbryio/ytsync/v5/shared"
|
||||
ytUtils "github.com/lbryio/ytsync/v5/util"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
|
||||
"github.com/lbryio/ytsync/manager"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var Version string
|
||||
@@ -20,30 +24,17 @@ var Version string
|
||||
const defaultMaxTries = 3
|
||||
|
||||
var (
|
||||
stopOnError bool
|
||||
maxTries int
|
||||
takeOverExistingChannel bool
|
||||
refill int
|
||||
limit int
|
||||
skipSpaceCheck bool
|
||||
syncUpdate bool
|
||||
singleRun bool
|
||||
syncStatus string
|
||||
channelID string
|
||||
syncFrom int64
|
||||
syncUntil int64
|
||||
concurrentJobs int
|
||||
videosLimit int
|
||||
maxVideoSize int
|
||||
maxVideoLength float64
|
||||
removeDBUnpublished bool
|
||||
upgradeMetadata bool
|
||||
cliFlags shared.SyncFlags
|
||||
maxVideoLength int
|
||||
)
|
||||
|
||||
func main() {
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
log.SetLevel(log.DebugLevel)
|
||||
|
||||
http.Handle("/metrics", promhttp.Handler())
|
||||
go func() {
|
||||
log.Error(http.ListenAndServe(":2112", nil))
|
||||
}()
|
||||
cmd := &cobra.Command{
|
||||
Use: "ytsync",
|
||||
Short: "Publish youtube channels into LBRY network automatically.",
|
||||
@@ -51,23 +42,25 @@ func main() {
|
||||
Args: cobra.RangeArgs(0, 0),
|
||||
}
|
||||
|
||||
cmd.Flags().BoolVar(&stopOnError, "stop-on-error", false, "If a publish fails, stop all publishing and exit")
|
||||
cmd.Flags().IntVar(&maxTries, "max-tries", defaultMaxTries, "Number of times to try a publish that fails")
|
||||
cmd.Flags().BoolVar(&takeOverExistingChannel, "takeover-existing-channel", false, "If channel exists and we don't own it, take over the channel")
|
||||
cmd.Flags().IntVar(&limit, "limit", 0, "limit the amount of channels to sync")
|
||||
cmd.Flags().BoolVar(&skipSpaceCheck, "skip-space-check", false, "Do not perform free space check on startup")
|
||||
cmd.Flags().BoolVar(&syncUpdate, "update", false, "Update previously synced channels instead of syncing new ones")
|
||||
cmd.Flags().BoolVar(&singleRun, "run-once", false, "Whether the process should be stopped after one cycle or not")
|
||||
cmd.Flags().BoolVar(&removeDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published")
|
||||
cmd.Flags().BoolVar(&upgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version")
|
||||
cmd.Flags().StringVar(&syncStatus, "status", "", "Specify which queue to pull from. Overrides --update")
|
||||
cmd.Flags().StringVar(&channelID, "channelID", "", "If specified, only this channel will be synced.")
|
||||
cmd.Flags().Int64Var(&syncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)")
|
||||
cmd.Flags().Int64Var(&syncUntil, "before", time.Now().AddDate(1, 0, 0).Unix(), "Specify until when to pull jobs [Unix time](Default: current Unix time)")
|
||||
cmd.Flags().IntVar(&concurrentJobs, "concurrent-jobs", 1, "how many jobs to process concurrently")
|
||||
cmd.Flags().IntVar(&videosLimit, "videos-limit", 1000, "how many videos to process per channel")
|
||||
cmd.Flags().IntVar(&maxVideoSize, "max-size", 2048, "Maximum video size to process (in MB)")
|
||||
cmd.Flags().Float64Var(&maxVideoLength, "max-length", 2.0, "Maximum video length to process (in hours)")
|
||||
cmd.Flags().IntVar(&cliFlags.MaxTries, "max-tries", defaultMaxTries, "Number of times to try a publish that fails")
|
||||
cmd.Flags().BoolVar(&cliFlags.TakeOverExistingChannel, "takeover-existing-channel", false, "If channel exists and we don't own it, take over the channel")
|
||||
cmd.Flags().IntVar(&cliFlags.Limit, "limit", 0, "limit the amount of channels to sync")
|
||||
cmd.Flags().BoolVar(&cliFlags.SkipSpaceCheck, "skip-space-check", false, "Do not perform free space check on startup")
|
||||
cmd.Flags().BoolVar(&cliFlags.SyncUpdate, "update", false, "Update previously synced channels instead of syncing new ones")
|
||||
cmd.Flags().BoolVar(&cliFlags.SingleRun, "run-once", false, "Whether the process should be stopped after one cycle or not")
|
||||
cmd.Flags().BoolVar(&cliFlags.RemoveDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published")
|
||||
cmd.Flags().BoolVar(&cliFlags.UpgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version")
|
||||
cmd.Flags().BoolVar(&cliFlags.DisableTransfers, "no-transfers", false, "Skips the transferring process of videos, channels and supports")
|
||||
cmd.Flags().BoolVar(&cliFlags.QuickSync, "quick", false, "Look up only the last 50 videos from youtube")
|
||||
cmd.Flags().StringVar(&cliFlags.Status, "status", "", "Specify which queue to pull from. Overrides --update")
|
||||
cmd.Flags().StringVar(&cliFlags.SecondaryStatus, "status2", "", "Specify which secondary queue to pull from.")
|
||||
cmd.Flags().StringVar(&cliFlags.ChannelID, "channelID", "", "If specified, only this channel will be synced.")
|
||||
cmd.Flags().Int64Var(&cliFlags.SyncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)")
|
||||
cmd.Flags().Int64Var(&cliFlags.SyncUntil, "before", time.Now().AddDate(1, 0, 0).Unix(), "Specify until when to pull jobs [Unix time](Default: current Unix time)")
|
||||
cmd.Flags().IntVar(&cliFlags.ConcurrentJobs, "concurrent-jobs", 1, "how many jobs to process concurrently")
|
||||
cmd.Flags().IntVar(&cliFlags.VideosLimit, "videos-limit", 0, "how many videos to process per channel (leave 0 for automatic detection)")
|
||||
cmd.Flags().IntVar(&cliFlags.MaxVideoSize, "max-size", 2048, "Maximum video size to process (in MB)")
|
||||
cmd.Flags().IntVar(&maxVideoLength, "max-length", 2, "Maximum video length to process (in hours)")
|
||||
|
||||
if err := cmd.Execute(); err != nil {
|
||||
fmt.Println(err)
|
||||
@@ -87,33 +80,33 @@ func ytSync(cmd *cobra.Command, args []string) {
|
||||
log.Error("could not detect system hostname")
|
||||
hostname = "ytsync-unknown"
|
||||
}
|
||||
if len(hostname) > 30 {
|
||||
hostname = hostname[0:30]
|
||||
}
|
||||
|
||||
util.InitSlack(os.Getenv("SLACK_TOKEN"), os.Getenv("SLACK_CHANNEL"), hostname)
|
||||
}
|
||||
|
||||
if syncStatus != "" && !util.InSlice(syncStatus, manager.SyncStatuses) {
|
||||
log.Errorf("status must be one of the following: %v\n", manager.SyncStatuses)
|
||||
if cliFlags.Status != "" && !util.InSlice(cliFlags.Status, shared.SyncStatuses) {
|
||||
log.Errorf("status must be one of the following: %v\n", shared.SyncStatuses)
|
||||
return
|
||||
}
|
||||
|
||||
if stopOnError && maxTries != defaultMaxTries {
|
||||
log.Errorln("--stop-on-error and --max-tries are mutually exclusive")
|
||||
return
|
||||
}
|
||||
if maxTries < 1 {
|
||||
if cliFlags.MaxTries < 1 {
|
||||
log.Errorln("setting --max-tries less than 1 doesn't make sense")
|
||||
return
|
||||
}
|
||||
|
||||
if limit < 0 {
|
||||
if cliFlags.Limit < 0 {
|
||||
log.Errorln("setting --limit less than 0 (unlimited) doesn't make sense")
|
||||
return
|
||||
}
|
||||
cliFlags.MaxVideoLength = time.Duration(maxVideoLength) * time.Hour
|
||||
|
||||
apiURL := os.Getenv("LBRY_WEB_API")
|
||||
apiToken := os.Getenv("LBRY_API_TOKEN")
|
||||
youtubeAPIKey := os.Getenv("YOUTUBE_API_KEY")
|
||||
blobsDir := os.Getenv("BLOBS_DIRECTORY")
|
||||
lbrycrdString := os.Getenv("LBRYCRD_STRING")
|
||||
lbrycrdDsn := os.Getenv("LBRYCRD_STRING")
|
||||
awsS3ID := os.Getenv("AWS_S3_ID")
|
||||
awsS3Secret := os.Getenv("AWS_S3_SECRET")
|
||||
awsS3Region := os.Getenv("AWS_S3_REGION")
|
||||
@@ -146,58 +139,34 @@ func ytSync(cmd *cobra.Command, args []string) {
|
||||
log.Errorln("AWS S3 Bucket was not defined. Please set the environment variable AWS_S3_BUCKET")
|
||||
return
|
||||
}
|
||||
if lbrycrdString == "" {
|
||||
if lbrycrdDsn == "" {
|
||||
log.Infoln("Using default (local) lbrycrd instance. Set LBRYCRD_STRING if you want to use something else")
|
||||
}
|
||||
if blobsDir == "" {
|
||||
usr, err := user.Current()
|
||||
if err != nil {
|
||||
log.Errorln(err.Error())
|
||||
return
|
||||
}
|
||||
blobsDir = usr.HomeDir + "/.lbrynet/blobfiles/"
|
||||
}
|
||||
|
||||
syncProperties := &sdk.SyncProperties{
|
||||
SyncFrom: syncFrom,
|
||||
SyncUntil: syncUntil,
|
||||
YoutubeChannelID: channelID,
|
||||
}
|
||||
blobsDir := ytUtils.GetBlobsDir()
|
||||
|
||||
apiConfig := &sdk.APIConfig{
|
||||
YoutubeAPIKey: youtubeAPIKey,
|
||||
ApiURL: apiURL,
|
||||
ApiToken: apiToken,
|
||||
HostName: hostname,
|
||||
}
|
||||
awsConfig := &shared.AwsConfigs{
|
||||
AwsS3ID: awsS3ID,
|
||||
AwsS3Secret: awsS3Secret,
|
||||
AwsS3Region: awsS3Region,
|
||||
AwsS3Bucket: awsS3Bucket,
|
||||
}
|
||||
sm := manager.NewSyncManager(
|
||||
stopOnError,
|
||||
maxTries,
|
||||
takeOverExistingChannel,
|
||||
refill,
|
||||
limit,
|
||||
skipSpaceCheck,
|
||||
syncUpdate,
|
||||
concurrentJobs,
|
||||
concurrentJobs,
|
||||
cliFlags,
|
||||
blobsDir,
|
||||
videosLimit,
|
||||
maxVideoSize,
|
||||
lbrycrdString,
|
||||
awsS3ID,
|
||||
awsS3Secret,
|
||||
awsS3Region,
|
||||
awsS3Bucket,
|
||||
syncStatus,
|
||||
singleRun,
|
||||
syncProperties,
|
||||
lbrycrdDsn,
|
||||
awsConfig,
|
||||
apiConfig,
|
||||
maxVideoLength,
|
||||
removeDBUnpublished,
|
||||
upgradeMetadata,
|
||||
)
|
||||
err := sm.Start()
|
||||
if err != nil {
|
||||
manager.SendErrorToSlack(err.Error())
|
||||
ytUtils.SendErrorToSlack(errors.FullTrace(err))
|
||||
}
|
||||
manager.SendInfoToSlack("Syncing process terminated!")
|
||||
ytUtils.SendInfoToSlack("Syncing process terminated!")
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
package manager
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
|
||||
"google.golang.org/api/googleapi/transport"
|
||||
"google.golang.org/api/youtube/v3"
|
||||
)
|
||||
|
||||
func (s *Sync) CountVideos() (uint64, error) {
|
||||
client := &http.Client{
|
||||
Transport: &transport.APIKey{Key: s.APIConfig.YoutubeAPIKey},
|
||||
}
|
||||
|
||||
service, err := youtube.New(client)
|
||||
if err != nil {
|
||||
return 0, errors.Prefix("error creating YouTube service", err)
|
||||
}
|
||||
|
||||
response, err := service.Channels.List("statistics").Id(s.YoutubeChannelID).Do()
|
||||
if err != nil {
|
||||
return 0, errors.Prefix("error getting channels", err)
|
||||
}
|
||||
|
||||
if len(response.Items) < 1 {
|
||||
return 0, errors.Err("youtube channel not found")
|
||||
}
|
||||
|
||||
return response.Items[0].Statistics.VideoCount, nil
|
||||
}
|
||||
+100
-158
@@ -3,194 +3,137 @@ package manager
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/ytsync/blobs_reflector"
|
||||
"github.com/lbryio/ytsync/namer"
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
"github.com/lbryio/ytsync/v5/blobs_reflector"
|
||||
"github.com/lbryio/ytsync/v5/ip_manager"
|
||||
"github.com/lbryio/ytsync/v5/namer"
|
||||
"github.com/lbryio/ytsync/v5/sdk"
|
||||
"github.com/lbryio/ytsync/v5/shared"
|
||||
logUtils "github.com/lbryio/ytsync/v5/util"
|
||||
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
"github.com/lbryio/lbry.go/extras/util"
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type SyncManager struct {
|
||||
stopOnError bool
|
||||
maxTries int
|
||||
takeOverExistingChannel bool
|
||||
refill int
|
||||
limit int
|
||||
skipSpaceCheck bool
|
||||
syncUpdate bool
|
||||
concurrentJobs int
|
||||
concurrentVideos int
|
||||
blobsDir string
|
||||
videosLimit int
|
||||
maxVideoSize int
|
||||
maxVideoLength float64
|
||||
lbrycrdString string
|
||||
awsS3ID string
|
||||
awsS3Secret string
|
||||
awsS3Region string
|
||||
syncStatus string
|
||||
awsS3Bucket string
|
||||
singleRun bool
|
||||
syncProperties *sdk.SyncProperties
|
||||
apiConfig *sdk.APIConfig
|
||||
removeDBUnpublished bool
|
||||
upgradeMetadata bool
|
||||
CliFlags shared.SyncFlags
|
||||
ApiConfig *sdk.APIConfig
|
||||
LbrycrdDsn string
|
||||
AwsConfigs *shared.AwsConfigs
|
||||
|
||||
blobsDir string
|
||||
channelsToSync []Sync
|
||||
}
|
||||
|
||||
func NewSyncManager(stopOnError bool, maxTries int, takeOverExistingChannel bool, refill int, limit int,
|
||||
skipSpaceCheck bool, syncUpdate bool, concurrentJobs int, concurrentVideos int, blobsDir string, videosLimit int,
|
||||
maxVideoSize int, lbrycrdString string, awsS3ID string, awsS3Secret string, awsS3Region string, awsS3Bucket string,
|
||||
syncStatus string, singleRun bool, syncProperties *sdk.SyncProperties, apiConfig *sdk.APIConfig, maxVideoLength float64, removeDBUnpublished bool, upgradeMetadata bool) *SyncManager {
|
||||
func NewSyncManager(cliFlags shared.SyncFlags, blobsDir, lbrycrdDsn string, awsConfigs *shared.AwsConfigs, apiConfig *sdk.APIConfig) *SyncManager {
|
||||
return &SyncManager{
|
||||
stopOnError: stopOnError,
|
||||
maxTries: maxTries,
|
||||
takeOverExistingChannel: takeOverExistingChannel,
|
||||
refill: refill,
|
||||
limit: limit,
|
||||
skipSpaceCheck: skipSpaceCheck,
|
||||
syncUpdate: syncUpdate,
|
||||
concurrentJobs: concurrentJobs,
|
||||
concurrentVideos: concurrentVideos,
|
||||
blobsDir: blobsDir,
|
||||
videosLimit: videosLimit,
|
||||
maxVideoSize: maxVideoSize,
|
||||
maxVideoLength: maxVideoLength,
|
||||
lbrycrdString: lbrycrdString,
|
||||
awsS3ID: awsS3ID,
|
||||
awsS3Secret: awsS3Secret,
|
||||
awsS3Region: awsS3Region,
|
||||
awsS3Bucket: awsS3Bucket,
|
||||
syncStatus: syncStatus,
|
||||
singleRun: singleRun,
|
||||
syncProperties: syncProperties,
|
||||
apiConfig: apiConfig,
|
||||
removeDBUnpublished: removeDBUnpublished,
|
||||
upgradeMetadata: upgradeMetadata,
|
||||
CliFlags: cliFlags,
|
||||
blobsDir: blobsDir,
|
||||
LbrycrdDsn: lbrycrdDsn,
|
||||
AwsConfigs: awsConfigs,
|
||||
ApiConfig: apiConfig,
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
StatusPending = "pending" // waiting for permission to sync
|
||||
StatusPendingEmail = "pendingemail" // permission granted but missing email
|
||||
StatusQueued = "queued" // in sync queue. will be synced soon
|
||||
StatusPendingUpgrade = "pendingupgrade" // in sync queue. will be synced soon
|
||||
StatusSyncing = "syncing" // syncing now
|
||||
StatusSynced = "synced" // done
|
||||
StatusFailed = "failed"
|
||||
StatusFinalized = "finalized" // no more changes allowed
|
||||
StatusAbandoned = "abandoned" // deleted on youtube or banned
|
||||
)
|
||||
|
||||
var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusPendingUpgrade, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned}
|
||||
|
||||
const (
|
||||
VideoStatusPublished = "published"
|
||||
VideoStatusFailed = "failed"
|
||||
VideoStatusUpgradeFailed = "upgradefailed"
|
||||
VideoStatusUnpublished = "unpublished"
|
||||
)
|
||||
func (s *SyncManager) enqueueChannel(channel *shared.YoutubeChannel) {
|
||||
s.channelsToSync = append(s.channelsToSync, Sync{
|
||||
DbChannelData: channel,
|
||||
Manager: s,
|
||||
namer: namer.NewNamer(),
|
||||
hardVideoFailure: hardVideoFailure{
|
||||
lock: &sync.Mutex{},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func (s *SyncManager) Start() error {
|
||||
|
||||
syncCount := 0
|
||||
for {
|
||||
err := s.checkUsedSpace()
|
||||
if logUtils.ShouldCleanOnStartup() {
|
||||
err := logUtils.CleanForStartup()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
var syncs []Sync
|
||||
var lastChannelProcessed string
|
||||
var secondLastChannelProcessed string
|
||||
syncCount := 0
|
||||
for {
|
||||
s.channelsToSync = make([]Sync, 0, 10) // reset sync queue
|
||||
err := s.checkUsedSpace()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
shouldInterruptLoop := false
|
||||
|
||||
isSingleChannelSync := s.syncProperties.YoutubeChannelID != ""
|
||||
if isSingleChannelSync {
|
||||
channels, err := s.apiConfig.FetchChannels("", s.syncProperties)
|
||||
if s.CliFlags.IsSingleChannelSync() {
|
||||
channels, err := s.ApiConfig.FetchChannels("", &s.CliFlags)
|
||||
if err != nil {
|
||||
return err
|
||||
return errors.Err(err)
|
||||
}
|
||||
if len(channels) != 1 {
|
||||
return errors.Err("Expected 1 channel, %d returned", len(channels))
|
||||
}
|
||||
lbryChannelName := channels[0].DesiredChannelName
|
||||
syncs = make([]Sync, 1)
|
||||
syncs[0] = Sync{
|
||||
APIConfig: s.apiConfig,
|
||||
YoutubeChannelID: s.syncProperties.YoutubeChannelID,
|
||||
LbryChannelName: lbryChannelName,
|
||||
lbryChannelID: channels[0].ChannelClaimID,
|
||||
StopOnError: s.stopOnError,
|
||||
MaxTries: s.maxTries,
|
||||
ConcurrentVideos: s.concurrentVideos,
|
||||
TakeOverExistingChannel: s.takeOverExistingChannel,
|
||||
Refill: s.refill,
|
||||
Manager: s,
|
||||
LbrycrdString: s.lbrycrdString,
|
||||
AwsS3ID: s.awsS3ID,
|
||||
AwsS3Secret: s.awsS3Secret,
|
||||
AwsS3Region: s.awsS3Region,
|
||||
AwsS3Bucket: s.awsS3Bucket,
|
||||
namer: namer.NewNamer(),
|
||||
Fee: channels[0].Fee,
|
||||
}
|
||||
s.enqueueChannel(&channels[0])
|
||||
shouldInterruptLoop = true
|
||||
} else {
|
||||
var queuesToSync []string
|
||||
//TODO: implement scrambling to avoid starvation of queues
|
||||
if s.syncStatus != "" {
|
||||
queuesToSync = append(queuesToSync, s.syncStatus)
|
||||
} else if s.syncUpdate {
|
||||
queuesToSync = append(queuesToSync, StatusSyncing, StatusSynced)
|
||||
if s.CliFlags.Status != "" {
|
||||
queuesToSync = append(queuesToSync, shared.StatusSyncing, s.CliFlags.Status)
|
||||
} else if s.CliFlags.SyncUpdate {
|
||||
queuesToSync = append(queuesToSync, shared.StatusSyncing, shared.StatusSynced)
|
||||
} else {
|
||||
queuesToSync = append(queuesToSync, StatusSyncing, StatusQueued)
|
||||
queuesToSync = append(queuesToSync, shared.StatusSyncing, shared.StatusQueued)
|
||||
}
|
||||
if s.CliFlags.SecondaryStatus != "" {
|
||||
queuesToSync = append(queuesToSync, s.CliFlags.SecondaryStatus)
|
||||
}
|
||||
queues:
|
||||
for _, q := range queuesToSync {
|
||||
channels, err := s.apiConfig.FetchChannels(q, s.syncProperties)
|
||||
channels, err := s.ApiConfig.FetchChannels(q, &s.CliFlags)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Infof("There are %d channels in the \"%s\" queue", len(channels), q)
|
||||
if len(channels) > 0 {
|
||||
c := channels[0]
|
||||
syncs = append(syncs, Sync{
|
||||
APIConfig: s.apiConfig,
|
||||
YoutubeChannelID: c.ChannelId,
|
||||
LbryChannelName: c.DesiredChannelName,
|
||||
lbryChannelID: c.ChannelClaimID,
|
||||
StopOnError: s.stopOnError,
|
||||
MaxTries: s.maxTries,
|
||||
ConcurrentVideos: s.concurrentVideos,
|
||||
TakeOverExistingChannel: s.takeOverExistingChannel,
|
||||
Refill: s.refill,
|
||||
Manager: s,
|
||||
LbrycrdString: s.lbrycrdString,
|
||||
AwsS3ID: s.awsS3ID,
|
||||
AwsS3Secret: s.awsS3Secret,
|
||||
AwsS3Region: s.awsS3Region,
|
||||
AwsS3Bucket: s.awsS3Bucket,
|
||||
namer: namer.NewNamer(),
|
||||
Fee: c.Fee,
|
||||
})
|
||||
continue
|
||||
log.Infof("Currently processing the \"%s\" queue with %d channels", q, len(channels))
|
||||
for _, c := range channels {
|
||||
s.enqueueChannel(&c)
|
||||
queueAll := q == shared.StatusFailed || q == shared.StatusSyncing
|
||||
if !queueAll {
|
||||
break queues
|
||||
}
|
||||
}
|
||||
log.Infof("Drained the \"%s\" queue", q)
|
||||
}
|
||||
}
|
||||
if len(syncs) == 0 {
|
||||
if len(s.channelsToSync) == 0 {
|
||||
log.Infoln("No channels to sync. Pausing 5 minutes!")
|
||||
time.Sleep(5 * time.Minute)
|
||||
}
|
||||
for _, sync := range syncs {
|
||||
for _, sync := range s.channelsToSync {
|
||||
if lastChannelProcessed == sync.DbChannelData.ChannelId && secondLastChannelProcessed == lastChannelProcessed {
|
||||
util.SendToSlack("We just killed a sync for %s to stop looping! (%s)", sync.DbChannelData.DesiredChannelName, sync.DbChannelData.ChannelId)
|
||||
stopTheLoops := errors.Err("Found channel %s running 3 times, set it to failed, and reprocess later", sync.DbChannelData.DesiredChannelName)
|
||||
sync.setChannelTerminationStatus(&stopTheLoops)
|
||||
continue
|
||||
}
|
||||
secondLastChannelProcessed = lastChannelProcessed
|
||||
lastChannelProcessed = sync.DbChannelData.ChannelId
|
||||
shouldNotCount := false
|
||||
SendInfoToSlack("Syncing %s (%s) to LBRY! total processed channels since startup: %d", sync.LbryChannelName, sync.YoutubeChannelID, syncCount+1)
|
||||
logUtils.SendInfoToSlack("Syncing %s (%s) to LBRY! total processed channels since startup: %d", sync.DbChannelData.DesiredChannelName, sync.DbChannelData.ChannelId, syncCount+1)
|
||||
err := sync.FullCycle()
|
||||
//TODO: THIS IS A TEMPORARY WORK AROUND FOR THE STUPID IP LOCKUP BUG
|
||||
ipPool, _ := ip_manager.GetIPPool(sync.grp)
|
||||
if ipPool != nil {
|
||||
ipPool.ReleaseAll()
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "quotaExceeded") {
|
||||
logUtils.SleepUntilQuotaReset()
|
||||
}
|
||||
fatalErrors := []string{
|
||||
"default_wallet already exists",
|
||||
"WALLET HAS NOT BEEN MOVED TO THE WALLET BACKUP DIR",
|
||||
@@ -199,48 +142,47 @@ func (s *SyncManager) Start() error {
|
||||
"failure uploading wallet",
|
||||
"the channel in the wallet is different than the channel in the database",
|
||||
"this channel does not belong to this wallet!",
|
||||
"Invalid status code: 429",
|
||||
"HTTP Error 429",
|
||||
"You already have a stream claim published under the name",
|
||||
}
|
||||
|
||||
if util.SubstringInSlice(err.Error(), fatalErrors) {
|
||||
return errors.Prefix("@Nikooo777 this requires manual intervention! Exiting...", err)
|
||||
}
|
||||
shouldNotCount = strings.Contains(err.Error(), "this youtube channel is being managed by another server")
|
||||
if !shouldNotCount {
|
||||
SendInfoToSlack("A non fatal error was reported by the sync process. %s\nContinuing...", err.Error())
|
||||
logUtils.SendInfoToSlack("A non fatal error was reported by the sync process.\n%s", errors.FullTrace(err))
|
||||
}
|
||||
}
|
||||
err = logUtils.CleanupMetadata()
|
||||
if err != nil {
|
||||
log.Errorf("something went wrong while trying to clear out the video metadata directory: %s", errors.FullTrace(err))
|
||||
}
|
||||
err = blobs_reflector.ReflectAndClean()
|
||||
if err != nil {
|
||||
return errors.Prefix("@Nikooo777 something went wrong while reflecting blobs", err)
|
||||
}
|
||||
SendInfoToSlack("Syncing %s (%s) reached an end. total processed channels since startup: %d", sync.LbryChannelName, sync.YoutubeChannelID, syncCount+1)
|
||||
logUtils.SendInfoToSlack("%s (%s) reached an end. Total processed channels since startup: %d", sync.DbChannelData.DesiredChannelName, sync.DbChannelData.ChannelId, syncCount+1)
|
||||
if !shouldNotCount {
|
||||
syncCount++
|
||||
}
|
||||
if sync.IsInterrupted() || (s.limit != 0 && syncCount >= s.limit) {
|
||||
if sync.IsInterrupted() || (s.CliFlags.Limit != 0 && syncCount >= s.CliFlags.Limit) {
|
||||
shouldInterruptLoop = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if shouldInterruptLoop || s.singleRun {
|
||||
if shouldInterruptLoop || s.CliFlags.SingleRun {
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (s *SyncManager) GetS3AWSConfig() aws.Config {
|
||||
return aws.Config{
|
||||
Credentials: credentials.NewStaticCredentials(s.awsS3ID, s.awsS3Secret, ""),
|
||||
Region: &s.awsS3Region,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *SyncManager) checkUsedSpace() error {
|
||||
usedPctile, err := GetUsedSpace(s.blobsDir)
|
||||
usedPctile, err := GetUsedSpace(logUtils.GetBlobsDir())
|
||||
if err != nil {
|
||||
return err
|
||||
return errors.Err(err)
|
||||
}
|
||||
if usedPctile >= 0.90 && !s.skipSpaceCheck {
|
||||
if usedPctile >= 0.90 && !s.CliFlags.SkipSpaceCheck {
|
||||
return errors.Err(fmt.Sprintf("more than 90%% of the space has been used. use --skip-space-check to ignore. Used: %.1f%%", usedPctile*100))
|
||||
}
|
||||
log.Infof("disk usage: %.1f%%", usedPctile*100)
|
||||
|
||||
@@ -0,0 +1,254 @@
|
||||
package manager
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
|
||||
logUtils "github.com/lbryio/ytsync/v5/util"
|
||||
)
|
||||
|
||||
func (s *Sync) getS3Downloader() (*s3manager.Downloader, error) {
|
||||
s3Session, err := session.NewSession(s.Manager.AwsConfigs.GetS3AWSConfig())
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("error starting session: ", err)
|
||||
}
|
||||
downloader := s3manager.NewDownloader(s3Session)
|
||||
return downloader, nil
|
||||
}
|
||||
func (s *Sync) getS3Uploader() (*s3manager.Uploader, error) {
|
||||
s3Session, err := session.NewSession(s.Manager.AwsConfigs.GetS3AWSConfig())
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("error starting session: ", err)
|
||||
}
|
||||
uploader := s3manager.NewUploader(s3Session)
|
||||
return uploader, nil
|
||||
}
|
||||
|
||||
func (s *Sync) downloadWallet() error {
|
||||
defaultWalletDir, defaultTempWalletDir, key, err := s.getWalletPaths()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
downloader, err := s.getS3Downloader()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out, err := os.Create(defaultTempWalletDir)
|
||||
if err != nil {
|
||||
return errors.Prefix("error creating temp wallet: ", err)
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{
|
||||
Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket),
|
||||
Key: key,
|
||||
})
|
||||
if err != nil {
|
||||
// Casting to the awserr.Error type will allow you to inspect the error
|
||||
// code returned by the service in code. The error code can be used
|
||||
// to switch on context specific functionality. In this case a context
|
||||
// specific error message is printed to the user based on the bucket
|
||||
// and key existing.
|
||||
//
|
||||
// For information on other S3 API error codes see:
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
|
||||
if aerr, ok := err.(awserr.Error); ok {
|
||||
code := aerr.Code()
|
||||
if code == s3.ErrCodeNoSuchKey {
|
||||
return errors.Err("wallet not on S3")
|
||||
}
|
||||
}
|
||||
return err
|
||||
} else if bytesWritten == 0 {
|
||||
return errors.Err("zero bytes written")
|
||||
}
|
||||
|
||||
err = os.Rename(defaultTempWalletDir, defaultWalletDir)
|
||||
if err != nil {
|
||||
return errors.Prefix("error replacing temp wallet for default wallet: ", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Sync) downloadBlockchainDB() error {
|
||||
if logUtils.IsRegTest() {
|
||||
return nil // tests fail if we re-use the same blockchain DB
|
||||
}
|
||||
defaultBDBDir, defaultTempBDBDir, key, err := s.getBlockchainDBPaths()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
files, err := filepath.Glob(defaultBDBDir + "*")
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
for _, f := range files {
|
||||
err = os.Remove(f)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
}
|
||||
if s.DbChannelData.WipeDB {
|
||||
return nil
|
||||
}
|
||||
downloader, err := s.getS3Downloader()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
out, err := os.Create(defaultTempBDBDir)
|
||||
if err != nil {
|
||||
return errors.Prefix("error creating temp wallet: ", err)
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{
|
||||
Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket),
|
||||
Key: key,
|
||||
})
|
||||
if err != nil {
|
||||
// Casting to the awserr.Error type will allow you to inspect the error
|
||||
// code returned by the service in code. The error code can be used
|
||||
// to switch on context specific functionality. In this case a context
|
||||
// specific error message is printed to the user based on the bucket
|
||||
// and key existing.
|
||||
//
|
||||
// For information on other S3 API error codes see:
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
|
||||
if aerr, ok := err.(awserr.Error); ok {
|
||||
code := aerr.Code()
|
||||
if code == s3.ErrCodeNoSuchKey {
|
||||
return nil // let ytsync sync the database by itself
|
||||
}
|
||||
}
|
||||
return errors.Err(err)
|
||||
} else if bytesWritten == 0 {
|
||||
return errors.Err("zero bytes written")
|
||||
}
|
||||
|
||||
err = os.Rename(defaultTempBDBDir, defaultBDBDir)
|
||||
if err != nil {
|
||||
return errors.Prefix("error replacing temp blockchain.db for default blockchain.db: ", err)
|
||||
}
|
||||
log.Printf("blockchain.db downloaded to %s", defaultBDBDir)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Sync) getWalletPaths() (defaultWallet, tempWallet string, key *string, err error) {
|
||||
defaultWallet = os.Getenv("HOME") + "/.lbryum/wallets/default_wallet"
|
||||
tempWallet = os.Getenv("HOME") + "/.lbryum/wallets/tmp_wallet"
|
||||
key = aws.String("/wallets/" + s.DbChannelData.ChannelId)
|
||||
if logUtils.IsRegTest() {
|
||||
defaultWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet"
|
||||
tempWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/tmp_wallet"
|
||||
key = aws.String("/regtest/" + s.DbChannelData.ChannelId)
|
||||
}
|
||||
|
||||
lbryumDir := os.Getenv("LBRYUM_DIR")
|
||||
if lbryumDir != "" {
|
||||
defaultWallet = lbryumDir + "/wallets/default_wallet"
|
||||
tempWallet = lbryumDir + "/wallets/tmp_wallet"
|
||||
}
|
||||
|
||||
if _, err := os.Stat(defaultWallet); !os.IsNotExist(err) {
|
||||
return "", "", nil, errors.Err("default_wallet already exists")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (s *Sync) getBlockchainDBPaths() (defaultDB, tempDB string, key *string, err error) {
|
||||
lbryumDir := os.Getenv("LBRYUM_DIR")
|
||||
if lbryumDir == "" {
|
||||
if logUtils.IsRegTest() {
|
||||
lbryumDir = os.Getenv("HOME") + "/.lbryum_regtest"
|
||||
} else {
|
||||
lbryumDir = os.Getenv("HOME") + "/.lbryum"
|
||||
}
|
||||
}
|
||||
defaultDB = lbryumDir + "/lbc_mainnet/blockchain.db"
|
||||
tempDB = lbryumDir + "/lbc_mainnet/tmp_blockchain.db"
|
||||
key = aws.String("/blockchain_dbs/" + s.DbChannelData.ChannelId)
|
||||
if logUtils.IsRegTest() {
|
||||
defaultDB = lbryumDir + "/lbc_regtest/blockchain.db"
|
||||
tempDB = lbryumDir + "/lbc_regtest/tmp_blockchain.db"
|
||||
key = aws.String("/regtest_dbs/" + s.DbChannelData.ChannelId)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (s *Sync) uploadWallet() error {
|
||||
defaultWalletDir := logUtils.GetDefaultWalletPath()
|
||||
key := aws.String("/wallets/" + s.DbChannelData.ChannelId)
|
||||
if logUtils.IsRegTest() {
|
||||
key = aws.String("/regtest/" + s.DbChannelData.ChannelId)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(defaultWalletDir); os.IsNotExist(err) {
|
||||
return errors.Err("default_wallet does not exist")
|
||||
}
|
||||
|
||||
uploader, err := s.getS3Uploader()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
file, err := os.Open(defaultWalletDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
_, err = uploader.Upload(&s3manager.UploadInput{
|
||||
Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket),
|
||||
Key: key,
|
||||
Body: file,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Println("wallet uploaded to S3")
|
||||
|
||||
return os.Remove(defaultWalletDir)
|
||||
}
|
||||
|
||||
func (s *Sync) uploadBlockchainDB() error {
|
||||
defaultBDBDir, _, key, err := s.getBlockchainDBPaths()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(defaultBDBDir); os.IsNotExist(err) {
|
||||
return errors.Err("blockchain.db does not exist")
|
||||
}
|
||||
|
||||
uploader, err := s.getS3Uploader()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
file, err := os.Open(defaultBDBDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
_, err = uploader.Upload(&s3manager.UploadInput{
|
||||
Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket),
|
||||
Key: key,
|
||||
Body: file,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Println("blockchain.db uploaded to S3")
|
||||
return os.Remove(defaultBDBDir)
|
||||
}
|
||||
+219
-159
@@ -3,34 +3,41 @@ package manager
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
"github.com/lbryio/lbry.go/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/extras/util"
|
||||
"github.com/lbryio/lbry.go/lbrycrd"
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
"github.com/lbryio/ytsync/v5/shared"
|
||||
"github.com/lbryio/ytsync/v5/timing"
|
||||
logUtils "github.com/lbryio/ytsync/v5/util"
|
||||
"github.com/lbryio/ytsync/v5/ytapi"
|
||||
|
||||
"github.com/lbryio/ytsync/tagsManager"
|
||||
"github.com/lbryio/ytsync/thumbs"
|
||||
"github.com/lbryio/ytsync/v5/tags_manager"
|
||||
"github.com/lbryio/ytsync/v5/thumbs"
|
||||
|
||||
"github.com/shopspring/decimal"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"google.golang.org/api/googleapi/transport"
|
||||
"google.golang.org/api/youtube/v3"
|
||||
)
|
||||
|
||||
func (s *Sync) enableAddressReuse() error {
|
||||
accountsResponse, err := s.daemon.AccountList()
|
||||
accountsResponse, err := s.daemon.AccountList(1, 50)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
accounts := accountsResponse.LBCMainnet
|
||||
if os.Getenv("REGTEST") == "true" {
|
||||
accounts = accountsResponse.LBCRegtest
|
||||
accounts := make([]jsonrpc.Account, 0, len(accountsResponse.Items))
|
||||
ledger := "lbc_mainnet"
|
||||
if logUtils.IsRegTest() {
|
||||
ledger = "lbc_regtest"
|
||||
}
|
||||
for _, a := range accountsResponse.Items {
|
||||
if *a.Ledger == ledger {
|
||||
accounts = append(accounts, a)
|
||||
}
|
||||
}
|
||||
|
||||
for _, a := range accounts {
|
||||
_, err = s.daemon.AccountSet(a.ID, jsonrpc.AccountSettings{
|
||||
ChangeMaxUses: util.PtrToInt(1000),
|
||||
@@ -43,6 +50,10 @@ func (s *Sync) enableAddressReuse() error {
|
||||
return nil
|
||||
}
|
||||
func (s *Sync) walletSetup() error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("walletSetup").Add(time.Since(start))
|
||||
}(start)
|
||||
//prevent unnecessary concurrent execution and publishing while refilling/reallocating UTXOs
|
||||
s.walletMux.Lock()
|
||||
defer s.walletMux.Unlock()
|
||||
@@ -57,17 +68,13 @@ func (s *Sync) walletSetup() error {
|
||||
} else if balanceResp == nil {
|
||||
return errors.Err("no response")
|
||||
}
|
||||
balance, err := strconv.ParseFloat((string)(*balanceResp), 64)
|
||||
balance, err := strconv.ParseFloat(balanceResp.Available.String(), 64)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
log.Debugf("Starting balance is %.4f", balance)
|
||||
|
||||
n, err := s.CountVideos()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
videosOnYoutube := int(n)
|
||||
videosOnYoutube := int(s.DbChannelData.TotalVideos)
|
||||
|
||||
log.Debugf("Source channel has %d videos", videosOnYoutube)
|
||||
if videosOnYoutube == 0 {
|
||||
@@ -92,22 +99,27 @@ func (s *Sync) walletSetup() error {
|
||||
|
||||
log.Debugf("We already allocated credits for %d published videos and %d failed videos", publishedCount, failedCount)
|
||||
|
||||
if videosOnYoutube > s.Manager.videosLimit {
|
||||
videosOnYoutube = s.Manager.videosLimit
|
||||
if videosOnYoutube > s.Manager.CliFlags.VideosToSync(s.DbChannelData.TotalSubscribers) {
|
||||
videosOnYoutube = s.Manager.CliFlags.VideosToSync(s.DbChannelData.TotalSubscribers)
|
||||
}
|
||||
unallocatedVideos := videosOnYoutube - (publishedCount + failedCount)
|
||||
requiredBalance := float64(unallocatedVideos)*(publishAmount+estimatedMaxTxFee) + channelClaimAmount
|
||||
if s.Manager.upgradeMetadata {
|
||||
channelFee := channelClaimAmount
|
||||
channelAlreadyClaimed := s.DbChannelData.ChannelClaimID != ""
|
||||
if channelAlreadyClaimed {
|
||||
channelFee = 0.0
|
||||
}
|
||||
requiredBalance := float64(unallocatedVideos)*(publishAmount+estimatedMaxTxFee) + channelFee
|
||||
if s.Manager.CliFlags.UpgradeMetadata {
|
||||
requiredBalance += float64(notUpgradedCount) * 0.001
|
||||
}
|
||||
|
||||
refillAmount := 0.0
|
||||
if balance < requiredBalance || balance < minimumAccountBalance {
|
||||
refillAmount = math.Max(requiredBalance-balance, minimumRefillAmount)
|
||||
refillAmount = math.Max(math.Max(requiredBalance-balance, minimumAccountBalance-balance), minimumRefillAmount)
|
||||
}
|
||||
|
||||
if s.Refill > 0 {
|
||||
refillAmount += float64(s.Refill)
|
||||
if s.Manager.CliFlags.Refill > 0 {
|
||||
refillAmount += float64(s.Manager.CliFlags.Refill)
|
||||
}
|
||||
|
||||
if refillAmount > 0 {
|
||||
@@ -117,14 +129,17 @@ func (s *Sync) walletSetup() error {
|
||||
}
|
||||
}
|
||||
|
||||
claimAddress, err := s.daemon.AddressList(nil)
|
||||
claimAddress, err := s.daemon.AddressList(nil, nil, 1, 20)
|
||||
if err != nil {
|
||||
return err
|
||||
} else if claimAddress == nil {
|
||||
return errors.Err("could not get unused address")
|
||||
return errors.Err("could not get an address")
|
||||
}
|
||||
s.claimAddress = string((*claimAddress)[0]) //TODO: remove claimAddress completely
|
||||
if s.claimAddress == "" {
|
||||
if s.DbChannelData.PublishAddress.Address == "" || !s.shouldTransfer() {
|
||||
s.DbChannelData.PublishAddress.Address = string(claimAddress.Items[0].Address)
|
||||
s.DbChannelData.PublishAddress.IsMine = true
|
||||
}
|
||||
if s.DbChannelData.PublishAddress.Address == "" {
|
||||
return errors.Err("found blank claim address")
|
||||
}
|
||||
|
||||
@@ -136,27 +151,47 @@ func (s *Sync) walletSetup() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Sync) ensureEnoughUTXOs() error {
|
||||
accounts, err := s.daemon.AccountList()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
accountsNet := (*accounts).LBCMainnet
|
||||
if os.Getenv("REGTEST") == "true" {
|
||||
accountsNet = (*accounts).LBCRegtest
|
||||
}
|
||||
defaultAccount := ""
|
||||
for _, account := range accountsNet {
|
||||
if account.IsDefault {
|
||||
defaultAccount = account.ID
|
||||
break
|
||||
func (s *Sync) getDefaultAccount() (string, error) {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("getDefaultAccount").Add(time.Since(start))
|
||||
}(start)
|
||||
if s.defaultAccountID == "" {
|
||||
accountsResponse, err := s.daemon.AccountList(1, 50)
|
||||
if err != nil {
|
||||
return "", errors.Err(err)
|
||||
}
|
||||
ledger := "lbc_mainnet"
|
||||
if logUtils.IsRegTest() {
|
||||
ledger = "lbc_regtest"
|
||||
}
|
||||
for _, a := range accountsResponse.Items {
|
||||
if *a.Ledger == ledger {
|
||||
if a.IsDefault {
|
||||
s.defaultAccountID = a.ID
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if s.defaultAccountID == "" {
|
||||
return "", errors.Err("No default account found")
|
||||
}
|
||||
}
|
||||
if defaultAccount == "" {
|
||||
return errors.Err("No default account found")
|
||||
return s.defaultAccountID, nil
|
||||
}
|
||||
|
||||
func (s *Sync) ensureEnoughUTXOs() error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("ensureEnoughUTXOs").Add(time.Since(start))
|
||||
}(start)
|
||||
defaultAccount, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
utxolist, err := s.daemon.UTXOList(&defaultAccount)
|
||||
utxolist, err := s.daemon.UTXOList(&defaultAccount, 1, 10000)
|
||||
if err != nil {
|
||||
return err
|
||||
} else if utxolist == nil {
|
||||
@@ -168,9 +203,9 @@ func (s *Sync) ensureEnoughUTXOs() error {
|
||||
count := 0
|
||||
confirmedCount := 0
|
||||
|
||||
for _, utxo := range *utxolist {
|
||||
for _, utxo := range utxolist.Items {
|
||||
amount, _ := strconv.ParseFloat(utxo.Amount, 64)
|
||||
if utxo.IsMine && utxo.Type == "payment" && amount > 0.001 {
|
||||
if utxo.IsMyOutput && utxo.Type == "payment" && amount > 0.001 {
|
||||
if utxo.Confirmations > 0 {
|
||||
confirmedCount++
|
||||
}
|
||||
@@ -187,16 +222,29 @@ func (s *Sync) ensureEnoughUTXOs() error {
|
||||
return errors.Err("no response")
|
||||
}
|
||||
|
||||
balanceAmount, err := strconv.ParseFloat((string)(*balance), 64)
|
||||
balanceAmount, err := strconv.ParseFloat(balance.Available.String(), 64)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
//this is dumb but sometimes the balance is negative and it breaks everything, so let's check again
|
||||
if balanceAmount < 0 {
|
||||
log.Infof("negative balance of %.2f found. Waiting to retry...", balanceAmount)
|
||||
time.Sleep(10 * time.Second)
|
||||
balanceAmount, err = strconv.ParseFloat(balance.Available.String(), 64)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
}
|
||||
maxUTXOs := uint64(500)
|
||||
desiredUTXOCount := uint64(math.Floor((balanceAmount) / 0.1))
|
||||
if desiredUTXOCount > maxUTXOs {
|
||||
desiredUTXOCount = maxUTXOs
|
||||
}
|
||||
log.Infof("Splitting balance of %s evenly between %d UTXOs", *balance, desiredUTXOCount)
|
||||
if desiredUTXOCount < uint64(confirmedCount) {
|
||||
return nil
|
||||
}
|
||||
availableBalance, _ := balance.Available.Float64()
|
||||
log.Infof("Splitting balance of %.3f evenly between %d UTXOs", availableBalance, desiredUTXOCount)
|
||||
|
||||
broadcastFee := 0.1
|
||||
prefillTx, err := s.daemon.AccountFund(defaultAccount, defaultAccount, fmt.Sprintf("%.4f", balanceAmount-broadcastFee), desiredUTXOCount, false)
|
||||
@@ -223,6 +271,9 @@ func (s *Sync) ensureEnoughUTXOs() error {
|
||||
}
|
||||
|
||||
func (s *Sync) waitForNewBlock() error {
|
||||
defer func(start time.Time) { timing.TimedComponent("waitForNewBlock").Add(time.Since(start)) }(time.Now())
|
||||
|
||||
log.Printf("regtest: %t, docker: %t", logUtils.IsRegTest(), logUtils.IsUsingDocker())
|
||||
status, err := s.daemon.Status()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -235,68 +286,89 @@ func (s *Sync) waitForNewBlock() error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
currentBlock := status.Wallet.Blocks
|
||||
for i := 0; status.Wallet.Blocks <= currentBlock; i++ {
|
||||
if i%3 == 0 {
|
||||
log.Printf("Waiting for new block (%d)...", currentBlock+1)
|
||||
}
|
||||
if logUtils.IsRegTest() && logUtils.IsUsingDocker() {
|
||||
err = s.GenerateRegtestBlock()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
time.Sleep(10 * time.Second)
|
||||
status, err = s.daemon.Status()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
time.Sleep(5 * time.Second)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Sync) GenerateRegtestBlock() error {
|
||||
lbrycrd, err := logUtils.GetLbrycrdClient(s.Manager.LbrycrdDsn)
|
||||
if err != nil {
|
||||
return errors.Prefix("error getting lbrycrd client: ", err)
|
||||
}
|
||||
|
||||
txs, err := lbrycrd.Generate(1)
|
||||
if err != nil {
|
||||
return errors.Prefix("error generating new block: ", err)
|
||||
}
|
||||
|
||||
for _, tx := range txs {
|
||||
log.Info("Generated tx: ", tx.String())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Sync) ensureChannelOwnership() error {
|
||||
if s.LbryChannelName == "" {
|
||||
defer func(start time.Time) { timing.TimedComponent("ensureChannelOwnership").Add(time.Since(start)) }(time.Now())
|
||||
|
||||
if s.DbChannelData.DesiredChannelName == "" {
|
||||
return errors.Err("no channel name set")
|
||||
}
|
||||
//@TODO: get rid of this when imported channels are supported
|
||||
if s.YoutubeChannelID == "UCW-thz5HxE-goYq8yPds1Gw" {
|
||||
return nil
|
||||
}
|
||||
channels, err := s.daemon.ChannelList(nil, 1, 50)
|
||||
|
||||
channels, err := s.daemon.ChannelList(nil, 1, 500, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
} else if channels == nil {
|
||||
return errors.Err("no channel response")
|
||||
}
|
||||
//special case for wallets we don't retain full control anymore
|
||||
if len((*channels).Items) > 1 {
|
||||
// This wallet is probably not under our control anymore but we still want to publish to it
|
||||
// here we shall check if within all the channels there is one that was created by ytsync
|
||||
SendInfoToSlack("we are dealing with a wallet that has multiple channels. This indicates that the wallet was probably transferred but we still want to sync their content. YoutubeID: %s", s.YoutubeChannelID)
|
||||
if s.lbryChannelID == "" {
|
||||
|
||||
var channelToUse *jsonrpc.Transaction
|
||||
if len((*channels).Items) > 0 {
|
||||
if s.DbChannelData.ChannelClaimID == "" {
|
||||
return errors.Err("this channel does not have a recorded claimID in the database. To prevent failures, updates are not supported until an entry is manually added in the database")
|
||||
}
|
||||
for _, c := range (*channels).Items {
|
||||
if c.ClaimID != s.lbryChannelID {
|
||||
if c.Name != s.LbryChannelName {
|
||||
return errors.Err("the channel in the wallet is different than the channel in the database")
|
||||
}
|
||||
return nil // we have the ytsync channel and both the claimID and the channelName from the database are correct
|
||||
log.Debugf("checking listed channel %s (%s)", c.ClaimID, c.Name)
|
||||
if c.ClaimID != s.DbChannelData.ChannelClaimID {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
channelUsesOldMetadata := false
|
||||
if len((*channels).Items) == 1 {
|
||||
channel := ((*channels).Items)[0]
|
||||
if channel.Name == s.LbryChannelName {
|
||||
channelUsesOldMetadata = channel.Value.GetThumbnail() == nil
|
||||
//TODO: eventually get rid of this when the whole db is filled
|
||||
if s.lbryChannelID == "" {
|
||||
err = s.Manager.apiConfig.SetChannelClaimID(s.YoutubeChannelID, channel.ClaimID)
|
||||
} else if channel.ClaimID != s.lbryChannelID {
|
||||
if c.Name != s.DbChannelData.DesiredChannelName {
|
||||
return errors.Err("the channel in the wallet is different than the channel in the database")
|
||||
}
|
||||
s.lbryChannelID = channel.ClaimID
|
||||
if !channelUsesOldMetadata {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
return errors.Err("this channel does not belong to this wallet! Expected: %s, found: %s", s.LbryChannelName, channel.Name)
|
||||
channelToUse = &c
|
||||
break
|
||||
}
|
||||
if channelToUse == nil {
|
||||
return errors.Err("this wallet has channels but not a single one is ours! Expected claim_id: %s (%s)", s.DbChannelData.ChannelClaimID, s.DbChannelData.DesiredChannelName)
|
||||
}
|
||||
} else if s.DbChannelData.TransferState == shared.TransferStateComplete {
|
||||
return errors.Err("the channel was transferred but appears to have been abandoned!")
|
||||
} else if s.DbChannelData.ChannelClaimID != "" {
|
||||
return errors.Err("the database has a channel recorded (%s) but nothing was found in our control", s.DbChannelData.ChannelClaimID)
|
||||
}
|
||||
|
||||
channelUsesOldMetadata := false
|
||||
if channelToUse != nil {
|
||||
channelUsesOldMetadata = channelToUse.Value.GetThumbnail() == nil
|
||||
if !channelUsesOldMetadata {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -308,47 +380,44 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
} else if balanceResp == nil {
|
||||
return errors.Err("no response")
|
||||
}
|
||||
balance, err := decimal.NewFromString((string)(*balanceResp))
|
||||
balance, err := decimal.NewFromString(balanceResp.Available.String())
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
if balance.LessThan(decimal.NewFromFloat(channelBidAmount)) {
|
||||
err = s.addCredits(channelBidAmount + 0.1)
|
||||
err = s.addCredits(channelBidAmount + 0.3)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
client := &http.Client{
|
||||
Transport: &transport.APIKey{Key: s.APIConfig.YoutubeAPIKey},
|
||||
}
|
||||
|
||||
service, err := youtube.New(client)
|
||||
channelInfo, err := ytapi.ChannelInfo(s.DbChannelData.ChannelId)
|
||||
if err != nil {
|
||||
return errors.Prefix("error creating YouTube service", err)
|
||||
if strings.Contains(err.Error(), "invalid character 'e' looking for beginning of value") {
|
||||
logUtils.SendInfoToSlack("failed to get channel data for %s. Waiting 1 minute to retry", s.DbChannelData.ChannelId)
|
||||
time.Sleep(1 * time.Minute)
|
||||
channelInfo, err = ytapi.ChannelInfo(s.DbChannelData.ChannelId)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
response, err := service.Channels.List("snippet,brandingSettings").Id(s.YoutubeChannelID).Do()
|
||||
if err != nil {
|
||||
return errors.Prefix("error getting channel details", err)
|
||||
}
|
||||
|
||||
if len(response.Items) < 1 {
|
||||
return errors.Err("youtube channel not found")
|
||||
}
|
||||
|
||||
channelInfo := response.Items[0].Snippet
|
||||
channelBranding := response.Items[0].BrandingSettings
|
||||
|
||||
thumbnail := thumbs.GetBestThumbnail(channelInfo.Thumbnails)
|
||||
thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail.Url, s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
|
||||
thumbnail := channelInfo.Header.C4TabbedHeaderRenderer.Avatar.Thumbnails[len(channelInfo.Header.C4TabbedHeaderRenderer.Avatar.Thumbnails)-1].URL
|
||||
thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail, s.DbChannelData.ChannelId, *s.Manager.AwsConfigs.GetS3AWSConfig())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var bannerURL *string
|
||||
if channelBranding.Image != nil && channelBranding.Image.BannerImageUrl != "" {
|
||||
bURL, err := thumbs.MirrorThumbnail(channelBranding.Image.BannerImageUrl, "banner-"+s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
|
||||
if channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails != nil {
|
||||
bURL, err := thumbs.MirrorThumbnail(channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails[len(channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails)-1].URL,
|
||||
"banner-"+s.DbChannelData.ChannelId,
|
||||
*s.Manager.AwsConfigs.GetS3AWSConfig(),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -356,34 +425,43 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
}
|
||||
|
||||
var languages []string = nil
|
||||
if channelInfo.DefaultLanguage != "" {
|
||||
languages = []string{channelInfo.DefaultLanguage}
|
||||
}
|
||||
//we don't have this data without the API
|
||||
//if channelInfo.DefaultLanguage != "" {
|
||||
// if channelInfo.DefaultLanguage == "iw" {
|
||||
// channelInfo.DefaultLanguage = "he"
|
||||
// }
|
||||
// languages = []string{channelInfo.DefaultLanguage}
|
||||
//}
|
||||
var locations []jsonrpc.Location = nil
|
||||
if channelInfo.Country != "" {
|
||||
locations = []jsonrpc.Location{{Country: util.PtrToString(channelInfo.Country)}}
|
||||
if channelInfo.Topbar.DesktopTopbarRenderer.CountryCode != "" {
|
||||
locations = []jsonrpc.Location{{Country: &channelInfo.Topbar.DesktopTopbarRenderer.CountryCode}}
|
||||
}
|
||||
var c *jsonrpc.TransactionSummary
|
||||
claimCreateOptions := jsonrpc.ClaimCreateOptions{
|
||||
Title: &channelInfo.Title,
|
||||
Description: &channelInfo.Description,
|
||||
Tags: tagsManager.GetTagsForChannel(s.YoutubeChannelID),
|
||||
Title: &channelInfo.Microformat.MicroformatDataRenderer.Title,
|
||||
Description: &channelInfo.Metadata.ChannelMetadataRenderer.Description,
|
||||
Tags: tags_manager.GetTagsForChannel(s.DbChannelData.ChannelId),
|
||||
Languages: languages,
|
||||
Locations: locations,
|
||||
ThumbnailURL: &thumbnailURL,
|
||||
}
|
||||
if channelUsesOldMetadata {
|
||||
c, err = s.daemon.ChannelUpdate(s.lbryChannelID, jsonrpc.ChannelUpdateOptions{
|
||||
ClearTags: util.PtrToBool(true),
|
||||
ClearLocations: util.PtrToBool(true),
|
||||
ClearLanguages: util.PtrToBool(true),
|
||||
ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
|
||||
ClaimCreateOptions: claimCreateOptions,
|
||||
CoverURL: bannerURL,
|
||||
},
|
||||
})
|
||||
if s.DbChannelData.TransferState <= 1 {
|
||||
c, err = s.daemon.ChannelUpdate(s.DbChannelData.ChannelClaimID, jsonrpc.ChannelUpdateOptions{
|
||||
ClearTags: util.PtrToBool(true),
|
||||
ClearLocations: util.PtrToBool(true),
|
||||
ClearLanguages: util.PtrToBool(true),
|
||||
ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
|
||||
ClaimCreateOptions: claimCreateOptions,
|
||||
CoverURL: bannerURL,
|
||||
},
|
||||
})
|
||||
} else {
|
||||
logUtils.SendInfoToSlack("%s (%s) has a channel with old metadata but isn't in our control anymore. Ignoring", s.DbChannelData.DesiredChannelName, s.DbChannelData.ChannelClaimID)
|
||||
return nil
|
||||
}
|
||||
} else {
|
||||
c, err = s.daemon.ChannelCreate(s.LbryChannelName, channelBidAmount, jsonrpc.ChannelCreateOptions{
|
||||
c, err = s.daemon.ChannelCreate(s.DbChannelData.DesiredChannelName, channelBidAmount, jsonrpc.ChannelCreateOptions{
|
||||
ClaimCreateOptions: claimCreateOptions,
|
||||
CoverURL: bannerURL,
|
||||
})
|
||||
@@ -392,45 +470,27 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.lbryChannelID = c.Outputs[0].ClaimID
|
||||
return s.Manager.apiConfig.SetChannelClaimID(s.YoutubeChannelID, s.lbryChannelID)
|
||||
}
|
||||
|
||||
func allUTXOsConfirmed(utxolist *jsonrpc.UTXOListResponse) bool {
|
||||
if utxolist == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
if len(*utxolist) < 1 {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, utxo := range *utxolist {
|
||||
if utxo.Confirmations <= 0 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
s.DbChannelData.ChannelClaimID = c.Outputs[0].ClaimID
|
||||
return s.Manager.ApiConfig.SetChannelClaimID(s.DbChannelData.ChannelId, s.DbChannelData.ChannelClaimID)
|
||||
}
|
||||
|
||||
func (s *Sync) addCredits(amountToAdd float64) error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("addCredits").Add(time.Since(start))
|
||||
}(start)
|
||||
log.Printf("Adding %f credits", amountToAdd)
|
||||
var lbrycrdd *lbrycrd.Client
|
||||
var err error
|
||||
if s.LbrycrdString == "" {
|
||||
lbrycrdd, err = lbrycrd.NewWithDefaultURL()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
lbrycrdd, err = lbrycrd.New(s.LbrycrdString)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
lbrycrdd, err := logUtils.GetLbrycrdClient(s.Manager.LbrycrdDsn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
addressResp, err := s.daemon.AddressUnused(nil)
|
||||
defaultAccount, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
addressResp, err := s.daemon.AddressUnused(&defaultAccount)
|
||||
if err != nil {
|
||||
return err
|
||||
} else if addressResp == nil {
|
||||
|
||||
@@ -0,0 +1,349 @@
|
||||
package manager
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
"github.com/lbryio/ytsync/v5/shared"
|
||||
"github.com/lbryio/ytsync/v5/timing"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func waitConfirmations(s *Sync) error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("waitConfirmations").Add(time.Since(start))
|
||||
}(start)
|
||||
defaultAccount, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
allConfirmed := false
|
||||
waitCount := 0
|
||||
waiting:
|
||||
for !allConfirmed && waitCount < 2 {
|
||||
utxolist, err := s.daemon.UTXOList(&defaultAccount, 1, 10000)
|
||||
if err != nil {
|
||||
return err
|
||||
} else if utxolist == nil {
|
||||
return errors.Err("no response")
|
||||
}
|
||||
|
||||
for _, utxo := range utxolist.Items {
|
||||
if utxo.Confirmations <= 0 {
|
||||
err = s.waitForNewBlock()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
waitCount++
|
||||
continue waiting
|
||||
}
|
||||
}
|
||||
allConfirmed = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type abandonResponse struct {
|
||||
ClaimID string
|
||||
Error error
|
||||
Amount float64
|
||||
}
|
||||
|
||||
func abandonSupports(s *Sync) (float64, error) {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("abandonSupports").Add(time.Since(start))
|
||||
}(start)
|
||||
totalPages := uint64(1)
|
||||
var allSupports []jsonrpc.Claim
|
||||
defaultAccount, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
for page := uint64(1); page <= totalPages; page++ {
|
||||
supports, err := s.daemon.SupportList(&defaultAccount, page, 50)
|
||||
if err != nil {
|
||||
supports, err = s.daemon.SupportList(&defaultAccount, page, 50)
|
||||
if err != nil {
|
||||
return 0, errors.Prefix("cannot list supports", err)
|
||||
}
|
||||
}
|
||||
allSupports = append(allSupports, (*supports).Items...)
|
||||
totalPages = (*supports).TotalPages
|
||||
}
|
||||
producerWG := &stop.Group{}
|
||||
|
||||
claimIDChan := make(chan string, len(allSupports))
|
||||
abandonRspChan := make(chan abandonResponse, len(allSupports))
|
||||
alreadyAbandoned := make(map[string]bool, len(allSupports))
|
||||
producerWG.Add(1)
|
||||
go func() {
|
||||
defer producerWG.Done()
|
||||
for _, support := range allSupports {
|
||||
_, ok := alreadyAbandoned[support.ClaimID]
|
||||
if ok {
|
||||
continue
|
||||
}
|
||||
alreadyAbandoned[support.ClaimID] = true
|
||||
claimIDChan <- support.ClaimID
|
||||
}
|
||||
}()
|
||||
consumerWG := &stop.Group{}
|
||||
//TODO: remove this once the SDK team fixes their RPC bugs....
|
||||
s.daemon.SetRPCTimeout(60 * time.Second)
|
||||
defer s.daemon.SetRPCTimeout(5 * time.Minute)
|
||||
for i := 0; i < s.Manager.CliFlags.ConcurrentJobs; i++ {
|
||||
consumerWG.Add(1)
|
||||
go func() {
|
||||
defer consumerWG.Done()
|
||||
outer:
|
||||
for {
|
||||
claimID, more := <-claimIDChan
|
||||
if !more {
|
||||
return
|
||||
} else {
|
||||
summary, err := s.daemon.TxoSpend(util.PtrToString("support"), &claimID, nil, nil, nil, &defaultAccount)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "Client.Timeout exceeded while awaiting headers") {
|
||||
log.Errorf("Support abandon for %s timed out, retrying...", claimID)
|
||||
summary, err = s.daemon.TxoSpend(util.PtrToString("support"), &claimID, nil, nil, nil, &defaultAccount)
|
||||
if err != nil {
|
||||
//TODO GUESS HOW MUCH LBC WAS RELEASED THAT WE DON'T KNOW ABOUT, because screw you SDK
|
||||
abandonRspChan <- abandonResponse{
|
||||
ClaimID: claimID,
|
||||
Error: err,
|
||||
Amount: 0, // this is likely wrong, but oh well... there is literally nothing I can do about it
|
||||
}
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
abandonRspChan <- abandonResponse{
|
||||
ClaimID: claimID,
|
||||
Error: err,
|
||||
Amount: 0,
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
if summary == nil || len(*summary) < 1 {
|
||||
abandonRspChan <- abandonResponse{
|
||||
ClaimID: claimID,
|
||||
Error: errors.Err("error abandoning supports: no outputs while abandoning %s", claimID),
|
||||
Amount: 0,
|
||||
}
|
||||
continue
|
||||
}
|
||||
var outputAmount float64
|
||||
for _, tx := range *summary {
|
||||
amount, err := strconv.ParseFloat(tx.Outputs[0].Amount, 64)
|
||||
if err != nil {
|
||||
abandonRspChan <- abandonResponse{
|
||||
ClaimID: claimID,
|
||||
Error: errors.Err(err),
|
||||
Amount: 0,
|
||||
}
|
||||
continue outer
|
||||
}
|
||||
outputAmount += amount
|
||||
}
|
||||
if err != nil {
|
||||
abandonRspChan <- abandonResponse{
|
||||
ClaimID: claimID,
|
||||
Error: errors.Err(err),
|
||||
Amount: 0,
|
||||
}
|
||||
continue
|
||||
}
|
||||
log.Infof("Abandoned supports of %.4f LBC for claim %s", outputAmount, claimID)
|
||||
abandonRspChan <- abandonResponse{
|
||||
ClaimID: claimID,
|
||||
Error: nil,
|
||||
Amount: outputAmount,
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
producerWG.Wait()
|
||||
close(claimIDChan)
|
||||
consumerWG.Wait()
|
||||
close(abandonRspChan)
|
||||
|
||||
totalAbandoned := 0.0
|
||||
for r := range abandonRspChan {
|
||||
if r.Error != nil {
|
||||
log.Errorf("Failed abandoning supports for %s: %s", r.ClaimID, r.Error.Error())
|
||||
continue
|
||||
}
|
||||
totalAbandoned += r.Amount
|
||||
}
|
||||
return totalAbandoned, nil
|
||||
}
|
||||
|
||||
type updateInfo struct {
|
||||
ClaimID string
|
||||
streamUpdateOptions *jsonrpc.StreamUpdateOptions
|
||||
videoStatus *shared.VideoStatus
|
||||
}
|
||||
|
||||
func transferVideos(s *Sync) error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("transferVideos").Add(time.Since(start))
|
||||
}(start)
|
||||
cleanTransfer := true
|
||||
|
||||
streamChan := make(chan updateInfo, s.Manager.CliFlags.ConcurrentJobs)
|
||||
account, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
streams, err := s.daemon.StreamList(&account, 1, 30000)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
producerWG := &stop.Group{}
|
||||
producerWG.Add(1)
|
||||
go func() {
|
||||
defer producerWG.Done()
|
||||
for _, video := range s.syncedVideos {
|
||||
if !video.Published || video.Transferred || video.MetadataVersion != shared.LatestMetadataVersion {
|
||||
continue
|
||||
}
|
||||
|
||||
var stream *jsonrpc.Claim = nil
|
||||
for _, c := range streams.Items {
|
||||
if c.ClaimID != video.ClaimID || (c.SigningChannel != nil && c.SigningChannel.ClaimID != s.DbChannelData.ChannelClaimID) {
|
||||
continue
|
||||
}
|
||||
stream = &c
|
||||
break
|
||||
}
|
||||
if stream == nil {
|
||||
return
|
||||
}
|
||||
|
||||
streamUpdateOptions := jsonrpc.StreamUpdateOptions{
|
||||
StreamCreateOptions: &jsonrpc.StreamCreateOptions{
|
||||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
|
||||
ClaimAddress: &s.DbChannelData.PublishAddress.Address,
|
||||
FundingAccountIDs: []string{
|
||||
account,
|
||||
},
|
||||
},
|
||||
},
|
||||
Bid: util.PtrToString("0.005"), // Todo - Dont hardcode
|
||||
}
|
||||
videoStatus := shared.VideoStatus{
|
||||
ChannelID: s.DbChannelData.ChannelId,
|
||||
VideoID: video.VideoID,
|
||||
ClaimID: video.ClaimID,
|
||||
ClaimName: video.ClaimName,
|
||||
Status: shared.VideoStatusPublished,
|
||||
IsTransferred: util.PtrToBool(true),
|
||||
}
|
||||
streamChan <- updateInfo{
|
||||
ClaimID: video.ClaimID,
|
||||
streamUpdateOptions: &streamUpdateOptions,
|
||||
videoStatus: &videoStatus,
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
consumerWG := &stop.Group{}
|
||||
for i := 0; i < s.Manager.CliFlags.ConcurrentJobs; i++ {
|
||||
consumerWG.Add(1)
|
||||
go func(worker int) {
|
||||
defer consumerWG.Done()
|
||||
for {
|
||||
ui, more := <-streamChan
|
||||
if !more {
|
||||
return
|
||||
} else {
|
||||
err := s.streamUpdate(&ui)
|
||||
if err != nil {
|
||||
cleanTransfer = false
|
||||
}
|
||||
}
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
producerWG.Wait()
|
||||
close(streamChan)
|
||||
consumerWG.Wait()
|
||||
|
||||
if !cleanTransfer {
|
||||
return errors.Err("A video has failed to transfer for the channel...skipping channel transfer")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Sync) streamUpdate(ui *updateInfo) error {
|
||||
start := time.Now()
|
||||
result, updateError := s.daemon.StreamUpdate(ui.ClaimID, *ui.streamUpdateOptions)
|
||||
timing.TimedComponent("transferStreamUpdate").Add(time.Since(start))
|
||||
if updateError != nil {
|
||||
ui.videoStatus.FailureReason = updateError.Error()
|
||||
ui.videoStatus.Status = shared.VideoStatusTranferFailed
|
||||
ui.videoStatus.IsTransferred = util.PtrToBool(false)
|
||||
} else {
|
||||
ui.videoStatus.IsTransferred = util.PtrToBool(len(result.Outputs) != 0)
|
||||
}
|
||||
log.Infof("TRANSFERRED %t", *ui.videoStatus.IsTransferred)
|
||||
statusErr := s.Manager.ApiConfig.MarkVideoStatus(*ui.videoStatus)
|
||||
if statusErr != nil {
|
||||
return errors.Prefix(statusErr.Error(), updateError)
|
||||
}
|
||||
return errors.Err(updateError)
|
||||
}
|
||||
|
||||
func transferChannel(s *Sync) error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("transferChannel").Add(time.Since(start))
|
||||
}(start)
|
||||
account, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
channelClaims, err := s.daemon.ChannelList(&account, 1, 50, nil)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
var channelClaim *jsonrpc.Transaction = nil
|
||||
for _, c := range channelClaims.Items {
|
||||
if c.ClaimID != s.DbChannelData.ChannelClaimID {
|
||||
continue
|
||||
}
|
||||
channelClaim = &c
|
||||
break
|
||||
}
|
||||
if channelClaim == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
updateOptions := jsonrpc.ChannelUpdateOptions{
|
||||
Bid: util.PtrToString(fmt.Sprintf("%.6f", channelClaimAmount-0.005)),
|
||||
ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
|
||||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
|
||||
ClaimAddress: &s.DbChannelData.PublishAddress.Address,
|
||||
},
|
||||
},
|
||||
}
|
||||
result, err := s.daemon.ChannelUpdate(s.DbChannelData.ChannelClaimID, updateOptions)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
log.Infof("TRANSFERRED %t", len(result.Outputs) != 0)
|
||||
|
||||
return nil
|
||||
}
|
||||
+610
-490
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,31 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"os"
|
||||
"regexp"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
var (
|
||||
Durations = promauto.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: "ytsync",
|
||||
Subsystem: getHostname(),
|
||||
Name: "duration",
|
||||
Help: "The durations of the individual modules",
|
||||
}, []string{"path"})
|
||||
)
|
||||
|
||||
func getHostname() string {
|
||||
hostname, err := os.Hostname()
|
||||
if err != nil {
|
||||
hostname = "ytsync_unknown"
|
||||
}
|
||||
reg, err := regexp.Compile("[^a-zA-Z0-9_]+")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
return reg.ReplaceAllString(hostname, "_")
|
||||
}
|
||||
+30
-5
@@ -10,7 +10,7 @@ import (
|
||||
"sync"
|
||||
)
|
||||
|
||||
var titleRegexp = regexp.MustCompile(`[^a-zA-Z0-9]+`)
|
||||
var claimNameRegexp = regexp.MustCompile(`[=&#:$@%??;、\\"/<>%{}||^~\x60[\]\s]`)
|
||||
|
||||
type Namer struct {
|
||||
mu *sync.Mutex
|
||||
@@ -43,9 +43,16 @@ func (n *Namer) GetNextName(prefix string) string {
|
||||
}
|
||||
|
||||
//if for some reasons the title can't be converted in a valid claim name (too short or not latin) then we use a hash
|
||||
attempt = 1
|
||||
if len(name) < 2 {
|
||||
sum := md5.Sum([]byte(prefix))
|
||||
name = fmt.Sprintf("%s-%d", hex.EncodeToString(sum[:])[:15], attempt)
|
||||
for {
|
||||
name = fmt.Sprintf("%s-%d", hex.EncodeToString(sum[:])[:15], attempt)
|
||||
if _, exists := n.names[name]; !exists {
|
||||
break
|
||||
}
|
||||
attempt++
|
||||
}
|
||||
}
|
||||
|
||||
n.names[name] = true
|
||||
@@ -61,18 +68,21 @@ func getClaimNameFromTitle(title string, attempt int) string {
|
||||
}
|
||||
maxLen := 40 - len(suffix)
|
||||
|
||||
chunks := strings.Split(strings.ToLower(strings.Trim(titleRegexp.ReplaceAllString(title, "-"), "-")), "-")
|
||||
chunks := strings.Split(strings.ToLower(strings.Trim(claimNameRegexp.ReplaceAllString(title, "-"), "-")), "-")
|
||||
|
||||
name := chunks[0]
|
||||
if len(name) > maxLen {
|
||||
return name[:maxLen]
|
||||
return truncateUnicode(name, maxLen) + suffix
|
||||
}
|
||||
|
||||
for _, chunk := range chunks[1:] {
|
||||
if chunk == "" {
|
||||
continue
|
||||
}
|
||||
tmpName := name + "-" + chunk
|
||||
if len(tmpName) > maxLen {
|
||||
if len(name) < 20 {
|
||||
name = tmpName[:maxLen]
|
||||
name = truncateUnicode(tmpName, maxLen-len(name))
|
||||
}
|
||||
break
|
||||
}
|
||||
@@ -81,3 +91,18 @@ func getClaimNameFromTitle(title string, attempt int) string {
|
||||
|
||||
return name + suffix
|
||||
}
|
||||
|
||||
func truncateUnicode(name string, limit int) string {
|
||||
reNameBlacklist := regexp.MustCompile(`(&|>|<|\/|:|\n|\r)*`)
|
||||
name = reNameBlacklist.ReplaceAllString(name, "")
|
||||
result := name
|
||||
chars := 0
|
||||
for i := range name {
|
||||
if chars >= limit {
|
||||
result = name[:i]
|
||||
break
|
||||
}
|
||||
chars++
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
package namer
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func Test_getClaimNameFromTitle(t *testing.T) {
|
||||
name := getClaimNameFromTitle("СтопХам - \"В ожидании ответа\"", 0)
|
||||
assert.Equal(t, "стопхам-в-ожидании", name)
|
||||
name = getClaimNameFromTitle("SADB - \"A Weak Woman With a Strong Hood\"", 0)
|
||||
assert.Equal(t, "sadb-a-weak-woman-with-a-strong-hood", name)
|
||||
name = getClaimNameFromTitle("錢包整理術 5 Tips、哪種錢包最NG?|有錢人默默在做的「錢包整理術」 ft.@SHIN LI", 0)
|
||||
assert.Equal(t, "錢包整理術-5-tips-哪種錢包最ng", name)
|
||||
name = getClaimNameFromTitle("اسرع-طريقة-لتختيم", 0)
|
||||
assert.Equal(t, "اسرع-طريقة-لتختيم", name)
|
||||
name = getClaimNameFromTitle("شكرا على 380 مشترك😍😍😍😍 لي يريد دعم ادا وصلنا المقطع 40 لايك وراح ادعم قناتين", 0)
|
||||
assert.Equal(t, "شكرا-على-380-مشترك😍😍😍", name)
|
||||
name = getClaimNameFromTitle("test-@", 0)
|
||||
assert.Equal(t, "test", name)
|
||||
name = getClaimNameFromTitle("『あなたはただの空の殻でした』", 0)
|
||||
assert.Equal(t, "『あなたはただの空の殻でした』", name)
|
||||
name = getClaimNameFromTitle("精靈樂章-這樣的夥伴沒問題嗎 幽暗隕石坑(夢魘) 王有無敵狀態...要會閃不然會被秒(無課)", 2)
|
||||
assert.Equal(t, "精靈樂章-這樣的夥伴沒問題嗎-2", name)
|
||||
name = getClaimNameFromTitle("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 50)
|
||||
assert.Equal(t, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-50", name)
|
||||
}
|
||||
+263
-58
@@ -11,8 +11,11 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
"github.com/lbryio/lbry.go/extras/null"
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/null"
|
||||
"github.com/lbryio/ytsync/v5/shared"
|
||||
|
||||
"github.com/lbryio/ytsync/v5/util"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
@@ -28,47 +31,42 @@ type APIConfig struct {
|
||||
HostName string
|
||||
}
|
||||
|
||||
type SyncProperties struct {
|
||||
SyncFrom int64
|
||||
SyncUntil int64
|
||||
YoutubeChannelID string
|
||||
}
|
||||
|
||||
type Fee struct {
|
||||
Amount string `json:"amount"`
|
||||
Address string `json:"address"`
|
||||
Currency string `json:"currency"`
|
||||
}
|
||||
type YoutubeChannel struct {
|
||||
ChannelId string `json:"channel_id"`
|
||||
TotalVideos uint `json:"total_videos"`
|
||||
DesiredChannelName string `json:"desired_channel_name"`
|
||||
Fee *Fee `json:"fee"`
|
||||
ChannelClaimID string `json:"channel_claim_id"`
|
||||
}
|
||||
|
||||
func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeChannel, error) {
|
||||
func (a *APIConfig) FetchChannels(status string, cliFlags *shared.SyncFlags) ([]shared.YoutubeChannel, error) {
|
||||
type apiJobsResponse struct {
|
||||
Success bool `json:"success"`
|
||||
Error null.String `json:"error"`
|
||||
Data []YoutubeChannel `json:"data"`
|
||||
Success bool `json:"success"`
|
||||
Error null.String `json:"error"`
|
||||
Data []shared.YoutubeChannel `json:"data"`
|
||||
}
|
||||
endpoint := a.ApiURL + "/yt/jobs"
|
||||
res, _ := http.PostForm(endpoint, url.Values{
|
||||
res, err := http.PostForm(endpoint, url.Values{
|
||||
"auth_token": {a.ApiToken},
|
||||
"sync_status": {status},
|
||||
"min_videos": {strconv.Itoa(1)},
|
||||
"after": {strconv.Itoa(int(cp.SyncFrom))},
|
||||
"before": {strconv.Itoa(int(cp.SyncUntil))},
|
||||
"after": {strconv.Itoa(int(cliFlags.SyncFrom))},
|
||||
"before": {strconv.Itoa(int(cliFlags.SyncUntil))},
|
||||
"sync_server": {a.HostName},
|
||||
"channel_id": {cp.YoutubeChannelID},
|
||||
"channel_id": {cliFlags.ChannelID},
|
||||
})
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.FetchChannels(status, cliFlags)
|
||||
}
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
if res.StatusCode != http.StatusOK {
|
||||
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
|
||||
log.Debugln(string(body))
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.FetchChannels(status, cliFlags)
|
||||
}
|
||||
var response apiJobsResponse
|
||||
err := json.Unmarshal(body, &response)
|
||||
err = json.Unmarshal(body, &response)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
if response.Data == nil {
|
||||
return nil, errors.Err(response.Error)
|
||||
@@ -85,6 +83,8 @@ type SyncedVideo struct {
|
||||
ClaimID string `json:"claim_id"`
|
||||
Size int64 `json:"size"`
|
||||
MetadataVersion int8 `json:"metadata_version"`
|
||||
Transferred bool `json:"transferred"`
|
||||
IsLbryFirst bool `json:"is_lbry_first"`
|
||||
}
|
||||
|
||||
func sanitizeFailureReason(s *string) {
|
||||
@@ -95,7 +95,50 @@ func sanitizeFailureReason(s *string) {
|
||||
*s = (*s)[:MaxReasonLength]
|
||||
}
|
||||
}
|
||||
func (a *APIConfig) SetChannelStatus(channelID string, status string, failureReason string) (map[string]SyncedVideo, map[string]bool, error) {
|
||||
|
||||
func (a *APIConfig) SetChannelCert(certHex string, channelID string) error {
|
||||
type apiSetChannelCertResponse struct {
|
||||
Success bool `json:"success"`
|
||||
Error null.String `json:"error"`
|
||||
Data string `json:"data"`
|
||||
}
|
||||
|
||||
endpoint := a.ApiURL + "/yt/channel_cert"
|
||||
|
||||
res, err := http.PostForm(endpoint, url.Values{
|
||||
"channel_claim_id": {channelID},
|
||||
"channel_cert": {certHex},
|
||||
"auth_token": {a.ApiToken},
|
||||
})
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.SetChannelCert(certHex, channelID)
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
if res.StatusCode != http.StatusOK {
|
||||
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
|
||||
log.Debugln(string(body))
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.SetChannelCert(certHex, channelID)
|
||||
}
|
||||
var response apiSetChannelCertResponse
|
||||
err = json.Unmarshal(body, &response)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
if !response.Error.IsNull() {
|
||||
return errors.Err(response.Error.String)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *APIConfig) SetChannelStatus(channelID string, status string, failureReason string, transferState *int) (map[string]SyncedVideo, map[string]bool, error) {
|
||||
type apiChannelStatusResponse struct {
|
||||
Success bool `json:"success"`
|
||||
Error null.String `json:"error"`
|
||||
@@ -104,19 +147,37 @@ func (a *APIConfig) SetChannelStatus(channelID string, status string, failureRea
|
||||
endpoint := a.ApiURL + "/yt/channel_status"
|
||||
|
||||
sanitizeFailureReason(&failureReason)
|
||||
res, _ := http.PostForm(endpoint, url.Values{
|
||||
params := url.Values{
|
||||
"channel_id": {channelID},
|
||||
"sync_server": {a.HostName},
|
||||
"auth_token": {a.ApiToken},
|
||||
"sync_status": {status},
|
||||
"failure_reason": {failureReason},
|
||||
})
|
||||
}
|
||||
if transferState != nil {
|
||||
params.Add("transfer_state", strconv.Itoa(*transferState))
|
||||
}
|
||||
res, err := http.PostForm(endpoint, params)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.SetChannelStatus(channelID, status, failureReason, transferState)
|
||||
}
|
||||
return nil, nil, errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
if res.StatusCode >= http.StatusInternalServerError {
|
||||
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
|
||||
log.Debugln(string(body))
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.SetChannelStatus(channelID, status, failureReason, transferState)
|
||||
}
|
||||
var response apiChannelStatusResponse
|
||||
err := json.Unmarshal(body, &response)
|
||||
err = json.Unmarshal(body, &response)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return nil, nil, errors.Err(err)
|
||||
}
|
||||
if !response.Error.IsNull() {
|
||||
return nil, nil, errors.Err(response.Error.String)
|
||||
@@ -142,15 +203,29 @@ func (a *APIConfig) SetChannelClaimID(channelID string, channelClaimID string) e
|
||||
Data string `json:"data"`
|
||||
}
|
||||
endpoint := a.ApiURL + "/yt/set_channel_claim_id"
|
||||
res, _ := http.PostForm(endpoint, url.Values{
|
||||
res, err := http.PostForm(endpoint, url.Values{
|
||||
"channel_id": {channelID},
|
||||
"auth_token": {a.ApiToken},
|
||||
"channel_claim_id": {channelClaimID},
|
||||
})
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.SetChannelClaimID(channelID, channelClaimID)
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
if res.StatusCode != http.StatusOK {
|
||||
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
|
||||
log.Debugln(string(body))
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.SetChannelClaimID(channelID, channelClaimID)
|
||||
}
|
||||
var response apiChannelStatusResponse
|
||||
err := json.Unmarshal(body, &response)
|
||||
err = json.Unmarshal(body, &response)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
@@ -176,15 +251,29 @@ func (a *APIConfig) DeleteVideos(videos []string) error {
|
||||
"video_ids": {videoIDs},
|
||||
"auth_token": {a.ApiToken},
|
||||
}
|
||||
res, _ := http.PostForm(endpoint, vals)
|
||||
res, err := http.PostForm(endpoint, vals)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.DeleteVideos(videos)
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
if res.StatusCode != http.StatusOK {
|
||||
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
|
||||
log.Debugln(string(body))
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.DeleteVideos(videos)
|
||||
}
|
||||
var response struct {
|
||||
Success bool `json:"success"`
|
||||
Error null.String `json:"error"`
|
||||
Data null.String `json:"data"`
|
||||
}
|
||||
err := json.Unmarshal(body, &response)
|
||||
err = json.Unmarshal(body, &response)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
@@ -198,42 +287,59 @@ func (a *APIConfig) DeleteVideos(videos []string) error {
|
||||
return errors.Err("invalid API response. Status code: %d", res.StatusCode)
|
||||
}
|
||||
|
||||
func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status string, claimID string, claimName string, failureReason string, size *int64, metadataVersion uint) error {
|
||||
func (a *APIConfig) MarkVideoStatus(status shared.VideoStatus) error {
|
||||
endpoint := a.ApiURL + "/yt/video_status"
|
||||
|
||||
sanitizeFailureReason(&failureReason)
|
||||
sanitizeFailureReason(&status.FailureReason)
|
||||
vals := url.Values{
|
||||
"youtube_channel_id": {channelID},
|
||||
"video_id": {videoID},
|
||||
"status": {status},
|
||||
"youtube_channel_id": {status.ChannelID},
|
||||
"video_id": {status.VideoID},
|
||||
"status": {status.Status},
|
||||
"auth_token": {a.ApiToken},
|
||||
}
|
||||
if status == VideoStatusPublished || status == VideoStatusUpgradeFailed {
|
||||
if claimID == "" || claimName == "" {
|
||||
return errors.Err("claimID (%s) or claimName (%s) missing", claimID, claimName)
|
||||
if status.Status == VideoStatusPublished || status.Status == VideoStatusUpgradeFailed {
|
||||
if status.ClaimID == "" || status.ClaimName == "" {
|
||||
return errors.Err("claimID (%s) or claimName (%s) missing", status.ClaimID, status.ClaimName)
|
||||
}
|
||||
vals.Add("published_at", strconv.FormatInt(time.Now().Unix(), 10))
|
||||
vals.Add("claim_id", claimID)
|
||||
vals.Add("claim_name", claimName)
|
||||
if metadataVersion > 0 {
|
||||
vals.Add("metadata_version", fmt.Sprintf("%d", metadataVersion))
|
||||
vals.Add("claim_id", status.ClaimID)
|
||||
vals.Add("claim_name", status.ClaimName)
|
||||
if status.MetaDataVersion > 0 {
|
||||
vals.Add("metadata_version", fmt.Sprintf("%d", status.MetaDataVersion))
|
||||
}
|
||||
if size != nil {
|
||||
vals.Add("size", strconv.FormatInt(*size, 10))
|
||||
if status.Size != nil {
|
||||
vals.Add("size", strconv.FormatInt(*status.Size, 10))
|
||||
}
|
||||
}
|
||||
if failureReason != "" {
|
||||
vals.Add("failure_reason", failureReason)
|
||||
if status.FailureReason != "" {
|
||||
vals.Add("failure_reason", status.FailureReason)
|
||||
}
|
||||
if status.IsTransferred != nil {
|
||||
vals.Add("transferred", strconv.FormatBool(*status.IsTransferred))
|
||||
}
|
||||
res, err := http.PostForm(endpoint, vals)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s for %s. Waiting to retry", endpoint, status.ClaimName)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.MarkVideoStatus(status)
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
res, _ := http.PostForm(endpoint, vals)
|
||||
defer res.Body.Close()
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
if res.StatusCode != http.StatusOK {
|
||||
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
|
||||
log.Debugln(string(body))
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.MarkVideoStatus(status)
|
||||
}
|
||||
var response struct {
|
||||
Success bool `json:"success"`
|
||||
Error null.String `json:"error"`
|
||||
Data null.String `json:"data"`
|
||||
}
|
||||
err := json.Unmarshal(body, &response)
|
||||
err = json.Unmarshal(body, &response)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -245,3 +351,102 @@ func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status str
|
||||
}
|
||||
return errors.Err("invalid API response. Status code: %d", res.StatusCode)
|
||||
}
|
||||
|
||||
func (a *APIConfig) VideoState(videoID string) (string, error) {
|
||||
endpoint := a.ApiURL + "/yt/video_state"
|
||||
vals := url.Values{
|
||||
"video_id": {videoID},
|
||||
"auth_token": {a.ApiToken},
|
||||
}
|
||||
|
||||
res, err := http.PostForm(endpoint, vals)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.VideoState(videoID)
|
||||
}
|
||||
return "", errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
if res.StatusCode == http.StatusNotFound {
|
||||
return "not_found", nil
|
||||
}
|
||||
if res.StatusCode != http.StatusOK {
|
||||
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
|
||||
log.Debugln(string(body))
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.VideoState(videoID)
|
||||
}
|
||||
var response struct {
|
||||
Success bool `json:"success"`
|
||||
Error null.String `json:"error"`
|
||||
Data null.String `json:"data"`
|
||||
}
|
||||
err = json.Unmarshal(body, &response)
|
||||
if err != nil {
|
||||
return "", errors.Err(err)
|
||||
}
|
||||
if !response.Error.IsNull() {
|
||||
return "", errors.Err(response.Error.String)
|
||||
}
|
||||
if !response.Data.IsNull() {
|
||||
return response.Data.String, nil
|
||||
}
|
||||
return "", errors.Err("invalid API response. Status code: %d", res.StatusCode)
|
||||
}
|
||||
|
||||
type VideoRelease struct {
|
||||
ID uint64 `json:"id"`
|
||||
YoutubeDataID uint64 `json:"youtube_data_id"`
|
||||
VideoID string `json:"video_id"`
|
||||
ReleaseTime string `json:"release_time"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
}
|
||||
|
||||
func (a *APIConfig) GetReleasedDate(videoID string) (*VideoRelease, error) {
|
||||
endpoint := a.ApiURL + "/yt/released"
|
||||
vals := url.Values{
|
||||
"video_id": {videoID},
|
||||
"auth_token": {a.ApiToken},
|
||||
}
|
||||
|
||||
res, err := http.PostForm(endpoint, vals)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.GetReleasedDate(videoID)
|
||||
}
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
if res.StatusCode == http.StatusNotFound {
|
||||
return nil, nil
|
||||
}
|
||||
if res.StatusCode != http.StatusOK {
|
||||
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
|
||||
log.Debugln(string(body))
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.GetReleasedDate(videoID)
|
||||
}
|
||||
var response struct {
|
||||
Success bool `json:"success"`
|
||||
Error null.String `json:"error"`
|
||||
Data VideoRelease `json:"data"`
|
||||
}
|
||||
err = json.Unmarshal(body, &response)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
if !response.Error.IsNull() {
|
||||
return nil, errors.Err(response.Error.String)
|
||||
}
|
||||
if response.Data.ReleaseTime != "" {
|
||||
return &response.Data, nil
|
||||
}
|
||||
return nil, errors.Err("invalid API response. Status code: %d", res.StatusCode)
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
package shared
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
)
|
||||
|
||||
type Fee struct {
|
||||
Amount string `json:"amount"`
|
||||
Address string `json:"address"`
|
||||
Currency string `json:"currency"`
|
||||
}
|
||||
type YoutubeChannel struct {
|
||||
ChannelId string `json:"channel_id"`
|
||||
TotalVideos uint `json:"total_videos"`
|
||||
TotalSubscribers uint `json:"total_subscribers"`
|
||||
DesiredChannelName string `json:"desired_channel_name"`
|
||||
Fee *Fee `json:"fee"`
|
||||
ChannelClaimID string `json:"channel_claim_id"`
|
||||
TransferState int `json:"transfer_state"`
|
||||
PublishAddress PublishAddress `json:"publish_address"`
|
||||
PublicKey string `json:"public_key"`
|
||||
LengthLimit int `json:"length_limit"`
|
||||
SizeLimit int `json:"size_limit"`
|
||||
LastUploadedVideo string `json:"last_uploaded_video"`
|
||||
WipeDB bool `json:"wipe_db"`
|
||||
}
|
||||
|
||||
type PublishAddress struct {
|
||||
Address string `json:"address"`
|
||||
IsMine bool `json:"is_mine"`
|
||||
}
|
||||
|
||||
func (p *PublishAddress) UnmarshalJSON(data []byte) error {
|
||||
var s string
|
||||
if err := json.Unmarshal(data, &s); err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
p.Address = s
|
||||
p.IsMine = false
|
||||
return nil
|
||||
}
|
||||
|
||||
var NeverRetryFailures = []string{
|
||||
"Error extracting sts from embedded url response",
|
||||
"Unable to extract signature tokens",
|
||||
"the video is too big to sync, skipping for now",
|
||||
"video is too long to process",
|
||||
"video is too short to process",
|
||||
"This video contains content from",
|
||||
"no compatible format available for this video",
|
||||
"Watch this video on YouTube.",
|
||||
"have blocked it on copyright grounds",
|
||||
"giving up after 0 fragment retries",
|
||||
"Sign in to confirm your age",
|
||||
}
|
||||
|
||||
type SyncFlags struct {
|
||||
TakeOverExistingChannel bool
|
||||
SkipSpaceCheck bool
|
||||
SyncUpdate bool
|
||||
SingleRun bool
|
||||
RemoveDBUnpublished bool
|
||||
UpgradeMetadata bool
|
||||
DisableTransfers bool
|
||||
QuickSync bool
|
||||
MaxTries int
|
||||
Refill int
|
||||
Limit int
|
||||
Status string
|
||||
SecondaryStatus string
|
||||
ChannelID string
|
||||
SyncFrom int64
|
||||
SyncUntil int64
|
||||
ConcurrentJobs int
|
||||
VideosLimit int
|
||||
MaxVideoSize int
|
||||
MaxVideoLength time.Duration
|
||||
}
|
||||
|
||||
// VideosToSync dynamically figures out how many videos should be synced for a given subs count if nothing was otherwise specified
|
||||
func (f *SyncFlags) VideosToSync(totalSubscribers uint) int {
|
||||
if f.VideosLimit > 0 {
|
||||
return f.VideosLimit
|
||||
}
|
||||
defaultVideosToSync := map[int]int{
|
||||
10000: 1000,
|
||||
5000: 500,
|
||||
1000: 400,
|
||||
800: 250,
|
||||
600: 200,
|
||||
200: 80,
|
||||
100: 50,
|
||||
1: 10,
|
||||
}
|
||||
videosToSync := 0
|
||||
for s, r := range defaultVideosToSync {
|
||||
if int(totalSubscribers) >= s && r > videosToSync {
|
||||
videosToSync = r
|
||||
}
|
||||
}
|
||||
return videosToSync
|
||||
}
|
||||
|
||||
func (f *SyncFlags) IsSingleChannelSync() bool {
|
||||
return f.ChannelID != ""
|
||||
}
|
||||
|
||||
type VideoStatus struct {
|
||||
ChannelID string
|
||||
VideoID string
|
||||
Status string
|
||||
ClaimID string
|
||||
ClaimName string
|
||||
FailureReason string
|
||||
Size *int64
|
||||
MetaDataVersion uint
|
||||
IsTransferred *bool
|
||||
}
|
||||
|
||||
const (
|
||||
StatusPending = "pending" // waiting for permission to sync
|
||||
StatusPendingEmail = "pendingemail" // permission granted but missing email
|
||||
StatusQueued = "queued" // in sync queue. will be synced soon
|
||||
StatusPendingUpgrade = "pendingupgrade" // in sync queue. will be synced soon
|
||||
StatusSyncing = "syncing" // syncing now
|
||||
StatusSynced = "synced" // done
|
||||
StatusWipeDb = "pendingdbwipe" // in sync queue. lbryum database will be pruned
|
||||
StatusFailed = "failed"
|
||||
StatusFinalized = "finalized" // no more changes allowed
|
||||
StatusAbandoned = "abandoned" // deleted on youtube or banned
|
||||
)
|
||||
|
||||
var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusPendingUpgrade, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned, StatusWipeDb}
|
||||
|
||||
const LatestMetadataVersion = 2
|
||||
|
||||
const (
|
||||
VideoStatusPublished = "published"
|
||||
VideoStatusFailed = "failed"
|
||||
VideoStatusUpgradeFailed = "upgradefailed"
|
||||
VideoStatusUnpublished = "unpublished"
|
||||
VideoStatusTranferFailed = "transferfailed"
|
||||
)
|
||||
|
||||
const (
|
||||
TransferStateNotTouched = iota
|
||||
TransferStatePending
|
||||
TransferStateComplete
|
||||
TransferStateManual
|
||||
)
|
||||
|
||||
type AwsConfigs struct {
|
||||
AwsS3ID string
|
||||
AwsS3Secret string
|
||||
AwsS3Region string
|
||||
AwsS3Bucket string
|
||||
}
|
||||
|
||||
func (a *AwsConfigs) GetS3AWSConfig() *aws.Config {
|
||||
return &aws.Config{
|
||||
Credentials: credentials.NewStaticCredentials(a.AwsS3ID, a.AwsS3Secret, ""),
|
||||
Region: &a.AwsS3Region,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package shared
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gotest.tools/assert"
|
||||
)
|
||||
|
||||
func TestSyncFlags_VideosToSync(t *testing.T) {
|
||||
f := SyncFlags{}
|
||||
assert.Equal(t, f.VideosToSync(0), 0)
|
||||
assert.Equal(t, f.VideosToSync(1), 10)
|
||||
assert.Equal(t, f.VideosToSync(5), 10)
|
||||
assert.Equal(t, f.VideosToSync(10), 10)
|
||||
assert.Equal(t, f.VideosToSync(101), 50)
|
||||
assert.Equal(t, f.VideosToSync(500), 80)
|
||||
assert.Equal(t, f.VideosToSync(21000), 1000)
|
||||
f.VideosLimit = 1337
|
||||
assert.Equal(t, f.VideosToSync(21), 1337)
|
||||
}
|
||||
+2
-2
@@ -4,8 +4,8 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/lbryio/lbry.go/extras/jsonrpc"
|
||||
"github.com/lbryio/ytsync/namer"
|
||||
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
||||
"github.com/lbryio/ytsync/v5/namer"
|
||||
)
|
||||
|
||||
type SyncSummary struct {
|
||||
|
||||
+496
-164
@@ -1,12 +1,13 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
@@ -14,21 +15,30 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
"github.com/lbryio/lbry.go/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/extras/util"
|
||||
"github.com/abadojack/whatlanggo"
|
||||
"github.com/lbryio/ytsync/v5/downloader"
|
||||
"github.com/lbryio/ytsync/v5/downloader/ytdl"
|
||||
"github.com/lbryio/ytsync/v5/shared"
|
||||
"github.com/vbauerster/mpb/v7"
|
||||
"github.com/vbauerster/mpb/v7/decor"
|
||||
"gopkg.in/vansante/go-ffprobe.v2"
|
||||
|
||||
"github.com/lbryio/ytsync/namer"
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
"github.com/lbryio/ytsync/tagsManager"
|
||||
"github.com/lbryio/ytsync/thumbs"
|
||||
"github.com/lbryio/ytsync/v5/ip_manager"
|
||||
"github.com/lbryio/ytsync/v5/namer"
|
||||
"github.com/lbryio/ytsync/v5/sdk"
|
||||
"github.com/lbryio/ytsync/v5/tags_manager"
|
||||
"github.com/lbryio/ytsync/v5/thumbs"
|
||||
"github.com/lbryio/ytsync/v5/timing"
|
||||
logUtils "github.com/lbryio/ytsync/v5/util"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
|
||||
"github.com/ChannelMeter/iso8601duration"
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/nikooo777/ytdl"
|
||||
"github.com/shopspring/decimal"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"google.golang.org/api/youtube/v3"
|
||||
)
|
||||
|
||||
type YoutubeVideo struct {
|
||||
@@ -38,10 +48,10 @@ type YoutubeVideo struct {
|
||||
playlistPosition int64
|
||||
size *int64
|
||||
maxVideoSize int64
|
||||
maxVideoLength float64
|
||||
maxVideoLength time.Duration
|
||||
publishedAt time.Time
|
||||
dir string
|
||||
youtubeInfo *youtube.Video
|
||||
youtubeInfo *ytdl.YtdlVideo
|
||||
youtubeChannelID string
|
||||
tags []string
|
||||
awsConfig aws.Config
|
||||
@@ -49,10 +59,12 @@ type YoutubeVideo struct {
|
||||
lbryChannelID string
|
||||
mocked bool
|
||||
walletLock *sync.RWMutex
|
||||
stopGroup *stop.Group
|
||||
pool *ip_manager.IPPool
|
||||
progressBars *mpb.Progress
|
||||
progressBarWg *sync.WaitGroup
|
||||
}
|
||||
|
||||
const reflectorURL = "http://blobs.lbry.io/"
|
||||
|
||||
var youtubeCategories = map[string]string{
|
||||
"1": "film & animation",
|
||||
"2": "autos & vehicles",
|
||||
@@ -88,22 +100,25 @@ var youtubeCategories = map[string]string{
|
||||
"44": "trailers",
|
||||
}
|
||||
|
||||
func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64, awsConfig aws.Config) *YoutubeVideo {
|
||||
publishedAt, _ := time.Parse(time.RFC3339Nano, videoData.Snippet.PublishedAt) // ignore parse errors
|
||||
func NewYoutubeVideo(directory string, videoData *ytdl.YtdlVideo, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) (*YoutubeVideo, error) {
|
||||
// youtube-dl returns times in local timezone sometimes. this could break in the future
|
||||
// maybe we can file a PR to choose the timezone we want from youtube-dl
|
||||
return &YoutubeVideo{
|
||||
id: videoData.Id,
|
||||
title: videoData.Snippet.Title,
|
||||
description: videoData.Snippet.Description,
|
||||
id: videoData.ID,
|
||||
title: videoData.Title,
|
||||
description: videoData.Description,
|
||||
playlistPosition: playlistPosition,
|
||||
publishedAt: publishedAt,
|
||||
publishedAt: videoData.UploadDateForReal,
|
||||
dir: directory,
|
||||
youtubeInfo: videoData,
|
||||
awsConfig: awsConfig,
|
||||
mocked: false,
|
||||
youtubeChannelID: videoData.Snippet.ChannelId,
|
||||
}
|
||||
youtubeChannelID: videoData.ChannelID,
|
||||
stopGroup: stopGroup,
|
||||
pool: pool,
|
||||
}, nil
|
||||
}
|
||||
func NewMockedVideo(directory string, videoID string, youtubeChannelID string, awsConfig aws.Config) *YoutubeVideo {
|
||||
func NewMockedVideo(directory string, videoID string, youtubeChannelID string, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo {
|
||||
return &YoutubeVideo{
|
||||
id: videoID,
|
||||
playlistPosition: 0,
|
||||
@@ -111,6 +126,8 @@ func NewMockedVideo(directory string, videoID string, youtubeChannelID string, a
|
||||
awsConfig: awsConfig,
|
||||
mocked: true,
|
||||
youtubeChannelID: youtubeChannelID,
|
||||
stopGroup: stopGroup,
|
||||
pool: pool,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -161,84 +178,49 @@ func (v *YoutubeVideo) getFullPath() string {
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) getAbbrevDescription() string {
|
||||
maxLines := 10
|
||||
maxLength := 2800
|
||||
description := strings.TrimSpace(v.description)
|
||||
additionalDescription := "\nhttps://www.youtube.com/watch?v=" + v.id
|
||||
khanAcademyClaimID := "5fc52291980268b82413ca4c0ace1b8d749f3ffb"
|
||||
if v.lbryChannelID == khanAcademyClaimID {
|
||||
additionalDescription = additionalDescription + "\nNote: All Khan Academy content is available for free at (www.khanacademy.org)"
|
||||
}
|
||||
if strings.Count(description, "\n") < maxLines {
|
||||
return description + "\n..." + additionalDescription
|
||||
if len(description) > maxLength {
|
||||
description = description[:maxLength]
|
||||
}
|
||||
return strings.Join(strings.Split(description, "\n")[:maxLines], "\n") + "\n..." + additionalDescription
|
||||
return description + "\n..." + additionalDescription
|
||||
}
|
||||
|
||||
var ipPool []string
|
||||
var IPIndex int
|
||||
|
||||
func getNextIP() (string, error) {
|
||||
if len(ipPool) < 1 {
|
||||
IPIndex = 0
|
||||
addrs, err := net.InterfaceAddrs()
|
||||
func checkCookiesIntegrity() error {
|
||||
fi, err := os.Stat("cookies.txt")
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
if fi.Size() == 0 {
|
||||
log.Errorf("cookies were cleared out. Attempting a restore from cookies-backup.txt")
|
||||
input, err := ioutil.ReadFile("cookies-backup.txt")
|
||||
if err != nil {
|
||||
return "", errors.Err(err)
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
for _, address := range addrs {
|
||||
if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
|
||||
if ipnet.IP.To16() != nil && len(ipnet.IP) == net.IPv6len {
|
||||
ipPool = append(ipPool, ipnet.IP.String())
|
||||
}
|
||||
}
|
||||
err = ioutil.WriteFile("cookies.txt", input, 0644)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
}
|
||||
nextIP := ipPool[IPIndex]
|
||||
if IPIndex == len(ipPool)-1 {
|
||||
IPIndex = 0
|
||||
} else {
|
||||
IPIndex++
|
||||
}
|
||||
return nextIP, nil
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) fallbackDownload() error {
|
||||
sourceAddress, err := getNextIP()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
cmd := exec.Command("youtube-dl",
|
||||
"--no-progress",
|
||||
"-fbestvideo[ext=mp4,height<=1080,filesize<2000M]+best[ext=mp4,height<=1080,filesize<2000M]",
|
||||
"-o"+strings.TrimRight(v.getFullPath(), ".mp4"),
|
||||
"--merge-output-format",
|
||||
"mp4",
|
||||
"-6",
|
||||
"--source-address",
|
||||
sourceAddress,
|
||||
"https://www.youtube.com/watch?v="+v.ID())
|
||||
|
||||
log.Printf("Running command and waiting for it to finish...")
|
||||
output, err := cmd.CombinedOutput()
|
||||
log.Debugln(string(output))
|
||||
if err != nil {
|
||||
log.Printf("Command finished with error: %v", errors.Err(string(output)))
|
||||
_ = v.delete()
|
||||
return errors.Err(err)
|
||||
}
|
||||
fi, err := os.Stat(v.getFullPath())
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
videoSize := fi.Size()
|
||||
v.size = &videoSize
|
||||
return nil
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) download() error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("download").Add(time.Since(start))
|
||||
}(start)
|
||||
if v.youtubeInfo.IsLive != nil {
|
||||
return errors.Err("video is a live stream and hasn't completed yet")
|
||||
}
|
||||
videoPath := v.getFullPath()
|
||||
|
||||
err := os.Mkdir(v.videoDir(), 0750)
|
||||
err := os.Mkdir(v.videoDir(), 0777)
|
||||
if err != nil && !strings.Contains(err.Error(), "file exists") {
|
||||
return errors.Wrap(err, 0)
|
||||
}
|
||||
@@ -250,77 +232,369 @@ func (v *YoutubeVideo) download() error {
|
||||
log.Debugln(v.id + " already exists at " + videoPath)
|
||||
return nil
|
||||
}
|
||||
qualities := []string{
|
||||
"1080",
|
||||
"720",
|
||||
"480",
|
||||
"320",
|
||||
}
|
||||
dur := time.Duration(v.youtubeInfo.Duration) * time.Second
|
||||
if dur.Hours() > 2 { //for videos longer than 2 hours only sync up to 720p
|
||||
qualities = []string{
|
||||
"720",
|
||||
"480",
|
||||
"320",
|
||||
}
|
||||
}
|
||||
|
||||
videoUrl := "https://www.youtube.com/watch?v=" + v.id
|
||||
videoInfo, err := ytdl.GetVideoInfo(videoUrl)
|
||||
metadataPath := path.Join(logUtils.GetVideoMetadataDir(), v.id+".info.json")
|
||||
_, err = os.Stat(metadataPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return errors.Err("metadata information for video %s is missing! Why?", v.id)
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
codec := []string{"H.264"}
|
||||
ext := []string{"mp4"}
|
||||
metadata, err := parseVideoMetadata(metadataPath)
|
||||
|
||||
//Filter requires a [] interface{}
|
||||
codecFilter := make([]interface{}, len(codec))
|
||||
for i, v := range codec {
|
||||
codecFilter[i] = v
|
||||
err = checkCookiesIntegrity()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
//Filter requires a [] interface{}
|
||||
extFilter := make([]interface{}, len(ext))
|
||||
for i, v := range ext {
|
||||
extFilter[i] = v
|
||||
ytdlArgs := []string{
|
||||
"--no-progress",
|
||||
"-o" + strings.TrimSuffix(v.getFullPath(), ".mp4"),
|
||||
"--merge-output-format",
|
||||
"mp4",
|
||||
"--postprocessor-args",
|
||||
"ffmpeg:-movflags faststart",
|
||||
"--abort-on-unavailable-fragment",
|
||||
"--fragment-retries",
|
||||
"1",
|
||||
"--cookies",
|
||||
"cookies.txt",
|
||||
"--load-info-json",
|
||||
metadataPath,
|
||||
}
|
||||
|
||||
formats := videoInfo.Formats.Filter(ytdl.FormatVideoEncodingKey, codecFilter).Filter(ytdl.FormatExtensionKey, extFilter)
|
||||
if len(formats) == 0 {
|
||||
return errors.Err("no compatible format available for this video")
|
||||
userAgent := []string{"--user-agent", downloader.ChromeUA}
|
||||
if v.maxVideoSize > 0 {
|
||||
ytdlArgs = append(ytdlArgs,
|
||||
"--max-filesize",
|
||||
fmt.Sprintf("%dM", v.maxVideoSize),
|
||||
)
|
||||
}
|
||||
maxRetryAttempts := 5
|
||||
isLengthLimitSet := v.maxVideoLength > 0.01
|
||||
if isLengthLimitSet && videoInfo.Duration.Hours() > v.maxVideoLength {
|
||||
return errors.Err("video is too long to process")
|
||||
if v.maxVideoLength > 0 {
|
||||
ytdlArgs = append(ytdlArgs,
|
||||
"--match-filter",
|
||||
fmt.Sprintf("duration <= %d", int(v.maxVideoLength.Seconds())),
|
||||
)
|
||||
}
|
||||
|
||||
for i := 0; i < len(formats) && i < maxRetryAttempts; i++ {
|
||||
formatIndex := i
|
||||
if i == maxRetryAttempts-1 {
|
||||
formatIndex = len(formats) - 1
|
||||
var sourceAddress string
|
||||
for {
|
||||
sourceAddress, err = v.pool.GetIP(v.id)
|
||||
if err != nil {
|
||||
if errors.Is(err, ip_manager.ErrAllThrottled) {
|
||||
select {
|
||||
case <-v.stopGroup.Ch():
|
||||
return errors.Err("interrupted by user")
|
||||
default:
|
||||
time.Sleep(ip_manager.IPCooldownPeriod)
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
var downloadedFile *os.File
|
||||
downloadedFile, err = os.Create(videoPath)
|
||||
break
|
||||
}
|
||||
defer v.pool.ReleaseIP(sourceAddress)
|
||||
|
||||
ytdlArgs = append(ytdlArgs,
|
||||
"--source-address",
|
||||
sourceAddress,
|
||||
fmt.Sprintf("https://www.youtube.com/watch?v=%s", v.id),
|
||||
)
|
||||
|
||||
for i := 0; i < len(qualities); i++ {
|
||||
quality := qualities[i]
|
||||
argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][vcodec!*=av01][height<="+quality+"]+bestaudio[ext!=webm][format_id!=258][format_id!=251][format_id!=256][format_id!=327]")
|
||||
argsWithFilters = append(argsWithFilters, userAgent...)
|
||||
cmd := exec.Command("yt-dlp", argsWithFilters...)
|
||||
log.Printf("Running command yt-dlp %s", strings.Join(argsWithFilters, " "))
|
||||
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
err = videoInfo.Download(formats[formatIndex], downloadedFile)
|
||||
_ = downloadedFile.Close()
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
//delete the video and ignore the error
|
||||
_ = v.delete()
|
||||
return errors.Err(err.Error())
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
ticker := time.NewTicker(400 * time.Millisecond)
|
||||
done := make(chan bool, 1)
|
||||
v.progressBarWg.Add(1)
|
||||
go func() {
|
||||
defer v.progressBarWg.Done()
|
||||
//get size of the video before downloading
|
||||
cmd := exec.Command("yt-dlp", append(argsWithFilters, "-s")...)
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
log.Errorf("error while getting final file size: %s", errors.FullTrace(err))
|
||||
return
|
||||
}
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
log.Errorf("error while getting final file size: %s", errors.FullTrace(err))
|
||||
return
|
||||
}
|
||||
outLog, _ := ioutil.ReadAll(stdout)
|
||||
err = cmd.Wait()
|
||||
output := string(outLog)
|
||||
parts := strings.Split(output, ": ")
|
||||
if len(parts) != 3 {
|
||||
log.Errorf("couldn't parse audio and video parts from the output (%s)", output)
|
||||
return
|
||||
}
|
||||
formats := strings.Split(parts[2], "+")
|
||||
if len(formats) != 2 {
|
||||
log.Errorf("couldn't parse formats from the output (%s)", output)
|
||||
return
|
||||
}
|
||||
log.Debugf("'%s'", output)
|
||||
videoFormat := formats[0]
|
||||
audioFormat := strings.Replace(formats[1], "\n", "", -1)
|
||||
|
||||
videoSize := 0
|
||||
audioSize := 0
|
||||
if metadata != nil {
|
||||
for _, f := range metadata.Formats {
|
||||
if f.FormatID == videoFormat {
|
||||
videoSize = f.Filesize
|
||||
}
|
||||
if f.FormatID == audioFormat {
|
||||
audioSize = f.Filesize
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log.Debugf("(%s) - videoSize: %d (%s), audiosize: %d (%s)", v.id, videoSize, videoFormat, audioSize, audioFormat)
|
||||
bar := v.progressBars.AddBar(int64(videoSize+audioSize),
|
||||
mpb.PrependDecorators(
|
||||
decor.CountersKibiByte("% .2f / % .2f "),
|
||||
// simple name decorator
|
||||
decor.Name(fmt.Sprintf("id: %s src-ip: (%s)", v.id, sourceAddress)),
|
||||
// decor.DSyncWidth bit enables column width synchronization
|
||||
decor.Percentage(decor.WCSyncSpace),
|
||||
),
|
||||
mpb.AppendDecorators(
|
||||
decor.EwmaETA(decor.ET_STYLE_GO, 90),
|
||||
decor.Name(" ] "),
|
||||
decor.EwmaSpeed(decor.UnitKiB, "% .2f ", 60),
|
||||
decor.OnComplete(
|
||||
// ETA decorator with ewma age of 60
|
||||
decor.EwmaETA(decor.ET_STYLE_GO, 60), "done",
|
||||
),
|
||||
),
|
||||
mpb.BarRemoveOnComplete(),
|
||||
)
|
||||
defer func() {
|
||||
bar.Completed()
|
||||
bar.Abort(true)
|
||||
}()
|
||||
for {
|
||||
select {
|
||||
case <-done:
|
||||
return
|
||||
case <-ticker.C:
|
||||
size, err := logUtils.DirSize(v.videoDir())
|
||||
if err != nil {
|
||||
log.Errorf("error while getting size of download directory: %s", errors.FullTrace(err))
|
||||
return
|
||||
}
|
||||
bar.SetCurrent(size)
|
||||
if size > int64(videoSize+audioSize) {
|
||||
bar.SetTotal(size+2048, false)
|
||||
}
|
||||
bar.DecoratorEwmaUpdate(400 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
errorLog, _ := ioutil.ReadAll(stderr)
|
||||
outLog, _ := ioutil.ReadAll(stdout)
|
||||
err = cmd.Wait()
|
||||
|
||||
//stop the progress bar
|
||||
ticker.Stop()
|
||||
done <- true
|
||||
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "exit status 1") {
|
||||
if strings.Contains(string(errorLog), "HTTP Error 429") || strings.Contains(string(errorLog), "returned non-zero exit status 8") {
|
||||
v.pool.SetThrottled(sourceAddress)
|
||||
} else if strings.Contains(string(errorLog), "giving up after 0 fragment retries") {
|
||||
if i == (len(qualities) - 1) {
|
||||
return errors.Err(string(errorLog))
|
||||
}
|
||||
continue //this bypasses the yt throttling IP redistribution... TODO: don't
|
||||
} else if strings.Contains(string(errorLog), "YouTube said: Unable to extract video data") && !strings.Contains(userAgent[1], "Googlebot") {
|
||||
i-- //do not lower quality when trying a different user agent
|
||||
userAgent = []string{downloader.GoogleBotUA}
|
||||
log.Infof("trying different user agent for video %s", v.ID())
|
||||
continue
|
||||
}
|
||||
return errors.Err(string(errorLog))
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
log.Debugln(string(outLog))
|
||||
|
||||
if strings.Contains(string(outLog), "does not pass filter duration") {
|
||||
_ = v.delete("does not pass filter duration")
|
||||
return errors.Err("video is too long to process")
|
||||
}
|
||||
if strings.Contains(string(outLog), "File is larger than max-filesize") {
|
||||
_ = v.delete("File is larger than max-filesize")
|
||||
return errors.Err("the video is too big to sync, skipping for now")
|
||||
}
|
||||
if string(errorLog) != "" {
|
||||
if strings.Contains(string(errorLog), "HTTP Error 429") {
|
||||
v.pool.SetThrottled(sourceAddress)
|
||||
}
|
||||
log.Printf("Command finished with error: %v", errors.Err(string(errorLog)))
|
||||
_ = v.delete("due to error")
|
||||
return errors.Err(string(errorLog))
|
||||
}
|
||||
fi, err := os.Stat(v.getFullPath())
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
err = os.Chmod(v.getFullPath(), 0777)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
videoSize := fi.Size()
|
||||
v.size = &videoSize
|
||||
|
||||
isVideoSizeLimitSet := v.maxVideoSize > 0
|
||||
if isVideoSizeLimitSet && videoSize > v.maxVideoSize {
|
||||
//delete the video and ignore the error
|
||||
_ = v.delete()
|
||||
err = errors.Err("file is too big and there is no other format available")
|
||||
} else {
|
||||
break
|
||||
}
|
||||
break
|
||||
}
|
||||
return errors.Err(err)
|
||||
return nil
|
||||
}
|
||||
|
||||
type ytMetadata struct {
|
||||
ID string `json:"id"`
|
||||
Title string `json:"title"`
|
||||
Formats []struct {
|
||||
Asr int `json:"asr"`
|
||||
Filesize int `json:"filesize"`
|
||||
FormatID string `json:"format_id"`
|
||||
FormatNote string `json:"format_note"`
|
||||
Fps interface{} `json:"fps"`
|
||||
Height interface{} `json:"height"`
|
||||
Quality int `json:"quality"`
|
||||
Tbr float64 `json:"tbr"`
|
||||
URL string `json:"url"`
|
||||
Width interface{} `json:"width"`
|
||||
Ext string `json:"ext"`
|
||||
Vcodec string `json:"vcodec"`
|
||||
Acodec string `json:"acodec"`
|
||||
Abr float64 `json:"abr,omitempty"`
|
||||
DownloaderOptions struct {
|
||||
HTTPChunkSize int `json:"http_chunk_size"`
|
||||
} `json:"downloader_options,omitempty"`
|
||||
Container string `json:"container,omitempty"`
|
||||
Format string `json:"format"`
|
||||
Protocol string `json:"protocol"`
|
||||
HTTPHeaders struct {
|
||||
UserAgent string `json:"User-Agent"`
|
||||
AcceptCharset string `json:"Accept-Charset"`
|
||||
Accept string `json:"Accept"`
|
||||
AcceptEncoding string `json:"Accept-Encoding"`
|
||||
AcceptLanguage string `json:"Accept-Language"`
|
||||
} `json:"http_headers"`
|
||||
Vbr float64 `json:"vbr,omitempty"`
|
||||
} `json:"formats"`
|
||||
Thumbnails []struct {
|
||||
Height int `json:"height"`
|
||||
URL string `json:"url"`
|
||||
Width int `json:"width"`
|
||||
Resolution string `json:"resolution"`
|
||||
ID string `json:"id"`
|
||||
} `json:"thumbnails"`
|
||||
Description string `json:"description"`
|
||||
UploadDate string `json:"upload_date"`
|
||||
Uploader string `json:"uploader"`
|
||||
UploaderID string `json:"uploader_id"`
|
||||
UploaderURL string `json:"uploader_url"`
|
||||
ChannelID string `json:"channel_id"`
|
||||
ChannelURL string `json:"channel_url"`
|
||||
Duration int `json:"duration"`
|
||||
ViewCount int `json:"view_count"`
|
||||
AverageRating float64 `json:"average_rating"`
|
||||
AgeLimit int `json:"age_limit"`
|
||||
WebpageURL string `json:"webpage_url"`
|
||||
Categories []string `json:"categories"`
|
||||
Tags []interface{} `json:"tags"`
|
||||
IsLive interface{} `json:"is_live"`
|
||||
LikeCount int `json:"like_count"`
|
||||
DislikeCount int `json:"dislike_count"`
|
||||
Channel string `json:"channel"`
|
||||
Extractor string `json:"extractor"`
|
||||
WebpageURLBasename string `json:"webpage_url_basename"`
|
||||
ExtractorKey string `json:"extractor_key"`
|
||||
Playlist interface{} `json:"playlist"`
|
||||
PlaylistIndex interface{} `json:"playlist_index"`
|
||||
Thumbnail string `json:"thumbnail"`
|
||||
DisplayID string `json:"display_id"`
|
||||
Format string `json:"format"`
|
||||
FormatID string `json:"format_id"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
Resolution interface{} `json:"resolution"`
|
||||
Fps int `json:"fps"`
|
||||
Vcodec string `json:"vcodec"`
|
||||
Vbr float64 `json:"vbr"`
|
||||
StretchedRatio interface{} `json:"stretched_ratio"`
|
||||
Acodec string `json:"acodec"`
|
||||
Abr float64 `json:"abr"`
|
||||
Ext string `json:"ext"`
|
||||
Fulltitle string `json:"fulltitle"`
|
||||
Filename string `json:"_filename"`
|
||||
}
|
||||
|
||||
func parseVideoMetadata(metadataPath string) (*ytMetadata, error) {
|
||||
f, err := os.Open(metadataPath)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
// defer the closing of our jsonFile so that we can parse it later on
|
||||
defer f.Close()
|
||||
// read our opened jsonFile as a byte array.
|
||||
byteValue, _ := ioutil.ReadAll(f)
|
||||
|
||||
// we initialize our Users array
|
||||
var m ytMetadata
|
||||
|
||||
// we unmarshal our byteArray which contains our
|
||||
// jsonFile's content into 'users' which we defined above
|
||||
err = json.Unmarshal(byteValue, &m)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
return &m, nil
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) videoDir() string {
|
||||
return v.dir + "/" + v.id
|
||||
return path.Join(v.dir, v.id)
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) getDownloadedPath() (string, error) {
|
||||
files, err := ioutil.ReadDir(v.videoDir())
|
||||
log.Infoln(v.videoDir())
|
||||
@@ -335,20 +609,20 @@ func (v *YoutubeVideo) getDownloadedPath() (string, error) {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(v.getFullPath(), strings.TrimSuffix(f.Name(), filepath.Ext(f.Name()))) {
|
||||
return v.videoDir() + "/" + f.Name(), nil
|
||||
return path.Join(v.videoDir(), f.Name()), nil
|
||||
}
|
||||
}
|
||||
return "", errors.Err("could not find any downloaded videos")
|
||||
|
||||
}
|
||||
func (v *YoutubeVideo) delete() error {
|
||||
func (v *YoutubeVideo) delete(reason string) error {
|
||||
videoPath, err := v.getDownloadedPath()
|
||||
if err != nil {
|
||||
log.Errorln(err)
|
||||
return err
|
||||
}
|
||||
err = os.Remove(videoPath)
|
||||
log.Debugf("%s deleted from disk (%s)", v.id, videoPath)
|
||||
log.Debugf("%s deleted from disk for '%s' (%s)", v.id, reason, videoPath)
|
||||
|
||||
if err != nil {
|
||||
err = errors.Prefix("delete error", err)
|
||||
@@ -360,14 +634,17 @@ func (v *YoutubeVideo) delete() error {
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) triggerThumbnailSave() (err error) {
|
||||
thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Snippet.Thumbnails)
|
||||
v.thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.Url, v.ID(), v.awsConfig)
|
||||
thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Thumbnails)
|
||||
v.thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.URL, v.ID(), v.awsConfig)
|
||||
return err
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("publish").Add(time.Since(start))
|
||||
}(start)
|
||||
languages, locations, tags := v.getMetadata()
|
||||
|
||||
var fee *jsonrpc.Fee
|
||||
if params.Fee != nil {
|
||||
feeAmount, err := decimal.NewFromString(params.Fee.Amount)
|
||||
@@ -380,7 +657,17 @@ func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, params SyncParams) (*Sync
|
||||
FeeCurrency: jsonrpc.Currency(params.Fee.Currency),
|
||||
}
|
||||
}
|
||||
|
||||
urlsRegex := regexp.MustCompile(`(?m) ?(f|ht)(tp)(s?)(://)(.*)[.|/](.*)`)
|
||||
descriptionSample := urlsRegex.ReplaceAllString(v.description, "")
|
||||
info := whatlanggo.Detect(descriptionSample)
|
||||
info2 := whatlanggo.Detect(v.title)
|
||||
if info.IsReliable() && info.Lang.Iso6391() != "" {
|
||||
language := info.Lang.Iso6391()
|
||||
languages = []string{language}
|
||||
} else if info2.IsReliable() && info2.Lang.Iso6391() != "" {
|
||||
language := info2.Lang.Iso6391()
|
||||
languages = []string{language}
|
||||
}
|
||||
options := jsonrpc.StreamCreateOptions{
|
||||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
|
||||
Title: &v.title,
|
||||
@@ -390,6 +677,9 @@ func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, params SyncParams) (*Sync
|
||||
ThumbnailURL: &v.thumbnailURL,
|
||||
Tags: tags,
|
||||
Locations: locations,
|
||||
FundingAccountIDs: []string{
|
||||
params.DefaultAccount,
|
||||
},
|
||||
},
|
||||
Fee: fee,
|
||||
License: util.PtrToString("Copyrighted (contact publisher)"),
|
||||
@@ -413,15 +703,18 @@ type SyncParams struct {
|
||||
ChannelID string
|
||||
MaxVideoSize int
|
||||
Namer *namer.Namer
|
||||
MaxVideoLength float64
|
||||
Fee *sdk.Fee
|
||||
MaxVideoLength time.Duration
|
||||
Fee *shared.Fee
|
||||
DefaultAccount string
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo, reprocess bool, walletLock *sync.RWMutex) (*SyncSummary, error) {
|
||||
v.maxVideoSize = int64(params.MaxVideoSize) * 1024 * 1024
|
||||
func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo, reprocess bool, walletLock *sync.RWMutex, pbWg *sync.WaitGroup, pb *mpb.Progress) (*SyncSummary, error) {
|
||||
v.maxVideoSize = int64(params.MaxVideoSize)
|
||||
v.maxVideoLength = params.MaxVideoLength
|
||||
v.lbryChannelID = params.ChannelID
|
||||
v.walletLock = walletLock
|
||||
v.progressBars = pb
|
||||
v.progressBarWg = pbWg
|
||||
if reprocess && existingVideoData != nil && existingVideoData.Published {
|
||||
summary, err := v.reprocess(daemon, params, existingVideoData)
|
||||
return summary, errors.Prefix("upgrade failed", err)
|
||||
@@ -430,17 +723,41 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
|
||||
err := v.download()
|
||||
var err error
|
||||
|
||||
dur := time.Duration(v.youtubeInfo.Duration) * time.Second
|
||||
minDuration := 7 * time.Second
|
||||
|
||||
if dur > v.maxVideoLength {
|
||||
logUtils.SendErrorToSlack("%s is %s long and the limit is %s", v.id, dur.String(), v.maxVideoLength.String())
|
||||
return nil, errors.Err("video is too long to process")
|
||||
}
|
||||
if dur < minDuration {
|
||||
logUtils.SendErrorToSlack("%s is %s long and the minimum is %s", v.id, dur.String(), minDuration.String())
|
||||
return nil, errors.Err("video is too short to process")
|
||||
}
|
||||
for {
|
||||
err = v.download()
|
||||
if err != nil && strings.Contains(err.Error(), "HTTP Error 429") {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return nil, errors.Prefix("download error", err)
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
//log.Debugln("Downloaded " + v.id)
|
||||
ctx, cancelFn := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancelFn()
|
||||
|
||||
data, err := ffprobe.ProbeURL(ctx, v.getFullPath())
|
||||
if err != nil {
|
||||
log.Errorf("standard downloader failed: %s. Trying fallback downloader\n", err.Error())
|
||||
fallBackErr := v.fallbackDownload()
|
||||
if fallBackErr != nil {
|
||||
log.Errorf("fallback downloader failed: %s\n", fallBackErr.Error())
|
||||
return nil, errors.Prefix("download error", err) //return original error
|
||||
log.Errorf("failure in probing downloaded video: %s", err.Error())
|
||||
} else {
|
||||
if data.Format.Duration() < minDuration {
|
||||
return nil, errors.Err("video is too short to process")
|
||||
}
|
||||
}
|
||||
log.Debugln("Downloaded " + v.id)
|
||||
|
||||
err = v.triggerThumbnailSave()
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("thumbnail error", err)
|
||||
@@ -449,7 +766,7 @@ func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncPar
|
||||
|
||||
summary, err := v.publish(daemon, params)
|
||||
//delete the video in all cases (and ignore the error)
|
||||
_ = v.delete()
|
||||
_ = v.delete("finished download and publish")
|
||||
|
||||
return summary, errors.Prefix("publish error", err)
|
||||
}
|
||||
@@ -459,31 +776,37 @@ func (v *YoutubeVideo) getMetadata() (languages []string, locations []jsonrpc.Lo
|
||||
locations = nil
|
||||
tags = nil
|
||||
if !v.mocked {
|
||||
if v.youtubeInfo.Snippet.DefaultLanguage != "" {
|
||||
languages = []string{v.youtubeInfo.Snippet.DefaultLanguage}
|
||||
}
|
||||
/*
|
||||
if v.youtubeInfo.Snippet.DefaultLanguage != "" {
|
||||
if v.youtubeInfo.Snippet.DefaultLanguage == "iw" {
|
||||
v.youtubeInfo.Snippet.DefaultLanguage = "he"
|
||||
}
|
||||
languages = []string{v.youtubeInfo.Snippet.DefaultLanguage}
|
||||
}*/
|
||||
|
||||
if v.youtubeInfo.RecordingDetails != nil && v.youtubeInfo.RecordingDetails.Location != nil {
|
||||
/*if v.youtubeInfo.!= nil && v.youtubeInfo.RecordingDetails.Location != nil {
|
||||
locations = []jsonrpc.Location{{
|
||||
Latitude: util.PtrToString(fmt.Sprintf("%.7f", v.youtubeInfo.RecordingDetails.Location.Latitude)),
|
||||
Longitude: util.PtrToString(fmt.Sprintf("%.7f", v.youtubeInfo.RecordingDetails.Location.Longitude)),
|
||||
}}
|
||||
}
|
||||
tags = v.youtubeInfo.Snippet.Tags
|
||||
}*/
|
||||
tags = v.youtubeInfo.Tags
|
||||
}
|
||||
tags, err := tagsManager.SanitizeTags(tags, v.youtubeChannelID)
|
||||
tags, err := tags_manager.SanitizeTags(tags, v.youtubeChannelID)
|
||||
if err != nil {
|
||||
log.Errorln(err.Error())
|
||||
}
|
||||
if !v.mocked {
|
||||
tags = append(tags, youtubeCategories[v.youtubeInfo.Snippet.CategoryId])
|
||||
for _, category := range v.youtubeInfo.Categories {
|
||||
tags = append(tags, youtubeCategories[category])
|
||||
}
|
||||
}
|
||||
|
||||
return languages, locations, tags
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo) (*SyncSummary, error) {
|
||||
c, err := daemon.ClaimSearch(nil, &existingVideoData.ClaimID, nil, nil)
|
||||
c, err := daemon.ClaimSearch(nil, &existingVideoData.ClaimID, nil, nil, 1, 20)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
@@ -501,8 +824,8 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
|
||||
if v.mocked {
|
||||
return nil, errors.Err("could not find thumbnail for mocked video")
|
||||
}
|
||||
thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Snippet.Thumbnails)
|
||||
thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.Url, v.ID(), v.awsConfig)
|
||||
thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Thumbnails)
|
||||
thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.URL, v.ID(), v.awsConfig)
|
||||
} else {
|
||||
thumbnailURL = thumbs.ThumbnailEndpoint + v.ID()
|
||||
}
|
||||
@@ -513,8 +836,15 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
|
||||
videoSize = uint64(existingVideoData.Size)
|
||||
} else {
|
||||
log.Infof("%s: the video must be republished as we can't get the right size", v.ID())
|
||||
//return v.downloadAndPublish(daemon, params) //TODO: actually republish the video. NB: the current claim should be abandoned first
|
||||
return nil, errors.Err("the video must be republished as we can't get the right size")
|
||||
if !v.mocked {
|
||||
_, err = daemon.StreamAbandon(currentClaim.Txid, currentClaim.Nout, nil, true)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
return v.downloadAndPublish(daemon, params)
|
||||
|
||||
}
|
||||
return nil, errors.Err("the video must be republished as we can't get the right size but it doesn't exist on youtube anymore")
|
||||
}
|
||||
}
|
||||
v.size = util.PtrToInt64(int64(videoSize))
|
||||
@@ -536,6 +866,9 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
|
||||
ThumbnailURL: &thumbnailURL,
|
||||
Languages: languages,
|
||||
Locations: locations,
|
||||
FundingAccountIDs: []string{
|
||||
params.DefaultAccount,
|
||||
},
|
||||
},
|
||||
Author: util.PtrToString(""),
|
||||
License: util.PtrToString("Copyrighted (contact publisher)"),
|
||||
@@ -548,10 +881,12 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
|
||||
v.walletLock.RLock()
|
||||
defer v.walletLock.RUnlock()
|
||||
if v.mocked {
|
||||
start := time.Now()
|
||||
pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
|
||||
StreamCreateOptions: streamCreateOptions,
|
||||
FileSize: &videoSize,
|
||||
})
|
||||
timing.TimedComponent("StreamUpdate").Add(time.Since(start))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -562,15 +897,11 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
|
||||
}, nil
|
||||
}
|
||||
|
||||
videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
streamCreateOptions.ClaimCreateOptions.Title = &v.title
|
||||
streamCreateOptions.ClaimCreateOptions.Description = util.PtrToString(v.getAbbrevDescription())
|
||||
streamCreateOptions.Duration = util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds())))
|
||||
streamCreateOptions.Duration = util.PtrToUint64(uint64(v.youtubeInfo.Duration))
|
||||
streamCreateOptions.ReleaseTime = util.PtrToInt64(v.publishedAt.Unix())
|
||||
start := time.Now()
|
||||
pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
|
||||
ClearLanguages: util.PtrToBool(true),
|
||||
ClearLocations: util.PtrToBool(true),
|
||||
@@ -578,6 +909,7 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
|
||||
StreamCreateOptions: streamCreateOptions,
|
||||
FileSize: &videoSize,
|
||||
})
|
||||
timing.TimedComponent("StreamUpdate").Add(time.Since(start))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"testing"
|
||||
|
||||
"github.com/abadojack/whatlanggo"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestLanguageDetection(t *testing.T) {
|
||||
description := `Om lättkränkta muslimer, och den bristande logiken i vad som anses vara att vanära profeten. Från Moderata riksdagspolitikern Hanif Balis podcast "God Ton", avsnitt 108, från oktober 2020, efter terrordådet där en fransk lärare fick huvudet avskuret efter att undervisat sin mångkulturella klass om frihet.`
|
||||
info := whatlanggo.Detect(description)
|
||||
logrus.Infof("confidence: %.2f", info.Confidence)
|
||||
assert.True(t, info.IsReliable())
|
||||
assert.True(t, info.Lang.Iso6391() != "")
|
||||
assert.Equal(t, "sv", info.Lang.Iso6391())
|
||||
|
||||
description = `🥳週四直播 | 晚上來開個賽車🔰歡迎各位一起來玩! - PonPonLin蹦蹦林`
|
||||
info = whatlanggo.Detect(description)
|
||||
logrus.Infof("confidence: %.2f", info.Confidence)
|
||||
assert.True(t, info.IsReliable())
|
||||
assert.True(t, info.Lang.Iso6391() != "")
|
||||
assert.Equal(t, "zh", info.Lang.Iso6391())
|
||||
|
||||
description = `成為這個頻道的會員並獲得獎勵:
|
||||
https://www.youtube.com/channel/UCOQFrooz-YGHjYb7s3-MrsQ/join
|
||||
_____________________________________________
|
||||
想聽我既音樂作品可以去下面LINK
|
||||
streetvoice 街聲:
|
||||
https://streetvoice.com/CTLam331/
|
||||
_____________________________________________
|
||||
想學結他、鋼琴
|
||||
有關音樂制作工作
|
||||
都可以搵我~
|
||||
大家快D訂閱喇
|
||||
不定期出片
|
||||
|
||||
|
||||
|
||||
|
||||
Website: http://ctlam331.wixsite.com/ctlamusic
|
||||
FB PAGE:https://www.facebook.com/ctlam331
|
||||
IG:ctlamusic`
|
||||
urlsRegex := regexp.MustCompile(`(?m) ?(f|ht)(tp)(s?)(://)(.*)[.|/](.*)`)
|
||||
descriptionSample := urlsRegex.ReplaceAllString(description, "")
|
||||
info = whatlanggo.Detect(descriptionSample)
|
||||
logrus.Infof("confidence: %.2f", info.Confidence)
|
||||
assert.True(t, info.IsReliable())
|
||||
assert.True(t, info.Lang.Iso6391() != "")
|
||||
assert.Equal(t, "zh", info.Lang.Iso6391())
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package tagsManager
|
||||
package tags_manager
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
@@ -71,7 +71,7 @@ func SanitizeTags(tags []string, youtubeChannelID string) ([]string, error) {
|
||||
return sanitizedTags, nil
|
||||
}
|
||||
|
||||
const TagMaxLength = 255
|
||||
const TagMaxLength = 50
|
||||
|
||||
func normalizeTag(t string) (string, error) {
|
||||
t = strings.ToLower(t)
|
||||
@@ -1,4 +1,4 @@
|
||||
package tagsManager
|
||||
package tags_manager
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
+38
-20
@@ -1,17 +1,19 @@
|
||||
package thumbs
|
||||
|
||||
import (
|
||||
"google.golang.org/api/youtube/v3"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/lbryio/errors.go"
|
||||
"github.com/lbryio/ytsync/v5/downloader/ytdl"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
"github.com/prometheus/common/log"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type thumbnailUploader struct {
|
||||
@@ -25,13 +27,15 @@ const thumbnailPath = "/tmp/ytsync_thumbnails/"
|
||||
const ThumbnailEndpoint = "https://thumbnails.lbry.com/"
|
||||
|
||||
func (u *thumbnailUploader) downloadThumbnail() error {
|
||||
_ = os.Mkdir(thumbnailPath, 0750)
|
||||
_ = os.Mkdir(thumbnailPath, 0777)
|
||||
img, err := os.Create("/tmp/ytsync_thumbnails/" + u.name)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer img.Close()
|
||||
|
||||
if strings.HasPrefix(u.originalUrl, "//") {
|
||||
u.originalUrl = "https:" + u.originalUrl
|
||||
}
|
||||
resp, err := http.Get(u.originalUrl)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
@@ -61,12 +65,14 @@ func (u *thumbnailUploader) uploadThumbnail() error {
|
||||
uploader := s3manager.NewUploader(s3Session)
|
||||
|
||||
_, err = uploader.Upload(&s3manager.UploadInput{
|
||||
Bucket: aws.String("thumbnails.lbry.com"),
|
||||
Key: key,
|
||||
Body: thumb,
|
||||
ACL: aws.String("public-read"),
|
||||
ContentType: aws.String("image/jpeg"),
|
||||
Bucket: aws.String("thumbnails.lbry.com"),
|
||||
Key: key,
|
||||
Body: thumb,
|
||||
ACL: aws.String("public-read"),
|
||||
ContentType: aws.String("image/jpeg"),
|
||||
CacheControl: aws.String("public, max-age=2592000"),
|
||||
})
|
||||
|
||||
u.mirroredUrl = ThumbnailEndpoint + u.name
|
||||
return errors.Err(err)
|
||||
}
|
||||
@@ -94,18 +100,30 @@ func MirrorThumbnail(url string, name string, s3Config aws.Config) (string, erro
|
||||
return "", err
|
||||
}
|
||||
|
||||
ownS3Config := s3Config.Copy(&aws.Config{Endpoint: aws.String("s3.lbry.tech")})
|
||||
|
||||
tu2 := thumbnailUploader{
|
||||
originalUrl: url,
|
||||
name: name,
|
||||
s3Config: *ownS3Config,
|
||||
}
|
||||
//own S3
|
||||
err = tu2.uploadThumbnail()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return tu.mirroredUrl, nil
|
||||
}
|
||||
|
||||
func GetBestThumbnail(thumbnails *youtube.ThumbnailDetails) *youtube.Thumbnail {
|
||||
if thumbnails.Maxres != nil {
|
||||
return thumbnails.Maxres
|
||||
} else if thumbnails.High != nil {
|
||||
return thumbnails.High
|
||||
} else if thumbnails.Medium != nil {
|
||||
return thumbnails.Medium
|
||||
} else if thumbnails.Standard != nil {
|
||||
return thumbnails.Standard
|
||||
func GetBestThumbnail(thumbnails []ytdl.Thumbnail) *ytdl.Thumbnail {
|
||||
var bestWidth *ytdl.Thumbnail
|
||||
for _, thumbnail := range thumbnails {
|
||||
if bestWidth == nil {
|
||||
bestWidth = &thumbnail
|
||||
} else if bestWidth.Width < thumbnail.Width {
|
||||
bestWidth = &thumbnail
|
||||
}
|
||||
}
|
||||
return thumbnails.Default
|
||||
return bestWidth
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
package timing
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/ytsync/v5/metrics"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type Timing struct {
|
||||
component string
|
||||
milliseconds int64
|
||||
min int64
|
||||
max int64
|
||||
invocations int32
|
||||
}
|
||||
|
||||
var timings *sync.Map
|
||||
|
||||
func TimedComponent(component string) *Timing {
|
||||
if timings == nil {
|
||||
timings = &sync.Map{}
|
||||
}
|
||||
stored, _ := timings.LoadOrStore(component, &Timing{
|
||||
component: component,
|
||||
milliseconds: 0,
|
||||
min: int64(99999999),
|
||||
})
|
||||
t, _ := stored.(*Timing)
|
||||
return t
|
||||
}
|
||||
|
||||
func ClearTimings() {
|
||||
if timings == nil {
|
||||
return
|
||||
}
|
||||
timings.Range(func(key interface{}, value interface{}) bool {
|
||||
timings.Delete(key)
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
func Report() {
|
||||
var totalTime time.Duration
|
||||
timings.Range(func(key interface{}, value interface{}) bool {
|
||||
totalTime += value.(*Timing).Get()
|
||||
return true
|
||||
})
|
||||
timings.Range(func(key interface{}, value interface{}) bool {
|
||||
component := key
|
||||
componentRuntime := value.(*Timing).Get().String()
|
||||
percentTime := float64(value.(*Timing).Get()) / float64(totalTime) * 100
|
||||
invocations := value.(*Timing).Invocations()
|
||||
avgTime := (time.Duration(int64(float64(value.(*Timing).Get()) / float64(value.(*Timing).Invocations())))).String()
|
||||
minRuntime := value.(*Timing).Min().String()
|
||||
maxRuntime := value.(*Timing).Max().String()
|
||||
logrus.Printf("component %s ran for %s (%.2f%% of the total time) - invoked %d times with an average of %s per call, a minimum of %s and a maximum of %s",
|
||||
component,
|
||||
componentRuntime,
|
||||
percentTime,
|
||||
invocations,
|
||||
avgTime,
|
||||
minRuntime,
|
||||
maxRuntime,
|
||||
)
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
func (t *Timing) Add(d time.Duration) {
|
||||
metrics.Durations.WithLabelValues(t.component).Observe(d.Seconds())
|
||||
atomic.AddInt64(&t.milliseconds, d.Milliseconds())
|
||||
for {
|
||||
oldMin := atomic.LoadInt64(&t.min)
|
||||
if d.Milliseconds() < oldMin {
|
||||
if atomic.CompareAndSwapInt64(&t.min, oldMin, d.Milliseconds()) {
|
||||
break
|
||||
}
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
for {
|
||||
oldMax := atomic.LoadInt64(&t.max)
|
||||
if d.Milliseconds() > oldMax {
|
||||
if atomic.CompareAndSwapInt64(&t.max, oldMax, d.Milliseconds()) {
|
||||
break
|
||||
}
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
atomic.AddInt32(&t.invocations, 1)
|
||||
}
|
||||
|
||||
func (t *Timing) Get() time.Duration {
|
||||
ms := atomic.LoadInt64(&t.milliseconds)
|
||||
return time.Duration(ms) * time.Millisecond
|
||||
}
|
||||
|
||||
func (t *Timing) Invocations() int32 {
|
||||
return atomic.LoadInt32(&t.invocations)
|
||||
}
|
||||
|
||||
func (t *Timing) Min() time.Duration {
|
||||
ms := atomic.LoadInt64(&t.min)
|
||||
return time.Duration(ms) * time.Millisecond
|
||||
}
|
||||
func (t *Timing) Max() time.Duration {
|
||||
ms := atomic.LoadInt64(&t.max)
|
||||
return time.Duration(ms) * time.Millisecond
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// SendErrorToSlack Sends an error message to the default channel and to the process log.
|
||||
func SendErrorToSlack(format string, a ...interface{}) {
|
||||
message := format
|
||||
if len(a) > 0 {
|
||||
message = fmt.Sprintf(format, a...)
|
||||
}
|
||||
log.Errorln(message)
|
||||
err := util.SendToSlack(":sos: " + message)
|
||||
if err != nil {
|
||||
log.Errorln(err)
|
||||
}
|
||||
}
|
||||
|
||||
// SendInfoToSlack Sends an info message to the default channel and to the process log.
|
||||
func SendInfoToSlack(format string, a ...interface{}) {
|
||||
message := format
|
||||
if len(a) > 0 {
|
||||
message = fmt.Sprintf(format, a...)
|
||||
}
|
||||
log.Infoln(message)
|
||||
err := util.SendToSlack(":information_source: " + message)
|
||||
if err != nil {
|
||||
log.Errorln(err)
|
||||
}
|
||||
}
|
||||
+421
@@ -0,0 +1,421 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/user"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/lbrycrd"
|
||||
"github.com/lbryio/ytsync/v5/timing"
|
||||
|
||||
"github.com/docker/docker/api/types"
|
||||
"github.com/docker/docker/api/types/filters"
|
||||
"github.com/docker/docker/client"
|
||||
"github.com/mitchellh/go-ps"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func GetBlobsDir() string {
|
||||
blobsDir := os.Getenv("BLOBS_DIRECTORY")
|
||||
if blobsDir == "" {
|
||||
usr, err := user.Current()
|
||||
if err != nil {
|
||||
log.Error(err.Error())
|
||||
return ""
|
||||
}
|
||||
blobsDir = usr.HomeDir + "/.lbrynet/blobfiles/"
|
||||
}
|
||||
|
||||
return blobsDir
|
||||
}
|
||||
|
||||
func IsBlobReflectionOff() bool {
|
||||
return os.Getenv("REFLECT_BLOBS") == "false"
|
||||
}
|
||||
|
||||
func GetLBRYNetDir() string {
|
||||
lbrynetDir := os.Getenv("LBRYNET_DIR")
|
||||
if lbrynetDir == "" {
|
||||
usr, err := user.Current()
|
||||
if err != nil {
|
||||
log.Errorln(err.Error())
|
||||
return ""
|
||||
}
|
||||
return usr.HomeDir + "/.lbrynet/"
|
||||
}
|
||||
return lbrynetDir
|
||||
}
|
||||
|
||||
func GetLbryumDir() string {
|
||||
lbryumDir := os.Getenv("LBRYUM_DIR")
|
||||
if lbryumDir == "" {
|
||||
usr, err := user.Current()
|
||||
if err != nil {
|
||||
log.Errorln(err.Error())
|
||||
return ""
|
||||
}
|
||||
return usr.HomeDir + "/.lbryum/"
|
||||
}
|
||||
return lbryumDir + "/"
|
||||
}
|
||||
|
||||
const ALL = true
|
||||
const ONLINE = false
|
||||
|
||||
func GetLBRYNetContainer(all bool) (*types.Container, error) {
|
||||
return getDockerContainer("lbrynet", all)
|
||||
}
|
||||
|
||||
func getDockerContainer(name string, all bool) (*types.Container, error) {
|
||||
cli, err := client.NewEnvClient()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
filters := filters.NewArgs()
|
||||
filters.Add("name", name)
|
||||
containers, err := cli.ContainerList(context.Background(), types.ContainerListOptions{All: all, Filters: filters})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if len(containers) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
if len(containers) > 1 {
|
||||
return nil, errors.Err("more than one %s container found", name)
|
||||
}
|
||||
|
||||
return &containers[0], nil
|
||||
}
|
||||
|
||||
func IsUsingDocker() bool {
|
||||
useDocker, err := strconv.ParseBool(os.Getenv("LBRYNET_USE_DOCKER"))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return useDocker
|
||||
}
|
||||
|
||||
func IsRegTest() bool {
|
||||
usesRegtest, err := strconv.ParseBool(os.Getenv("REGTEST"))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return usesRegtest
|
||||
}
|
||||
|
||||
func GetLbrycrdClient(lbrycrdString string) (*lbrycrd.Client, error) {
|
||||
chainName := os.Getenv("CHAINNAME")
|
||||
chainParams, ok := lbrycrd.ChainParamsMap[chainName]
|
||||
if !ok {
|
||||
chainParams = lbrycrd.MainNetParams
|
||||
}
|
||||
var lbrycrdd *lbrycrd.Client
|
||||
var err error
|
||||
if lbrycrdString == "" {
|
||||
lbrycrdd, err = lbrycrd.NewWithDefaultURL(&chainParams)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
lbrycrdd, err = lbrycrd.New(lbrycrdString, &chainParams)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return lbrycrdd, nil
|
||||
}
|
||||
|
||||
func ShouldCleanOnStartup() bool {
|
||||
shouldClean, err := strconv.ParseBool(os.Getenv("CLEAN_ON_STARTUP"))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return shouldClean
|
||||
}
|
||||
|
||||
func IsLbrynetRunning() (bool, error) {
|
||||
if IsUsingDocker() {
|
||||
container, err := GetLBRYNetContainer(ONLINE)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return container != nil, nil
|
||||
}
|
||||
|
||||
processes, err := ps.Processes()
|
||||
if err != nil {
|
||||
return true, errors.Err(err)
|
||||
}
|
||||
var daemonProcessId = -1
|
||||
for _, p := range processes {
|
||||
if p.Executable() == "lbrynet" {
|
||||
daemonProcessId = p.Pid()
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
running := daemonProcessId != -1
|
||||
return running, nil
|
||||
}
|
||||
|
||||
func CleanForStartup() error {
|
||||
if !IsRegTest() {
|
||||
return errors.Err("never cleanup wallet outside of regtest and with caution. this should only be done in local testing and requires regtest to be on")
|
||||
}
|
||||
|
||||
running, err := IsLbrynetRunning()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if running {
|
||||
err := StopDaemon()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
err = CleanupLbrynet()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
lbrycrd, err := GetLbrycrdClient(os.Getenv("LBRYCRD_STRING"))
|
||||
if err != nil {
|
||||
return errors.Prefix("error getting lbrycrd client: ", err)
|
||||
}
|
||||
height, err := lbrycrd.GetBlockCount()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
const minBlocksForUTXO = 200
|
||||
if height < minBlocksForUTXO {
|
||||
//Start reg test with some credits
|
||||
txs, err := lbrycrd.Generate(uint32(minBlocksForUTXO) - uint32(height))
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
log.Debugf("REGTEST: Generated %d transactions to get some LBC!", len(txs))
|
||||
}
|
||||
|
||||
defaultWalletDir := GetDefaultWalletPath()
|
||||
_, err = os.Stat(defaultWalletDir)
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return errors.Err(os.Remove(defaultWalletDir))
|
||||
}
|
||||
|
||||
func CleanupLbrynet() error {
|
||||
//make sure lbrynet is off
|
||||
running, err := IsLbrynetRunning()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if running {
|
||||
return errors.Prefix("cannot cleanup lbrynet as the daemon is running", err)
|
||||
}
|
||||
lbrynetDir := GetLBRYNetDir()
|
||||
files, err := filepath.Glob(lbrynetDir + "lbrynet.sqlite*")
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
for _, f := range files {
|
||||
err = os.Remove(f)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
}
|
||||
blobsDir := GetBlobsDir()
|
||||
err = os.RemoveAll(blobsDir)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
err = os.Mkdir(blobsDir, 0777)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
lbryumDir := GetLbryumDir()
|
||||
ledger := "lbc_mainnet"
|
||||
if IsRegTest() {
|
||||
ledger = "lbc_regtest"
|
||||
}
|
||||
lbryumDir = lbryumDir + ledger
|
||||
|
||||
files, err = filepath.Glob(lbryumDir + "/blockchain.db*")
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
for _, f := range files {
|
||||
err = os.Remove(f)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var metadataDirInitialized = false
|
||||
|
||||
func GetVideoMetadataDir() string {
|
||||
dir := "./videos_metadata"
|
||||
if !metadataDirInitialized {
|
||||
metadataDirInitialized = true
|
||||
_ = os.MkdirAll(dir, 0755)
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
func CleanupMetadata() error {
|
||||
dir := GetVideoMetadataDir()
|
||||
err := os.RemoveAll(dir)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
metadataDirInitialized = false
|
||||
return nil
|
||||
}
|
||||
|
||||
func SleepUntilQuotaReset() {
|
||||
PST, _ := time.LoadLocation("America/Los_Angeles")
|
||||
t := time.Now().In(PST)
|
||||
n := time.Date(t.Year(), t.Month(), t.Day(), 24, 2, 0, 0, PST)
|
||||
d := n.Sub(t)
|
||||
if d < 0 {
|
||||
n = n.Add(24 * time.Hour)
|
||||
d = n.Sub(t)
|
||||
}
|
||||
log.Infof("gotta sleep %s until the quota resets", d.String())
|
||||
time.Sleep(d)
|
||||
}
|
||||
|
||||
func StartDaemon() error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("startDaemon").Add(time.Since(start))
|
||||
}(start)
|
||||
if IsUsingDocker() {
|
||||
return startDaemonViaDocker()
|
||||
}
|
||||
return startDaemonViaSystemd()
|
||||
}
|
||||
|
||||
func StopDaemon() error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("stopDaemon").Add(time.Since(start))
|
||||
}(start)
|
||||
if IsUsingDocker() {
|
||||
return stopDaemonViaDocker()
|
||||
}
|
||||
return stopDaemonViaSystemd()
|
||||
}
|
||||
|
||||
func startDaemonViaDocker() error {
|
||||
container, err := GetLBRYNetContainer(true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cli, err := client.NewEnvClient()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
err = cli.ContainerStart(context.Background(), container.ID, types.ContainerStartOptions{})
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func stopDaemonViaDocker() error {
|
||||
container, err := GetLBRYNetContainer(ONLINE)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cli, err := client.NewEnvClient()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
err = cli.ContainerStop(context.Background(), container.ID, nil)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func startDaemonViaSystemd() error {
|
||||
err := exec.Command("/usr/bin/sudo", "/bin/systemctl", "start", "lbrynet.service").Run()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func stopDaemonViaSystemd() error {
|
||||
err := exec.Command("/usr/bin/sudo", "/bin/systemctl", "stop", "lbrynet.service").Run()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func GetDefaultWalletPath() string {
|
||||
defaultWalletDir := os.Getenv("HOME") + "/.lbryum/wallets/default_wallet"
|
||||
if IsRegTest() {
|
||||
defaultWalletDir = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet"
|
||||
}
|
||||
|
||||
walletPath := os.Getenv("LBRYUM_DIR")
|
||||
if walletPath != "" {
|
||||
defaultWalletDir = walletPath + "/wallets/default_wallet"
|
||||
}
|
||||
return defaultWalletDir
|
||||
}
|
||||
func GetBlockchainDBPath() string {
|
||||
lbryumDir := os.Getenv("LBRYUM_DIR")
|
||||
if lbryumDir == "" {
|
||||
if IsRegTest() {
|
||||
lbryumDir = os.Getenv("HOME") + "/.lbryum_regtest"
|
||||
} else {
|
||||
lbryumDir = os.Getenv("HOME") + "/.lbryum"
|
||||
}
|
||||
}
|
||||
defaultDB := lbryumDir + "/lbc_mainnet/blockchain.db"
|
||||
if IsRegTest() {
|
||||
defaultDB = lbryumDir + "/lbc_regtest/blockchain.db"
|
||||
}
|
||||
return defaultDB
|
||||
}
|
||||
func GetBlockchainDirectoryName() string {
|
||||
ledger := "lbc_mainnet"
|
||||
if IsRegTest() {
|
||||
ledger = "lbc_regtest"
|
||||
}
|
||||
return ledger
|
||||
}
|
||||
|
||||
func DirSize(path string) (int64, error) {
|
||||
var size int64
|
||||
err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
size += info.Size()
|
||||
}
|
||||
return err
|
||||
})
|
||||
return size, err
|
||||
}
|
||||
+393
@@ -0,0 +1,393 @@
|
||||
package ytapi
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/ytsync/v5/shared"
|
||||
logUtils "github.com/lbryio/ytsync/v5/util"
|
||||
"github.com/vbauerster/mpb/v7"
|
||||
|
||||
"github.com/lbryio/ytsync/v5/downloader/ytdl"
|
||||
|
||||
"github.com/lbryio/ytsync/v5/downloader"
|
||||
"github.com/lbryio/ytsync/v5/ip_manager"
|
||||
"github.com/lbryio/ytsync/v5/sdk"
|
||||
"github.com/lbryio/ytsync/v5/sources"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type Video interface {
|
||||
Size() *int64
|
||||
ID() string
|
||||
IDAndNum() string
|
||||
PlaylistPosition() int
|
||||
PublishedAt() time.Time
|
||||
Sync(*jsonrpc.Client, sources.SyncParams, *sdk.SyncedVideo, bool, *sync.RWMutex, *sync.WaitGroup, *mpb.Progress) (*sources.SyncSummary, error)
|
||||
}
|
||||
|
||||
type byPublishedAt []Video
|
||||
|
||||
func (a byPublishedAt) Len() int { return len(a) }
|
||||
func (a byPublishedAt) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
|
||||
func (a byPublishedAt) Less(i, j int) bool { return a[i].PublishedAt().Before(a[j].PublishedAt()) }
|
||||
|
||||
type VideoParams struct {
|
||||
VideoDir string
|
||||
S3Config aws.Config
|
||||
Stopper *stop.Group
|
||||
IPPool *ip_manager.IPPool
|
||||
}
|
||||
|
||||
var mostRecentlyFailedChannel string // TODO: fix this hack!
|
||||
|
||||
func GetVideosToSync(config *sdk.APIConfig, channelID string, syncedVideos map[string]sdk.SyncedVideo, quickSync bool, maxVideos int, videoParams VideoParams, lastUploadedVideo string) ([]Video, error) {
|
||||
var videos []Video
|
||||
if quickSync && maxVideos > 50 {
|
||||
maxVideos = 50
|
||||
}
|
||||
allVideos, err := downloader.GetPlaylistVideoIDs(channelID, maxVideos, videoParams.Stopper.Ch(), videoParams.IPPool)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
videoIDs := make([]string, 0, len(allVideos))
|
||||
for _, video := range allVideos {
|
||||
sv, ok := syncedVideos[video]
|
||||
if ok && util.SubstringInSlice(sv.FailureReason, shared.NeverRetryFailures) {
|
||||
continue
|
||||
}
|
||||
videoIDs = append(videoIDs, video)
|
||||
}
|
||||
log.Infof("Got info for %d videos from youtube downloader", len(videoIDs))
|
||||
|
||||
playlistMap := make(map[string]int64)
|
||||
for i, videoID := range videoIDs {
|
||||
playlistMap[videoID] = int64(i)
|
||||
}
|
||||
//this will ensure that we at least try to sync the video that was marked as last uploaded video in the database.
|
||||
if lastUploadedVideo != "" {
|
||||
_, ok := playlistMap[lastUploadedVideo]
|
||||
if !ok {
|
||||
playlistMap[lastUploadedVideo] = 0
|
||||
videoIDs = append(videoIDs, lastUploadedVideo)
|
||||
}
|
||||
}
|
||||
|
||||
if len(videoIDs) < 1 {
|
||||
if channelID == mostRecentlyFailedChannel {
|
||||
return nil, errors.Err("playlist items not found")
|
||||
}
|
||||
mostRecentlyFailedChannel = channelID
|
||||
}
|
||||
|
||||
vids, err := getVideos(config, channelID, videoIDs, videoParams.Stopper.Ch(), videoParams.IPPool)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, item := range vids {
|
||||
positionInList := playlistMap[item.ID]
|
||||
videoToAdd, err := sources.NewYoutubeVideo(videoParams.VideoDir, item, positionInList, videoParams.S3Config, videoParams.Stopper, videoParams.IPPool)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
videos = append(videos, videoToAdd)
|
||||
}
|
||||
|
||||
for k, v := range syncedVideos {
|
||||
if !v.Published {
|
||||
continue
|
||||
}
|
||||
if _, ok := playlistMap[k]; !ok {
|
||||
videos = append(videos, sources.NewMockedVideo(videoParams.VideoDir, k, channelID, videoParams.S3Config, videoParams.Stopper, videoParams.IPPool))
|
||||
}
|
||||
}
|
||||
|
||||
sort.Sort(byPublishedAt(videos))
|
||||
|
||||
return videos, nil
|
||||
}
|
||||
|
||||
// CountVideosInChannel is unused for now... keeping it here just in case
|
||||
func CountVideosInChannel(channelID string) (int, error) {
|
||||
url := "https://socialblade.com/youtube/channel/" + channelID
|
||||
|
||||
req, _ := http.NewRequest("GET", url, nil)
|
||||
|
||||
req.Header.Add("User-Agent", downloader.ChromeUA)
|
||||
req.Header.Add("Accept", "*/*")
|
||||
req.Header.Add("Host", "socialblade.com")
|
||||
|
||||
res, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return 0, errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
var line string
|
||||
scanner := bufio.NewScanner(res.Body)
|
||||
for scanner.Scan() {
|
||||
if strings.Contains(scanner.Text(), "youtube-stats-header-uploads") {
|
||||
line = scanner.Text()
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if line == "" {
|
||||
return 0, errors.Err("upload count line not found")
|
||||
}
|
||||
|
||||
matches := regexp.MustCompile(">([0-9]+)<").FindStringSubmatch(line)
|
||||
if len(matches) != 2 {
|
||||
return 0, errors.Err("upload count not found with regex")
|
||||
}
|
||||
|
||||
num, err := strconv.Atoi(matches[1])
|
||||
if err != nil {
|
||||
return 0, errors.Err(err)
|
||||
}
|
||||
|
||||
return num, nil
|
||||
}
|
||||
|
||||
func ChannelInfo(channelID string) (*YoutubeStatsResponse, error) {
|
||||
url := "https://www.youtube.com/channel/" + channelID + "/about"
|
||||
|
||||
req, _ := http.NewRequest("GET", url, nil)
|
||||
|
||||
req.Header.Add("User-Agent", downloader.ChromeUA)
|
||||
req.Header.Add("Accept", "*/*")
|
||||
|
||||
res, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
pageBody := string(body)
|
||||
dataStartIndex := strings.Index(pageBody, "window[\"ytInitialData\"] = ") + 26
|
||||
if dataStartIndex == 25 {
|
||||
dataStartIndex = strings.Index(pageBody, "var ytInitialData = ") + 20
|
||||
}
|
||||
dataEndIndex := strings.Index(pageBody, "]}}};") + 4
|
||||
if dataEndIndex < dataStartIndex {
|
||||
return nil, errors.Err("start index is lower than end index. cannot extract channel info!")
|
||||
}
|
||||
data := pageBody[dataStartIndex:dataEndIndex]
|
||||
var decodedResponse YoutubeStatsResponse
|
||||
err = json.Unmarshal([]byte(data), &decodedResponse)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
return &decodedResponse, nil
|
||||
}
|
||||
|
||||
func getVideos(config *sdk.APIConfig, channelID string, videoIDs []string, stopChan stop.Chan, ipPool *ip_manager.IPPool) ([]*ytdl.YtdlVideo, error) {
|
||||
var videos []*ytdl.YtdlVideo
|
||||
for _, videoID := range videoIDs {
|
||||
if len(videoID) < 5 {
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case <-stopChan:
|
||||
return videos, errors.Err("interrupted by user")
|
||||
default:
|
||||
}
|
||||
|
||||
//ip, err := ipPool.GetIP(videoID)
|
||||
//if err != nil {
|
||||
// return nil, err
|
||||
//}
|
||||
//video, err := downloader.GetVideoInformation(videoID, &net.TCPAddr{IP: net.ParseIP(ip)})
|
||||
state, err := config.VideoState(videoID)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
if state == "published" {
|
||||
continue
|
||||
}
|
||||
video, err := downloader.GetVideoInformation(config, videoID, stopChan, nil, ipPool)
|
||||
if err != nil {
|
||||
errSDK := config.MarkVideoStatus(shared.VideoStatus{
|
||||
ChannelID: channelID,
|
||||
VideoID: videoID,
|
||||
Status: "failed",
|
||||
FailureReason: err.Error(),
|
||||
})
|
||||
logUtils.SendErrorToSlack(fmt.Sprintf("Skipping video (%s): %s", videoID, errors.FullTrace(err)))
|
||||
if errSDK != nil {
|
||||
return nil, errors.Err(errSDK)
|
||||
}
|
||||
} else {
|
||||
videos = append(videos, video)
|
||||
}
|
||||
}
|
||||
return videos, nil
|
||||
}
|
||||
|
||||
type YoutubeStatsResponse struct {
|
||||
Contents struct {
|
||||
TwoColumnBrowseResultsRenderer struct {
|
||||
Tabs []struct {
|
||||
TabRenderer struct {
|
||||
Title string `json:"title"`
|
||||
Selected bool `json:"selected"`
|
||||
Content struct {
|
||||
SectionListRenderer struct {
|
||||
Contents []struct {
|
||||
ItemSectionRenderer struct {
|
||||
Contents []struct {
|
||||
ChannelAboutFullMetadataRenderer struct {
|
||||
Description struct {
|
||||
SimpleText string `json:"simpleText"`
|
||||
} `json:"description"`
|
||||
ViewCountText struct {
|
||||
SimpleText string `json:"simpleText"`
|
||||
} `json:"viewCountText"`
|
||||
JoinedDateText struct {
|
||||
Runs []struct {
|
||||
Text string `json:"text"`
|
||||
} `json:"runs"`
|
||||
} `json:"joinedDateText"`
|
||||
CanonicalChannelURL string `json:"canonicalChannelUrl"`
|
||||
BypassBusinessEmailCaptcha bool `json:"bypassBusinessEmailCaptcha"`
|
||||
Title struct {
|
||||
SimpleText string `json:"simpleText"`
|
||||
} `json:"title"`
|
||||
Avatar struct {
|
||||
Thumbnails []struct {
|
||||
URL string `json:"url"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
} `json:"thumbnails"`
|
||||
} `json:"avatar"`
|
||||
ShowDescription bool `json:"showDescription"`
|
||||
DescriptionLabel struct {
|
||||
Runs []struct {
|
||||
Text string `json:"text"`
|
||||
} `json:"runs"`
|
||||
} `json:"descriptionLabel"`
|
||||
DetailsLabel struct {
|
||||
Runs []struct {
|
||||
Text string `json:"text"`
|
||||
} `json:"runs"`
|
||||
} `json:"detailsLabel"`
|
||||
ChannelID string `json:"channelId"`
|
||||
} `json:"channelAboutFullMetadataRenderer"`
|
||||
} `json:"contents"`
|
||||
} `json:"itemSectionRenderer"`
|
||||
} `json:"contents"`
|
||||
} `json:"sectionListRenderer"`
|
||||
} `json:"content"`
|
||||
} `json:"tabRenderer"`
|
||||
} `json:"tabs"`
|
||||
} `json:"twoColumnBrowseResultsRenderer"`
|
||||
} `json:"contents"`
|
||||
Header struct {
|
||||
C4TabbedHeaderRenderer struct {
|
||||
ChannelID string `json:"channelId"`
|
||||
Title string `json:"title"`
|
||||
Avatar struct {
|
||||
Thumbnails []struct {
|
||||
URL string `json:"url"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
} `json:"thumbnails"`
|
||||
} `json:"avatar"`
|
||||
Banner struct {
|
||||
Thumbnails []struct {
|
||||
URL string `json:"url"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
} `json:"thumbnails"`
|
||||
} `json:"banner"`
|
||||
VisitTracking struct {
|
||||
RemarketingPing string `json:"remarketingPing"`
|
||||
} `json:"visitTracking"`
|
||||
SubscriberCountText struct {
|
||||
SimpleText string `json:"simpleText"`
|
||||
} `json:"subscriberCountText"`
|
||||
} `json:"c4TabbedHeaderRenderer"`
|
||||
} `json:"header"`
|
||||
Metadata struct {
|
||||
ChannelMetadataRenderer struct {
|
||||
Title string `json:"title"`
|
||||
Description string `json:"description"`
|
||||
RssURL string `json:"rssUrl"`
|
||||
ChannelConversionURL string `json:"channelConversionUrl"`
|
||||
ExternalID string `json:"externalId"`
|
||||
Keywords string `json:"keywords"`
|
||||
OwnerUrls []string `json:"ownerUrls"`
|
||||
Avatar struct {
|
||||
Thumbnails []struct {
|
||||
URL string `json:"url"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
} `json:"thumbnails"`
|
||||
} `json:"avatar"`
|
||||
ChannelURL string `json:"channelUrl"`
|
||||
IsFamilySafe bool `json:"isFamilySafe"`
|
||||
VanityChannelURL string `json:"vanityChannelUrl"`
|
||||
} `json:"channelMetadataRenderer"`
|
||||
} `json:"metadata"`
|
||||
Topbar struct {
|
||||
DesktopTopbarRenderer struct {
|
||||
CountryCode string `json:"countryCode"`
|
||||
} `json:"desktopTopbarRenderer"`
|
||||
} `json:"topbar"`
|
||||
Microformat struct {
|
||||
MicroformatDataRenderer struct {
|
||||
URLCanonical string `json:"urlCanonical"`
|
||||
Title string `json:"title"`
|
||||
Description string `json:"description"`
|
||||
Thumbnail struct {
|
||||
Thumbnails []struct {
|
||||
URL string `json:"url"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
} `json:"thumbnails"`
|
||||
} `json:"thumbnail"`
|
||||
SiteName string `json:"siteName"`
|
||||
AppName string `json:"appName"`
|
||||
AndroidPackage string `json:"androidPackage"`
|
||||
IosAppStoreID string `json:"iosAppStoreId"`
|
||||
IosAppArguments string `json:"iosAppArguments"`
|
||||
OgType string `json:"ogType"`
|
||||
URLApplinksWeb string `json:"urlApplinksWeb"`
|
||||
URLApplinksIos string `json:"urlApplinksIos"`
|
||||
URLApplinksAndroid string `json:"urlApplinksAndroid"`
|
||||
URLTwitterIos string `json:"urlTwitterIos"`
|
||||
URLTwitterAndroid string `json:"urlTwitterAndroid"`
|
||||
TwitterCardType string `json:"twitterCardType"`
|
||||
TwitterSiteHandle string `json:"twitterSiteHandle"`
|
||||
SchemaDotOrgType string `json:"schemaDotOrgType"`
|
||||
Noindex bool `json:"noindex"`
|
||||
Unlisted bool `json:"unlisted"`
|
||||
FamilySafe bool `json:"familySafe"`
|
||||
Tags []string `json:"tags"`
|
||||
} `json:"microformatDataRenderer"`
|
||||
} `json:"microformat"`
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package ytapi
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestChannelInfo(t *testing.T) {
|
||||
info, err := ChannelInfo("UCNQfQvFMPnInwsU_iGYArJQ")
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, info)
|
||||
}
|
||||
Reference in new issue
Block a user