Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
24cf937e14 | ||
|
|
0b002c8228 | ||
|
|
a56166ee51 | ||
|
|
ddca850c17 | ||
|
|
31ce612e2e | ||
|
|
eb8900c66a | ||
|
|
67da4142d5 | ||
|
|
fb0e567caf | ||
|
|
7e83b17b69 | ||
|
|
7c02c5b92d | ||
|
|
2a33f44317 | ||
|
|
b7037900f8 | ||
|
|
5d230a6b54 | ||
|
|
f0280b51b4 | ||
|
|
775e4881cb | ||
|
|
ec9f46f552 | ||
|
|
6c6e93cefc | ||
|
|
eab9bcf1ff | ||
|
|
751bc84ce5 | ||
|
|
41fd9f6844 | ||
|
|
7f1906d58b | ||
|
|
5be3551abe | ||
|
|
fc18151d77 | ||
|
|
13543b20e9 | ||
|
|
4eba7200d8 | ||
|
|
cd11e82676 | ||
|
|
c7c220ecd3 | ||
|
|
4d56339756 | ||
|
|
140353097e | ||
|
|
a1caea4a28 | ||
|
|
b1e10e7b09 | ||
|
|
ad27425471 | ||
|
|
81e9378b9d | ||
|
|
64040ea67a | ||
|
|
d99e200178 | ||
|
|
357aebbcce | ||
|
|
0eef62b5fd | ||
|
|
a3dd3dc626 | ||
|
|
f942bf8025 | ||
|
|
a05864404d | ||
|
|
70ad891dfa | ||
|
|
8e61cde0a0 | ||
|
|
8600077caa | ||
|
|
e7fdd21bac | ||
|
|
0bb6b6d833 | ||
|
|
b59ef28267 | ||
|
|
e926a2c1f6 | ||
|
|
1369ed0b48 | ||
|
|
f39fc11697 | ||
|
|
bffc0823be | ||
|
|
edbb22fcf0 | ||
|
|
843303301a | ||
|
|
be7fd7ddd8 | ||
|
|
bfcfe5f36b | ||
|
|
4c4da93ef3 | ||
|
|
c2b5878daa | ||
|
|
749d5095c9 | ||
|
|
ba77b61ae0 | ||
|
|
07915ce697 | ||
|
|
4b2221ce5e | ||
|
|
1d5b69bfe6 | ||
|
|
4d915587e0 | ||
|
|
83f2c28c20 | ||
|
|
47e467057d | ||
|
|
1ae32d638b | ||
|
|
7907ee3579 | ||
|
|
414ed1c130 | ||
|
|
3bb17b09d3 | ||
|
|
046f46267e | ||
|
|
adc3ffe194 | ||
|
|
76301d40ba | ||
|
|
963c450730 | ||
|
|
6e7bb994d0 | ||
|
|
95890cbabf | ||
|
|
69c2a91ec7 | ||
|
|
fbe09a692c | ||
|
|
0ccc6e7979 | ||
|
|
3eb74cc8b5 | ||
|
|
9bfb1656fe | ||
|
|
5d40d24804 | ||
|
|
79d3aadda8 | ||
|
|
eba59a0806 | ||
|
|
3fd4909cd1 | ||
|
|
b59372c1d8 | ||
|
|
c9e713e9df | ||
|
|
bc72d2f1c4 | ||
|
|
df539e5d01 | ||
|
|
1b89104101 | ||
|
|
3dc30c3b98 | ||
|
|
259df8d257 | ||
|
|
5460224cd6 | ||
|
|
29479a73a7 | ||
|
|
625f339709 | ||
|
|
d6fcfd888a | ||
|
|
5f7d9d4940 | ||
|
|
826fcb78ea | ||
|
|
45e87c3007 | ||
|
|
cdee0d0de7 | ||
|
|
e30f21dc85 | ||
|
|
df88c7e952 | ||
|
|
62a1a0eed5 | ||
|
|
93f9185c71 | ||
|
|
ef62242f42 | ||
|
|
4a356e86da | ||
|
|
ac95ca57af | ||
|
|
c28d5a716b | ||
|
|
59471f9c26 | ||
|
|
8028c6621c | ||
|
|
8abf6ad255 | ||
|
|
1f9a645c57 | ||
|
|
a5657aaf46 | ||
|
|
b6861dae9b | ||
|
|
cff147bb95 | ||
|
|
54d04dcf2c | ||
|
|
65c4f99c1c | ||
|
|
68c0fd9ed7 | ||
|
|
2bde06e4b9 | ||
|
|
5df6db6e96 | ||
|
|
62dfdc1adb | ||
|
|
fe55304184 | ||
|
|
bb76500f04 | ||
|
|
4e2efec66e | ||
|
|
8d650e032c | ||
|
|
f5d97957e7 | ||
|
|
36f539ef5d | ||
|
|
77461f71af | ||
|
|
caca13de61 | ||
|
|
7bdc117ba0 | ||
|
|
fc575dac80 | ||
|
|
d520ff2c68 | ||
|
|
25e49f8ff2 | ||
|
|
77d0efcf3c | ||
|
|
291a105269 | ||
|
|
682117a030 | ||
|
|
6d2e4aa6e6 | ||
|
|
77749f05ba | ||
|
|
10a855a8ba | ||
|
|
a9e76149e8 | ||
|
|
dd4aebdba4 | ||
|
|
b409d36de0 | ||
|
|
30af4a0136 | ||
|
|
a3b660fab5 | ||
|
|
65daeec216 | ||
|
|
ad1704fb39 | ||
|
|
44e6cb5ddc | ||
|
|
5171acc007 | ||
|
|
946314da94 | ||
|
|
5961327adf | ||
|
|
d2bbb36312 | ||
|
|
c7272e988a | ||
|
|
5b9846bcca | ||
|
|
8d849b8f06 | ||
|
|
c41c368f34 | ||
|
|
7ce4c4a2ce | ||
|
|
0d473cc4f6 | ||
|
|
2f701a887a | ||
|
|
8d56ef852f | ||
|
|
849ff11bfd | ||
|
|
45982f30c5 | ||
|
|
07413b499a | ||
|
|
92d8ef75f8 | ||
|
|
1b08bb0e61 | ||
|
|
bdd55c9965 | ||
|
|
14668c339e | ||
|
|
7d38aa7b29 | ||
|
|
3c3ea2138e | ||
|
|
7b23235f83 | ||
|
|
79ae630bc4 | ||
|
|
0d8db26e96 | ||
|
|
bacb91e82a | ||
|
|
577411cf50 | ||
|
|
e98061f2eb | ||
|
|
4548c41082 | ||
|
|
f7c80c2e5d | ||
|
|
2d6e53be32 | ||
|
|
419434db28 | ||
|
|
cd6ddaaa85 | ||
|
|
4d16a1d6e8 | ||
|
|
1fb1077a94 | ||
|
|
2c9c999c71 | ||
|
|
c754562f96 | ||
|
|
7a7de03c0f | ||
|
|
1b9ed266e0 | ||
|
|
eb2f6273e4 | ||
|
|
45cea808ed | ||
|
|
9a5b6b4e56 | ||
|
|
d63aba568d | ||
|
|
4cc0b71279 | ||
|
|
b8d66cb8c0 | ||
|
|
eff76d22eb | ||
|
|
8ebdf33213 | ||
|
|
81616c1d96 | ||
|
|
b477d18a16 | ||
|
|
5cb7f72ef7 | ||
|
|
1b1d7f5ec0 | ||
|
|
adeabc97f8 | ||
|
|
d73c6945dd | ||
|
|
7a76a4b79d | ||
|
|
cd57e1ab08 | ||
|
|
b90e367e82 | ||
|
|
e477759395 | ||
|
|
6438048f40 | ||
|
|
73937a541b | ||
|
|
517c2c56a3 | ||
|
|
c84e1e6fb9 | ||
|
|
f26712b022 | ||
|
|
5e56ac0516 | ||
|
|
0308da2e4c | ||
|
|
df63b91f61 | ||
|
|
03161a5a02 | ||
|
|
6dc2c39ee6 | ||
|
|
87211e802d | ||
|
|
3ba0c97634 | ||
|
|
9d65fce6bc | ||
|
|
d5ed4ce753 | ||
|
|
0aedf80386 | ||
|
|
c193dae95a | ||
|
|
0c47510785 | ||
|
|
8f556a86c8 | ||
|
|
64ab2490fe | ||
|
|
1760935edf | ||
|
|
60f2585f33 | ||
|
|
df08d42d9b | ||
|
|
1b55033bb8 | ||
|
|
1331e5769f | ||
|
|
43f745ae20 | ||
|
|
8c2a8262e6 | ||
|
|
6f486717da | ||
|
|
a3fcd67611 | ||
|
|
ae1ffb60c5 | ||
|
|
fa24d83ae9 | ||
|
|
2f11d99a97 | ||
|
|
bab01a6354 | ||
|
|
89fe1e99d6 | ||
|
|
ec81af0857 | ||
|
|
51cb7f5e34 | ||
|
|
8ad2acc936 | ||
|
|
b8caf3f82b | ||
|
|
b34d328cba | ||
|
|
a0dcb2216d | ||
|
|
9d6a7e03cd | ||
|
|
e44e272689 | ||
|
|
87f7d4c9f9 | ||
|
|
173e62d286 | ||
|
|
013760d48d | ||
|
|
5dbad2d2e4 | ||
|
|
545ab07013 | ||
|
|
c402a1cdef |
No files matched your search
+6
-1
@@ -1 +1,6 @@
|
||||
bin/
|
||||
bin/
|
||||
e2e/persist
|
||||
e2e/supporty/supporty
|
||||
.env
|
||||
blobsfiles
|
||||
ytsync_docker
|
||||
+8
-8
@@ -9,12 +9,13 @@ builds:
|
||||
- amd64
|
||||
ldflags:
|
||||
- -X "{{ .Env.IMPORTPATH }}/meta.semVersion={{ .Tag }}" -X "{{ .Env.IMPORTPATH }}/meta.version={{ .Env.VERSIONSHORT }}" -X "{{ .Env.IMPORTPATH }}/meta.versionLong={{ .Env.VERSIONLONG }}" -X "{{ .Env.IMPORTPATH }}/meta.commitMsg={{ .Env.COMMITMSG }}"
|
||||
archive:
|
||||
name_template: '{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}'
|
||||
replacements:
|
||||
linux: Linux
|
||||
amd64: x86_64
|
||||
format: zip
|
||||
archives:
|
||||
- id: zip
|
||||
name_template: '{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}'
|
||||
replacements:
|
||||
linux: Linux
|
||||
amd64: x86_64
|
||||
format: zip
|
||||
checksum:
|
||||
name_template: 'checksums.txt'
|
||||
snapshot:
|
||||
@@ -24,5 +25,4 @@ changelog:
|
||||
filters:
|
||||
exclude:
|
||||
- '^docs:'
|
||||
- '^test:'
|
||||
|
||||
- '^test:'
|
||||
+31
-6
@@ -1,15 +1,40 @@
|
||||
os: linux
|
||||
dist: xenial
|
||||
dist: bionic
|
||||
language: go
|
||||
go_import_path: github.com/lbryio/ytsync
|
||||
go:
|
||||
- 1.11.5
|
||||
- master
|
||||
- 1.14.4
|
||||
|
||||
install: true
|
||||
|
||||
cache:
|
||||
directories:
|
||||
- $HOME/.cache/go-build
|
||||
- $HOME/gopath/pkg/mod
|
||||
|
||||
services:
|
||||
- docker
|
||||
|
||||
addons:
|
||||
apt:
|
||||
update: true
|
||||
packages:
|
||||
- ffmpeg
|
||||
- tree
|
||||
- python3-pip
|
||||
|
||||
before_script:
|
||||
- sudo pip3 install -U youtube-dl
|
||||
- sudo add-apt-repository -y ppa:savoury1/ffmpeg4
|
||||
|
||||
env:
|
||||
global:
|
||||
- GO111MODULE=on
|
||||
#GITHUB_TOKEN
|
||||
- secure: "Ps3KocRP5xnM3/uA99CeYhDTVxRIuW7fGyrtqBeRWZW0cXzeA4XCTKxqcFbrPUPw67XkrBVgE58JDdWoQEJ7tm67PjMm/ltp5Evhx/QAJDh+YSofXyGDVpG1mrTZFI66R3NVVJLkSGALMkuWWXvfYZeU//AworJbyRoaIK/CVt5OP23i5N4tdd5UXc5dfLuYqnKRynyMmCkz9c3yEIQMXoPhG2hx7l7L2BeMJvcKmVhkSN7nQayjnrbUXGm/IRqrb88lvkyBevN5E3IB2V5IKEieIPZjbD/N0IfcnAt89Z96tgDhtIbx3ZvXm92lsvHA8buqQpG9d2AmSi6GKs64lQcnGeM5o0wER2JHWl1OSa1Nr/UAo5Xb/PM65Yt3yZE8AuMKHBmbfDSBzdkTXx58AeDzFUd3kMXD/fFjeQQWyXFlOss3ygH9SObl827Txmz9OJqZaxabs5Q3AP6m3EjKjz7zfLfrgpcxJM2WBiU1bN0ZxUgZkImy/CHk5gCZ7vhcnaLiDO4HZnzY/aRJwKYQPE5i0O2nHpIfovqkc0DFBA7U/7Cjin7e1E0UZvF3meLOxMqkfc6X7QTxqQpt2Tej6jlpdxw4CTLwGUhGkAw9IAPkUB3L0EbZ1/ksGhNvGDvUeSTq8hYdMAPmA+k9jS6653V4SQ+qBMy5++tbr5AeZQI="
|
||||
|
||||
script:
|
||||
#- ./e2e/e2e.sh # Hold until we can resolve the /var/tmp issue - talk to beamer/niko
|
||||
- make
|
||||
|
||||
deploy:
|
||||
provider: script
|
||||
skip_cleanup: true
|
||||
@@ -17,4 +42,4 @@ deploy:
|
||||
file: bin/ytsync
|
||||
on:
|
||||
repo: lbryio/ytsync
|
||||
tags: true
|
||||
tags: true
|
||||
@@ -0,0 +1,111 @@
|
||||
From 30380338ba9af01696c94b61f0597131638eaec1 Mon Sep 17 00:00:00 2001
|
||||
From: Niko Storni <niko@lbry.io>
|
||||
Date: Mon, 16 Dec 2019 00:13:36 +0100
|
||||
Subject: [PATCH] lbry-patch
|
||||
|
||||
---
|
||||
youtube_dl/extractor/youtube.py | 45 +++++++++++++++++++++++++--------
|
||||
1 file changed, 35 insertions(+), 10 deletions(-)
|
||||
|
||||
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
|
||||
index b913d07a6..cd66a5b01 100644
|
||||
--- a/youtube_dl/extractor/youtube.py
|
||||
+++ b/youtube_dl/extractor/youtube.py
|
||||
@@ -10,6 +10,7 @@ import random
|
||||
import re
|
||||
import time
|
||||
import traceback
|
||||
+import subprocess
|
||||
|
||||
from .common import InfoExtractor, SearchInfoExtractor
|
||||
from ..jsinterp import JSInterpreter
|
||||
@@ -536,6 +537,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
|
||||
_GEO_BYPASS = False
|
||||
|
||||
+ _WGET_429_RATE_LIMIT = 8191
|
||||
+ _WGET_BINARY = "wget"
|
||||
+
|
||||
IE_NAME = 'youtube'
|
||||
_TESTS = [
|
||||
{
|
||||
@@ -1254,6 +1258,17 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
""" Return a string representation of a signature """
|
||||
return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
|
||||
|
||||
+ def _rate_limit_download(self, url, video_id, note=None):
|
||||
+ if note is None:
|
||||
+ self.report_download_webpage(video_id)
|
||||
+ elif note is not False:
|
||||
+ if video_id is None:
|
||||
+ self.to_screen('%s' % (note,))
|
||||
+ else:
|
||||
+ self.to_screen('%s: %s' % (video_id, note))
|
||||
+ source_address = self._downloader.params.get('source_address')
|
||||
+ return subprocess.run([self._WGET_BINARY, '-q', '--limit-rate', str(self._WGET_429_RATE_LIMIT), '--bind-address', source_address, '-O', '-', url], check=True, stdout=subprocess.PIPE).stdout.decode(encoding='UTF-8')
|
||||
+
|
||||
def _extract_signature_function(self, video_id, player_url, example_sig):
|
||||
id_m = re.match(
|
||||
r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player(?:-new)?|(?:/[a-z]{2,3}_[A-Z]{2})?/base)?\.(?P<ext>[a-z]+)$',
|
||||
@@ -1678,7 +1693,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
|
||||
# Get video webpage
|
||||
url = proto + '://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1&bpctr=9999999999' % video_id
|
||||
- video_webpage = self._download_webpage(url, video_id)
|
||||
+ video_webpage = self._rate_limit_download(url, video_id)
|
||||
|
||||
# Attempt to extract SWF player URL
|
||||
mobj = re.search(r'swfConfig.*?"(https?:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
|
||||
@@ -1736,10 +1751,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
r'"sts"\s*:\s*(\d+)', embed_webpage, 'sts', default=''),
|
||||
})
|
||||
video_info_url = proto + '://www.youtube.com/get_video_info?' + data
|
||||
- video_info_webpage = self._download_webpage(
|
||||
+ video_info_webpage = self._rate_limit_download(
|
||||
video_info_url, video_id,
|
||||
- note='Refetching age-gated info webpage',
|
||||
- errnote='unable to download video info webpage')
|
||||
+ note='Refetching age-gated info webpage')
|
||||
video_info = compat_parse_qs(video_info_webpage)
|
||||
pl_response = video_info.get('player_response', [None])[0]
|
||||
player_response = extract_player_response(pl_response, video_id)
|
||||
@@ -1777,7 +1791,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
# The general idea is to take a union of itags of both DASH manifests (for example
|
||||
# video with such 'manifest behavior' see https://github.com/ytdl-org/youtube-dl/issues/6093)
|
||||
self.report_video_info_webpage_download(video_id)
|
||||
- for el in ('embedded', 'detailpage', 'vevo', ''):
|
||||
+ for el in ('', 'embedded', 'detailpage', 'vevo'):
|
||||
query = {
|
||||
'video_id': video_id,
|
||||
'ps': 'default',
|
||||
@@ -1789,11 +1803,22 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
query['el'] = el
|
||||
if sts:
|
||||
query['sts'] = sts
|
||||
- video_info_webpage = self._download_webpage(
|
||||
- '%s://www.youtube.com/get_video_info' % proto,
|
||||
- video_id, note=False,
|
||||
- errnote='unable to download video info webpage',
|
||||
- fatal=False, query=query)
|
||||
+
|
||||
+ if el == '':
|
||||
+ base_url = 'https://youtube.com/get_video_info?video_id={}'.format(video_id)
|
||||
+ else:
|
||||
+ base_url = 'https://youtube.com/get_video_info'
|
||||
+
|
||||
+ for q in query:
|
||||
+ if q is None or q is "":
|
||||
+ continue
|
||||
+ if query[q] is None or query[q] is "":
|
||||
+ continue
|
||||
+
|
||||
+ base_url = base_url + "?{}={}".format(q, query[q])
|
||||
+
|
||||
+ video_info_webpage = self._rate_limit_download(base_url, video_id)
|
||||
+
|
||||
if not video_info_webpage:
|
||||
continue
|
||||
get_video_info = compat_parse_qs(video_info_webpage)
|
||||
--
|
||||
2.17.1
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2017-2020 LBRY Inc.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the
|
||||
"Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish,
|
||||
distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the
|
||||
following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
@@ -13,7 +13,7 @@ LDFLAGS = -ldflags "-X ${IMPORT_PATH}/meta.Version=${VERSION} -X ${IMPORT_PATH}/
|
||||
|
||||
|
||||
build:
|
||||
mkdir -p ${BIN_DIR} && CGO_ENABLED=0 go build ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/${BINARY} main.go
|
||||
mkdir -p ${BIN_DIR} && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/${BINARY} main.go
|
||||
|
||||
clean:
|
||||
if [ -f ${BIN_DIR}/${BINARY} ]; then rm ${BIN_DIR}/${BINARY}; fi
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
package blobs_reflector
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/user"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/reflector.go/cmd"
|
||||
"github.com/lbryio/reflector.go/db"
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/lbryio/ytsync/v5/util"
|
||||
)
|
||||
|
||||
var dbHandle *db.SQL
|
||||
|
||||
func ReflectAndClean() error {
|
||||
err := reflectBlobs()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return util.CleanupLbrynet()
|
||||
}
|
||||
|
||||
func loadConfig(path string) (cmd.Config, error) {
|
||||
var c cmd.Config
|
||||
|
||||
raw, err := ioutil.ReadFile(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return c, errors.Err("config file not found")
|
||||
}
|
||||
return c, err
|
||||
}
|
||||
|
||||
err = json.Unmarshal(raw, &c)
|
||||
return c, err
|
||||
}
|
||||
|
||||
func reflectBlobs() error {
|
||||
if util.IsBlobReflectionOff() {
|
||||
return nil
|
||||
}
|
||||
//make sure lbrynet is off
|
||||
running, err := util.IsLbrynetRunning()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if running {
|
||||
return errors.Prefix("cannot reflect blobs as the daemon is running", err)
|
||||
}
|
||||
logrus.SetLevel(logrus.InfoLevel)
|
||||
defer logrus.SetLevel(logrus.DebugLevel)
|
||||
ex, err := os.Executable()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
exPath := filepath.Dir(ex)
|
||||
config, err := loadConfig(exPath + "/prism_config.json")
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
if dbHandle == nil {
|
||||
dbHandle = new(db.SQL)
|
||||
err = dbHandle.Connect(config.DBConn)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
}
|
||||
st := store.NewDBBackedS3Store(
|
||||
store.NewS3BlobStore(config.AwsID, config.AwsSecret, config.BucketRegion, config.BucketName),
|
||||
dbHandle)
|
||||
|
||||
uploadWorkers := 10
|
||||
uploader := reflector.NewUploader(dbHandle, st, uploadWorkers, false)
|
||||
usr, err := user.Current()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
blobsDir := usr.HomeDir + "/.lbrynet/blobfiles/"
|
||||
err = uploader.Upload(blobsDir)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
if uploader.GetSummary().Err > 0 {
|
||||
return errors.Err("not al blobs were reflected. Errors: %d", uploader.GetSummary().Err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,364 @@
|
||||
package downloader
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/lbryio/ytsync/v5/downloader/ytdl"
|
||||
"github.com/lbryio/ytsync/v5/ip_manager"
|
||||
"github.com/lbryio/ytsync/v5/sdk"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
|
||||
args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName, "--get-id", "--flat-playlist", "--cookies", "cookies.txt"}
|
||||
ids, err := run(channelName, args, stopChan, pool)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
videoIDs := make([]string, maxVideos)
|
||||
for i, v := range ids {
|
||||
logrus.Debugf("%d - video id %s", i, v)
|
||||
if i >= maxVideos {
|
||||
break
|
||||
}
|
||||
videoIDs[i] = v
|
||||
}
|
||||
return videoIDs, nil
|
||||
}
|
||||
|
||||
const releaseTimeFormat = "2006-01-02, 15:04:05 (MST)"
|
||||
|
||||
func GetVideoInformation(config *sdk.APIConfig, videoID string, stopChan stop.Chan, ip *net.TCPAddr, pool *ip_manager.IPPool) (*ytdl.YtdlVideo, error) {
|
||||
args := []string{"--skip-download", "--print-json", "https://www.youtube.com/watch?v=" + videoID, "--cookies", "cookies.txt"}
|
||||
results, err := run(videoID, args, stopChan, pool)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
var video *ytdl.YtdlVideo
|
||||
err = json.Unmarshal([]byte(results[0]), &video)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
// now get an accurate time
|
||||
const maxTries = 5
|
||||
tries := 0
|
||||
GetTime:
|
||||
tries++
|
||||
t, err := getUploadTime(config, videoID, ip, video.UploadDate)
|
||||
if err != nil {
|
||||
//slack(":warning: Upload time error: %v", err)
|
||||
if tries <= maxTries && (errors.Is(err, errNotScraped) || errors.Is(err, errUploadTimeEmpty) || errors.Is(err, errStatusParse) || errors.Is(err, errConnectionIssue)) {
|
||||
err := triggerScrape(videoID, ip)
|
||||
if err == nil {
|
||||
time.Sleep(2 * time.Second) // let them scrape it
|
||||
goto GetTime
|
||||
} else {
|
||||
//slack("triggering scrape returned error: %v", err)
|
||||
}
|
||||
} else if !errors.Is(err, errNotScraped) && !errors.Is(err, errUploadTimeEmpty) {
|
||||
//slack(":warning: Error while trying to get accurate upload time for %s: %v", videoID, err)
|
||||
if t == "" {
|
||||
return nil, errors.Err(err)
|
||||
} else {
|
||||
t = "" //TODO: get rid of the other piece below?
|
||||
}
|
||||
}
|
||||
// do fallback below
|
||||
}
|
||||
//slack("After all that, upload time for %s is %s", videoID, t)
|
||||
|
||||
if t != "" {
|
||||
parsed, err := time.Parse("2006-01-02, 15:04:05 (MST)", t) // this will probably be UTC, but Go's timezone parsing is fucked up. it ignores the timezone in the date
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
//slack(":exclamation: Got an accurate time for %s", videoID)
|
||||
video.UploadDateForReal = parsed
|
||||
} else { //TODO: this is the piece that isn't needed!
|
||||
slack(":warning: Could not get accurate time for %s. Falling back to time from upload ytdl: %s.", videoID, video.UploadDate)
|
||||
// fall back to UploadDate from youtube-dl
|
||||
video.UploadDateForReal, err = time.Parse("20060102", video.UploadDate)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return video, nil
|
||||
}
|
||||
|
||||
var errNotScraped = errors.Base("not yet scraped by caa.iti.gr")
|
||||
var errUploadTimeEmpty = errors.Base("upload time is empty")
|
||||
var errStatusParse = errors.Base("could not parse status, got number, need string")
|
||||
var errConnectionIssue = errors.Base("there was a connection issue with the api")
|
||||
|
||||
func slack(format string, a ...interface{}) {
|
||||
fmt.Printf(format+"\n", a...)
|
||||
util.SendToSlack(format, a...)
|
||||
}
|
||||
|
||||
func triggerScrape(videoID string, ip *net.TCPAddr) error {
|
||||
//slack("Triggering scrape for %s", videoID)
|
||||
u, err := url.Parse("https://caa.iti.gr/verify_videoV3")
|
||||
q := u.Query()
|
||||
q.Set("twtimeline", "0")
|
||||
q.Set("url", "https://www.youtube.com/watch?v="+videoID)
|
||||
u.RawQuery = q.Encode()
|
||||
//slack("GET %s", u.String())
|
||||
|
||||
client := getClient(ip)
|
||||
req, err := http.NewRequest(http.MethodGet, u.String(), nil)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36")
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
var response struct {
|
||||
Message string `json:"message"`
|
||||
Status string `json:"status"`
|
||||
VideoURL string `json:"video_url"`
|
||||
}
|
||||
err = json.NewDecoder(res.Body).Decode(&response)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "cannot unmarshal number") {
|
||||
return errors.Err(errStatusParse)
|
||||
}
|
||||
if strings.Contains(err.Error(), "no route to host") {
|
||||
return errors.Err(errConnectionIssue)
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
switch response.Status {
|
||||
case "removed_video":
|
||||
return errors.Err("video previously removed from service")
|
||||
case "no_video":
|
||||
return errors.Err("they say 'video cannot be found'. wtf?")
|
||||
default:
|
||||
spew.Dump(response)
|
||||
}
|
||||
|
||||
return nil
|
||||
//https://caa.iti.gr/caa/api/v4/videos/reports/h-tuxHS5lSM
|
||||
}
|
||||
|
||||
func getUploadTime(config *sdk.APIConfig, videoID string, ip *net.TCPAddr, uploadDate string) (string, error) {
|
||||
//slack("Getting upload time for %s", videoID)
|
||||
release, err := config.GetReleasedDate(videoID)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
ytdlUploadDate, err := time.Parse("20060102", uploadDate)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
if release != nil {
|
||||
//const sqlTimeFormat = "2006-01-02 15:04:05"
|
||||
sqlTime, err := time.ParseInLocation(time.RFC3339, release.ReleaseTime, time.UTC)
|
||||
if err == nil {
|
||||
if sqlTime.Day() != ytdlUploadDate.Day() {
|
||||
logrus.Infof("upload day from APIs differs from the ytdl one by more than 1 day.")
|
||||
} else {
|
||||
return sqlTime.Format(releaseTimeFormat), nil
|
||||
}
|
||||
} else {
|
||||
logrus.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
if time.Now().AddDate(0, 0, -3).After(ytdlUploadDate) {
|
||||
return ytdlUploadDate.Format(releaseTimeFormat), nil
|
||||
}
|
||||
client := getClient(ip)
|
||||
req, err := http.NewRequest(http.MethodGet, "https://caa.iti.gr/get_verificationV3?url=https://www.youtube.com/watch?v="+videoID, nil)
|
||||
if err != nil {
|
||||
return ytdlUploadDate.Format(releaseTimeFormat), errors.Err(err)
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36")
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return ytdlUploadDate.Format(releaseTimeFormat), errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
var uploadTime struct {
|
||||
Time string `json:"video_upload_time"`
|
||||
Message string `json:"message"`
|
||||
Status string `json:"status"`
|
||||
}
|
||||
err = json.NewDecoder(res.Body).Decode(&uploadTime)
|
||||
if err != nil {
|
||||
return ytdlUploadDate.Format(releaseTimeFormat), errors.Err(err)
|
||||
}
|
||||
|
||||
if uploadTime.Status == "ERROR1" {
|
||||
return ytdlUploadDate.Format(releaseTimeFormat), errNotScraped
|
||||
}
|
||||
|
||||
if uploadTime.Status == "" && strings.HasPrefix(uploadTime.Message, "CANNOT_RETRIEVE_REPORT_FOR_VIDEO_") {
|
||||
return ytdlUploadDate.Format(releaseTimeFormat), errors.Err("cannot retrieve report for video")
|
||||
}
|
||||
|
||||
if uploadTime.Time == "" {
|
||||
return ytdlUploadDate.Format(releaseTimeFormat), errUploadTimeEmpty
|
||||
}
|
||||
|
||||
return uploadTime.Time, nil
|
||||
}
|
||||
|
||||
func getClient(ip *net.TCPAddr) *http.Client {
|
||||
if ip == nil {
|
||||
return http.DefaultClient
|
||||
}
|
||||
|
||||
return &http.Client{
|
||||
Transport: &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
DialContext: (&net.Dialer{
|
||||
LocalAddr: ip,
|
||||
Timeout: 30 * time.Second,
|
||||
KeepAlive: 30 * time.Second,
|
||||
}).DialContext,
|
||||
MaxIdleConns: 100,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
TLSHandshakeTimeout: 10 * time.Second,
|
||||
ExpectContinueTimeout: 1 * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
googleBotUA = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
|
||||
chromeUA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"
|
||||
maxAttempts = 3
|
||||
extractionError = "YouTube said: Unable to extract video data"
|
||||
throttledError = "HTTP Error 429"
|
||||
AlternateThrottledError = "returned non-zero exit status 8"
|
||||
youtubeDlError = "exit status 1"
|
||||
)
|
||||
|
||||
func run(use string, args []string, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
|
||||
var useragent []string
|
||||
var lastError error
|
||||
for attempts := 0; attempts < maxAttempts; attempts++ {
|
||||
sourceAddress, err := getIPFromPool(use, stopChan, pool)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
argsForCommand := append(args, "--source-address", sourceAddress)
|
||||
argsForCommand = append(argsForCommand, useragent...)
|
||||
cmd := exec.Command("youtube-dl", argsForCommand...)
|
||||
|
||||
res, err := runCmd(cmd, stopChan)
|
||||
pool.ReleaseIP(sourceAddress)
|
||||
if err == nil {
|
||||
return res, nil
|
||||
}
|
||||
lastError = err
|
||||
if strings.Contains(err.Error(), youtubeDlError) {
|
||||
if strings.Contains(err.Error(), extractionError) {
|
||||
logrus.Warnf("known extraction error: %s", errors.FullTrace(err))
|
||||
useragent = nextUA(useragent)
|
||||
}
|
||||
if strings.Contains(err.Error(), throttledError) || strings.Contains(err.Error(), AlternateThrottledError) {
|
||||
pool.SetThrottled(sourceAddress)
|
||||
//we don't want throttle errors to count toward the max retries
|
||||
attempts--
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil, lastError
|
||||
}
|
||||
|
||||
func nextUA(current []string) []string {
|
||||
if len(current) == 0 {
|
||||
return []string{"--user-agent", googleBotUA}
|
||||
}
|
||||
return []string{"--user-agent", chromeUA}
|
||||
}
|
||||
|
||||
func runCmd(cmd *exec.Cmd, stopChan stop.Chan) ([]string, error) {
|
||||
logrus.Infof("running youtube-dl cmd: %s", strings.Join(cmd.Args, " "))
|
||||
var err error
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
err = cmd.Start()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
outLog, err := ioutil.ReadAll(stdout)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
errorLog, err := ioutil.ReadAll(stderr)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- cmd.Wait()
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-stopChan:
|
||||
err := cmd.Process.Kill()
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("failed to kill command after stopper cancellation", err)
|
||||
}
|
||||
return nil, errors.Err("canceled by stopper")
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("youtube-dl "+strings.Join(cmd.Args, " ")+" ["+string(errorLog)+"]", err)
|
||||
}
|
||||
return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil
|
||||
}
|
||||
}
|
||||
|
||||
func getIPFromPool(use string, stopChan stop.Chan, pool *ip_manager.IPPool) (sourceAddress string, err error) {
|
||||
for {
|
||||
sourceAddress, err = pool.GetIP(use)
|
||||
if err != nil {
|
||||
if errors.Is(err, ip_manager.ErrAllThrottled) {
|
||||
select {
|
||||
case <-stopChan:
|
||||
return "", errors.Err("interrupted by user")
|
||||
|
||||
default:
|
||||
time.Sleep(ip_manager.IPCooldownPeriod)
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package downloader
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/lbryio/ytsync/v5/sdk"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestGetPlaylistVideoIDs(t *testing.T) {
|
||||
videoIDs, err := GetPlaylistVideoIDs("UCJ0-OtVpF0wOKEqT2Z1HEtA", 50, nil, nil)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
for _, id := range videoIDs {
|
||||
println(id)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetVideoInformation(t *testing.T) {
|
||||
video, err := GetVideoInformation(nil, "zj7pXM9gE5M", nil, nil, nil)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
if video != nil {
|
||||
logrus.Info(video.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func Test_getUploadTime(t *testing.T) {
|
||||
configs := sdk.APIConfig{
|
||||
YoutubeAPIKey: "",
|
||||
ApiURL: "https://api.lbry.com",
|
||||
ApiToken: "Ht4NETrL5oWKyAaZkuSV68BKhtXkiLh5",
|
||||
HostName: "test",
|
||||
}
|
||||
got, err := getUploadTime(&configs, "kDGOHNpRjzc", nil, "20060102")
|
||||
assert.NoError(t, err)
|
||||
t.Log(got)
|
||||
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
package ytdl
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
type YtdlVideo struct {
|
||||
UploadDate string `json:"upload_date"`
|
||||
UploadDateForReal time.Time // you need to manually set this since the value in the API doesn't include the time
|
||||
Extractor string `json:"extractor"`
|
||||
Series interface{} `json:"series"`
|
||||
Format string `json:"format"`
|
||||
Vbr interface{} `json:"vbr"`
|
||||
Chapters interface{} `json:"chapters"`
|
||||
Height int `json:"height"`
|
||||
LikeCount interface{} `json:"like_count"`
|
||||
Duration int `json:"duration"`
|
||||
Fulltitle string `json:"fulltitle"`
|
||||
PlaylistIndex interface{} `json:"playlist_index"`
|
||||
Album interface{} `json:"album"`
|
||||
ViewCount int `json:"view_count"`
|
||||
Playlist interface{} `json:"playlist"`
|
||||
Title string `json:"title"`
|
||||
Filename string `json:"_filename"`
|
||||
Creator interface{} `json:"creator"`
|
||||
Ext string `json:"ext"`
|
||||
ID string `json:"id"`
|
||||
DislikeCount interface{} `json:"dislike_count"`
|
||||
AverageRating float64 `json:"average_rating"`
|
||||
Abr int `json:"abr"`
|
||||
UploaderURL string `json:"uploader_url"`
|
||||
Categories []string `json:"categories"`
|
||||
Fps float64 `json:"fps"`
|
||||
StretchedRatio interface{} `json:"stretched_ratio"`
|
||||
SeasonNumber interface{} `json:"season_number"`
|
||||
Annotations interface{} `json:"annotations"`
|
||||
WebpageURLBasename string `json:"webpage_url_basename"`
|
||||
Acodec string `json:"acodec"`
|
||||
DisplayID string `json:"display_id"`
|
||||
//RequestedFormats []RequestedFormat `json:"requested_formats"`
|
||||
//AutomaticCaptions struct{} `json:"automatic_captions"`
|
||||
Description string `json:"description"`
|
||||
Tags []string `json:"tags"`
|
||||
Track interface{} `json:"track"`
|
||||
RequestedSubtitles interface{} `json:"requested_subtitles"`
|
||||
StartTime interface{} `json:"start_time"`
|
||||
Uploader string `json:"uploader"`
|
||||
ExtractorKey string `json:"extractor_key"`
|
||||
FormatID string `json:"format_id"`
|
||||
EpisodeNumber interface{} `json:"episode_number"`
|
||||
UploaderID string `json:"uploader_id"`
|
||||
//Subtitles struct{} `json:"subtitles"`
|
||||
ReleaseYear interface{} `json:"release_year"`
|
||||
Thumbnails []Thumbnail `json:"thumbnails"`
|
||||
License interface{} `json:"license"`
|
||||
Artist interface{} `json:"artist"`
|
||||
AgeLimit int `json:"age_limit"`
|
||||
ReleaseDate interface{} `json:"release_date"`
|
||||
AltTitle interface{} `json:"alt_title"`
|
||||
Thumbnail string `json:"thumbnail"`
|
||||
ChannelID string `json:"channel_id"`
|
||||
IsLive interface{} `json:"is_live"`
|
||||
Width int `json:"width"`
|
||||
EndTime interface{} `json:"end_time"`
|
||||
WebpageURL string `json:"webpage_url"`
|
||||
//Formats []Format `json:"formats"`
|
||||
ChannelURL string `json:"channel_url"`
|
||||
Resolution interface{} `json:"resolution"`
|
||||
Vcodec string `json:"vcodec"`
|
||||
}
|
||||
|
||||
type RequestedFormat struct {
|
||||
Asr interface{} `json:"asr"`
|
||||
Tbr float64 `json:"tbr"`
|
||||
Container string `json:"container"`
|
||||
Language interface{} `json:"language"`
|
||||
Format string `json:"format"`
|
||||
URL string `json:"url"`
|
||||
Vcodec string `json:"vcodec"`
|
||||
FormatNote string `json:"format_note"`
|
||||
Height int `json:"height"`
|
||||
Width int `json:"width"`
|
||||
Ext string `json:"ext"`
|
||||
FragmentBaseURL string `json:"fragment_base_url"`
|
||||
Filesize interface{} `json:"filesize"`
|
||||
Fps float64 `json:"fps"`
|
||||
ManifestURL string `json:"manifest_url"`
|
||||
Protocol string `json:"protocol"`
|
||||
FormatID string `json:"format_id"`
|
||||
HTTPHeaders struct {
|
||||
AcceptCharset string `json:"Accept-Charset"`
|
||||
AcceptLanguage string `json:"Accept-Language"`
|
||||
AcceptEncoding string `json:"Accept-Encoding"`
|
||||
Accept string `json:"Accept"`
|
||||
UserAgent string `json:"User-Agent"`
|
||||
} `json:"http_headers"`
|
||||
Fragments []struct {
|
||||
Path string `json:"path"`
|
||||
Duration float64 `json:"duration,omitempty"`
|
||||
} `json:"fragments"`
|
||||
Acodec string `json:"acodec"`
|
||||
Abr int `json:"abr,omitempty"`
|
||||
}
|
||||
|
||||
type Format struct {
|
||||
Asr int `json:"asr"`
|
||||
Tbr float64 `json:"tbr"`
|
||||
Protocol string `json:"protocol"`
|
||||
Format string `json:"format"`
|
||||
FormatNote string `json:"format_note"`
|
||||
Height interface{} `json:"height"`
|
||||
ManifestURL string `json:"manifest_url,omitempty"`
|
||||
FormatID string `json:"format_id"`
|
||||
Container string `json:"container,omitempty"`
|
||||
Language interface{} `json:"language,omitempty"`
|
||||
HTTPHeaders HTTPHeaders `json:"http_headers"`
|
||||
URL string `json:"url"`
|
||||
Vcodec string `json:"vcodec"`
|
||||
Abr int `json:"abr,omitempty"`
|
||||
Width interface{} `json:"width"`
|
||||
Ext string `json:"ext"`
|
||||
FragmentBaseURL string `json:"fragment_base_url,omitempty"`
|
||||
Filesize interface{} `json:"filesize"`
|
||||
Fps float64 `json:"fps"`
|
||||
Fragments []struct {
|
||||
Path string `json:"path"`
|
||||
Duration float64 `json:"duration,omitempty"`
|
||||
} `json:"fragments,omitempty"`
|
||||
Acodec string `json:"acodec"`
|
||||
PlayerURL interface{} `json:"player_url,omitempty"`
|
||||
}
|
||||
|
||||
type Thumbnail struct {
|
||||
URL string `json:"url"`
|
||||
Width int `json:"width"`
|
||||
Resolution string `json:"resolution"`
|
||||
ID string `json:"id"`
|
||||
Height int `json:"height"`
|
||||
}
|
||||
|
||||
type HTTPHeaders struct {
|
||||
AcceptCharset string `json:"Accept-Charset"`
|
||||
AcceptLanguage string `json:"Accept-Language"`
|
||||
AcceptEncoding string `json:"Accept-Encoding"`
|
||||
Accept string `json:"Accept"`
|
||||
UserAgent string `json:"User-Agent"`
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
version: '3.4'
|
||||
|
||||
services:
|
||||
###########
|
||||
## MYSQL ##
|
||||
###########
|
||||
mysql:
|
||||
image: mysql:5.7.23
|
||||
restart: "no"
|
||||
ports:
|
||||
- 3306:3306
|
||||
volumes:
|
||||
- "../persist/chainquery/db:/var/lib/mysql"
|
||||
## This one may need to be tweaked based on where you run this docker-compose from.
|
||||
- "../stuff/my.cnf:/etc/mysql/conf.d/chainquery-optimizations.cnf"
|
||||
################
|
||||
## Chainquery ##
|
||||
################
|
||||
chainquery:
|
||||
image: lbry/chainquery:v1.8.1
|
||||
restart: "no"
|
||||
ports:
|
||||
- 6300:6300
|
||||
depends_on:
|
||||
- mysql
|
||||
## TODO: Uncomment this in a docker-compose.override.yml to allow for external configurations.
|
||||
volumes:
|
||||
- "../persist/chainquery/config/chainqueryconfig.toml:/etc/chainquery/chainqueryconfig.toml"
|
||||
@@ -0,0 +1,33 @@
|
||||
## Get the latest source and extract it for the app container.
|
||||
## Design choices, two RUN layers intended to keep builds faster, the zipped
|
||||
FROM ubuntu:18.04 as prep
|
||||
LABEL MAINTAINER="leopere [at] nixc [dot] us"
|
||||
RUN apt-get update && \
|
||||
apt-get -y install unzip curl telnet wait-for-it && \
|
||||
apt-get autoclean -y && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
WORKDIR /
|
||||
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
|
||||
COPY ./start.sh start
|
||||
COPY ./healthcheck.sh healthcheck
|
||||
ARG VERSION="master"
|
||||
RUN curl -s -o /chainquery http://build.lbry.io/chainquery/branch-"${VERSION}"/chainquery && \
|
||||
chmod +x /chainquery
|
||||
|
||||
|
||||
FROM ubuntu:18.04 as app
|
||||
RUN apt-get update && \
|
||||
apt-get -y install telnet wait-for-it && \
|
||||
apt-get autoclean -y && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
ARG VERSION="master"
|
||||
ADD https://raw.githubusercontent.com/lbryio/chainquery/"${VERSION}"/config/default/chainqueryconfig.toml /etc/lbry/chainqueryconfig.toml.orig
|
||||
RUN adduser chainquery --gecos GECOS --shell /bin/bash --disabled-password --home /home/chainquery && \
|
||||
chown -R chainquery:chainquery /etc/lbry
|
||||
COPY --from=prep ./healthcheck /chainquery /start /usr/bin/
|
||||
HEALTHCHECK --interval=1m --timeout=30s \
|
||||
CMD healthcheck
|
||||
EXPOSE 6300
|
||||
USER chainquery
|
||||
STOPSIGNAL SIGINT
|
||||
CMD ["start"]
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/bin/bash
|
||||
if [ $# -eq 0 ]
|
||||
then
|
||||
echo "No docker tag argument supplied. Use './build.sh <tag>'"
|
||||
exit 1
|
||||
fi
|
||||
docker build --no-cache --build-arg VERSION=$1 --tag lbry/chainquery:$1 .
|
||||
docker push lbry/chainquery:$1
|
||||
Executable
+2
@@ -0,0 +1,2 @@
|
||||
#!/usr/bin/env bash
|
||||
curl --fail http://localhost:6300/api/status || exit 1
|
||||
@@ -0,0 +1,9 @@
|
||||
# Default Homebrew MySQL server config
|
||||
[mysqld]
|
||||
# Only allow connections from localhost
|
||||
innodb_log_file_size=5G
|
||||
key_buffer_size=1G
|
||||
innodb_flush_log_at_trx_commit = 0
|
||||
innodb_autoinc_lock_mode=2
|
||||
innodb_buffer_pool_size=1G
|
||||
innodb_log_buffer_size=1G
|
||||
Executable
+51
@@ -0,0 +1,51 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
## Config setup
|
||||
|
||||
## Setup Values
|
||||
DEBUGMODE=$(echo "debugmode=$DEBUGMODE")
|
||||
LBRYCRDURL=$(echo "lbrycrdurl=\"rpc://$RPC_USER:$RPC_PASSWORD@10.5.1.2:9245\"")
|
||||
MYSQLDSN=$(echo "mysqldsn=\"$MYSQL_USER:$MYSQL_PASSWORD@tcp($MYSQL_SERVER:3306)/$MYSQL_DATABASE\"")
|
||||
APIMYSQLDSN=$(echo "apimysqldsn=\"$MYSQL_USER:$MYSQL_PASSWORD@tcp($MYSQL_SERVER:3306)/$MYSQL_DATABASE\"")
|
||||
|
||||
## Setup Defaults
|
||||
DEBUGMODE_DEFAULT='#DEFAULT-debugmode=false'
|
||||
LBRYCRDURL_DEFAULT='#DEFAULT-lbrycrdurl="rpc://lbry:lbry@localhost:9245"'
|
||||
MYSQLDSN_DEFAULT='#DEFAULT-mysqldsn="lbry:lbry@tcp(localhost:3306)/chainquery"'
|
||||
APIMYSQLDSN_DEFAULT='#DEFAULT-apimysqldsn="lbry:lbry@tcp(localhost:3306)/chainquery"'
|
||||
|
||||
## Add setup value variable name to this list to get processed on container start
|
||||
CONFIG_SETTINGS=(
|
||||
DEBUGMODE
|
||||
LBRYCRDURL
|
||||
MYSQLDSN
|
||||
APIMYSQLDSN
|
||||
)
|
||||
|
||||
function set_configs() {
|
||||
## Set configs on container start if not already set.
|
||||
for i in "${!CONFIG_SETTINGS[@]}"; do
|
||||
## Indirect references http://tldp.org/LDP/abs/html/ivr.html
|
||||
eval FROM_STRING=\$"${CONFIG_SETTINGS[$i]}_DEFAULT"
|
||||
eval TO_STRING=\$${CONFIG_SETTINGS[$i]}
|
||||
## TODO: Add a bit more magic to make sure that you're only configuring things if not set by config mounts.
|
||||
sed -i "s~$FROM_STRING~"$TO_STRING"~g" /etc/lbry/chainqueryconfig.toml
|
||||
done
|
||||
echo "Reading config for debugging."
|
||||
cat /etc/lbry/chainqueryconfig.toml
|
||||
}
|
||||
|
||||
if [[ ! -f /etc/lbry/chainqueryconfig.toml ]]; then
|
||||
echo "[INFO]: Did not find chainqueryconfig.toml"
|
||||
echo " Installing default and configuring with provided environment variables if any."
|
||||
## Install fresh copy of config file.
|
||||
echo "cp -v /etc/lbry/chainqueryconfig.toml.orig /etc/lbry/chainqueryconfig.toml"
|
||||
cp -v /etc/lbry/chainqueryconfig.toml.orig /etc/lbry/chainqueryconfig.toml
|
||||
ls -lAh /etc/lbry/
|
||||
set_configs
|
||||
else
|
||||
echo "[INFO]: Found a copy of chainqueryconfig.toml in /etc/lbry"
|
||||
fi
|
||||
|
||||
## For now keeping this simple. Potentially eventually add all command args as envvars for the Dockerfile or use safe way to add args via docker-compose.yml
|
||||
chainquery serve --configpath "/etc/lbry/"
|
||||
@@ -0,0 +1,99 @@
|
||||
#Debug mode outputs specific information to the console
|
||||
#DEFAULT: false
|
||||
#debugmode=
|
||||
|
||||
#DebugQueryMode outputs SQL Boiler queries to the console.
|
||||
#DEFAULT: false
|
||||
#debugquerymode=
|
||||
|
||||
#LBRYcrd URL is required for chainquery to query the blockchain
|
||||
#DEFAULT: "rpc://lbry:lbry@localhost:9245"
|
||||
lbrycrdurl="rpc://lbry:lbry@lbrycrd:29245"
|
||||
|
||||
#MySQL DSN is required for chainquery to store information.
|
||||
#DEFAULT: "lbry:lbry@tcp(localhost:3306)/chainquery"
|
||||
#SUGGESTED: "lbry:lbry@unix(/var/run/mysqld/mysqld.sock)/chainquery"
|
||||
mysqldsn="lbry:lbry@tcp(mysql:3306)/chainquery"
|
||||
|
||||
#API MySQL DSN is required for chainquery to expose a SQL query service
|
||||
#DEFAULT: "lbry:lbry@tcp(localhost:3306)/chainquery"
|
||||
#SUGGESTED: "lbry:lbry@unix(/var/run/mysqld/mysqld.sock)/chainquery"
|
||||
apimysqldsn="lbry:lbry@tcp(mysql:3306)/chainquery"
|
||||
|
||||
#API Host and Port is required for the API Server to bind and listen on.
|
||||
#DEFAULT: "0.0.0.0:6300"
|
||||
#apihostport=
|
||||
|
||||
#Profile mode enables and disables the reporting of a profile for chainquery
|
||||
#DEFAULT: false
|
||||
#profilemode=
|
||||
|
||||
#Daemon mode tells chainquery how hard it should work catch up processing the blockchain
|
||||
#deamonmode=0 #BeastMode it continuously process block after block until caughtup.
|
||||
#daemonmode=1 #SlowAndSteadyMode it will process block with a frequency of 1 block every 100ms
|
||||
#daemonmode=2 #DelayMode it will process a block with a configured delay frequency (set via 'processingdelay')
|
||||
#daemonmode=3 #DaemonMode it will process a block every iteration of the daemon.
|
||||
#DEFAULT: 0
|
||||
#deamonmode=
|
||||
|
||||
#Default client timeout is for communication with the api of chainquery
|
||||
#DEFAULT: 20 #Measured in seconds
|
||||
#defaultclienttimeout=
|
||||
|
||||
#Processing delay is used to determine how frequently chainquery should process a block
|
||||
# It is only used if Daemon mode is set to delay mode
|
||||
#DEFAULT: 100 #Measured in milliseconds
|
||||
#processingdelay=
|
||||
|
||||
#Daemon delay is the frequency at which chainquery checks for work to do.
|
||||
#DEFAULT: 1 #Measured in seconds
|
||||
#daemondelay=
|
||||
|
||||
#Profiling options - will output the time take for certain opertions related to the below category
|
||||
#DEFAULT: false (for all 3 params)
|
||||
#daemonprofile=
|
||||
#lbrycrdprofile=
|
||||
#mysqlprofile=
|
||||
|
||||
#Slack Hook URL allows slack integration. All logging info level and above is posted to a slack channel.
|
||||
#DEFAULT: ""
|
||||
#slackhookurl=
|
||||
|
||||
#Slack Channel is the channel that you want the messages to appear. Works together with the hook url.
|
||||
#DEFAULT: ""
|
||||
#slackchannel=
|
||||
|
||||
#Slack Log Level tells chainquery what level of logging will be sent to the slack channel. It will log all levels below
|
||||
# it as well. Panic=0,Fatal=1,Error=2,Warning=3,Info=4,Debug=5
|
||||
#DEFAULT: 0
|
||||
#slackloglevel=
|
||||
|
||||
#The command that should be executed to trigger a self update of the software. For linux, for example, `<yourscript>.sh`
|
||||
#DEFAULT: ""
|
||||
#autoupdatecommand=
|
||||
|
||||
#Twilio service of chainquery to send specifically important information to key users of the Chainquery install.
|
||||
#DEFAULT:
|
||||
##twiliosid=""
|
||||
##twilioauthtoken=""
|
||||
##smsrecipients=["",""]
|
||||
##smsfromphonenumber=""
|
||||
#twiliosid=
|
||||
#twilioauthtoken=
|
||||
#smsrecipients=
|
||||
#smsfromphonenumber=
|
||||
|
||||
#API Keys - Disallowed by default unless keys are entered.
|
||||
#DEFAULT: []
|
||||
#apikeys=
|
||||
|
||||
#Max Failures - Specifies the number of failures that can happen in processing a transaction. This is for parallel
|
||||
#transaction processing which puts a transaction to the back of the processing queue if it fails. It can fail say if its
|
||||
#source output to spend is not already processed.
|
||||
#DEFAULT: 1000
|
||||
#maxfailures=
|
||||
|
||||
#Block Chain Name - Specifies the chain params for parsing blocks, transactions, claims, and addresses. valid choices are
|
||||
#lbrycrd_main, lbrycrd_testnet, and lbrycrd_regtest.
|
||||
#DEFAULT: "lbrycrd_main"
|
||||
blockchainname="lbrycrd_regtest"
|
||||
@@ -0,0 +1,10 @@
|
||||
blockchain_name: lbrycrd_regtest
|
||||
lbryum_servers:
|
||||
- walletserver:50001
|
||||
reflect_streams: false
|
||||
save_blobs: true
|
||||
save_files: false
|
||||
share_usage_data: false
|
||||
tcp_port: 3333
|
||||
udp_port: 4444
|
||||
use_upnp: false
|
||||
Executable
+24
@@ -0,0 +1,24 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
#Add a ytsync user
|
||||
ADDYTSYNCUSER='INSERT INTO user (given_name) VALUE("ytsync user")'
|
||||
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCUSER"
|
||||
#Insert an auth token for the user to be used by ytsync
|
||||
ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(1,"ytsyntoken")'
|
||||
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN"
|
||||
#Give priveledges to ytsync user
|
||||
ASSIGNGROOP='INSERT INTO user_groop (user_id, groop_id) VALUE( 1,3)'
|
||||
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ASSIGNGROOP"
|
||||
|
||||
#Add youtuber to sync
|
||||
ADDYTSYNCER='INSERT INTO user (given_name) VALUE("youtuber")'
|
||||
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCER"
|
||||
#Insert an auth token for the youtuber to be used by ytsync
|
||||
ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(2,"youtubertoken")'
|
||||
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN"
|
||||
#Add their youtube channel to be synced
|
||||
ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers,should_sync,redeemable,total_views,reviewed,last_uploaded_video,length_limit,size_limit)
|
||||
VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@test"$(date +%s)"','UCNQfQvFMPnInwsU_iGYArJQ','BeamerAtLBRY','queued','2019-08-01 00:00:00','sync',10,10,1,1,10000,1,'7bBV2Z-9wpo',60,2048)"
|
||||
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTCHANNEL"
|
||||
@@ -0,0 +1,117 @@
|
||||
version: "3.5"
|
||||
services:
|
||||
#############
|
||||
## Lbrycrd ##
|
||||
#############
|
||||
lbrycrd:
|
||||
image: lbry/lbrycrd:v0.17.3.2-deprecatedrpc
|
||||
restart: "no"
|
||||
ports:
|
||||
- "15201:29246"
|
||||
- "15200:29245"
|
||||
expose:
|
||||
- "29246"
|
||||
- "29245"
|
||||
## host volumes for persistent data such as wallet private keys.
|
||||
volumes:
|
||||
- "./persist:/data"
|
||||
environment:
|
||||
- RUN_MODE=regtest
|
||||
###################
|
||||
## Wallet Server ##
|
||||
###################
|
||||
walletserver:
|
||||
image: lbry/wallet-server:v0.73.1
|
||||
restart: always
|
||||
environment:
|
||||
- DB_DIRECTORY=/database
|
||||
- MAX_SEND=1000000000000000000000
|
||||
- DAEMON_URL=http://lbry:lbry@lbrycrd:29245
|
||||
- MAX_SUBS=1000000000000
|
||||
- BANDWIDTH_LIMIT=80000000000
|
||||
- SESSION_TIMEOUT=10000000000000000000000000
|
||||
- TCP_PORT=50001
|
||||
ports:
|
||||
- "15300:50001"
|
||||
expose:
|
||||
- "50001"
|
||||
depends_on:
|
||||
- lbrycrd
|
||||
ulimits:
|
||||
nofile:
|
||||
soft: 90000
|
||||
hard: 90000
|
||||
#command: lbry.wallet.server.coin.LBC
|
||||
command: lbry.wallet.server.coin.LBCRegTest
|
||||
#############
|
||||
## Lbrynet ##
|
||||
#############
|
||||
lbrynet:
|
||||
image: lbry/lbrynet:v0.73.1
|
||||
restart: always
|
||||
ports:
|
||||
- "15100:5279"
|
||||
- "15101:5280"
|
||||
expose:
|
||||
- "5279"
|
||||
- "5280"
|
||||
depends_on:
|
||||
- walletserver
|
||||
environment:
|
||||
- LBRY_STREAMING_SERVER=0.0.0.0:5280
|
||||
volumes:
|
||||
- "./persist/.lbrynet:/home/lbrynet"
|
||||
- ".:/etc/lbry" #Put your daemon_settings.yml here
|
||||
# /private/var/tmp for OSX and /var/tmp for Linux
|
||||
- "${LOCAL_TMP_DIR}"
|
||||
###########
|
||||
## MySQL ##
|
||||
###########
|
||||
mysql:
|
||||
image: mysql/mysql-server:5.7.27
|
||||
restart: "no"
|
||||
ports:
|
||||
- "15500:3306"
|
||||
expose:
|
||||
- "3306"
|
||||
environment:
|
||||
- MYSQL_ALLOW_EMPTY_PASSWORD=true
|
||||
- MYSQL_DATABASE=lbry
|
||||
- MYSQL_USER=lbry
|
||||
- MYSQL_PASSWORD=lbry
|
||||
- MYSQL_LOG_CONSOLE=true
|
||||
volumes:
|
||||
- "./init.sql:/docker-entrypoint-initdb.d/init.sql"
|
||||
- "./chainquery/docker/my.cnf:/etc/mysql/conf.d/chainquery-optimizations.cnf"
|
||||
###################
|
||||
## Internal APIs ##
|
||||
###################
|
||||
internalapis:
|
||||
image: lbry/internal-apis:master
|
||||
restart: "no"
|
||||
ports:
|
||||
- "15400:8080"
|
||||
expose:
|
||||
- "8080"
|
||||
depends_on:
|
||||
- mysql
|
||||
- lbrycrd
|
||||
environment:
|
||||
- MYSQL_DSN=lbry:lbry@tcp(mysql:3306)/lbry
|
||||
- LBRYCRD_CONNECT=rpc://lbry:lbry@lbrycrd:29245
|
||||
entrypoint: wait-for-it -t 0 chainquery:6300 -- wait-for-it -t 0 lbrycrd:29245 -- ./latest serve
|
||||
################
|
||||
## Chainquery ##
|
||||
################
|
||||
chainquery:
|
||||
image: lbry/chainquery:v2.0.8
|
||||
restart: "no"
|
||||
ports:
|
||||
- 6300:6300
|
||||
depends_on:
|
||||
- lbrycrd
|
||||
- mysql
|
||||
## TODO: Uncomment this in a docker-compose.override.yml to allow for external configurations.
|
||||
volumes:
|
||||
- ./chainqueryconfig.toml:/etc/lbry/chainqueryconfig.toml
|
||||
entrypoint: wait-for-it -t 0 lbrycrd:29245 -- wait-for-it -t 0 mysql:3306 -- start
|
||||
Executable
+94
@@ -0,0 +1,94 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
#Always compile ytsync
|
||||
make
|
||||
#Always compile supporty
|
||||
cd e2e/supporty && make && cd ../..
|
||||
|
||||
#OVERRIDE this in your .env file if running from mac. Check docker-compose.yml for details
|
||||
export LOCAL_TMP_DIR="/var/tmp:/var/tmp"
|
||||
|
||||
#Private Variables Set in local installations: SLACK_TOKEN,YOUTUBE_API_KEY,AWS_S3_ID,AWS_S3_SECRET,AWS_S3_REGION,AWS_S3_BUCKET
|
||||
touch -a .env && set -o allexport; source ./.env; set +o allexport
|
||||
echo "LOCAL_TMP_DIR=$LOCAL_TMP_DIR"
|
||||
# Compose settings - docker only
|
||||
export SLACK_CHANNEL="ytsync-travis"
|
||||
export LBRY_API_TOKEN="ytsyntoken"
|
||||
export LBRY_WEB_API="http://localhost:15400"
|
||||
export LBRYNET_ADDRESS="http://localhost:15100"
|
||||
export LBRYCRD_STRING="tcp://lbry:lbry@localhost:15200"
|
||||
export LBRYNET_USE_DOCKER=true
|
||||
export REFLECT_BLOBS=false
|
||||
export CLEAN_ON_STARTUP=true
|
||||
export REGTEST=true
|
||||
# Local settings
|
||||
export BLOBS_DIRECTORY="$(pwd)/e2e/blobsfiles"
|
||||
export LBRYNET_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbrynet/"
|
||||
export LBRYUM_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbryum"
|
||||
export TMP_DIR="/var/tmp"
|
||||
export CHAINNAME="lbrycrd_regtest"
|
||||
export UID
|
||||
|
||||
cd ./e2e
|
||||
docker-compose stop
|
||||
docker-compose rm -f
|
||||
echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
|
||||
docker-compose pull
|
||||
if [[ -d persist ]]; then rm -rf persist; fi
|
||||
mkdir -m 0777 -p ./persist
|
||||
mkdir -m 777 -p ./persist/.walletserver
|
||||
mkdir -m 777 -p ./persist/.lbrynet
|
||||
#sudo chown -Rv 999:999 ./persist/.walletserver
|
||||
#sudo chown -Rv 1000:1000 ./persist/.lbrynet
|
||||
docker-compose up -d
|
||||
printf 'waiting for internal apis'
|
||||
until curl --output /dev/null --silent --head --fail http://localhost:15400; do
|
||||
printf '.'
|
||||
sleep 1
|
||||
done
|
||||
echo "successfully started..."
|
||||
|
||||
#Data Setup for test
|
||||
./data_setup.sh
|
||||
|
||||
# Execute the sync test!
|
||||
./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 --quick #Force channel intended...just in case. This channel lines up with the api container
|
||||
status=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM youtube_data WHERE id=1')
|
||||
videoStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM synced_video WHERE id=1')
|
||||
videoClaimID1=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT publish.claim_id FROM synced_video INNER JOIN publish ON publish.id = synced_video.publish_id WHERE synced_video.id=1')
|
||||
videoClaimID2=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT publish.claim_id FROM synced_video INNER JOIN publish ON publish.id = synced_video.publish_id WHERE synced_video.id=2')
|
||||
videoClaimAddress1=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT claim_address FROM claim WHERE id=2')
|
||||
videoClaimAddress2=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT claim_address FROM claim WHERE id=3')
|
||||
# Create Supports for published claim
|
||||
./supporty/supporty @BeamerTest "${videoClaimID1}" "${videoClaimAddress1}" lbrycrd_regtest 1.0
|
||||
./supporty/supporty @BeamerTest "${videoClaimID2}" "${videoClaimAddress2}" lbrycrd_regtest 2.0
|
||||
./supporty/supporty @BeamerTest "${videoClaimID1}" "${videoClaimAddress1}" lbrycrd_regtest 3.0
|
||||
./supporty/supporty @BeamerTest "${videoClaimID2}" "${videoClaimAddress2}" lbrycrd_regtest 3.0
|
||||
curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"generate","params":[1]}' -H 'content-type:text/plain;' --user lbry:lbry http://localhost:15200
|
||||
# Reset status for transfer test
|
||||
mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e "UPDATE youtube_data SET status = 'queued' WHERE id = 1"
|
||||
# Trigger transfer api
|
||||
curl -i -H 'Accept: application/json' -H 'Content-Type: application/json' 'http://localhost:15400/yt/transfer?auth_token=youtubertoken&address=n4eYeXAYmHo4YRUDEfsEhucy8y5LKRMcHg&public_key=tpubDA9GDAntyJu4hD3wU7175p7CuV6DWbYXfyb2HedBA3yuBp9HZ4n3QE4Ex6RHCSiEuVp2nKAL1Lzf2ZLo9ApaFgNaJjG6Xo1wB3iEeVbrDZp'
|
||||
# Execute the transfer test!
|
||||
./../bin/ytsync --channelID UCNQfQvFMPnInwsU_iGYArJQ --videos-limit 2 --concurrent-jobs 4 --quick #Force channel intended...just in case. This channel lines up with the api container
|
||||
# Check that the channel and the video are marked as transferred and that all supports are spent
|
||||
channelTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT distinct transfer_state FROM youtube_data')
|
||||
videoTransferStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT distinct transferred FROM synced_video')
|
||||
nrUnspentSupports=$(mysql -u lbry -plbry -ss -D chainquery -h "127.0.0.1" -P 15500 -e 'SELECT COUNT(*) FROM chainquery.support INNER JOIN output ON output.transaction_hash = support.transaction_hash_id AND output.vout = support.vout WHERE output.is_spent = 0')
|
||||
if [[ $status != "synced" || $videoStatus != "published" || $channelTransferStatus != "2" || $videoTransferStatus != "1" || $nrUnspentSupports != "1" ]]; then
|
||||
echo "~~!!!~~~FAILED~~~!!!~~"
|
||||
echo "Channel Status: $status"
|
||||
echo "Video Status: $videoStatus"
|
||||
echo "Channel Transfer Status: $channelTransferStatus"
|
||||
echo "Video Transfer Status: $videoTransferStatus"
|
||||
echo "Nr Unspent Supports: $nrUnspentSupports"
|
||||
#docker-compose logs --tail="all" lbrycrd
|
||||
#docker-compose logs --tail="all" walletserver
|
||||
#docker-compose logs --tail="all" lbrynet
|
||||
#docker-compose logs --tail="all" internalapis
|
||||
exit 1;
|
||||
else
|
||||
echo "SUCCESSSSSSSSSSSSS!"
|
||||
fi;
|
||||
@@ -0,0 +1,3 @@
|
||||
CREATE DATABASE IF NOT EXISTS chainquery;
|
||||
GRANT ALL PRIVILEGES ON chainquery.* TO 'lbry'@'%';
|
||||
FLUSH PRIVILEGES;
|
||||
@@ -0,0 +1,26 @@
|
||||
version: "3"
|
||||
networks:
|
||||
lbry-network:
|
||||
external: true
|
||||
|
||||
services:
|
||||
#############
|
||||
## Lbrycrd ##
|
||||
#############
|
||||
lbrycrd:
|
||||
image: lbry/lbrycrd:v0.12.4.1
|
||||
restart: always
|
||||
networks:
|
||||
lbry-network:
|
||||
ipv4_address: 10.6.1.1
|
||||
ports:
|
||||
- "15201:29246"
|
||||
- "15200:29245"
|
||||
expose:
|
||||
- "29246"
|
||||
- "29245"
|
||||
## host volumes for persistent data such as wallet private keys.
|
||||
volumes:
|
||||
- "../persist/data:/data"
|
||||
environment:
|
||||
- RUN_MODE=regtest
|
||||
@@ -0,0 +1,37 @@
|
||||
FROM ubuntu:18.04 as prep
|
||||
LABEL MAINTAINER="leopere [at] nixc [dot] us"
|
||||
## TODO: Implement version pinning. `apt-get install curl=<version>`
|
||||
RUN apt-get update && \
|
||||
apt-get -y install unzip curl build-essential && \
|
||||
apt-get autoclean -y && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
WORKDIR /
|
||||
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
|
||||
COPY ./start.sh start
|
||||
COPY ./healthcheck.sh healthcheck
|
||||
COPY ./advance_blocks.sh advance
|
||||
COPY ./fix-permissions.c fix-permissions.c
|
||||
|
||||
## Add lbrycrd - Change the version below to create an image for a different tag/version
|
||||
ARG VERSION="v0.12.4.1"
|
||||
RUN URL=$(curl -s https://api.github.com/repos/lbryio/lbrycrd/releases/$(if [ "${VERSION}" = 'latest' ]; then echo "latest"; else echo "tags/${VERSION}"; fi) | grep browser_download_url | grep lbrycrd-linux.zip | cut -d'"' -f4) && echo $URL && curl -L -o /lbrycrd-linux.zip $URL
|
||||
|
||||
RUN unzip ./lbrycrd-linux.zip && \
|
||||
gcc fix-permissions.c -o fix-permissions && \
|
||||
chmod +x ./lbrycrdd ./lbrycrd-cli ./lbrycrd-tx ./start ./healthcheck ./fix-permissions ./advance
|
||||
|
||||
FROM ubuntu:18.04 as app
|
||||
COPY --from=prep /lbrycrdd /lbrycrd-cli /lbrycrd-tx /start /healthcheck /fix-permissions /advance /usr/bin/
|
||||
RUN addgroup --gid 1000 lbrycrd && \
|
||||
adduser lbrycrd --uid 1000 --gid 1000 --gecos GECOS --shell /bin/bash --disabled-password --home /data && \
|
||||
mkdir /etc/lbry && \
|
||||
chown lbrycrd /etc/lbry && \
|
||||
chmod a+s /usr/bin/fix-permissions
|
||||
VOLUME ["/data"]
|
||||
WORKDIR /data
|
||||
## TODO: Implement healthcheck.
|
||||
# HEALTHCHECK ["healthcheck"]
|
||||
EXPOSE 9246 9245 11337 29245
|
||||
|
||||
USER lbrycrd
|
||||
CMD ["start"]
|
||||
@@ -0,0 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
while true; do
|
||||
lbrycrd-cli -conf=/etc/lbry/lbrycrd.conf generate 100 >> /tmp/output.log
|
||||
sleep 2
|
||||
done
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/bin/bash
|
||||
if [ $# -eq 0 ]
|
||||
then
|
||||
echo "No docker tag argument supplied. Use './build.sh <tag>'"
|
||||
exit 1
|
||||
fi
|
||||
docker build --build-arg VERSION=$1 --tag lbry/lbrycrd:$1 .
|
||||
docker push lbry/lbrycrd:$1
|
||||
@@ -0,0 +1,9 @@
|
||||
#include <unistd.h>
|
||||
int main() {
|
||||
// This program needs to run with setuid == root
|
||||
// This needs to be in a compiled language because you cannot setuid bash scripts
|
||||
setuid(0);
|
||||
execle("/bin/bash", "bash", "-c",
|
||||
"/bin/chown -R lbrycrd:lbrycrd /data && /bin/chmod -R 755 /data/",
|
||||
(char*) NULL, (char*) NULL);
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
## TODO: Implement a healthcheck for lbrycrd.
|
||||
curl --data-binary '{"jsonrpc":"1.0","id":"curltext","method":"getinfo","params":[]}' -H 'content-type:text/plain;' http://$RPC_USER:$RPC_PASSWORD@127.0.0.1:9246
|
||||
## OR
|
||||
lbrycrd-cli getinfo
|
||||
@@ -0,0 +1,127 @@
|
||||
#!/usr/bin/env bash
|
||||
CONFIG_PATH=/etc/lbry/lbrycrd.conf
|
||||
|
||||
function override_config_option() {
|
||||
# Remove existing config line from a config file
|
||||
# and replace with environment fed value.
|
||||
# Does nothing if the variable does not exist.
|
||||
# var Name of ENV variable
|
||||
# option Name of config option
|
||||
# config Path of config file
|
||||
local var=$1 option=$2 config=$3
|
||||
if [[ -v $var ]]; then
|
||||
# Remove the existing config option:
|
||||
sed -i "/^$option\W*=/d" "$config"
|
||||
# Add the value from the environment:
|
||||
echo "$option=${!var}" >> "$config"
|
||||
fi
|
||||
}
|
||||
|
||||
function set_config() {
|
||||
if [ -d "$CONFIG_PATH" ]; then
|
||||
echo "$CONFIG_PATH is a directory when it should be a file."
|
||||
exit 1
|
||||
elif [ -f "$CONFIG_PATH" ]; then
|
||||
echo "Merging the mounted config file with environment variables."
|
||||
local MERGED_CONFIG=/tmp/lbrycrd_merged.conf
|
||||
cat $CONFIG_PATH > $MERGED_CONFIG
|
||||
echo "" >> $MERGED_CONFIG
|
||||
override_config_option PORT port $MERGED_CONFIG
|
||||
override_config_option RPC_USER rpcuser $MERGED_CONFIG
|
||||
override_config_option RPC_PASSWORD rpcpassword $MERGED_CONFIG
|
||||
override_config_option RPC_ALLOW_IP rpcallowip $MERGED_CONFIG
|
||||
override_config_option RPC_PORT rpcport $MERGED_CONFIG
|
||||
override_config_option RPC_BIND rpcbind $MERGED_CONFIG
|
||||
# Make the new merged config file the new CONFIG_PATH
|
||||
# This ensures that the original file the user mounted remains unmodified
|
||||
CONFIG_PATH=$MERGED_CONFIG
|
||||
else
|
||||
echo "Creating a fresh config file from environment variables."
|
||||
## Set config params
|
||||
{
|
||||
echo "port=${PORT=9246}"
|
||||
echo "rpcuser=${RPC_USER=lbry}"
|
||||
echo "rpcpassword=${RPC_PASSWORD=lbry}"
|
||||
echo "rpcallowip=${RPC_ALLOW_IP=127.0.0.1/24}"
|
||||
echo "rpcport=${RPC_PORT=9245}"
|
||||
echo "rpcbind=${RPC_BIND=0.0.0.0}"
|
||||
echo "deprecatedrpc=accounts"
|
||||
echo "deprecatedrpc=validateaddress"
|
||||
echo "deprecatedrpc=signrawtransaction"
|
||||
} >> $CONFIG_PATH
|
||||
fi
|
||||
echo "Config: "
|
||||
cat $CONFIG_PATH
|
||||
}
|
||||
|
||||
## Ensure perms are correct prior to running main binary
|
||||
/usr/bin/fix-permissions
|
||||
|
||||
## You can optionally specify a run mode if you want to use lbry defined presets for compatibility.
|
||||
case $RUN_MODE in
|
||||
default )
|
||||
set_config
|
||||
lbrycrdd -server -conf=$CONFIG_PATH -printtoconsole
|
||||
;;
|
||||
## If it's a first run you need to do a full index including all transactions
|
||||
## tx index creates an index of every single transaction in the block history if
|
||||
## not specified it will only create an index for transactions that are related to the wallet or have unspent outputs.
|
||||
## This is generally specific to chainquery.
|
||||
reindex )
|
||||
## Apply this RUN_MODE in the case you need to update a dataset. NOTE: you do not need to use `RUN_MODE reindex` for more than one complete run.
|
||||
set_config
|
||||
lbrycrdd -server -txindex -reindex -conf=$CONFIG_PATH -printtoconsole
|
||||
;;
|
||||
chainquery )
|
||||
## If your only goal is to run Chainquery against this instance of lbrycrd and you're starting a
|
||||
## fresh local dataset use this run mode.
|
||||
set_config
|
||||
lbrycrdd -server -txindex -conf=$CONFIG_PATH -printtoconsole
|
||||
;;
|
||||
regtest )
|
||||
## Set config params
|
||||
## TODO: Make this more automagic in the future.
|
||||
mkdir -p "$(dirname $CONFIG_PATH)"
|
||||
echo "rpcuser=lbry" > $CONFIG_PATH
|
||||
echo "rpcpassword=lbry" >> $CONFIG_PATH
|
||||
echo "rpcport=29245" >> $CONFIG_PATH
|
||||
echo "rpcbind=0.0.0.0" >> $CONFIG_PATH
|
||||
echo "rpcallowip=0.0.0.0/0" >> $CONFIG_PATH
|
||||
echo "regtest=1" >> $CONFIG_PATH
|
||||
echo "txindex=1" >> $CONFIG_PATH
|
||||
echo "server=1" >> $CONFIG_PATH
|
||||
echo "printtoconsole=1" >> $CONFIG_PATH
|
||||
echo "deprecatedrpc=accounts" >> $CONFIG_PATH
|
||||
echo "deprecatedrpc=validateaddress" >> $CONFIG_PATH
|
||||
echo "deprecatedrpc=signrawtransaction" >> $CONFIG_PATH
|
||||
echo "vbparams=segwit:0:999999999999" >> $CONFIG_PATH
|
||||
echo "addresstype=legacy" >> $CONFIG_PATH
|
||||
|
||||
#nohup advance &>/dev/null &
|
||||
lbrycrdd -conf=$CONFIG_PATH $1
|
||||
;;
|
||||
testnet )
|
||||
## Set config params
|
||||
## TODO: Make this more automagic in the future.
|
||||
mkdir -p "$(dirname $CONFIG_PATH)"
|
||||
echo "rpcuser=lbry" > $CONFIG_PATH
|
||||
echo "rpcpassword=lbry" >> $CONFIG_PATH
|
||||
echo "rpcport=29245" >> $CONFIG_PATH
|
||||
echo "rpcbind=0.0.0.0" >> $CONFIG_PATH
|
||||
echo "rpcallowip=0.0.0.0/0" >> $CONFIG_PATH
|
||||
echo "testnet=1" >> $CONFIG_PATH
|
||||
echo "txindex=1" >> $CONFIG_PATH
|
||||
echo "server=1" >> $CONFIG_PATH
|
||||
echo "printtoconsole=1" >> $CONFIG_PATH
|
||||
echo "deprecatedrpc=accounts" >> $CONFIG_PATH
|
||||
echo "deprecatedrpc=validateaddress" >> $CONFIG_PATH
|
||||
echo "deprecatedrpc=signrawtransaction" >> $CONFIG_PATH
|
||||
|
||||
#nohup advance &>/dev/null &
|
||||
lbrycrdd -conf=$CONFIG_PATH $1
|
||||
;;
|
||||
* )
|
||||
echo "Error, you must define a RUN_MODE environment variable."
|
||||
echo "Available options are testnet, regtest, chainquery, default, and reindex"
|
||||
;;
|
||||
esac
|
||||
@@ -0,0 +1,23 @@
|
||||
version: "3"
|
||||
networks:
|
||||
lbry-network:
|
||||
external: true
|
||||
|
||||
services:
|
||||
#############
|
||||
## Lbrynet ##
|
||||
#############
|
||||
lbrynet:
|
||||
image: lbry/lbrynet:v0.73.1
|
||||
restart: "no"
|
||||
networks:
|
||||
lbry-network:
|
||||
ipv4_address: 10.6.1.3
|
||||
ports:
|
||||
- "15100:5279"
|
||||
- "15101:5280"
|
||||
environment:
|
||||
- LBRY_STREAMING_SERVER=0.0.0.0:5280
|
||||
volumes:
|
||||
- "../persist/data/.lbrynet:/home/lbrynet"
|
||||
- "./settings:/etc/lbry" #Put your daemon_settings.yml here
|
||||
@@ -0,0 +1,27 @@
|
||||
## This base image is for running the latest lbrynet-daemon release.
|
||||
FROM ubuntu:18.04 as prep
|
||||
LABEL MAINTAINER="leopere [at] nixc [dot] us"
|
||||
RUN apt-get update && apt-get -y install unzip curl telnet wait-for-it
|
||||
|
||||
## Add lbrynet
|
||||
ARG VERSION="latest"
|
||||
RUN URL=$(curl -s https://api.github.com/repos/lbryio/lbry-sdk/releases/$(if [ "${VERSION}" = 'latest' ]; then echo "latest"; else echo "tags/${VERSION}"; fi) | grep browser_download_url | grep lbrynet-linux.zip | cut -d'"' -f4) && echo $URL && curl -L -o /lbrynet.linux.zip $URL
|
||||
|
||||
COPY start.sh /usr/bin/start
|
||||
COPY checkmount.sh /usr/bin/checkmount
|
||||
RUN unzip /lbrynet.linux.zip -d /lbrynet/ && \
|
||||
mv /lbrynet/lbrynet /usr/bin && \
|
||||
chmod a+x /usr/bin/checkmount /usr/bin/start /usr/bin/lbrynet
|
||||
|
||||
FROM ubuntu:18.04 as app
|
||||
COPY --from=prep /usr/bin/start /usr/bin/checkmount /usr/bin/lbrynet /usr/bin/
|
||||
RUN adduser lbrynet --gecos GECOS --shell /bin/bash --disabled-password --home /home/lbrynet
|
||||
## Daemon port [Intended for internal use]
|
||||
## LBRYNET talks to peers on port 3333 [Intended for external use] this port is used to discover other lbrynet daemons with blobs.
|
||||
## Expose 5566 Reflector port to listen on
|
||||
## Expose 5279 Port the daemon API will listen on
|
||||
## the lbryumx aka Wallet port [Intended for internal use]
|
||||
#EXPOSE 4444 3333 5566 5279 50001
|
||||
USER lbrynet
|
||||
ENTRYPOINT ["/usr/bin/checkmount"]
|
||||
CMD ["start"]
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/bin/bash
|
||||
if [ $# -eq 0 ]
|
||||
then
|
||||
echo "No docker tag argument supplied. Use './build.sh <tag>'"
|
||||
exit 1
|
||||
fi
|
||||
docker build --build-arg VERSION=$1 --tag lbry/lbrynet:$1 .
|
||||
docker push lbry/lbrynet:$1
|
||||
@@ -0,0 +1,12 @@
|
||||
#!/bin/bash
|
||||
|
||||
## TODO: Make a bit more aware of the run mode of this appliance in case there is ever a test mode enabled in the start.sh
|
||||
mountpoint=/home/lbrynet
|
||||
|
||||
if ! grep -qs ".* $mountpoint " /proc/mounts; then
|
||||
echo "$mountpoint not mounted, refusing to run."
|
||||
## TODO: We should have documentation that this error references directly with a URL as to why it won't run without a volume.
|
||||
exit 1
|
||||
else
|
||||
bash -c "$*"
|
||||
fi
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/bin/bash
|
||||
lbrynet start \
|
||||
--api="${API_BIND_IP:-0.0.0.0}":"${API_PORT:-5279}" \
|
||||
--config="${CONFIG_PATH:-/etc/lbry/daemon_settings.yml}"
|
||||
@@ -0,0 +1,13 @@
|
||||
#blockchain_name: lbrycrd_main
|
||||
#blockchain_name: lbrycrd_testnet
|
||||
blockchain_name: lbrycrd_regtest
|
||||
lbryum_servers:
|
||||
# - spv1.lbry.com:50001 #Production Wallet Server
|
||||
- walletserver:50001
|
||||
save_blobs: true
|
||||
save_files: false
|
||||
reflect_streams: false #for the love of god, don't upload regtest streams to reflector!
|
||||
share_usage_data: false
|
||||
tcp_port: 3333
|
||||
udp_port: 4444
|
||||
use_upnp: true
|
||||
@@ -0,0 +1,12 @@
|
||||
BINARY=supporty
|
||||
|
||||
DIR = $(shell cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)
|
||||
BIN_DIR = ${DIR}
|
||||
|
||||
.PHONY: build clean test lint
|
||||
.DEFAULT_GOAL: build
|
||||
|
||||
|
||||
build:
|
||||
mkdir -p ${BIN_DIR} && CGO_ENABLED=0 go build -asmflags -trimpath=${DIR} -o ${BIN_DIR}/${BINARY} supporty.go
|
||||
chmod +x ${BIN_DIR}/${BINARY}
|
||||
@@ -0,0 +1,43 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/lbryio/ytsync/v5/util"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if len(os.Args) != 6 {
|
||||
logrus.Info(strings.Join(os.Args, ","))
|
||||
logrus.Fatal("Not enough arguments: name, claimID, address, blockchainName, claimAmount")
|
||||
}
|
||||
println("Supporty!")
|
||||
lbrycrd, err := util.GetLbrycrdClient(os.Getenv("LBRYCRD_STRING"))
|
||||
if err != nil {
|
||||
logrus.Fatal(err)
|
||||
}
|
||||
if lbrycrd == nil {
|
||||
logrus.Fatal("Lbrycrd Client is nil")
|
||||
}
|
||||
amount, err := strconv.ParseFloat(os.Args[5], 64)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
name := os.Args[1]
|
||||
claimid := os.Args[2]
|
||||
claimAddress := os.Args[3]
|
||||
blockChainName := os.Args[4]
|
||||
logrus.Infof("Supporting %s[%s] with %.2f LBC on chain %s at address %s", name, claimid, amount, blockChainName, claimAddress)
|
||||
hash, err := lbrycrd.SupportClaim(name, claimid, claimAddress, blockChainName, amount)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
if hash == nil {
|
||||
logrus.Fatal("Tx not created!")
|
||||
}
|
||||
logrus.Info("Tx: ", hash.String())
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
version: "3"
|
||||
networks:
|
||||
lbry-network:
|
||||
external: true
|
||||
|
||||
services:
|
||||
###################
|
||||
## Wallet Server ##
|
||||
###################
|
||||
walletserver:
|
||||
image: lbry/wallet-server:v0.73.1
|
||||
restart: always
|
||||
networks:
|
||||
lbry-network:
|
||||
ipv4_address: 10.6.1.2
|
||||
volumes:
|
||||
- "../persist/data/.walletserver/database:/database"
|
||||
environment:
|
||||
- DB_DIRECTORY=/database
|
||||
- MAX_SEND=1000000000000000000000
|
||||
- DAEMON_URL=http://lbry:lbry@lbrycrd:29245/
|
||||
- MAX_SUBS=1000000000000
|
||||
- BANDWIDTH_LIMIT=80000000000
|
||||
- SESSION_TIMEOUT=10000000000000000000000000
|
||||
- TCP_PORT=50001
|
||||
#network_mode: host
|
||||
#network_mode: bridge
|
||||
ports:
|
||||
- "50001:50001"
|
||||
expose:
|
||||
- "50001"
|
||||
ulimits:
|
||||
nofile: 90000
|
||||
# command: lbry.wallet.server.coin.LBC
|
||||
command: lbry.wallet.server.coin.LBCRegTest
|
||||
@@ -1,44 +1,38 @@
|
||||
module github.com/lbryio/ytsync
|
||||
module github.com/lbryio/ytsync/v5
|
||||
|
||||
replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.37.4 // indirect
|
||||
github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61
|
||||
github.com/PuerkitoBio/goquery v1.5.0 // indirect
|
||||
github.com/aws/aws-sdk-go v1.17.3
|
||||
github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 // indirect
|
||||
github.com/go-ini/ini v1.42.0 // indirect
|
||||
github.com/golang/protobuf v1.3.1 // indirect
|
||||
github.com/gopherjs/gopherjs v0.0.0-20190411002643-bd77b112433e // indirect
|
||||
github.com/gorilla/websocket v1.4.0 // indirect
|
||||
github.com/hashicorp/golang-lru v0.5.1 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.0.0 // indirect
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
|
||||
github.com/kr/pretty v0.1.0 // indirect
|
||||
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c
|
||||
github.com/lbryio/lbry.go v1.0.15
|
||||
github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2 // indirect
|
||||
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936
|
||||
github.com/mitchellh/mapstructure v1.1.2 // indirect
|
||||
github.com/nikooo777/ytdl v0.0.0-20190215151411-9c7832eaf457
|
||||
github.com/onsi/ginkgo v1.8.0 // indirect
|
||||
github.com/onsi/gomega v1.5.0 // indirect
|
||||
github.com/prometheus/common v0.3.0
|
||||
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24
|
||||
github.com/sirupsen/logrus v1.4.1
|
||||
github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3 // indirect
|
||||
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect
|
||||
github.com/spf13/cobra v0.0.0-20190109003409-7547e83b2d85
|
||||
github.com/spf13/pflag v1.0.3 // indirect
|
||||
github.com/ybbus/jsonrpc v2.1.2+incompatible // indirect
|
||||
go.opencensus.io v0.20.2 // indirect
|
||||
golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480 // indirect
|
||||
golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2 // indirect
|
||||
golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a // indirect
|
||||
google.golang.org/api v0.3.2
|
||||
google.golang.org/appengine v1.5.0 // indirect
|
||||
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7 // indirect
|
||||
google.golang.org/grpc v1.20.0 // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
|
||||
gopkg.in/ini.v1 v1.42.0 // indirect
|
||||
gopkg.in/yaml.v2 v2.2.2 // indirect
|
||||
github.com/Microsoft/go-winio v0.4.14 // indirect
|
||||
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a
|
||||
github.com/aws/aws-sdk-go v1.25.9
|
||||
github.com/davecgh/go-spew v1.1.1
|
||||
github.com/docker/distribution v2.7.1+incompatible // indirect
|
||||
github.com/docker/docker v1.13.1
|
||||
github.com/docker/go-connections v0.4.0 // indirect
|
||||
github.com/docker/go-units v0.4.0 // indirect
|
||||
github.com/go-sql-driver/mysql v1.4.1 // indirect
|
||||
github.com/google/btree v1.0.0 // indirect
|
||||
github.com/hashicorp/go-immutable-radix v1.1.0 // indirect
|
||||
github.com/hashicorp/go-sockaddr v1.0.2 // indirect
|
||||
github.com/hashicorp/golang-lru v0.5.3 // indirect
|
||||
github.com/hashicorp/memberlist v0.1.5 // indirect
|
||||
github.com/hashicorp/serf v0.8.5 // indirect
|
||||
github.com/lbryio/lbry.go/v2 v2.6.1-0.20200710180140-fcade7475323
|
||||
github.com/lbryio/reflector.go v1.0.6-0.20190828131602-ce3d4403dbc6
|
||||
github.com/miekg/dns v1.1.22 // indirect
|
||||
github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b
|
||||
github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
|
||||
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 // indirect
|
||||
github.com/prometheus/client_golang v0.9.2
|
||||
github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5
|
||||
github.com/sirupsen/logrus v1.4.2
|
||||
github.com/spf13/cobra v0.0.5
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
github.com/stretchr/testify v1.4.0
|
||||
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e // indirect
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect
|
||||
google.golang.org/appengine v1.6.5 // indirect
|
||||
)
|
||||
|
||||
go 1.13
|
||||
@@ -1,32 +1,29 @@
|
||||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.37.4 h1:glPeL3BQJsbF6aIIYfZizMwc5LTYz250bDMjttbBGAU=
|
||||
cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7hw=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 h1:N5Vqww5QISEHsWHOWDEx4PzdIay3Cg0Jp7zItq2ZAro=
|
||||
github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61/go.mod h1:GnKXcK+7DYNy/8w2Ex//Uql4IgfaU82Cd5rWKb7ah00=
|
||||
github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP7EJk=
|
||||
github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg=
|
||||
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
|
||||
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
|
||||
github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
|
||||
github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU=
|
||||
github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
|
||||
github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
|
||||
github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
|
||||
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
|
||||
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
|
||||
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
|
||||
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
|
||||
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM=
|
||||
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg=
|
||||
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
|
||||
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
|
||||
github.com/aws/aws-sdk-go v1.17.3 h1:KBXxg7Jh0TxE5zmpNB2DwKmJeDUqh0O6jhy25TuYOmc=
|
||||
github.com/aws/aws-sdk-go v1.17.3/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
||||
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA=
|
||||
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
|
||||
github.com/aws/aws-sdk-go v1.16.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
||||
github.com/aws/aws-sdk-go v1.25.9 h1:WtVzerf5wSgPwlTTwl+ktCq/0GCS5MI9ZlLIcjsTr+Q=
|
||||
github.com/aws/aws-sdk-go v1.25.9/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||
github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32 h1:qkOC5Gd33k54tobS36cXdAzJbeHaduLtnLQQwNoIi78=
|
||||
github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8=
|
||||
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
|
||||
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo=
|
||||
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
|
||||
github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803 h1:j3AgPKKZtZStM2nyhrDSLSYgT7YHrZKdSkq1OYeLjvM=
|
||||
github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
|
||||
github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d h1:yJzD/yFppdVCf6ApMkVy8cUxV0XrxdP9rVf6D87/Mng=
|
||||
github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
|
||||
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd h1:R/opQEbFEy9JGkIguV40SvRY1uliPX8ifOvi6ICsFCw=
|
||||
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg=
|
||||
github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY=
|
||||
@@ -34,17 +31,26 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku
|
||||
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3SkEwmHoWBmX1DNXhXZqlTpq6s4tyJGc=
|
||||
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
|
||||
github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
|
||||
github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 h1:o64h9XF42kVEUuhuer2ehqrlX8rZmvQSU0+Vpj1rF6Q=
|
||||
github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61/go.mod h1:Rp8e0DCtEKwXFOC6JPJQVTz8tuGoGvw6Xfexggh/ed0=
|
||||
github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
|
||||
github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
|
||||
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
|
||||
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
|
||||
github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
|
||||
github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
|
||||
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
|
||||
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
|
||||
github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g=
|
||||
github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
|
||||
github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
|
||||
github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo=
|
||||
github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
|
||||
github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
|
||||
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
|
||||
github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
|
||||
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
|
||||
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
|
||||
github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
|
||||
github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
|
||||
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
|
||||
@@ -52,62 +58,97 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
|
||||
github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
|
||||
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
|
||||
github.com/go-ini/ini v1.38.2/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-ini/ini v1.42.0 h1:TWr1wGj35+UiWHlBA8er89seFXxzwFn11spilrrj+38=
|
||||
github.com/go-ini/ini v1.42.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
|
||||
github.com/go-ini/ini v1.48.0 h1:TvO60hO/2xgaaTWp2P0wUe4CFxwdMzfbkv3+343Xzqw=
|
||||
github.com/go-ini/ini v1.48.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
|
||||
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
github.com/go-ozzo/ozzo-validation v3.6.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
|
||||
github.com/go-sql-driver/mysql v0.0.0-20180719071942-99ff426eb706/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
||||
github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
|
||||
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
|
||||
github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
|
||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
|
||||
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
|
||||
github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
|
||||
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
|
||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20190411002643-bd77b112433e h1:XWcjeEtTFTOVA9Fs1w7n2XBftk5ib4oZrhzWk0B+3eA=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20190411002643-bd77b112433e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f h1:TyqzGm2z1h3AGhjOoRYyeLcW4WlW81MDQkWa+rx/000=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
|
||||
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
|
||||
github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
|
||||
github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
|
||||
github.com/gorilla/rpc v1.1.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ=
|
||||
github.com/gorilla/rpc v1.2.0 h1:WvvdC2lNeT1SP32zrIce5l0ECBfbAlmrmSBsuc57wfk=
|
||||
github.com/gorilla/rpc v1.2.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ=
|
||||
github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
|
||||
github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
|
||||
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
|
||||
github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM=
|
||||
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
|
||||
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
|
||||
github.com/hashicorp/go-immutable-radix v1.1.0 h1:vN9wG1D6KG6YHRTWr8512cxGOVgTMEfgEdSj/hr8MPc=
|
||||
github.com/hashicorp/go-immutable-radix v1.1.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
|
||||
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
|
||||
github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI=
|
||||
github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
|
||||
github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
|
||||
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
|
||||
github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
|
||||
github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
|
||||
github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc=
|
||||
github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A=
|
||||
github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
|
||||
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE=
|
||||
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
|
||||
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk=
|
||||
github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
|
||||
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
|
||||
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
|
||||
github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
|
||||
github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
|
||||
github.com/hashicorp/memberlist v0.1.4/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
|
||||
github.com/hashicorp/memberlist v0.1.5 h1:AYBsgJOW9gab/toO5tEB8lWetVgDKZycqkebJ8xxpqM=
|
||||
github.com/hashicorp/memberlist v0.1.5/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
|
||||
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
|
||||
github.com/hashicorp/serf v0.8.5 h1:ZynDUIQiA8usmRgPdGPHFdPnb1wgGI9tK3mO9hcAJjc=
|
||||
github.com/hashicorp/serf v0.8.5/go.mod h1:UpNcs7fFbpKIyZaUuSW6EPiH+eZC7OuyFD+wc1oal+k=
|
||||
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf h1:WfD7VjIE6z8dIvMsI4/s+1qr5EL+zoIGev1BQj1eoJ8=
|
||||
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h1:hyb9oH7vZsitZCiBt0ZvifOrB+qc8PS5IiilCIb87rg=
|
||||
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
|
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
|
||||
github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22 h1:jKUP9TQ0c7X3w6+IPyMit07RE42MtTWNd77sN2cHngQ=
|
||||
github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22/go.mod h1:u0Jo4f2dNlTJeeOywkM6bLwxq6gC3pZ9rEFHn3AhTdk=
|
||||
github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07 h1:+kBG/8rjCa6vxJZbUjAiE4MQmBEBYc8nLEb51frnvBY=
|
||||
github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07/go.mod h1:j1kV/8f3jowErEq4XyeypkCdvg5EeHkf0YCKCcq5Ybo=
|
||||
github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
|
||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
@@ -115,173 +156,232 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M=
|
||||
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8=
|
||||
github.com/lbryio/lbry.go v1.0.15 h1:g4g9cDDUsobmWgTA+1jEIb5k2fRCP0/NvPOMXduP8xY=
|
||||
github.com/lbryio/lbry.go v1.0.15/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002 h1:urfYK5ElpUrAv90auPLldoVC60LwiGAcY0OE6HJB9KI=
|
||||
github.com/lbryio/lbry.go v0.0.0-20190828131228-f3a1fbdd5303/go.mod h1:qR+Ui0hYhemIU4fXqM3d1P9eiaRFlof777VJgV7KJ8w=
|
||||
github.com/lbryio/lbry.go v1.1.2 h1:Dyxc+glT/rVWJwHfIf7vjlPYYbjzrQz5ARmJd5Hp69c=
|
||||
github.com/lbryio/lbry.go v1.1.2/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
|
||||
github.com/lbryio/lbry.go/v2 v2.6.1-0.20200710180140-fcade7475323 h1:Fvngg+hybcA1O2783YBw07S2MGpemSO+vWYSk97QHdM=
|
||||
github.com/lbryio/lbry.go/v2 v2.6.1-0.20200710180140-fcade7475323/go.mod h1:LgFKEpZzJE72DVgSXXfg+2IGOhrC9Lzj6eLma18iNz8=
|
||||
github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19 h1:/zWD8dVIl7bV1TdJWqPqy9tpqixzX2Qxgit48h3hQcY=
|
||||
github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46 h1:LemfR+rMxhf7nnOrzy2HqS7Me7SZ5gEwOcNFzKC8ySQ=
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
|
||||
github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4=
|
||||
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c h1:m3O7561xBQ00lfUVayW4c6SnpVbUDQtPUwGcGYSUYQA=
|
||||
github.com/lbryio/reflector.go v1.0.6-0.20190828131602-ce3d4403dbc6 h1:8k12lI18EEgMHkgGrgtRvJGl2Bq1vDSEbE4s+QaQ5Qc=
|
||||
github.com/lbryio/reflector.go v1.0.6-0.20190828131602-ce3d4403dbc6/go.mod h1:Q1Cnuv5iLsB2rS4Vr0o0EJr4Gs/EgDXFH2V4WtoM5o8=
|
||||
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
|
||||
github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5 h1:AsEBgzv3DhuYHI/GiQh2HxvTP71HCCE9E/tzGUzGdtU=
|
||||
github.com/lbryio/types v0.0.0-20191009145016-1bb8107e04f8 h1:jSNW/rK6DQsz7Zh+iv1zR384PeQdHt0gS4hKY17tkuM=
|
||||
github.com/lbryio/types v0.0.0-20191009145016-1bb8107e04f8/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
|
||||
github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0=
|
||||
github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg=
|
||||
github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2 h1:CTzpAplpuXay3gDDqbErv/OSV6SoKZ9UTsGxLKfxB8M=
|
||||
github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg=
|
||||
github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
|
||||
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 h1:mG83tLXWSRdcXMWfkoumVwhcCbf3jHF9QKv/m37BkM0=
|
||||
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCWffGOaDcjkw1iB7W9DVLp6GXmfcJY/7YZCWPA4=
|
||||
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
|
||||
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936 h1:kw1v0NlnN+GZcU8Ma8CLF2Zzgjfx95gs3/GN3vYAPpo=
|
||||
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk=
|
||||
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
|
||||
github.com/miekg/dns v1.1.22 h1:Jm64b3bO9kP43ddLjL2EY3Io6bmy1qGb9Xxz6TqS6rc=
|
||||
github.com/miekg/dns v1.1.22/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
|
||||
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b h1:9+ke9YJ9KGWw5ANXK6ozjoK47uI3uNbXv4YVINBnGm8=
|
||||
github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk=
|
||||
github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
|
||||
github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
|
||||
github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
|
||||
github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/mitchellh/mapstructure v0.0.0-20180511142126-bb74f1db0675/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
|
||||
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/nikooo777/ytdl v0.0.0-20190215151411-9c7832eaf457 h1:S6VzSeZkOv8pKevvKF1isVp9HF294LxVsYw5JrV0FdQ=
|
||||
github.com/nikooo777/ytdl v0.0.0-20190215151411-9c7832eaf457/go.mod h1:Z/EmaCtMz3DxZsLaTnbHjqjZrOH4MHQBvvxtMObRGAY=
|
||||
github.com/nlopes/slack v0.5.0 h1:NbIae8Kd0NpqaEI3iUrsuS0KbcEDhzhc939jLW5fNm0=
|
||||
github.com/nlopes/slack v0.5.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM=
|
||||
github.com/nlopes/slack v0.6.0 h1:jt0jxVQGhssx1Ib7naAOZEZcGdtIhTzkP0nopK0AsRA=
|
||||
github.com/nlopes/slack v0.6.0/go.mod h1:JzQ9m3PMAqcpeCam7UaHSuBuupz7CmpjehYMayT6YOk=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
|
||||
github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.10.2 h1:uqH7bpe+ERSiDa34FDOF7RikN6RzXgduUF8yarlZp94=
|
||||
github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
|
||||
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
|
||||
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
|
||||
github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
|
||||
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=
|
||||
github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
|
||||
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
||||
github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
|
||||
github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
||||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||
github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
|
||||
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc=
|
||||
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
|
||||
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
|
||||
github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740=
|
||||
github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
|
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8=
|
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
||||
github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
||||
github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
|
||||
github.com/prometheus/common v0.3.0 h1:taZ4h8Tkxv2kNyoSctBvfXEHmBmxrwmIidZTIaHons4=
|
||||
github.com/prometheus/common v0.3.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
|
||||
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
||||
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8=
|
||||
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
|
||||
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE=
|
||||
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
|
||||
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
|
||||
github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
|
||||
github.com/sebdah/goldie v0.0.0-20180424091453-8784dd1ab561/go.mod h1:lvjGftC8oe7XPtyrOidaMi0rp5B9+XY/ZRUynGnuaxQ=
|
||||
github.com/sebdah/goldie v0.0.0-20190531093107-d313ffb52c77 h1:Msb6XRY62jQOueNNlB5LGin1rljK7c49NLniGwJG2bg=
|
||||
github.com/sebdah/goldie v0.0.0-20190531093107-d313ffb52c77/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4=
|
||||
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/shopspring/decimal v0.0.0-20180607144847-19e3cb6c2930/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
|
||||
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 h1:pntxY8Ary0t43dCZ5dqY4YTJCObLY1kIXl0uzMv+7DE=
|
||||
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
|
||||
github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5 h1:Gojs/hac/DoYEM7WEICT45+hNWczIeuL5D21e5/HPAw=
|
||||
github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k=
|
||||
github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
|
||||
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3 h1:hBSHahWMEgzwRyS6dRpxY0XyjZsHyQ61s084wo5PJe0=
|
||||
github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w=
|
||||
github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
|
||||
github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
|
||||
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs=
|
||||
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8=
|
||||
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
|
||||
github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
|
||||
github.com/spf13/cobra v0.0.0-20190109003409-7547e83b2d85 h1:UQHWkFUuJBy5rWN1DxosG/efssLu7u0fXXSTC2HHKfQ=
|
||||
github.com/spf13/cobra v0.0.0-20190109003409-7547e83b2d85/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
|
||||
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
|
||||
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
|
||||
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
|
||||
github.com/spf13/cobra v0.0.0-20180722215644-7c4570c3ebeb/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
|
||||
github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
|
||||
github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
|
||||
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
|
||||
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
|
||||
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
|
||||
github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
|
||||
github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o=
|
||||
github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
|
||||
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
|
||||
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
|
||||
github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d h1:tQo6hjclyv3RHUgZOl6iWb2Y44A/sN9bf9LAYfuioEg=
|
||||
github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE=
|
||||
github.com/ybbus/jsonrpc v2.1.2+incompatible h1:V4mkE9qhbDQ92/MLMIhlhMSbz8jNXdagC3xBR5NDwaQ=
|
||||
github.com/ybbus/jsonrpc v2.1.2+incompatible/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE=
|
||||
go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
|
||||
go.opencensus.io v0.20.2 h1:NAfh7zF0/3/HqtMvJNZ/RFrSlCE6ZTlHmKfhL/Dm1Jk=
|
||||
go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
|
||||
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
|
||||
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480 h1:O5YqonU5IWby+w98jVUG9h7zlCWCcH4RHyPVReBmhzk=
|
||||
golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
|
||||
golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
|
||||
golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc h1:c0o/qxkaO2LF5t6fQrT4b5hzyggAkLLlCUjqfRxd8Q4=
|
||||
golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2 h1:iC0Y6EDq+rhnAePxGvJs2kzUAYcwESqdcGRPzEUfzTU=
|
||||
golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20191009170851-d66e71096ffb h1:TR699M2v0qoKTOHxeLgp6zPqaQNs74f01a/ob9W0qko=
|
||||
golang.org/x/net v0.0.0-20191009170851-d66e71096ffb/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
|
||||
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a h1:tImsplftrFpALCYumobsd0K86vlAs/eXGFms2txfJfA=
|
||||
golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190520201301-c432e742b0af h1:NXfmMfXz6JqGfG3ikSxcz2N93j6DgScr19Oo2uwFu88=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190520201301-c432e742b0af/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f h1:hjzMYz/7Ea1mNKfOnFOfktR0mlA5jqhvywClCMHM/qw=
|
||||
golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 h1:xQwXv67TxFo9nC1GJFyab5eq/5B590r6RlnL/G8Sz7w=
|
||||
golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
|
||||
google.golang.org/api v0.3.2 h1:iTp+3yyl/KOtxa/d1/JUE0GGSoR6FuW5udver22iwpw=
|
||||
google.golang.org/api v0.3.2/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
|
||||
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
|
||||
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7 h1:ZUjXAXmrAyrmmCPHgCA/vChHcpsX27MZ3yBonD/z1KE=
|
||||
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03 h1:4HYDjxeNXAOTv3o1N2tjo8UUSlhQgAD52FVkwxnWgM8=
|
||||
google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.0 h1:DlsSIrgEBuZAUFJcta2B5i/lzeHHbnfkNFAfFXLVFYQ=
|
||||
google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
|
||||
google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s=
|
||||
google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk=
|
||||
gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/ini.v1 v1.48.0 h1:URjZc+8ugRY5mL5uUeQH/a63JcHwdX9xZaWvmNWD7z8=
|
||||
gopkg.in/ini.v1 v1.48.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 h1:FpCr9V8wuOei4BAen+93HtVJ+XSi+KPbaPKm0Vj5R64=
|
||||
gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79/go.mod h1:gWkaRU7CoXpezCBWfWjm3999QqS+1pYPXGbqQCTMzo8=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
|
||||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
|
||||
gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
|
||||
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
@@ -0,0 +1,227 @@
|
||||
package ip_manager
|
||||
|
||||
import (
|
||||
"net"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/asaskevich/govalidator"
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/ytsync/v5/util"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const IPCooldownPeriod = 20 * time.Second
|
||||
const unbanTimeout = 3 * time.Hour
|
||||
|
||||
var stopper = stop.New()
|
||||
|
||||
type IPPool struct {
|
||||
ips []throttledIP
|
||||
lock *sync.RWMutex
|
||||
stopGrp *stop.Group
|
||||
}
|
||||
|
||||
type throttledIP struct {
|
||||
IP string
|
||||
UsedForVideo string
|
||||
LastUse time.Time
|
||||
Throttled bool
|
||||
InUse bool
|
||||
}
|
||||
|
||||
var ipPoolInstance *IPPool
|
||||
|
||||
func GetIPPool(stopGrp *stop.Group) (*IPPool, error) {
|
||||
if ipPoolInstance != nil {
|
||||
return ipPoolInstance, nil
|
||||
}
|
||||
addrs, err := net.InterfaceAddrs()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
var pool []throttledIP
|
||||
for _, address := range addrs {
|
||||
if ipnet, ok := address.(*net.IPNet); ok && ipnet.IP.IsGlobalUnicast() {
|
||||
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) {
|
||||
pool = append(pool, throttledIP{
|
||||
IP: ipnet.IP.String(),
|
||||
LastUse: time.Now().Add(-5 * time.Minute),
|
||||
})
|
||||
} else if ipnet.IP.To4() != nil && govalidator.IsIPv4(ipnet.IP.String()) {
|
||||
pool = append(pool, throttledIP{
|
||||
IP: ipnet.IP.String(),
|
||||
LastUse: time.Now().Add(-5 * time.Minute),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
ipPoolInstance = &IPPool{
|
||||
ips: pool,
|
||||
lock: &sync.RWMutex{},
|
||||
stopGrp: stopGrp,
|
||||
}
|
||||
//ticker := time.NewTicker(10 * time.Second)
|
||||
//go func() {
|
||||
// for {
|
||||
// select {
|
||||
// case <-stopGrp.Ch():
|
||||
// return
|
||||
// case <-ticker.C:
|
||||
// ipPoolInstance.lock.RLock()
|
||||
// for _, ip := range ipPoolInstance.ips {
|
||||
// log.Debugf("IP: %s\tInUse: %t\tVideoID: %s\tThrottled: %t\tLastUse: %.1f", ip.IP, ip.InUse, ip.UsedForVideo, ip.Throttled, time.Since(ip.LastUse).Seconds())
|
||||
// }
|
||||
// ipPoolInstance.lock.RUnlock()
|
||||
// }
|
||||
// }
|
||||
//}()
|
||||
return ipPoolInstance, nil
|
||||
}
|
||||
|
||||
// AllThrottled checks whether the IPs provided are all throttled.
|
||||
// returns false if at least one IP is not throttled
|
||||
// Not thread safe, should use locking when called
|
||||
func AllThrottled(ips []throttledIP) bool {
|
||||
for _, i := range ips {
|
||||
if !i.Throttled {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// AllInUse checks whether the IPs provided are all currently in use.
|
||||
// returns false if at least one IP is not in use AND is not throttled
|
||||
// Not thread safe, should use locking when called
|
||||
func AllInUse(ips []throttledIP) bool {
|
||||
for _, i := range ips {
|
||||
if !i.InUse && !i.Throttled {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (i *IPPool) ReleaseIP(ip string) {
|
||||
i.lock.Lock()
|
||||
defer i.lock.Unlock()
|
||||
for j := range i.ips {
|
||||
localIP := &i.ips[j]
|
||||
if localIP.IP == ip {
|
||||
localIP.InUse = false
|
||||
localIP.LastUse = time.Now()
|
||||
return
|
||||
}
|
||||
}
|
||||
util.SendErrorToSlack("something went wrong while releasing the IP %s as we reached the end of the function", ip)
|
||||
}
|
||||
|
||||
func (i *IPPool) ReleaseAll() {
|
||||
i.lock.Lock()
|
||||
defer i.lock.Unlock()
|
||||
for j := range i.ips {
|
||||
if i.ips[j].Throttled {
|
||||
continue
|
||||
}
|
||||
localIP := &i.ips[j]
|
||||
localIP.InUse = false
|
||||
}
|
||||
}
|
||||
|
||||
func (i *IPPool) SetThrottled(ip string) {
|
||||
i.lock.Lock()
|
||||
defer i.lock.Unlock()
|
||||
var tIP *throttledIP
|
||||
for j, _ := range i.ips {
|
||||
localIP := &i.ips[j]
|
||||
if localIP.IP == ip {
|
||||
if localIP.Throttled {
|
||||
return
|
||||
}
|
||||
localIP.Throttled = true
|
||||
tIP = localIP
|
||||
break
|
||||
}
|
||||
}
|
||||
util.SendErrorToSlack("%s set to throttled", ip)
|
||||
|
||||
stopper.Add(1)
|
||||
go func(tIP *throttledIP) {
|
||||
defer stopper.Done()
|
||||
unbanTimer := time.NewTimer(unbanTimeout)
|
||||
select {
|
||||
case <-unbanTimer.C:
|
||||
i.lock.Lock()
|
||||
tIP.Throttled = false
|
||||
i.lock.Unlock()
|
||||
util.SendInfoToSlack("%s set back to not throttled", ip)
|
||||
case <-i.stopGrp.Ch():
|
||||
unbanTimer.Stop()
|
||||
}
|
||||
}(tIP)
|
||||
}
|
||||
|
||||
var ErrAllInUse = errors.Base("all IPs are in use, try again")
|
||||
var ErrAllThrottled = errors.Base("all IPs are throttled")
|
||||
var ErrResourceLock = errors.Base("error getting next ip, did you forget to lock on the resource?")
|
||||
var ErrInterruptedByUser = errors.Base("interrupted by user")
|
||||
|
||||
func (i *IPPool) nextIP(forVideo string) (*throttledIP, error) {
|
||||
i.lock.Lock()
|
||||
defer i.lock.Unlock()
|
||||
|
||||
sort.Slice(i.ips, func(j, k int) bool {
|
||||
return i.ips[j].LastUse.Before(i.ips[k].LastUse)
|
||||
})
|
||||
|
||||
if !AllThrottled(i.ips) {
|
||||
if AllInUse(i.ips) {
|
||||
return nil, errors.Err(ErrAllInUse)
|
||||
}
|
||||
|
||||
var nextIP *throttledIP
|
||||
for j := range i.ips {
|
||||
ip := &i.ips[j]
|
||||
if ip.InUse || ip.Throttled {
|
||||
continue
|
||||
}
|
||||
nextIP = ip
|
||||
break
|
||||
}
|
||||
if nextIP == nil {
|
||||
return nil, errors.Err(ErrResourceLock)
|
||||
}
|
||||
nextIP.InUse = true
|
||||
nextIP.UsedForVideo = forVideo
|
||||
return nextIP, nil
|
||||
}
|
||||
return nil, errors.Err(ErrAllThrottled)
|
||||
}
|
||||
|
||||
func (i *IPPool) GetIP(forVideo string) (string, error) {
|
||||
for {
|
||||
ip, err := i.nextIP(forVideo)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrAllInUse) {
|
||||
select {
|
||||
case <-i.stopGrp.Ch():
|
||||
return "", errors.Err(ErrInterruptedByUser)
|
||||
default:
|
||||
time.Sleep(5 * time.Second)
|
||||
continue
|
||||
}
|
||||
} else if errors.Is(err, ErrAllThrottled) {
|
||||
return "throttled", err
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
if time.Since(ip.LastUse) < IPCooldownPeriod {
|
||||
log.Debugf("The IP %s is too hot, waiting for %.1f seconds before continuing", ip.IP, (IPCooldownPeriod - time.Since(ip.LastUse)).Seconds())
|
||||
time.Sleep(IPCooldownPeriod - time.Since(ip.LastUse))
|
||||
}
|
||||
return ip.IP, nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package ip_manager
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAll(t *testing.T) {
|
||||
pool, err := GetIPPool()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ip, err := pool.GetIP()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log(ip)
|
||||
pool.ReleaseIP(ip)
|
||||
ip2, err := pool.GetIP()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ip == ip2 && len(pool.ips) > 1 {
|
||||
t.Fatalf("the same IP was returned twice! %s, %s", ip, ip2)
|
||||
}
|
||||
t.Log(ip2)
|
||||
pool.ReleaseIP(ip2)
|
||||
|
||||
for range pool.ips {
|
||||
_, err = pool.GetIP()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
next, err := pool.nextIP()
|
||||
if err != nil {
|
||||
t.Logf("%s", err.Error())
|
||||
} else {
|
||||
t.Fatal(next)
|
||||
}
|
||||
}
|
||||
@@ -3,16 +3,20 @@ package main
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/user"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/extras/util"
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
"github.com/lbryio/ytsync/v5/manager"
|
||||
"github.com/lbryio/ytsync/v5/sdk"
|
||||
"github.com/lbryio/ytsync/v5/shared"
|
||||
ytUtils "github.com/lbryio/ytsync/v5/util"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
|
||||
"github.com/lbryio/ytsync/manager"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var Version string
|
||||
@@ -20,30 +24,17 @@ var Version string
|
||||
const defaultMaxTries = 3
|
||||
|
||||
var (
|
||||
stopOnError bool
|
||||
maxTries int
|
||||
takeOverExistingChannel bool
|
||||
refill int
|
||||
limit int
|
||||
skipSpaceCheck bool
|
||||
syncUpdate bool
|
||||
singleRun bool
|
||||
syncStatus string
|
||||
channelID string
|
||||
syncFrom int64
|
||||
syncUntil int64
|
||||
concurrentJobs int
|
||||
videosLimit int
|
||||
maxVideoSize int
|
||||
maxVideoLength float64
|
||||
removeDBUnpublished bool
|
||||
upgradeMetadata bool
|
||||
cliFlags shared.SyncFlags
|
||||
maxVideoLength int
|
||||
)
|
||||
|
||||
func main() {
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
log.SetLevel(log.DebugLevel)
|
||||
|
||||
http.Handle("/metrics", promhttp.Handler())
|
||||
go func() {
|
||||
log.Error(http.ListenAndServe(":2112", nil))
|
||||
}()
|
||||
cmd := &cobra.Command{
|
||||
Use: "ytsync",
|
||||
Short: "Publish youtube channels into LBRY network automatically.",
|
||||
@@ -51,23 +42,25 @@ func main() {
|
||||
Args: cobra.RangeArgs(0, 0),
|
||||
}
|
||||
|
||||
cmd.Flags().BoolVar(&stopOnError, "stop-on-error", false, "If a publish fails, stop all publishing and exit")
|
||||
cmd.Flags().IntVar(&maxTries, "max-tries", defaultMaxTries, "Number of times to try a publish that fails")
|
||||
cmd.Flags().BoolVar(&takeOverExistingChannel, "takeover-existing-channel", false, "If channel exists and we don't own it, take over the channel")
|
||||
cmd.Flags().IntVar(&limit, "limit", 0, "limit the amount of channels to sync")
|
||||
cmd.Flags().BoolVar(&skipSpaceCheck, "skip-space-check", false, "Do not perform free space check on startup")
|
||||
cmd.Flags().BoolVar(&syncUpdate, "update", false, "Update previously synced channels instead of syncing new ones")
|
||||
cmd.Flags().BoolVar(&singleRun, "run-once", false, "Whether the process should be stopped after one cycle or not")
|
||||
cmd.Flags().BoolVar(&removeDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published")
|
||||
cmd.Flags().BoolVar(&upgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version")
|
||||
cmd.Flags().StringVar(&syncStatus, "status", "", "Specify which queue to pull from. Overrides --update")
|
||||
cmd.Flags().StringVar(&channelID, "channelID", "", "If specified, only this channel will be synced.")
|
||||
cmd.Flags().Int64Var(&syncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)")
|
||||
cmd.Flags().Int64Var(&syncUntil, "before", time.Now().AddDate(1, 0, 0).Unix(), "Specify until when to pull jobs [Unix time](Default: current Unix time)")
|
||||
cmd.Flags().IntVar(&concurrentJobs, "concurrent-jobs", 1, "how many jobs to process concurrently")
|
||||
cmd.Flags().IntVar(&videosLimit, "videos-limit", 1000, "how many videos to process per channel")
|
||||
cmd.Flags().IntVar(&maxVideoSize, "max-size", 2048, "Maximum video size to process (in MB)")
|
||||
cmd.Flags().Float64Var(&maxVideoLength, "max-length", 2.0, "Maximum video length to process (in hours)")
|
||||
cmd.Flags().BoolVar(&cliFlags.StopOnError, "stop-on-error", false, "If a publish fails, stop all publishing and exit")
|
||||
cmd.Flags().IntVar(&cliFlags.MaxTries, "max-tries", defaultMaxTries, "Number of times to try a publish that fails")
|
||||
cmd.Flags().BoolVar(&cliFlags.TakeOverExistingChannel, "takeover-existing-channel", false, "If channel exists and we don't own it, take over the channel")
|
||||
cmd.Flags().IntVar(&cliFlags.Limit, "limit", 0, "limit the amount of channels to sync")
|
||||
cmd.Flags().BoolVar(&cliFlags.SkipSpaceCheck, "skip-space-check", false, "Do not perform free space check on startup")
|
||||
cmd.Flags().BoolVar(&cliFlags.SyncUpdate, "update", false, "Update previously synced channels instead of syncing new ones")
|
||||
cmd.Flags().BoolVar(&cliFlags.SingleRun, "run-once", false, "Whether the process should be stopped after one cycle or not")
|
||||
cmd.Flags().BoolVar(&cliFlags.RemoveDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published")
|
||||
cmd.Flags().BoolVar(&cliFlags.UpgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version")
|
||||
cmd.Flags().BoolVar(&cliFlags.DisableTransfers, "no-transfers", false, "Skips the transferring process of videos, channels and supports")
|
||||
cmd.Flags().BoolVar(&cliFlags.QuickSync, "quick", false, "Look up only the last 50 videos from youtube")
|
||||
cmd.Flags().StringVar(&cliFlags.SyncStatus, "status", "", "Specify which queue to pull from. Overrides --update")
|
||||
cmd.Flags().StringVar(&cliFlags.ChannelID, "channelID", "", "If specified, only this channel will be synced.")
|
||||
cmd.Flags().Int64Var(&cliFlags.SyncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)")
|
||||
cmd.Flags().Int64Var(&cliFlags.SyncUntil, "before", time.Now().AddDate(1, 0, 0).Unix(), "Specify until when to pull jobs [Unix time](Default: current Unix time)")
|
||||
cmd.Flags().IntVar(&cliFlags.ConcurrentJobs, "concurrent-jobs", 1, "how many jobs to process concurrently")
|
||||
cmd.Flags().IntVar(&cliFlags.VideosLimit, "videos-limit", 1000, "how many videos to process per channel")
|
||||
cmd.Flags().IntVar(&cliFlags.MaxVideoSize, "max-size", 2048, "Maximum video size to process (in MB)")
|
||||
cmd.Flags().IntVar(&maxVideoLength, "max-length", 2, "Maximum video length to process (in hours)")
|
||||
|
||||
if err := cmd.Execute(); err != nil {
|
||||
fmt.Println(err)
|
||||
@@ -87,33 +80,37 @@ func ytSync(cmd *cobra.Command, args []string) {
|
||||
log.Error("could not detect system hostname")
|
||||
hostname = "ytsync-unknown"
|
||||
}
|
||||
if len(hostname) > 30 {
|
||||
hostname = hostname[0:30]
|
||||
}
|
||||
|
||||
util.InitSlack(os.Getenv("SLACK_TOKEN"), os.Getenv("SLACK_CHANNEL"), hostname)
|
||||
}
|
||||
|
||||
if syncStatus != "" && !util.InSlice(syncStatus, manager.SyncStatuses) {
|
||||
log.Errorf("status must be one of the following: %v\n", manager.SyncStatuses)
|
||||
if cliFlags.SyncStatus != "" && !util.InSlice(cliFlags.SyncStatus, shared.SyncStatuses) {
|
||||
log.Errorf("status must be one of the following: %v\n", shared.SyncStatuses)
|
||||
return
|
||||
}
|
||||
|
||||
if stopOnError && maxTries != defaultMaxTries {
|
||||
if cliFlags.StopOnError && cliFlags.MaxTries != defaultMaxTries {
|
||||
log.Errorln("--stop-on-error and --max-tries are mutually exclusive")
|
||||
return
|
||||
}
|
||||
if maxTries < 1 {
|
||||
if cliFlags.MaxTries < 1 {
|
||||
log.Errorln("setting --max-tries less than 1 doesn't make sense")
|
||||
return
|
||||
}
|
||||
|
||||
if limit < 0 {
|
||||
if cliFlags.Limit < 0 {
|
||||
log.Errorln("setting --limit less than 0 (unlimited) doesn't make sense")
|
||||
return
|
||||
}
|
||||
cliFlags.MaxVideoLength = time.Duration(maxVideoLength) * time.Hour
|
||||
|
||||
apiURL := os.Getenv("LBRY_WEB_API")
|
||||
apiToken := os.Getenv("LBRY_API_TOKEN")
|
||||
youtubeAPIKey := os.Getenv("YOUTUBE_API_KEY")
|
||||
blobsDir := os.Getenv("BLOBS_DIRECTORY")
|
||||
lbrycrdString := os.Getenv("LBRYCRD_STRING")
|
||||
lbrycrdDsn := os.Getenv("LBRYCRD_STRING")
|
||||
awsS3ID := os.Getenv("AWS_S3_ID")
|
||||
awsS3Secret := os.Getenv("AWS_S3_SECRET")
|
||||
awsS3Region := os.Getenv("AWS_S3_REGION")
|
||||
@@ -146,58 +143,34 @@ func ytSync(cmd *cobra.Command, args []string) {
|
||||
log.Errorln("AWS S3 Bucket was not defined. Please set the environment variable AWS_S3_BUCKET")
|
||||
return
|
||||
}
|
||||
if lbrycrdString == "" {
|
||||
if lbrycrdDsn == "" {
|
||||
log.Infoln("Using default (local) lbrycrd instance. Set LBRYCRD_STRING if you want to use something else")
|
||||
}
|
||||
if blobsDir == "" {
|
||||
usr, err := user.Current()
|
||||
if err != nil {
|
||||
log.Errorln(err.Error())
|
||||
return
|
||||
}
|
||||
blobsDir = usr.HomeDir + "/.lbrynet/blobfiles/"
|
||||
}
|
||||
|
||||
syncProperties := &sdk.SyncProperties{
|
||||
SyncFrom: syncFrom,
|
||||
SyncUntil: syncUntil,
|
||||
YoutubeChannelID: channelID,
|
||||
}
|
||||
blobsDir := ytUtils.GetBlobsDir()
|
||||
|
||||
apiConfig := &sdk.APIConfig{
|
||||
YoutubeAPIKey: youtubeAPIKey,
|
||||
ApiURL: apiURL,
|
||||
ApiToken: apiToken,
|
||||
HostName: hostname,
|
||||
}
|
||||
awsConfig := &shared.AwsConfigs{
|
||||
AwsS3ID: awsS3ID,
|
||||
AwsS3Secret: awsS3Secret,
|
||||
AwsS3Region: awsS3Region,
|
||||
AwsS3Bucket: awsS3Bucket,
|
||||
}
|
||||
sm := manager.NewSyncManager(
|
||||
stopOnError,
|
||||
maxTries,
|
||||
takeOverExistingChannel,
|
||||
refill,
|
||||
limit,
|
||||
skipSpaceCheck,
|
||||
syncUpdate,
|
||||
concurrentJobs,
|
||||
concurrentJobs,
|
||||
cliFlags,
|
||||
blobsDir,
|
||||
videosLimit,
|
||||
maxVideoSize,
|
||||
lbrycrdString,
|
||||
awsS3ID,
|
||||
awsS3Secret,
|
||||
awsS3Region,
|
||||
awsS3Bucket,
|
||||
syncStatus,
|
||||
singleRun,
|
||||
syncProperties,
|
||||
lbrycrdDsn,
|
||||
awsConfig,
|
||||
apiConfig,
|
||||
maxVideoLength,
|
||||
removeDBUnpublished,
|
||||
upgradeMetadata,
|
||||
)
|
||||
err := sm.Start()
|
||||
if err != nil {
|
||||
manager.SendErrorToSlack(err.Error())
|
||||
ytUtils.SendErrorToSlack(errors.FullTrace(err))
|
||||
}
|
||||
manager.SendInfoToSlack("Syncing process terminated!")
|
||||
ytUtils.SendInfoToSlack("Syncing process terminated!")
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
package manager
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
|
||||
"google.golang.org/api/googleapi/transport"
|
||||
"google.golang.org/api/youtube/v3"
|
||||
)
|
||||
|
||||
func (s *Sync) CountVideos() (uint64, error) {
|
||||
client := &http.Client{
|
||||
Transport: &transport.APIKey{Key: s.APIConfig.YoutubeAPIKey},
|
||||
}
|
||||
|
||||
service, err := youtube.New(client)
|
||||
if err != nil {
|
||||
return 0, errors.Prefix("error creating YouTube service", err)
|
||||
}
|
||||
|
||||
response, err := service.Channels.List("statistics").Id(s.YoutubeChannelID).Do()
|
||||
if err != nil {
|
||||
return 0, errors.Prefix("error getting channels", err)
|
||||
}
|
||||
|
||||
if len(response.Items) < 1 {
|
||||
return 0, errors.Err("youtube channel not found")
|
||||
}
|
||||
|
||||
return response.Items[0].Statistics.VideoCount, nil
|
||||
}
|
||||
+91
-155
@@ -6,190 +6,125 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/ytsync/namer"
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
"github.com/lbryio/ytsync/v5/blobs_reflector"
|
||||
"github.com/lbryio/ytsync/v5/ip_manager"
|
||||
"github.com/lbryio/ytsync/v5/namer"
|
||||
"github.com/lbryio/ytsync/v5/sdk"
|
||||
"github.com/lbryio/ytsync/v5/shared"
|
||||
logUtils "github.com/lbryio/ytsync/v5/util"
|
||||
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
"github.com/lbryio/lbry.go/extras/util"
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type SyncManager struct {
|
||||
stopOnError bool
|
||||
maxTries int
|
||||
takeOverExistingChannel bool
|
||||
refill int
|
||||
limit int
|
||||
skipSpaceCheck bool
|
||||
syncUpdate bool
|
||||
concurrentJobs int
|
||||
concurrentVideos int
|
||||
blobsDir string
|
||||
videosLimit int
|
||||
maxVideoSize int
|
||||
maxVideoLength float64
|
||||
lbrycrdString string
|
||||
awsS3ID string
|
||||
awsS3Secret string
|
||||
awsS3Region string
|
||||
syncStatus string
|
||||
awsS3Bucket string
|
||||
singleRun bool
|
||||
syncProperties *sdk.SyncProperties
|
||||
apiConfig *sdk.APIConfig
|
||||
removeDBUnpublished bool
|
||||
upgradeMetadata bool
|
||||
CliFlags shared.SyncFlags
|
||||
ApiConfig *sdk.APIConfig
|
||||
LbrycrdDsn string
|
||||
AwsConfigs *shared.AwsConfigs
|
||||
|
||||
blobsDir string
|
||||
channelsToSync []Sync
|
||||
}
|
||||
|
||||
func NewSyncManager(stopOnError bool, maxTries int, takeOverExistingChannel bool, refill int, limit int,
|
||||
skipSpaceCheck bool, syncUpdate bool, concurrentJobs int, concurrentVideos int, blobsDir string, videosLimit int,
|
||||
maxVideoSize int, lbrycrdString string, awsS3ID string, awsS3Secret string, awsS3Region string, awsS3Bucket string,
|
||||
syncStatus string, singleRun bool, syncProperties *sdk.SyncProperties, apiConfig *sdk.APIConfig, maxVideoLength float64, removeDBUnpublished bool, upgradeMetadata bool) *SyncManager {
|
||||
func NewSyncManager(cliFlags shared.SyncFlags, blobsDir, lbrycrdDsn string, awsConfigs *shared.AwsConfigs, apiConfig *sdk.APIConfig) *SyncManager {
|
||||
return &SyncManager{
|
||||
stopOnError: stopOnError,
|
||||
maxTries: maxTries,
|
||||
takeOverExistingChannel: takeOverExistingChannel,
|
||||
refill: refill,
|
||||
limit: limit,
|
||||
skipSpaceCheck: skipSpaceCheck,
|
||||
syncUpdate: syncUpdate,
|
||||
concurrentJobs: concurrentJobs,
|
||||
concurrentVideos: concurrentVideos,
|
||||
blobsDir: blobsDir,
|
||||
videosLimit: videosLimit,
|
||||
maxVideoSize: maxVideoSize,
|
||||
maxVideoLength: maxVideoLength,
|
||||
lbrycrdString: lbrycrdString,
|
||||
awsS3ID: awsS3ID,
|
||||
awsS3Secret: awsS3Secret,
|
||||
awsS3Region: awsS3Region,
|
||||
awsS3Bucket: awsS3Bucket,
|
||||
syncStatus: syncStatus,
|
||||
singleRun: singleRun,
|
||||
syncProperties: syncProperties,
|
||||
apiConfig: apiConfig,
|
||||
removeDBUnpublished: removeDBUnpublished,
|
||||
upgradeMetadata: upgradeMetadata,
|
||||
CliFlags: cliFlags,
|
||||
blobsDir: blobsDir,
|
||||
LbrycrdDsn: lbrycrdDsn,
|
||||
AwsConfigs: awsConfigs,
|
||||
ApiConfig: apiConfig,
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
StatusPending = "pending" // waiting for permission to sync
|
||||
StatusPendingEmail = "pendingemail" // permission granted but missing email
|
||||
StatusQueued = "queued" // in sync queue. will be synced soon
|
||||
StatusPendingUpgrade = "pendingupgrade" // in sync queue. will be synced soon
|
||||
StatusSyncing = "syncing" // syncing now
|
||||
StatusSynced = "synced" // done
|
||||
StatusFailed = "failed"
|
||||
StatusFinalized = "finalized" // no more changes allowed
|
||||
StatusAbandoned = "abandoned" // deleted on youtube or banned
|
||||
)
|
||||
|
||||
var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusPendingUpgrade, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned}
|
||||
|
||||
const (
|
||||
VideoStatusPublished = "published"
|
||||
VideoStatusFailed = "failed"
|
||||
VideoStatusUpgradeFailed = "upgradefailed"
|
||||
VideoStatusUnpublished = "unpublished"
|
||||
)
|
||||
func (s *SyncManager) enqueueChannel(channel *shared.YoutubeChannel) {
|
||||
s.channelsToSync = append(s.channelsToSync, Sync{
|
||||
DbChannelData: channel,
|
||||
Manager: s,
|
||||
namer: namer.NewNamer(),
|
||||
})
|
||||
}
|
||||
|
||||
func (s *SyncManager) Start() error {
|
||||
|
||||
syncCount := 0
|
||||
for {
|
||||
err := s.checkUsedSpace()
|
||||
if logUtils.ShouldCleanOnStartup() {
|
||||
err := logUtils.CleanForStartup()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
var syncs []Sync
|
||||
var lastChannelProcessed string
|
||||
syncCount := 0
|
||||
for {
|
||||
s.channelsToSync = make([]Sync, 0, 10) // reset sync queue
|
||||
err := s.checkUsedSpace()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
shouldInterruptLoop := false
|
||||
|
||||
isSingleChannelSync := s.syncProperties.YoutubeChannelID != ""
|
||||
if isSingleChannelSync {
|
||||
channels, err := s.apiConfig.FetchChannels("", s.syncProperties)
|
||||
if s.CliFlags.IsSingleChannelSync() {
|
||||
channels, err := s.ApiConfig.FetchChannels("", &s.CliFlags)
|
||||
if err != nil {
|
||||
return err
|
||||
return errors.Err(err)
|
||||
}
|
||||
if len(channels) != 1 {
|
||||
return errors.Err("Expected 1 channel, %d returned", len(channels))
|
||||
}
|
||||
lbryChannelName := channels[0].DesiredChannelName
|
||||
syncs = make([]Sync, 1)
|
||||
syncs[0] = Sync{
|
||||
APIConfig: s.apiConfig,
|
||||
YoutubeChannelID: s.syncProperties.YoutubeChannelID,
|
||||
LbryChannelName: lbryChannelName,
|
||||
lbryChannelID: channels[0].ChannelClaimID,
|
||||
StopOnError: s.stopOnError,
|
||||
MaxTries: s.maxTries,
|
||||
ConcurrentVideos: s.concurrentVideos,
|
||||
TakeOverExistingChannel: s.takeOverExistingChannel,
|
||||
Refill: s.refill,
|
||||
Manager: s,
|
||||
LbrycrdString: s.lbrycrdString,
|
||||
AwsS3ID: s.awsS3ID,
|
||||
AwsS3Secret: s.awsS3Secret,
|
||||
AwsS3Region: s.awsS3Region,
|
||||
AwsS3Bucket: s.awsS3Bucket,
|
||||
namer: namer.NewNamer(),
|
||||
Fee: channels[0].Fee,
|
||||
}
|
||||
s.enqueueChannel(&channels[0])
|
||||
shouldInterruptLoop = true
|
||||
} else {
|
||||
var queuesToSync []string
|
||||
//TODO: implement scrambling to avoid starvation of queues
|
||||
if s.syncStatus != "" {
|
||||
queuesToSync = append(queuesToSync, s.syncStatus)
|
||||
} else if s.syncUpdate {
|
||||
queuesToSync = append(queuesToSync, StatusSyncing, StatusSynced)
|
||||
if s.CliFlags.SyncStatus != "" {
|
||||
queuesToSync = append(queuesToSync, s.CliFlags.SyncStatus)
|
||||
} else if s.CliFlags.SyncUpdate {
|
||||
queuesToSync = append(queuesToSync, shared.StatusSyncing, shared.StatusSynced)
|
||||
} else {
|
||||
queuesToSync = append(queuesToSync, StatusSyncing, StatusQueued)
|
||||
queuesToSync = append(queuesToSync, shared.StatusSyncing, shared.StatusQueued)
|
||||
}
|
||||
queues:
|
||||
for _, q := range queuesToSync {
|
||||
channels, err := s.apiConfig.FetchChannels(q, s.syncProperties)
|
||||
channels, err := s.ApiConfig.FetchChannels(q, &s.CliFlags)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Infof("There are %d channels in the \"%s\" queue", len(channels), q)
|
||||
if len(channels) > 0 {
|
||||
c := channels[0]
|
||||
syncs = append(syncs, Sync{
|
||||
APIConfig: s.apiConfig,
|
||||
YoutubeChannelID: c.ChannelId,
|
||||
LbryChannelName: c.DesiredChannelName,
|
||||
lbryChannelID: c.ChannelClaimID,
|
||||
StopOnError: s.stopOnError,
|
||||
MaxTries: s.maxTries,
|
||||
ConcurrentVideos: s.concurrentVideos,
|
||||
TakeOverExistingChannel: s.takeOverExistingChannel,
|
||||
Refill: s.refill,
|
||||
Manager: s,
|
||||
LbrycrdString: s.lbrycrdString,
|
||||
AwsS3ID: s.awsS3ID,
|
||||
AwsS3Secret: s.awsS3Secret,
|
||||
AwsS3Region: s.awsS3Region,
|
||||
AwsS3Bucket: s.awsS3Bucket,
|
||||
namer: namer.NewNamer(),
|
||||
Fee: c.Fee,
|
||||
})
|
||||
continue
|
||||
log.Infof("Currently processing the \"%s\" queue with %d channels", q, len(channels))
|
||||
for _, c := range channels {
|
||||
s.enqueueChannel(&c)
|
||||
queueAll := q == shared.StatusFailed || q == shared.StatusSyncing
|
||||
if !queueAll {
|
||||
break queues
|
||||
}
|
||||
}
|
||||
log.Infof("Drained the \"%s\" queue", q)
|
||||
}
|
||||
}
|
||||
if len(syncs) == 0 {
|
||||
if len(s.channelsToSync) == 0 {
|
||||
log.Infoln("No channels to sync. Pausing 5 minutes!")
|
||||
time.Sleep(5 * time.Minute)
|
||||
}
|
||||
for _, sync := range syncs {
|
||||
for _, sync := range s.channelsToSync {
|
||||
if lastChannelProcessed == sync.DbChannelData.ChannelId {
|
||||
util.SendToSlack("We just killed a sync for %s to stop looping! (%s)", sync.DbChannelData.DesiredChannelName, sync.DbChannelData.ChannelId)
|
||||
stopTheLoops := errors.Err("Found channel %s running twice, set it to failed, and reprocess later", sync.DbChannelData.DesiredChannelName)
|
||||
sync.setChannelTerminationStatus(&stopTheLoops)
|
||||
continue
|
||||
}
|
||||
lastChannelProcessed = sync.DbChannelData.ChannelId
|
||||
shouldNotCount := false
|
||||
SendInfoToSlack("Syncing %s (%s) to LBRY! total processed channels since startup: %d", sync.LbryChannelName, sync.YoutubeChannelID, syncCount+1)
|
||||
logUtils.SendInfoToSlack("Syncing %s (%s) to LBRY! total processed channels since startup: %d", sync.DbChannelData.DesiredChannelName, sync.DbChannelData.ChannelId, syncCount+1)
|
||||
err := sync.FullCycle()
|
||||
//TODO: THIS IS A TEMPORARY WORK AROUND FOR THE STUPID IP LOCKUP BUG
|
||||
ipPool, _ := ip_manager.GetIPPool(sync.grp)
|
||||
if ipPool != nil {
|
||||
ipPool.ReleaseAll()
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "quotaExceeded") {
|
||||
logUtils.SleepUntilQuotaReset()
|
||||
}
|
||||
fatalErrors := []string{
|
||||
"default_wallet already exists",
|
||||
"WALLET HAS NOT BEEN MOVED TO THE WALLET BACKUP DIR",
|
||||
@@ -198,42 +133,43 @@ func (s *SyncManager) Start() error {
|
||||
"failure uploading wallet",
|
||||
"the channel in the wallet is different than the channel in the database",
|
||||
"this channel does not belong to this wallet!",
|
||||
"You already have a stream claim published under the name",
|
||||
}
|
||||
|
||||
if util.SubstringInSlice(err.Error(), fatalErrors) {
|
||||
return errors.Prefix("@Nikooo777 this requires manual intervention! Exiting...", err)
|
||||
}
|
||||
shouldNotCount = strings.Contains(err.Error(), "this youtube channel is being managed by another server")
|
||||
if !shouldNotCount {
|
||||
SendInfoToSlack("A non fatal error was reported by the sync process. %s\nContinuing...", err.Error())
|
||||
logUtils.SendInfoToSlack("A non fatal error was reported by the sync process.\n%s", errors.FullTrace(err))
|
||||
}
|
||||
}
|
||||
SendInfoToSlack("Syncing %s (%s) reached an end. total processed channels since startup: %d", sync.LbryChannelName, sync.YoutubeChannelID, syncCount+1)
|
||||
err = blobs_reflector.ReflectAndClean()
|
||||
if err != nil {
|
||||
return errors.Prefix("@Nikooo777 something went wrong while reflecting blobs", err)
|
||||
}
|
||||
logUtils.SendInfoToSlack("%s (%s) reached an end. Total processed channels since startup: %d", sync.DbChannelData.DesiredChannelName, sync.DbChannelData.ChannelId, syncCount+1)
|
||||
if !shouldNotCount {
|
||||
syncCount++
|
||||
}
|
||||
if sync.IsInterrupted() || (s.limit != 0 && syncCount >= s.limit) {
|
||||
if sync.IsInterrupted() || (s.CliFlags.Limit != 0 && syncCount >= s.CliFlags.Limit) {
|
||||
shouldInterruptLoop = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if shouldInterruptLoop || s.singleRun {
|
||||
if shouldInterruptLoop || s.CliFlags.SingleRun {
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (s *SyncManager) GetS3AWSConfig() aws.Config {
|
||||
return aws.Config{
|
||||
Credentials: credentials.NewStaticCredentials(s.awsS3ID, s.awsS3Secret, ""),
|
||||
Region: &s.awsS3Region,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *SyncManager) checkUsedSpace() error {
|
||||
usedPctile, err := GetUsedSpace(s.blobsDir)
|
||||
usedPctile, err := GetUsedSpace(logUtils.GetBlobsDir())
|
||||
if err != nil {
|
||||
return err
|
||||
return errors.Err(err)
|
||||
}
|
||||
if usedPctile >= 0.90 && !s.skipSpaceCheck {
|
||||
if usedPctile >= 0.90 && !s.CliFlags.SkipSpaceCheck {
|
||||
return errors.Err(fmt.Sprintf("more than 90%% of the space has been used. use --skip-space-check to ignore. Used: %.1f%%", usedPctile*100))
|
||||
}
|
||||
log.Infof("disk usage: %.1f%%", usedPctile*100)
|
||||
|
||||
@@ -0,0 +1,250 @@
|
||||
package manager
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
|
||||
logUtils "github.com/lbryio/ytsync/v5/util"
|
||||
)
|
||||
|
||||
func (s *Sync) getS3Downloader() (*s3manager.Downloader, error) {
|
||||
s3Session, err := session.NewSession(s.Manager.AwsConfigs.GetS3AWSConfig())
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("error starting session: ", err)
|
||||
}
|
||||
downloader := s3manager.NewDownloader(s3Session)
|
||||
return downloader, nil
|
||||
}
|
||||
func (s *Sync) getS3Uploader() (*s3manager.Uploader, error) {
|
||||
s3Session, err := session.NewSession(s.Manager.AwsConfigs.GetS3AWSConfig())
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("error starting session: ", err)
|
||||
}
|
||||
uploader := s3manager.NewUploader(s3Session)
|
||||
return uploader, nil
|
||||
}
|
||||
|
||||
func (s *Sync) downloadWallet() error {
|
||||
defaultWalletDir, defaultTempWalletDir, key, err := s.getWalletPaths()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
downloader, err := s.getS3Downloader()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out, err := os.Create(defaultTempWalletDir)
|
||||
if err != nil {
|
||||
return errors.Prefix("error creating temp wallet: ", err)
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{
|
||||
Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket),
|
||||
Key: key,
|
||||
})
|
||||
if err != nil {
|
||||
// Casting to the awserr.Error type will allow you to inspect the error
|
||||
// code returned by the service in code. The error code can be used
|
||||
// to switch on context specific functionality. In this case a context
|
||||
// specific error message is printed to the user based on the bucket
|
||||
// and key existing.
|
||||
//
|
||||
// For information on other S3 API error codes see:
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
|
||||
if aerr, ok := err.(awserr.Error); ok {
|
||||
code := aerr.Code()
|
||||
if code == s3.ErrCodeNoSuchKey {
|
||||
return errors.Err("wallet not on S3")
|
||||
}
|
||||
}
|
||||
return err
|
||||
} else if bytesWritten == 0 {
|
||||
return errors.Err("zero bytes written")
|
||||
}
|
||||
|
||||
err = os.Rename(defaultTempWalletDir, defaultWalletDir)
|
||||
if err != nil {
|
||||
return errors.Prefix("error replacing temp wallet for default wallet: ", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Sync) downloadBlockchainDB() error {
|
||||
if logUtils.IsRegTest() {
|
||||
return nil // tests fail if we re-use the same blockchain DB
|
||||
}
|
||||
defaultBDBDir, defaultTempBDBDir, key, err := s.getBlockchainDBPaths()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
files, err := filepath.Glob(defaultBDBDir + "*")
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
for _, f := range files {
|
||||
err = os.Remove(f)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
}
|
||||
|
||||
downloader, err := s.getS3Downloader()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
out, err := os.Create(defaultTempBDBDir)
|
||||
if err != nil {
|
||||
return errors.Prefix("error creating temp wallet: ", err)
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{
|
||||
Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket),
|
||||
Key: key,
|
||||
})
|
||||
if err != nil {
|
||||
// Casting to the awserr.Error type will allow you to inspect the error
|
||||
// code returned by the service in code. The error code can be used
|
||||
// to switch on context specific functionality. In this case a context
|
||||
// specific error message is printed to the user based on the bucket
|
||||
// and key existing.
|
||||
//
|
||||
// For information on other S3 API error codes see:
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
|
||||
if aerr, ok := err.(awserr.Error); ok {
|
||||
code := aerr.Code()
|
||||
if code == s3.ErrCodeNoSuchKey {
|
||||
return nil // let ytsync sync the database by itself
|
||||
}
|
||||
}
|
||||
return errors.Err(err)
|
||||
} else if bytesWritten == 0 {
|
||||
return errors.Err("zero bytes written")
|
||||
}
|
||||
|
||||
err = os.Rename(defaultTempBDBDir, defaultBDBDir)
|
||||
if err != nil {
|
||||
return errors.Prefix("error replacing temp blockchain.db for default blockchain.db: ", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Sync) getWalletPaths() (defaultWallet, tempWallet string, key *string, err error) {
|
||||
defaultWallet = os.Getenv("HOME") + "/.lbryum/wallets/default_wallet"
|
||||
tempWallet = os.Getenv("HOME") + "/.lbryum/wallets/tmp_wallet"
|
||||
key = aws.String("/wallets/" + s.DbChannelData.ChannelId)
|
||||
if logUtils.IsRegTest() {
|
||||
defaultWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet"
|
||||
tempWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/tmp_wallet"
|
||||
key = aws.String("/regtest/" + s.DbChannelData.ChannelId)
|
||||
}
|
||||
|
||||
lbryumDir := os.Getenv("LBRYUM_DIR")
|
||||
if lbryumDir != "" {
|
||||
defaultWallet = lbryumDir + "/wallets/default_wallet"
|
||||
tempWallet = lbryumDir + "/wallets/tmp_wallet"
|
||||
}
|
||||
|
||||
if _, err := os.Stat(defaultWallet); !os.IsNotExist(err) {
|
||||
return "", "", nil, errors.Err("default_wallet already exists")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (s *Sync) getBlockchainDBPaths() (defaultDB, tempDB string, key *string, err error) {
|
||||
lbryumDir := os.Getenv("LBRYUM_DIR")
|
||||
if lbryumDir == "" {
|
||||
if logUtils.IsRegTest() {
|
||||
lbryumDir = os.Getenv("HOME") + "/.lbryum_regtest"
|
||||
} else {
|
||||
lbryumDir = os.Getenv("HOME") + "/.lbryum"
|
||||
}
|
||||
}
|
||||
defaultDB = lbryumDir + "/lbc_mainnet/blockchain.db"
|
||||
tempDB = lbryumDir + "/lbc_mainnet/tmp_blockchain.db"
|
||||
key = aws.String("/blockchain_dbs/" + s.DbChannelData.ChannelId)
|
||||
if logUtils.IsRegTest() {
|
||||
defaultDB = lbryumDir + "/lbc_regtest/blockchain.db"
|
||||
tempDB = lbryumDir + "/lbc_regtest/tmp_blockchain.db"
|
||||
key = aws.String("/regtest_dbs/" + s.DbChannelData.ChannelId)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (s *Sync) uploadWallet() error {
|
||||
defaultWalletDir := logUtils.GetDefaultWalletPath()
|
||||
key := aws.String("/wallets/" + s.DbChannelData.ChannelId)
|
||||
if logUtils.IsRegTest() {
|
||||
key = aws.String("/regtest/" + s.DbChannelData.ChannelId)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(defaultWalletDir); os.IsNotExist(err) {
|
||||
return errors.Err("default_wallet does not exist")
|
||||
}
|
||||
|
||||
uploader, err := s.getS3Uploader()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
file, err := os.Open(defaultWalletDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
_, err = uploader.Upload(&s3manager.UploadInput{
|
||||
Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket),
|
||||
Key: key,
|
||||
Body: file,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return os.Remove(defaultWalletDir)
|
||||
}
|
||||
|
||||
func (s *Sync) uploadBlockchainDB() error {
|
||||
defaultBDBDir, _, key, err := s.getBlockchainDBPaths()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(defaultBDBDir); os.IsNotExist(err) {
|
||||
return errors.Err("blockchain.db does not exist")
|
||||
}
|
||||
|
||||
uploader, err := s.getS3Uploader()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
file, err := os.Open(defaultBDBDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
_, err = uploader.Upload(&s3manager.UploadInput{
|
||||
Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket),
|
||||
Key: key,
|
||||
Body: file,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return os.Remove(defaultBDBDir)
|
||||
}
|
||||
+200
-157
@@ -3,34 +3,40 @@ package manager
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
"github.com/lbryio/lbry.go/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/extras/util"
|
||||
"github.com/lbryio/lbry.go/lbrycrd"
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
"github.com/lbryio/ytsync/v5/shared"
|
||||
"github.com/lbryio/ytsync/v5/timing"
|
||||
logUtils "github.com/lbryio/ytsync/v5/util"
|
||||
"github.com/lbryio/ytsync/v5/ytapi"
|
||||
|
||||
"github.com/lbryio/ytsync/tagsManager"
|
||||
"github.com/lbryio/ytsync/thumbs"
|
||||
"github.com/lbryio/ytsync/v5/tags_manager"
|
||||
"github.com/lbryio/ytsync/v5/thumbs"
|
||||
|
||||
"github.com/shopspring/decimal"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"google.golang.org/api/googleapi/transport"
|
||||
"google.golang.org/api/youtube/v3"
|
||||
)
|
||||
|
||||
func (s *Sync) enableAddressReuse() error {
|
||||
accountsResponse, err := s.daemon.AccountList()
|
||||
accountsResponse, err := s.daemon.AccountList(1, 50)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
accounts := accountsResponse.LBCMainnet
|
||||
if os.Getenv("REGTEST") == "true" {
|
||||
accounts = accountsResponse.LBCRegtest
|
||||
accounts := make([]jsonrpc.Account, 0, len(accountsResponse.Items))
|
||||
ledger := "lbc_mainnet"
|
||||
if logUtils.IsRegTest() {
|
||||
ledger = "lbc_regtest"
|
||||
}
|
||||
for _, a := range accountsResponse.Items {
|
||||
if *a.Ledger == ledger {
|
||||
accounts = append(accounts, a)
|
||||
}
|
||||
}
|
||||
|
||||
for _, a := range accounts {
|
||||
_, err = s.daemon.AccountSet(a.ID, jsonrpc.AccountSettings{
|
||||
ChangeMaxUses: util.PtrToInt(1000),
|
||||
@@ -43,6 +49,10 @@ func (s *Sync) enableAddressReuse() error {
|
||||
return nil
|
||||
}
|
||||
func (s *Sync) walletSetup() error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("walletSetup").Add(time.Since(start))
|
||||
}(start)
|
||||
//prevent unnecessary concurrent execution and publishing while refilling/reallocating UTXOs
|
||||
s.walletMux.Lock()
|
||||
defer s.walletMux.Unlock()
|
||||
@@ -57,17 +67,17 @@ func (s *Sync) walletSetup() error {
|
||||
} else if balanceResp == nil {
|
||||
return errors.Err("no response")
|
||||
}
|
||||
balance, err := strconv.ParseFloat((string)(*balanceResp), 64)
|
||||
balance, err := strconv.ParseFloat(balanceResp.Available.String(), 64)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
log.Debugf("Starting balance is %.4f", balance)
|
||||
|
||||
n, err := s.CountVideos()
|
||||
videosOnYoutube, err := ytapi.CountVideosInChannel(s.DbChannelData.ChannelId)
|
||||
if err != nil {
|
||||
return err
|
||||
logUtils.SendErrorToSlack(errors.FullTrace(errors.Prefix("failed to get video count through socialblade. Falling back to API count", err)))
|
||||
videosOnYoutube = int(s.DbChannelData.TotalVideos)
|
||||
}
|
||||
videosOnYoutube := int(n)
|
||||
|
||||
log.Debugf("Source channel has %d videos", videosOnYoutube)
|
||||
if videosOnYoutube == 0 {
|
||||
@@ -92,22 +102,27 @@ func (s *Sync) walletSetup() error {
|
||||
|
||||
log.Debugf("We already allocated credits for %d published videos and %d failed videos", publishedCount, failedCount)
|
||||
|
||||
if videosOnYoutube > s.Manager.videosLimit {
|
||||
videosOnYoutube = s.Manager.videosLimit
|
||||
if videosOnYoutube > s.Manager.CliFlags.VideosLimit {
|
||||
videosOnYoutube = s.Manager.CliFlags.VideosLimit
|
||||
}
|
||||
unallocatedVideos := videosOnYoutube - (publishedCount + failedCount)
|
||||
requiredBalance := float64(unallocatedVideos)*(publishAmount+estimatedMaxTxFee) + channelClaimAmount
|
||||
if s.Manager.upgradeMetadata {
|
||||
channelFee := channelClaimAmount
|
||||
channelAlreadyClaimed := s.DbChannelData.ChannelClaimID != ""
|
||||
if channelAlreadyClaimed {
|
||||
channelFee = 0.0
|
||||
}
|
||||
requiredBalance := float64(unallocatedVideos)*(publishAmount+estimatedMaxTxFee) + channelFee
|
||||
if s.Manager.CliFlags.UpgradeMetadata {
|
||||
requiredBalance += float64(notUpgradedCount) * 0.001
|
||||
}
|
||||
|
||||
refillAmount := 0.0
|
||||
if balance < requiredBalance || balance < minimumAccountBalance {
|
||||
refillAmount = math.Max(requiredBalance-balance, minimumRefillAmount)
|
||||
refillAmount = math.Max(math.Max(requiredBalance-balance, minimumAccountBalance-balance), minimumRefillAmount)
|
||||
}
|
||||
|
||||
if s.Refill > 0 {
|
||||
refillAmount += float64(s.Refill)
|
||||
if s.Manager.CliFlags.Refill > 0 {
|
||||
refillAmount += float64(s.Manager.CliFlags.Refill)
|
||||
}
|
||||
|
||||
if refillAmount > 0 {
|
||||
@@ -117,14 +132,16 @@ func (s *Sync) walletSetup() error {
|
||||
}
|
||||
}
|
||||
|
||||
claimAddress, err := s.daemon.AddressList(nil)
|
||||
claimAddress, err := s.daemon.AddressList(nil, nil, 1, 20)
|
||||
if err != nil {
|
||||
return err
|
||||
} else if claimAddress == nil {
|
||||
return errors.Err("could not get unused address")
|
||||
return errors.Err("could not get an address")
|
||||
}
|
||||
s.claimAddress = string((*claimAddress)[0]) //TODO: remove claimAddress completely
|
||||
if s.claimAddress == "" {
|
||||
if s.DbChannelData.PublishAddress == "" || !s.shouldTransfer() {
|
||||
s.DbChannelData.PublishAddress = string(claimAddress.Items[0].Address)
|
||||
}
|
||||
if s.DbChannelData.PublishAddress == "" {
|
||||
return errors.Err("found blank claim address")
|
||||
}
|
||||
|
||||
@@ -136,27 +153,47 @@ func (s *Sync) walletSetup() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Sync) ensureEnoughUTXOs() error {
|
||||
accounts, err := s.daemon.AccountList()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
accountsNet := (*accounts).LBCMainnet
|
||||
if os.Getenv("REGTEST") == "true" {
|
||||
accountsNet = (*accounts).LBCRegtest
|
||||
}
|
||||
defaultAccount := ""
|
||||
for _, account := range accountsNet {
|
||||
if account.IsDefault {
|
||||
defaultAccount = account.ID
|
||||
break
|
||||
func (s *Sync) getDefaultAccount() (string, error) {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("getDefaultAccount").Add(time.Since(start))
|
||||
}(start)
|
||||
if s.defaultAccountID == "" {
|
||||
accountsResponse, err := s.daemon.AccountList(1, 50)
|
||||
if err != nil {
|
||||
return "", errors.Err(err)
|
||||
}
|
||||
ledger := "lbc_mainnet"
|
||||
if logUtils.IsRegTest() {
|
||||
ledger = "lbc_regtest"
|
||||
}
|
||||
for _, a := range accountsResponse.Items {
|
||||
if *a.Ledger == ledger {
|
||||
if a.IsDefault {
|
||||
s.defaultAccountID = a.ID
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if s.defaultAccountID == "" {
|
||||
return "", errors.Err("No default account found")
|
||||
}
|
||||
}
|
||||
if defaultAccount == "" {
|
||||
return errors.Err("No default account found")
|
||||
return s.defaultAccountID, nil
|
||||
}
|
||||
|
||||
func (s *Sync) ensureEnoughUTXOs() error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("ensureEnoughUTXOs").Add(time.Since(start))
|
||||
}(start)
|
||||
defaultAccount, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
utxolist, err := s.daemon.UTXOList(&defaultAccount)
|
||||
utxolist, err := s.daemon.UTXOList(&defaultAccount, 1, 10000)
|
||||
if err != nil {
|
||||
return err
|
||||
} else if utxolist == nil {
|
||||
@@ -168,9 +205,9 @@ func (s *Sync) ensureEnoughUTXOs() error {
|
||||
count := 0
|
||||
confirmedCount := 0
|
||||
|
||||
for _, utxo := range *utxolist {
|
||||
for _, utxo := range utxolist.Items {
|
||||
amount, _ := strconv.ParseFloat(utxo.Amount, 64)
|
||||
if utxo.IsMine && utxo.Type == "payment" && amount > 0.001 {
|
||||
if utxo.IsMyOutput && utxo.Type == "payment" && amount > 0.001 {
|
||||
if utxo.Confirmations > 0 {
|
||||
confirmedCount++
|
||||
}
|
||||
@@ -187,7 +224,7 @@ func (s *Sync) ensureEnoughUTXOs() error {
|
||||
return errors.Err("no response")
|
||||
}
|
||||
|
||||
balanceAmount, err := strconv.ParseFloat((string)(*balance), 64)
|
||||
balanceAmount, err := strconv.ParseFloat(balance.Available.String(), 64)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
@@ -196,7 +233,11 @@ func (s *Sync) ensureEnoughUTXOs() error {
|
||||
if desiredUTXOCount > maxUTXOs {
|
||||
desiredUTXOCount = maxUTXOs
|
||||
}
|
||||
log.Infof("Splitting balance of %s evenly between %d UTXOs", *balance, desiredUTXOCount)
|
||||
if desiredUTXOCount < uint64(confirmedCount) {
|
||||
return nil
|
||||
}
|
||||
availableBalance, _ := balance.Available.Float64()
|
||||
log.Infof("Splitting balance of %.3f evenly between %d UTXOs", availableBalance, desiredUTXOCount)
|
||||
|
||||
broadcastFee := 0.1
|
||||
prefillTx, err := s.daemon.AccountFund(defaultAccount, defaultAccount, fmt.Sprintf("%.4f", balanceAmount-broadcastFee), desiredUTXOCount, false)
|
||||
@@ -223,6 +264,9 @@ func (s *Sync) ensureEnoughUTXOs() error {
|
||||
}
|
||||
|
||||
func (s *Sync) waitForNewBlock() error {
|
||||
defer func(start time.Time) { timing.TimedComponent("waitForNewBlock").Add(time.Since(start)) }(time.Now())
|
||||
|
||||
log.Printf("regtest: %t, docker: %t", logUtils.IsRegTest(), logUtils.IsUsingDocker())
|
||||
status, err := s.daemon.Status()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -235,11 +279,18 @@ func (s *Sync) waitForNewBlock() error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
currentBlock := status.Wallet.Blocks
|
||||
for i := 0; status.Wallet.Blocks <= currentBlock; i++ {
|
||||
if i%3 == 0 {
|
||||
log.Printf("Waiting for new block (%d)...", currentBlock+1)
|
||||
}
|
||||
if logUtils.IsRegTest() && logUtils.IsUsingDocker() {
|
||||
err = s.GenerateRegtestBlock()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
time.Sleep(10 * time.Second)
|
||||
status, err = s.daemon.Status()
|
||||
if err != nil {
|
||||
@@ -249,54 +300,67 @@ func (s *Sync) waitForNewBlock() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Sync) GenerateRegtestBlock() error {
|
||||
lbrycrd, err := logUtils.GetLbrycrdClient(s.Manager.LbrycrdDsn)
|
||||
if err != nil {
|
||||
return errors.Prefix("error getting lbrycrd client: ", err)
|
||||
}
|
||||
|
||||
txs, err := lbrycrd.Generate(1)
|
||||
if err != nil {
|
||||
return errors.Prefix("error generating new block: ", err)
|
||||
}
|
||||
|
||||
for _, tx := range txs {
|
||||
log.Info("Generated tx: ", tx.String())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Sync) ensureChannelOwnership() error {
|
||||
if s.LbryChannelName == "" {
|
||||
defer func(start time.Time) { timing.TimedComponent("ensureChannelOwnership").Add(time.Since(start)) }(time.Now())
|
||||
|
||||
if s.DbChannelData.DesiredChannelName == "" {
|
||||
return errors.Err("no channel name set")
|
||||
}
|
||||
//@TODO: get rid of this when imported channels are supported
|
||||
if s.YoutubeChannelID == "UCW-thz5HxE-goYq8yPds1Gw" {
|
||||
return nil
|
||||
}
|
||||
channels, err := s.daemon.ChannelList(nil, 1, 50)
|
||||
|
||||
channels, err := s.daemon.ChannelList(nil, 1, 50, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
} else if channels == nil {
|
||||
return errors.Err("no channel response")
|
||||
}
|
||||
//special case for wallets we don't retain full control anymore
|
||||
if len((*channels).Items) > 1 {
|
||||
// This wallet is probably not under our control anymore but we still want to publish to it
|
||||
// here we shall check if within all the channels there is one that was created by ytsync
|
||||
SendInfoToSlack("we are dealing with a wallet that has multiple channels. This indicates that the wallet was probably transferred but we still want to sync their content. YoutubeID: %s", s.YoutubeChannelID)
|
||||
if s.lbryChannelID == "" {
|
||||
|
||||
var channelToUse *jsonrpc.Transaction
|
||||
if len((*channels).Items) > 0 {
|
||||
if s.DbChannelData.ChannelClaimID == "" {
|
||||
return errors.Err("this channel does not have a recorded claimID in the database. To prevent failures, updates are not supported until an entry is manually added in the database")
|
||||
}
|
||||
for _, c := range (*channels).Items {
|
||||
if c.ClaimID != s.lbryChannelID {
|
||||
if c.Name != s.LbryChannelName {
|
||||
return errors.Err("the channel in the wallet is different than the channel in the database")
|
||||
}
|
||||
return nil // we have the ytsync channel and both the claimID and the channelName from the database are correct
|
||||
log.Debugf("checking listed channel %s (%s)", c.ClaimID, c.Name)
|
||||
if c.ClaimID != s.DbChannelData.ChannelClaimID {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
channelUsesOldMetadata := false
|
||||
if len((*channels).Items) == 1 {
|
||||
channel := ((*channels).Items)[0]
|
||||
if channel.Name == s.LbryChannelName {
|
||||
channelUsesOldMetadata = channel.Value.GetThumbnail() == nil
|
||||
//TODO: eventually get rid of this when the whole db is filled
|
||||
if s.lbryChannelID == "" {
|
||||
err = s.Manager.apiConfig.SetChannelClaimID(s.YoutubeChannelID, channel.ClaimID)
|
||||
} else if channel.ClaimID != s.lbryChannelID {
|
||||
if c.Name != s.DbChannelData.DesiredChannelName {
|
||||
return errors.Err("the channel in the wallet is different than the channel in the database")
|
||||
}
|
||||
s.lbryChannelID = channel.ClaimID
|
||||
if !channelUsesOldMetadata {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
return errors.Err("this channel does not belong to this wallet! Expected: %s, found: %s", s.LbryChannelName, channel.Name)
|
||||
channelToUse = &c
|
||||
break
|
||||
}
|
||||
if channelToUse == nil {
|
||||
return errors.Err("this wallet has channels but not a single one is ours! Expected claim_id: %s (%s)", s.DbChannelData.ChannelClaimID, s.DbChannelData.DesiredChannelName)
|
||||
}
|
||||
} else if s.DbChannelData.TransferState == shared.TransferStateComplete {
|
||||
return errors.Err("the channel was transferred but appears to have been abandoned!")
|
||||
} else if s.DbChannelData.ChannelClaimID != "" {
|
||||
return errors.Err("the database has a channel recorded (%s) but nothing was found in our control", s.DbChannelData.ChannelClaimID)
|
||||
}
|
||||
|
||||
channelUsesOldMetadata := false
|
||||
if channelToUse != nil {
|
||||
channelUsesOldMetadata = channelToUse.Value.GetThumbnail() == nil
|
||||
if !channelUsesOldMetadata {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -308,47 +372,35 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
} else if balanceResp == nil {
|
||||
return errors.Err("no response")
|
||||
}
|
||||
balance, err := decimal.NewFromString((string)(*balanceResp))
|
||||
balance, err := decimal.NewFromString(balanceResp.Available.String())
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
if balance.LessThan(decimal.NewFromFloat(channelBidAmount)) {
|
||||
err = s.addCredits(channelBidAmount + 0.1)
|
||||
err = s.addCredits(channelBidAmount + 0.3)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
client := &http.Client{
|
||||
Transport: &transport.APIKey{Key: s.APIConfig.YoutubeAPIKey},
|
||||
}
|
||||
|
||||
service, err := youtube.New(client)
|
||||
channelInfo, err := ytapi.ChannelInfo(s.DbChannelData.ChannelId)
|
||||
if err != nil {
|
||||
return errors.Prefix("error creating YouTube service", err)
|
||||
return err
|
||||
}
|
||||
|
||||
response, err := service.Channels.List("snippet,brandingSettings").Id(s.YoutubeChannelID).Do()
|
||||
if err != nil {
|
||||
return errors.Prefix("error getting channel details", err)
|
||||
}
|
||||
|
||||
if len(response.Items) < 1 {
|
||||
return errors.Err("youtube channel not found")
|
||||
}
|
||||
|
||||
channelInfo := response.Items[0].Snippet
|
||||
channelBranding := response.Items[0].BrandingSettings
|
||||
|
||||
thumbnail := thumbs.GetBestThumbnail(channelInfo.Thumbnails)
|
||||
thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail.Url, s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
|
||||
thumbnail := channelInfo.Header.C4TabbedHeaderRenderer.Avatar.Thumbnails[len(channelInfo.Header.C4TabbedHeaderRenderer.Avatar.Thumbnails)-1].URL
|
||||
thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail, s.DbChannelData.ChannelId, *s.Manager.AwsConfigs.GetS3AWSConfig())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var bannerURL *string
|
||||
if channelBranding.Image != nil && channelBranding.Image.BannerImageUrl != "" {
|
||||
bURL, err := thumbs.MirrorThumbnail(channelBranding.Image.BannerImageUrl, "banner-"+s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
|
||||
if channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails != nil {
|
||||
bURL, err := thumbs.MirrorThumbnail(channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails[len(channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails)-1].URL,
|
||||
"banner-"+s.DbChannelData.ChannelId,
|
||||
*s.Manager.AwsConfigs.GetS3AWSConfig(),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -356,34 +408,43 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
}
|
||||
|
||||
var languages []string = nil
|
||||
if channelInfo.DefaultLanguage != "" {
|
||||
languages = []string{channelInfo.DefaultLanguage}
|
||||
}
|
||||
//we don't have this data without the API
|
||||
//if channelInfo.DefaultLanguage != "" {
|
||||
// if channelInfo.DefaultLanguage == "iw" {
|
||||
// channelInfo.DefaultLanguage = "he"
|
||||
// }
|
||||
// languages = []string{channelInfo.DefaultLanguage}
|
||||
//}
|
||||
var locations []jsonrpc.Location = nil
|
||||
if channelInfo.Country != "" {
|
||||
locations = []jsonrpc.Location{{Country: util.PtrToString(channelInfo.Country)}}
|
||||
if channelInfo.Topbar.DesktopTopbarRenderer.CountryCode != "" {
|
||||
locations = []jsonrpc.Location{{Country: &channelInfo.Topbar.DesktopTopbarRenderer.CountryCode}}
|
||||
}
|
||||
var c *jsonrpc.TransactionSummary
|
||||
claimCreateOptions := jsonrpc.ClaimCreateOptions{
|
||||
Title: &channelInfo.Title,
|
||||
Description: &channelInfo.Description,
|
||||
Tags: tagsManager.GetTagsForChannel(s.YoutubeChannelID),
|
||||
Title: &channelInfo.Microformat.MicroformatDataRenderer.Title,
|
||||
Description: &channelInfo.Microformat.MicroformatDataRenderer.Description,
|
||||
Tags: tags_manager.GetTagsForChannel(s.DbChannelData.ChannelId),
|
||||
Languages: languages,
|
||||
Locations: locations,
|
||||
ThumbnailURL: &thumbnailURL,
|
||||
}
|
||||
if channelUsesOldMetadata {
|
||||
c, err = s.daemon.ChannelUpdate(s.lbryChannelID, jsonrpc.ChannelUpdateOptions{
|
||||
ClearTags: util.PtrToBool(true),
|
||||
ClearLocations: util.PtrToBool(true),
|
||||
ClearLanguages: util.PtrToBool(true),
|
||||
ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
|
||||
ClaimCreateOptions: claimCreateOptions,
|
||||
CoverURL: bannerURL,
|
||||
},
|
||||
})
|
||||
if s.DbChannelData.TransferState <= 1 {
|
||||
c, err = s.daemon.ChannelUpdate(s.DbChannelData.ChannelClaimID, jsonrpc.ChannelUpdateOptions{
|
||||
ClearTags: util.PtrToBool(true),
|
||||
ClearLocations: util.PtrToBool(true),
|
||||
ClearLanguages: util.PtrToBool(true),
|
||||
ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
|
||||
ClaimCreateOptions: claimCreateOptions,
|
||||
CoverURL: bannerURL,
|
||||
},
|
||||
})
|
||||
} else {
|
||||
logUtils.SendInfoToSlack("%s (%s) has a channel with old metadata but isn't in our control anymore. Ignoring", s.DbChannelData.DesiredChannelName, s.DbChannelData.ChannelClaimID)
|
||||
return nil
|
||||
}
|
||||
} else {
|
||||
c, err = s.daemon.ChannelCreate(s.LbryChannelName, channelBidAmount, jsonrpc.ChannelCreateOptions{
|
||||
c, err = s.daemon.ChannelCreate(s.DbChannelData.DesiredChannelName, channelBidAmount, jsonrpc.ChannelCreateOptions{
|
||||
ClaimCreateOptions: claimCreateOptions,
|
||||
CoverURL: bannerURL,
|
||||
})
|
||||
@@ -392,45 +453,27 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.lbryChannelID = c.Outputs[0].ClaimID
|
||||
return s.Manager.apiConfig.SetChannelClaimID(s.YoutubeChannelID, s.lbryChannelID)
|
||||
}
|
||||
|
||||
func allUTXOsConfirmed(utxolist *jsonrpc.UTXOListResponse) bool {
|
||||
if utxolist == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
if len(*utxolist) < 1 {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, utxo := range *utxolist {
|
||||
if utxo.Confirmations <= 0 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
s.DbChannelData.ChannelClaimID = c.Outputs[0].ClaimID
|
||||
return s.Manager.ApiConfig.SetChannelClaimID(s.DbChannelData.ChannelId, s.DbChannelData.ChannelClaimID)
|
||||
}
|
||||
|
||||
func (s *Sync) addCredits(amountToAdd float64) error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("addCredits").Add(time.Since(start))
|
||||
}(start)
|
||||
log.Printf("Adding %f credits", amountToAdd)
|
||||
var lbrycrdd *lbrycrd.Client
|
||||
var err error
|
||||
if s.LbrycrdString == "" {
|
||||
lbrycrdd, err = lbrycrd.NewWithDefaultURL()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
lbrycrdd, err = lbrycrd.New(s.LbrycrdString)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
lbrycrdd, err := logUtils.GetLbrycrdClient(s.Manager.LbrycrdDsn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
addressResp, err := s.daemon.AddressUnused(nil)
|
||||
defaultAccount, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
addressResp, err := s.daemon.AddressUnused(&defaultAccount)
|
||||
if err != nil {
|
||||
return err
|
||||
} else if addressResp == nil {
|
||||
|
||||
@@ -0,0 +1,349 @@
|
||||
package manager
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
"github.com/lbryio/ytsync/v5/shared"
|
||||
"github.com/lbryio/ytsync/v5/timing"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func waitConfirmations(s *Sync) error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("waitConfirmations").Add(time.Since(start))
|
||||
}(start)
|
||||
defaultAccount, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
allConfirmed := false
|
||||
waitCount := 0
|
||||
waiting:
|
||||
for !allConfirmed && waitCount < 2 {
|
||||
utxolist, err := s.daemon.UTXOList(&defaultAccount, 1, 10000)
|
||||
if err != nil {
|
||||
return err
|
||||
} else if utxolist == nil {
|
||||
return errors.Err("no response")
|
||||
}
|
||||
|
||||
for _, utxo := range utxolist.Items {
|
||||
if utxo.Confirmations <= 0 {
|
||||
err = s.waitForNewBlock()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
waitCount++
|
||||
continue waiting
|
||||
}
|
||||
}
|
||||
allConfirmed = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type abandonResponse struct {
|
||||
ClaimID string
|
||||
Error error
|
||||
Amount float64
|
||||
}
|
||||
|
||||
func abandonSupports(s *Sync) (float64, error) {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("abandonSupports").Add(time.Since(start))
|
||||
}(start)
|
||||
totalPages := uint64(1)
|
||||
var allSupports []jsonrpc.Claim
|
||||
defaultAccount, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
for page := uint64(1); page <= totalPages; page++ {
|
||||
supports, err := s.daemon.SupportList(&defaultAccount, page, 50)
|
||||
if err != nil {
|
||||
supports, err = s.daemon.SupportList(&defaultAccount, page, 50)
|
||||
if err != nil {
|
||||
return 0, errors.Prefix("cannot list supports", err)
|
||||
}
|
||||
}
|
||||
allSupports = append(allSupports, (*supports).Items...)
|
||||
totalPages = (*supports).TotalPages
|
||||
}
|
||||
producerWG := &stop.Group{}
|
||||
|
||||
claimIDChan := make(chan string, len(allSupports))
|
||||
abandonRspChan := make(chan abandonResponse, len(allSupports))
|
||||
alreadyAbandoned := make(map[string]bool, len(allSupports))
|
||||
producerWG.Add(1)
|
||||
go func() {
|
||||
defer producerWG.Done()
|
||||
for _, support := range allSupports {
|
||||
_, ok := alreadyAbandoned[support.ClaimID]
|
||||
if ok {
|
||||
continue
|
||||
}
|
||||
alreadyAbandoned[support.ClaimID] = true
|
||||
claimIDChan <- support.ClaimID
|
||||
}
|
||||
}()
|
||||
consumerWG := &stop.Group{}
|
||||
//TODO: remove this once the SDK team fixes their RPC bugs....
|
||||
s.daemon.SetRPCTimeout(60 * time.Second)
|
||||
defer s.daemon.SetRPCTimeout(5 * time.Minute)
|
||||
for i := 0; i < s.Manager.CliFlags.ConcurrentJobs; i++ {
|
||||
consumerWG.Add(1)
|
||||
go func() {
|
||||
defer consumerWG.Done()
|
||||
outer:
|
||||
for {
|
||||
claimID, more := <-claimIDChan
|
||||
if !more {
|
||||
return
|
||||
} else {
|
||||
summary, err := s.daemon.TxoSpend(util.PtrToString("support"), &claimID, nil, nil, nil, &defaultAccount)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "Client.Timeout exceeded while awaiting headers") {
|
||||
log.Errorf("Support abandon for %s timed out, retrying...", claimID)
|
||||
summary, err = s.daemon.TxoSpend(util.PtrToString("support"), &claimID, nil, nil, nil, &defaultAccount)
|
||||
if err != nil {
|
||||
//TODO GUESS HOW MUCH LBC WAS RELEASED THAT WE DON'T KNOW ABOUT, because screw you SDK
|
||||
abandonRspChan <- abandonResponse{
|
||||
ClaimID: claimID,
|
||||
Error: err,
|
||||
Amount: 0, // this is likely wrong, but oh well... there is literally nothing I can do about it
|
||||
}
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
abandonRspChan <- abandonResponse{
|
||||
ClaimID: claimID,
|
||||
Error: err,
|
||||
Amount: 0,
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
if summary == nil || len(*summary) < 1 {
|
||||
abandonRspChan <- abandonResponse{
|
||||
ClaimID: claimID,
|
||||
Error: errors.Err("error abandoning supports: no outputs while abandoning %s", claimID),
|
||||
Amount: 0,
|
||||
}
|
||||
continue
|
||||
}
|
||||
var outputAmount float64
|
||||
for _, tx := range *summary {
|
||||
amount, err := strconv.ParseFloat(tx.Outputs[0].Amount, 64)
|
||||
if err != nil {
|
||||
abandonRspChan <- abandonResponse{
|
||||
ClaimID: claimID,
|
||||
Error: errors.Err(err),
|
||||
Amount: 0,
|
||||
}
|
||||
continue outer
|
||||
}
|
||||
outputAmount += amount
|
||||
}
|
||||
if err != nil {
|
||||
abandonRspChan <- abandonResponse{
|
||||
ClaimID: claimID,
|
||||
Error: errors.Err(err),
|
||||
Amount: 0,
|
||||
}
|
||||
continue
|
||||
}
|
||||
log.Infof("Abandoned supports of %.4f LBC for claim %s", outputAmount, claimID)
|
||||
abandonRspChan <- abandonResponse{
|
||||
ClaimID: claimID,
|
||||
Error: nil,
|
||||
Amount: outputAmount,
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
producerWG.Wait()
|
||||
close(claimIDChan)
|
||||
consumerWG.Wait()
|
||||
close(abandonRspChan)
|
||||
|
||||
totalAbandoned := 0.0
|
||||
for r := range abandonRspChan {
|
||||
if r.Error != nil {
|
||||
log.Errorf("Failed abandoning supports for %s: %s", r.ClaimID, r.Error.Error())
|
||||
continue
|
||||
}
|
||||
totalAbandoned += r.Amount
|
||||
}
|
||||
return totalAbandoned, nil
|
||||
}
|
||||
|
||||
type updateInfo struct {
|
||||
ClaimID string
|
||||
streamUpdateOptions *jsonrpc.StreamUpdateOptions
|
||||
videoStatus *shared.VideoStatus
|
||||
}
|
||||
|
||||
func transferVideos(s *Sync) error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("transferVideos").Add(time.Since(start))
|
||||
}(start)
|
||||
cleanTransfer := true
|
||||
|
||||
streamChan := make(chan updateInfo, s.Manager.CliFlags.ConcurrentJobs)
|
||||
account, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
streams, err := s.daemon.StreamList(&account, 1, 30000)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
producerWG := &stop.Group{}
|
||||
producerWG.Add(1)
|
||||
go func() {
|
||||
defer producerWG.Done()
|
||||
for _, video := range s.syncedVideos {
|
||||
if !video.Published || video.Transferred || video.MetadataVersion != shared.LatestMetadataVersion {
|
||||
continue
|
||||
}
|
||||
|
||||
var stream *jsonrpc.Claim = nil
|
||||
for _, c := range streams.Items {
|
||||
if c.ClaimID != video.ClaimID || (c.SigningChannel != nil && c.SigningChannel.ClaimID != s.DbChannelData.ChannelClaimID) {
|
||||
continue
|
||||
}
|
||||
stream = &c
|
||||
break
|
||||
}
|
||||
if stream == nil {
|
||||
return
|
||||
}
|
||||
|
||||
streamUpdateOptions := jsonrpc.StreamUpdateOptions{
|
||||
StreamCreateOptions: &jsonrpc.StreamCreateOptions{
|
||||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
|
||||
ClaimAddress: &s.DbChannelData.PublishAddress,
|
||||
FundingAccountIDs: []string{
|
||||
account,
|
||||
},
|
||||
},
|
||||
},
|
||||
Bid: util.PtrToString("0.005"), // Todo - Dont hardcode
|
||||
}
|
||||
videoStatus := shared.VideoStatus{
|
||||
ChannelID: s.DbChannelData.ChannelId,
|
||||
VideoID: video.VideoID,
|
||||
ClaimID: video.ClaimID,
|
||||
ClaimName: video.ClaimName,
|
||||
Status: shared.VideoStatusPublished,
|
||||
IsTransferred: util.PtrToBool(true),
|
||||
}
|
||||
streamChan <- updateInfo{
|
||||
ClaimID: video.ClaimID,
|
||||
streamUpdateOptions: &streamUpdateOptions,
|
||||
videoStatus: &videoStatus,
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
consumerWG := &stop.Group{}
|
||||
for i := 0; i < s.Manager.CliFlags.ConcurrentJobs; i++ {
|
||||
consumerWG.Add(1)
|
||||
go func(worker int) {
|
||||
defer consumerWG.Done()
|
||||
for {
|
||||
ui, more := <-streamChan
|
||||
if !more {
|
||||
return
|
||||
} else {
|
||||
err := s.streamUpdate(&ui)
|
||||
if err != nil {
|
||||
cleanTransfer = false
|
||||
}
|
||||
}
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
producerWG.Wait()
|
||||
close(streamChan)
|
||||
consumerWG.Wait()
|
||||
|
||||
if !cleanTransfer {
|
||||
return errors.Err("A video has failed to transfer for the channel...skipping channel transfer")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Sync) streamUpdate(ui *updateInfo) error {
|
||||
start := time.Now()
|
||||
result, updateError := s.daemon.StreamUpdate(ui.ClaimID, *ui.streamUpdateOptions)
|
||||
timing.TimedComponent("transferStreamUpdate").Add(time.Since(start))
|
||||
if updateError != nil {
|
||||
ui.videoStatus.FailureReason = updateError.Error()
|
||||
ui.videoStatus.Status = shared.VideoStatusTranferFailed
|
||||
ui.videoStatus.IsTransferred = util.PtrToBool(false)
|
||||
} else {
|
||||
ui.videoStatus.IsTransferred = util.PtrToBool(len(result.Outputs) != 0)
|
||||
}
|
||||
log.Infof("TRANSFERRED %t", *ui.videoStatus.IsTransferred)
|
||||
statusErr := s.Manager.ApiConfig.MarkVideoStatus(*ui.videoStatus)
|
||||
if statusErr != nil {
|
||||
return errors.Prefix(statusErr.Error(), updateError)
|
||||
}
|
||||
return errors.Err(updateError)
|
||||
}
|
||||
|
||||
func transferChannel(s *Sync) error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("transferChannel").Add(time.Since(start))
|
||||
}(start)
|
||||
account, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
channelClaims, err := s.daemon.ChannelList(&account, 1, 50, nil)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
var channelClaim *jsonrpc.Transaction = nil
|
||||
for _, c := range channelClaims.Items {
|
||||
if c.ClaimID != s.DbChannelData.ChannelClaimID {
|
||||
continue
|
||||
}
|
||||
channelClaim = &c
|
||||
break
|
||||
}
|
||||
if channelClaim == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
updateOptions := jsonrpc.ChannelUpdateOptions{
|
||||
Bid: util.PtrToString(fmt.Sprintf("%.6f", channelClaimAmount-0.005)),
|
||||
ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
|
||||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
|
||||
ClaimAddress: &s.DbChannelData.PublishAddress,
|
||||
},
|
||||
},
|
||||
}
|
||||
result, err := s.daemon.ChannelUpdate(s.DbChannelData.ChannelClaimID, updateOptions)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
log.Infof("TRANSFERRED %t", len(result.Outputs) != 0)
|
||||
|
||||
return nil
|
||||
}
|
||||
+516
-444
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,31 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"os"
|
||||
"regexp"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
var (
|
||||
Durations = promauto.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: "ytsync",
|
||||
Subsystem: getHostname(),
|
||||
Name: "duration",
|
||||
Help: "The durations of the individual modules",
|
||||
}, []string{"path"})
|
||||
)
|
||||
|
||||
func getHostname() string {
|
||||
hostname, err := os.Hostname()
|
||||
if err != nil {
|
||||
hostname = "ytsync_unknown"
|
||||
}
|
||||
reg, err := regexp.Compile("[^a-zA-Z0-9_]+")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
return reg.ReplaceAllString(hostname, "_")
|
||||
}
|
||||
+8
-1
@@ -43,9 +43,16 @@ func (n *Namer) GetNextName(prefix string) string {
|
||||
}
|
||||
|
||||
//if for some reasons the title can't be converted in a valid claim name (too short or not latin) then we use a hash
|
||||
attempt = 1
|
||||
if len(name) < 2 {
|
||||
sum := md5.Sum([]byte(prefix))
|
||||
name = fmt.Sprintf("%s-%d", hex.EncodeToString(sum[:])[:15], attempt)
|
||||
for {
|
||||
name = fmt.Sprintf("%s-%d", hex.EncodeToString(sum[:])[:15], attempt)
|
||||
if _, exists := n.names[name]; !exists {
|
||||
break
|
||||
}
|
||||
attempt++
|
||||
}
|
||||
}
|
||||
|
||||
n.names[name] = true
|
||||
|
||||
+263
-58
@@ -11,8 +11,11 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
"github.com/lbryio/lbry.go/extras/null"
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/null"
|
||||
"github.com/lbryio/ytsync/v5/shared"
|
||||
|
||||
"github.com/lbryio/ytsync/v5/util"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
@@ -28,47 +31,42 @@ type APIConfig struct {
|
||||
HostName string
|
||||
}
|
||||
|
||||
type SyncProperties struct {
|
||||
SyncFrom int64
|
||||
SyncUntil int64
|
||||
YoutubeChannelID string
|
||||
}
|
||||
|
||||
type Fee struct {
|
||||
Amount string `json:"amount"`
|
||||
Address string `json:"address"`
|
||||
Currency string `json:"currency"`
|
||||
}
|
||||
type YoutubeChannel struct {
|
||||
ChannelId string `json:"channel_id"`
|
||||
TotalVideos uint `json:"total_videos"`
|
||||
DesiredChannelName string `json:"desired_channel_name"`
|
||||
Fee *Fee `json:"fee"`
|
||||
ChannelClaimID string `json:"channel_claim_id"`
|
||||
}
|
||||
|
||||
func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeChannel, error) {
|
||||
func (a *APIConfig) FetchChannels(status string, cliFlags *shared.SyncFlags) ([]shared.YoutubeChannel, error) {
|
||||
type apiJobsResponse struct {
|
||||
Success bool `json:"success"`
|
||||
Error null.String `json:"error"`
|
||||
Data []YoutubeChannel `json:"data"`
|
||||
Success bool `json:"success"`
|
||||
Error null.String `json:"error"`
|
||||
Data []shared.YoutubeChannel `json:"data"`
|
||||
}
|
||||
endpoint := a.ApiURL + "/yt/jobs"
|
||||
res, _ := http.PostForm(endpoint, url.Values{
|
||||
res, err := http.PostForm(endpoint, url.Values{
|
||||
"auth_token": {a.ApiToken},
|
||||
"sync_status": {status},
|
||||
"min_videos": {strconv.Itoa(1)},
|
||||
"after": {strconv.Itoa(int(cp.SyncFrom))},
|
||||
"before": {strconv.Itoa(int(cp.SyncUntil))},
|
||||
"after": {strconv.Itoa(int(cliFlags.SyncFrom))},
|
||||
"before": {strconv.Itoa(int(cliFlags.SyncUntil))},
|
||||
"sync_server": {a.HostName},
|
||||
"channel_id": {cp.YoutubeChannelID},
|
||||
"channel_id": {cliFlags.ChannelID},
|
||||
})
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.FetchChannels(status, cliFlags)
|
||||
}
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
if res.StatusCode != http.StatusOK {
|
||||
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
|
||||
log.Debugln(string(body))
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.FetchChannels(status, cliFlags)
|
||||
}
|
||||
var response apiJobsResponse
|
||||
err := json.Unmarshal(body, &response)
|
||||
err = json.Unmarshal(body, &response)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
if response.Data == nil {
|
||||
return nil, errors.Err(response.Error)
|
||||
@@ -85,6 +83,8 @@ type SyncedVideo struct {
|
||||
ClaimID string `json:"claim_id"`
|
||||
Size int64 `json:"size"`
|
||||
MetadataVersion int8 `json:"metadata_version"`
|
||||
Transferred bool `json:"transferred"`
|
||||
IsLbryFirst bool `json:"is_lbry_first"`
|
||||
}
|
||||
|
||||
func sanitizeFailureReason(s *string) {
|
||||
@@ -95,7 +95,50 @@ func sanitizeFailureReason(s *string) {
|
||||
*s = (*s)[:MaxReasonLength]
|
||||
}
|
||||
}
|
||||
func (a *APIConfig) SetChannelStatus(channelID string, status string, failureReason string) (map[string]SyncedVideo, map[string]bool, error) {
|
||||
|
||||
func (a *APIConfig) SetChannelCert(certHex string, channelID string) error {
|
||||
type apiSetChannelCertResponse struct {
|
||||
Success bool `json:"success"`
|
||||
Error null.String `json:"error"`
|
||||
Data string `json:"data"`
|
||||
}
|
||||
|
||||
endpoint := a.ApiURL + "/yt/channel_cert"
|
||||
|
||||
res, err := http.PostForm(endpoint, url.Values{
|
||||
"channel_claim_id": {channelID},
|
||||
"channel_cert": {certHex},
|
||||
"auth_token": {a.ApiToken},
|
||||
})
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.SetChannelCert(certHex, channelID)
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
if res.StatusCode != http.StatusOK {
|
||||
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
|
||||
log.Debugln(string(body))
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.SetChannelCert(certHex, channelID)
|
||||
}
|
||||
var response apiSetChannelCertResponse
|
||||
err = json.Unmarshal(body, &response)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
if !response.Error.IsNull() {
|
||||
return errors.Err(response.Error.String)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *APIConfig) SetChannelStatus(channelID string, status string, failureReason string, transferState *int) (map[string]SyncedVideo, map[string]bool, error) {
|
||||
type apiChannelStatusResponse struct {
|
||||
Success bool `json:"success"`
|
||||
Error null.String `json:"error"`
|
||||
@@ -104,19 +147,37 @@ func (a *APIConfig) SetChannelStatus(channelID string, status string, failureRea
|
||||
endpoint := a.ApiURL + "/yt/channel_status"
|
||||
|
||||
sanitizeFailureReason(&failureReason)
|
||||
res, _ := http.PostForm(endpoint, url.Values{
|
||||
params := url.Values{
|
||||
"channel_id": {channelID},
|
||||
"sync_server": {a.HostName},
|
||||
"auth_token": {a.ApiToken},
|
||||
"sync_status": {status},
|
||||
"failure_reason": {failureReason},
|
||||
})
|
||||
}
|
||||
if transferState != nil {
|
||||
params.Add("transfer_state", strconv.Itoa(*transferState))
|
||||
}
|
||||
res, err := http.PostForm(endpoint, params)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.SetChannelStatus(channelID, status, failureReason, transferState)
|
||||
}
|
||||
return nil, nil, errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
if res.StatusCode >= http.StatusInternalServerError {
|
||||
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
|
||||
log.Debugln(string(body))
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.SetChannelStatus(channelID, status, failureReason, transferState)
|
||||
}
|
||||
var response apiChannelStatusResponse
|
||||
err := json.Unmarshal(body, &response)
|
||||
err = json.Unmarshal(body, &response)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return nil, nil, errors.Err(err)
|
||||
}
|
||||
if !response.Error.IsNull() {
|
||||
return nil, nil, errors.Err(response.Error.String)
|
||||
@@ -142,15 +203,29 @@ func (a *APIConfig) SetChannelClaimID(channelID string, channelClaimID string) e
|
||||
Data string `json:"data"`
|
||||
}
|
||||
endpoint := a.ApiURL + "/yt/set_channel_claim_id"
|
||||
res, _ := http.PostForm(endpoint, url.Values{
|
||||
res, err := http.PostForm(endpoint, url.Values{
|
||||
"channel_id": {channelID},
|
||||
"auth_token": {a.ApiToken},
|
||||
"channel_claim_id": {channelClaimID},
|
||||
})
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.SetChannelClaimID(channelID, channelClaimID)
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
if res.StatusCode != http.StatusOK {
|
||||
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
|
||||
log.Debugln(string(body))
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.SetChannelClaimID(channelID, channelClaimID)
|
||||
}
|
||||
var response apiChannelStatusResponse
|
||||
err := json.Unmarshal(body, &response)
|
||||
err = json.Unmarshal(body, &response)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
@@ -176,15 +251,29 @@ func (a *APIConfig) DeleteVideos(videos []string) error {
|
||||
"video_ids": {videoIDs},
|
||||
"auth_token": {a.ApiToken},
|
||||
}
|
||||
res, _ := http.PostForm(endpoint, vals)
|
||||
res, err := http.PostForm(endpoint, vals)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.DeleteVideos(videos)
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
if res.StatusCode != http.StatusOK {
|
||||
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
|
||||
log.Debugln(string(body))
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.DeleteVideos(videos)
|
||||
}
|
||||
var response struct {
|
||||
Success bool `json:"success"`
|
||||
Error null.String `json:"error"`
|
||||
Data null.String `json:"data"`
|
||||
}
|
||||
err := json.Unmarshal(body, &response)
|
||||
err = json.Unmarshal(body, &response)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
@@ -198,42 +287,59 @@ func (a *APIConfig) DeleteVideos(videos []string) error {
|
||||
return errors.Err("invalid API response. Status code: %d", res.StatusCode)
|
||||
}
|
||||
|
||||
func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status string, claimID string, claimName string, failureReason string, size *int64, metadataVersion uint) error {
|
||||
func (a *APIConfig) MarkVideoStatus(status shared.VideoStatus) error {
|
||||
endpoint := a.ApiURL + "/yt/video_status"
|
||||
|
||||
sanitizeFailureReason(&failureReason)
|
||||
sanitizeFailureReason(&status.FailureReason)
|
||||
vals := url.Values{
|
||||
"youtube_channel_id": {channelID},
|
||||
"video_id": {videoID},
|
||||
"status": {status},
|
||||
"youtube_channel_id": {status.ChannelID},
|
||||
"video_id": {status.VideoID},
|
||||
"status": {status.Status},
|
||||
"auth_token": {a.ApiToken},
|
||||
}
|
||||
if status == VideoStatusPublished || status == VideoStatusUpgradeFailed {
|
||||
if claimID == "" || claimName == "" {
|
||||
return errors.Err("claimID (%s) or claimName (%s) missing", claimID, claimName)
|
||||
if status.Status == VideoStatusPublished || status.Status == VideoStatusUpgradeFailed {
|
||||
if status.ClaimID == "" || status.ClaimName == "" {
|
||||
return errors.Err("claimID (%s) or claimName (%s) missing", status.ClaimID, status.ClaimName)
|
||||
}
|
||||
vals.Add("published_at", strconv.FormatInt(time.Now().Unix(), 10))
|
||||
vals.Add("claim_id", claimID)
|
||||
vals.Add("claim_name", claimName)
|
||||
if metadataVersion > 0 {
|
||||
vals.Add("metadata_version", fmt.Sprintf("%d", metadataVersion))
|
||||
vals.Add("claim_id", status.ClaimID)
|
||||
vals.Add("claim_name", status.ClaimName)
|
||||
if status.MetaDataVersion > 0 {
|
||||
vals.Add("metadata_version", fmt.Sprintf("%d", status.MetaDataVersion))
|
||||
}
|
||||
if size != nil {
|
||||
vals.Add("size", strconv.FormatInt(*size, 10))
|
||||
if status.Size != nil {
|
||||
vals.Add("size", strconv.FormatInt(*status.Size, 10))
|
||||
}
|
||||
}
|
||||
if failureReason != "" {
|
||||
vals.Add("failure_reason", failureReason)
|
||||
if status.FailureReason != "" {
|
||||
vals.Add("failure_reason", status.FailureReason)
|
||||
}
|
||||
if status.IsTransferred != nil {
|
||||
vals.Add("transferred", strconv.FormatBool(*status.IsTransferred))
|
||||
}
|
||||
res, err := http.PostForm(endpoint, vals)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.MarkVideoStatus(status)
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
res, _ := http.PostForm(endpoint, vals)
|
||||
defer res.Body.Close()
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
if res.StatusCode != http.StatusOK {
|
||||
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
|
||||
log.Debugln(string(body))
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.MarkVideoStatus(status)
|
||||
}
|
||||
var response struct {
|
||||
Success bool `json:"success"`
|
||||
Error null.String `json:"error"`
|
||||
Data null.String `json:"data"`
|
||||
}
|
||||
err := json.Unmarshal(body, &response)
|
||||
err = json.Unmarshal(body, &response)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -245,3 +351,102 @@ func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status str
|
||||
}
|
||||
return errors.Err("invalid API response. Status code: %d", res.StatusCode)
|
||||
}
|
||||
|
||||
func (a *APIConfig) VideoState(videoID string) (string, error) {
|
||||
endpoint := a.ApiURL + "/yt/video_state"
|
||||
vals := url.Values{
|
||||
"video_id": {videoID},
|
||||
"auth_token": {a.ApiToken},
|
||||
}
|
||||
|
||||
res, err := http.PostForm(endpoint, vals)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.VideoState(videoID)
|
||||
}
|
||||
return "", errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
if res.StatusCode == http.StatusNotFound {
|
||||
return "not_found", nil
|
||||
}
|
||||
if res.StatusCode != http.StatusOK {
|
||||
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
|
||||
log.Debugln(string(body))
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.VideoState(videoID)
|
||||
}
|
||||
var response struct {
|
||||
Success bool `json:"success"`
|
||||
Error null.String `json:"error"`
|
||||
Data null.String `json:"data"`
|
||||
}
|
||||
err = json.Unmarshal(body, &response)
|
||||
if err != nil {
|
||||
return "", errors.Err(err)
|
||||
}
|
||||
if !response.Error.IsNull() {
|
||||
return "", errors.Err(response.Error.String)
|
||||
}
|
||||
if !response.Data.IsNull() {
|
||||
return response.Data.String, nil
|
||||
}
|
||||
return "", errors.Err("invalid API response. Status code: %d", res.StatusCode)
|
||||
}
|
||||
|
||||
type VideoRelease struct {
|
||||
ID uint64 `json:"id"`
|
||||
YoutubeDataID uint64 `json:"youtube_data_id"`
|
||||
VideoID string `json:"video_id"`
|
||||
ReleaseTime string `json:"release_time"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
}
|
||||
|
||||
func (a *APIConfig) GetReleasedDate(videoID string) (*VideoRelease, error) {
|
||||
endpoint := a.ApiURL + "/yt/released"
|
||||
vals := url.Values{
|
||||
"video_id": {videoID},
|
||||
"auth_token": {a.ApiToken},
|
||||
}
|
||||
|
||||
res, err := http.PostForm(endpoint, vals)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.GetReleasedDate(videoID)
|
||||
}
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
if res.StatusCode == http.StatusNotFound {
|
||||
return nil, nil
|
||||
}
|
||||
if res.StatusCode != http.StatusOK {
|
||||
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
|
||||
log.Debugln(string(body))
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.GetReleasedDate(videoID)
|
||||
}
|
||||
var response struct {
|
||||
Success bool `json:"success"`
|
||||
Error null.String `json:"error"`
|
||||
Data VideoRelease `json:"data"`
|
||||
}
|
||||
err = json.Unmarshal(body, &response)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
if !response.Error.IsNull() {
|
||||
return nil, errors.Err(response.Error.String)
|
||||
}
|
||||
if response.Data.ReleaseTime != "" {
|
||||
return &response.Data, nil
|
||||
}
|
||||
return nil, errors.Err("invalid API response. Status code: %d", res.StatusCode)
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
package shared
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
)
|
||||
|
||||
type Fee struct {
|
||||
Amount string `json:"amount"`
|
||||
Address string `json:"address"`
|
||||
Currency string `json:"currency"`
|
||||
}
|
||||
type YoutubeChannel struct {
|
||||
ChannelId string `json:"channel_id"`
|
||||
TotalVideos uint `json:"total_videos"`
|
||||
TotalSubscribers uint `json:"total_subscribers"`
|
||||
DesiredChannelName string `json:"desired_channel_name"`
|
||||
Fee *Fee `json:"fee"`
|
||||
ChannelClaimID string `json:"channel_claim_id"`
|
||||
TransferState int `json:"transfer_state"`
|
||||
PublishAddress string `json:"publish_address"`
|
||||
PublicKey string `json:"public_key"`
|
||||
LengthLimit int `json:"length_limit"`
|
||||
SizeLimit int `json:"size_limit"`
|
||||
LastUploadedVideo string `json:"last_uploaded_video"`
|
||||
}
|
||||
|
||||
var NeverRetryFailures = []string{
|
||||
"Error extracting sts from embedded url response",
|
||||
"Unable to extract signature tokens",
|
||||
"the video is too big to sync, skipping for now",
|
||||
"video is too long to process",
|
||||
"This video contains content from",
|
||||
"no compatible format available for this video",
|
||||
"Watch this video on YouTube.",
|
||||
"have blocked it on copyright grounds",
|
||||
"giving up after 0 fragment retries",
|
||||
"Sign in to confirm your age",
|
||||
}
|
||||
|
||||
type SyncFlags struct {
|
||||
StopOnError bool
|
||||
TakeOverExistingChannel bool
|
||||
SkipSpaceCheck bool
|
||||
SyncUpdate bool
|
||||
SingleRun bool
|
||||
RemoveDBUnpublished bool
|
||||
UpgradeMetadata bool
|
||||
DisableTransfers bool
|
||||
QuickSync bool
|
||||
MaxTries int
|
||||
Refill int
|
||||
Limit int
|
||||
SyncStatus string
|
||||
ChannelID string
|
||||
SyncFrom int64
|
||||
SyncUntil int64
|
||||
ConcurrentJobs int
|
||||
VideosLimit int
|
||||
MaxVideoSize int
|
||||
MaxVideoLength time.Duration
|
||||
}
|
||||
|
||||
func (f *SyncFlags) IsSingleChannelSync() bool {
|
||||
return f.ChannelID != ""
|
||||
}
|
||||
|
||||
type VideoStatus struct {
|
||||
ChannelID string
|
||||
VideoID string
|
||||
Status string
|
||||
ClaimID string
|
||||
ClaimName string
|
||||
FailureReason string
|
||||
Size *int64
|
||||
MetaDataVersion uint
|
||||
IsTransferred *bool
|
||||
}
|
||||
|
||||
const (
|
||||
StatusPending = "pending" // waiting for permission to sync
|
||||
StatusPendingEmail = "pendingemail" // permission granted but missing email
|
||||
StatusQueued = "queued" // in sync queue. will be synced soon
|
||||
StatusPendingUpgrade = "pendingupgrade" // in sync queue. will be synced soon
|
||||
StatusSyncing = "syncing" // syncing now
|
||||
StatusSynced = "synced" // done
|
||||
StatusFailed = "failed"
|
||||
StatusFinalized = "finalized" // no more changes allowed
|
||||
StatusAbandoned = "abandoned" // deleted on youtube or banned
|
||||
)
|
||||
|
||||
var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusPendingUpgrade, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned}
|
||||
|
||||
const LatestMetadataVersion = 2
|
||||
|
||||
const (
|
||||
VideoStatusPublished = "published"
|
||||
VideoStatusFailed = "failed"
|
||||
VideoStatusUpgradeFailed = "upgradefailed"
|
||||
VideoStatusUnpublished = "unpublished"
|
||||
VideoStatusTranferFailed = "transferfailed"
|
||||
)
|
||||
|
||||
const (
|
||||
TransferStateNotTouched = iota
|
||||
TransferStatePending
|
||||
TransferStateComplete
|
||||
TransferStateManual
|
||||
)
|
||||
|
||||
type AwsConfigs struct {
|
||||
AwsS3ID string
|
||||
AwsS3Secret string
|
||||
AwsS3Region string
|
||||
AwsS3Bucket string
|
||||
}
|
||||
|
||||
func (a *AwsConfigs) GetS3AWSConfig() *aws.Config {
|
||||
return &aws.Config{
|
||||
Credentials: credentials.NewStaticCredentials(a.AwsS3ID, a.AwsS3Secret, ""),
|
||||
Region: &a.AwsS3Region,
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -4,8 +4,8 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/lbryio/lbry.go/extras/jsonrpc"
|
||||
"github.com/lbryio/ytsync/namer"
|
||||
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
||||
"github.com/lbryio/ytsync/v5/namer"
|
||||
)
|
||||
|
||||
type SyncSummary struct {
|
||||
|
||||
+236
-143
@@ -3,7 +3,6 @@ package sources
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
@@ -13,21 +12,25 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
"github.com/lbryio/lbry.go/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/extras/util"
|
||||
"github.com/lbryio/ytsync/v5/downloader/ytdl"
|
||||
"github.com/lbryio/ytsync/v5/shared"
|
||||
|
||||
"github.com/lbryio/ytsync/namer"
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
"github.com/lbryio/ytsync/tagsManager"
|
||||
"github.com/lbryio/ytsync/thumbs"
|
||||
"github.com/lbryio/ytsync/v5/ip_manager"
|
||||
"github.com/lbryio/ytsync/v5/namer"
|
||||
"github.com/lbryio/ytsync/v5/sdk"
|
||||
"github.com/lbryio/ytsync/v5/tags_manager"
|
||||
"github.com/lbryio/ytsync/v5/thumbs"
|
||||
"github.com/lbryio/ytsync/v5/timing"
|
||||
logUtils "github.com/lbryio/ytsync/v5/util"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
|
||||
"github.com/ChannelMeter/iso8601duration"
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/nikooo777/ytdl"
|
||||
"github.com/shopspring/decimal"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"google.golang.org/api/youtube/v3"
|
||||
)
|
||||
|
||||
type YoutubeVideo struct {
|
||||
@@ -37,10 +40,10 @@ type YoutubeVideo struct {
|
||||
playlistPosition int64
|
||||
size *int64
|
||||
maxVideoSize int64
|
||||
maxVideoLength float64
|
||||
maxVideoLength time.Duration
|
||||
publishedAt time.Time
|
||||
dir string
|
||||
youtubeInfo *youtube.Video
|
||||
youtubeInfo *ytdl.YtdlVideo
|
||||
youtubeChannelID string
|
||||
tags []string
|
||||
awsConfig aws.Config
|
||||
@@ -48,10 +51,10 @@ type YoutubeVideo struct {
|
||||
lbryChannelID string
|
||||
mocked bool
|
||||
walletLock *sync.RWMutex
|
||||
stopGroup *stop.Group
|
||||
pool *ip_manager.IPPool
|
||||
}
|
||||
|
||||
const reflectorURL = "http://blobs.lbry.io/"
|
||||
|
||||
var youtubeCategories = map[string]string{
|
||||
"1": "film & animation",
|
||||
"2": "autos & vehicles",
|
||||
@@ -87,22 +90,25 @@ var youtubeCategories = map[string]string{
|
||||
"44": "trailers",
|
||||
}
|
||||
|
||||
func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64, awsConfig aws.Config) *YoutubeVideo {
|
||||
publishedAt, _ := time.Parse(time.RFC3339Nano, videoData.Snippet.PublishedAt) // ignore parse errors
|
||||
func NewYoutubeVideo(directory string, videoData *ytdl.YtdlVideo, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) (*YoutubeVideo, error) {
|
||||
// youtube-dl returns times in local timezone sometimes. this could break in the future
|
||||
// maybe we can file a PR to choose the timezone we want from youtube-dl
|
||||
return &YoutubeVideo{
|
||||
id: videoData.Id,
|
||||
title: videoData.Snippet.Title,
|
||||
description: videoData.Snippet.Description,
|
||||
id: videoData.ID,
|
||||
title: videoData.Title,
|
||||
description: videoData.Description,
|
||||
playlistPosition: playlistPosition,
|
||||
publishedAt: publishedAt,
|
||||
publishedAt: videoData.UploadDateForReal,
|
||||
dir: directory,
|
||||
youtubeInfo: videoData,
|
||||
awsConfig: awsConfig,
|
||||
mocked: false,
|
||||
youtubeChannelID: videoData.Snippet.ChannelId,
|
||||
}
|
||||
youtubeChannelID: videoData.ChannelID,
|
||||
stopGroup: stopGroup,
|
||||
pool: pool,
|
||||
}, nil
|
||||
}
|
||||
func NewMockedVideo(directory string, videoID string, youtubeChannelID string, awsConfig aws.Config) *YoutubeVideo {
|
||||
func NewMockedVideo(directory string, videoID string, youtubeChannelID string, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo {
|
||||
return &YoutubeVideo{
|
||||
id: videoID,
|
||||
playlistPosition: 0,
|
||||
@@ -110,6 +116,8 @@ func NewMockedVideo(directory string, videoID string, youtubeChannelID string, a
|
||||
awsConfig: awsConfig,
|
||||
mocked: true,
|
||||
youtubeChannelID: youtubeChannelID,
|
||||
stopGroup: stopGroup,
|
||||
pool: pool,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -160,49 +168,30 @@ func (v *YoutubeVideo) getFullPath() string {
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) getAbbrevDescription() string {
|
||||
maxLines := 10
|
||||
maxLength := 2800
|
||||
description := strings.TrimSpace(v.description)
|
||||
additionalDescription := "\nhttps://www.youtube.com/watch?v=" + v.id
|
||||
khanAcademyClaimID := "5fc52291980268b82413ca4c0ace1b8d749f3ffb"
|
||||
if v.lbryChannelID == khanAcademyClaimID {
|
||||
additionalDescription = additionalDescription + "\nNote: All Khan Academy content is available for free at (www.khanacademy.org)"
|
||||
}
|
||||
if strings.Count(description, "\n") < maxLines {
|
||||
return description + "\n..." + additionalDescription
|
||||
if len(description) > maxLength {
|
||||
description = description[:maxLength]
|
||||
}
|
||||
return strings.Join(strings.Split(description, "\n")[:maxLines], "\n") + "\n..." + additionalDescription
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) fallbackDownload() error {
|
||||
cmd := exec.Command("youtube-dl",
|
||||
"--no-progress",
|
||||
"-fbestvideo[ext=mp4,height<=1080,filesize<2000M]+best[ext=mp4,height<=1080,filesize<2000M]",
|
||||
"-o"+strings.TrimRight(v.getFullPath(), ".mp4"),
|
||||
"--merge-output-format",
|
||||
"mp4",
|
||||
"https://www.youtube.com/watch?v="+v.ID())
|
||||
|
||||
log.Printf("Running command and waiting for it to finish...")
|
||||
output, err := cmd.CombinedOutput()
|
||||
log.Debugln(string(output))
|
||||
if err != nil {
|
||||
log.Printf("Command finished with error: %v", errors.Err(string(output)))
|
||||
_ = v.delete()
|
||||
return errors.Err(err)
|
||||
}
|
||||
fi, err := os.Stat(v.getFullPath())
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
videoSize := fi.Size()
|
||||
v.size = &videoSize
|
||||
return nil
|
||||
return description + "\n..." + additionalDescription
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) download() error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("download").Add(time.Since(start))
|
||||
}(start)
|
||||
if v.youtubeInfo.IsLive != nil {
|
||||
return errors.Err("video is a live stream and hasn't completed yet")
|
||||
}
|
||||
videoPath := v.getFullPath()
|
||||
|
||||
err := os.Mkdir(v.videoDir(), 0750)
|
||||
err := os.Mkdir(v.videoDir(), 0777)
|
||||
if err != nil && !strings.Contains(err.Error(), "file exists") {
|
||||
return errors.Wrap(err, 0)
|
||||
}
|
||||
@@ -214,72 +203,145 @@ func (v *YoutubeVideo) download() error {
|
||||
log.Debugln(v.id + " already exists at " + videoPath)
|
||||
return nil
|
||||
}
|
||||
|
||||
videoUrl := "https://www.youtube.com/watch?v=" + v.id
|
||||
videoInfo, err := ytdl.GetVideoInfo(videoUrl)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
qualities := []string{
|
||||
"1080",
|
||||
"720",
|
||||
"480",
|
||||
"320",
|
||||
}
|
||||
|
||||
codec := []string{"H.264"}
|
||||
ext := []string{"mp4"}
|
||||
|
||||
//Filter requires a [] interface{}
|
||||
codecFilter := make([]interface{}, len(codec))
|
||||
for i, v := range codec {
|
||||
codecFilter[i] = v
|
||||
}
|
||||
|
||||
//Filter requires a [] interface{}
|
||||
extFilter := make([]interface{}, len(ext))
|
||||
for i, v := range ext {
|
||||
extFilter[i] = v
|
||||
}
|
||||
|
||||
formats := videoInfo.Formats.Filter(ytdl.FormatVideoEncodingKey, codecFilter).Filter(ytdl.FormatExtensionKey, extFilter)
|
||||
if len(formats) == 0 {
|
||||
return errors.Err("no compatible format available for this video")
|
||||
}
|
||||
maxRetryAttempts := 5
|
||||
isLengthLimitSet := v.maxVideoLength > 0.01
|
||||
if isLengthLimitSet && videoInfo.Duration.Hours() > v.maxVideoLength {
|
||||
return errors.Err("video is too long to process")
|
||||
}
|
||||
|
||||
for i := 0; i < len(formats) && i < maxRetryAttempts; i++ {
|
||||
formatIndex := i
|
||||
if i == maxRetryAttempts-1 {
|
||||
formatIndex = len(formats) - 1
|
||||
dur := time.Duration(v.youtubeInfo.Duration) * time.Second
|
||||
if dur.Hours() > 2 { //for videos longer than 2 hours only sync up to 720p
|
||||
qualities = []string{
|
||||
"720",
|
||||
"480",
|
||||
"320",
|
||||
}
|
||||
var downloadedFile *os.File
|
||||
downloadedFile, err = os.Create(videoPath)
|
||||
}
|
||||
|
||||
ytdlArgs := []string{
|
||||
"--no-progress",
|
||||
"-o" + strings.TrimSuffix(v.getFullPath(), ".mp4"),
|
||||
"--merge-output-format",
|
||||
"mp4",
|
||||
"--rm-cache-dir",
|
||||
"--postprocessor-args",
|
||||
"-movflags faststart",
|
||||
"--abort-on-unavailable-fragment",
|
||||
"--fragment-retries",
|
||||
"1",
|
||||
"--cookies",
|
||||
"cookies.txt",
|
||||
}
|
||||
userAgent := []string{"--user-agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"}
|
||||
if v.maxVideoSize > 0 {
|
||||
ytdlArgs = append(ytdlArgs,
|
||||
"--max-filesize",
|
||||
fmt.Sprintf("%dM", v.maxVideoSize),
|
||||
)
|
||||
}
|
||||
if v.maxVideoLength > 0 {
|
||||
ytdlArgs = append(ytdlArgs,
|
||||
"--match-filter",
|
||||
fmt.Sprintf("duration <= %d", int(v.maxVideoLength.Seconds())),
|
||||
)
|
||||
}
|
||||
|
||||
var sourceAddress string
|
||||
for {
|
||||
sourceAddress, err = v.pool.GetIP(v.id)
|
||||
if err != nil {
|
||||
if errors.Is(err, ip_manager.ErrAllThrottled) {
|
||||
select {
|
||||
case <-v.stopGroup.Ch():
|
||||
return errors.Err("interrupted by user")
|
||||
default:
|
||||
time.Sleep(ip_manager.IPCooldownPeriod)
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
defer v.pool.ReleaseIP(sourceAddress)
|
||||
|
||||
ytdlArgs = append(ytdlArgs,
|
||||
"--source-address",
|
||||
sourceAddress,
|
||||
"https://www.youtube.com/watch?v="+v.ID(),
|
||||
)
|
||||
|
||||
for i := 0; i < len(qualities); i++ {
|
||||
quality := qualities[i]
|
||||
argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][height<="+quality+"]+bestaudio[ext!=webm]")
|
||||
argsWithFilters = append(argsWithFilters, userAgent...)
|
||||
cmd := exec.Command("youtube-dl", argsWithFilters...)
|
||||
log.Printf("Running command youtube-dl %s", strings.Join(argsWithFilters, " "))
|
||||
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
err = videoInfo.Download(formats[formatIndex], downloadedFile)
|
||||
_ = downloadedFile.Close()
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
//delete the video and ignore the error
|
||||
_ = v.delete()
|
||||
return errors.Err(err.Error())
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
errorLog, _ := ioutil.ReadAll(stderr)
|
||||
outLog, _ := ioutil.ReadAll(stdout)
|
||||
err = cmd.Wait()
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "exit status 1") {
|
||||
if strings.Contains(string(errorLog), "HTTP Error 429") || strings.Contains(string(errorLog), "returned non-zero exit status 8") {
|
||||
v.pool.SetThrottled(sourceAddress)
|
||||
} else if strings.Contains(string(errorLog), "giving up after 0 fragment retries") {
|
||||
if i == (len(qualities) - 1) {
|
||||
return errors.Err(string(errorLog))
|
||||
}
|
||||
continue //this bypasses the yt throttling IP redistribution... TODO: don't
|
||||
} else if strings.Contains(string(errorLog), "YouTube said: Unable to extract video data") && !strings.Contains(userAgent[1], "Googlebot") {
|
||||
i-- //do not lower quality when trying a different user agent
|
||||
userAgent = []string{"--user-agent", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"}
|
||||
log.Infof("trying different user agent for video %s", v.ID())
|
||||
continue
|
||||
}
|
||||
return errors.Err(string(errorLog))
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
log.Debugln(string(outLog))
|
||||
|
||||
if strings.Contains(string(outLog), "does not pass filter duration") {
|
||||
_ = v.delete("does not pass filter duration")
|
||||
return errors.Err("video is too long to process")
|
||||
}
|
||||
if strings.Contains(string(outLog), "File is larger than max-filesize") {
|
||||
_ = v.delete("File is larger than max-filesize")
|
||||
return errors.Err("the video is too big to sync, skipping for now")
|
||||
}
|
||||
if string(errorLog) != "" {
|
||||
log.Printf("Command finished with error: %v", errors.Err(string(errorLog)))
|
||||
_ = v.delete("due to error")
|
||||
return errors.Err(string(errorLog))
|
||||
}
|
||||
fi, err := os.Stat(v.getFullPath())
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
err = os.Chmod(v.getFullPath(), 0777)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
videoSize := fi.Size()
|
||||
v.size = &videoSize
|
||||
|
||||
isVideoSizeLimitSet := v.maxVideoSize > 0
|
||||
if isVideoSizeLimitSet && videoSize > v.maxVideoSize {
|
||||
//delete the video and ignore the error
|
||||
_ = v.delete()
|
||||
err = errors.Err("file is too big and there is no other format available")
|
||||
} else {
|
||||
break
|
||||
}
|
||||
break
|
||||
}
|
||||
return errors.Err(err)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) videoDir() string {
|
||||
@@ -305,14 +367,14 @@ func (v *YoutubeVideo) getDownloadedPath() (string, error) {
|
||||
return "", errors.Err("could not find any downloaded videos")
|
||||
|
||||
}
|
||||
func (v *YoutubeVideo) delete() error {
|
||||
func (v *YoutubeVideo) delete(reason string) error {
|
||||
videoPath, err := v.getDownloadedPath()
|
||||
if err != nil {
|
||||
log.Errorln(err)
|
||||
return err
|
||||
}
|
||||
err = os.Remove(videoPath)
|
||||
log.Debugf("%s deleted from disk (%s)", v.id, videoPath)
|
||||
log.Debugf("%s deleted from disk for '%s' (%s)", v.id, reason, videoPath)
|
||||
|
||||
if err != nil {
|
||||
err = errors.Prefix("delete error", err)
|
||||
@@ -324,14 +386,17 @@ func (v *YoutubeVideo) delete() error {
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) triggerThumbnailSave() (err error) {
|
||||
thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Snippet.Thumbnails)
|
||||
v.thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.Url, v.ID(), v.awsConfig)
|
||||
thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Thumbnails)
|
||||
v.thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.URL, v.ID(), v.awsConfig)
|
||||
return err
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("publish").Add(time.Since(start))
|
||||
}(start)
|
||||
languages, locations, tags := v.getMetadata()
|
||||
|
||||
var fee *jsonrpc.Fee
|
||||
if params.Fee != nil {
|
||||
feeAmount, err := decimal.NewFromString(params.Fee.Amount)
|
||||
@@ -354,6 +419,9 @@ func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, params SyncParams) (*Sync
|
||||
ThumbnailURL: &v.thumbnailURL,
|
||||
Tags: tags,
|
||||
Locations: locations,
|
||||
FundingAccountIDs: []string{
|
||||
params.DefaultAccount,
|
||||
},
|
||||
},
|
||||
Fee: fee,
|
||||
License: util.PtrToString("Copyrighted (contact publisher)"),
|
||||
@@ -377,12 +445,13 @@ type SyncParams struct {
|
||||
ChannelID string
|
||||
MaxVideoSize int
|
||||
Namer *namer.Namer
|
||||
MaxVideoLength float64
|
||||
Fee *sdk.Fee
|
||||
MaxVideoLength time.Duration
|
||||
Fee *shared.Fee
|
||||
DefaultAccount string
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo, reprocess bool, walletLock *sync.RWMutex) (*SyncSummary, error) {
|
||||
v.maxVideoSize = int64(params.MaxVideoSize) * 1024 * 1024
|
||||
v.maxVideoSize = int64(params.MaxVideoSize)
|
||||
v.maxVideoLength = params.MaxVideoLength
|
||||
v.lbryChannelID = params.ChannelID
|
||||
v.walletLock = walletLock
|
||||
@@ -394,15 +463,24 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
|
||||
err := v.download()
|
||||
if err != nil {
|
||||
log.Errorf("standard downloader failed: %s. Trying fallback downloader\n", err.Error())
|
||||
fallBackErr := v.fallbackDownload()
|
||||
if fallBackErr != nil {
|
||||
log.Errorf("fallback downloader failed: %s\n", fallBackErr.Error())
|
||||
return nil, errors.Prefix("download error", err) //return original error
|
||||
}
|
||||
var err error
|
||||
|
||||
dur := time.Duration(v.youtubeInfo.Duration) * time.Second
|
||||
|
||||
if dur > v.maxVideoLength {
|
||||
logUtils.SendErrorToSlack("%s is %s long and the limit is %s", v.id, dur.String(), v.maxVideoLength.String())
|
||||
return nil, errors.Err("video is too long to process")
|
||||
}
|
||||
for {
|
||||
err = v.download()
|
||||
if err != nil && strings.Contains(err.Error(), "HTTP Error 429") {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return nil, errors.Prefix("download error", err)
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
log.Debugln("Downloaded " + v.id)
|
||||
|
||||
err = v.triggerThumbnailSave()
|
||||
@@ -413,7 +491,7 @@ func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncPar
|
||||
|
||||
summary, err := v.publish(daemon, params)
|
||||
//delete the video in all cases (and ignore the error)
|
||||
_ = v.delete()
|
||||
_ = v.delete("finished download and publish")
|
||||
|
||||
return summary, errors.Prefix("publish error", err)
|
||||
}
|
||||
@@ -423,31 +501,37 @@ func (v *YoutubeVideo) getMetadata() (languages []string, locations []jsonrpc.Lo
|
||||
locations = nil
|
||||
tags = nil
|
||||
if !v.mocked {
|
||||
if v.youtubeInfo.Snippet.DefaultLanguage != "" {
|
||||
languages = []string{v.youtubeInfo.Snippet.DefaultLanguage}
|
||||
}
|
||||
/*
|
||||
if v.youtubeInfo.Snippet.DefaultLanguage != "" {
|
||||
if v.youtubeInfo.Snippet.DefaultLanguage == "iw" {
|
||||
v.youtubeInfo.Snippet.DefaultLanguage = "he"
|
||||
}
|
||||
languages = []string{v.youtubeInfo.Snippet.DefaultLanguage}
|
||||
}*/
|
||||
|
||||
if v.youtubeInfo.RecordingDetails != nil && v.youtubeInfo.RecordingDetails.Location != nil {
|
||||
/*if v.youtubeInfo.!= nil && v.youtubeInfo.RecordingDetails.Location != nil {
|
||||
locations = []jsonrpc.Location{{
|
||||
Latitude: util.PtrToString(fmt.Sprintf("%.7f", v.youtubeInfo.RecordingDetails.Location.Latitude)),
|
||||
Longitude: util.PtrToString(fmt.Sprintf("%.7f", v.youtubeInfo.RecordingDetails.Location.Longitude)),
|
||||
}}
|
||||
}
|
||||
tags = v.youtubeInfo.Snippet.Tags
|
||||
}*/
|
||||
tags = v.youtubeInfo.Tags
|
||||
}
|
||||
tags, err := tagsManager.SanitizeTags(tags, v.youtubeChannelID)
|
||||
tags, err := tags_manager.SanitizeTags(tags, v.youtubeChannelID)
|
||||
if err != nil {
|
||||
log.Errorln(err.Error())
|
||||
}
|
||||
if !v.mocked {
|
||||
tags = append(tags, youtubeCategories[v.youtubeInfo.Snippet.CategoryId])
|
||||
for _, category := range v.youtubeInfo.Categories {
|
||||
tags = append(tags, youtubeCategories[category])
|
||||
}
|
||||
}
|
||||
|
||||
return languages, locations, tags
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo) (*SyncSummary, error) {
|
||||
c, err := daemon.ClaimSearch(nil, &existingVideoData.ClaimID, nil, nil)
|
||||
c, err := daemon.ClaimSearch(nil, &existingVideoData.ClaimID, nil, nil, 1, 20)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
@@ -465,8 +549,8 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
|
||||
if v.mocked {
|
||||
return nil, errors.Err("could not find thumbnail for mocked video")
|
||||
}
|
||||
thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Snippet.Thumbnails)
|
||||
thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.Url, v.ID(), v.awsConfig)
|
||||
thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Thumbnails)
|
||||
thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.URL, v.ID(), v.awsConfig)
|
||||
} else {
|
||||
thumbnailURL = thumbs.ThumbnailEndpoint + v.ID()
|
||||
}
|
||||
@@ -477,8 +561,15 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
|
||||
videoSize = uint64(existingVideoData.Size)
|
||||
} else {
|
||||
log.Infof("%s: the video must be republished as we can't get the right size", v.ID())
|
||||
//return v.downloadAndPublish(daemon, params) //TODO: actually republish the video. NB: the current claim should be abandoned first
|
||||
return nil, errors.Err("the video must be republished as we can't get the right size")
|
||||
if !v.mocked {
|
||||
_, err = daemon.StreamAbandon(currentClaim.Txid, currentClaim.Nout, nil, true)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
return v.downloadAndPublish(daemon, params)
|
||||
|
||||
}
|
||||
return nil, errors.Err("the video must be republished as we can't get the right size but it doesn't exist on youtube anymore")
|
||||
}
|
||||
}
|
||||
v.size = util.PtrToInt64(int64(videoSize))
|
||||
@@ -500,6 +591,9 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
|
||||
ThumbnailURL: &thumbnailURL,
|
||||
Languages: languages,
|
||||
Locations: locations,
|
||||
FundingAccountIDs: []string{
|
||||
params.DefaultAccount,
|
||||
},
|
||||
},
|
||||
Author: util.PtrToString(""),
|
||||
License: util.PtrToString("Copyrighted (contact publisher)"),
|
||||
@@ -512,10 +606,12 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
|
||||
v.walletLock.RLock()
|
||||
defer v.walletLock.RUnlock()
|
||||
if v.mocked {
|
||||
start := time.Now()
|
||||
pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
|
||||
StreamCreateOptions: streamCreateOptions,
|
||||
FileSize: &videoSize,
|
||||
})
|
||||
timing.TimedComponent("StreamUpdate").Add(time.Since(start))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -526,15 +622,11 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
|
||||
}, nil
|
||||
}
|
||||
|
||||
videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
streamCreateOptions.ClaimCreateOptions.Title = &v.title
|
||||
streamCreateOptions.ClaimCreateOptions.Description = util.PtrToString(v.getAbbrevDescription())
|
||||
streamCreateOptions.Duration = util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds())))
|
||||
streamCreateOptions.Duration = util.PtrToUint64(uint64(v.youtubeInfo.Duration))
|
||||
streamCreateOptions.ReleaseTime = util.PtrToInt64(v.publishedAt.Unix())
|
||||
start := time.Now()
|
||||
pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
|
||||
ClearLanguages: util.PtrToBool(true),
|
||||
ClearLocations: util.PtrToBool(true),
|
||||
@@ -542,6 +634,7 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
|
||||
StreamCreateOptions: streamCreateOptions,
|
||||
FileSize: &videoSize,
|
||||
})
|
||||
timing.TimedComponent("StreamUpdate").Add(time.Since(start))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package tagsManager
|
||||
package tags_manager
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
@@ -71,7 +71,7 @@ func SanitizeTags(tags []string, youtubeChannelID string) ([]string, error) {
|
||||
return sanitizedTags, nil
|
||||
}
|
||||
|
||||
const TagMaxLength = 255
|
||||
const TagMaxLength = 50
|
||||
|
||||
func normalizeTag(t string) (string, error) {
|
||||
t = strings.ToLower(t)
|
||||
@@ -1,4 +1,4 @@
|
||||
package tagsManager
|
||||
package tags_manager
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
+24
-20
@@ -1,17 +1,19 @@
|
||||
package thumbs
|
||||
|
||||
import (
|
||||
"google.golang.org/api/youtube/v3"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/lbryio/errors.go"
|
||||
"github.com/lbryio/ytsync/v5/downloader/ytdl"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
"github.com/prometheus/common/log"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type thumbnailUploader struct {
|
||||
@@ -25,13 +27,15 @@ const thumbnailPath = "/tmp/ytsync_thumbnails/"
|
||||
const ThumbnailEndpoint = "https://thumbnails.lbry.com/"
|
||||
|
||||
func (u *thumbnailUploader) downloadThumbnail() error {
|
||||
_ = os.Mkdir(thumbnailPath, 0750)
|
||||
_ = os.Mkdir(thumbnailPath, 0777)
|
||||
img, err := os.Create("/tmp/ytsync_thumbnails/" + u.name)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer img.Close()
|
||||
|
||||
if strings.HasPrefix(u.originalUrl, "//") {
|
||||
u.originalUrl = "https:" + u.originalUrl
|
||||
}
|
||||
resp, err := http.Get(u.originalUrl)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
@@ -61,11 +65,12 @@ func (u *thumbnailUploader) uploadThumbnail() error {
|
||||
uploader := s3manager.NewUploader(s3Session)
|
||||
|
||||
_, err = uploader.Upload(&s3manager.UploadInput{
|
||||
Bucket: aws.String("thumbnails.lbry.com"),
|
||||
Key: key,
|
||||
Body: thumb,
|
||||
ACL: aws.String("public-read"),
|
||||
ContentType: aws.String("image/jpeg"),
|
||||
Bucket: aws.String("thumbnails.lbry.com"),
|
||||
Key: key,
|
||||
Body: thumb,
|
||||
ACL: aws.String("public-read"),
|
||||
ContentType: aws.String("image/jpeg"),
|
||||
CacheControl: aws.String("public, max-age=2592000"),
|
||||
})
|
||||
u.mirroredUrl = ThumbnailEndpoint + u.name
|
||||
return errors.Err(err)
|
||||
@@ -97,15 +102,14 @@ func MirrorThumbnail(url string, name string, s3Config aws.Config) (string, erro
|
||||
return tu.mirroredUrl, nil
|
||||
}
|
||||
|
||||
func GetBestThumbnail(thumbnails *youtube.ThumbnailDetails) *youtube.Thumbnail {
|
||||
if thumbnails.Maxres != nil {
|
||||
return thumbnails.Maxres
|
||||
} else if thumbnails.High != nil {
|
||||
return thumbnails.High
|
||||
} else if thumbnails.Medium != nil {
|
||||
return thumbnails.Medium
|
||||
} else if thumbnails.Standard != nil {
|
||||
return thumbnails.Standard
|
||||
func GetBestThumbnail(thumbnails []ytdl.Thumbnail) *ytdl.Thumbnail {
|
||||
var bestWidth *ytdl.Thumbnail
|
||||
for _, thumbnail := range thumbnails {
|
||||
if bestWidth == nil {
|
||||
bestWidth = &thumbnail
|
||||
} else if bestWidth.Width < thumbnail.Width {
|
||||
bestWidth = &thumbnail
|
||||
}
|
||||
}
|
||||
return thumbnails.Default
|
||||
return bestWidth
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
package timing
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/ytsync/v5/metrics"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type Timing struct {
|
||||
component string
|
||||
milliseconds int64
|
||||
min int64
|
||||
max int64
|
||||
invocations int32
|
||||
}
|
||||
|
||||
var timings *sync.Map
|
||||
|
||||
func TimedComponent(component string) *Timing {
|
||||
if timings == nil {
|
||||
timings = &sync.Map{}
|
||||
}
|
||||
stored, _ := timings.LoadOrStore(component, &Timing{
|
||||
component: component,
|
||||
milliseconds: 0,
|
||||
min: int64(99999999),
|
||||
})
|
||||
t, _ := stored.(*Timing)
|
||||
return t
|
||||
}
|
||||
|
||||
func ClearTimings() {
|
||||
if timings == nil {
|
||||
return
|
||||
}
|
||||
timings.Range(func(key interface{}, value interface{}) bool {
|
||||
timings.Delete(key)
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
func Report() {
|
||||
var totalTime time.Duration
|
||||
timings.Range(func(key interface{}, value interface{}) bool {
|
||||
totalTime += value.(*Timing).Get()
|
||||
return true
|
||||
})
|
||||
timings.Range(func(key interface{}, value interface{}) bool {
|
||||
component := key
|
||||
componentRuntime := value.(*Timing).Get().String()
|
||||
percentTime := float64(value.(*Timing).Get()) / float64(totalTime) * 100
|
||||
invocations := value.(*Timing).Invocations()
|
||||
avgTime := (time.Duration(int64(float64(value.(*Timing).Get()) / float64(value.(*Timing).Invocations())))).String()
|
||||
minRuntime := value.(*Timing).Min().String()
|
||||
maxRuntime := value.(*Timing).Max().String()
|
||||
logrus.Printf("component %s ran for %s (%.2f%% of the total time) - invoked %d times with an average of %s per call, a minimum of %s and a maximum of %s",
|
||||
component,
|
||||
componentRuntime,
|
||||
percentTime,
|
||||
invocations,
|
||||
avgTime,
|
||||
minRuntime,
|
||||
maxRuntime,
|
||||
)
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
func (t *Timing) Add(d time.Duration) {
|
||||
metrics.Durations.WithLabelValues(t.component).Observe(d.Seconds())
|
||||
atomic.AddInt64(&t.milliseconds, d.Milliseconds())
|
||||
for {
|
||||
oldMin := atomic.LoadInt64(&t.min)
|
||||
if d.Milliseconds() < oldMin {
|
||||
if atomic.CompareAndSwapInt64(&t.min, oldMin, d.Milliseconds()) {
|
||||
break
|
||||
}
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
for {
|
||||
oldMax := atomic.LoadInt64(&t.max)
|
||||
if d.Milliseconds() > oldMax {
|
||||
if atomic.CompareAndSwapInt64(&t.max, oldMax, d.Milliseconds()) {
|
||||
break
|
||||
}
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
atomic.AddInt32(&t.invocations, 1)
|
||||
}
|
||||
|
||||
func (t *Timing) Get() time.Duration {
|
||||
ms := atomic.LoadInt64(&t.milliseconds)
|
||||
return time.Duration(ms) * time.Millisecond
|
||||
}
|
||||
|
||||
func (t *Timing) Invocations() int32 {
|
||||
return atomic.LoadInt32(&t.invocations)
|
||||
}
|
||||
|
||||
func (t *Timing) Min() time.Duration {
|
||||
ms := atomic.LoadInt64(&t.min)
|
||||
return time.Duration(ms) * time.Millisecond
|
||||
}
|
||||
func (t *Timing) Max() time.Duration {
|
||||
ms := atomic.LoadInt64(&t.max)
|
||||
return time.Duration(ms) * time.Millisecond
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// SendErrorToSlack Sends an error message to the default channel and to the process log.
|
||||
func SendErrorToSlack(format string, a ...interface{}) {
|
||||
message := format
|
||||
if len(a) > 0 {
|
||||
message = fmt.Sprintf(format, a...)
|
||||
}
|
||||
log.Errorln(message)
|
||||
err := util.SendToSlack(":sos: " + message)
|
||||
if err != nil {
|
||||
log.Errorln(err)
|
||||
}
|
||||
}
|
||||
|
||||
// SendInfoToSlack Sends an info message to the default channel and to the process log.
|
||||
func SendInfoToSlack(format string, a ...interface{}) {
|
||||
message := format
|
||||
if len(a) > 0 {
|
||||
message = fmt.Sprintf(format, a...)
|
||||
}
|
||||
log.Infoln(message)
|
||||
err := util.SendToSlack(":information_source: " + message)
|
||||
if err != nil {
|
||||
log.Errorln(err)
|
||||
}
|
||||
}
|
||||
+396
@@ -0,0 +1,396 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/user"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/lbrycrd"
|
||||
"github.com/lbryio/ytsync/v5/timing"
|
||||
|
||||
"github.com/docker/docker/api/types"
|
||||
"github.com/docker/docker/api/types/filters"
|
||||
"github.com/docker/docker/client"
|
||||
"github.com/mitchellh/go-ps"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func GetBlobsDir() string {
|
||||
blobsDir := os.Getenv("BLOBS_DIRECTORY")
|
||||
if blobsDir == "" {
|
||||
usr, err := user.Current()
|
||||
if err != nil {
|
||||
log.Error(err.Error())
|
||||
return ""
|
||||
}
|
||||
blobsDir = usr.HomeDir + "/.lbrynet/blobfiles/"
|
||||
}
|
||||
|
||||
return blobsDir
|
||||
}
|
||||
|
||||
func IsBlobReflectionOff() bool {
|
||||
return os.Getenv("REFLECT_BLOBS") == "false"
|
||||
}
|
||||
|
||||
func GetLBRYNetDir() string {
|
||||
lbrynetDir := os.Getenv("LBRYNET_DIR")
|
||||
if lbrynetDir == "" {
|
||||
usr, err := user.Current()
|
||||
if err != nil {
|
||||
log.Errorln(err.Error())
|
||||
return ""
|
||||
}
|
||||
return usr.HomeDir + "/.lbrynet/"
|
||||
}
|
||||
return lbrynetDir
|
||||
}
|
||||
|
||||
func GetLbryumDir() string {
|
||||
lbryumDir := os.Getenv("LBRYUM_DIR")
|
||||
if lbryumDir == "" {
|
||||
usr, err := user.Current()
|
||||
if err != nil {
|
||||
log.Errorln(err.Error())
|
||||
return ""
|
||||
}
|
||||
return usr.HomeDir + "/.lbryum/"
|
||||
}
|
||||
return lbryumDir + "/"
|
||||
}
|
||||
|
||||
const ALL = true
|
||||
const ONLINE = false
|
||||
|
||||
func GetLBRYNetContainer(all bool) (*types.Container, error) {
|
||||
return getDockerContainer("lbrynet", all)
|
||||
}
|
||||
|
||||
func getDockerContainer(name string, all bool) (*types.Container, error) {
|
||||
cli, err := client.NewEnvClient()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
filters := filters.NewArgs()
|
||||
filters.Add("name", name)
|
||||
containers, err := cli.ContainerList(context.Background(), types.ContainerListOptions{All: all, Filters: filters})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if len(containers) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
if len(containers) > 1 {
|
||||
return nil, errors.Err("more than one %s container found", name)
|
||||
}
|
||||
|
||||
return &containers[0], nil
|
||||
}
|
||||
|
||||
func IsUsingDocker() bool {
|
||||
useDocker, err := strconv.ParseBool(os.Getenv("LBRYNET_USE_DOCKER"))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return useDocker
|
||||
}
|
||||
|
||||
func IsRegTest() bool {
|
||||
usesRegtest, err := strconv.ParseBool(os.Getenv("REGTEST"))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return usesRegtest
|
||||
}
|
||||
|
||||
func GetLbrycrdClient(lbrycrdString string) (*lbrycrd.Client, error) {
|
||||
chainName := os.Getenv("CHAINNAME")
|
||||
chainParams, ok := lbrycrd.ChainParamsMap[chainName]
|
||||
if !ok {
|
||||
chainParams = lbrycrd.MainNetParams
|
||||
}
|
||||
var lbrycrdd *lbrycrd.Client
|
||||
var err error
|
||||
if lbrycrdString == "" {
|
||||
lbrycrdd, err = lbrycrd.NewWithDefaultURL(&chainParams)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
lbrycrdd, err = lbrycrd.New(lbrycrdString, &chainParams)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return lbrycrdd, nil
|
||||
}
|
||||
|
||||
func ShouldCleanOnStartup() bool {
|
||||
shouldClean, err := strconv.ParseBool(os.Getenv("CLEAN_ON_STARTUP"))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return shouldClean
|
||||
}
|
||||
|
||||
func IsLbrynetRunning() (bool, error) {
|
||||
if IsUsingDocker() {
|
||||
container, err := GetLBRYNetContainer(ONLINE)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return container != nil, nil
|
||||
}
|
||||
|
||||
processes, err := ps.Processes()
|
||||
if err != nil {
|
||||
return true, errors.Err(err)
|
||||
}
|
||||
var daemonProcessId = -1
|
||||
for _, p := range processes {
|
||||
if p.Executable() == "lbrynet" {
|
||||
daemonProcessId = p.Pid()
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
running := daemonProcessId != -1
|
||||
return running, nil
|
||||
}
|
||||
|
||||
func CleanForStartup() error {
|
||||
if !IsRegTest() {
|
||||
return errors.Err("never cleanup wallet outside of regtest and with caution. this should only be done in local testing and requires regtest to be on")
|
||||
}
|
||||
|
||||
running, err := IsLbrynetRunning()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if running {
|
||||
err := StopDaemon()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
err = CleanupLbrynet()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
lbrycrd, err := GetLbrycrdClient(os.Getenv("LBRYCRD_STRING"))
|
||||
if err != nil {
|
||||
return errors.Prefix("error getting lbrycrd client: ", err)
|
||||
}
|
||||
height, err := lbrycrd.GetBlockCount()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
const minBlocksForUTXO = 200
|
||||
if height < minBlocksForUTXO {
|
||||
//Start reg test with some credits
|
||||
txs, err := lbrycrd.Generate(uint32(minBlocksForUTXO) - uint32(height))
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
log.Debugf("REGTEST: Generated %d transactions to get some LBC!", len(txs))
|
||||
}
|
||||
|
||||
defaultWalletDir := GetDefaultWalletPath()
|
||||
_, err = os.Stat(defaultWalletDir)
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return errors.Err(os.Remove(defaultWalletDir))
|
||||
}
|
||||
|
||||
func CleanupLbrynet() error {
|
||||
//make sure lbrynet is off
|
||||
running, err := IsLbrynetRunning()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if running {
|
||||
return errors.Prefix("cannot cleanup lbrynet as the daemon is running", err)
|
||||
}
|
||||
lbrynetDir := GetLBRYNetDir()
|
||||
files, err := filepath.Glob(lbrynetDir + "lbrynet.sqlite*")
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
for _, f := range files {
|
||||
err = os.Remove(f)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
}
|
||||
blobsDir := GetBlobsDir()
|
||||
err = os.RemoveAll(blobsDir)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
err = os.Mkdir(blobsDir, 0777)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
lbryumDir := GetLbryumDir()
|
||||
ledger := "lbc_mainnet"
|
||||
if IsRegTest() {
|
||||
ledger = "lbc_regtest"
|
||||
}
|
||||
lbryumDir = lbryumDir + ledger
|
||||
|
||||
db, err := os.Stat(lbryumDir + "/blockchain.db")
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
dbSizeLimit := int64(2 * 1024 * 1024 * 1024)
|
||||
if db.Size() > dbSizeLimit {
|
||||
files, err := filepath.Glob(lbryumDir + "/blockchain.db*")
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
for _, f := range files {
|
||||
err = os.Remove(f)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func SleepUntilQuotaReset() {
|
||||
PST, _ := time.LoadLocation("America/Los_Angeles")
|
||||
t := time.Now().In(PST)
|
||||
n := time.Date(t.Year(), t.Month(), t.Day(), 24, 2, 0, 0, PST)
|
||||
d := n.Sub(t)
|
||||
if d < 0 {
|
||||
n = n.Add(24 * time.Hour)
|
||||
d = n.Sub(t)
|
||||
}
|
||||
log.Infof("gotta sleep %s until the quota resets", d.String())
|
||||
time.Sleep(d)
|
||||
}
|
||||
|
||||
func StartDaemon() error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("startDaemon").Add(time.Since(start))
|
||||
}(start)
|
||||
if IsUsingDocker() {
|
||||
return startDaemonViaDocker()
|
||||
}
|
||||
return startDaemonViaSystemd()
|
||||
}
|
||||
|
||||
func StopDaemon() error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("stopDaemon").Add(time.Since(start))
|
||||
}(start)
|
||||
if IsUsingDocker() {
|
||||
return stopDaemonViaDocker()
|
||||
}
|
||||
return stopDaemonViaSystemd()
|
||||
}
|
||||
|
||||
func startDaemonViaDocker() error {
|
||||
container, err := GetLBRYNetContainer(true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cli, err := client.NewEnvClient()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
err = cli.ContainerStart(context.Background(), container.ID, types.ContainerStartOptions{})
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func stopDaemonViaDocker() error {
|
||||
container, err := GetLBRYNetContainer(ONLINE)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cli, err := client.NewEnvClient()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
err = cli.ContainerStop(context.Background(), container.ID, nil)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func startDaemonViaSystemd() error {
|
||||
err := exec.Command("/usr/bin/sudo", "/bin/systemctl", "start", "lbrynet.service").Run()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func stopDaemonViaSystemd() error {
|
||||
err := exec.Command("/usr/bin/sudo", "/bin/systemctl", "stop", "lbrynet.service").Run()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func GetDefaultWalletPath() string {
|
||||
defaultWalletDir := os.Getenv("HOME") + "/.lbryum/wallets/default_wallet"
|
||||
if IsRegTest() {
|
||||
defaultWalletDir = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet"
|
||||
}
|
||||
|
||||
walletPath := os.Getenv("LBRYUM_DIR")
|
||||
if walletPath != "" {
|
||||
defaultWalletDir = walletPath + "/wallets/default_wallet"
|
||||
}
|
||||
return defaultWalletDir
|
||||
}
|
||||
func GetBlockchainDBPath() string {
|
||||
lbryumDir := os.Getenv("LBRYUM_DIR")
|
||||
if lbryumDir == "" {
|
||||
if IsRegTest() {
|
||||
lbryumDir = os.Getenv("HOME") + "/.lbryum_regtest"
|
||||
} else {
|
||||
lbryumDir = os.Getenv("HOME") + "/.lbryum"
|
||||
}
|
||||
}
|
||||
defaultDB := lbryumDir + "/lbc_mainnet/blockchain.db"
|
||||
if IsRegTest() {
|
||||
defaultDB = lbryumDir + "/lbc_regtest/blockchain.db"
|
||||
}
|
||||
return defaultDB
|
||||
}
|
||||
func GetBlockchainDirectoryName() string {
|
||||
ledger := "lbc_mainnet"
|
||||
if IsRegTest() {
|
||||
ledger = "lbc_regtest"
|
||||
}
|
||||
return ledger
|
||||
}
|
||||
+379
@@ -0,0 +1,379 @@
|
||||
package ytapi
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/ytsync/v5/shared"
|
||||
logUtils "github.com/lbryio/ytsync/v5/util"
|
||||
|
||||
"github.com/lbryio/ytsync/v5/downloader/ytdl"
|
||||
|
||||
"github.com/lbryio/ytsync/v5/downloader"
|
||||
"github.com/lbryio/ytsync/v5/ip_manager"
|
||||
"github.com/lbryio/ytsync/v5/sdk"
|
||||
"github.com/lbryio/ytsync/v5/sources"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type Video interface {
|
||||
Size() *int64
|
||||
ID() string
|
||||
IDAndNum() string
|
||||
PlaylistPosition() int
|
||||
PublishedAt() time.Time
|
||||
Sync(*jsonrpc.Client, sources.SyncParams, *sdk.SyncedVideo, bool, *sync.RWMutex) (*sources.SyncSummary, error)
|
||||
}
|
||||
|
||||
type byPublishedAt []Video
|
||||
|
||||
func (a byPublishedAt) Len() int { return len(a) }
|
||||
func (a byPublishedAt) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
|
||||
func (a byPublishedAt) Less(i, j int) bool { return a[i].PublishedAt().Before(a[j].PublishedAt()) }
|
||||
|
||||
type VideoParams struct {
|
||||
VideoDir string
|
||||
S3Config aws.Config
|
||||
Stopper *stop.Group
|
||||
IPPool *ip_manager.IPPool
|
||||
}
|
||||
|
||||
var mostRecentlyFailedChannel string // TODO: fix this hack!
|
||||
|
||||
func GetVideosToSync(config *sdk.APIConfig, channelID string, syncedVideos map[string]sdk.SyncedVideo, quickSync bool, maxVideos int, videoParams VideoParams, lastUploadedVideo string) ([]Video, error) {
|
||||
var videos []Video
|
||||
if quickSync && maxVideos > 50 {
|
||||
maxVideos = 50
|
||||
}
|
||||
allVideos, err := downloader.GetPlaylistVideoIDs(channelID, maxVideos, videoParams.Stopper.Ch(), videoParams.IPPool)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
videoIDs := make([]string, 0, len(allVideos))
|
||||
for _, video := range allVideos {
|
||||
sv, ok := syncedVideos[video]
|
||||
if ok && util.SubstringInSlice(sv.FailureReason, shared.NeverRetryFailures) {
|
||||
continue
|
||||
}
|
||||
videoIDs = append(videoIDs, video)
|
||||
}
|
||||
log.Infof("Got info for %d videos from youtube downloader", len(videoIDs))
|
||||
|
||||
playlistMap := make(map[string]int64)
|
||||
for i, videoID := range videoIDs {
|
||||
playlistMap[videoID] = int64(i)
|
||||
}
|
||||
//this will ensure that we at least try to sync the video that was marked as last uploaded video in the database.
|
||||
if lastUploadedVideo != "" {
|
||||
_, ok := playlistMap[lastUploadedVideo]
|
||||
if !ok {
|
||||
playlistMap[lastUploadedVideo] = 0
|
||||
videoIDs = append(videoIDs, lastUploadedVideo)
|
||||
}
|
||||
}
|
||||
|
||||
if len(videoIDs) < 1 {
|
||||
if channelID == mostRecentlyFailedChannel {
|
||||
return nil, errors.Err("playlist items not found")
|
||||
}
|
||||
mostRecentlyFailedChannel = channelID
|
||||
}
|
||||
|
||||
vids, err := getVideos(config, channelID, videoIDs, videoParams.Stopper.Ch(), videoParams.IPPool)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, item := range vids {
|
||||
positionInList := playlistMap[item.ID]
|
||||
videoToAdd, err := sources.NewYoutubeVideo(videoParams.VideoDir, item, positionInList, videoParams.S3Config, videoParams.Stopper, videoParams.IPPool)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
videos = append(videos, videoToAdd)
|
||||
}
|
||||
|
||||
for k, v := range syncedVideos {
|
||||
if !v.Published {
|
||||
continue
|
||||
}
|
||||
if _, ok := playlistMap[k]; !ok {
|
||||
videos = append(videos, sources.NewMockedVideo(videoParams.VideoDir, k, channelID, videoParams.S3Config, videoParams.Stopper, videoParams.IPPool))
|
||||
}
|
||||
}
|
||||
|
||||
sort.Sort(byPublishedAt(videos))
|
||||
|
||||
return videos, nil
|
||||
}
|
||||
|
||||
func CountVideosInChannel(channelID string) (int, error) {
|
||||
res, err := http.Get("https://socialblade.com/youtube/channel/" + channelID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
var line string
|
||||
|
||||
scanner := bufio.NewScanner(res.Body)
|
||||
for scanner.Scan() {
|
||||
if strings.Contains(scanner.Text(), "youtube-stats-header-uploads") {
|
||||
line = scanner.Text()
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if line == "" {
|
||||
return 0, errors.Err("upload count line not found")
|
||||
}
|
||||
|
||||
matches := regexp.MustCompile(">([0-9]+)<").FindStringSubmatch(line)
|
||||
if len(matches) != 2 {
|
||||
return 0, errors.Err("upload count not found with regex")
|
||||
}
|
||||
|
||||
num, err := strconv.Atoi(matches[1])
|
||||
if err != nil {
|
||||
return 0, errors.Err(err)
|
||||
}
|
||||
|
||||
return num, nil
|
||||
}
|
||||
|
||||
func ChannelInfo(channelID string) (*YoutubeStatsResponse, error) {
|
||||
url := "https://www.youtube.com/channel/" + channelID + "/about"
|
||||
|
||||
req, _ := http.NewRequest("GET", url, nil)
|
||||
|
||||
req.Header.Add("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36")
|
||||
req.Header.Add("Accept", "*/*")
|
||||
|
||||
res, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
pageBody := string(body)
|
||||
dataStartIndex := strings.Index(pageBody, "window[\"ytInitialData\"] = ") + 26
|
||||
dataEndIndex := strings.Index(pageBody, "]}}};") + 4
|
||||
|
||||
data := pageBody[dataStartIndex:dataEndIndex]
|
||||
var decodedResponse YoutubeStatsResponse
|
||||
err = json.Unmarshal([]byte(data), &decodedResponse)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
return &decodedResponse, nil
|
||||
}
|
||||
|
||||
func getVideos(config *sdk.APIConfig, channelID string, videoIDs []string, stopChan stop.Chan, ipPool *ip_manager.IPPool) ([]*ytdl.YtdlVideo, error) {
|
||||
var videos []*ytdl.YtdlVideo
|
||||
for _, videoID := range videoIDs {
|
||||
if len(videoID) < 5 {
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case <-stopChan:
|
||||
return videos, errors.Err("canceled by stopper")
|
||||
default:
|
||||
}
|
||||
|
||||
//ip, err := ipPool.GetIP(videoID)
|
||||
//if err != nil {
|
||||
// return nil, err
|
||||
//}
|
||||
//video, err := downloader.GetVideoInformation(videoID, &net.TCPAddr{IP: net.ParseIP(ip)})
|
||||
state, err := config.VideoState(videoID)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
if state == "published" {
|
||||
continue
|
||||
}
|
||||
video, err := downloader.GetVideoInformation(config, videoID, stopChan, nil, ipPool)
|
||||
if err != nil {
|
||||
errSDK := config.MarkVideoStatus(shared.VideoStatus{
|
||||
ChannelID: channelID,
|
||||
VideoID: videoID,
|
||||
Status: "failed",
|
||||
FailureReason: err.Error(),
|
||||
})
|
||||
logUtils.SendErrorToSlack(fmt.Sprintf("Skipping video (%s): %s", videoID, errors.FullTrace(err)))
|
||||
if errSDK != nil {
|
||||
return nil, errors.Err(errSDK)
|
||||
}
|
||||
} else {
|
||||
videos = append(videos, video)
|
||||
}
|
||||
}
|
||||
return videos, nil
|
||||
}
|
||||
|
||||
type YoutubeStatsResponse struct {
|
||||
Contents struct {
|
||||
TwoColumnBrowseResultsRenderer struct {
|
||||
Tabs []struct {
|
||||
TabRenderer struct {
|
||||
Title string `json:"title"`
|
||||
Selected bool `json:"selected"`
|
||||
Content struct {
|
||||
SectionListRenderer struct {
|
||||
Contents []struct {
|
||||
ItemSectionRenderer struct {
|
||||
Contents []struct {
|
||||
ChannelAboutFullMetadataRenderer struct {
|
||||
Description struct {
|
||||
SimpleText string `json:"simpleText"`
|
||||
} `json:"description"`
|
||||
ViewCountText struct {
|
||||
SimpleText string `json:"simpleText"`
|
||||
} `json:"viewCountText"`
|
||||
JoinedDateText struct {
|
||||
Runs []struct {
|
||||
Text string `json:"text"`
|
||||
} `json:"runs"`
|
||||
} `json:"joinedDateText"`
|
||||
CanonicalChannelURL string `json:"canonicalChannelUrl"`
|
||||
BypassBusinessEmailCaptcha bool `json:"bypassBusinessEmailCaptcha"`
|
||||
Title struct {
|
||||
SimpleText string `json:"simpleText"`
|
||||
} `json:"title"`
|
||||
Avatar struct {
|
||||
Thumbnails []struct {
|
||||
URL string `json:"url"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
} `json:"thumbnails"`
|
||||
} `json:"avatar"`
|
||||
ShowDescription bool `json:"showDescription"`
|
||||
DescriptionLabel struct {
|
||||
Runs []struct {
|
||||
Text string `json:"text"`
|
||||
} `json:"runs"`
|
||||
} `json:"descriptionLabel"`
|
||||
DetailsLabel struct {
|
||||
Runs []struct {
|
||||
Text string `json:"text"`
|
||||
} `json:"runs"`
|
||||
} `json:"detailsLabel"`
|
||||
ChannelID string `json:"channelId"`
|
||||
} `json:"channelAboutFullMetadataRenderer"`
|
||||
} `json:"contents"`
|
||||
} `json:"itemSectionRenderer"`
|
||||
} `json:"contents"`
|
||||
} `json:"sectionListRenderer"`
|
||||
} `json:"content"`
|
||||
} `json:"tabRenderer"`
|
||||
} `json:"tabs"`
|
||||
} `json:"twoColumnBrowseResultsRenderer"`
|
||||
} `json:"contents"`
|
||||
Header struct {
|
||||
C4TabbedHeaderRenderer struct {
|
||||
ChannelID string `json:"channelId"`
|
||||
Title string `json:"title"`
|
||||
Avatar struct {
|
||||
Thumbnails []struct {
|
||||
URL string `json:"url"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
} `json:"thumbnails"`
|
||||
} `json:"avatar"`
|
||||
Banner struct {
|
||||
Thumbnails []struct {
|
||||
URL string `json:"url"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
} `json:"thumbnails"`
|
||||
} `json:"banner"`
|
||||
VisitTracking struct {
|
||||
RemarketingPing string `json:"remarketingPing"`
|
||||
} `json:"visitTracking"`
|
||||
SubscriberCountText struct {
|
||||
SimpleText string `json:"simpleText"`
|
||||
} `json:"subscriberCountText"`
|
||||
} `json:"c4TabbedHeaderRenderer"`
|
||||
} `json:"header"`
|
||||
Metadata struct {
|
||||
ChannelMetadataRenderer struct {
|
||||
Title string `json:"title"`
|
||||
Description string `json:"description"`
|
||||
RssURL string `json:"rssUrl"`
|
||||
ChannelConversionURL string `json:"channelConversionUrl"`
|
||||
ExternalID string `json:"externalId"`
|
||||
Keywords string `json:"keywords"`
|
||||
OwnerUrls []string `json:"ownerUrls"`
|
||||
Avatar struct {
|
||||
Thumbnails []struct {
|
||||
URL string `json:"url"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
} `json:"thumbnails"`
|
||||
} `json:"avatar"`
|
||||
ChannelURL string `json:"channelUrl"`
|
||||
IsFamilySafe bool `json:"isFamilySafe"`
|
||||
VanityChannelURL string `json:"vanityChannelUrl"`
|
||||
} `json:"channelMetadataRenderer"`
|
||||
} `json:"metadata"`
|
||||
Topbar struct {
|
||||
DesktopTopbarRenderer struct {
|
||||
CountryCode string `json:"countryCode"`
|
||||
} `json:"desktopTopbarRenderer"`
|
||||
} `json:"topbar"`
|
||||
Microformat struct {
|
||||
MicroformatDataRenderer struct {
|
||||
URLCanonical string `json:"urlCanonical"`
|
||||
Title string `json:"title"`
|
||||
Description string `json:"description"`
|
||||
Thumbnail struct {
|
||||
Thumbnails []struct {
|
||||
URL string `json:"url"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
} `json:"thumbnails"`
|
||||
} `json:"thumbnail"`
|
||||
SiteName string `json:"siteName"`
|
||||
AppName string `json:"appName"`
|
||||
AndroidPackage string `json:"androidPackage"`
|
||||
IosAppStoreID string `json:"iosAppStoreId"`
|
||||
IosAppArguments string `json:"iosAppArguments"`
|
||||
OgType string `json:"ogType"`
|
||||
URLApplinksWeb string `json:"urlApplinksWeb"`
|
||||
URLApplinksIos string `json:"urlApplinksIos"`
|
||||
URLApplinksAndroid string `json:"urlApplinksAndroid"`
|
||||
URLTwitterIos string `json:"urlTwitterIos"`
|
||||
URLTwitterAndroid string `json:"urlTwitterAndroid"`
|
||||
TwitterCardType string `json:"twitterCardType"`
|
||||
TwitterSiteHandle string `json:"twitterSiteHandle"`
|
||||
SchemaDotOrgType string `json:"schemaDotOrgType"`
|
||||
Noindex bool `json:"noindex"`
|
||||
Unlisted bool `json:"unlisted"`
|
||||
FamilySafe bool `json:"familySafe"`
|
||||
Tags []string `json:"tags"`
|
||||
} `json:"microformatDataRenderer"`
|
||||
} `json:"microformat"`
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
package ytapi
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestChannelInfo(t *testing.T) {
|
||||
_, _, err := ChannelInfo("", "UCNQfQvFMPnInwsU_iGYArJQ")
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
Reference in new issue
Block a user