Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
053c618458 | ||
|
|
2cf3526c74 | ||
|
|
d7e194cb5c | ||
|
|
9a8b1922fe | ||
|
|
12d627bd35 | ||
|
|
9397a40c9f | ||
|
|
6a4093f3f8 | ||
|
|
bac2969295 | ||
|
|
5dfd8dee1b | ||
|
|
06c43c3f71 | ||
|
|
881d86799b | ||
|
|
f7e4366a67 | ||
|
|
5b5f7e1c68 | ||
|
|
fecf67118c | ||
|
|
beade71aa6 | ||
|
|
ca8ff505d4 | ||
|
|
f5f12e1560 | ||
|
|
c4693c221f | ||
|
|
692c796770 | ||
|
|
de798ade36 | ||
|
|
a93c2c4538 | ||
|
|
53e0b7c47b | ||
|
|
070dda8739 | ||
|
|
f773569920 | ||
|
|
ad6fa4d725 | ||
|
|
d4ca71a89d | ||
|
|
c53cf4c1b3 | ||
|
|
ad5a30da9e | ||
|
|
ecda80b02d | ||
|
|
24cf937e14 | ||
|
|
0b002c8228 | ||
|
|
a56166ee51 | ||
|
|
ddca850c17 | ||
|
|
31ce612e2e | ||
|
|
eb8900c66a | ||
|
|
67da4142d5 | ||
|
|
fb0e567caf | ||
|
|
7e83b17b69 | ||
|
|
7c02c5b92d | ||
|
|
2a33f44317 | ||
|
|
b7037900f8 | ||
|
|
5d230a6b54 | ||
|
|
f0280b51b4 | ||
|
|
775e4881cb | ||
|
|
ec9f46f552 | ||
|
|
6c6e93cefc | ||
|
|
eab9bcf1ff | ||
|
|
751bc84ce5 | ||
|
|
41fd9f6844 | ||
|
|
7f1906d58b | ||
|
|
5be3551abe | ||
|
|
fc18151d77 | ||
|
|
13543b20e9 | ||
|
|
4eba7200d8 | ||
|
|
cd11e82676 | ||
|
|
c7c220ecd3 | ||
|
|
4d56339756 | ||
|
|
140353097e | ||
|
|
a1caea4a28 | ||
|
|
b1e10e7b09 | ||
|
|
ad27425471 | ||
|
|
81e9378b9d | ||
|
|
64040ea67a | ||
|
|
d99e200178 | ||
|
|
357aebbcce | ||
|
|
0eef62b5fd | ||
|
|
a3dd3dc626 | ||
|
|
f942bf8025 | ||
|
|
a05864404d | ||
|
|
70ad891dfa | ||
|
|
8e61cde0a0 | ||
|
|
8600077caa | ||
|
|
e7fdd21bac | ||
|
|
0bb6b6d833 | ||
|
|
b59ef28267 | ||
|
|
e926a2c1f6 | ||
|
|
1369ed0b48 | ||
|
|
f39fc11697 | ||
|
|
bffc0823be | ||
|
|
edbb22fcf0 | ||
|
|
843303301a | ||
|
|
be7fd7ddd8 | ||
|
|
bfcfe5f36b | ||
|
|
4c4da93ef3 | ||
|
|
c2b5878daa | ||
|
|
749d5095c9 | ||
|
|
ba77b61ae0 | ||
|
|
07915ce697 | ||
|
|
4b2221ce5e | ||
|
|
1d5b69bfe6 | ||
|
|
4d915587e0 | ||
|
|
83f2c28c20 | ||
|
|
47e467057d | ||
|
|
1ae32d638b | ||
|
|
7907ee3579 | ||
|
|
414ed1c130 | ||
|
|
3bb17b09d3 | ||
|
|
046f46267e | ||
|
|
adc3ffe194 | ||
|
|
76301d40ba | ||
|
|
963c450730 | ||
|
|
6e7bb994d0 | ||
|
|
95890cbabf | ||
|
|
69c2a91ec7 | ||
|
|
fbe09a692c | ||
|
|
0ccc6e7979 | ||
|
|
3eb74cc8b5 | ||
|
|
9bfb1656fe | ||
|
|
5d40d24804 | ||
|
|
79d3aadda8 | ||
|
|
eba59a0806 | ||
|
|
3fd4909cd1 | ||
|
|
b59372c1d8 | ||
|
|
c9e713e9df | ||
|
|
bc72d2f1c4 | ||
|
|
df539e5d01 | ||
|
|
1b89104101 | ||
|
|
3dc30c3b98 | ||
|
|
259df8d257 | ||
|
|
5460224cd6 | ||
|
|
29479a73a7 | ||
|
|
625f339709 | ||
|
|
d6fcfd888a | ||
|
|
5f7d9d4940 | ||
|
|
826fcb78ea | ||
|
|
45e87c3007 | ||
|
|
cdee0d0de7 | ||
|
|
e30f21dc85 | ||
|
|
df88c7e952 |
No files matched your search
+4
-6
@@ -2,8 +2,7 @@ os: linux
|
||||
dist: bionic
|
||||
language: go
|
||||
go:
|
||||
- 1.12.7
|
||||
- master
|
||||
- 1.14.4
|
||||
|
||||
install: true
|
||||
|
||||
@@ -21,15 +20,14 @@ addons:
|
||||
packages:
|
||||
- ffmpeg
|
||||
- tree
|
||||
- python-pip
|
||||
- python3-pip
|
||||
|
||||
before_script:
|
||||
- sudo pip install -U youtube-dl
|
||||
- sudo add-apt-repository -y ppa:jonathonf/ffmpeg-4
|
||||
- sudo pip3 install -U youtube-dl
|
||||
- sudo add-apt-repository -y ppa:savoury1/ffmpeg4
|
||||
|
||||
env:
|
||||
global:
|
||||
- GO111MODULE=on
|
||||
#GITHUB_TOKEN
|
||||
- secure: "Ps3KocRP5xnM3/uA99CeYhDTVxRIuW7fGyrtqBeRWZW0cXzeA4XCTKxqcFbrPUPw67XkrBVgE58JDdWoQEJ7tm67PjMm/ltp5Evhx/QAJDh+YSofXyGDVpG1mrTZFI66R3NVVJLkSGALMkuWWXvfYZeU//AworJbyRoaIK/CVt5OP23i5N4tdd5UXc5dfLuYqnKRynyMmCkz9c3yEIQMXoPhG2hx7l7L2BeMJvcKmVhkSN7nQayjnrbUXGm/IRqrb88lvkyBevN5E3IB2V5IKEieIPZjbD/N0IfcnAt89Z96tgDhtIbx3ZvXm92lsvHA8buqQpG9d2AmSi6GKs64lQcnGeM5o0wER2JHWl1OSa1Nr/UAo5Xb/PM65Yt3yZE8AuMKHBmbfDSBzdkTXx58AeDzFUd3kMXD/fFjeQQWyXFlOss3ygH9SObl827Txmz9OJqZaxabs5Q3AP6m3EjKjz7zfLfrgpcxJM2WBiU1bN0ZxUgZkImy/CHk5gCZ7vhcnaLiDO4HZnzY/aRJwKYQPE5i0O2nHpIfovqkc0DFBA7U/7Cjin7e1E0UZvF3meLOxMqkfc6X7QTxqQpt2Tej6jlpdxw4CTLwGUhGkAw9IAPkUB3L0EbZ1/ksGhNvGDvUeSTq8hYdMAPmA+k9jS6653V4SQ+qBMy5++tbr5AeZQI="
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2017-2019 LBRY Inc.
|
||||
Copyright (c) 2017-2020 LBRY Inc.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the
|
||||
"Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish,
|
||||
|
||||
@@ -13,7 +13,7 @@ LDFLAGS = -ldflags "-X ${IMPORT_PATH}/meta.Version=${VERSION} -X ${IMPORT_PATH}/
|
||||
|
||||
|
||||
build:
|
||||
mkdir -p ${BIN_DIR} && CGO_ENABLED=0 go build ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/${BINARY} main.go
|
||||
mkdir -p ${BIN_DIR} && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/${BINARY} main.go
|
||||
|
||||
clean:
|
||||
if [ -f ${BIN_DIR}/${BINARY} ]; then rm ${BIN_DIR}/${BINARY}; fi
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/lbryio/ytsync/util"
|
||||
"github.com/lbryio/ytsync/v5/util"
|
||||
)
|
||||
|
||||
var dbHandle *db.SQL
|
||||
@@ -46,6 +46,7 @@ func reflectBlobs() error {
|
||||
if util.IsBlobReflectionOff() {
|
||||
return nil
|
||||
}
|
||||
logrus.Infoln("reflecting blobs...")
|
||||
//make sure lbrynet is off
|
||||
running, err := util.IsLbrynetRunning()
|
||||
if err != nil {
|
||||
@@ -72,12 +73,10 @@ func reflectBlobs() error {
|
||||
return errors.Err(err)
|
||||
}
|
||||
}
|
||||
st := store.NewDBBackedS3Store(
|
||||
store.NewS3BlobStore(config.AwsID, config.AwsSecret, config.BucketRegion, config.BucketName),
|
||||
dbHandle)
|
||||
st := store.NewDBBackedStore(store.NewS3BlobStore(config.AwsID, config.AwsSecret, config.BucketRegion, config.BucketName), dbHandle)
|
||||
|
||||
uploadWorkers := 10
|
||||
uploader := reflector.NewUploader(dbHandle, st, uploadWorkers, false)
|
||||
uploader := reflector.NewUploader(dbHandle, st, uploadWorkers, false, false)
|
||||
usr, err := user.Current()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
|
||||
@@ -0,0 +1,368 @@
|
||||
package downloader
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/lbryio/ytsync/v5/downloader/ytdl"
|
||||
"github.com/lbryio/ytsync/v5/ip_manager"
|
||||
"github.com/lbryio/ytsync/v5/sdk"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func GetPlaylistVideoIDs(channelName string, maxVideos int, stopChan stop.Chan, pool *ip_manager.IPPool) ([]string, error) {
|
||||
args := []string{"--skip-download", "https://www.youtube.com/channel/" + channelName, "--get-id", "--flat-playlist", "--cookies", "cookies.txt"}
|
||||
ids, err := run(channelName, args, stopChan, pool, true)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
videoIDs := make([]string, 0, maxVideos)
|
||||
for i, v := range ids {
|
||||
logrus.Debugf("%d - video id %s", i, v)
|
||||
if i >= maxVideos {
|
||||
break
|
||||
}
|
||||
videoIDs = append(videoIDs, v)
|
||||
}
|
||||
return videoIDs, nil
|
||||
}
|
||||
|
||||
const releaseTimeFormat = "2006-01-02, 15:04:05 (MST)"
|
||||
|
||||
func GetVideoInformation(config *sdk.APIConfig, videoID string, stopChan stop.Chan, ip *net.TCPAddr, pool *ip_manager.IPPool) (*ytdl.YtdlVideo, error) {
|
||||
args := []string{"--skip-download", "--print-json", "https://www.youtube.com/watch?v=" + videoID, "--cookies", "cookies.txt"}
|
||||
results, err := run(videoID, args, stopChan, pool, false)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
var video *ytdl.YtdlVideo
|
||||
err = json.Unmarshal([]byte(results[0]), &video)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
// now get an accurate time
|
||||
const maxTries = 5
|
||||
tries := 0
|
||||
GetTime:
|
||||
tries++
|
||||
t, err := getUploadTime(config, videoID, ip, video.UploadDate)
|
||||
if err != nil {
|
||||
//slack(":warning: Upload time error: %v", err)
|
||||
if tries <= maxTries && (errors.Is(err, errNotScraped) || errors.Is(err, errUploadTimeEmpty) || errors.Is(err, errStatusParse) || errors.Is(err, errConnectionIssue)) {
|
||||
err := triggerScrape(videoID, ip)
|
||||
if err == nil {
|
||||
time.Sleep(2 * time.Second) // let them scrape it
|
||||
goto GetTime
|
||||
} else {
|
||||
//slack("triggering scrape returned error: %v", err)
|
||||
}
|
||||
} else if !errors.Is(err, errNotScraped) && !errors.Is(err, errUploadTimeEmpty) {
|
||||
//slack(":warning: Error while trying to get accurate upload time for %s: %v", videoID, err)
|
||||
if t == "" {
|
||||
return nil, errors.Err(err)
|
||||
} else {
|
||||
t = "" //TODO: get rid of the other piece below?
|
||||
}
|
||||
}
|
||||
// do fallback below
|
||||
}
|
||||
//slack("After all that, upload time for %s is %s", videoID, t)
|
||||
|
||||
if t != "" {
|
||||
parsed, err := time.Parse("2006-01-02, 15:04:05 (MST)", t) // this will probably be UTC, but Go's timezone parsing is fucked up. it ignores the timezone in the date
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
//slack(":exclamation: Got an accurate time for %s", videoID)
|
||||
video.UploadDateForReal = parsed
|
||||
} else { //TODO: this is the piece that isn't needed!
|
||||
slack(":warning: Could not get accurate time for %s. Falling back to time from upload ytdl: %s.", videoID, video.UploadDate)
|
||||
// fall back to UploadDate from youtube-dl
|
||||
video.UploadDateForReal, err = time.Parse("20060102", video.UploadDate)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return video, nil
|
||||
}
|
||||
|
||||
var errNotScraped = errors.Base("not yet scraped by caa.iti.gr")
|
||||
var errUploadTimeEmpty = errors.Base("upload time is empty")
|
||||
var errStatusParse = errors.Base("could not parse status, got number, need string")
|
||||
var errConnectionIssue = errors.Base("there was a connection issue with the api")
|
||||
|
||||
func slack(format string, a ...interface{}) {
|
||||
fmt.Printf(format+"\n", a...)
|
||||
util.SendToSlack(format, a...)
|
||||
}
|
||||
|
||||
func triggerScrape(videoID string, ip *net.TCPAddr) error {
|
||||
//slack("Triggering scrape for %s", videoID)
|
||||
u, err := url.Parse("https://caa.iti.gr/verify_videoV3")
|
||||
q := u.Query()
|
||||
q.Set("twtimeline", "0")
|
||||
q.Set("url", "https://www.youtube.com/watch?v="+videoID)
|
||||
u.RawQuery = q.Encode()
|
||||
//slack("GET %s", u.String())
|
||||
|
||||
client := getClient(ip)
|
||||
req, err := http.NewRequest(http.MethodGet, u.String(), nil)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36")
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
var response struct {
|
||||
Message string `json:"message"`
|
||||
Status string `json:"status"`
|
||||
VideoURL string `json:"video_url"`
|
||||
}
|
||||
err = json.NewDecoder(res.Body).Decode(&response)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "cannot unmarshal number") {
|
||||
return errors.Err(errStatusParse)
|
||||
}
|
||||
if strings.Contains(err.Error(), "no route to host") {
|
||||
return errors.Err(errConnectionIssue)
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
switch response.Status {
|
||||
case "removed_video":
|
||||
return errors.Err("video previously removed from service")
|
||||
case "no_video":
|
||||
return errors.Err("they say 'video cannot be found'. wtf?")
|
||||
default:
|
||||
spew.Dump(response)
|
||||
}
|
||||
|
||||
return nil
|
||||
//https://caa.iti.gr/caa/api/v4/videos/reports/h-tuxHS5lSM
|
||||
}
|
||||
|
||||
func getUploadTime(config *sdk.APIConfig, videoID string, ip *net.TCPAddr, uploadDate string) (string, error) {
|
||||
//slack("Getting upload time for %s", videoID)
|
||||
release, err := config.GetReleasedDate(videoID)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
ytdlUploadDate, err := time.Parse("20060102", uploadDate)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
if release != nil {
|
||||
//const sqlTimeFormat = "2006-01-02 15:04:05"
|
||||
sqlTime, err := time.ParseInLocation(time.RFC3339, release.ReleaseTime, time.UTC)
|
||||
if err == nil {
|
||||
if sqlTime.Day() != ytdlUploadDate.Day() {
|
||||
logrus.Infof("upload day from APIs differs from the ytdl one by more than 1 day.")
|
||||
} else {
|
||||
return sqlTime.Format(releaseTimeFormat), nil
|
||||
}
|
||||
} else {
|
||||
logrus.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
if time.Now().AddDate(0, 0, -3).After(ytdlUploadDate) {
|
||||
return ytdlUploadDate.Format(releaseTimeFormat), nil
|
||||
}
|
||||
client := getClient(ip)
|
||||
req, err := http.NewRequest(http.MethodGet, "https://caa.iti.gr/get_verificationV3?url=https://www.youtube.com/watch?v="+videoID, nil)
|
||||
if err != nil {
|
||||
return ytdlUploadDate.Format(releaseTimeFormat), errors.Err(err)
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36")
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return ytdlUploadDate.Format(releaseTimeFormat), errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
var uploadTime struct {
|
||||
Time string `json:"video_upload_time"`
|
||||
Message string `json:"message"`
|
||||
Status string `json:"status"`
|
||||
}
|
||||
err = json.NewDecoder(res.Body).Decode(&uploadTime)
|
||||
if err != nil {
|
||||
return ytdlUploadDate.Format(releaseTimeFormat), errors.Err(err)
|
||||
}
|
||||
|
||||
if uploadTime.Status == "ERROR1" {
|
||||
return ytdlUploadDate.Format(releaseTimeFormat), errNotScraped
|
||||
}
|
||||
|
||||
if uploadTime.Status == "" && strings.HasPrefix(uploadTime.Message, "CANNOT_RETRIEVE_REPORT_FOR_VIDEO_") {
|
||||
return ytdlUploadDate.Format(releaseTimeFormat), errors.Err("cannot retrieve report for video")
|
||||
}
|
||||
|
||||
if uploadTime.Time == "" {
|
||||
return ytdlUploadDate.Format(releaseTimeFormat), errUploadTimeEmpty
|
||||
}
|
||||
|
||||
return uploadTime.Time, nil
|
||||
}
|
||||
|
||||
func getClient(ip *net.TCPAddr) *http.Client {
|
||||
if ip == nil {
|
||||
return http.DefaultClient
|
||||
}
|
||||
|
||||
return &http.Client{
|
||||
Transport: &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
DialContext: (&net.Dialer{
|
||||
LocalAddr: ip,
|
||||
Timeout: 30 * time.Second,
|
||||
KeepAlive: 30 * time.Second,
|
||||
}).DialContext,
|
||||
MaxIdleConns: 100,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
TLSHandshakeTimeout: 10 * time.Second,
|
||||
ExpectContinueTimeout: 1 * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
googleBotUA = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
|
||||
chromeUA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36"
|
||||
maxAttempts = 3
|
||||
extractionError = "YouTube said: Unable to extract video data"
|
||||
throttledError = "HTTP Error 429"
|
||||
AlternateThrottledError = "returned non-zero exit status 8"
|
||||
youtubeDlError = "exit status 1"
|
||||
)
|
||||
|
||||
func run(use string, args []string, stopChan stop.Chan, pool *ip_manager.IPPool, dlc bool) ([]string, error) {
|
||||
var useragent []string
|
||||
var lastError error
|
||||
for attempts := 0; attempts < maxAttempts; attempts++ {
|
||||
sourceAddress, err := getIPFromPool(use, stopChan, pool)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
argsForCommand := append(args, "--source-address", sourceAddress)
|
||||
argsForCommand = append(argsForCommand, useragent...)
|
||||
binary := "youtube-dl"
|
||||
if dlc {
|
||||
binary = "youtube-dlc"
|
||||
}
|
||||
cmd := exec.Command(binary, argsForCommand...)
|
||||
|
||||
res, err := runCmd(cmd, stopChan)
|
||||
pool.ReleaseIP(sourceAddress)
|
||||
if err == nil {
|
||||
return res, nil
|
||||
}
|
||||
lastError = err
|
||||
if strings.Contains(err.Error(), youtubeDlError) {
|
||||
if strings.Contains(err.Error(), extractionError) {
|
||||
logrus.Warnf("known extraction error: %s", errors.FullTrace(err))
|
||||
useragent = nextUA(useragent)
|
||||
}
|
||||
if strings.Contains(err.Error(), throttledError) || strings.Contains(err.Error(), AlternateThrottledError) {
|
||||
pool.SetThrottled(sourceAddress)
|
||||
//we don't want throttle errors to count toward the max retries
|
||||
attempts--
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil, lastError
|
||||
}
|
||||
|
||||
func nextUA(current []string) []string {
|
||||
if len(current) == 0 {
|
||||
return []string{"--user-agent", googleBotUA}
|
||||
}
|
||||
return []string{"--user-agent", chromeUA}
|
||||
}
|
||||
|
||||
func runCmd(cmd *exec.Cmd, stopChan stop.Chan) ([]string, error) {
|
||||
logrus.Infof("running youtube-dl(c) cmd: %s", strings.Join(cmd.Args, " "))
|
||||
var err error
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
err = cmd.Start()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
outLog, err := ioutil.ReadAll(stdout)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
errorLog, err := ioutil.ReadAll(stderr)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- cmd.Wait()
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-stopChan:
|
||||
err := cmd.Process.Kill()
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("failed to kill command after stopper cancellation", err)
|
||||
}
|
||||
return nil, errors.Err("interrupted by user")
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("youtube-dl(c) "+strings.Join(cmd.Args, " ")+" ["+string(errorLog)+"]", err)
|
||||
}
|
||||
return strings.Split(strings.Replace(string(outLog), "\r\n", "\n", -1), "\n"), nil
|
||||
}
|
||||
}
|
||||
|
||||
func getIPFromPool(use string, stopChan stop.Chan, pool *ip_manager.IPPool) (sourceAddress string, err error) {
|
||||
for {
|
||||
sourceAddress, err = pool.GetIP(use)
|
||||
if err != nil {
|
||||
if errors.Is(err, ip_manager.ErrAllThrottled) {
|
||||
select {
|
||||
case <-stopChan:
|
||||
return "", errors.Err("interrupted by user")
|
||||
|
||||
default:
|
||||
time.Sleep(ip_manager.IPCooldownPeriod)
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package downloader
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/lbryio/ytsync/v5/sdk"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestGetPlaylistVideoIDs(t *testing.T) {
|
||||
videoIDs, err := GetPlaylistVideoIDs("UCJ0-OtVpF0wOKEqT2Z1HEtA", 50, nil, nil)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
for _, id := range videoIDs {
|
||||
println(id)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetVideoInformation(t *testing.T) {
|
||||
video, err := GetVideoInformation(nil, "zj7pXM9gE5M", nil, nil, nil)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
if video != nil {
|
||||
logrus.Info(video.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func Test_getUploadTime(t *testing.T) {
|
||||
configs := sdk.APIConfig{
|
||||
YoutubeAPIKey: "",
|
||||
ApiURL: "https://api.lbry.com",
|
||||
ApiToken: "Ht4NETrL5oWKyAaZkuSV68BKhtXkiLh5",
|
||||
HostName: "test",
|
||||
}
|
||||
got, err := getUploadTime(&configs, "kDGOHNpRjzc", nil, "20060102")
|
||||
assert.NoError(t, err)
|
||||
t.Log(got)
|
||||
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
package ytdl
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
type YtdlVideo struct {
|
||||
UploadDate string `json:"upload_date"`
|
||||
UploadDateForReal time.Time // you need to manually set this since the value in the API doesn't include the time
|
||||
Extractor string `json:"extractor"`
|
||||
Series interface{} `json:"series"`
|
||||
Format string `json:"format"`
|
||||
Vbr interface{} `json:"vbr"`
|
||||
Chapters interface{} `json:"chapters"`
|
||||
Height int `json:"height"`
|
||||
LikeCount interface{} `json:"like_count"`
|
||||
Duration int `json:"duration"`
|
||||
Fulltitle string `json:"fulltitle"`
|
||||
PlaylistIndex interface{} `json:"playlist_index"`
|
||||
Album interface{} `json:"album"`
|
||||
ViewCount int `json:"view_count"`
|
||||
Playlist interface{} `json:"playlist"`
|
||||
Title string `json:"title"`
|
||||
Filename string `json:"_filename"`
|
||||
Creator interface{} `json:"creator"`
|
||||
Ext string `json:"ext"`
|
||||
ID string `json:"id"`
|
||||
DislikeCount interface{} `json:"dislike_count"`
|
||||
AverageRating float64 `json:"average_rating"`
|
||||
Abr int `json:"abr"`
|
||||
UploaderURL string `json:"uploader_url"`
|
||||
Categories []string `json:"categories"`
|
||||
Fps float64 `json:"fps"`
|
||||
StretchedRatio interface{} `json:"stretched_ratio"`
|
||||
SeasonNumber interface{} `json:"season_number"`
|
||||
Annotations interface{} `json:"annotations"`
|
||||
WebpageURLBasename string `json:"webpage_url_basename"`
|
||||
Acodec string `json:"acodec"`
|
||||
DisplayID string `json:"display_id"`
|
||||
//RequestedFormats []RequestedFormat `json:"requested_formats"`
|
||||
//AutomaticCaptions struct{} `json:"automatic_captions"`
|
||||
Description string `json:"description"`
|
||||
Tags []string `json:"tags"`
|
||||
Track interface{} `json:"track"`
|
||||
RequestedSubtitles interface{} `json:"requested_subtitles"`
|
||||
StartTime interface{} `json:"start_time"`
|
||||
Uploader string `json:"uploader"`
|
||||
ExtractorKey string `json:"extractor_key"`
|
||||
FormatID string `json:"format_id"`
|
||||
EpisodeNumber interface{} `json:"episode_number"`
|
||||
UploaderID string `json:"uploader_id"`
|
||||
//Subtitles struct{} `json:"subtitles"`
|
||||
ReleaseYear interface{} `json:"release_year"`
|
||||
Thumbnails []Thumbnail `json:"thumbnails"`
|
||||
License interface{} `json:"license"`
|
||||
Artist interface{} `json:"artist"`
|
||||
AgeLimit int `json:"age_limit"`
|
||||
ReleaseDate interface{} `json:"release_date"`
|
||||
AltTitle interface{} `json:"alt_title"`
|
||||
Thumbnail string `json:"thumbnail"`
|
||||
ChannelID string `json:"channel_id"`
|
||||
IsLive interface{} `json:"is_live"`
|
||||
Width int `json:"width"`
|
||||
EndTime interface{} `json:"end_time"`
|
||||
WebpageURL string `json:"webpage_url"`
|
||||
//Formats []Format `json:"formats"`
|
||||
ChannelURL string `json:"channel_url"`
|
||||
Resolution interface{} `json:"resolution"`
|
||||
Vcodec string `json:"vcodec"`
|
||||
}
|
||||
|
||||
type RequestedFormat struct {
|
||||
Asr interface{} `json:"asr"`
|
||||
Tbr float64 `json:"tbr"`
|
||||
Container string `json:"container"`
|
||||
Language interface{} `json:"language"`
|
||||
Format string `json:"format"`
|
||||
URL string `json:"url"`
|
||||
Vcodec string `json:"vcodec"`
|
||||
FormatNote string `json:"format_note"`
|
||||
Height int `json:"height"`
|
||||
Width int `json:"width"`
|
||||
Ext string `json:"ext"`
|
||||
FragmentBaseURL string `json:"fragment_base_url"`
|
||||
Filesize interface{} `json:"filesize"`
|
||||
Fps float64 `json:"fps"`
|
||||
ManifestURL string `json:"manifest_url"`
|
||||
Protocol string `json:"protocol"`
|
||||
FormatID string `json:"format_id"`
|
||||
HTTPHeaders struct {
|
||||
AcceptCharset string `json:"Accept-Charset"`
|
||||
AcceptLanguage string `json:"Accept-Language"`
|
||||
AcceptEncoding string `json:"Accept-Encoding"`
|
||||
Accept string `json:"Accept"`
|
||||
UserAgent string `json:"User-Agent"`
|
||||
} `json:"http_headers"`
|
||||
Fragments []struct {
|
||||
Path string `json:"path"`
|
||||
Duration float64 `json:"duration,omitempty"`
|
||||
} `json:"fragments"`
|
||||
Acodec string `json:"acodec"`
|
||||
Abr int `json:"abr,omitempty"`
|
||||
}
|
||||
|
||||
type Format struct {
|
||||
Asr int `json:"asr"`
|
||||
Tbr float64 `json:"tbr"`
|
||||
Protocol string `json:"protocol"`
|
||||
Format string `json:"format"`
|
||||
FormatNote string `json:"format_note"`
|
||||
Height interface{} `json:"height"`
|
||||
ManifestURL string `json:"manifest_url,omitempty"`
|
||||
FormatID string `json:"format_id"`
|
||||
Container string `json:"container,omitempty"`
|
||||
Language interface{} `json:"language,omitempty"`
|
||||
HTTPHeaders HTTPHeaders `json:"http_headers"`
|
||||
URL string `json:"url"`
|
||||
Vcodec string `json:"vcodec"`
|
||||
Abr int `json:"abr,omitempty"`
|
||||
Width interface{} `json:"width"`
|
||||
Ext string `json:"ext"`
|
||||
FragmentBaseURL string `json:"fragment_base_url,omitempty"`
|
||||
Filesize interface{} `json:"filesize"`
|
||||
Fps float64 `json:"fps"`
|
||||
Fragments []struct {
|
||||
Path string `json:"path"`
|
||||
Duration float64 `json:"duration,omitempty"`
|
||||
} `json:"fragments,omitempty"`
|
||||
Acodec string `json:"acodec"`
|
||||
PlayerURL interface{} `json:"player_url,omitempty"`
|
||||
}
|
||||
|
||||
type Thumbnail struct {
|
||||
URL string `json:"url"`
|
||||
Width int `json:"width"`
|
||||
Resolution string `json:"resolution"`
|
||||
ID string `json:"id"`
|
||||
Height int `json:"height"`
|
||||
}
|
||||
|
||||
type HTTPHeaders struct {
|
||||
AcceptCharset string `json:"Accept-Charset"`
|
||||
AcceptLanguage string `json:"Accept-Language"`
|
||||
AcceptEncoding string `json:"Accept-Encoding"`
|
||||
Accept string `json:"Accept"`
|
||||
UserAgent string `json:"User-Agent"`
|
||||
}
|
||||
+2
-2
@@ -19,6 +19,6 @@ mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCER"
|
||||
ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(2,"youtubertoken")'
|
||||
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN"
|
||||
#Add their youtube channel to be synced
|
||||
ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers)
|
||||
VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@beamertest','UCNQfQvFMPnInwsU_iGYArJQ','BeamerAtLBRY','queued','2019-08-01 00:00:00','sync',1,0)"
|
||||
ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers,should_sync,redeemable,total_views,reviewed,last_uploaded_video,length_limit,size_limit,reward_amount,reward_expiration)
|
||||
VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@test"$(date +%s)"','UCNQfQvFMPnInwsU_iGYArJQ','BeamerAtLBRY','queued','2019-08-01 00:00:00','sync',10,10,1,1,10000,1,'7bBV2Z-9wpo',60,2048,0,'2019-08-01 00:00:00')"
|
||||
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTCHANNEL"
|
||||
@@ -21,7 +21,7 @@ services:
|
||||
## Wallet Server ##
|
||||
###################
|
||||
walletserver:
|
||||
image: lbry/wallet-server:v0.50.1
|
||||
image: lbry/wallet-server:v0.83.0
|
||||
restart: always
|
||||
environment:
|
||||
- DB_DIRECTORY=/database
|
||||
@@ -47,7 +47,7 @@ services:
|
||||
## Lbrynet ##
|
||||
#############
|
||||
lbrynet:
|
||||
image: lbry/lbrynet:v0.48.1
|
||||
image: lbry/lbrynet:v0.83.0
|
||||
restart: always
|
||||
ports:
|
||||
- "15100:5279"
|
||||
|
||||
+2
-1
@@ -26,8 +26,9 @@ export REGTEST=true
|
||||
# Local settings
|
||||
export BLOBS_DIRECTORY="$(pwd)/e2e/blobsfiles"
|
||||
export LBRYNET_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbrynet/"
|
||||
export LBRYNET_WALLETS_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbryum"
|
||||
export LBRYUM_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbryum"
|
||||
export TMP_DIR="/var/tmp"
|
||||
export CHAINNAME="lbrycrd_regtest"
|
||||
export UID
|
||||
|
||||
cd ./e2e
|
||||
|
||||
@@ -8,7 +8,7 @@ services:
|
||||
## Lbrynet ##
|
||||
#############
|
||||
lbrynet:
|
||||
image: lbry/lbrynet:v0.48.1
|
||||
image: lbry/lbrynet:v0.83.0
|
||||
restart: "no"
|
||||
networks:
|
||||
lbry-network:
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/lbryio/ytsync/util"
|
||||
"github.com/lbryio/ytsync/v5/util"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
@@ -8,7 +8,7 @@ services:
|
||||
## Wallet Server ##
|
||||
###################
|
||||
walletserver:
|
||||
image: lbry/wallet-server:v0.38.6
|
||||
image: lbry/wallet-server:v0.73.1
|
||||
restart: always
|
||||
networks:
|
||||
lbry-network:
|
||||
|
||||
@@ -1,37 +1,40 @@
|
||||
module github.com/lbryio/ytsync
|
||||
module github.com/lbryio/ytsync/v5
|
||||
|
||||
replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.46.3 // indirect
|
||||
github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61
|
||||
github.com/Microsoft/go-winio v0.4.14 // indirect
|
||||
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a
|
||||
github.com/abadojack/whatlanggo v1.0.1
|
||||
github.com/asaskevich/govalidator v0.0.0-20200819183940-29e1ff8eb0bb
|
||||
github.com/aws/aws-sdk-go v1.25.9
|
||||
github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1
|
||||
github.com/docker/distribution v2.7.1+incompatible // indirect
|
||||
github.com/docker/docker v1.13.1
|
||||
github.com/docker/go-connections v0.4.0 // indirect
|
||||
github.com/docker/go-units v0.4.0 // indirect
|
||||
github.com/go-sql-driver/mysql v1.4.1 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc // indirect
|
||||
github.com/hashicorp/go-immutable-radix v1.1.0 // indirect
|
||||
github.com/hashicorp/go-sockaddr v1.0.2 // indirect
|
||||
github.com/hashicorp/golang-lru v0.5.3 // indirect
|
||||
github.com/hashicorp/memberlist v0.1.5 // indirect
|
||||
github.com/hashicorp/serf v0.8.5 // indirect
|
||||
github.com/lbryio/lbry.go/v2 v2.4.1-0.20191206194319-06764c3d007e
|
||||
github.com/lbryio/reflector.go v1.0.6-0.20190828131602-ce3d4403dbc6
|
||||
github.com/kr/pretty v0.2.1 // indirect
|
||||
github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128
|
||||
github.com/lbryio/reflector.go v1.1.3-0.20201006141750-a80e0f5b0dc6
|
||||
github.com/lbryio/types v0.0.0-20200605192618-366870b2862d // indirect
|
||||
github.com/miekg/dns v1.1.22 // indirect
|
||||
github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b
|
||||
github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
|
||||
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 // indirect
|
||||
github.com/prometheus/client_golang v0.9.2
|
||||
github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5
|
||||
github.com/sirupsen/logrus v1.4.2
|
||||
github.com/smartystreets/goconvey v1.6.4 // indirect
|
||||
github.com/spf13/cobra v0.0.5
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
go.opencensus.io v0.22.1 // indirect
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect
|
||||
google.golang.org/api v0.11.0
|
||||
github.com/stretchr/testify v1.4.0
|
||||
google.golang.org/appengine v1.6.5 // indirect
|
||||
gopkg.in/ini.v1 v1.60.2 // indirect
|
||||
gopkg.in/vansante/go-ffprobe.v2 v2.0.2
|
||||
gotest.tools v2.2.0+incompatible
|
||||
)
|
||||
|
||||
go 1.13
|
||||
@@ -1,67 +1,67 @@
|
||||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg=
|
||||
cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
|
||||
cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
|
||||
cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
|
||||
cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
|
||||
cloud.google.com/go v0.46.3 h1:AVXDdKsrtX33oR9fbCMu/+c1o8Ofjq6Ku/MInaLVg5Y=
|
||||
cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
|
||||
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
|
||||
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
|
||||
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
|
||||
cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo=
|
||||
dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU=
|
||||
dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU=
|
||||
dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4=
|
||||
dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU=
|
||||
git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
|
||||
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 h1:N5Vqww5QISEHsWHOWDEx4PzdIay3Cg0Jp7zItq2ZAro=
|
||||
github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61/go.mod h1:GnKXcK+7DYNy/8w2Ex//Uql4IgfaU82Cd5rWKb7ah00=
|
||||
github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
|
||||
github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU=
|
||||
github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
|
||||
github.com/aead/siphash v1.0.1 h1:FwHfE/T45KPKYuuSAKyyvE+oPWcaQ+CUmFW0bPlM+kg=
|
||||
github.com/StackExchange/wmi v0.0.0-20170410192909-ea383cf3ba6e/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
|
||||
github.com/abadojack/whatlanggo v1.0.1 h1:19N6YogDnf71CTHm3Mp2qhYfkRdyvbgwWdd2EPxJRG4=
|
||||
github.com/abadojack/whatlanggo v1.0.1/go.mod h1:66WiQbSbJBIlOZMsvbKe5m6pzQovxCH9B/K8tQB2uoc=
|
||||
github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
|
||||
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
|
||||
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
|
||||
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
|
||||
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
|
||||
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM=
|
||||
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg=
|
||||
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
|
||||
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf h1:eg0MeVzsP1G42dRafH3vf+al2vQIJU0YHX+1Tw87oco=
|
||||
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
|
||||
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA=
|
||||
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
|
||||
github.com/asaskevich/govalidator v0.0.0-20200819183940-29e1ff8eb0bb h1:kvlW1qyM1aU3xeyeIVTU2jx5fSvjKpsU3aXvuaCMg3Q=
|
||||
github.com/asaskevich/govalidator v0.0.0-20200819183940-29e1ff8eb0bb/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
|
||||
github.com/aws/aws-sdk-go v1.16.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
||||
github.com/aws/aws-sdk-go v1.25.9 h1:WtVzerf5wSgPwlTTwl+ktCq/0GCS5MI9ZlLIcjsTr+Q=
|
||||
github.com/aws/aws-sdk-go v1.25.9/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
|
||||
github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32 h1:qkOC5Gd33k54tobS36cXdAzJbeHaduLtnLQQwNoIi78=
|
||||
github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8=
|
||||
github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
|
||||
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo=
|
||||
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
|
||||
github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803 h1:j3AgPKKZtZStM2nyhrDSLSYgT7YHrZKdSkq1OYeLjvM=
|
||||
github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
|
||||
github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d h1:yJzD/yFppdVCf6ApMkVy8cUxV0XrxdP9rVf6D87/Mng=
|
||||
github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
|
||||
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd h1:R/opQEbFEy9JGkIguV40SvRY1uliPX8ifOvi6ICsFCw=
|
||||
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg=
|
||||
github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd h1:qdGvebPBDuYDPGi1WCPjy1tGyMpmDK8IEapSsszn7HE=
|
||||
github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY=
|
||||
github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723 h1:ZA/jbKoGcVAnER6pCHPEkGdZOV7U1oLUedErBHCUMs0=
|
||||
github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
|
||||
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3SkEwmHoWBmX1DNXhXZqlTpq6s4tyJGc=
|
||||
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
|
||||
github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
|
||||
github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 h1:o64h9XF42kVEUuhuer2ehqrlX8rZmvQSU0+Vpj1rF6Q=
|
||||
github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61/go.mod h1:Rp8e0DCtEKwXFOC6JPJQVTz8tuGoGvw6Xfexggh/ed0=
|
||||
github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
|
||||
github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
|
||||
github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
|
||||
github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
|
||||
github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
|
||||
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
|
||||
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
|
||||
github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
|
||||
github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
|
||||
github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
|
||||
github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
|
||||
github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
|
||||
github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo=
|
||||
@@ -70,76 +70,90 @@ github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKoh
|
||||
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
|
||||
github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
|
||||
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
|
||||
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
|
||||
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
|
||||
github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
|
||||
github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
|
||||
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
|
||||
github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk=
|
||||
github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
|
||||
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
|
||||
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
|
||||
github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
|
||||
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
|
||||
github.com/go-ini/ini v1.38.2/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-ini/ini v1.48.0 h1:TvO60hO/2xgaaTWp2P0wUe4CFxwdMzfbkv3+343Xzqw=
|
||||
github.com/go-ini/ini v1.48.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
|
||||
github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
|
||||
github.com/go-ozzo/ozzo-validation v3.6.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
|
||||
github.com/go-sql-driver/mysql v0.0.0-20180719071942-99ff426eb706 h1:P3NPKb7qq581SeMCB+dU1SuCX1kQh8VoQ/4HmT2ftQY=
|
||||
github.com/go-sql-driver/mysql v0.0.0-20180719071942-99ff426eb706/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
||||
github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
|
||||
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
|
||||
github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE=
|
||||
github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
|
||||
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc h1:55rEp52jU6bkyslZ1+C/7NGfpQsEc6pxGLAGDOctqbw=
|
||||
github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
|
||||
github.com/golang/mock v1.4.0 h1:Rd1kQnQu0Hq3qvJppYSG0HtP+f5LPPUiDswTLiEegLg=
|
||||
github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
|
||||
github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc=
|
||||
github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
|
||||
github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
|
||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
|
||||
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw=
|
||||
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
|
||||
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
|
||||
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
|
||||
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
|
||||
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
|
||||
github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
|
||||
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
|
||||
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
|
||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
|
||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
|
||||
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
|
||||
github.com/google/gops v0.3.7 h1:KtVAagOM0FIq+02DiQrKBTnLhYpWBMowaufcj+W1Exw=
|
||||
github.com/google/gops v0.3.7/go.mod h1:bj0cwMmX1X4XIJFTjR99R5sCxNssNJ8HebFNvoQlmgY=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
|
||||
github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f h1:TyqzGm2z1h3AGhjOoRYyeLcW4WlW81MDQkWa+rx/000=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8=
|
||||
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
|
||||
github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk=
|
||||
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
|
||||
github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
|
||||
github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
|
||||
github.com/gorilla/rpc v1.1.0 h1:marKfvVP0Gpd/jHlVBKCQ8RAoUPdX7K1Nuh6l1BNh7A=
|
||||
github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc=
|
||||
github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
|
||||
github.com/gorilla/rpc v1.1.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ=
|
||||
github.com/gorilla/rpc v1.2.0 h1:WvvdC2lNeT1SP32zrIce5l0ECBfbAlmrmSBsuc57wfk=
|
||||
github.com/gorilla/rpc v1.2.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ=
|
||||
github.com/gorilla/schema v1.0.2/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU=
|
||||
github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
|
||||
github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
|
||||
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
|
||||
github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM=
|
||||
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
|
||||
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
|
||||
github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0=
|
||||
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
|
||||
github.com/hashicorp/go-immutable-radix v1.1.0 h1:vN9wG1D6KG6YHRTWr8512cxGOVgTMEfgEdSj/hr8MPc=
|
||||
github.com/hashicorp/go-immutable-radix v1.1.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
|
||||
@@ -149,7 +163,6 @@ github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iP
|
||||
github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
|
||||
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
|
||||
github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
|
||||
github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs=
|
||||
github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
|
||||
github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc=
|
||||
github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A=
|
||||
@@ -158,20 +171,18 @@ github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b
|
||||
github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE=
|
||||
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
|
||||
github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk=
|
||||
github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
|
||||
github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
|
||||
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
|
||||
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
|
||||
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
|
||||
github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
|
||||
github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
|
||||
github.com/hashicorp/memberlist v0.1.4 h1:gkyML/r71w3FL8gUi74Vk76avkj/9lYAY9lvg0OcoGs=
|
||||
github.com/hashicorp/memberlist v0.1.4/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
|
||||
github.com/hashicorp/memberlist v0.1.5 h1:AYBsgJOW9gab/toO5tEB8lWetVgDKZycqkebJ8xxpqM=
|
||||
github.com/hashicorp/memberlist v0.1.5/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
|
||||
github.com/hashicorp/serf v0.8.2 h1:YZ7UKsJv+hKjqGVUUbtE3HNj79Eln2oQ75tniF6iPt0=
|
||||
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
|
||||
github.com/hashicorp/serf v0.8.5 h1:ZynDUIQiA8usmRgPdGPHFdPnb1wgGI9tK3mO9hcAJjc=
|
||||
github.com/hashicorp/serf v0.8.5/go.mod h1:UpNcs7fFbpKIyZaUuSW6EPiH+eZC7OuyFD+wc1oal+k=
|
||||
@@ -181,61 +192,81 @@ github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf h1:WfD7V
|
||||
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h1:hyb9oH7vZsitZCiBt0ZvifOrB+qc8PS5IiilCIb87rg=
|
||||
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
|
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89 h1:12K8AlpT0/6QUXSfV0yi4Q0jkbq8NDtIKFtF61AoqV0=
|
||||
github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
|
||||
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
|
||||
github.com/jmoiron/sqlx v0.0.0-20170430194603-d9bd385d68c0/go.mod h1:IiEW3SEiiErVyFdH8NTuWjSifiEQKUoyK3LNqr2kCHU=
|
||||
github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22 h1:jKUP9TQ0c7X3w6+IPyMit07RE42MtTWNd77sN2cHngQ=
|
||||
github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22/go.mod h1:u0Jo4f2dNlTJeeOywkM6bLwxq6gC3pZ9rEFHn3AhTdk=
|
||||
github.com/johntdyer/slackrus v0.0.0-20170926115001-3992f319fd0a/go.mod h1:j1kV/8f3jowErEq4XyeypkCdvg5EeHkf0YCKCcq5Ybo=
|
||||
github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07 h1:+kBG/8rjCa6vxJZbUjAiE4MQmBEBYc8nLEb51frnvBY=
|
||||
github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07/go.mod h1:j1kV/8f3jowErEq4XyeypkCdvg5EeHkf0YCKCcq5Ybo=
|
||||
github.com/jrick/logrotate v1.0.0 h1:lQ1bL/n9mBNeIXoTUoYRlK4dHuNJVofX9oWqBtPnSzI=
|
||||
github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
|
||||
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE=
|
||||
github.com/jteeuwen/go-bindata v3.0.7+incompatible/go.mod h1:JVvhzYOiGBnFSYRyV00iY8q7/0PThjIYav1p9h5dmKs=
|
||||
github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
|
||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1 h1:PJPDf8OUfOK1bb/NeTKd4f1QXZItOX389VN3B6qC8ro=
|
||||
github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
|
||||
github.com/keybase/go-ps v0.0.0-20161005175911-668c8856d999/go.mod h1:hY+WOq6m2FpbvyrI93sMaypsttvaIL5nhVR92dTMUcQ=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23 h1:FOOIBWrEkLgmlgGfMuZT83xIwfPDxEI2OHu6xUmJMFE=
|
||||
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M=
|
||||
github.com/lbryio/chainquery v1.9.0 h1:NfBZ3eKYwD3PqXU/vt+2tF3ox3WUWoW4J5YdEQ0rxw0=
|
||||
github.com/lbryio/chainquery v1.9.0/go.mod h1:7G8l7jNtANS1I7fQOvtzbiHsv6qKVmN4codXHc3C4kk=
|
||||
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8=
|
||||
github.com/lbryio/lbry.go v0.0.0-20190828131228-f3a1fbdd5303/go.mod h1:qR+Ui0hYhemIU4fXqM3d1P9eiaRFlof777VJgV7KJ8w=
|
||||
github.com/lbryio/lbry.go v1.1.1-0.20190825202001-8fa28d3d656f/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
|
||||
github.com/lbryio/lbry.go v1.1.2 h1:Dyxc+glT/rVWJwHfIf7vjlPYYbjzrQz5ARmJd5Hp69c=
|
||||
github.com/lbryio/lbry.go v1.1.2/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
|
||||
github.com/lbryio/lbry.go/v2 v2.4.1-0.20191206194319-06764c3d007e h1:wN6r46L8EjgC2JyOImagQXaQvyF8Bd2QPjfXqv7oDAg=
|
||||
github.com/lbryio/lbry.go/v2 v2.4.1-0.20191206194319-06764c3d007e/go.mod h1:gbs/+PVII96AqsRgf+hY/F7sJkb/ciiX2yy3j+NApd0=
|
||||
github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128 h1:VL209c+AGKixMFpxT+TsOAPzBPuoUzyjXf47iNe7OzY=
|
||||
github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128/go.mod h1:RqOv4V5eWY/JGmduCPcQVdN19SEYnNY3SuF+arTKIU4=
|
||||
github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19 h1:/zWD8dVIl7bV1TdJWqPqy9tpqixzX2Qxgit48h3hQcY=
|
||||
github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46 h1:LemfR+rMxhf7nnOrzy2HqS7Me7SZ5gEwOcNFzKC8ySQ=
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
|
||||
github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4=
|
||||
github.com/lbryio/reflector.go v1.0.6-0.20190828131602-ce3d4403dbc6 h1:8k12lI18EEgMHkgGrgtRvJGl2Bq1vDSEbE4s+QaQ5Qc=
|
||||
github.com/lbryio/reflector.go v1.0.6-0.20190828131602-ce3d4403dbc6/go.mod h1:Q1Cnuv5iLsB2rS4Vr0o0EJr4Gs/EgDXFH2V4WtoM5o8=
|
||||
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c h1:m3O7561xBQ00lfUVayW4c6SnpVbUDQtPUwGcGYSUYQA=
|
||||
github.com/lbryio/reflector.go v1.1.3-0.20201006141750-a80e0f5b0dc6 h1:c702F5SrB2gdxQcPBvORu+KNiGW5C/asKuPFjYyrme8=
|
||||
github.com/lbryio/reflector.go v1.1.3-0.20201006141750-a80e0f5b0dc6/go.mod h1:kD8X5EswPGVEARKN6hYeaxAB29XxZkXCl0+DLE3e+bc=
|
||||
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
|
||||
github.com/lbryio/types v0.0.0-20191009145016-1bb8107e04f8 h1:jSNW/rK6DQsz7Zh+iv1zR384PeQdHt0gS4hKY17tkuM=
|
||||
github.com/lbryio/types v0.0.0-20191009145016-1bb8107e04f8/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
|
||||
github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5 h1:AsEBgzv3DhuYHI/GiQh2HxvTP71HCCE9E/tzGUzGdtU=
|
||||
github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
|
||||
github.com/lbryio/types v0.0.0-20200605192618-366870b2862d h1:Pg/4L7ExGXsseTMy1d/LK9TY55c8CP2gtmafmYzed0Q=
|
||||
github.com/lbryio/types v0.0.0-20200605192618-366870b2862d/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
|
||||
github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
github.com/lucas-clemente/quic-go v0.18.0 h1:JhQDdqxdwdmGdKsKgXi1+coHRoGhvU6z0rNzOJqZ/4o=
|
||||
github.com/lucas-clemente/quic-go v0.18.0/go.mod h1:yXttHsSNxQi8AWijC/vLP+OJczXqzHSOcJrM5ITUlCg=
|
||||
github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
|
||||
github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0=
|
||||
github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6 h1:iOAVXzZyXtW408TMYejlUPo6BIn92HmOacWtIfNyYns=
|
||||
github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg=
|
||||
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 h1:mG83tLXWSRdcXMWfkoumVwhcCbf3jHF9QKv/m37BkM0=
|
||||
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCWffGOaDcjkw1iB7W9DVLp6GXmfcJY/7YZCWPA4=
|
||||
github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
|
||||
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
|
||||
github.com/marten-seemann/qpack v0.2.0 h1:/r1rhZoOmgxVKBqPNnYilZBDEyw+6OUHCbBzA5jc2y0=
|
||||
github.com/marten-seemann/qpack v0.2.0/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc=
|
||||
github.com/marten-seemann/qtls v0.10.0 h1:ECsuYUKalRL240rRD4Ri33ISb7kAQ3qGDlrrl55b2pc=
|
||||
github.com/marten-seemann/qtls v0.10.0/go.mod h1:UvMd1oaYDACI99/oZUYLzMCkBXQVT0aGm99sJhbT8hs=
|
||||
github.com/marten-seemann/qtls-go1-15 v0.1.0 h1:i/YPXVxz8q9umso/5y474CNcHmTpA+5DH+mFPjx6PZg=
|
||||
github.com/marten-seemann/qtls-go1-15 v0.1.0/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I=
|
||||
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
|
||||
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA=
|
||||
github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
|
||||
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
|
||||
github.com/miekg/dns v1.1.22 h1:Jm64b3bO9kP43ddLjL2EY3Io6bmy1qGb9Xxz6TqS6rc=
|
||||
github.com/miekg/dns v1.1.22/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
|
||||
@@ -250,270 +281,335 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F
|
||||
github.com/mitchellh/mapstructure v0.0.0-20180511142126-bb74f1db0675/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
|
||||
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/nlopes/slack v0.5.0 h1:NbIae8Kd0NpqaEI3iUrsuS0KbcEDhzhc939jLW5fNm0=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
|
||||
github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
|
||||
github.com/nlopes/slack v0.5.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM=
|
||||
github.com/nlopes/slack v0.6.0 h1:jt0jxVQGhssx1Ib7naAOZEZcGdtIhTzkP0nopK0AsRA=
|
||||
github.com/nlopes/slack v0.6.0/go.mod h1:JzQ9m3PMAqcpeCam7UaHSuBuupz7CmpjehYMayT6YOk=
|
||||
github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
|
||||
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
|
||||
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.10.2 h1:uqH7bpe+ERSiDa34FDOF7RikN6RzXgduUF8yarlZp94=
|
||||
github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
|
||||
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
|
||||
github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA=
|
||||
github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
|
||||
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
|
||||
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
|
||||
github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
|
||||
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
|
||||
github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=
|
||||
github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
|
||||
github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
|
||||
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
||||
github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
|
||||
github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
||||
github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
|
||||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||
github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6 h1:2bae6N0SZjgzk+Zg8mzTsfmpwHXY9VBNp9UdjhaElA0=
|
||||
github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
|
||||
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc=
|
||||
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/profile v1.3.0/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
|
||||
github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740=
|
||||
github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
|
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8=
|
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
||||
github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
|
||||
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8=
|
||||
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
|
||||
github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE=
|
||||
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rubenv/sql-migrate v0.0.0-20170330050058-38004e7a77f2/go.mod h1:WS0rl9eEliYI8DPnr3TOwz4439pay+qNgzJoVya/DmY=
|
||||
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
|
||||
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
|
||||
github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
|
||||
github.com/sebdah/goldie v0.0.0-20180424091453-8784dd1ab561 h1:IY+sDBJR/wRtsxq+626xJnt4Tw7/ROA9cDIR8MMhWyg=
|
||||
github.com/sebdah/goldie v0.0.0-20180424091453-8784dd1ab561/go.mod h1:lvjGftC8oe7XPtyrOidaMi0rp5B9+XY/ZRUynGnuaxQ=
|
||||
github.com/sebdah/goldie v0.0.0-20190531093107-d313ffb52c77 h1:Msb6XRY62jQOueNNlB5LGin1rljK7c49NLniGwJG2bg=
|
||||
github.com/sebdah/goldie v0.0.0-20190531093107-d313ffb52c77/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4=
|
||||
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/sfreiberg/gotwilio v0.0.0-20180612161623-8fb7259ba8bf/go.mod h1:60PiR0SAnAcYSiwrXB6BaxeqHdXMf172toCosHfV+Yk=
|
||||
github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
|
||||
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
|
||||
github.com/shopspring/decimal v0.0.0-20180607144847-19e3cb6c2930/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
|
||||
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
|
||||
github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5 h1:Gojs/hac/DoYEM7WEICT45+hNWczIeuL5D21e5/HPAw=
|
||||
github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
|
||||
github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY=
|
||||
github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM=
|
||||
github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0=
|
||||
github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
|
||||
github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
|
||||
github.com/shurcooL/gofontwoff v0.0.0-20180329035133-29b52fc0a18d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw=
|
||||
github.com/shurcooL/gopherjslib v0.0.0-20160914041154-feb6d3990c2c/go.mod h1:8d3azKNyqcHP1GaQE/c6dDgjkgSx2BZ4IoEi4F1reUI=
|
||||
github.com/shurcooL/highlight_diff v0.0.0-20170515013008-09bb4053de1b/go.mod h1:ZpfEhSmds4ytuByIcDnOLkTHGUI6KNqRNPDLHDk+mUU=
|
||||
github.com/shurcooL/highlight_go v0.0.0-20181028180052-98c3abbbae20/go.mod h1:UDKB5a1T23gOMUJrI+uSuH0VRDStOiUVSjBTRDVBVag=
|
||||
github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg=
|
||||
github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw=
|
||||
github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y=
|
||||
github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg=
|
||||
github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q=
|
||||
github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ=
|
||||
github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I=
|
||||
github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b5uSkrEVM1jQUspwbixRBhaIjIzL2xazXp6kntxYle0=
|
||||
github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ=
|
||||
github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk=
|
||||
github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
|
||||
github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4=
|
||||
github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k=
|
||||
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
|
||||
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w=
|
||||
github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
|
||||
github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c h1:Ho+uVpkel/udgjbwB5Lktg9BtvJSh2DT0Hi6LPSyI2w=
|
||||
github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
|
||||
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8=
|
||||
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
|
||||
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||
github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE=
|
||||
github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA=
|
||||
github.com/spf13/afero v1.1.1/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
|
||||
github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
|
||||
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
|
||||
github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
|
||||
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
|
||||
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
|
||||
github.com/spf13/cobra v0.0.0-20180722215644-7c4570c3ebeb/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
|
||||
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
|
||||
github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
|
||||
github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
|
||||
github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
|
||||
github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
|
||||
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
|
||||
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
|
||||
github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
|
||||
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
|
||||
github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M=
|
||||
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
|
||||
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
|
||||
github.com/uber-go/atomic v1.3.2 h1:Azu9lPBWRNKzYXSIwRfgRuDuS0YKsK4NFhiQv98gkxo=
|
||||
github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
|
||||
github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o=
|
||||
github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
|
||||
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
|
||||
github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
|
||||
github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
|
||||
github.com/volatiletech/inflect v0.0.0-20170731032912-e7201282ae8d h1:gI4/tqP6lCY5k6Sg+4k9qSoBXmPwG+xXgMpK7jivD4M=
|
||||
github.com/volatiletech/inflect v0.0.0-20170731032912-e7201282ae8d/go.mod h1:jspfvgf53t5NLUT4o9L1IX0kIBNKamGq1tWc/MgWK9Q=
|
||||
github.com/volatiletech/null v8.0.0+incompatible h1:7wP8m5d/gZ6kW/9GnrLtMCRre2dlEnaQ9Km5OXlK4zg=
|
||||
github.com/volatiletech/null v8.0.0+incompatible/go.mod h1:0wD98JzdqB+rLyZ70fN05VDbXbafIb0KU0MdVhCzmOQ=
|
||||
github.com/volatiletech/sqlboiler v3.4.0+incompatible h1:saQ6WxZ9wEJp33q3w/DHs7an7SYi1H7Yzf4/moxCbJU=
|
||||
github.com/volatiletech/sqlboiler v3.4.0+incompatible/go.mod h1:jLfDkkHWPbS2cWRLkyC20vQWaIQsASEY7gM7zSo11Yw=
|
||||
github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
|
||||
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
|
||||
github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d h1:tQo6hjclyv3RHUgZOl6iWb2Y44A/sN9bf9LAYfuioEg=
|
||||
github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE=
|
||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
|
||||
go.opencensus.io v0.22.1 h1:8dP3SGL7MPB94crU3bEPplMPe83FI4EouesJUeFHv50=
|
||||
go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA=
|
||||
go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4=
|
||||
github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0=
|
||||
go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
|
||||
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
|
||||
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
go.uber.org/atomic v1.5.1 h1:rsqfU5vBkVknbhUGbAUwQKR2H4ItV8tjJ+6kJX4cxHM=
|
||||
go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
|
||||
go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
|
||||
golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw=
|
||||
golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c h1:Vj5n4GlwjmQteupaxJ9+0FNOmBrHfq7vN4btdGoDZgI=
|
||||
golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
|
||||
golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc h1:c0o/qxkaO2LF5t6fQrT4b5hzyggAkLLlCUjqfRxd8Q4=
|
||||
golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
|
||||
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
|
||||
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
|
||||
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
|
||||
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE=
|
||||
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
|
||||
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c h1:uOCk1iQW6Vc18bnC13MfzScl+wdKBmM9Y9kU7Z83/lw=
|
||||
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20191009170851-d66e71096ffb h1:TR699M2v0qoKTOHxeLgp6zPqaQNs74f01a/ob9W0qko=
|
||||
golang.org/x/net v0.0.0-20191009170851-d66e71096ffb/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
|
||||
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI=
|
||||
golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20171017063910-8dbc5d05d6ed/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190520201301-c432e742b0af/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5 h1:sM3evRHxE/1RuMe1FYAL3j7C7fUfIjkbE+NiDAYUF8U=
|
||||
golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f h1:hjzMYz/7Ea1mNKfOnFOfktR0mlA5jqhvywClCMHM/qw=
|
||||
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4=
|
||||
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=
|
||||
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 h1:xQwXv67TxFo9nC1GJFyab5eq/5B590r6RlnL/G8Sz7w=
|
||||
golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4 h1:Toz2IK7k8rbltAXwNAxKcn9OzqyNfMUhUNjz3sL0NMk=
|
||||
golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
|
||||
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
|
||||
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
|
||||
google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
|
||||
google.golang.org/api v0.11.0 h1:n/qM3q0/rV2F0pox7o0CvNhlPvZAo7pLbef122cbLJ0=
|
||||
google.golang.org/api v0.11.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
|
||||
google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
|
||||
google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
|
||||
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
|
||||
google.golang.org/appengine v1.6.2/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
|
||||
google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
|
||||
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19 h1:Lj2SnHtxkRGJDqnGaSjo+CCdIieEnwVazbOXILwQemk=
|
||||
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
|
||||
google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||
google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
|
||||
google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03 h1:4HYDjxeNXAOTv3o1N2tjo8UUSlhQgAD52FVkwxnWgM8=
|
||||
google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
|
||||
google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
|
||||
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
|
||||
google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s=
|
||||
google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
|
||||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
|
||||
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
|
||||
google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
|
||||
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/ini.v1 v1.41.0 h1:Ka3ViY6gNYSKiVy71zXBEqKplnV35ImDLVG+8uoIklE=
|
||||
gopkg.in/gorp.v1 v1.7.1/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw=
|
||||
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
|
||||
gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/ini.v1 v1.48.0 h1:URjZc+8ugRY5mL5uUeQH/a63JcHwdX9xZaWvmNWD7z8=
|
||||
gopkg.in/ini.v1 v1.48.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/ini.v1 v1.60.2 h1:7i8mqModL63zqi8nQn8Q3+0zvSCZy1AxhBgthKfi4WU=
|
||||
gopkg.in/ini.v1 v1.60.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 h1:FpCr9V8wuOei4BAen+93HtVJ+XSi+KPbaPKm0Vj5R64=
|
||||
gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79/go.mod h1:gWkaRU7CoXpezCBWfWjm3999QqS+1pYPXGbqQCTMzo8=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/vansante/go-ffprobe.v2 v2.0.2 h1:DdxSfFnlqeawPIVbIQEI6LR6OQHQNR7tNgWb2mWuC4w=
|
||||
gopkg.in/vansante/go-ffprobe.v2 v2.0.2/go.mod h1:qF0AlAjk7Nqzqf3y333Ly+KxN3cKF2JqA3JT5ZheUGE=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
|
||||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
|
||||
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
|
||||
gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
|
||||
grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o=
|
||||
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
|
||||
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
|
||||
rsc.io/goversion v1.0.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo=
|
||||
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
|
||||
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
|
||||
sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck=
|
||||
sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
|
||||
+21
-24
@@ -9,12 +9,12 @@ import (
|
||||
"github.com/asaskevich/govalidator"
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/ytsync/util"
|
||||
"github.com/lbryio/ytsync/v5/util"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const IPCooldownPeriod = 20 * time.Second
|
||||
const unbanTimeout = 3 * time.Hour
|
||||
const unbanTimeout = 48 * time.Hour
|
||||
|
||||
var stopper = stop.New()
|
||||
|
||||
@@ -43,15 +43,13 @@ func GetIPPool(stopGrp *stop.Group) (*IPPool, error) {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
var pool []throttledIP
|
||||
ipv6Added := false
|
||||
for _, address := range addrs {
|
||||
if ipnet, ok := address.(*net.IPNet); ok && ipnet.IP.IsGlobalUnicast() {
|
||||
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) && !ipv6Added {
|
||||
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) {
|
||||
pool = append(pool, throttledIP{
|
||||
IP: ipnet.IP.String(),
|
||||
LastUse: time.Now().Add(-5 * time.Minute),
|
||||
})
|
||||
ipv6Added = true
|
||||
} else if ipnet.IP.To4() != nil && govalidator.IsIPv4(ipnet.IP.String()) {
|
||||
pool = append(pool, throttledIP{
|
||||
IP: ipnet.IP.String(),
|
||||
@@ -65,21 +63,21 @@ func GetIPPool(stopGrp *stop.Group) (*IPPool, error) {
|
||||
lock: &sync.RWMutex{},
|
||||
stopGrp: stopGrp,
|
||||
}
|
||||
ticker := time.NewTicker(10 * time.Second)
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-stopGrp.Ch():
|
||||
return
|
||||
case <-ticker.C:
|
||||
ipPoolInstance.lock.RLock()
|
||||
for _, ip := range ipPoolInstance.ips {
|
||||
log.Debugf("IP: %s\tInUse: %t\tVideoID: %s\tThrottled: %t\tLastUse: %.1f", ip.IP, ip.InUse, ip.UsedForVideo, ip.Throttled, time.Since(ip.LastUse).Seconds())
|
||||
}
|
||||
ipPoolInstance.lock.RUnlock()
|
||||
}
|
||||
}
|
||||
}()
|
||||
//ticker := time.NewTicker(10 * time.Second)
|
||||
//go func() {
|
||||
// for {
|
||||
// select {
|
||||
// case <-stopGrp.Ch():
|
||||
// return
|
||||
// case <-ticker.C:
|
||||
// ipPoolInstance.lock.RLock()
|
||||
// for _, ip := range ipPoolInstance.ips {
|
||||
// log.Debugf("IP: %s\tInUse: %t\tVideoID: %s\tThrottled: %t\tLastUse: %.1f", ip.IP, ip.InUse, ip.UsedForVideo, ip.Throttled, time.Since(ip.LastUse).Seconds())
|
||||
// }
|
||||
// ipPoolInstance.lock.RUnlock()
|
||||
// }
|
||||
// }
|
||||
//}()
|
||||
return ipPoolInstance, nil
|
||||
}
|
||||
|
||||
@@ -110,7 +108,7 @@ func AllInUse(ips []throttledIP) bool {
|
||||
func (i *IPPool) ReleaseIP(ip string) {
|
||||
i.lock.Lock()
|
||||
defer i.lock.Unlock()
|
||||
for j, _ := range i.ips {
|
||||
for j := range i.ips {
|
||||
localIP := &i.ips[j]
|
||||
if localIP.IP == ip {
|
||||
localIP.InUse = false
|
||||
@@ -124,13 +122,12 @@ func (i *IPPool) ReleaseIP(ip string) {
|
||||
func (i *IPPool) ReleaseAll() {
|
||||
i.lock.Lock()
|
||||
defer i.lock.Unlock()
|
||||
for j, _ := range i.ips {
|
||||
for j := range i.ips {
|
||||
if i.ips[j].Throttled {
|
||||
continue
|
||||
}
|
||||
localIP := &i.ips[j]
|
||||
localIP.InUse = false
|
||||
localIP.LastUse = time.Now()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -186,7 +183,7 @@ func (i *IPPool) nextIP(forVideo string) (*throttledIP, error) {
|
||||
}
|
||||
|
||||
var nextIP *throttledIP
|
||||
for j, _ := range i.ips {
|
||||
for j := range i.ips {
|
||||
ip := &i.ips[j]
|
||||
if ip.InUse || ip.Throttled {
|
||||
continue
|
||||
|
||||
@@ -3,14 +3,17 @@ package main
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
"github.com/lbryio/ytsync/manager"
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
ytUtils "github.com/lbryio/ytsync/util"
|
||||
"github.com/lbryio/ytsync/v5/manager"
|
||||
"github.com/lbryio/ytsync/v5/sdk"
|
||||
"github.com/lbryio/ytsync/v5/shared"
|
||||
ytUtils "github.com/lbryio/ytsync/v5/util"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
@@ -21,24 +24,17 @@ var Version string
|
||||
const defaultMaxTries = 3
|
||||
|
||||
var (
|
||||
flags sdk.SyncFlags
|
||||
maxTries int
|
||||
refill int
|
||||
limit int
|
||||
syncStatus string
|
||||
channelID string
|
||||
syncFrom int64
|
||||
syncUntil int64
|
||||
concurrentJobs int
|
||||
videosLimit int
|
||||
maxVideoSize int
|
||||
maxVideoLength float64
|
||||
cliFlags shared.SyncFlags
|
||||
maxVideoLength int
|
||||
)
|
||||
|
||||
func main() {
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
log.SetLevel(log.DebugLevel)
|
||||
|
||||
http.Handle("/metrics", promhttp.Handler())
|
||||
go func() {
|
||||
log.Error(http.ListenAndServe(":2112", nil))
|
||||
}()
|
||||
cmd := &cobra.Command{
|
||||
Use: "ytsync",
|
||||
Short: "Publish youtube channels into LBRY network automatically.",
|
||||
@@ -46,25 +42,25 @@ func main() {
|
||||
Args: cobra.RangeArgs(0, 0),
|
||||
}
|
||||
|
||||
cmd.Flags().BoolVar(&flags.StopOnError, "stop-on-error", false, "If a publish fails, stop all publishing and exit")
|
||||
cmd.Flags().IntVar(&maxTries, "max-tries", defaultMaxTries, "Number of times to try a publish that fails")
|
||||
cmd.Flags().BoolVar(&flags.TakeOverExistingChannel, "takeover-existing-channel", false, "If channel exists and we don't own it, take over the channel")
|
||||
cmd.Flags().IntVar(&limit, "limit", 0, "limit the amount of channels to sync")
|
||||
cmd.Flags().BoolVar(&flags.SkipSpaceCheck, "skip-space-check", false, "Do not perform free space check on startup")
|
||||
cmd.Flags().BoolVar(&flags.SyncUpdate, "update", false, "Update previously synced channels instead of syncing new ones")
|
||||
cmd.Flags().BoolVar(&flags.SingleRun, "run-once", false, "Whether the process should be stopped after one cycle or not")
|
||||
cmd.Flags().BoolVar(&flags.RemoveDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published")
|
||||
cmd.Flags().BoolVar(&flags.UpgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version")
|
||||
cmd.Flags().BoolVar(&flags.DisableTransfers, "no-transfers", false, "Skips the transferring process of videos, channels and supports")
|
||||
cmd.Flags().BoolVar(&flags.QuickSync, "quick", false, "Look up only the last 50 videos from youtube")
|
||||
cmd.Flags().StringVar(&syncStatus, "status", "", "Specify which queue to pull from. Overrides --update")
|
||||
cmd.Flags().StringVar(&channelID, "channelID", "", "If specified, only this channel will be synced.")
|
||||
cmd.Flags().Int64Var(&syncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)")
|
||||
cmd.Flags().Int64Var(&syncUntil, "before", time.Now().AddDate(1, 0, 0).Unix(), "Specify until when to pull jobs [Unix time](Default: current Unix time)")
|
||||
cmd.Flags().IntVar(&concurrentJobs, "concurrent-jobs", 1, "how many jobs to process concurrently")
|
||||
cmd.Flags().IntVar(&videosLimit, "videos-limit", 1000, "how many videos to process per channel")
|
||||
cmd.Flags().IntVar(&maxVideoSize, "max-size", 2048, "Maximum video size to process (in MB)")
|
||||
cmd.Flags().Float64Var(&maxVideoLength, "max-length", 2.0, "Maximum video length to process (in hours)")
|
||||
cmd.Flags().IntVar(&cliFlags.MaxTries, "max-tries", defaultMaxTries, "Number of times to try a publish that fails")
|
||||
cmd.Flags().BoolVar(&cliFlags.TakeOverExistingChannel, "takeover-existing-channel", false, "If channel exists and we don't own it, take over the channel")
|
||||
cmd.Flags().IntVar(&cliFlags.Limit, "limit", 0, "limit the amount of channels to sync")
|
||||
cmd.Flags().BoolVar(&cliFlags.SkipSpaceCheck, "skip-space-check", false, "Do not perform free space check on startup")
|
||||
cmd.Flags().BoolVar(&cliFlags.SyncUpdate, "update", false, "Update previously synced channels instead of syncing new ones")
|
||||
cmd.Flags().BoolVar(&cliFlags.SingleRun, "run-once", false, "Whether the process should be stopped after one cycle or not")
|
||||
cmd.Flags().BoolVar(&cliFlags.RemoveDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published")
|
||||
cmd.Flags().BoolVar(&cliFlags.UpgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version")
|
||||
cmd.Flags().BoolVar(&cliFlags.DisableTransfers, "no-transfers", false, "Skips the transferring process of videos, channels and supports")
|
||||
cmd.Flags().BoolVar(&cliFlags.QuickSync, "quick", false, "Look up only the last 50 videos from youtube")
|
||||
cmd.Flags().StringVar(&cliFlags.Status, "status", "", "Specify which queue to pull from. Overrides --update")
|
||||
cmd.Flags().StringVar(&cliFlags.SecondaryStatus, "status2", "", "Specify which secondary queue to pull from.")
|
||||
cmd.Flags().StringVar(&cliFlags.ChannelID, "channelID", "", "If specified, only this channel will be synced.")
|
||||
cmd.Flags().Int64Var(&cliFlags.SyncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)")
|
||||
cmd.Flags().Int64Var(&cliFlags.SyncUntil, "before", time.Now().AddDate(1, 0, 0).Unix(), "Specify until when to pull jobs [Unix time](Default: current Unix time)")
|
||||
cmd.Flags().IntVar(&cliFlags.ConcurrentJobs, "concurrent-jobs", 1, "how many jobs to process concurrently")
|
||||
cmd.Flags().IntVar(&cliFlags.VideosLimit, "videos-limit", 0, "how many videos to process per channel (leave 0 for automatic detection)")
|
||||
cmd.Flags().IntVar(&cliFlags.MaxVideoSize, "max-size", 2048, "Maximum video size to process (in MB)")
|
||||
cmd.Flags().IntVar(&maxVideoLength, "max-length", 2, "Maximum video length to process (in hours)")
|
||||
|
||||
if err := cmd.Execute(); err != nil {
|
||||
fmt.Println(err)
|
||||
@@ -91,29 +87,26 @@ func ytSync(cmd *cobra.Command, args []string) {
|
||||
util.InitSlack(os.Getenv("SLACK_TOKEN"), os.Getenv("SLACK_CHANNEL"), hostname)
|
||||
}
|
||||
|
||||
if syncStatus != "" && !util.InSlice(syncStatus, manager.SyncStatuses) {
|
||||
log.Errorf("status must be one of the following: %v\n", manager.SyncStatuses)
|
||||
if cliFlags.Status != "" && !util.InSlice(cliFlags.Status, shared.SyncStatuses) {
|
||||
log.Errorf("status must be one of the following: %v\n", shared.SyncStatuses)
|
||||
return
|
||||
}
|
||||
|
||||
if flags.StopOnError && maxTries != defaultMaxTries {
|
||||
log.Errorln("--stop-on-error and --max-tries are mutually exclusive")
|
||||
return
|
||||
}
|
||||
if maxTries < 1 {
|
||||
if cliFlags.MaxTries < 1 {
|
||||
log.Errorln("setting --max-tries less than 1 doesn't make sense")
|
||||
return
|
||||
}
|
||||
|
||||
if limit < 0 {
|
||||
if cliFlags.Limit < 0 {
|
||||
log.Errorln("setting --limit less than 0 (unlimited) doesn't make sense")
|
||||
return
|
||||
}
|
||||
cliFlags.MaxVideoLength = time.Duration(maxVideoLength) * time.Hour
|
||||
|
||||
apiURL := os.Getenv("LBRY_WEB_API")
|
||||
apiToken := os.Getenv("LBRY_API_TOKEN")
|
||||
youtubeAPIKey := os.Getenv("YOUTUBE_API_KEY")
|
||||
lbrycrdString := os.Getenv("LBRYCRD_STRING")
|
||||
lbrycrdDsn := os.Getenv("LBRYCRD_STRING")
|
||||
awsS3ID := os.Getenv("AWS_S3_ID")
|
||||
awsS3Secret := os.Getenv("AWS_S3_SECRET")
|
||||
awsS3Region := os.Getenv("AWS_S3_REGION")
|
||||
@@ -146,42 +139,30 @@ func ytSync(cmd *cobra.Command, args []string) {
|
||||
log.Errorln("AWS S3 Bucket was not defined. Please set the environment variable AWS_S3_BUCKET")
|
||||
return
|
||||
}
|
||||
if lbrycrdString == "" {
|
||||
if lbrycrdDsn == "" {
|
||||
log.Infoln("Using default (local) lbrycrd instance. Set LBRYCRD_STRING if you want to use something else")
|
||||
}
|
||||
|
||||
blobsDir := ytUtils.GetBlobsDir()
|
||||
|
||||
syncProperties := &sdk.SyncProperties{
|
||||
SyncFrom: syncFrom,
|
||||
SyncUntil: syncUntil,
|
||||
YoutubeChannelID: channelID,
|
||||
}
|
||||
apiConfig := &sdk.APIConfig{
|
||||
YoutubeAPIKey: youtubeAPIKey,
|
||||
ApiURL: apiURL,
|
||||
ApiToken: apiToken,
|
||||
HostName: hostname,
|
||||
}
|
||||
awsConfig := &shared.AwsConfigs{
|
||||
AwsS3ID: awsS3ID,
|
||||
AwsS3Secret: awsS3Secret,
|
||||
AwsS3Region: awsS3Region,
|
||||
AwsS3Bucket: awsS3Bucket,
|
||||
}
|
||||
sm := manager.NewSyncManager(
|
||||
flags,
|
||||
maxTries,
|
||||
refill,
|
||||
limit,
|
||||
concurrentJobs,
|
||||
concurrentJobs,
|
||||
cliFlags,
|
||||
blobsDir,
|
||||
videosLimit,
|
||||
maxVideoSize,
|
||||
lbrycrdString,
|
||||
awsS3ID,
|
||||
awsS3Secret,
|
||||
awsS3Region,
|
||||
awsS3Bucket,
|
||||
syncStatus,
|
||||
syncProperties,
|
||||
lbrycrdDsn,
|
||||
awsConfig,
|
||||
apiConfig,
|
||||
maxVideoLength,
|
||||
)
|
||||
err := sm.Start()
|
||||
if err != nil {
|
||||
|
||||
@@ -1,32 +0,0 @@
|
||||
package manager
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
|
||||
"google.golang.org/api/googleapi/transport"
|
||||
"google.golang.org/api/youtube/v3"
|
||||
)
|
||||
|
||||
func (s *Sync) CountVideos() (uint64, error) {
|
||||
client := &http.Client{
|
||||
Transport: &transport.APIKey{Key: s.APIConfig.YoutubeAPIKey},
|
||||
}
|
||||
|
||||
service, err := youtube.New(client)
|
||||
if err != nil {
|
||||
return 0, errors.Prefix("error creating YouTube service", err)
|
||||
}
|
||||
|
||||
response, err := service.Channels.List("statistics").Id(s.YoutubeChannelID).Do()
|
||||
if err != nil {
|
||||
return 0, errors.Prefix("error getting channels", err)
|
||||
}
|
||||
|
||||
if len(response.Items) < 1 {
|
||||
return 0, errors.Err("youtube channel not found")
|
||||
}
|
||||
|
||||
return response.Items[0].Statistics.VideoCount, nil
|
||||
}
|
||||
+73
-153
@@ -3,101 +3,54 @@ package manager
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/ytsync/blobs_reflector"
|
||||
"github.com/lbryio/ytsync/ip_manager"
|
||||
"github.com/lbryio/ytsync/namer"
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
logUtils "github.com/lbryio/ytsync/util"
|
||||
"github.com/lbryio/ytsync/v5/blobs_reflector"
|
||||
"github.com/lbryio/ytsync/v5/ip_manager"
|
||||
"github.com/lbryio/ytsync/v5/namer"
|
||||
"github.com/lbryio/ytsync/v5/sdk"
|
||||
"github.com/lbryio/ytsync/v5/shared"
|
||||
logUtils "github.com/lbryio/ytsync/v5/util"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type SyncManager struct {
|
||||
SyncFlags sdk.SyncFlags
|
||||
maxTries int
|
||||
refill int
|
||||
limit int
|
||||
concurrentJobs int
|
||||
concurrentVideos int
|
||||
blobsDir string
|
||||
videosLimit int
|
||||
maxVideoSize int
|
||||
maxVideoLength float64
|
||||
lbrycrdString string
|
||||
awsS3ID string
|
||||
awsS3Secret string
|
||||
awsS3Region string
|
||||
syncStatus string
|
||||
awsS3Bucket string
|
||||
syncProperties *sdk.SyncProperties
|
||||
apiConfig *sdk.APIConfig
|
||||
CliFlags shared.SyncFlags
|
||||
ApiConfig *sdk.APIConfig
|
||||
LbrycrdDsn string
|
||||
AwsConfigs *shared.AwsConfigs
|
||||
|
||||
blobsDir string
|
||||
channelsToSync []Sync
|
||||
}
|
||||
|
||||
func NewSyncManager(syncFlags sdk.SyncFlags, maxTries int, refill int, limit int, concurrentJobs int, concurrentVideos int, blobsDir string, videosLimit int,
|
||||
maxVideoSize int, lbrycrdString string, awsS3ID string, awsS3Secret string, awsS3Region string, awsS3Bucket string,
|
||||
syncStatus string, syncProperties *sdk.SyncProperties, apiConfig *sdk.APIConfig, maxVideoLength float64) *SyncManager {
|
||||
func NewSyncManager(cliFlags shared.SyncFlags, blobsDir, lbrycrdDsn string, awsConfigs *shared.AwsConfigs, apiConfig *sdk.APIConfig) *SyncManager {
|
||||
return &SyncManager{
|
||||
SyncFlags: syncFlags,
|
||||
maxTries: maxTries,
|
||||
refill: refill,
|
||||
limit: limit,
|
||||
concurrentJobs: concurrentJobs,
|
||||
concurrentVideos: concurrentVideos,
|
||||
blobsDir: blobsDir,
|
||||
videosLimit: videosLimit,
|
||||
maxVideoSize: maxVideoSize,
|
||||
maxVideoLength: maxVideoLength,
|
||||
lbrycrdString: lbrycrdString,
|
||||
awsS3ID: awsS3ID,
|
||||
awsS3Secret: awsS3Secret,
|
||||
awsS3Region: awsS3Region,
|
||||
awsS3Bucket: awsS3Bucket,
|
||||
syncStatus: syncStatus,
|
||||
syncProperties: syncProperties,
|
||||
apiConfig: apiConfig,
|
||||
CliFlags: cliFlags,
|
||||
blobsDir: blobsDir,
|
||||
LbrycrdDsn: lbrycrdDsn,
|
||||
AwsConfigs: awsConfigs,
|
||||
ApiConfig: apiConfig,
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
StatusPending = "pending" // waiting for permission to sync
|
||||
StatusPendingEmail = "pendingemail" // permission granted but missing email
|
||||
StatusQueued = "queued" // in sync queue. will be synced soon
|
||||
StatusPendingUpgrade = "pendingupgrade" // in sync queue. will be synced soon
|
||||
StatusSyncing = "syncing" // syncing now
|
||||
StatusSynced = "synced" // done
|
||||
StatusFailed = "failed"
|
||||
StatusFinalized = "finalized" // no more changes allowed
|
||||
StatusAbandoned = "abandoned" // deleted on youtube or banned
|
||||
)
|
||||
const LatestMetadataVersion = 2
|
||||
|
||||
var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusPendingUpgrade, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned}
|
||||
|
||||
const (
|
||||
VideoStatusPublished = "published"
|
||||
VideoStatusFailed = "failed"
|
||||
VideoStatusUpgradeFailed = "upgradefailed"
|
||||
VideoStatusUnpublished = "unpublished"
|
||||
VideoStatusTranferFailed = "transferfailed"
|
||||
)
|
||||
|
||||
const (
|
||||
TransferStateNotTouched = iota
|
||||
TransferStatePending
|
||||
TransferStateComplete
|
||||
TransferStateManual
|
||||
)
|
||||
func (s *SyncManager) enqueueChannel(channel *shared.YoutubeChannel) {
|
||||
s.channelsToSync = append(s.channelsToSync, Sync{
|
||||
DbChannelData: channel,
|
||||
Manager: s,
|
||||
namer: namer.NewNamer(),
|
||||
hardVideoFailure: hardVideoFailure{
|
||||
lock: &sync.Mutex{},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func (s *SyncManager) Start() error {
|
||||
|
||||
if logUtils.ShouldCleanOnStartup() {
|
||||
err := logUtils.CleanForStartup()
|
||||
if err != nil {
|
||||
@@ -105,102 +58,71 @@ func (s *SyncManager) Start() error {
|
||||
}
|
||||
}
|
||||
|
||||
var lastChannelProcessed string
|
||||
var secondLastChannelProcessed string
|
||||
syncCount := 0
|
||||
for {
|
||||
s.channelsToSync = make([]Sync, 0, 10) // reset sync queue
|
||||
err := s.checkUsedSpace()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
var syncs []Sync
|
||||
shouldInterruptLoop := false
|
||||
|
||||
isSingleChannelSync := s.syncProperties.YoutubeChannelID != ""
|
||||
if isSingleChannelSync {
|
||||
channels, err := s.apiConfig.FetchChannels("", s.syncProperties)
|
||||
if s.CliFlags.IsSingleChannelSync() {
|
||||
channels, err := s.ApiConfig.FetchChannels("", &s.CliFlags)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
if len(channels) != 1 {
|
||||
return errors.Err("Expected 1 channel, %d returned", len(channels))
|
||||
}
|
||||
lbryChannelName := channels[0].DesiredChannelName
|
||||
syncs = make([]Sync, 1)
|
||||
syncs[0] = Sync{
|
||||
APIConfig: s.apiConfig,
|
||||
YoutubeChannelID: s.syncProperties.YoutubeChannelID,
|
||||
LbryChannelName: lbryChannelName,
|
||||
lbryChannelID: channels[0].ChannelClaimID,
|
||||
MaxTries: s.maxTries,
|
||||
ConcurrentVideos: s.concurrentVideos,
|
||||
Refill: s.refill,
|
||||
Manager: s,
|
||||
LbrycrdString: s.lbrycrdString,
|
||||
AwsS3ID: s.awsS3ID,
|
||||
AwsS3Secret: s.awsS3Secret,
|
||||
AwsS3Region: s.awsS3Region,
|
||||
AwsS3Bucket: s.awsS3Bucket,
|
||||
namer: namer.NewNamer(),
|
||||
Fee: channels[0].Fee,
|
||||
clientPublishAddress: channels[0].PublishAddress,
|
||||
publicKey: channels[0].PublicKey,
|
||||
transferState: channels[0].TransferState,
|
||||
}
|
||||
s.enqueueChannel(&channels[0])
|
||||
shouldInterruptLoop = true
|
||||
} else {
|
||||
var queuesToSync []string
|
||||
if s.syncStatus != "" {
|
||||
queuesToSync = append(queuesToSync, s.syncStatus)
|
||||
} else if s.SyncFlags.SyncUpdate {
|
||||
queuesToSync = append(queuesToSync, StatusSyncing, StatusSynced)
|
||||
if s.CliFlags.Status != "" {
|
||||
queuesToSync = append(queuesToSync, shared.StatusSyncing, s.CliFlags.Status)
|
||||
} else if s.CliFlags.SyncUpdate {
|
||||
queuesToSync = append(queuesToSync, shared.StatusSyncing, shared.StatusSynced)
|
||||
} else {
|
||||
queuesToSync = append(queuesToSync, StatusSyncing, StatusQueued)
|
||||
queuesToSync = append(queuesToSync, shared.StatusSyncing, shared.StatusQueued)
|
||||
}
|
||||
if s.CliFlags.SecondaryStatus != "" {
|
||||
queuesToSync = append(queuesToSync, s.CliFlags.SecondaryStatus)
|
||||
}
|
||||
queues:
|
||||
for _, q := range queuesToSync {
|
||||
//temporary override for sync-until to give tom the time to review the channels
|
||||
if q == StatusQueued {
|
||||
s.syncProperties.SyncUntil = time.Now().Add(-8 * time.Hour).Unix()
|
||||
}
|
||||
channels, err := s.apiConfig.FetchChannels(q, s.syncProperties)
|
||||
channels, err := s.ApiConfig.FetchChannels(q, &s.CliFlags)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for i, c := range channels {
|
||||
log.Infof("There are %d channels in the \"%s\" queue", len(channels)-i, q)
|
||||
syncs = append(syncs, Sync{
|
||||
APIConfig: s.apiConfig,
|
||||
YoutubeChannelID: c.ChannelId,
|
||||
LbryChannelName: c.DesiredChannelName,
|
||||
lbryChannelID: c.ChannelClaimID,
|
||||
MaxTries: s.maxTries,
|
||||
ConcurrentVideos: s.concurrentVideos,
|
||||
Refill: s.refill,
|
||||
Manager: s,
|
||||
LbrycrdString: s.lbrycrdString,
|
||||
AwsS3ID: s.awsS3ID,
|
||||
AwsS3Secret: s.awsS3Secret,
|
||||
AwsS3Region: s.awsS3Region,
|
||||
AwsS3Bucket: s.awsS3Bucket,
|
||||
namer: namer.NewNamer(),
|
||||
Fee: c.Fee,
|
||||
clientPublishAddress: c.PublishAddress,
|
||||
publicKey: c.PublicKey,
|
||||
transferState: c.TransferState,
|
||||
})
|
||||
if q != StatusFailed {
|
||||
continue queues
|
||||
log.Infof("Currently processing the \"%s\" queue with %d channels", q, len(channels))
|
||||
for _, c := range channels {
|
||||
s.enqueueChannel(&c)
|
||||
queueAll := q == shared.StatusFailed || q == shared.StatusSyncing
|
||||
if !queueAll {
|
||||
break queues
|
||||
}
|
||||
}
|
||||
log.Infof("Drained the \"%s\" queue", q)
|
||||
}
|
||||
}
|
||||
if len(syncs) == 0 {
|
||||
if len(s.channelsToSync) == 0 {
|
||||
log.Infoln("No channels to sync. Pausing 5 minutes!")
|
||||
time.Sleep(5 * time.Minute)
|
||||
}
|
||||
for _, sync := range syncs {
|
||||
for _, sync := range s.channelsToSync {
|
||||
if lastChannelProcessed == sync.DbChannelData.ChannelId && secondLastChannelProcessed == lastChannelProcessed {
|
||||
util.SendToSlack("We just killed a sync for %s to stop looping! (%s)", sync.DbChannelData.DesiredChannelName, sync.DbChannelData.ChannelId)
|
||||
stopTheLoops := errors.Err("Found channel %s running 3 times, set it to failed, and reprocess later", sync.DbChannelData.DesiredChannelName)
|
||||
sync.setChannelTerminationStatus(&stopTheLoops)
|
||||
continue
|
||||
}
|
||||
secondLastChannelProcessed = lastChannelProcessed
|
||||
lastChannelProcessed = sync.DbChannelData.ChannelId
|
||||
shouldNotCount := false
|
||||
logUtils.SendInfoToSlack("Syncing %s (%s) to LBRY! total processed channels since startup: %d", sync.LbryChannelName, sync.YoutubeChannelID, syncCount+1)
|
||||
logUtils.SendInfoToSlack("Syncing %s (%s) to LBRY! total processed channels since startup: %d", sync.DbChannelData.DesiredChannelName, sync.DbChannelData.ChannelId, syncCount+1)
|
||||
err := sync.FullCycle()
|
||||
//TODO: THIS IS A TEMPORARY WORK AROUND FOR THE STUPID IP LOCKUP BUG
|
||||
ipPool, _ := ip_manager.GetIPPool(sync.grp)
|
||||
@@ -209,6 +131,9 @@ func (s *SyncManager) Start() error {
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "quotaExceeded") {
|
||||
logUtils.SleepUntilQuotaReset()
|
||||
}
|
||||
fatalErrors := []string{
|
||||
"default_wallet already exists",
|
||||
"WALLET HAS NOT BEEN MOVED TO THE WALLET BACKUP DIR",
|
||||
@@ -218,47 +143,42 @@ func (s *SyncManager) Start() error {
|
||||
"the channel in the wallet is different than the channel in the database",
|
||||
"this channel does not belong to this wallet!",
|
||||
"You already have a stream claim published under the name",
|
||||
"Daily Limit Exceeded",
|
||||
}
|
||||
|
||||
if util.SubstringInSlice(err.Error(), fatalErrors) {
|
||||
return errors.Prefix("@Nikooo777 this requires manual intervention! Exiting...", err)
|
||||
}
|
||||
shouldNotCount = strings.Contains(err.Error(), "this youtube channel is being managed by another server")
|
||||
if !shouldNotCount {
|
||||
logUtils.SendInfoToSlack("A non fatal error was reported by the sync process. %s\nContinuing...", err.Error())
|
||||
logUtils.SendInfoToSlack("A non fatal error was reported by the sync process.\n%s", errors.FullTrace(err))
|
||||
}
|
||||
}
|
||||
err = blobs_reflector.ReflectAndClean()
|
||||
if err != nil {
|
||||
return errors.Prefix("@Nikooo777 something went wrong while reflecting blobs", err)
|
||||
}
|
||||
logUtils.SendInfoToSlack("Syncing %s (%s) reached an end. total processed channels since startup: %d", sync.LbryChannelName, sync.YoutubeChannelID, syncCount+1)
|
||||
logUtils.SendInfoToSlack("%s (%s) reached an end. Total processed channels since startup: %d", sync.DbChannelData.DesiredChannelName, sync.DbChannelData.ChannelId, syncCount+1)
|
||||
if !shouldNotCount {
|
||||
syncCount++
|
||||
}
|
||||
if sync.IsInterrupted() || (s.limit != 0 && syncCount >= s.limit) {
|
||||
if sync.IsInterrupted() || (s.CliFlags.Limit != 0 && syncCount >= s.CliFlags.Limit) {
|
||||
shouldInterruptLoop = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if shouldInterruptLoop || s.SyncFlags.SingleRun {
|
||||
if shouldInterruptLoop || s.CliFlags.SingleRun {
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (s *SyncManager) GetS3AWSConfig() aws.Config {
|
||||
return aws.Config{
|
||||
Credentials: credentials.NewStaticCredentials(s.awsS3ID, s.awsS3Secret, ""),
|
||||
Region: &s.awsS3Region,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *SyncManager) checkUsedSpace() error {
|
||||
usedPctile, err := GetUsedSpace(logUtils.GetBlobsDir())
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
if usedPctile >= 0.90 && !s.SyncFlags.SkipSpaceCheck {
|
||||
if usedPctile >= 0.90 && !s.CliFlags.SkipSpaceCheck {
|
||||
return errors.Err(fmt.Sprintf("more than 90%% of the space has been used. use --skip-space-check to ignore. Used: %.1f%%", usedPctile*100))
|
||||
}
|
||||
log.Infof("disk usage: %.1f%%", usedPctile*100)
|
||||
|
||||
@@ -0,0 +1,254 @@
|
||||
package manager
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
|
||||
logUtils "github.com/lbryio/ytsync/v5/util"
|
||||
)
|
||||
|
||||
func (s *Sync) getS3Downloader() (*s3manager.Downloader, error) {
|
||||
s3Session, err := session.NewSession(s.Manager.AwsConfigs.GetS3AWSConfig())
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("error starting session: ", err)
|
||||
}
|
||||
downloader := s3manager.NewDownloader(s3Session)
|
||||
return downloader, nil
|
||||
}
|
||||
func (s *Sync) getS3Uploader() (*s3manager.Uploader, error) {
|
||||
s3Session, err := session.NewSession(s.Manager.AwsConfigs.GetS3AWSConfig())
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("error starting session: ", err)
|
||||
}
|
||||
uploader := s3manager.NewUploader(s3Session)
|
||||
return uploader, nil
|
||||
}
|
||||
|
||||
func (s *Sync) downloadWallet() error {
|
||||
defaultWalletDir, defaultTempWalletDir, key, err := s.getWalletPaths()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
downloader, err := s.getS3Downloader()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out, err := os.Create(defaultTempWalletDir)
|
||||
if err != nil {
|
||||
return errors.Prefix("error creating temp wallet: ", err)
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{
|
||||
Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket),
|
||||
Key: key,
|
||||
})
|
||||
if err != nil {
|
||||
// Casting to the awserr.Error type will allow you to inspect the error
|
||||
// code returned by the service in code. The error code can be used
|
||||
// to switch on context specific functionality. In this case a context
|
||||
// specific error message is printed to the user based on the bucket
|
||||
// and key existing.
|
||||
//
|
||||
// For information on other S3 API error codes see:
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
|
||||
if aerr, ok := err.(awserr.Error); ok {
|
||||
code := aerr.Code()
|
||||
if code == s3.ErrCodeNoSuchKey {
|
||||
return errors.Err("wallet not on S3")
|
||||
}
|
||||
}
|
||||
return err
|
||||
} else if bytesWritten == 0 {
|
||||
return errors.Err("zero bytes written")
|
||||
}
|
||||
|
||||
err = os.Rename(defaultTempWalletDir, defaultWalletDir)
|
||||
if err != nil {
|
||||
return errors.Prefix("error replacing temp wallet for default wallet: ", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Sync) downloadBlockchainDB() error {
|
||||
if logUtils.IsRegTest() {
|
||||
return nil // tests fail if we re-use the same blockchain DB
|
||||
}
|
||||
defaultBDBDir, defaultTempBDBDir, key, err := s.getBlockchainDBPaths()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
files, err := filepath.Glob(defaultBDBDir + "*")
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
for _, f := range files {
|
||||
err = os.Remove(f)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
}
|
||||
if s.DbChannelData.WipeDB {
|
||||
return nil
|
||||
}
|
||||
downloader, err := s.getS3Downloader()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
out, err := os.Create(defaultTempBDBDir)
|
||||
if err != nil {
|
||||
return errors.Prefix("error creating temp wallet: ", err)
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{
|
||||
Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket),
|
||||
Key: key,
|
||||
})
|
||||
if err != nil {
|
||||
// Casting to the awserr.Error type will allow you to inspect the error
|
||||
// code returned by the service in code. The error code can be used
|
||||
// to switch on context specific functionality. In this case a context
|
||||
// specific error message is printed to the user based on the bucket
|
||||
// and key existing.
|
||||
//
|
||||
// For information on other S3 API error codes see:
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
|
||||
if aerr, ok := err.(awserr.Error); ok {
|
||||
code := aerr.Code()
|
||||
if code == s3.ErrCodeNoSuchKey {
|
||||
return nil // let ytsync sync the database by itself
|
||||
}
|
||||
}
|
||||
return errors.Err(err)
|
||||
} else if bytesWritten == 0 {
|
||||
return errors.Err("zero bytes written")
|
||||
}
|
||||
|
||||
err = os.Rename(defaultTempBDBDir, defaultBDBDir)
|
||||
if err != nil {
|
||||
return errors.Prefix("error replacing temp blockchain.db for default blockchain.db: ", err)
|
||||
}
|
||||
log.Printf("blockchain.db downloaded to %s", defaultBDBDir)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Sync) getWalletPaths() (defaultWallet, tempWallet string, key *string, err error) {
|
||||
defaultWallet = os.Getenv("HOME") + "/.lbryum/wallets/default_wallet"
|
||||
tempWallet = os.Getenv("HOME") + "/.lbryum/wallets/tmp_wallet"
|
||||
key = aws.String("/wallets/" + s.DbChannelData.ChannelId)
|
||||
if logUtils.IsRegTest() {
|
||||
defaultWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet"
|
||||
tempWallet = os.Getenv("HOME") + "/.lbryum_regtest/wallets/tmp_wallet"
|
||||
key = aws.String("/regtest/" + s.DbChannelData.ChannelId)
|
||||
}
|
||||
|
||||
lbryumDir := os.Getenv("LBRYUM_DIR")
|
||||
if lbryumDir != "" {
|
||||
defaultWallet = lbryumDir + "/wallets/default_wallet"
|
||||
tempWallet = lbryumDir + "/wallets/tmp_wallet"
|
||||
}
|
||||
|
||||
if _, err := os.Stat(defaultWallet); !os.IsNotExist(err) {
|
||||
return "", "", nil, errors.Err("default_wallet already exists")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (s *Sync) getBlockchainDBPaths() (defaultDB, tempDB string, key *string, err error) {
|
||||
lbryumDir := os.Getenv("LBRYUM_DIR")
|
||||
if lbryumDir == "" {
|
||||
if logUtils.IsRegTest() {
|
||||
lbryumDir = os.Getenv("HOME") + "/.lbryum_regtest"
|
||||
} else {
|
||||
lbryumDir = os.Getenv("HOME") + "/.lbryum"
|
||||
}
|
||||
}
|
||||
defaultDB = lbryumDir + "/lbc_mainnet/blockchain.db"
|
||||
tempDB = lbryumDir + "/lbc_mainnet/tmp_blockchain.db"
|
||||
key = aws.String("/blockchain_dbs/" + s.DbChannelData.ChannelId)
|
||||
if logUtils.IsRegTest() {
|
||||
defaultDB = lbryumDir + "/lbc_regtest/blockchain.db"
|
||||
tempDB = lbryumDir + "/lbc_regtest/tmp_blockchain.db"
|
||||
key = aws.String("/regtest_dbs/" + s.DbChannelData.ChannelId)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (s *Sync) uploadWallet() error {
|
||||
defaultWalletDir := logUtils.GetDefaultWalletPath()
|
||||
key := aws.String("/wallets/" + s.DbChannelData.ChannelId)
|
||||
if logUtils.IsRegTest() {
|
||||
key = aws.String("/regtest/" + s.DbChannelData.ChannelId)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(defaultWalletDir); os.IsNotExist(err) {
|
||||
return errors.Err("default_wallet does not exist")
|
||||
}
|
||||
|
||||
uploader, err := s.getS3Uploader()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
file, err := os.Open(defaultWalletDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
_, err = uploader.Upload(&s3manager.UploadInput{
|
||||
Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket),
|
||||
Key: key,
|
||||
Body: file,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Println("wallet uploaded to S3")
|
||||
|
||||
return os.Remove(defaultWalletDir)
|
||||
}
|
||||
|
||||
func (s *Sync) uploadBlockchainDB() error {
|
||||
defaultBDBDir, _, key, err := s.getBlockchainDBPaths()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(defaultBDBDir); os.IsNotExist(err) {
|
||||
return errors.Err("blockchain.db does not exist")
|
||||
}
|
||||
|
||||
uploader, err := s.getS3Uploader()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
file, err := os.Open(defaultBDBDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
_, err = uploader.Upload(&s3manager.UploadInput{
|
||||
Bucket: aws.String(s.Manager.AwsConfigs.AwsS3Bucket),
|
||||
Key: key,
|
||||
Body: file,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Println("blockchain.db uploaded to S3")
|
||||
return os.Remove(defaultBDBDir)
|
||||
}
|
||||
+115
-78
@@ -3,22 +3,23 @@ package manager
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
logUtils "github.com/lbryio/ytsync/util"
|
||||
"github.com/lbryio/ytsync/v5/shared"
|
||||
"github.com/lbryio/ytsync/v5/timing"
|
||||
logUtils "github.com/lbryio/ytsync/v5/util"
|
||||
"github.com/lbryio/ytsync/v5/ytapi"
|
||||
|
||||
"github.com/lbryio/ytsync/tags_manager"
|
||||
"github.com/lbryio/ytsync/thumbs"
|
||||
"github.com/lbryio/ytsync/v5/tags_manager"
|
||||
"github.com/lbryio/ytsync/v5/thumbs"
|
||||
|
||||
"github.com/shopspring/decimal"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"google.golang.org/api/googleapi/transport"
|
||||
"google.golang.org/api/youtube/v3"
|
||||
)
|
||||
|
||||
func (s *Sync) enableAddressReuse() error {
|
||||
@@ -49,6 +50,10 @@ func (s *Sync) enableAddressReuse() error {
|
||||
return nil
|
||||
}
|
||||
func (s *Sync) walletSetup() error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("walletSetup").Add(time.Since(start))
|
||||
}(start)
|
||||
//prevent unnecessary concurrent execution and publishing while refilling/reallocating UTXOs
|
||||
s.walletMux.Lock()
|
||||
defer s.walletMux.Unlock()
|
||||
@@ -69,11 +74,7 @@ func (s *Sync) walletSetup() error {
|
||||
}
|
||||
log.Debugf("Starting balance is %.4f", balance)
|
||||
|
||||
n, err := s.CountVideos()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
videosOnYoutube := int(n)
|
||||
videosOnYoutube := int(s.DbChannelData.TotalVideos)
|
||||
|
||||
log.Debugf("Source channel has %d videos", videosOnYoutube)
|
||||
if videosOnYoutube == 0 {
|
||||
@@ -98,17 +99,17 @@ func (s *Sync) walletSetup() error {
|
||||
|
||||
log.Debugf("We already allocated credits for %d published videos and %d failed videos", publishedCount, failedCount)
|
||||
|
||||
if videosOnYoutube > s.Manager.videosLimit {
|
||||
videosOnYoutube = s.Manager.videosLimit
|
||||
if videosOnYoutube > s.Manager.CliFlags.VideosToSync(s.DbChannelData.TotalSubscribers) {
|
||||
videosOnYoutube = s.Manager.CliFlags.VideosToSync(s.DbChannelData.TotalSubscribers)
|
||||
}
|
||||
unallocatedVideos := videosOnYoutube - (publishedCount + failedCount)
|
||||
channelFee := channelClaimAmount
|
||||
channelAlreadyClaimed := s.lbryChannelID != ""
|
||||
channelAlreadyClaimed := s.DbChannelData.ChannelClaimID != ""
|
||||
if channelAlreadyClaimed {
|
||||
channelFee = 0.0
|
||||
}
|
||||
requiredBalance := float64(unallocatedVideos)*(publishAmount+estimatedMaxTxFee) + channelFee
|
||||
if s.Manager.SyncFlags.UpgradeMetadata {
|
||||
if s.Manager.CliFlags.UpgradeMetadata {
|
||||
requiredBalance += float64(notUpgradedCount) * 0.001
|
||||
}
|
||||
|
||||
@@ -117,8 +118,8 @@ func (s *Sync) walletSetup() error {
|
||||
refillAmount = math.Max(math.Max(requiredBalance-balance, minimumAccountBalance-balance), minimumRefillAmount)
|
||||
}
|
||||
|
||||
if s.Refill > 0 {
|
||||
refillAmount += float64(s.Refill)
|
||||
if s.Manager.CliFlags.Refill > 0 {
|
||||
refillAmount += float64(s.Manager.CliFlags.Refill)
|
||||
}
|
||||
|
||||
if refillAmount > 0 {
|
||||
@@ -134,12 +135,11 @@ func (s *Sync) walletSetup() error {
|
||||
} else if claimAddress == nil {
|
||||
return errors.Err("could not get an address")
|
||||
}
|
||||
s.claimAddress = string(claimAddress.Items[0].Address)
|
||||
if s.claimAddress == "" {
|
||||
return errors.Err("found blank claim address")
|
||||
if s.DbChannelData.PublishAddress == "" || !s.shouldTransfer() {
|
||||
s.DbChannelData.PublishAddress = string(claimAddress.Items[0].Address)
|
||||
}
|
||||
if s.shouldTransfer() {
|
||||
s.claimAddress = s.clientPublishAddress
|
||||
if s.DbChannelData.PublishAddress == "" {
|
||||
return errors.Err("found blank claim address")
|
||||
}
|
||||
|
||||
err = s.ensureEnoughUTXOs()
|
||||
@@ -151,6 +151,10 @@ func (s *Sync) walletSetup() error {
|
||||
}
|
||||
|
||||
func (s *Sync) getDefaultAccount() (string, error) {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("getDefaultAccount").Add(time.Since(start))
|
||||
}(start)
|
||||
if s.defaultAccountID == "" {
|
||||
accountsResponse, err := s.daemon.AccountList(1, 50)
|
||||
if err != nil {
|
||||
@@ -177,6 +181,10 @@ func (s *Sync) getDefaultAccount() (string, error) {
|
||||
}
|
||||
|
||||
func (s *Sync) ensureEnoughUTXOs() error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("ensureEnoughUTXOs").Add(time.Since(start))
|
||||
}(start)
|
||||
defaultAccount, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -196,7 +204,7 @@ func (s *Sync) ensureEnoughUTXOs() error {
|
||||
|
||||
for _, utxo := range utxolist.Items {
|
||||
amount, _ := strconv.ParseFloat(utxo.Amount, 64)
|
||||
if utxo.IsMine && utxo.Type == "payment" && amount > 0.001 {
|
||||
if utxo.IsMyOutput && utxo.Type == "payment" && amount > 0.001 {
|
||||
if utxo.Confirmations > 0 {
|
||||
confirmedCount++
|
||||
}
|
||||
@@ -217,11 +225,23 @@ func (s *Sync) ensureEnoughUTXOs() error {
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
//this is dumb but sometimes the balance is negative and it breaks everything, so let's check again
|
||||
if balanceAmount < 0 {
|
||||
log.Infof("negative balance of %.2f found. Waiting to retry...", balanceAmount)
|
||||
time.Sleep(10 * time.Second)
|
||||
balanceAmount, err = strconv.ParseFloat(balance.Available.String(), 64)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
}
|
||||
maxUTXOs := uint64(500)
|
||||
desiredUTXOCount := uint64(math.Floor((balanceAmount) / 0.1))
|
||||
if desiredUTXOCount > maxUTXOs {
|
||||
desiredUTXOCount = maxUTXOs
|
||||
}
|
||||
if desiredUTXOCount < uint64(confirmedCount) {
|
||||
return nil
|
||||
}
|
||||
availableBalance, _ := balance.Available.Float64()
|
||||
log.Infof("Splitting balance of %.3f evenly between %d UTXOs", availableBalance, desiredUTXOCount)
|
||||
|
||||
@@ -250,11 +270,14 @@ func (s *Sync) ensureEnoughUTXOs() error {
|
||||
}
|
||||
|
||||
func (s *Sync) waitForNewBlock() error {
|
||||
defer func(start time.Time) { timing.TimedComponent("waitForNewBlock").Add(time.Since(start)) }(time.Now())
|
||||
|
||||
log.Printf("regtest: %t, docker: %t", logUtils.IsRegTest(), logUtils.IsUsingDocker())
|
||||
status, err := s.daemon.Status()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for status.Wallet.Blocks == 0 || status.Wallet.BlocksBehind != 0 {
|
||||
time.Sleep(5 * time.Second)
|
||||
status, err = s.daemon.Status()
|
||||
@@ -280,18 +303,21 @@ func (s *Sync) waitForNewBlock() error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
time.Sleep(5 * time.Second)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Sync) GenerateRegtestBlock() error {
|
||||
lbrycrd, err := logUtils.GetLbrycrdClient(s.LbrycrdString)
|
||||
lbrycrd, err := logUtils.GetLbrycrdClient(s.Manager.LbrycrdDsn)
|
||||
if err != nil {
|
||||
return errors.Prefix("error getting lbrycrd client: ", err)
|
||||
}
|
||||
|
||||
txs, err := lbrycrd.Generate(1)
|
||||
if err != nil {
|
||||
return errors.Prefix("error generating new block: ", err)
|
||||
}
|
||||
|
||||
for _, tx := range txs {
|
||||
log.Info("Generated tx: ", tx.String())
|
||||
}
|
||||
@@ -299,14 +325,13 @@ func (s *Sync) GenerateRegtestBlock() error {
|
||||
}
|
||||
|
||||
func (s *Sync) ensureChannelOwnership() error {
|
||||
if s.LbryChannelName == "" {
|
||||
defer func(start time.Time) { timing.TimedComponent("ensureChannelOwnership").Add(time.Since(start)) }(time.Now())
|
||||
|
||||
if s.DbChannelData.DesiredChannelName == "" {
|
||||
return errors.Err("no channel name set")
|
||||
}
|
||||
|
||||
if s.transferState == TransferStateComplete {
|
||||
return nil
|
||||
}
|
||||
channels, err := s.daemon.ChannelList(nil, 1, 50, nil)
|
||||
channels, err := s.daemon.ChannelList(nil, 1, 500, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
} else if channels == nil {
|
||||
@@ -315,23 +340,27 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
|
||||
var channelToUse *jsonrpc.Transaction
|
||||
if len((*channels).Items) > 0 {
|
||||
if s.lbryChannelID == "" {
|
||||
if s.DbChannelData.ChannelClaimID == "" {
|
||||
return errors.Err("this channel does not have a recorded claimID in the database. To prevent failures, updates are not supported until an entry is manually added in the database")
|
||||
}
|
||||
for _, c := range (*channels).Items {
|
||||
log.Debugf("checking listed channel %s (%s)", c.ClaimID, c.Name)
|
||||
if c.ClaimID != s.lbryChannelID {
|
||||
if c.ClaimID != s.DbChannelData.ChannelClaimID {
|
||||
continue
|
||||
}
|
||||
if c.Name != s.LbryChannelName {
|
||||
if c.Name != s.DbChannelData.DesiredChannelName {
|
||||
return errors.Err("the channel in the wallet is different than the channel in the database")
|
||||
}
|
||||
channelToUse = &c
|
||||
break
|
||||
}
|
||||
if channelToUse == nil {
|
||||
return errors.Err("this wallet has channels but not a single one is ours! Expected claim_id: %s (%s)", s.lbryChannelID, s.LbryChannelName)
|
||||
return errors.Err("this wallet has channels but not a single one is ours! Expected claim_id: %s (%s)", s.DbChannelData.ChannelClaimID, s.DbChannelData.DesiredChannelName)
|
||||
}
|
||||
} else if s.DbChannelData.TransferState == shared.TransferStateComplete {
|
||||
return errors.Err("the channel was transferred but appears to have been abandoned!")
|
||||
} else if s.DbChannelData.ChannelClaimID != "" {
|
||||
return errors.Err("the database has a channel recorded (%s) but nothing was found in our control", s.DbChannelData.ChannelClaimID)
|
||||
}
|
||||
|
||||
channelUsesOldMetadata := false
|
||||
@@ -361,36 +390,33 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
client := &http.Client{
|
||||
Transport: &transport.APIKey{Key: s.APIConfig.YoutubeAPIKey},
|
||||
}
|
||||
|
||||
service, err := youtube.New(client)
|
||||
channelInfo, err := ytapi.ChannelInfo(s.DbChannelData.ChannelId)
|
||||
if err != nil {
|
||||
return errors.Prefix("error creating YouTube service", err)
|
||||
if strings.Contains(err.Error(), "invalid character 'e' looking for beginning of value") {
|
||||
logUtils.SendInfoToSlack("failed to get channel data for %s. Waiting 1 minute to retry", s.DbChannelData.ChannelId)
|
||||
time.Sleep(1 * time.Minute)
|
||||
channelInfo, err = ytapi.ChannelInfo(s.DbChannelData.ChannelId)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
response, err := service.Channels.List("snippet,brandingSettings").Id(s.YoutubeChannelID).Do()
|
||||
if err != nil {
|
||||
return errors.Prefix("error getting channel details", err)
|
||||
}
|
||||
|
||||
if len(response.Items) < 1 {
|
||||
return errors.Err("youtube channel not found")
|
||||
}
|
||||
|
||||
channelInfo := response.Items[0].Snippet
|
||||
channelBranding := response.Items[0].BrandingSettings
|
||||
|
||||
thumbnail := thumbs.GetBestThumbnail(channelInfo.Thumbnails)
|
||||
thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail.Url, s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
|
||||
thumbnail := channelInfo.Header.C4TabbedHeaderRenderer.Avatar.Thumbnails[len(channelInfo.Header.C4TabbedHeaderRenderer.Avatar.Thumbnails)-1].URL
|
||||
thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail, s.DbChannelData.ChannelId, *s.Manager.AwsConfigs.GetS3AWSConfig())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var bannerURL *string
|
||||
if channelBranding.Image != nil && channelBranding.Image.BannerImageUrl != "" {
|
||||
bURL, err := thumbs.MirrorThumbnail(channelBranding.Image.BannerImageUrl, "banner-"+s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
|
||||
if channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails != nil {
|
||||
bURL, err := thumbs.MirrorThumbnail(channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails[len(channelInfo.Header.C4TabbedHeaderRenderer.Banner.Thumbnails)-1].URL,
|
||||
"banner-"+s.DbChannelData.ChannelId,
|
||||
*s.Manager.AwsConfigs.GetS3AWSConfig(),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -398,37 +424,43 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
}
|
||||
|
||||
var languages []string = nil
|
||||
if channelInfo.DefaultLanguage != "" {
|
||||
if channelInfo.DefaultLanguage == "iw" {
|
||||
channelInfo.DefaultLanguage = "he"
|
||||
}
|
||||
languages = []string{channelInfo.DefaultLanguage}
|
||||
}
|
||||
//we don't have this data without the API
|
||||
//if channelInfo.DefaultLanguage != "" {
|
||||
// if channelInfo.DefaultLanguage == "iw" {
|
||||
// channelInfo.DefaultLanguage = "he"
|
||||
// }
|
||||
// languages = []string{channelInfo.DefaultLanguage}
|
||||
//}
|
||||
var locations []jsonrpc.Location = nil
|
||||
if channelInfo.Country != "" {
|
||||
locations = []jsonrpc.Location{{Country: util.PtrToString(channelInfo.Country)}}
|
||||
if channelInfo.Topbar.DesktopTopbarRenderer.CountryCode != "" {
|
||||
locations = []jsonrpc.Location{{Country: &channelInfo.Topbar.DesktopTopbarRenderer.CountryCode}}
|
||||
}
|
||||
var c *jsonrpc.TransactionSummary
|
||||
claimCreateOptions := jsonrpc.ClaimCreateOptions{
|
||||
Title: &channelInfo.Title,
|
||||
Description: &channelInfo.Description,
|
||||
Tags: tags_manager.GetTagsForChannel(s.YoutubeChannelID),
|
||||
Title: &channelInfo.Microformat.MicroformatDataRenderer.Title,
|
||||
Description: &channelInfo.Metadata.ChannelMetadataRenderer.Description,
|
||||
Tags: tags_manager.GetTagsForChannel(s.DbChannelData.ChannelId),
|
||||
Languages: languages,
|
||||
Locations: locations,
|
||||
ThumbnailURL: &thumbnailURL,
|
||||
}
|
||||
if channelUsesOldMetadata {
|
||||
c, err = s.daemon.ChannelUpdate(s.lbryChannelID, jsonrpc.ChannelUpdateOptions{
|
||||
ClearTags: util.PtrToBool(true),
|
||||
ClearLocations: util.PtrToBool(true),
|
||||
ClearLanguages: util.PtrToBool(true),
|
||||
ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
|
||||
ClaimCreateOptions: claimCreateOptions,
|
||||
CoverURL: bannerURL,
|
||||
},
|
||||
})
|
||||
if s.DbChannelData.TransferState <= 1 {
|
||||
c, err = s.daemon.ChannelUpdate(s.DbChannelData.ChannelClaimID, jsonrpc.ChannelUpdateOptions{
|
||||
ClearTags: util.PtrToBool(true),
|
||||
ClearLocations: util.PtrToBool(true),
|
||||
ClearLanguages: util.PtrToBool(true),
|
||||
ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
|
||||
ClaimCreateOptions: claimCreateOptions,
|
||||
CoverURL: bannerURL,
|
||||
},
|
||||
})
|
||||
} else {
|
||||
logUtils.SendInfoToSlack("%s (%s) has a channel with old metadata but isn't in our control anymore. Ignoring", s.DbChannelData.DesiredChannelName, s.DbChannelData.ChannelClaimID)
|
||||
return nil
|
||||
}
|
||||
} else {
|
||||
c, err = s.daemon.ChannelCreate(s.LbryChannelName, channelBidAmount, jsonrpc.ChannelCreateOptions{
|
||||
c, err = s.daemon.ChannelCreate(s.DbChannelData.DesiredChannelName, channelBidAmount, jsonrpc.ChannelCreateOptions{
|
||||
ClaimCreateOptions: claimCreateOptions,
|
||||
CoverURL: bannerURL,
|
||||
})
|
||||
@@ -437,13 +469,18 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.lbryChannelID = c.Outputs[0].ClaimID
|
||||
return s.Manager.apiConfig.SetChannelClaimID(s.YoutubeChannelID, s.lbryChannelID)
|
||||
|
||||
s.DbChannelData.ChannelClaimID = c.Outputs[0].ClaimID
|
||||
return s.Manager.ApiConfig.SetChannelClaimID(s.DbChannelData.ChannelId, s.DbChannelData.ChannelClaimID)
|
||||
}
|
||||
|
||||
func (s *Sync) addCredits(amountToAdd float64) error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("addCredits").Add(time.Since(start))
|
||||
}(start)
|
||||
log.Printf("Adding %f credits", amountToAdd)
|
||||
lbrycrdd, err := logUtils.GetLbrycrdClient(s.LbrycrdString)
|
||||
lbrycrdd, err := logUtils.GetLbrycrdClient(s.Manager.LbrycrdDsn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
+61
-25
@@ -10,20 +10,25 @@ import (
|
||||
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
"github.com/lbryio/ytsync/v5/shared"
|
||||
"github.com/lbryio/ytsync/v5/timing"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func waitConfirmations(s *Sync) error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("waitConfirmations").Add(time.Since(start))
|
||||
}(start)
|
||||
defaultAccount, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
allConfirmed := false
|
||||
waitCount := 0
|
||||
waiting:
|
||||
for !allConfirmed {
|
||||
for !allConfirmed && waitCount < 2 {
|
||||
utxolist, err := s.daemon.UTXOList(&defaultAccount, 1, 10000)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -37,6 +42,7 @@ waiting:
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
waitCount++
|
||||
continue waiting
|
||||
}
|
||||
}
|
||||
@@ -52,6 +58,10 @@ type abandonResponse struct {
|
||||
}
|
||||
|
||||
func abandonSupports(s *Sync) (float64, error) {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("abandonSupports").Add(time.Since(start))
|
||||
}(start)
|
||||
totalPages := uint64(1)
|
||||
var allSupports []jsonrpc.Claim
|
||||
defaultAccount, err := s.getDefaultAccount()
|
||||
@@ -61,7 +71,10 @@ func abandonSupports(s *Sync) (float64, error) {
|
||||
for page := uint64(1); page <= totalPages; page++ {
|
||||
supports, err := s.daemon.SupportList(&defaultAccount, page, 50)
|
||||
if err != nil {
|
||||
return 0, errors.Prefix("cannot list claims", err)
|
||||
supports, err = s.daemon.SupportList(&defaultAccount, page, 50)
|
||||
if err != nil {
|
||||
return 0, errors.Prefix("cannot list supports", err)
|
||||
}
|
||||
}
|
||||
allSupports = append(allSupports, (*supports).Items...)
|
||||
totalPages = (*supports).TotalPages
|
||||
@@ -85,22 +98,23 @@ func abandonSupports(s *Sync) (float64, error) {
|
||||
}()
|
||||
consumerWG := &stop.Group{}
|
||||
//TODO: remove this once the SDK team fixes their RPC bugs....
|
||||
s.daemon.SetRPCTimeout(30 * time.Second)
|
||||
defer s.daemon.SetRPCTimeout(40 * time.Minute)
|
||||
for i := 0; i < s.ConcurrentVideos; i++ {
|
||||
s.daemon.SetRPCTimeout(60 * time.Second)
|
||||
defer s.daemon.SetRPCTimeout(5 * time.Minute)
|
||||
for i := 0; i < s.Manager.CliFlags.ConcurrentJobs; i++ {
|
||||
consumerWG.Add(1)
|
||||
go func() {
|
||||
defer consumerWG.Done()
|
||||
outer:
|
||||
for {
|
||||
claimID, more := <-claimIDChan
|
||||
if !more {
|
||||
return
|
||||
} else {
|
||||
summary, err := s.daemon.SupportAbandon(&claimID, nil, nil, nil, &defaultAccount)
|
||||
summary, err := s.daemon.TxoSpend(util.PtrToString("support"), &claimID, nil, nil, nil, &defaultAccount)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "Client.Timeout exceeded while awaiting headers") {
|
||||
log.Errorf("Support abandon for %s timed out, retrying...", claimID)
|
||||
summary, err = s.daemon.SupportAbandon(&claimID, nil, nil, nil, &defaultAccount)
|
||||
summary, err = s.daemon.TxoSpend(util.PtrToString("support"), &claimID, nil, nil, nil, &defaultAccount)
|
||||
if err != nil {
|
||||
//TODO GUESS HOW MUCH LBC WAS RELEASED THAT WE DON'T KNOW ABOUT, because screw you SDK
|
||||
abandonRspChan <- abandonResponse{
|
||||
@@ -119,7 +133,7 @@ func abandonSupports(s *Sync) (float64, error) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if len(summary.Outputs) < 1 {
|
||||
if summary == nil || len(*summary) < 1 {
|
||||
abandonRspChan <- abandonResponse{
|
||||
ClaimID: claimID,
|
||||
Error: errors.Err("error abandoning supports: no outputs while abandoning %s", claimID),
|
||||
@@ -127,7 +141,19 @@ func abandonSupports(s *Sync) (float64, error) {
|
||||
}
|
||||
continue
|
||||
}
|
||||
outputAmount, err := strconv.ParseFloat(summary.Outputs[0].Amount, 64)
|
||||
var outputAmount float64
|
||||
for _, tx := range *summary {
|
||||
amount, err := strconv.ParseFloat(tx.Outputs[0].Amount, 64)
|
||||
if err != nil {
|
||||
abandonRspChan <- abandonResponse{
|
||||
ClaimID: claimID,
|
||||
Error: errors.Err(err),
|
||||
Amount: 0,
|
||||
}
|
||||
continue outer
|
||||
}
|
||||
outputAmount += amount
|
||||
}
|
||||
if err != nil {
|
||||
abandonRspChan <- abandonResponse{
|
||||
ClaimID: claimID,
|
||||
@@ -166,13 +192,17 @@ func abandonSupports(s *Sync) (float64, error) {
|
||||
type updateInfo struct {
|
||||
ClaimID string
|
||||
streamUpdateOptions *jsonrpc.StreamUpdateOptions
|
||||
videoStatus *sdk.VideoStatus
|
||||
videoStatus *shared.VideoStatus
|
||||
}
|
||||
|
||||
func transferVideos(s *Sync) error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("transferVideos").Add(time.Since(start))
|
||||
}(start)
|
||||
cleanTransfer := true
|
||||
|
||||
streamChan := make(chan updateInfo, s.ConcurrentVideos)
|
||||
streamChan := make(chan updateInfo, s.Manager.CliFlags.ConcurrentJobs)
|
||||
account, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -186,13 +216,13 @@ func transferVideos(s *Sync) error {
|
||||
go func() {
|
||||
defer producerWG.Done()
|
||||
for _, video := range s.syncedVideos {
|
||||
if !video.Published || video.Transferred || video.MetadataVersion != LatestMetadataVersion {
|
||||
if !video.Published || video.Transferred || video.MetadataVersion != shared.LatestMetadataVersion {
|
||||
continue
|
||||
}
|
||||
|
||||
var stream *jsonrpc.Claim = nil
|
||||
for _, c := range streams.Items {
|
||||
if c.ClaimID != video.ClaimID || (c.SigningChannel != nil && c.SigningChannel.ClaimID != s.lbryChannelID) {
|
||||
if c.ClaimID != video.ClaimID || (c.SigningChannel != nil && c.SigningChannel.ClaimID != s.DbChannelData.ChannelClaimID) {
|
||||
continue
|
||||
}
|
||||
stream = &c
|
||||
@@ -205,7 +235,7 @@ func transferVideos(s *Sync) error {
|
||||
streamUpdateOptions := jsonrpc.StreamUpdateOptions{
|
||||
StreamCreateOptions: &jsonrpc.StreamCreateOptions{
|
||||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
|
||||
ClaimAddress: &s.clientPublishAddress,
|
||||
ClaimAddress: &s.DbChannelData.PublishAddress,
|
||||
FundingAccountIDs: []string{
|
||||
account,
|
||||
},
|
||||
@@ -213,12 +243,12 @@ func transferVideos(s *Sync) error {
|
||||
},
|
||||
Bid: util.PtrToString("0.005"), // Todo - Dont hardcode
|
||||
}
|
||||
videoStatus := sdk.VideoStatus{
|
||||
ChannelID: s.YoutubeChannelID,
|
||||
videoStatus := shared.VideoStatus{
|
||||
ChannelID: s.DbChannelData.ChannelId,
|
||||
VideoID: video.VideoID,
|
||||
ClaimID: video.ClaimID,
|
||||
ClaimName: video.ClaimName,
|
||||
Status: VideoStatusPublished,
|
||||
Status: shared.VideoStatusPublished,
|
||||
IsTransferred: util.PtrToBool(true),
|
||||
}
|
||||
streamChan <- updateInfo{
|
||||
@@ -230,7 +260,7 @@ func transferVideos(s *Sync) error {
|
||||
}()
|
||||
|
||||
consumerWG := &stop.Group{}
|
||||
for i := 0; i < s.ConcurrentVideos; i++ {
|
||||
for i := 0; i < s.Manager.CliFlags.ConcurrentJobs; i++ {
|
||||
consumerWG.Add(1)
|
||||
go func(worker int) {
|
||||
defer consumerWG.Done()
|
||||
@@ -258,16 +288,18 @@ func transferVideos(s *Sync) error {
|
||||
}
|
||||
|
||||
func (s *Sync) streamUpdate(ui *updateInfo) error {
|
||||
start := time.Now()
|
||||
result, updateError := s.daemon.StreamUpdate(ui.ClaimID, *ui.streamUpdateOptions)
|
||||
timing.TimedComponent("transferStreamUpdate").Add(time.Since(start))
|
||||
if updateError != nil {
|
||||
ui.videoStatus.FailureReason = updateError.Error()
|
||||
ui.videoStatus.Status = VideoStatusTranferFailed
|
||||
ui.videoStatus.Status = shared.VideoStatusTranferFailed
|
||||
ui.videoStatus.IsTransferred = util.PtrToBool(false)
|
||||
} else {
|
||||
ui.videoStatus.IsTransferred = util.PtrToBool(len(result.Outputs) != 0)
|
||||
}
|
||||
log.Infof("TRANSFERRED %t", *ui.videoStatus.IsTransferred)
|
||||
statusErr := s.APIConfig.MarkVideoStatus(*ui.videoStatus)
|
||||
statusErr := s.Manager.ApiConfig.MarkVideoStatus(*ui.videoStatus)
|
||||
if statusErr != nil {
|
||||
return errors.Prefix(statusErr.Error(), updateError)
|
||||
}
|
||||
@@ -275,6 +307,10 @@ func (s *Sync) streamUpdate(ui *updateInfo) error {
|
||||
}
|
||||
|
||||
func transferChannel(s *Sync) error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("transferChannel").Add(time.Since(start))
|
||||
}(start)
|
||||
account, err := s.getDefaultAccount()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -285,7 +321,7 @@ func transferChannel(s *Sync) error {
|
||||
}
|
||||
var channelClaim *jsonrpc.Transaction = nil
|
||||
for _, c := range channelClaims.Items {
|
||||
if c.ClaimID != s.lbryChannelID {
|
||||
if c.ClaimID != s.DbChannelData.ChannelClaimID {
|
||||
continue
|
||||
}
|
||||
channelClaim = &c
|
||||
@@ -299,11 +335,11 @@ func transferChannel(s *Sync) error {
|
||||
Bid: util.PtrToString(fmt.Sprintf("%.6f", channelClaimAmount-0.005)),
|
||||
ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
|
||||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
|
||||
ClaimAddress: &s.clientPublishAddress,
|
||||
ClaimAddress: &s.DbChannelData.PublishAddress,
|
||||
},
|
||||
},
|
||||
}
|
||||
result, err := s.daemon.ChannelUpdate(s.lbryChannelID, updateOptions)
|
||||
result, err := s.daemon.ChannelUpdate(s.DbChannelData.ChannelClaimID, updateOptions)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
+267
-441
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,31 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"os"
|
||||
"regexp"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
var (
|
||||
Durations = promauto.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: "ytsync",
|
||||
Subsystem: getHostname(),
|
||||
Name: "duration",
|
||||
Help: "The durations of the individual modules",
|
||||
}, []string{"path"})
|
||||
)
|
||||
|
||||
func getHostname() string {
|
||||
hostname, err := os.Hostname()
|
||||
if err != nil {
|
||||
hostname = "ytsync_unknown"
|
||||
}
|
||||
reg, err := regexp.Compile("[^a-zA-Z0-9_]+")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
return reg.ReplaceAllString(hostname, "_")
|
||||
}
|
||||
+142
-58
@@ -13,8 +13,9 @@ import (
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/null"
|
||||
"github.com/lbryio/ytsync/v5/shared"
|
||||
|
||||
"github.com/lbryio/ytsync/util"
|
||||
"github.com/lbryio/ytsync/v5/util"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
@@ -30,57 +31,28 @@ type APIConfig struct {
|
||||
HostName string
|
||||
}
|
||||
|
||||
type SyncProperties struct {
|
||||
SyncFrom int64
|
||||
SyncUntil int64
|
||||
YoutubeChannelID string
|
||||
}
|
||||
|
||||
type SyncFlags struct {
|
||||
StopOnError bool
|
||||
TakeOverExistingChannel bool
|
||||
SkipSpaceCheck bool
|
||||
SyncUpdate bool
|
||||
SingleRun bool
|
||||
RemoveDBUnpublished bool
|
||||
UpgradeMetadata bool
|
||||
DisableTransfers bool
|
||||
QuickSync bool
|
||||
}
|
||||
|
||||
type Fee struct {
|
||||
Amount string `json:"amount"`
|
||||
Address string `json:"address"`
|
||||
Currency string `json:"currency"`
|
||||
}
|
||||
type YoutubeChannel struct {
|
||||
ChannelId string `json:"channel_id"`
|
||||
TotalVideos uint `json:"total_videos"`
|
||||
DesiredChannelName string `json:"desired_channel_name"`
|
||||
Fee *Fee `json:"fee"`
|
||||
ChannelClaimID string `json:"channel_claim_id"`
|
||||
TransferState int `json:"transfer_state"`
|
||||
PublishAddress string `json:"publish_address"`
|
||||
PublicKey string `json:"public_key"`
|
||||
}
|
||||
|
||||
func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeChannel, error) {
|
||||
func (a *APIConfig) FetchChannels(status string, cliFlags *shared.SyncFlags) ([]shared.YoutubeChannel, error) {
|
||||
type apiJobsResponse struct {
|
||||
Success bool `json:"success"`
|
||||
Error null.String `json:"error"`
|
||||
Data []YoutubeChannel `json:"data"`
|
||||
Success bool `json:"success"`
|
||||
Error null.String `json:"error"`
|
||||
Data []shared.YoutubeChannel `json:"data"`
|
||||
}
|
||||
endpoint := a.ApiURL + "/yt/jobs"
|
||||
res, err := http.PostForm(endpoint, url.Values{
|
||||
"auth_token": {a.ApiToken},
|
||||
"sync_status": {status},
|
||||
"min_videos": {strconv.Itoa(1)},
|
||||
"after": {strconv.Itoa(int(cp.SyncFrom))},
|
||||
"before": {strconv.Itoa(int(cp.SyncUntil))},
|
||||
"after": {strconv.Itoa(int(cliFlags.SyncFrom))},
|
||||
"before": {strconv.Itoa(int(cliFlags.SyncUntil))},
|
||||
"sync_server": {a.HostName},
|
||||
"channel_id": {cp.YoutubeChannelID},
|
||||
"channel_id": {cliFlags.ChannelID},
|
||||
})
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.FetchChannels(status, cliFlags)
|
||||
}
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
@@ -89,7 +61,7 @@ func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeC
|
||||
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
|
||||
log.Debugln(string(body))
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.FetchChannels(status, cp)
|
||||
return a.FetchChannels(status, cliFlags)
|
||||
}
|
||||
var response apiJobsResponse
|
||||
err = json.Unmarshal(body, &response)
|
||||
@@ -112,6 +84,7 @@ type SyncedVideo struct {
|
||||
Size int64 `json:"size"`
|
||||
MetadataVersion int8 `json:"metadata_version"`
|
||||
Transferred bool `json:"transferred"`
|
||||
IsLbryFirst bool `json:"is_lbry_first"`
|
||||
}
|
||||
|
||||
func sanitizeFailureReason(s *string) {
|
||||
@@ -124,7 +97,6 @@ func sanitizeFailureReason(s *string) {
|
||||
}
|
||||
|
||||
func (a *APIConfig) SetChannelCert(certHex string, channelID string) error {
|
||||
|
||||
type apiSetChannelCertResponse struct {
|
||||
Success bool `json:"success"`
|
||||
Error null.String `json:"error"`
|
||||
@@ -139,13 +111,18 @@ func (a *APIConfig) SetChannelCert(certHex string, channelID string) error {
|
||||
"auth_token": {a.ApiToken},
|
||||
})
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.SetChannelCert(certHex, channelID)
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
if res.StatusCode != http.StatusOK {
|
||||
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
|
||||
log.Debugln(body)
|
||||
log.Debugln(string(body))
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.SetChannelCert(certHex, channelID)
|
||||
}
|
||||
@@ -182,6 +159,11 @@ func (a *APIConfig) SetChannelStatus(channelID string, status string, failureRea
|
||||
}
|
||||
res, err := http.PostForm(endpoint, params)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.SetChannelStatus(channelID, status, failureReason, transferState)
|
||||
}
|
||||
return nil, nil, errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
@@ -227,6 +209,11 @@ func (a *APIConfig) SetChannelClaimID(channelID string, channelClaimID string) e
|
||||
"channel_claim_id": {channelClaimID},
|
||||
})
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.SetChannelClaimID(channelID, channelClaimID)
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
@@ -266,6 +253,11 @@ func (a *APIConfig) DeleteVideos(videos []string) error {
|
||||
}
|
||||
res, err := http.PostForm(endpoint, vals)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.DeleteVideos(videos)
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
@@ -295,19 +287,7 @@ func (a *APIConfig) DeleteVideos(videos []string) error {
|
||||
return errors.Err("invalid API response. Status code: %d", res.StatusCode)
|
||||
}
|
||||
|
||||
type VideoStatus struct {
|
||||
ChannelID string
|
||||
VideoID string
|
||||
Status string
|
||||
ClaimID string
|
||||
ClaimName string
|
||||
FailureReason string
|
||||
Size *int64
|
||||
MetaDataVersion uint
|
||||
IsTransferred *bool
|
||||
}
|
||||
|
||||
func (a *APIConfig) MarkVideoStatus(status VideoStatus) error {
|
||||
func (a *APIConfig) MarkVideoStatus(status shared.VideoStatus) error {
|
||||
endpoint := a.ApiURL + "/yt/video_status"
|
||||
|
||||
sanitizeFailureReason(&status.FailureReason)
|
||||
@@ -339,6 +319,11 @@ func (a *APIConfig) MarkVideoStatus(status VideoStatus) error {
|
||||
}
|
||||
res, err := http.PostForm(endpoint, vals)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.MarkVideoStatus(status)
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
@@ -366,3 +351,102 @@ func (a *APIConfig) MarkVideoStatus(status VideoStatus) error {
|
||||
}
|
||||
return errors.Err("invalid API response. Status code: %d", res.StatusCode)
|
||||
}
|
||||
|
||||
func (a *APIConfig) VideoState(videoID string) (string, error) {
|
||||
endpoint := a.ApiURL + "/yt/video_state"
|
||||
vals := url.Values{
|
||||
"video_id": {videoID},
|
||||
"auth_token": {a.ApiToken},
|
||||
}
|
||||
|
||||
res, err := http.PostForm(endpoint, vals)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.VideoState(videoID)
|
||||
}
|
||||
return "", errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
if res.StatusCode == http.StatusNotFound {
|
||||
return "not_found", nil
|
||||
}
|
||||
if res.StatusCode != http.StatusOK {
|
||||
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
|
||||
log.Debugln(string(body))
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.VideoState(videoID)
|
||||
}
|
||||
var response struct {
|
||||
Success bool `json:"success"`
|
||||
Error null.String `json:"error"`
|
||||
Data null.String `json:"data"`
|
||||
}
|
||||
err = json.Unmarshal(body, &response)
|
||||
if err != nil {
|
||||
return "", errors.Err(err)
|
||||
}
|
||||
if !response.Error.IsNull() {
|
||||
return "", errors.Err(response.Error.String)
|
||||
}
|
||||
if !response.Data.IsNull() {
|
||||
return response.Data.String, nil
|
||||
}
|
||||
return "", errors.Err("invalid API response. Status code: %d", res.StatusCode)
|
||||
}
|
||||
|
||||
type VideoRelease struct {
|
||||
ID uint64 `json:"id"`
|
||||
YoutubeDataID uint64 `json:"youtube_data_id"`
|
||||
VideoID string `json:"video_id"`
|
||||
ReleaseTime string `json:"release_time"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
}
|
||||
|
||||
func (a *APIConfig) GetReleasedDate(videoID string) (*VideoRelease, error) {
|
||||
endpoint := a.ApiURL + "/yt/released"
|
||||
vals := url.Values{
|
||||
"video_id": {videoID},
|
||||
"auth_token": {a.ApiToken},
|
||||
}
|
||||
|
||||
res, err := http.PostForm(endpoint, vals)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "EOF") {
|
||||
util.SendErrorToSlack("EOF error while trying to call %s. Waiting to retry", endpoint)
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.GetReleasedDate(videoID)
|
||||
}
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
if res.StatusCode == http.StatusNotFound {
|
||||
return nil, nil
|
||||
}
|
||||
if res.StatusCode != http.StatusOK {
|
||||
util.SendErrorToSlack("Error %d while trying to call %s. Waiting to retry", res.StatusCode, endpoint)
|
||||
log.Debugln(string(body))
|
||||
time.Sleep(30 * time.Second)
|
||||
return a.GetReleasedDate(videoID)
|
||||
}
|
||||
var response struct {
|
||||
Success bool `json:"success"`
|
||||
Error null.String `json:"error"`
|
||||
Data VideoRelease `json:"data"`
|
||||
}
|
||||
err = json.Unmarshal(body, &response)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
if !response.Error.IsNull() {
|
||||
return nil, errors.Err(response.Error.String)
|
||||
}
|
||||
if response.Data.ReleaseTime != "" {
|
||||
return &response.Data, nil
|
||||
}
|
||||
return nil, errors.Err("invalid API response. Status code: %d", res.StatusCode)
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package shared
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
)
|
||||
|
||||
type Fee struct {
|
||||
Amount string `json:"amount"`
|
||||
Address string `json:"address"`
|
||||
Currency string `json:"currency"`
|
||||
}
|
||||
type YoutubeChannel struct {
|
||||
ChannelId string `json:"channel_id"`
|
||||
TotalVideos uint `json:"total_videos"`
|
||||
TotalSubscribers uint `json:"total_subscribers"`
|
||||
DesiredChannelName string `json:"desired_channel_name"`
|
||||
Fee *Fee `json:"fee"`
|
||||
ChannelClaimID string `json:"channel_claim_id"`
|
||||
TransferState int `json:"transfer_state"`
|
||||
PublishAddress string `json:"publish_address"`
|
||||
PublicKey string `json:"public_key"`
|
||||
LengthLimit int `json:"length_limit"`
|
||||
SizeLimit int `json:"size_limit"`
|
||||
LastUploadedVideo string `json:"last_uploaded_video"`
|
||||
WipeDB bool `json:"wipe_db"`
|
||||
}
|
||||
|
||||
var NeverRetryFailures = []string{
|
||||
"Error extracting sts from embedded url response",
|
||||
"Unable to extract signature tokens",
|
||||
"the video is too big to sync, skipping for now",
|
||||
"video is too long to process",
|
||||
"video is too short to process",
|
||||
"This video contains content from",
|
||||
"no compatible format available for this video",
|
||||
"Watch this video on YouTube.",
|
||||
"have blocked it on copyright grounds",
|
||||
"giving up after 0 fragment retries",
|
||||
"Sign in to confirm your age",
|
||||
}
|
||||
|
||||
type SyncFlags struct {
|
||||
TakeOverExistingChannel bool
|
||||
SkipSpaceCheck bool
|
||||
SyncUpdate bool
|
||||
SingleRun bool
|
||||
RemoveDBUnpublished bool
|
||||
UpgradeMetadata bool
|
||||
DisableTransfers bool
|
||||
QuickSync bool
|
||||
MaxTries int
|
||||
Refill int
|
||||
Limit int
|
||||
Status string
|
||||
SecondaryStatus string
|
||||
ChannelID string
|
||||
SyncFrom int64
|
||||
SyncUntil int64
|
||||
ConcurrentJobs int
|
||||
VideosLimit int
|
||||
MaxVideoSize int
|
||||
MaxVideoLength time.Duration
|
||||
}
|
||||
|
||||
// VideosToSync dynamically figures out how many videos should be synced for a given subs count if nothing was otherwise specified
|
||||
func (f *SyncFlags) VideosToSync(totalSubscribers uint) int {
|
||||
if f.VideosLimit > 0 {
|
||||
return f.VideosLimit
|
||||
}
|
||||
defaultVideosToSync := map[int]int{
|
||||
10000: 1000,
|
||||
5000: 500,
|
||||
1000: 400,
|
||||
800: 250,
|
||||
600: 200,
|
||||
200: 80,
|
||||
100: 50,
|
||||
1: 10,
|
||||
}
|
||||
videosToSync := 0
|
||||
for s, r := range defaultVideosToSync {
|
||||
if int(totalSubscribers) >= s && r > videosToSync {
|
||||
videosToSync = r
|
||||
}
|
||||
}
|
||||
return videosToSync
|
||||
}
|
||||
|
||||
func (f *SyncFlags) IsSingleChannelSync() bool {
|
||||
return f.ChannelID != ""
|
||||
}
|
||||
|
||||
type VideoStatus struct {
|
||||
ChannelID string
|
||||
VideoID string
|
||||
Status string
|
||||
ClaimID string
|
||||
ClaimName string
|
||||
FailureReason string
|
||||
Size *int64
|
||||
MetaDataVersion uint
|
||||
IsTransferred *bool
|
||||
}
|
||||
|
||||
const (
|
||||
StatusPending = "pending" // waiting for permission to sync
|
||||
StatusPendingEmail = "pendingemail" // permission granted but missing email
|
||||
StatusQueued = "queued" // in sync queue. will be synced soon
|
||||
StatusPendingUpgrade = "pendingupgrade" // in sync queue. will be synced soon
|
||||
StatusSyncing = "syncing" // syncing now
|
||||
StatusSynced = "synced" // done
|
||||
StatusWipeDb = "pendingdbwipe" // in sync queue. lbryum database will be pruned
|
||||
StatusFailed = "failed"
|
||||
StatusFinalized = "finalized" // no more changes allowed
|
||||
StatusAbandoned = "abandoned" // deleted on youtube or banned
|
||||
)
|
||||
|
||||
var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusPendingUpgrade, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned, StatusWipeDb}
|
||||
|
||||
const LatestMetadataVersion = 2
|
||||
|
||||
const (
|
||||
VideoStatusPublished = "published"
|
||||
VideoStatusFailed = "failed"
|
||||
VideoStatusUpgradeFailed = "upgradefailed"
|
||||
VideoStatusUnpublished = "unpublished"
|
||||
VideoStatusTranferFailed = "transferfailed"
|
||||
)
|
||||
|
||||
const (
|
||||
TransferStateNotTouched = iota
|
||||
TransferStatePending
|
||||
TransferStateComplete
|
||||
TransferStateManual
|
||||
)
|
||||
|
||||
type AwsConfigs struct {
|
||||
AwsS3ID string
|
||||
AwsS3Secret string
|
||||
AwsS3Region string
|
||||
AwsS3Bucket string
|
||||
}
|
||||
|
||||
func (a *AwsConfigs) GetS3AWSConfig() *aws.Config {
|
||||
return &aws.Config{
|
||||
Credentials: credentials.NewStaticCredentials(a.AwsS3ID, a.AwsS3Secret, ""),
|
||||
Region: &a.AwsS3Region,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package shared
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gotest.tools/assert"
|
||||
)
|
||||
|
||||
func TestSyncFlags_VideosToSync(t *testing.T) {
|
||||
f := SyncFlags{}
|
||||
assert.Equal(t, f.VideosToSync(0), 0)
|
||||
assert.Equal(t, f.VideosToSync(1), 10)
|
||||
assert.Equal(t, f.VideosToSync(5), 10)
|
||||
assert.Equal(t, f.VideosToSync(10), 10)
|
||||
assert.Equal(t, f.VideosToSync(101), 50)
|
||||
assert.Equal(t, f.VideosToSync(500), 80)
|
||||
assert.Equal(t, f.VideosToSync(21000), 1000)
|
||||
f.VideosLimit = 1337
|
||||
assert.Equal(t, f.VideosToSync(21), 1337)
|
||||
}
|
||||
+1
-1
@@ -5,7 +5,7 @@ import (
|
||||
"sync"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
||||
"github.com/lbryio/ytsync/namer"
|
||||
"github.com/lbryio/ytsync/v5/namer"
|
||||
)
|
||||
|
||||
type SyncSummary struct {
|
||||
|
||||
+108
-47
@@ -1,9 +1,9 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
@@ -13,22 +13,27 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/abadojack/whatlanggo"
|
||||
"github.com/lbryio/ytsync/v5/downloader/ytdl"
|
||||
"github.com/lbryio/ytsync/v5/shared"
|
||||
"gopkg.in/vansante/go-ffprobe.v2"
|
||||
|
||||
"github.com/lbryio/ytsync/v5/ip_manager"
|
||||
"github.com/lbryio/ytsync/v5/namer"
|
||||
"github.com/lbryio/ytsync/v5/sdk"
|
||||
"github.com/lbryio/ytsync/v5/tags_manager"
|
||||
"github.com/lbryio/ytsync/v5/thumbs"
|
||||
"github.com/lbryio/ytsync/v5/timing"
|
||||
logUtils "github.com/lbryio/ytsync/v5/util"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
|
||||
"github.com/lbryio/ytsync/ip_manager"
|
||||
"github.com/lbryio/ytsync/namer"
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
"github.com/lbryio/ytsync/tags_manager"
|
||||
"github.com/lbryio/ytsync/thumbs"
|
||||
|
||||
duration "github.com/ChannelMeter/iso8601duration"
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/shopspring/decimal"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"google.golang.org/api/youtube/v3"
|
||||
)
|
||||
|
||||
type YoutubeVideo struct {
|
||||
@@ -38,10 +43,10 @@ type YoutubeVideo struct {
|
||||
playlistPosition int64
|
||||
size *int64
|
||||
maxVideoSize int64
|
||||
maxVideoLength float64
|
||||
maxVideoLength time.Duration
|
||||
publishedAt time.Time
|
||||
dir string
|
||||
youtubeInfo *youtube.Video
|
||||
youtubeInfo *ytdl.YtdlVideo
|
||||
youtubeChannelID string
|
||||
tags []string
|
||||
awsConfig aws.Config
|
||||
@@ -88,22 +93,23 @@ var youtubeCategories = map[string]string{
|
||||
"44": "trailers",
|
||||
}
|
||||
|
||||
func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo {
|
||||
publishedAt, _ := time.Parse(time.RFC3339Nano, videoData.Snippet.PublishedAt) // ignore parse errors
|
||||
func NewYoutubeVideo(directory string, videoData *ytdl.YtdlVideo, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) (*YoutubeVideo, error) {
|
||||
// youtube-dl returns times in local timezone sometimes. this could break in the future
|
||||
// maybe we can file a PR to choose the timezone we want from youtube-dl
|
||||
return &YoutubeVideo{
|
||||
id: videoData.Id,
|
||||
title: videoData.Snippet.Title,
|
||||
description: videoData.Snippet.Description,
|
||||
id: videoData.ID,
|
||||
title: videoData.Title,
|
||||
description: videoData.Description,
|
||||
playlistPosition: playlistPosition,
|
||||
publishedAt: publishedAt,
|
||||
publishedAt: videoData.UploadDateForReal,
|
||||
dir: directory,
|
||||
youtubeInfo: videoData,
|
||||
awsConfig: awsConfig,
|
||||
mocked: false,
|
||||
youtubeChannelID: videoData.Snippet.ChannelId,
|
||||
youtubeChannelID: videoData.ChannelID,
|
||||
stopGroup: stopGroup,
|
||||
pool: pool,
|
||||
}
|
||||
}, nil
|
||||
}
|
||||
func NewMockedVideo(directory string, videoID string, youtubeChannelID string, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) *YoutubeVideo {
|
||||
return &YoutubeVideo{
|
||||
@@ -179,6 +185,13 @@ func (v *YoutubeVideo) getAbbrevDescription() string {
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) download() error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("download").Add(time.Since(start))
|
||||
}(start)
|
||||
if v.youtubeInfo.IsLive != nil {
|
||||
return errors.Err("video is a live stream and hasn't completed yet")
|
||||
}
|
||||
videoPath := v.getFullPath()
|
||||
|
||||
err := os.Mkdir(v.videoDir(), 0777)
|
||||
@@ -199,21 +212,30 @@ func (v *YoutubeVideo) download() error {
|
||||
"480",
|
||||
"320",
|
||||
}
|
||||
dur := time.Duration(v.youtubeInfo.Duration) * time.Second
|
||||
if dur.Hours() > 2 { //for videos longer than 2 hours only sync up to 720p
|
||||
qualities = []string{
|
||||
"720",
|
||||
"480",
|
||||
"320",
|
||||
}
|
||||
}
|
||||
|
||||
ytdlArgs := []string{
|
||||
"--no-progress",
|
||||
"-o" + strings.TrimSuffix(v.getFullPath(), ".mp4"),
|
||||
"--merge-output-format",
|
||||
"mp4",
|
||||
"--rm-cache-dir",
|
||||
"--postprocessor-args",
|
||||
"-movflags faststart",
|
||||
"--abort-on-unavailable-fragment",
|
||||
"--fragment-retries",
|
||||
"0",
|
||||
"--user-agent",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36",
|
||||
"1",
|
||||
"--cookies",
|
||||
"cookies.txt",
|
||||
}
|
||||
userAgent := []string{"--user-agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36"}
|
||||
if v.maxVideoSize > 0 {
|
||||
ytdlArgs = append(ytdlArgs,
|
||||
"--max-filesize",
|
||||
@@ -223,7 +245,7 @@ func (v *YoutubeVideo) download() error {
|
||||
if v.maxVideoLength > 0 {
|
||||
ytdlArgs = append(ytdlArgs,
|
||||
"--match-filter",
|
||||
fmt.Sprintf("duration <= %d", int(math.Round(v.maxVideoLength*3600))),
|
||||
fmt.Sprintf("duration <= %d", int(v.maxVideoLength.Seconds())),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -253,8 +275,10 @@ func (v *YoutubeVideo) download() error {
|
||||
"https://www.youtube.com/watch?v="+v.ID(),
|
||||
)
|
||||
|
||||
for i, quality := range qualities {
|
||||
for i := 0; i < len(qualities); i++ {
|
||||
quality := qualities[i]
|
||||
argsWithFilters := append(ytdlArgs, "-fbestvideo[ext=mp4][height<="+quality+"]+bestaudio[ext!=webm]")
|
||||
argsWithFilters = append(argsWithFilters, userAgent...)
|
||||
cmd := exec.Command("youtube-dl", argsWithFilters...)
|
||||
log.Printf("Running command youtube-dl %s", strings.Join(argsWithFilters, " "))
|
||||
|
||||
@@ -283,6 +307,11 @@ func (v *YoutubeVideo) download() error {
|
||||
return errors.Err(string(errorLog))
|
||||
}
|
||||
continue //this bypasses the yt throttling IP redistribution... TODO: don't
|
||||
} else if strings.Contains(string(errorLog), "YouTube said: Unable to extract video data") && !strings.Contains(userAgent[1], "Googlebot") {
|
||||
i-- //do not lower quality when trying a different user agent
|
||||
userAgent = []string{"--user-agent", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"}
|
||||
log.Infof("trying different user agent for video %s", v.ID())
|
||||
continue
|
||||
}
|
||||
return errors.Err(string(errorLog))
|
||||
}
|
||||
@@ -360,12 +389,16 @@ func (v *YoutubeVideo) delete(reason string) error {
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) triggerThumbnailSave() (err error) {
|
||||
thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Snippet.Thumbnails)
|
||||
v.thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.Url, v.ID(), v.awsConfig)
|
||||
thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Thumbnails)
|
||||
v.thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.URL, v.ID(), v.awsConfig)
|
||||
return err
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("publish").Add(time.Since(start))
|
||||
}(start)
|
||||
languages, locations, tags := v.getMetadata()
|
||||
var fee *jsonrpc.Fee
|
||||
if params.Fee != nil {
|
||||
@@ -379,7 +412,11 @@ func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, params SyncParams) (*Sync
|
||||
FeeCurrency: jsonrpc.Currency(params.Fee.Currency),
|
||||
}
|
||||
}
|
||||
|
||||
info := whatlanggo.Detect(v.getAbbrevDescription())
|
||||
if info.IsReliable() && info.Lang.Iso6391() != "" {
|
||||
language := info.Lang.Iso6391()
|
||||
languages = []string{language}
|
||||
}
|
||||
options := jsonrpc.StreamCreateOptions{
|
||||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
|
||||
Title: &v.title,
|
||||
@@ -415,8 +452,8 @@ type SyncParams struct {
|
||||
ChannelID string
|
||||
MaxVideoSize int
|
||||
Namer *namer.Namer
|
||||
MaxVideoLength float64
|
||||
Fee *sdk.Fee
|
||||
MaxVideoLength time.Duration
|
||||
Fee *shared.Fee
|
||||
DefaultAccount string
|
||||
}
|
||||
|
||||
@@ -434,6 +471,18 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV
|
||||
|
||||
func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
|
||||
var err error
|
||||
|
||||
dur := time.Duration(v.youtubeInfo.Duration) * time.Second
|
||||
minDuration := 7 * time.Second
|
||||
|
||||
if dur > v.maxVideoLength {
|
||||
logUtils.SendErrorToSlack("%s is %s long and the limit is %s", v.id, dur.String(), v.maxVideoLength.String())
|
||||
return nil, errors.Err("video is too long to process")
|
||||
}
|
||||
if dur < minDuration {
|
||||
logUtils.SendErrorToSlack("%s is %s long and the minimum is %s", v.id, dur.String(), minDuration.String())
|
||||
return nil, errors.Err("video is too short to process")
|
||||
}
|
||||
for {
|
||||
err = v.download()
|
||||
if err != nil && strings.Contains(err.Error(), "HTTP Error 429") {
|
||||
@@ -445,7 +494,17 @@ func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncPar
|
||||
}
|
||||
|
||||
log.Debugln("Downloaded " + v.id)
|
||||
ctx, cancelFn := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancelFn()
|
||||
|
||||
data, err := ffprobe.ProbeURL(ctx, v.getFullPath())
|
||||
if err != nil {
|
||||
log.Errorf("failure in probing downloaded video: %s", err.Error())
|
||||
} else {
|
||||
if data.Format.Duration() < minDuration {
|
||||
return nil, errors.Err("video is too short to process")
|
||||
}
|
||||
}
|
||||
err = v.triggerThumbnailSave()
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("thumbnail error", err)
|
||||
@@ -464,27 +523,30 @@ func (v *YoutubeVideo) getMetadata() (languages []string, locations []jsonrpc.Lo
|
||||
locations = nil
|
||||
tags = nil
|
||||
if !v.mocked {
|
||||
if v.youtubeInfo.Snippet.DefaultLanguage != "" {
|
||||
if v.youtubeInfo.Snippet.DefaultLanguage == "iw" {
|
||||
v.youtubeInfo.Snippet.DefaultLanguage = "he"
|
||||
}
|
||||
languages = []string{v.youtubeInfo.Snippet.DefaultLanguage}
|
||||
}
|
||||
/*
|
||||
if v.youtubeInfo.Snippet.DefaultLanguage != "" {
|
||||
if v.youtubeInfo.Snippet.DefaultLanguage == "iw" {
|
||||
v.youtubeInfo.Snippet.DefaultLanguage = "he"
|
||||
}
|
||||
languages = []string{v.youtubeInfo.Snippet.DefaultLanguage}
|
||||
}*/
|
||||
|
||||
if v.youtubeInfo.RecordingDetails != nil && v.youtubeInfo.RecordingDetails.Location != nil {
|
||||
/*if v.youtubeInfo.!= nil && v.youtubeInfo.RecordingDetails.Location != nil {
|
||||
locations = []jsonrpc.Location{{
|
||||
Latitude: util.PtrToString(fmt.Sprintf("%.7f", v.youtubeInfo.RecordingDetails.Location.Latitude)),
|
||||
Longitude: util.PtrToString(fmt.Sprintf("%.7f", v.youtubeInfo.RecordingDetails.Location.Longitude)),
|
||||
}}
|
||||
}
|
||||
tags = v.youtubeInfo.Snippet.Tags
|
||||
}*/
|
||||
tags = v.youtubeInfo.Tags
|
||||
}
|
||||
tags, err := tags_manager.SanitizeTags(tags, v.youtubeChannelID)
|
||||
if err != nil {
|
||||
log.Errorln(err.Error())
|
||||
}
|
||||
if !v.mocked {
|
||||
tags = append(tags, youtubeCategories[v.youtubeInfo.Snippet.CategoryId])
|
||||
for _, category := range v.youtubeInfo.Categories {
|
||||
tags = append(tags, youtubeCategories[category])
|
||||
}
|
||||
}
|
||||
|
||||
return languages, locations, tags
|
||||
@@ -509,8 +571,8 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
|
||||
if v.mocked {
|
||||
return nil, errors.Err("could not find thumbnail for mocked video")
|
||||
}
|
||||
thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Snippet.Thumbnails)
|
||||
thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.Url, v.ID(), v.awsConfig)
|
||||
thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Thumbnails)
|
||||
thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.URL, v.ID(), v.awsConfig)
|
||||
} else {
|
||||
thumbnailURL = thumbs.ThumbnailEndpoint + v.ID()
|
||||
}
|
||||
@@ -566,10 +628,12 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
|
||||
v.walletLock.RLock()
|
||||
defer v.walletLock.RUnlock()
|
||||
if v.mocked {
|
||||
start := time.Now()
|
||||
pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
|
||||
StreamCreateOptions: streamCreateOptions,
|
||||
FileSize: &videoSize,
|
||||
})
|
||||
timing.TimedComponent("StreamUpdate").Add(time.Since(start))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -580,15 +644,11 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
|
||||
}, nil
|
||||
}
|
||||
|
||||
videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
streamCreateOptions.ClaimCreateOptions.Title = &v.title
|
||||
streamCreateOptions.ClaimCreateOptions.Description = util.PtrToString(v.getAbbrevDescription())
|
||||
streamCreateOptions.Duration = util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds())))
|
||||
streamCreateOptions.Duration = util.PtrToUint64(uint64(v.youtubeInfo.Duration))
|
||||
streamCreateOptions.ReleaseTime = util.PtrToInt64(v.publishedAt.Unix())
|
||||
start := time.Now()
|
||||
pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
|
||||
ClearLanguages: util.PtrToBool(true),
|
||||
ClearLocations: util.PtrToBool(true),
|
||||
@@ -596,6 +656,7 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
|
||||
StreamCreateOptions: streamCreateOptions,
|
||||
FileSize: &videoSize,
|
||||
})
|
||||
timing.TimedComponent("StreamUpdate").Add(time.Since(start))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
+29
-12
@@ -4,6 +4,9 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/lbryio/ytsync/v5/downloader/ytdl"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
|
||||
@@ -11,7 +14,6 @@ import (
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"google.golang.org/api/youtube/v3"
|
||||
)
|
||||
|
||||
type thumbnailUploader struct {
|
||||
@@ -31,7 +33,9 @@ func (u *thumbnailUploader) downloadThumbnail() error {
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer img.Close()
|
||||
|
||||
if strings.HasPrefix(u.originalUrl, "//") {
|
||||
u.originalUrl = "https:" + u.originalUrl
|
||||
}
|
||||
resp, err := http.Get(u.originalUrl)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
@@ -68,6 +72,7 @@ func (u *thumbnailUploader) uploadThumbnail() error {
|
||||
ContentType: aws.String("image/jpeg"),
|
||||
CacheControl: aws.String("public, max-age=2592000"),
|
||||
})
|
||||
|
||||
u.mirroredUrl = ThumbnailEndpoint + u.name
|
||||
return errors.Err(err)
|
||||
}
|
||||
@@ -95,18 +100,30 @@ func MirrorThumbnail(url string, name string, s3Config aws.Config) (string, erro
|
||||
return "", err
|
||||
}
|
||||
|
||||
ownS3Config := s3Config.Copy(&aws.Config{Endpoint: aws.String("s3.lbry.tech")})
|
||||
|
||||
tu2 := thumbnailUploader{
|
||||
originalUrl: url,
|
||||
name: name,
|
||||
s3Config: *ownS3Config,
|
||||
}
|
||||
//own S3
|
||||
err = tu2.uploadThumbnail()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return tu.mirroredUrl, nil
|
||||
}
|
||||
|
||||
func GetBestThumbnail(thumbnails *youtube.ThumbnailDetails) *youtube.Thumbnail {
|
||||
if thumbnails.Maxres != nil {
|
||||
return thumbnails.Maxres
|
||||
} else if thumbnails.High != nil {
|
||||
return thumbnails.High
|
||||
} else if thumbnails.Medium != nil {
|
||||
return thumbnails.Medium
|
||||
} else if thumbnails.Standard != nil {
|
||||
return thumbnails.Standard
|
||||
func GetBestThumbnail(thumbnails []ytdl.Thumbnail) *ytdl.Thumbnail {
|
||||
var bestWidth *ytdl.Thumbnail
|
||||
for _, thumbnail := range thumbnails {
|
||||
if bestWidth == nil {
|
||||
bestWidth = &thumbnail
|
||||
} else if bestWidth.Width < thumbnail.Width {
|
||||
bestWidth = &thumbnail
|
||||
}
|
||||
}
|
||||
return thumbnails.Default
|
||||
return bestWidth
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
package timing
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/ytsync/v5/metrics"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type Timing struct {
|
||||
component string
|
||||
milliseconds int64
|
||||
min int64
|
||||
max int64
|
||||
invocations int32
|
||||
}
|
||||
|
||||
var timings *sync.Map
|
||||
|
||||
func TimedComponent(component string) *Timing {
|
||||
if timings == nil {
|
||||
timings = &sync.Map{}
|
||||
}
|
||||
stored, _ := timings.LoadOrStore(component, &Timing{
|
||||
component: component,
|
||||
milliseconds: 0,
|
||||
min: int64(99999999),
|
||||
})
|
||||
t, _ := stored.(*Timing)
|
||||
return t
|
||||
}
|
||||
|
||||
func ClearTimings() {
|
||||
if timings == nil {
|
||||
return
|
||||
}
|
||||
timings.Range(func(key interface{}, value interface{}) bool {
|
||||
timings.Delete(key)
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
func Report() {
|
||||
var totalTime time.Duration
|
||||
timings.Range(func(key interface{}, value interface{}) bool {
|
||||
totalTime += value.(*Timing).Get()
|
||||
return true
|
||||
})
|
||||
timings.Range(func(key interface{}, value interface{}) bool {
|
||||
component := key
|
||||
componentRuntime := value.(*Timing).Get().String()
|
||||
percentTime := float64(value.(*Timing).Get()) / float64(totalTime) * 100
|
||||
invocations := value.(*Timing).Invocations()
|
||||
avgTime := (time.Duration(int64(float64(value.(*Timing).Get()) / float64(value.(*Timing).Invocations())))).String()
|
||||
minRuntime := value.(*Timing).Min().String()
|
||||
maxRuntime := value.(*Timing).Max().String()
|
||||
logrus.Printf("component %s ran for %s (%.2f%% of the total time) - invoked %d times with an average of %s per call, a minimum of %s and a maximum of %s",
|
||||
component,
|
||||
componentRuntime,
|
||||
percentTime,
|
||||
invocations,
|
||||
avgTime,
|
||||
minRuntime,
|
||||
maxRuntime,
|
||||
)
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
func (t *Timing) Add(d time.Duration) {
|
||||
metrics.Durations.WithLabelValues(t.component).Observe(d.Seconds())
|
||||
atomic.AddInt64(&t.milliseconds, d.Milliseconds())
|
||||
for {
|
||||
oldMin := atomic.LoadInt64(&t.min)
|
||||
if d.Milliseconds() < oldMin {
|
||||
if atomic.CompareAndSwapInt64(&t.min, oldMin, d.Milliseconds()) {
|
||||
break
|
||||
}
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
for {
|
||||
oldMax := atomic.LoadInt64(&t.max)
|
||||
if d.Milliseconds() > oldMax {
|
||||
if atomic.CompareAndSwapInt64(&t.max, oldMax, d.Milliseconds()) {
|
||||
break
|
||||
}
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
atomic.AddInt32(&t.invocations, 1)
|
||||
}
|
||||
|
||||
func (t *Timing) Get() time.Duration {
|
||||
ms := atomic.LoadInt64(&t.milliseconds)
|
||||
return time.Duration(ms) * time.Millisecond
|
||||
}
|
||||
|
||||
func (t *Timing) Invocations() int32 {
|
||||
return atomic.LoadInt32(&t.invocations)
|
||||
}
|
||||
|
||||
func (t *Timing) Min() time.Duration {
|
||||
ms := atomic.LoadInt64(&t.min)
|
||||
return time.Duration(ms) * time.Millisecond
|
||||
}
|
||||
func (t *Timing) Max() time.Duration {
|
||||
ms := atomic.LoadInt64(&t.max)
|
||||
return time.Duration(ms) * time.Millisecond
|
||||
}
|
||||
+84
-3
@@ -7,9 +7,11 @@ import (
|
||||
"os/user"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/lbrycrd"
|
||||
"github.com/lbryio/ytsync/v5/timing"
|
||||
|
||||
"github.com/docker/docker/api/types"
|
||||
"github.com/docker/docker/api/types/filters"
|
||||
@@ -49,6 +51,19 @@ func GetLBRYNetDir() string {
|
||||
return lbrynetDir
|
||||
}
|
||||
|
||||
func GetLbryumDir() string {
|
||||
lbryumDir := os.Getenv("LBRYUM_DIR")
|
||||
if lbryumDir == "" {
|
||||
usr, err := user.Current()
|
||||
if err != nil {
|
||||
log.Errorln(err.Error())
|
||||
return ""
|
||||
}
|
||||
return usr.HomeDir + "/.lbryum/"
|
||||
}
|
||||
return lbryumDir + "/"
|
||||
}
|
||||
|
||||
const ALL = true
|
||||
const ONLINE = false
|
||||
|
||||
@@ -94,15 +109,20 @@ func IsRegTest() bool {
|
||||
}
|
||||
|
||||
func GetLbrycrdClient(lbrycrdString string) (*lbrycrd.Client, error) {
|
||||
chainName := os.Getenv("CHAINNAME")
|
||||
chainParams, ok := lbrycrd.ChainParamsMap[chainName]
|
||||
if !ok {
|
||||
chainParams = lbrycrd.MainNetParams
|
||||
}
|
||||
var lbrycrdd *lbrycrd.Client
|
||||
var err error
|
||||
if lbrycrdString == "" {
|
||||
lbrycrdd, err = lbrycrd.NewWithDefaultURL()
|
||||
lbrycrdd, err = lbrycrd.NewWithDefaultURL(&chainParams)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
lbrycrdd, err = lbrycrd.New(lbrycrdString)
|
||||
lbrycrdd, err = lbrycrd.New(lbrycrdString, &chainParams)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -220,10 +240,45 @@ func CleanupLbrynet() error {
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
lbryumDir := GetLbryumDir()
|
||||
ledger := "lbc_mainnet"
|
||||
if IsRegTest() {
|
||||
ledger = "lbc_regtest"
|
||||
}
|
||||
lbryumDir = lbryumDir + ledger
|
||||
|
||||
files, err = filepath.Glob(lbryumDir + "/blockchain.db*")
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
for _, f := range files {
|
||||
err = os.Remove(f)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func SleepUntilQuotaReset() {
|
||||
PST, _ := time.LoadLocation("America/Los_Angeles")
|
||||
t := time.Now().In(PST)
|
||||
n := time.Date(t.Year(), t.Month(), t.Day(), 24, 2, 0, 0, PST)
|
||||
d := n.Sub(t)
|
||||
if d < 0 {
|
||||
n = n.Add(24 * time.Hour)
|
||||
d = n.Sub(t)
|
||||
}
|
||||
log.Infof("gotta sleep %s until the quota resets", d.String())
|
||||
time.Sleep(d)
|
||||
}
|
||||
|
||||
func StartDaemon() error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("startDaemon").Add(time.Since(start))
|
||||
}(start)
|
||||
if IsUsingDocker() {
|
||||
return startDaemonViaDocker()
|
||||
}
|
||||
@@ -231,6 +286,10 @@ func StartDaemon() error {
|
||||
}
|
||||
|
||||
func StopDaemon() error {
|
||||
start := time.Now()
|
||||
defer func(start time.Time) {
|
||||
timing.TimedComponent("stopDaemon").Add(time.Since(start))
|
||||
}(start)
|
||||
if IsUsingDocker() {
|
||||
return stopDaemonViaDocker()
|
||||
}
|
||||
@@ -297,9 +356,31 @@ func GetDefaultWalletPath() string {
|
||||
defaultWalletDir = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet"
|
||||
}
|
||||
|
||||
walletPath := os.Getenv("LBRYNET_WALLETS_DIR")
|
||||
walletPath := os.Getenv("LBRYUM_DIR")
|
||||
if walletPath != "" {
|
||||
defaultWalletDir = walletPath + "/wallets/default_wallet"
|
||||
}
|
||||
return defaultWalletDir
|
||||
}
|
||||
func GetBlockchainDBPath() string {
|
||||
lbryumDir := os.Getenv("LBRYUM_DIR")
|
||||
if lbryumDir == "" {
|
||||
if IsRegTest() {
|
||||
lbryumDir = os.Getenv("HOME") + "/.lbryum_regtest"
|
||||
} else {
|
||||
lbryumDir = os.Getenv("HOME") + "/.lbryum"
|
||||
}
|
||||
}
|
||||
defaultDB := lbryumDir + "/lbc_mainnet/blockchain.db"
|
||||
if IsRegTest() {
|
||||
defaultDB = lbryumDir + "/lbc_regtest/blockchain.db"
|
||||
}
|
||||
return defaultDB
|
||||
}
|
||||
func GetBlockchainDirectoryName() string {
|
||||
ledger := "lbc_mainnet"
|
||||
if IsRegTest() {
|
||||
ledger = "lbc_regtest"
|
||||
}
|
||||
return ledger
|
||||
}
|
||||
+392
@@ -0,0 +1,392 @@
|
||||
package ytapi
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/ytsync/v5/shared"
|
||||
logUtils "github.com/lbryio/ytsync/v5/util"
|
||||
|
||||
"github.com/lbryio/ytsync/v5/downloader/ytdl"
|
||||
|
||||
"github.com/lbryio/ytsync/v5/downloader"
|
||||
"github.com/lbryio/ytsync/v5/ip_manager"
|
||||
"github.com/lbryio/ytsync/v5/sdk"
|
||||
"github.com/lbryio/ytsync/v5/sources"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type Video interface {
|
||||
Size() *int64
|
||||
ID() string
|
||||
IDAndNum() string
|
||||
PlaylistPosition() int
|
||||
PublishedAt() time.Time
|
||||
Sync(*jsonrpc.Client, sources.SyncParams, *sdk.SyncedVideo, bool, *sync.RWMutex) (*sources.SyncSummary, error)
|
||||
}
|
||||
|
||||
type byPublishedAt []Video
|
||||
|
||||
func (a byPublishedAt) Len() int { return len(a) }
|
||||
func (a byPublishedAt) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
|
||||
func (a byPublishedAt) Less(i, j int) bool { return a[i].PublishedAt().Before(a[j].PublishedAt()) }
|
||||
|
||||
type VideoParams struct {
|
||||
VideoDir string
|
||||
S3Config aws.Config
|
||||
Stopper *stop.Group
|
||||
IPPool *ip_manager.IPPool
|
||||
}
|
||||
|
||||
var mostRecentlyFailedChannel string // TODO: fix this hack!
|
||||
|
||||
func GetVideosToSync(config *sdk.APIConfig, channelID string, syncedVideos map[string]sdk.SyncedVideo, quickSync bool, maxVideos int, videoParams VideoParams, lastUploadedVideo string) ([]Video, error) {
|
||||
var videos []Video
|
||||
if quickSync && maxVideos > 50 {
|
||||
maxVideos = 50
|
||||
}
|
||||
allVideos, err := downloader.GetPlaylistVideoIDs(channelID, maxVideos, videoParams.Stopper.Ch(), videoParams.IPPool)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
videoIDs := make([]string, 0, len(allVideos))
|
||||
for _, video := range allVideos {
|
||||
sv, ok := syncedVideos[video]
|
||||
if ok && util.SubstringInSlice(sv.FailureReason, shared.NeverRetryFailures) {
|
||||
continue
|
||||
}
|
||||
videoIDs = append(videoIDs, video)
|
||||
}
|
||||
log.Infof("Got info for %d videos from youtube downloader", len(videoIDs))
|
||||
|
||||
playlistMap := make(map[string]int64)
|
||||
for i, videoID := range videoIDs {
|
||||
playlistMap[videoID] = int64(i)
|
||||
}
|
||||
//this will ensure that we at least try to sync the video that was marked as last uploaded video in the database.
|
||||
if lastUploadedVideo != "" {
|
||||
_, ok := playlistMap[lastUploadedVideo]
|
||||
if !ok {
|
||||
playlistMap[lastUploadedVideo] = 0
|
||||
videoIDs = append(videoIDs, lastUploadedVideo)
|
||||
}
|
||||
}
|
||||
|
||||
if len(videoIDs) < 1 {
|
||||
if channelID == mostRecentlyFailedChannel {
|
||||
return nil, errors.Err("playlist items not found")
|
||||
}
|
||||
mostRecentlyFailedChannel = channelID
|
||||
}
|
||||
|
||||
vids, err := getVideos(config, channelID, videoIDs, videoParams.Stopper.Ch(), videoParams.IPPool)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, item := range vids {
|
||||
positionInList := playlistMap[item.ID]
|
||||
videoToAdd, err := sources.NewYoutubeVideo(videoParams.VideoDir, item, positionInList, videoParams.S3Config, videoParams.Stopper, videoParams.IPPool)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
videos = append(videos, videoToAdd)
|
||||
}
|
||||
|
||||
for k, v := range syncedVideos {
|
||||
if !v.Published {
|
||||
continue
|
||||
}
|
||||
if _, ok := playlistMap[k]; !ok {
|
||||
videos = append(videos, sources.NewMockedVideo(videoParams.VideoDir, k, channelID, videoParams.S3Config, videoParams.Stopper, videoParams.IPPool))
|
||||
}
|
||||
}
|
||||
|
||||
sort.Sort(byPublishedAt(videos))
|
||||
|
||||
return videos, nil
|
||||
}
|
||||
|
||||
// CountVideosInChannel is unused for now... keeping it here just in case
|
||||
func CountVideosInChannel(channelID string) (int, error) {
|
||||
url := "https://socialblade.com/youtube/channel/" + channelID
|
||||
|
||||
req, _ := http.NewRequest("GET", url, nil)
|
||||
|
||||
req.Header.Add("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36")
|
||||
req.Header.Add("Accept", "*/*")
|
||||
req.Header.Add("Host", "socialblade.com")
|
||||
|
||||
res, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return 0, errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
var line string
|
||||
scanner := bufio.NewScanner(res.Body)
|
||||
for scanner.Scan() {
|
||||
if strings.Contains(scanner.Text(), "youtube-stats-header-uploads") {
|
||||
line = scanner.Text()
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if line == "" {
|
||||
return 0, errors.Err("upload count line not found")
|
||||
}
|
||||
|
||||
matches := regexp.MustCompile(">([0-9]+)<").FindStringSubmatch(line)
|
||||
if len(matches) != 2 {
|
||||
return 0, errors.Err("upload count not found with regex")
|
||||
}
|
||||
|
||||
num, err := strconv.Atoi(matches[1])
|
||||
if err != nil {
|
||||
return 0, errors.Err(err)
|
||||
}
|
||||
|
||||
return num, nil
|
||||
}
|
||||
|
||||
func ChannelInfo(channelID string) (*YoutubeStatsResponse, error) {
|
||||
url := "https://www.youtube.com/channel/" + channelID + "/about"
|
||||
|
||||
req, _ := http.NewRequest("GET", url, nil)
|
||||
|
||||
req.Header.Add("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36")
|
||||
req.Header.Add("Accept", "*/*")
|
||||
|
||||
res, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
pageBody := string(body)
|
||||
dataStartIndex := strings.Index(pageBody, "window[\"ytInitialData\"] = ") + 26
|
||||
if dataStartIndex == 25 {
|
||||
dataStartIndex = strings.Index(pageBody, "var ytInitialData = ") + 20
|
||||
}
|
||||
dataEndIndex := strings.Index(pageBody, "]}}};") + 4
|
||||
if dataEndIndex < dataStartIndex {
|
||||
return nil, errors.Err("start index is lower than end index. cannot extract channel info!")
|
||||
}
|
||||
data := pageBody[dataStartIndex:dataEndIndex]
|
||||
var decodedResponse YoutubeStatsResponse
|
||||
err = json.Unmarshal([]byte(data), &decodedResponse)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
return &decodedResponse, nil
|
||||
}
|
||||
|
||||
func getVideos(config *sdk.APIConfig, channelID string, videoIDs []string, stopChan stop.Chan, ipPool *ip_manager.IPPool) ([]*ytdl.YtdlVideo, error) {
|
||||
var videos []*ytdl.YtdlVideo
|
||||
for _, videoID := range videoIDs {
|
||||
if len(videoID) < 5 {
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case <-stopChan:
|
||||
return videos, errors.Err("interrupted by user")
|
||||
default:
|
||||
}
|
||||
|
||||
//ip, err := ipPool.GetIP(videoID)
|
||||
//if err != nil {
|
||||
// return nil, err
|
||||
//}
|
||||
//video, err := downloader.GetVideoInformation(videoID, &net.TCPAddr{IP: net.ParseIP(ip)})
|
||||
state, err := config.VideoState(videoID)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
if state == "published" {
|
||||
continue
|
||||
}
|
||||
video, err := downloader.GetVideoInformation(config, videoID, stopChan, nil, ipPool)
|
||||
if err != nil {
|
||||
errSDK := config.MarkVideoStatus(shared.VideoStatus{
|
||||
ChannelID: channelID,
|
||||
VideoID: videoID,
|
||||
Status: "failed",
|
||||
FailureReason: err.Error(),
|
||||
})
|
||||
logUtils.SendErrorToSlack(fmt.Sprintf("Skipping video (%s): %s", videoID, errors.FullTrace(err)))
|
||||
if errSDK != nil {
|
||||
return nil, errors.Err(errSDK)
|
||||
}
|
||||
} else {
|
||||
videos = append(videos, video)
|
||||
}
|
||||
}
|
||||
return videos, nil
|
||||
}
|
||||
|
||||
type YoutubeStatsResponse struct {
|
||||
Contents struct {
|
||||
TwoColumnBrowseResultsRenderer struct {
|
||||
Tabs []struct {
|
||||
TabRenderer struct {
|
||||
Title string `json:"title"`
|
||||
Selected bool `json:"selected"`
|
||||
Content struct {
|
||||
SectionListRenderer struct {
|
||||
Contents []struct {
|
||||
ItemSectionRenderer struct {
|
||||
Contents []struct {
|
||||
ChannelAboutFullMetadataRenderer struct {
|
||||
Description struct {
|
||||
SimpleText string `json:"simpleText"`
|
||||
} `json:"description"`
|
||||
ViewCountText struct {
|
||||
SimpleText string `json:"simpleText"`
|
||||
} `json:"viewCountText"`
|
||||
JoinedDateText struct {
|
||||
Runs []struct {
|
||||
Text string `json:"text"`
|
||||
} `json:"runs"`
|
||||
} `json:"joinedDateText"`
|
||||
CanonicalChannelURL string `json:"canonicalChannelUrl"`
|
||||
BypassBusinessEmailCaptcha bool `json:"bypassBusinessEmailCaptcha"`
|
||||
Title struct {
|
||||
SimpleText string `json:"simpleText"`
|
||||
} `json:"title"`
|
||||
Avatar struct {
|
||||
Thumbnails []struct {
|
||||
URL string `json:"url"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
} `json:"thumbnails"`
|
||||
} `json:"avatar"`
|
||||
ShowDescription bool `json:"showDescription"`
|
||||
DescriptionLabel struct {
|
||||
Runs []struct {
|
||||
Text string `json:"text"`
|
||||
} `json:"runs"`
|
||||
} `json:"descriptionLabel"`
|
||||
DetailsLabel struct {
|
||||
Runs []struct {
|
||||
Text string `json:"text"`
|
||||
} `json:"runs"`
|
||||
} `json:"detailsLabel"`
|
||||
ChannelID string `json:"channelId"`
|
||||
} `json:"channelAboutFullMetadataRenderer"`
|
||||
} `json:"contents"`
|
||||
} `json:"itemSectionRenderer"`
|
||||
} `json:"contents"`
|
||||
} `json:"sectionListRenderer"`
|
||||
} `json:"content"`
|
||||
} `json:"tabRenderer"`
|
||||
} `json:"tabs"`
|
||||
} `json:"twoColumnBrowseResultsRenderer"`
|
||||
} `json:"contents"`
|
||||
Header struct {
|
||||
C4TabbedHeaderRenderer struct {
|
||||
ChannelID string `json:"channelId"`
|
||||
Title string `json:"title"`
|
||||
Avatar struct {
|
||||
Thumbnails []struct {
|
||||
URL string `json:"url"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
} `json:"thumbnails"`
|
||||
} `json:"avatar"`
|
||||
Banner struct {
|
||||
Thumbnails []struct {
|
||||
URL string `json:"url"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
} `json:"thumbnails"`
|
||||
} `json:"banner"`
|
||||
VisitTracking struct {
|
||||
RemarketingPing string `json:"remarketingPing"`
|
||||
} `json:"visitTracking"`
|
||||
SubscriberCountText struct {
|
||||
SimpleText string `json:"simpleText"`
|
||||
} `json:"subscriberCountText"`
|
||||
} `json:"c4TabbedHeaderRenderer"`
|
||||
} `json:"header"`
|
||||
Metadata struct {
|
||||
ChannelMetadataRenderer struct {
|
||||
Title string `json:"title"`
|
||||
Description string `json:"description"`
|
||||
RssURL string `json:"rssUrl"`
|
||||
ChannelConversionURL string `json:"channelConversionUrl"`
|
||||
ExternalID string `json:"externalId"`
|
||||
Keywords string `json:"keywords"`
|
||||
OwnerUrls []string `json:"ownerUrls"`
|
||||
Avatar struct {
|
||||
Thumbnails []struct {
|
||||
URL string `json:"url"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
} `json:"thumbnails"`
|
||||
} `json:"avatar"`
|
||||
ChannelURL string `json:"channelUrl"`
|
||||
IsFamilySafe bool `json:"isFamilySafe"`
|
||||
VanityChannelURL string `json:"vanityChannelUrl"`
|
||||
} `json:"channelMetadataRenderer"`
|
||||
} `json:"metadata"`
|
||||
Topbar struct {
|
||||
DesktopTopbarRenderer struct {
|
||||
CountryCode string `json:"countryCode"`
|
||||
} `json:"desktopTopbarRenderer"`
|
||||
} `json:"topbar"`
|
||||
Microformat struct {
|
||||
MicroformatDataRenderer struct {
|
||||
URLCanonical string `json:"urlCanonical"`
|
||||
Title string `json:"title"`
|
||||
Description string `json:"description"`
|
||||
Thumbnail struct {
|
||||
Thumbnails []struct {
|
||||
URL string `json:"url"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
} `json:"thumbnails"`
|
||||
} `json:"thumbnail"`
|
||||
SiteName string `json:"siteName"`
|
||||
AppName string `json:"appName"`
|
||||
AndroidPackage string `json:"androidPackage"`
|
||||
IosAppStoreID string `json:"iosAppStoreId"`
|
||||
IosAppArguments string `json:"iosAppArguments"`
|
||||
OgType string `json:"ogType"`
|
||||
URLApplinksWeb string `json:"urlApplinksWeb"`
|
||||
URLApplinksIos string `json:"urlApplinksIos"`
|
||||
URLApplinksAndroid string `json:"urlApplinksAndroid"`
|
||||
URLTwitterIos string `json:"urlTwitterIos"`
|
||||
URLTwitterAndroid string `json:"urlTwitterAndroid"`
|
||||
TwitterCardType string `json:"twitterCardType"`
|
||||
TwitterSiteHandle string `json:"twitterSiteHandle"`
|
||||
SchemaDotOrgType string `json:"schemaDotOrgType"`
|
||||
Noindex bool `json:"noindex"`
|
||||
Unlisted bool `json:"unlisted"`
|
||||
FamilySafe bool `json:"familySafe"`
|
||||
Tags []string `json:"tags"`
|
||||
} `json:"microformatDataRenderer"`
|
||||
} `json:"microformat"`
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package ytapi
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestChannelInfo(t *testing.T) {
|
||||
info, err := ChannelInfo("UCNQfQvFMPnInwsU_iGYArJQ")
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, info)
|
||||
}
|
||||
Reference in new issue
Block a user