Compare commits

...
48 Commits
Author SHA1 Message Date
Niko Storni 64ab2490fe fix bugs
improve debugging
2019-07-22 02:27:14 +02:00
Niko Storni 1760935edf don't retry missing fragments
lower syncing quality for now
2019-07-22 02:09:18 +02:00
Niko Storni 60f2585f33 improve throttling
refactor slack wrapper
cleanup dependencies
2019-07-15 16:16:02 -04:00
Niko Storni df08d42d9b improve ip re-selection 2019-07-13 12:15:19 +02:00
Niko Storni 1b55033bb8 don't retry big videos 2019-07-13 11:40:50 +02:00
Niko Storni 1331e5769f actually use the functions I create... 2019-07-13 11:33:40 +02:00
Niko Storni 43f745ae20 improve ip shuffling (prevent ips in use from being dealt) 2019-07-12 23:54:48 +02:00
Niko Storni 8c2a8262e6 fix bugs 2019-07-12 23:20:01 +02:00
Niko Storni 6f486717da fix stop pattern
fix noretry string
fix youtube-dl param
2019-07-12 22:58:34 +02:00
Niko Storni a3fcd67611 better handle interruptions by user
refactor IP throttling in its own package
2019-07-12 21:32:49 +02:00
Niko Storni ae1ffb60c5 add ip throttling and better pooling
fix nil pointer dereference
2019-07-12 20:42:44 +02:00
Niko Storni fa24d83ae9 add max length filter 2019-07-12 02:39:24 +02:00
Niko Storni 2f11d99a97 stop on rare bug/failure 2019-07-12 02:21:27 +02:00
Niko Storni bab01a6354 prefer IPv4 over IPv6
youtube sucks and they ban /64 subnet blocks on IPv6
2019-07-12 01:54:34 +02:00
Niko Storni 89fe1e99d6 improve ipv6 validation
fix audio merge issue
2019-07-12 01:28:00 +02:00
Niko Storni ec81af0857 fix download filters 2019-07-12 01:04:45 +02:00
Niko Storni 51cb7f5e34 improve youtube antithrottle
fix logging
2019-07-11 19:14:15 +02:00
Niko Storni 8ad2acc936 remove original downloader
add locking to IP selection to avoid concurrent access
2019-07-11 17:32:46 +02:00
Niko Storni b8caf3f82b add IPv6 source pooling to bypass youtube throttling 2019-07-11 16:22:58 +02:00
Niko Storni b34d328cba halt ytsync when youtube throttles us 2019-07-10 22:47:01 +02:00
Niko Storni a0dcb2216d close db handles 2019-07-10 22:23:57 +02:00
Niko Storni 9d6a7e03cd update prism dependency 2019-07-10 21:07:48 +02:00
Niko Storni e44e272689 adjust config name 2019-07-10 18:04:05 +02:00
Niko Storni 87f7d4c9f9 recreate directory when emptied 2019-07-10 17:42:28 +02:00
Niko Storni 173e62d286 add check to avoid deleting unreflected blobs 2019-07-10 17:39:58 +02:00
Niko Storni 013760d48d delete blobs after reflection
remove 15 seconds wait after sync
2019-07-10 17:27:46 +02:00
Niko Storni 5dbad2d2e4 add automatic blob reflection
retry videos that fail due to tx issues
2019-07-10 15:46:54 +02:00
Niko Storni 545ab07013 fail channels that should fail 2019-07-05 17:35:20 +02:00
Niko Storni c402a1cdef add locking to map reads 2019-06-26 14:40:40 -04:00
Niko Storni a4c6b2da30 actually count UTXOs 2019-06-25 22:28:43 -04:00
Niko Storni 7f48220a80 remove dist folder... 2019-06-26 03:48:44 +02:00
Niko Storni 9e49110295 fix hash 2019-06-26 03:37:52 +02:00
Niko Storni 4506bdb638 fix bug in utxo counting 2019-06-25 21:12:15 -04:00
Niko Storni a8551bc9ee fix UTXO waiting logic 2019-06-25 20:45:27 -04:00
Niko Storni b5c9806dd0 improve space management 2019-06-24 20:43:50 -04:00
Niko Storni 8c349cf7ac fix bug in queues 2019-06-24 04:14:27 -04:00
Niko Storni f3f40c41ef fix refill balance 2019-06-22 06:08:11 -04:00
Niko Storni 05b65c4cb5 add noretry failure 2019-06-21 16:41:46 +02:00
Niko Storni 69ad13848b fix utxo count
fix one bug
add content type to uploaded thumbs
2019-06-20 21:45:17 +02:00
Niko Storni e496ad670a limit tag length 2019-06-20 17:43:46 +02:00
Niko 9d8eb32f9f Merge pull request #28 from lbryio/metadata_upgrade
Add support for new SDK (0.37.*) and support for upgrading channels and claims to new metadata
2019-06-13 20:14:13 +02:00
Niko Storni 6668840b11 fix description not containing youtube link
add workaround to SDK bug not spending the right UTXOs
fix bug in assignments
2019-06-13 19:33:58 +02:00
Niko Storni e893f68bc1 fix queue pulling
fix address reuse
2019-06-12 22:42:42 +02:00
Niko Storni 8da631eac3 add a limit to UTXOs generation
fix broadcastFee
fix spelling
2019-06-12 05:33:13 +02:00
Niko Storni 1729af0ae5 fix UTXOs management while publishing
fix account_set
improve logging
2019-06-12 05:25:01 +02:00
Niko Storni dce71d2451 add adjustments to video tracking
reduce minimum refill amount (min account balance supersedes)
revert account_fund change as --everything isn't supported with outputs
fix bugs
2019-06-12 04:44:56 +02:00
Niko Storni f4e75cf221 rewrite account funding algorithm
lock publishing when UTXOs management is in progress
spend everything when funding UTXOs
update lbry.go
2019-06-12 03:17:59 +02:00
Niko Storni ae3dad944d add blocking support for publishing and upgrading
add channel wide mappings
2019-06-11 19:18:39 +02:00
14 changed files with 1173 additions and 290 deletions

No files matched your search

+146
View File
@@ -0,0 +1,146 @@
package blobs_reflector
import (
"encoding/json"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/reflector.go/cmd"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/store"
"github.com/mitchellh/go-ps"
"io/ioutil"
"os"
"os/user"
"path/filepath"
log "github.com/sirupsen/logrus"
)
func ReflectAndClean() error {
err := reflectBlobs()
if err != nil {
return err
}
return cleanupLbrynet()
}
func loadConfig(path string) (cmd.Config, error) {
var c cmd.Config
raw, err := ioutil.ReadFile(path)
if err != nil {
if os.IsNotExist(err) {
return c, errors.Err("config file not found")
}
return c, err
}
err = json.Unmarshal(raw, &c)
return c, err
}
func reflectBlobs() error {
//make sure lbrynet is off
running, err := isLbrynetRunning()
if err != nil {
return err
}
if running {
return errors.Prefix("cannot reflect blobs as the daemon is running", err)
}
dbHandle := new(db.SQL)
ex, err := os.Executable()
if err != nil {
return errors.Err(err)
}
exPath := filepath.Dir(ex)
config, err := loadConfig(exPath + "/prism_config.json")
if err != nil {
return errors.Err(err)
}
err = dbHandle.Connect(config.DBConn)
if err != nil {
return errors.Err(err)
}
defer func() {
err := dbHandle.CloseDB()
if err != nil {
log.Errorf("failed to close db handle: %s", err.Error())
}
}()
st := store.NewDBBackedS3Store(
store.NewS3BlobStore(config.AwsID, config.AwsSecret, config.BucketRegion, config.BucketName),
dbHandle)
uploadWorkers := 10
uploader := reflector.NewUploader(dbHandle, st, uploadWorkers, false)
usr, err := user.Current()
if err != nil {
return errors.Err(err)
}
blobsDir := usr.HomeDir + "/.lbrynet/blobfiles/"
err = uploader.Upload(blobsDir)
if err != nil {
return errors.Err(err)
}
if uploader.GetSummary().Err > 0 {
return errors.Err("not al blobs were reflected. Errors: %d", uploader.GetSummary().Err)
}
return nil
}
func cleanupLbrynet() error {
//make sure lbrynet is off
running, err := isLbrynetRunning()
if err != nil {
return err
}
if running {
return errors.Prefix("cannot cleanup lbrynet as the daemon is running", err)
}
usr, err := user.Current()
if err != nil {
log.Errorln(err.Error())
return errors.Err(err)
}
lbrynetDir := usr.HomeDir + "/.lbrynet/"
files, err := filepath.Glob(lbrynetDir + "lbrynet.sqlite*")
if err != nil {
return errors.Err(err)
}
for _, f := range files {
err = os.Remove(f)
if err != nil {
return errors.Err(err)
}
}
blobsDir := lbrynetDir + "/blobfiles/"
err = os.RemoveAll(blobsDir)
if err != nil {
return errors.Err(err)
}
err = os.Mkdir(blobsDir, 0755)
if err != nil {
return errors.Err(err)
}
return nil
}
func isLbrynetRunning() (bool, error) {
processes, err := ps.Processes()
if err != nil {
return true, errors.Err(err)
}
var daemonProcessId = -1
for _, p := range processes {
if p.Executable() == "lbrynet" {
daemonProcessId = p.Pid()
break
}
}
running := daemonProcessId != -1
return running, nil
}
+12 -10
View File
@@ -3,25 +3,27 @@ module github.com/lbryio/ytsync
require (
cloud.google.com/go v0.37.4 // indirect
github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61
github.com/PuerkitoBio/goquery v1.5.0 // indirect
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf
github.com/aws/aws-sdk-go v1.17.3
github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 // indirect
github.com/go-ini/ini v1.42.0 // indirect
github.com/golang/protobuf v1.3.1 // indirect
github.com/go-sql-driver/mysql v1.4.1 // indirect
github.com/gopherjs/gopherjs v0.0.0-20190411002643-bd77b112433e // indirect
github.com/gorilla/websocket v1.4.0 // indirect
github.com/hashicorp/go-msgpack v0.5.5 // indirect
github.com/hashicorp/golang-lru v0.5.1 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/hashicorp/memberlist v0.1.4 // indirect
github.com/hashicorp/serf v0.8.2 // indirect
github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
github.com/kr/pretty v0.1.0 // indirect
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c
github.com/lbryio/lbry.go v1.0.9
github.com/lbryio/lbry.go v1.0.15
github.com/lbryio/reflector.go v1.0.6-0.20190710201919-7c7ed9da72ce
github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2 // indirect
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936
github.com/mitchellh/mapstructure v1.1.2 // indirect
github.com/nikooo777/ytdl v0.0.0-20190215151411-9c7832eaf457
github.com/onsi/ginkgo v1.8.0 // indirect
github.com/onsi/gomega v1.5.0 // indirect
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 // indirect
github.com/prometheus/common v0.3.0
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24
github.com/sirupsen/logrus v1.4.1
@@ -32,13 +34,13 @@ require (
github.com/ybbus/jsonrpc v2.1.2+incompatible // indirect
go.opencensus.io v0.20.2 // indirect
golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480 // indirect
golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2 // indirect
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c // indirect
golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a // indirect
golang.org/x/sync v0.0.0-20190423024810-112230192c58 // indirect
golang.org/x/text v0.3.2 // indirect
google.golang.org/api v0.3.2
google.golang.org/appengine v1.5.0 // indirect
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7 // indirect
google.golang.org/grpc v1.20.0 // indirect
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
gopkg.in/ini.v1 v1.42.0 // indirect
gopkg.in/yaml.v2 v2.2.2 // indirect
)
+147 -30
View File
@@ -5,41 +5,52 @@ cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7h
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 h1:N5Vqww5QISEHsWHOWDEx4PzdIay3Cg0Jp7zItq2ZAro=
github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61/go.mod h1:GnKXcK+7DYNy/8w2Ex//Uql4IgfaU82Cd5rWKb7ah00=
github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP7EJk=
github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg=
github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
github.com/aead/siphash v1.0.1 h1:FwHfE/T45KPKYuuSAKyyvE+oPWcaQ+CUmFW0bPlM+kg=
github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
github.com/armon/go-metrics v0.0.0-20180713145231-3c58d8115a78 h1:mdRSArcFLfW0VoL34LZAKSz6LkkK4jFxVx2xYavACMg=
github.com/armon/go-metrics v0.0.0-20180713145231-3c58d8115a78/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM=
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf h1:eg0MeVzsP1G42dRafH3vf+al2vQIJU0YHX+1Tw87oco=
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
github.com/aws/aws-sdk-go v1.16.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.17.3 h1:KBXxg7Jh0TxE5zmpNB2DwKmJeDUqh0O6jhy25TuYOmc=
github.com/aws/aws-sdk-go v1.17.3/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/btcsuite/btcd v0.0.0-20180531025944-86fed781132a/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ=
github.com/btcsuite/btcd v0.0.0-20190109040709-5bda5314ca95/go.mod h1:d3C0AkH6BRcvO8T0UEPu53cnw4IbV63x1bEjildYhO0=
github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32 h1:qkOC5Gd33k54tobS36cXdAzJbeHaduLtnLQQwNoIi78=
github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8=
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo=
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
github.com/btcsuite/btcutil v0.0.0-20180524032703-d4cc87b86016/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
github.com/btcsuite/btcutil v0.0.0-20180706230648-ab6388e0c60a/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803 h1:j3AgPKKZtZStM2nyhrDSLSYgT7YHrZKdSkq1OYeLjvM=
github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd h1:R/opQEbFEy9JGkIguV40SvRY1uliPX8ifOvi6ICsFCw=
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg=
github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd h1:qdGvebPBDuYDPGi1WCPjy1tGyMpmDK8IEapSsszn7HE=
github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY=
github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723 h1:ZA/jbKoGcVAnER6pCHPEkGdZOV7U1oLUedErBHCUMs0=
github.com/btcsuite/goleveldb v1.0.0/go.mod h1:QiK9vBlgftBg6rWQIj6wFzbPfRjiykIEhBH4obrXJ/I=
github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3SkEwmHoWBmX1DNXhXZqlTpq6s4tyJGc=
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 h1:o64h9XF42kVEUuhuer2ehqrlX8rZmvQSU0+Vpj1rF6Q=
github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61/go.mod h1:Rp8e0DCtEKwXFOC6JPJQVTz8tuGoGvw6Xfexggh/ed0=
github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -49,6 +60,7 @@ github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5m
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
@@ -56,27 +68,30 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
github.com/go-ini/ini v1.38.2/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-ini/ini v1.41.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-ini/ini v1.42.0 h1:TWr1wGj35+UiWHlBA8er89seFXxzwFn11spilrrj+38=
github.com/go-ini/ini v1.42.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-ozzo/ozzo-validation v3.5.0+incompatible h1:sUy/in/P6askYr16XJgTKq/0SZhiWsdg4WZGaLsGQkM=
github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
github.com/go-sql-driver/mysql v0.0.0-20180719071942-99ff426eb706/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.0 h1:kbxbvI4Un1LUWKxufD+BiE6AEExYYgkQLQmLFqA1LFk=
github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@@ -84,40 +99,84 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg=
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gopherjs/gopherjs v0.0.0-20190411002643-bd77b112433e h1:XWcjeEtTFTOVA9Fs1w7n2XBftk5ib4oZrhzWk0B+3eA=
github.com/gopherjs/gopherjs v0.0.0-20190411002643-bd77b112433e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8=
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk=
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/rpc v1.1.0 h1:marKfvVP0Gpd/jHlVBKCQ8RAoUPdX7K1Nuh6l1BNh7A=
github.com/gorilla/rpc v1.1.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ=
github.com/gorilla/websocket v1.2.0 h1:VJtLvh6VQym50czpZzx07z/kw9EgAxI3x1ZB8taTMQQ=
github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/hashicorp/errwrap v0.0.0-20180715044906-d6c0cd880357 h1:Rem2+U35z1QtPQc6r+WolF7yXiefXqDKyk+lN2pE164=
github.com/hashicorp/errwrap v0.0.0-20180715044906-d6c0cd880357/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-immutable-radix v0.0.0-20180129170900-7f3cd4390caa h1:0nA8i+6Rwqaq9xlpmVxxTwk6rxiEhX+E6Wh4vPNHiS8=
github.com/hashicorp/go-immutable-radix v0.0.0-20180129170900-7f3cd4390caa/go.mod h1:6ij3Z20p+OhOkCSrA0gImAWoHYQRGbnlcuk6XYTiaRw=
github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0=
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
github.com/hashicorp/go-msgpack v0.0.0-20150518234257-fa3f63826f7c h1:BTAbnbegUIMB6xmQCwWE8yRzbA4XSpnZY5hvRJC188I=
github.com/hashicorp/go-msgpack v0.0.0-20150518234257-fa3f63826f7c/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI=
github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
github.com/hashicorp/go-multierror v0.0.0-20180717150148-3d5d8f294aa0 h1:j30noezaCfvNLcdMYSvHLv81DxYRSt1grlpseG67vhU=
github.com/hashicorp/go-multierror v0.0.0-20180717150148-3d5d8f294aa0/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=
github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86 h1:7YOlAIO2YWnJZkQp7B5eFykaIY7C9JndqAFQyVV5BhM=
github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs=
github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE=
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
github.com/hashicorp/memberlist v0.1.0 h1:qSsCiC0WYD39lbSitKNt40e30uorm2Ss/d4JGU1hzH8=
github.com/hashicorp/memberlist v0.1.0/go.mod h1:ncdBp14cuox2iFOq3kDiquKU6fqsTBc3W6JvZwjxxsE=
github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
github.com/hashicorp/memberlist v0.1.4 h1:gkyML/r71w3FL8gUi74Vk76avkj/9lYAY9lvg0OcoGs=
github.com/hashicorp/memberlist v0.1.4/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
github.com/hashicorp/serf v0.0.0-20180530155958-984a73625de3 h1:NUr1hG6WO9sI1x8ofSimmpqfJ+rEHiHP/PLEA33rcfQ=
github.com/hashicorp/serf v0.0.0-20180530155958-984a73625de3/go.mod h1:h/Ru6tmZazX7WO/GDmwdpS975F019L4t5ng5IgwbNrE=
github.com/hashicorp/serf v0.8.2 h1:YZ7UKsJv+hKjqGVUUbtE3HNj79Eln2oQ75tniF6iPt0=
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf h1:WfD7VjIE6z8dIvMsI4/s+1qr5EL+zoIGev1BQj1eoJ8=
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h1:hyb9oH7vZsitZCiBt0ZvifOrB+qc8PS5IiilCIb87rg=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89 h1:12K8AlpT0/6QUXSfV0yi4Q0jkbq8NDtIKFtF61AoqV0=
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jrick/logrotate v1.0.0 h1:lQ1bL/n9mBNeIXoTUoYRlK4dHuNJVofX9oWqBtPnSzI=
github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22 h1:jKUP9TQ0c7X3w6+IPyMit07RE42MtTWNd77sN2cHngQ=
github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22/go.mod h1:u0Jo4f2dNlTJeeOywkM6bLwxq6gC3pZ9rEFHn3AhTdk=
github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07 h1:+kBG/8rjCa6vxJZbUjAiE4MQmBEBYc8nLEb51frnvBY=
github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07/go.mod h1:j1kV/8f3jowErEq4XyeypkCdvg5EeHkf0YCKCcq5Ybo=
github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE=
github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23 h1:FOOIBWrEkLgmlgGfMuZT83xIwfPDxEI2OHu6xUmJMFE=
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
github.com/kkdai/bstream v0.0.0-20181106074824-b3251f7901ec/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -129,66 +188,94 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M=
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8=
github.com/lbryio/lbry.go v1.0.9 h1:SAqopNiISazYuF38quA3WEfqOCzFtUed6QfCkAYQoJo=
github.com/lbryio/lbry.go v1.0.9/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
github.com/lbryio/lbry.go v0.0.0-20190109223729-30c312501602/go.mod h1:YEuFJD/oHNra6BFy+NfuvS84Wg6RMWJFGtiCCCc6MmQ=
github.com/lbryio/lbry.go v1.0.15 h1:g4g9cDDUsobmWgTA+1jEIb5k2fRCP0/NvPOMXduP8xY=
github.com/lbryio/lbry.go v1.0.15/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002 h1:urfYK5ElpUrAv90auPLldoVC60LwiGAcY0OE6HJB9KI=
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04 h1:Nze+C2HbeKvhjI/kVn+9Poj/UuEW5sOQxcsxqO7L3GI=
github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4=
github.com/lbryio/reflector.go v1.0.6-0.20190710201919-7c7ed9da72ce h1:ixsKssly++rkh0DWxNDrZyiRp00NieKZlndkqtpgj/c=
github.com/lbryio/reflector.go v1.0.6-0.20190710201919-7c7ed9da72ce/go.mod h1:7Y3YYeAKS6egH2WzwfU8f6+uNGjVHfLzKvwn+Nv3VMY=
github.com/lbryio/types v0.0.0-20181001180206-594241d24e00/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c h1:m3O7561xBQ00lfUVayW4c6SnpVbUDQtPUwGcGYSUYQA=
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5 h1:AsEBgzv3DhuYHI/GiQh2HxvTP71HCCE9E/tzGUzGdtU=
github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0=
github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6 h1:iOAVXzZyXtW408TMYejlUPo6BIn92HmOacWtIfNyYns=
github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg=
github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2 h1:CTzpAplpuXay3gDDqbErv/OSV6SoKZ9UTsGxLKfxB8M=
github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg=
github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 h1:mG83tLXWSRdcXMWfkoumVwhcCbf3jHF9QKv/m37BkM0=
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCWffGOaDcjkw1iB7W9DVLp6GXmfcJY/7YZCWPA4=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/miekg/dns v1.0.8 h1:Zi8HNpze3NeRWH1PQV6O71YcvJRQ6j0lORO6DAEmAAI=
github.com/miekg/dns v1.0.8/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936 h1:kw1v0NlnN+GZcU8Ma8CLF2Zzgjfx95gs3/GN3vYAPpo=
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk=
github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v0.0.0-20180511142126-bb74f1db0675/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/nikooo777/ytdl v0.0.0-20190215151411-9c7832eaf457 h1:S6VzSeZkOv8pKevvKF1isVp9HF294LxVsYw5JrV0FdQ=
github.com/nikooo777/ytdl v0.0.0-20190215151411-9c7832eaf457/go.mod h1:Z/EmaCtMz3DxZsLaTnbHjqjZrOH4MHQBvvxtMObRGAY=
github.com/nlopes/slack v0.2.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM=
github.com/nlopes/slack v0.4.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM=
github.com/nlopes/slack v0.5.0 h1:NbIae8Kd0NpqaEI3iUrsuS0KbcEDhzhc939jLW5fNm0=
github.com/nlopes/slack v0.5.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc=
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.3.0 h1:taZ4h8Tkxv2kNyoSctBvfXEHmBmxrwmIidZTIaHons4=
github.com/prometheus/common v0.3.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/sebdah/goldie v0.0.0-20180424091453-8784dd1ab561 h1:IY+sDBJR/wRtsxq+626xJnt4Tw7/ROA9cDIR8MMhWyg=
github.com/sebdah/goldie v0.0.0-20180424091453-8784dd1ab561/go.mod h1:lvjGftC8oe7XPtyrOidaMi0rp5B9+XY/ZRUynGnuaxQ=
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/shopspring/decimal v0.0.0-20180607144847-19e3cb6c2930/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 h1:pntxY8Ary0t43dCZ5dqY4YTJCObLY1kIXl0uzMv+7DE=
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
github.com/sirupsen/logrus v0.0.0-20180523074243-ea8897e79973/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k=
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
@@ -199,8 +286,12 @@ github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/spf13/cast v1.2.0 h1:HHl1DSRbEQN2i8tJmtS6ViPyHx35+p51amrdsiTCrkg=
github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.0-20180722215644-7c4570c3ebeb/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/cobra v0.0.0-20190109003409-7547e83b2d85 h1:UQHWkFUuJBy5rWN1DxosG/efssLu7u0fXXSTC2HHKfQ=
github.com/spf13/cobra v0.0.0-20190109003409-7547e83b2d85/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -208,6 +299,8 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
github.com/uber-go/atomic v1.3.2 h1:Azu9lPBWRNKzYXSIwRfgRuDuS0YKsK4NFhiQv98gkxo=
github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE=
github.com/ybbus/jsonrpc v2.1.2+incompatible h1:V4mkE9qhbDQ92/MLMIhlhMSbz8jNXdagC3xBR5NDwaQ=
@@ -215,43 +308,57 @@ github.com/ybbus/jsonrpc v2.1.2+incompatible/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G
go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
go.opencensus.io v0.20.2 h1:NAfh7zF0/3/HqtMvJNZ/RFrSlCE6ZTlHmKfhL/Dm1Jk=
go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480 h1:O5YqonU5IWby+w98jVUG9h7zlCWCcH4RHyPVReBmhzk=
golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20181217174547-8f45f776aaf1/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd h1:HuTn7WObtcDo9uEEU7rEqL0jYthdXAmZ6PP+meazmaU=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2 h1:iC0Y6EDq+rhnAePxGvJs2kzUAYcwESqdcGRPzEUfzTU=
golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c h1:uOCk1iQW6Vc18bnC13MfzScl+wdKBmM9Y9kU7Z83/lw=
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a h1:tImsplftrFpALCYumobsd0K86vlAs/eXGFms2txfJfA=
golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI=
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190520201301-c432e742b0af h1:NXfmMfXz6JqGfG3ikSxcz2N93j6DgScr19Oo2uwFu88=
@@ -259,8 +366,13 @@ golang.org/x/sys v0.0.0-20190520201301-c432e742b0af/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190109165630-d30e00c24034/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
@@ -276,14 +388,17 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190108161440-ae2f86662275/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7 h1:ZUjXAXmrAyrmmCPHgCA/vChHcpsX27MZ3yBonD/z1KE=
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.0 h1:DlsSIrgEBuZAUFJcta2B5i/lzeHHbnfkNFAfFXLVFYQ=
google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=
gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -291,6 +406,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk=
gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
@@ -306,3 +422,4 @@ gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190109154334-5bcec433c8ea/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+122
View File
@@ -0,0 +1,122 @@
package ipManager
import (
"github.com/asaskevich/govalidator"
"github.com/lbryio/ytsync/util"
log "github.com/sirupsen/logrus"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/stop"
"net"
"sync"
"time"
)
const IPCooldownPeriod = 25 * time.Second
const unbanTimeout = 3 * time.Hour
var ipv6Pool []string
var ipv4Pool []string
var throttledIPs map[string]bool
var ipInUse map[string]bool
var ipLastUsed map[string]time.Time
var ipMutex sync.Mutex
var stopper = stop.New()
func GetNextIP(ipv6 bool) (string, error) {
ipMutex.Lock()
defer ipMutex.Unlock()
if len(ipv4Pool) < 1 || len(ipv6Pool) < 1 {
throttledIPs = make(map[string]bool)
ipInUse = make(map[string]bool)
ipLastUsed = make(map[string]time.Time)
addrs, err := net.InterfaceAddrs()
if err != nil {
return "", errors.Err(err)
}
for _, address := range addrs {
if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) {
ipv6Pool = append(ipv6Pool, ipnet.IP.String())
ipLastUsed[ipnet.IP.String()] = time.Now().Add(-IPCooldownPeriod)
} else if ipnet.IP.To4() != nil && govalidator.IsIPv4(ipnet.IP.String()) {
ipv4Pool = append(ipv4Pool, ipnet.IP.String())
ipLastUsed[ipnet.IP.String()] = time.Now().Add(-IPCooldownPeriod)
}
}
}
}
nextIP := ""
if ipv6 {
nextIP = getLeastUsedIP(ipv6Pool)
} else {
nextIP = getLeastUsedIP(ipv4Pool)
}
if nextIP == "" {
return "throttled", errors.Err("all IPs are throttled")
}
lastUse := ipLastUsed[nextIP]
if time.Since(lastUse) < IPCooldownPeriod {
log.Debugf("The IP %s is too hot, waiting for %.1f seconds before continuing", nextIP, (IPCooldownPeriod - time.Since(lastUse)).Seconds())
time.Sleep(IPCooldownPeriod - time.Since(lastUse))
}
ipInUse[nextIP] = true
return nextIP, nil
}
func ReleaseIP(ip string) {
ipMutex.Lock()
defer ipMutex.Unlock()
ipLastUsed[ip] = time.Now()
ipInUse[ip] = false
}
func getLeastUsedIP(ipPool []string) string {
nextIP := ""
veryLastUse := time.Now()
for _, ip := range ipPool {
isThrottled := throttledIPs[ip]
if isThrottled {
continue
}
inUse := ipInUse[ip]
if inUse {
continue
}
lastUse := ipLastUsed[ip]
if lastUse.Before(veryLastUse) {
nextIP = ip
veryLastUse = lastUse
}
}
return nextIP
}
func SetIpThrottled(ip string, stopGrp *stop.Group) {
ipMutex.Lock()
isThrottled := throttledIPs[ip]
if isThrottled {
return
}
throttledIPs[ip] = true
ipMutex.Unlock()
util.SendErrorToSlack("%s set to throttled", ip)
stopper.Add(1)
go func() {
defer stopper.Done()
unbanTimer := time.NewTimer(unbanTimeout)
select {
case <-unbanTimer.C:
ipMutex.Lock()
throttledIPs[ip] = false
ipMutex.Unlock()
util.SendInfoToSlack("%s set back to not throttled", ip)
case <-stopGrp.Ch():
unbanTimer.Stop()
}
}()
}
+4 -3
View File
@@ -9,10 +9,11 @@ import (
"github.com/lbryio/lbry.go/extras/util"
"github.com/lbryio/ytsync/sdk"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/lbryio/ytsync/manager"
log "github.com/sirupsen/logrus"
logUtils "github.com/lbryio/ytsync/util"
)
var Version string
@@ -197,7 +198,7 @@ func ytSync(cmd *cobra.Command, args []string) {
)
err := sm.Start()
if err != nil {
manager.SendErrorToSlack(err.Error())
logUtils.SendErrorToSlack(err.Error())
}
manager.SendInfoToSlack("Syncing process terminated!")
logUtils.SendInfoToSlack("Syncing process terminated!")
}
+15 -5
View File
@@ -6,8 +6,10 @@ import (
"syscall"
"time"
"github.com/lbryio/ytsync/blobs_reflector"
"github.com/lbryio/ytsync/namer"
"github.com/lbryio/ytsync/sdk"
logUtils "github.com/lbryio/ytsync/util"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/util"
@@ -142,6 +144,7 @@ func (s *SyncManager) Start() error {
shouldInterruptLoop = true
} else {
var queuesToSync []string
//TODO: implement scrambling to avoid starvation of queues
if s.syncStatus != "" {
queuesToSync = append(queuesToSync, s.syncStatus)
} else if s.syncUpdate {
@@ -154,7 +157,9 @@ func (s *SyncManager) Start() error {
if err != nil {
return err
}
for _, c := range channels {
log.Infof("There are %d channels in the \"%s\" queue", len(channels), q)
if len(channels) > 0 {
c := channels[0]
syncs = append(syncs, Sync{
APIConfig: s.apiConfig,
YoutubeChannelID: c.ChannelId,
@@ -174,6 +179,7 @@ func (s *SyncManager) Start() error {
namer: namer.NewNamer(),
Fee: c.Fee,
})
continue
}
}
}
@@ -181,9 +187,9 @@ func (s *SyncManager) Start() error {
log.Infoln("No channels to sync. Pausing 5 minutes!")
time.Sleep(5 * time.Minute)
}
for i, sync := range syncs {
for _, sync := range syncs {
shouldNotCount := false
SendInfoToSlack("Syncing %s (%s) to LBRY! (iteration %d/%d - total processed channels: %d)", sync.LbryChannelName, sync.YoutubeChannelID, i+1, len(syncs), syncCount+1)
logUtils.SendInfoToSlack("Syncing %s (%s) to LBRY! total processed channels since startup: %d", sync.LbryChannelName, sync.YoutubeChannelID, syncCount+1)
err := sync.FullCycle()
if err != nil {
fatalErrors := []string{
@@ -200,10 +206,14 @@ func (s *SyncManager) Start() error {
}
shouldNotCount = strings.Contains(err.Error(), "this youtube channel is being managed by another server")
if !shouldNotCount {
SendInfoToSlack("A non fatal error was reported by the sync process. %s\nContinuing...", err.Error())
logUtils.SendInfoToSlack("A non fatal error was reported by the sync process. %s\nContinuing...", err.Error())
}
}
SendInfoToSlack("Syncing %s (%s) reached an end. (iteration %d/%d - total processed channels: %d)", sync.LbryChannelName, sync.YoutubeChannelID, i+1, len(syncs), syncCount+1)
err = blobs_reflector.ReflectAndClean()
if err != nil {
return errors.Prefix("@Nikooo777 something went wrong while reflecting blobs", err)
}
logUtils.SendInfoToSlack("Syncing %s (%s) reached an end. total processed channels since startup: %d", sync.LbryChannelName, sync.YoutubeChannelID, syncCount+1)
if !shouldNotCount {
syncCount++
}
+58 -50
View File
@@ -15,6 +15,7 @@ import (
"github.com/lbryio/ytsync/tagsManager"
"github.com/lbryio/ytsync/thumbs"
logUtils "github.com/lbryio/ytsync/util"
"github.com/shopspring/decimal"
log "github.com/sirupsen/logrus"
@@ -22,8 +23,6 @@ import (
"google.golang.org/api/youtube/v3"
)
const minimumRefillAmount = 4
func (s *Sync) enableAddressReuse() error {
accountsResponse, err := s.daemon.AccountList()
if err != nil {
@@ -35,7 +34,8 @@ func (s *Sync) enableAddressReuse() error {
}
for _, a := range accounts {
_, err = s.daemon.AccountSet(a.ID, jsonrpc.AccountSettings{
ChangeMaxUses: 1000,
ChangeMaxUses: util.PtrToInt(1000),
ReceivingMaxUses: util.PtrToInt(100),
})
if err != nil {
return errors.Err(err)
@@ -44,7 +44,7 @@ func (s *Sync) enableAddressReuse() error {
return nil
}
func (s *Sync) walletSetup() error {
//prevent unnecessary concurrent execution
//prevent unnecessary concurrent execution and publishing while refilling/reallocating UTXOs
s.walletMux.Lock()
defer s.walletMux.Unlock()
err := s.ensureChannelOwnership()
@@ -68,51 +68,51 @@ func (s *Sync) walletSetup() error {
if err != nil {
return err
}
numOnSource := int(n)
videosOnYoutube := int(n)
log.Debugf("Source channel has %d videos", numOnSource)
if numOnSource == 0 {
log.Debugf("Source channel has %d videos", videosOnYoutube)
if videosOnYoutube == 0 {
return nil
}
s.syncedVideosMux.RLock()
numPublished := len(s.syncedVideos) //should we only count published videos? Credits are allocated even for failed ones...
s.syncedVideosMux.RUnlock()
log.Debugf("We already allocated credits for %d videos", numPublished)
if numOnSource-numPublished > s.Manager.videosLimit {
numOnSource = s.Manager.videosLimit
}
minBalance := (float64(numOnSource)-float64(numPublished))*(publishAmount+0.1) + channelClaimAmount
if s.Manager.upgradeMetadata {
videosToUpgrade := 0
for _, v := range s.syncedVideos {
if v.Published && v.MetadataVersion < 2 {
videosToUpgrade++
publishedCount := 0
notUpgradedCount := 0
failedCount := 0
for _, sv := range s.syncedVideos {
if sv.Published {
publishedCount++
if sv.MetadataVersion < 2 {
notUpgradedCount++
}
} else {
failedCount++
}
minBalance += float64(videosToUpgrade) * 0.001
}
if numPublished > numOnSource && balance < minimumRefillAmount {
SendErrorToSlack("something is going on as we published more videos than those available on source: %d/%d", numPublished, numOnSource)
minBalance = minimumRefillAmount
s.syncedVideosMux.RUnlock()
log.Debugf("We already allocated credits for %d published videos and %d failed videos", publishedCount, failedCount)
if videosOnYoutube > s.Manager.videosLimit {
videosOnYoutube = s.Manager.videosLimit
}
unallocatedVideos := videosOnYoutube - (publishedCount + failedCount)
requiredBalance := float64(unallocatedVideos)*(publishAmount+estimatedMaxTxFee) + channelClaimAmount
if s.Manager.upgradeMetadata {
requiredBalance += float64(notUpgradedCount) * 0.001
}
refillAmount := 0.0
if balance < requiredBalance || balance < minimumAccountBalance {
refillAmount = math.Max(requiredBalance-balance, minimumRefillAmount)
}
amountToAdd := minBalance - balance
if s.Refill > 0 {
if amountToAdd < 0 {
amountToAdd = float64(s.Refill)
} else {
amountToAdd += float64(s.Refill)
}
refillAmount += float64(s.Refill)
}
if amountToAdd > 0 {
if amountToAdd < minimumRefillAmount {
amountToAdd = minimumRefillAmount
}
err := s.addCredits(amountToAdd)
if refillAmount > 0 {
err := s.addCredits(refillAmount)
if err != nil {
return errors.Err(err)
}
@@ -167,14 +167,19 @@ func (s *Sync) ensureEnoughUTXOs() error {
target := 40
slack := int(float32(0.1) * float32(target))
count := 0
confirmedCount := 0
for _, utxo := range *utxolist {
amount, _ := strconv.ParseFloat(utxo.Amount, 64)
if !utxo.IsClaim && !utxo.IsSupport && !utxo.IsUpdate && amount != 0.0 {
if utxo.IsMine && utxo.Type == "payment" && amount > 0.001 {
if utxo.Confirmations > 0 {
confirmedCount++
}
count++
}
}
log.Infof("utxo count: %d", count)
log.Infof("utxo count: %d (%d confirmed)", count, confirmedCount)
UTXOWaitThreshold := 16
if count < target-slack {
balance, err := s.daemon.AccountBalance(&defaultAccount)
if err != nil {
@@ -187,24 +192,27 @@ func (s *Sync) ensureEnoughUTXOs() error {
if err != nil {
return errors.Err(err)
}
broadcastFee := 0.01
amountToSplit := fmt.Sprintf("%.6f", balanceAmount-broadcastFee)
desiredUTXOCount := uint64(math.Floor((balanceAmount - broadcastFee) / 0.1))
maxUTXOs := uint64(500)
desiredUTXOCount := uint64(math.Floor((balanceAmount) / 0.1))
if desiredUTXOCount > maxUTXOs {
desiredUTXOCount = maxUTXOs
}
log.Infof("Splitting balance of %s evenly between %d UTXOs", *balance, desiredUTXOCount)
prefillTx, err := s.daemon.AccountFund(defaultAccount, defaultAccount, amountToSplit, desiredUTXOCount)
broadcastFee := 0.1
prefillTx, err := s.daemon.AccountFund(defaultAccount, defaultAccount, fmt.Sprintf("%.4f", balanceAmount-broadcastFee), desiredUTXOCount, false)
if err != nil {
return err
} else if prefillTx == nil {
return errors.Err("no response")
}
err = s.waitForNewBlock()
if err != nil {
return err
if confirmedCount < UTXOWaitThreshold {
err = s.waitForNewBlock()
if err != nil {
return err
}
}
} else if !allUTXOsConfirmed(utxolist) {
} else if confirmedCount < UTXOWaitThreshold {
log.Println("Waiting for previous txns to confirm")
err := s.waitForNewBlock()
if err != nil {
@@ -260,7 +268,7 @@ func (s *Sync) ensureChannelOwnership() error {
if len((*channels).Items) > 1 {
// This wallet is probably not under our control anymore but we still want to publish to it
// here we shall check if within all the channels there is one that was created by ytsync
SendInfoToSlack("we are dealing with a wallet that has multiple channels. This indicates that the wallet was probably transferred but we still want to sync their content. YoutubeID: %s", s.YoutubeChannelID)
logUtils.SendInfoToSlack("we are dealing with a wallet that has multiple channels. This indicates that the wallet was probably transferred but we still want to sync their content. YoutubeID: %s", s.YoutubeChannelID)
if s.lbryChannelID == "" {
return errors.Err("this channel does not have a recorded claimID in the database. To prevent failures, updates are not supported until an entry is manually added in the database")
}
@@ -399,7 +407,7 @@ func allUTXOsConfirmed(utxolist *jsonrpc.UTXOListResponse) bool {
}
for _, utxo := range *utxolist {
if utxo.Height == 0 {
if utxo.Confirmations <= 0 {
return false
}
}
+71 -75
View File
@@ -18,6 +18,7 @@ import (
"github.com/lbryio/ytsync/sdk"
"github.com/lbryio/ytsync/sources"
"github.com/lbryio/ytsync/thumbs"
logUtils "github.com/lbryio/ytsync/util"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/jsonrpc"
@@ -37,9 +38,12 @@ import (
)
const (
channelClaimAmount = 0.01
publishAmount = 0.01
maxReasonLength = 500
channelClaimAmount = 0.01
estimatedMaxTxFee = 0.1
minimumAccountBalance = 4.0
minimumRefillAmount = 1
publishAmount = 0.01
maxReasonLength = 500
)
type video interface {
@@ -48,7 +52,7 @@ type video interface {
IDAndNum() string
PlaylistPosition() int
PublishedAt() time.Time
Sync(*jsonrpc.Client, sources.SyncParams, *sdk.SyncedVideo, bool) (*sources.SyncSummary, error)
Sync(*jsonrpc.Client, sources.SyncParams, *sdk.SyncedVideo, bool, *sync.RWMutex) (*sources.SyncSummary, error)
}
// sorting videos
@@ -83,43 +87,21 @@ type Sync struct {
grp *stop.Group
lbryChannelID string
namer *namer.Namer
walletMux *sync.Mutex
walletMux *sync.RWMutex
queue chan video
}
func (s *Sync) AppendSyncedVideo(videoID string, published bool, failureReason string, claimName string) {
func (s *Sync) AppendSyncedVideo(videoID string, published bool, failureReason string, claimName string, claimID string, metadataVersion int8, size int64) {
s.syncedVideosMux.Lock()
defer s.syncedVideosMux.Unlock()
s.syncedVideos[videoID] = sdk.SyncedVideo{
VideoID: videoID,
Published: published,
FailureReason: failureReason,
}
}
// SendErrorToSlack Sends an error message to the default channel and to the process log.
func SendErrorToSlack(format string, a ...interface{}) {
message := format
if len(a) > 0 {
message = fmt.Sprintf(format, a...)
}
log.Errorln(message)
err := util.SendToSlack(":sos: " + message)
if err != nil {
log.Errorln(err)
}
}
// SendInfoToSlack Sends an info message to the default channel and to the process log.
func SendInfoToSlack(format string, a ...interface{}) {
message := format
if len(a) > 0 {
message = fmt.Sprintf(format, a...)
}
log.Infoln(message)
err := util.SendToSlack(":information_source: " + message)
if err != nil {
log.Errorln(err)
VideoID: videoID,
Published: published,
FailureReason: failureReason,
ClaimID: claimID,
ClaimName: claimName,
MetadataVersion: metadataVersion,
Size: size,
}
}
@@ -254,7 +236,7 @@ func (s *Sync) FullCycle() (e error) {
s.setExceptions()
s.syncedVideosMux = &sync.RWMutex{}
s.walletMux = &sync.Mutex{}
s.walletMux = &sync.RWMutex{}
s.grp = stop.New()
s.queue = make(chan video)
interruptChan := make(chan os.Signal, 1)
@@ -288,6 +270,7 @@ func (s *Sync) FullCycle() (e error) {
return errors.Wrap(err, 0)
}
defer deleteSyncFolder(s.videoDirectory)
log.Printf("Starting daemon")
err = startDaemonViaSystemd()
if err != nil {
@@ -303,17 +286,17 @@ func (s *Sync) FullCycle() (e error) {
return err
}
err = s.doSync()
if err != nil {
return err
} else {
// wait for reflection to finish???
wait := 15 * time.Second // should bump this up to a few min, but keeping it low for testing
log.Println("Waiting " + wait.String() + " to finish reflecting everything")
time.Sleep(wait)
}
return s.doSync()
}
return nil
func deleteSyncFolder(videoDirectory string) {
if !strings.Contains(videoDirectory, "/tmp/ytsync") {
_ = util.SendToSlack(errors.Err("Trying to delete an unexpected directory: %s", videoDirectory).Error())
}
err := os.RemoveAll(videoDirectory)
if err != nil {
_ = util.SendToSlack(err.Error())
}
}
func (s *Sync) setChannelTerminationStatus(e *error) {
@@ -322,7 +305,6 @@ func (s *Sync) setChannelTerminationStatus(e *error) {
noFailConditions := []string{
"this youtube channel is being managed by another server",
"interrupted during daemon startup",
"playlist items not found",
}
if util.SubstringInSlice((*e).Error(), noFailConditions) {
return
@@ -382,8 +364,8 @@ func (s *Sync) stopAndUploadWallet(e *error) {
}
}
func logShutdownError(shutdownErr error) {
SendErrorToSlack("error shutting down daemon: %v", shutdownErr)
SendErrorToSlack("WALLET HAS NOT BEEN MOVED TO THE WALLET BACKUP DIR")
logUtils.SendErrorToSlack("error shutting down daemon: %v", shutdownErr)
logUtils.SendErrorToSlack("WALLET HAS NOT BEEN MOVED TO THE WALLET BACKUP DIR")
}
var thumbnailHosts = []string{
@@ -457,7 +439,9 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total, fixed, removed int
tn := c.Value.GetThumbnail().GetUrl()
videoID := tn[strings.LastIndex(tn, "/")+1:]
videoIDMap[videoID] = c.ClaimID
s.syncedVideosMux.RLock()
pv, claimInDatabase := s.syncedVideos[videoID]
s.syncedVideosMux.RUnlock()
claimMetadataVersion := uint(1)
if strings.Contains(tn, thumbs.ThumbnailEndpoint) {
claimMetadataVersion = 2
@@ -473,7 +457,7 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total, fixed, removed int
if claimIDDiffers {
log.Debugf("%s: Mismatch in database for claimID. DB: %s - Blockchain: %s", videoID, pv.ClaimID, c.ClaimID)
}
if claimIDDiffers {
if claimNameDiffers {
log.Debugf("%s: Mismatch in database for claimName. DB: %s - Blockchain: %s", videoID, pv.ClaimName, c.Name)
}
if claimMarkedUnpublished {
@@ -544,7 +528,7 @@ func (s *Sync) doSync() error {
return errors.Prefix("error checking for duplicates", err)
}
if hasDupes {
SendInfoToSlack("Channel had dupes and was fixed!")
logUtils.SendInfoToSlack("Channel had dupes and was fixed!")
err = s.waitForNewBlock()
if err != nil {
return err
@@ -565,10 +549,10 @@ func (s *Sync) doSync() error {
return err
}
if nFixed > 0 {
SendInfoToSlack("%d claims had mismatched database info or were completely missing and were fixed", nFixed)
logUtils.SendInfoToSlack("%d claims had mismatched database info or were completely missing and were fixed", nFixed)
}
if nRemoved > 0 {
SendInfoToSlack("%d were marked as published but weren't actually published and thus removed from the database", nRemoved)
logUtils.SendInfoToSlack("%d were marked as published but weren't actually published and thus removed from the database", nRemoved)
}
}
pubsOnDB := 0
@@ -579,11 +563,11 @@ func (s *Sync) doSync() error {
}
if pubsOnWallet > pubsOnDB { //This case should never happen
SendInfoToSlack("We're claiming to have published %d videos but in reality we published %d (%s)", pubsOnDB, pubsOnWallet, s.YoutubeChannelID)
logUtils.SendInfoToSlack("We're claiming to have published %d videos but in reality we published %d (%s)", pubsOnDB, pubsOnWallet, s.YoutubeChannelID)
return errors.Err("not all published videos are in the database")
}
if pubsOnWallet < pubsOnDB {
SendInfoToSlack("we're claiming to have published %d videos but we only published %d (%s)", pubsOnDB, pubsOnWallet, s.YoutubeChannelID)
logUtils.SendInfoToSlack("we're claiming to have published %d videos but we only published %d (%s)", pubsOnDB, pubsOnWallet, s.YoutubeChannelID)
}
if s.StopOnError {
@@ -638,8 +622,11 @@ func (s *Sync) startWorker(workerNum int) {
err := s.processVideo(v)
if err != nil {
logMsg := "error processing video: " + err.Error()
logMsg := fmt.Sprintf("error processing video %s: %s", v.ID(), err.Error())
log.Errorln(logMsg)
if strings.Contains(strings.ToLower(err.Error()), "interrupted by user") {
return
}
fatalErrors := []string{
":5279: read: connection reset by peer",
"no space left on device",
@@ -647,13 +634,14 @@ func (s *Sync) startWorker(workerNum int) {
"Cannot publish using channel",
"cannot concatenate 'str' and 'NoneType' objects",
"more than 90% of the space has been used.",
"Couldn't find private key for id",
}
if util.SubstringInSlice(err.Error(), fatalErrors) || s.StopOnError {
s.grp.Stop()
} else if s.MaxTries > 1 {
errorsNoRetry := []string{
"non 200 status code received",
" reason: 'This video contains content from",
"This video contains content from",
"dont know which claim to update",
"uploader has not made this video available in your country",
"download error: AccessDenied: Access Denied",
@@ -668,6 +656,7 @@ func (s *Sync) startWorker(workerNum int) {
"Watch this video on YouTube.",
"have blocked it on copyright grounds",
"the video must be republished as we can't get the right size",
"HTTP Error 403",
}
if util.SubstringInSlice(err.Error(), errorsNoRetry) {
log.Println("This error should not be retried at all")
@@ -675,21 +664,22 @@ func (s *Sync) startWorker(workerNum int) {
if strings.Contains(err.Error(), "txn-mempool-conflict") ||
strings.Contains(err.Error(), "too-long-mempool-chain") {
log.Println("waiting for a block before retrying")
err = s.waitForNewBlock()
err := s.waitForNewBlock()
if err != nil {
s.grp.Stop()
SendErrorToSlack("something went wrong while waiting for a block: %v", err)
logUtils.SendErrorToSlack("something went wrong while waiting for a block: %v", err)
break
}
} else if util.SubstringInSlice(err.Error(), []string{
"Not enough funds to cover this transaction",
"failed: Not enough funds",
"Error in daemon: Insufficient funds, please deposit additional LBC"}) {
log.Println("refilling addresses before retrying")
err = s.walletSetup()
"Error in daemon: Insufficient funds, please deposit additional LBC",
}) {
log.Println("checking funds and UTXOs before retrying...")
err := s.walletSetup()
if err != nil {
s.grp.Stop()
SendErrorToSlack("failed to setup the wallet for a refill: %v", err)
logUtils.SendErrorToSlack("failed to setup the wallet for a refill: %v", err)
break
}
} else if strings.Contains(err.Error(), "Error in daemon: 'str' object has no attribute 'get'") {
@@ -698,27 +688,33 @@ func (s *Sync) startWorker(workerNum int) {
log.Println("Retrying")
continue
}
SendErrorToSlack("Video failed after %d retries, skipping. Stack: %s", tryCount, logMsg)
logUtils.SendErrorToSlack("Video failed after %d retries, skipping. Stack: %s", tryCount, logMsg)
}
s.syncedVideosMux.RLock()
existingClaim, ok := s.syncedVideos[v.ID()]
s.syncedVideosMux.RUnlock()
existingClaimID := ""
existingClaimName := ""
existingClaimSize := v.Size()
existingClaimSize := int64(0)
if v.Size() != nil {
existingClaimSize = *v.Size()
}
if ok {
existingClaimID = existingClaim.ClaimID
existingClaimName = existingClaim.ClaimName
if existingClaim.Size > 0 {
existingClaimSize = &existingClaim.Size
existingClaimSize = existingClaim.Size
}
}
videoStatus := VideoStatusFailed
if strings.Contains(err.Error(), "upgrade failed") {
videoStatus = VideoStatusUpgradeFailed
} else {
s.AppendSyncedVideo(v.ID(), false, err.Error(), existingClaimName, existingClaimID, 0, existingClaimSize)
}
s.AppendSyncedVideo(v.ID(), false, err.Error(), "")
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), videoStatus, existingClaimID, existingClaimName, err.Error(), existingClaimSize, 0)
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), videoStatus, existingClaimID, existingClaimName, err.Error(), &existingClaimSize, 0)
if err != nil {
SendErrorToSlack("Failed to mark video on the database: %s", err.Error())
logUtils.SendErrorToSlack("Failed to mark video on the database: %s", err.Error())
}
}
break
@@ -792,7 +788,7 @@ func (s *Sync) enqueueYoutubeVideos() error {
return errors.Prefix("error getting videos info", err)
}
for _, item := range videosListResponse.Items {
videos = append(videos, sources.NewYoutubeVideo(s.videoDirectory, item, playlistMap[item.Id].Position, s.Manager.GetS3AWSConfig()))
videos = append(videos, sources.NewYoutubeVideo(s.videoDirectory, item, playlistMap[item.Id].Position, s.Manager.GetS3AWSConfig(), s.grp))
}
log.Infof("Got info for %d videos from youtube API", len(videos))
@@ -808,7 +804,7 @@ func (s *Sync) enqueueYoutubeVideos() error {
}
_, ok := playlistMap[k]
if !ok {
videos = append(videos, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig()))
videos = append(videos, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig(), s.grp))
}
}
@@ -863,6 +859,7 @@ func (s *Sync) processVideo(v video) (err error) {
"Unable to extract signature tokens",
"the video is too big to sync, skipping for now",
"video is too long to process",
"This video contains content from",
"no compatible format available for this video",
"Watch this video on YouTube.",
"have blocked it on copyright grounds",
@@ -899,18 +896,17 @@ func (s *Sync) processVideo(v video) (err error) {
Fee: s.Fee,
}
summary, err := v.Sync(s.daemon, sp, &sv, videoRequiresUpgrade)
summary, err := v.Sync(s.daemon, sp, &sv, videoRequiresUpgrade, s.walletMux)
if err != nil {
return err
}
s.AppendSyncedVideo(v.ID(), true, "", summary.ClaimName, summary.ClaimID, newMetadataVersion, *v.Size())
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusPublished, summary.ClaimID, summary.ClaimName, "", v.Size(), 2)
if err != nil {
SendErrorToSlack("Failed to mark video on the database: %s", err.Error())
logUtils.SendErrorToSlack("Failed to mark video on the database: %s", err.Error())
}
s.AppendSyncedVideo(v.ID(), true, "", summary.ClaimName)
return nil
}
+10 -6
View File
@@ -54,7 +54,7 @@ func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeC
Data []YoutubeChannel `json:"data"`
}
endpoint := a.ApiURL + "/yt/jobs"
res, _ := http.PostForm(endpoint, url.Values{
res, err := http.PostForm(endpoint, url.Values{
"auth_token": {a.ApiToken},
"sync_status": {status},
"min_videos": {strconv.Itoa(1)},
@@ -63,10 +63,13 @@ func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeC
"sync_server": {a.HostName},
"channel_id": {cp.YoutubeChannelID},
})
if err != nil {
return nil, errors.Err(err)
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
var response apiJobsResponse
err := json.Unmarshal(body, &response)
err = json.Unmarshal(body, &response)
if err != nil {
return nil, err
}
@@ -164,8 +167,9 @@ func (a *APIConfig) SetChannelClaimID(channelID string, channelClaimID string) e
}
const (
VideoStatusPublished = "published"
VideoStatusFailed = "failed"
VideoStatusPublished = "published"
VideoStatusUpgradeFailed = "upgradefailed"
VideoStatusFailed = "failed"
)
func (a *APIConfig) DeleteVideos(videos []string) error {
@@ -207,9 +211,9 @@ func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status str
"status": {status},
"auth_token": {a.ApiToken},
}
if status == VideoStatusPublished {
if status == VideoStatusPublished || status == VideoStatusUpgradeFailed {
if claimID == "" || claimName == "" {
return errors.Err("claimID or claimName missing")
return errors.Err("claimID (%s) or claimName (%s) missing", claimID, claimName)
}
vals.Add("published_at", strconv.FormatInt(time.Now().Unix(), 10))
vals.Add("claim_id", claimID)
+4 -1
View File
@@ -2,6 +2,7 @@ package sources
import (
"strings"
"sync"
"github.com/lbryio/lbry.go/extras/jsonrpc"
"github.com/lbryio/ytsync/namer"
@@ -12,7 +13,9 @@ type SyncSummary struct {
ClaimName string
}
func publishAndRetryExistingNames(daemon *jsonrpc.Client, title, filename string, amount float64, options jsonrpc.StreamCreateOptions, namer *namer.Namer) (*SyncSummary, error) {
func publishAndRetryExistingNames(daemon *jsonrpc.Client, title, filename string, amount float64, options jsonrpc.StreamCreateOptions, namer *namer.Namer, walletLock *sync.RWMutex) (*SyncSummary, error) {
walletLock.RLock()
defer walletLock.RUnlock()
for {
name := namer.GetNextName(title)
response, err := daemon.StreamCreate(name, filename, amount, options)
+131 -91
View File
@@ -10,12 +10,15 @@ import (
"regexp"
"strconv"
"strings"
"sync"
"time"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/jsonrpc"
"github.com/lbryio/lbry.go/extras/stop"
"github.com/lbryio/lbry.go/extras/util"
"github.com/lbryio/ytsync/ipManager"
"github.com/lbryio/ytsync/namer"
"github.com/lbryio/ytsync/sdk"
"github.com/lbryio/ytsync/tagsManager"
@@ -23,7 +26,6 @@ import (
"github.com/ChannelMeter/iso8601duration"
"github.com/aws/aws-sdk-go/aws"
"github.com/nikooo777/ytdl"
"github.com/shopspring/decimal"
log "github.com/sirupsen/logrus"
"google.golang.org/api/youtube/v3"
@@ -46,10 +48,10 @@ type YoutubeVideo struct {
thumbnailURL string
lbryChannelID string
mocked bool
walletLock *sync.RWMutex
stopGroup *stop.Group
}
const reflectorURL = "http://blobs.lbry.io/"
var youtubeCategories = map[string]string{
"1": "film & animation",
"2": "autos & vehicles",
@@ -85,7 +87,7 @@ var youtubeCategories = map[string]string{
"44": "trailers",
}
func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64, awsConfig aws.Config) *YoutubeVideo {
func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group) *YoutubeVideo {
publishedAt, _ := time.Parse(time.RFC3339Nano, videoData.Snippet.PublishedAt) // ignore parse errors
return &YoutubeVideo{
id: videoData.Id,
@@ -98,9 +100,10 @@ func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPositio
awsConfig: awsConfig,
mocked: false,
youtubeChannelID: videoData.Snippet.ChannelId,
stopGroup: stopGroup,
}
}
func NewMockedVideo(directory string, videoID string, youtubeChannelID string, awsConfig aws.Config) *YoutubeVideo {
func NewMockedVideo(directory string, videoID string, youtubeChannelID string, awsConfig aws.Config, stopGroup *stop.Group) *YoutubeVideo {
return &YoutubeVideo{
id: videoID,
playlistPosition: 0,
@@ -108,6 +111,7 @@ func NewMockedVideo(directory string, videoID string, youtubeChannelID string, a
awsConfig: awsConfig,
mocked: true,
youtubeChannelID: youtubeChannelID,
stopGroup: stopGroup,
}
}
@@ -160,37 +164,18 @@ func (v *YoutubeVideo) getFullPath() string {
func (v *YoutubeVideo) getAbbrevDescription() string {
maxLines := 10
description := strings.TrimSpace(v.description)
if strings.Count(description, "\n") < maxLines {
return description
}
additionalDescription := "\nhttps://www.youtube.com/watch?v=" + v.id
khanAcademyClaimID := "5fc52291980268b82413ca4c0ace1b8d749f3ffb"
if v.lbryChannelID == khanAcademyClaimID {
additionalDescription = additionalDescription + "\nNote: All Khan Academy content is available for free at (www.khanacademy.org)"
}
if strings.Count(description, "\n") < maxLines {
return description + "\n..." + additionalDescription
}
return strings.Join(strings.Split(description, "\n")[:maxLines], "\n") + "\n..." + additionalDescription
}
func (v *YoutubeVideo) fallbackDownload() error {
cmd := exec.Command("youtube-dl",
v.ID(),
"--no-progress",
"-fbestvideo[ext=mp4,height<=1080,filesize<2000M]+best[ext=mp4,height<=1080,filesize<2000M]",
"-o"+strings.TrimRight(v.getFullPath(), ".mp4"),
"--merge-output-format",
"mp4")
log.Printf("Running command and waiting for it to finish...")
output, err := cmd.CombinedOutput()
log.Debugln(string(output))
if err != nil {
log.Printf("Command finished with error: %v", errors.Err(string(output)))
return errors.Err(err)
}
return nil
}
func (v *YoutubeVideo) download() error {
func (v *YoutubeVideo) download(useIPv6 bool) error {
videoPath := v.getFullPath()
err := os.Mkdir(v.videoDir(), 0750)
@@ -205,72 +190,119 @@ func (v *YoutubeVideo) download() error {
log.Debugln(v.id + " already exists at " + videoPath)
return nil
}
qualities := []string{
//"1080",
"720",
"480",
"320",
}
qualityIndex := 0
ytdlArgs := []string{
"--no-progress",
"--max-filesize",
fmt.Sprintf("%dM", v.maxVideoSize),
"--match-filter",
fmt.Sprintf("duration <= %d", int(math.Round(v.maxVideoLength*3600))),
"-fbestvideo[ext=mp4][height<=" + qualities[qualityIndex] + "]+bestaudio[ext!=webm]",
"-o\"" + strings.TrimSuffix(v.getFullPath(), ".mp4") + "\"",
"--merge-output-format",
"mp4",
"--abort-on-unavailable-fragment",
"--fragment-retries",
"0",
}
sourceAddress, err := ipManager.GetNextIP(useIPv6)
if err != nil {
if sourceAddress == "throttled" {
for {
select {
case <-v.stopGroup.Ch():
return errors.Err("interrupted by user")
default:
}
videoUrl := "https://www.youtube.com/watch?v=" + v.id
videoInfo, err := ytdl.GetVideoInfo(videoUrl)
time.Sleep(ipManager.IPCooldownPeriod)
sourceAddress, err = ipManager.GetNextIP(useIPv6)
if err == nil {
break
}
}
} else {
return errors.Err(err)
}
}
defer ipManager.ReleaseIP(sourceAddress)
if useIPv6 {
log.Infof("using IPv6: %s", sourceAddress)
ytdlArgs = append(ytdlArgs,
"-6",
"--source-address",
sourceAddress,
)
} else {
log.Infof("using IPv4: %s", sourceAddress)
ytdlArgs = append(ytdlArgs,
"-4",
"--source-address",
sourceAddress,
)
}
ytdlArgs = append(ytdlArgs, "https://www.youtube.com/watch?v="+v.ID())
runcmd:
cmd := exec.Command("youtube-dl", ytdlArgs...)
log.Printf("Running command youtube-dl %s", strings.Join(ytdlArgs, " "))
stderr, err := cmd.StderrPipe()
if err != nil {
return errors.Err(err)
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return errors.Err(err)
}
codec := []string{"H.264"}
ext := []string{"mp4"}
//Filter requires a [] interface{}
codecFilter := make([]interface{}, len(codec))
for i, v := range codec {
codecFilter[i] = v
if err := cmd.Start(); err != nil {
return errors.Err(err)
}
//Filter requires a [] interface{}
extFilter := make([]interface{}, len(ext))
for i, v := range ext {
extFilter[i] = v
}
errorLog, _ := ioutil.ReadAll(stderr)
outLog, _ := ioutil.ReadAll(stdout)
formats := videoInfo.Formats.Filter(ytdl.FormatVideoEncodingKey, codecFilter).Filter(ytdl.FormatExtensionKey, extFilter)
if len(formats) == 0 {
return errors.Err("no compatible format available for this video")
if err = cmd.Wait(); err != nil {
if strings.Contains(err.Error(), "exit status 1") {
if strings.Contains(string(errorLog), "HTTP Error 429") {
ipManager.SetIpThrottled(sourceAddress, v.stopGroup)
}
return errors.Err(string(errorLog))
} else if strings.Contains(string(errorLog), "giving up after 0 fragment retries") && qualityIndex < len(qualities)-1 {
qualityIndex++
goto runcmd
}
return errors.Err(err)
}
maxRetryAttempts := 5
isLengthLimitSet := v.maxVideoLength > 0.01
if isLengthLimitSet && videoInfo.Duration.Hours() > v.maxVideoLength {
log.Debugln(string(outLog))
if strings.Contains(string(outLog), "does not pass filter duration") {
_ = v.delete()
return errors.Err("video is too long to process")
}
for i := 0; i < len(formats) && i < maxRetryAttempts; i++ {
formatIndex := i
if i == maxRetryAttempts-1 {
formatIndex = len(formats) - 1
}
var downloadedFile *os.File
downloadedFile, err = os.Create(videoPath)
if err != nil {
return errors.Err(err)
}
err = videoInfo.Download(formats[formatIndex], downloadedFile)
_ = downloadedFile.Close()
if err != nil {
//delete the video and ignore the error
_ = v.delete()
return errors.Err(err.Error())
}
fi, err := os.Stat(v.getFullPath())
if err != nil {
return errors.Err(err)
}
videoSize := fi.Size()
v.size = &videoSize
isVideoSizeLimitSet := v.maxVideoSize > 0
if isVideoSizeLimitSet && videoSize > v.maxVideoSize {
//delete the video and ignore the error
_ = v.delete()
err = errors.Err("file is too big and there is no other format available")
} else {
break
}
if strings.Contains(string(outLog), "File is larger than max-filesize") {
_ = v.delete()
return errors.Err("the video is too big to sync, skipping for now")
}
return errors.Err(err)
if string(errorLog) != "" {
log.Printf("Command finished with error: %v", errors.Err(string(errorLog)))
_ = v.delete()
return errors.Err(string(errorLog))
}
fi, err := os.Stat(v.getFullPath())
if err != nil {
return errors.Err(err)
}
videoSize := fi.Size()
v.size = &videoSize
return nil
}
func (v *YoutubeVideo) videoDir() string {
@@ -355,7 +387,7 @@ func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, params SyncParams) (*Sync
if err != nil {
return nil, err
}
return publishAndRetryExistingNames(daemon, v.title, downloadPath, params.Amount, options, params.Namer)
return publishAndRetryExistingNames(daemon, v.title, downloadPath, params.Amount, options, params.Namer, v.walletLock)
}
func (v *YoutubeVideo) Size() *int64 {
@@ -372,10 +404,11 @@ type SyncParams struct {
Fee *sdk.Fee
}
func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo, reprocess bool) (*SyncSummary, error) {
v.maxVideoSize = int64(params.MaxVideoSize) * 1024 * 1024
func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo, reprocess bool, walletLock *sync.RWMutex) (*SyncSummary, error) {
v.maxVideoSize = int64(params.MaxVideoSize)
v.maxVideoLength = params.MaxVideoLength
v.lbryChannelID = params.ChannelID
v.walletLock = walletLock
if reprocess && existingVideoData != nil && existingVideoData.Published {
summary, err := v.reprocess(daemon, params, existingVideoData)
return summary, errors.Prefix("upgrade failed", err)
@@ -383,14 +416,19 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingV
return v.downloadAndPublish(daemon, params)
}
var isThrottled bool
func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
err := v.download()
err := v.download(isThrottled)
if err != nil {
log.Errorf("standard downloader failed: %s. Trying fallback downloader\n", err.Error())
fallBackErr := v.fallbackDownload()
if fallBackErr != nil {
log.Errorf("fallback downloader failed: %s\n", fallBackErr.Error())
return nil, errors.Prefix("download error", err) //return original error
if strings.Contains(err.Error(), "HTTP Error 429") && !isThrottled {
isThrottled = true
err = v.download(isThrottled)
if err != nil {
return nil, errors.Prefix("download error", err)
}
} else {
return nil, errors.Prefix("download error", err)
}
}
log.Debugln("Downloaded " + v.id)
@@ -499,6 +537,8 @@ func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, exis
Fee: fee,
}
v.walletLock.RLock()
defer v.walletLock.RUnlock()
if v.mocked {
pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
StreamCreateOptions: streamCreateOptions,
+414 -15
View File
@@ -71,6 +71,8 @@ func SanitizeTags(tags []string, youtubeChannelID string) ([]string, error) {
return sanitizedTags, nil
}
const TagMaxLength = 255
func normalizeTag(t string) (string, error) {
t = strings.ToLower(t)
multipleSpaces := regexp.MustCompile(`\s{2,}`)
@@ -87,7 +89,10 @@ func normalizeTag(t string) (string, error) {
if weirdChars.MatchString(t) {
log.Debugf("tag '%s' has weird stuff in it, skipping\n", t)
return "", nil
}
if len(t) > TagMaxLength {
log.Debugf("tag '%s' is too long, skipping\n", t)
return "", nil
}
return t, nil
}
@@ -140,23 +145,417 @@ func (ts *tagsSanitizer) add() {
}
const (
Lunduke = "UCkK9UDm_ZNrq_rIXCz3xCGA"
SwissExperiments = "UCNQfQvFMPnInwsU_iGYArJQ"
Juggling = "UC2fhTIbnQlFYaFzyTcmPkXg"
JustJuggling = "UCftqelpjmbFrUwr3VVzzVwA"
JordanBPeterson = "UCL_f53ZEJxp8TtlOkHwMV9Q"
DavidPakman = "UCvixJtaXuNdMPUGdOPcY8Ag"
TheLinuxGamer = "UCv1Kcz-CuGM6mxzL3B1_Eiw"
SwissExperiments = "UCNQfQvFMPnInwsU_iGYArJQ"
JustJuggling = "UCftqelpjmbFrUwr3VVzzVwA"
anupjanakpur = "UC_5tRfC4L2AbTz6mj6vZrKw"
PraveenMedia = "UC_fjE70lKNwM9AKIofQv-bA"
MisteriosDesvelados = "UC-FzxivscjYzonBDXgX2GLg"
kaipenchery = "UC-MU4K3Ghl-IdEX4J68tnTA"
Dhinchakpooja = "UC-stzLwoQF_R8Vnfb28d7Lg"
Karolajn = "UC-vYDJQiabg9BK8XTDPFspg"
thefantasio974 = "UC0GHeUcPxfNEZqbFyxy2frQ"
EduPrimitivo = "UC0odmP6ffEw3iVwTEyt-UKA"
isupportanupriyapatel = "UC0sxOzdHnmauMdOra0cxXVg"
theconfusedindian = "UC12ZPYxQbMA1loZE1OA609w"
guriaarts = "UC1fHp166o1Hd024fg3-d-jg"
minutodafisica = "UC1lUEcHrQwbusC5ext1himg"
khabir = "UC1yzNCXXk1h_tHgsqChE8mw"
OMapadaMinaMara = "UC2WMUPTbxQQ9hXq2_uutspg"
oliverjanich = "UC3cmEfpy4XED7YYEe69nIMA"
shyamsadhu = "UC3XAT8oBjL2RaqfblESHW_A"
EbaduRahmantech = "UC4950gpY6Qw1lCAcN6gNWwQ"
_1975oles = "UC4bGQWN4C8idymxlZxw77oA"
_8dsongsbollywood = "UC4nH7zmw41lRDbaiIddumjw"
EYESTV = "UC50CIbyHMydEEzEhV1ZNEBw"
Nono = "UC5yufoRPJy-1e1pR73UTxFQ"
lichtle = "UC6n_2v7YjwZ65F9h77nZPOA"
elabuelokraken = "UC7iWk2xziMR4hs_NzfsYytw"
cidvela = "UCaVOx5GCcSi2ELjWUdf05PA"
jayaskumar = "UCax9CJCQ6aY0bntP3mcIWKQ"
canaldarippi = "UCB_fba7yYMwa91F7rijBsVA"
FunMusicClassical = "UCB_X256IN7QiBtDkueo_6QQ"
minutodaterra = "UCB0zinWfy-dS_NqcOINYo3A"
franciscoalves = "UCbAm6YcZGk04obnFb-LuOSA"
criptomonedastv = "UCbK6Awel1-o-9JDVMFq808Q"
lux77 = "UCBYm4l3NX352goFtjSgZ7jA"
MrLuckyOficial = "UCEbMhGhZ1JoOYKgbgVyeQaA"
KotneKit = "UCf_1BQz3T151Eph0Sarj3Yw"
jaysgaming = "UCFAkedtc3jjDZQqqIULxqWg"
iamdineshthakur = "UCg0VInmUZoSdHD-DNmjifBA"
Musicreationz = "UCGJEtZB0Nj3rT3zZfyfjIZQ"
lafenixtv = "UCHAx8o0jj-8L_sqR5QM1Cwg"
guardeiafe = "UCHWjmG-Ce93VNcNZMMUyGQQ"
KHANEKIKHUSHBOO = "UCI8DRKcrfrHklRqSJHPhOwQ"
AjayGamingYT = "UCi9r5igvQblbI0PhJFj-IDg"
SzekelyVegan = "UCiqvKH1Ib_VPwh9bQf_hlXQ"
barzoy888 = "UCIYF1orTg4nDvv29ORoKXyA"
_7playstudiopersian = "UCJ3QUKcd9yOhiJzZBR2zT3Q"
eazypurple = "UCJ6Hd8S39g7pe65A6k42NZw"
GennadyM = "UCjZlOrmC7hi_1kMA6BKZXeQ"
GTGujarati = "UCKnYZmYFdFX4hP09Hh8k8jw"
annapavlidou = "UCKPSHyCFIDtMqzbJ_Qx-7aw"
Tamilmemesclub = "UCL_DPW38jcSGWy0Fu0A_hCg"
QuickReactionTeam = "UCl--LVcANGKHJYN7O8uY6OA"
rodrigojoseoficial = "UClMsSb57Jvexl1a0a2c2_CQ"
AzadChaiwala = "UCM5QNdoIefx6eumjPk8ZTMw"
lafenixextras = "UCM7zougvyPtfd9zw5HmAGlg"
JustSawAGhost = "UCmQYXE03n6qlF9FhtAPztcw"
mirchibangla = "UCmzj6hXrPZ_AwIZ8lgo-HuQ"
marsaguirre = "UCN3OMvdU7ySvE6meGjo9omg"
JustGoool = "UCn7kTV_syU_rI0jANxrpaFg"
nossocanal = "UCNDb9jdx5Jz-C7hWJv3y6Fw"
CSNN = "UCnfn-8PJbjYFAJ1fqzMdcew"
pedronedved = "UCNhb2Fz3sVLZcMb_arm1cMQ"
famoosh = "UCnQU2QZLVG1wxHoy_9LZucA"
Cenoritas = "UCo-VSGoYy_4IjXF9u4QqTiw"
fantabobgames = "UCo0U1tbk3YbqiLDhkeWOviQ"
thesoundofsoul = "UCOey6R7Ktnil8RZrY_xJYiA"
SaurabhMishraJournalist = "UCoHVCBoSfUOlQVa80dlN60g"
YoYoHoneySinghlive = "UCOZTp3nGj39-snzo4QSdduA"
CINEGLITZ = "UCP_p8JkLOwPcGAnxN1iVhZA"
lovetreta = "UCPi3uuGrh2mxnZJMfRhcjiw"
canalvendimeusofa = "UCpk58NDdaKdX0QiiA2e79tg"
CARIBEANVIDEOS = "UCPynGkfzH35RKyXRccCgzNg"
Dimon = "UCpZsZZ6KEqCHeAJ_Y0gEbyQ"
Anantvijaysoni = "UCQ3NeFKF2yo70xxBBUDeXXQ"
canalilhadebarbados = "UCR5L_Q8Tiljy7WKEQOrGKbg"
SotomayorTV2 = "UCR70CjRHxQilfEUgBBYpDhg"
Recipesarab = "UCRbMDfkH_bPUjQsv5dHcFQA"
_4ak10 = "UCsbrIjDPPXuVlWApYTnQECA"
Top50 = "UCSjXR8uvU4PijMM_kgyVQfw"
SoumyajitPyne = "UCSJXwgF3SePNfyojYG3yt6Q"
SneksShow = "UCsqZSkVccnyIxSE6pZuj13g"
casadosaber = "UCtvvTFp0XANyllOdmzZr9VQ"
Entarexyt = "UCuDkq6yAb5zdM2zIRG6VRDw"
UCurU6GLM4ggcLtWWAOTzlYw = "UCurU6GLM4ggcLtWWAOTzlYw"
promining = "UCviqKxlMnZqBipTzwixkGyg"
zapkids = "UCx2uGhYa9EuCNmbL72hVKGg"
Canalokok = "UCXyCAcjoWz9SMLGJT8dR0pA"
SmoothGames = "UCY8gvG25rOZ4hWWvcQH88PQ"
LIGATFA = "UCYhiIMOlDn_HEn3zvqZCDiw"
DeHamad = "UCZ5G07Vw7IaV81InBgYLCcg"
misszizi = "UCE55WTFs4ekJ_aWCoNEapbQ"
creationshub = "UCNfELkowZPIQ-Vegmb3pIBQ"
AlicePandora = "UCScRxEtwlt082_6ThI0YJbg"
Akito0405 = "UCXHWx1teSYIKwTGiscYaC-Q"
dashcambristol = "UCD0IC8bZI-MfIutkgPHlyWA"
adrianbonillap = "UCdmY7p_kC-QN8jS7rocmCSA"
SubhamVlogs = "UCXtdoCLRlnLIW3Skix0rHaQ"
DisciplesofJesusChristJeremiahPayne = "UC_y2rVsotcQcVF7LImVGs5Q"
ModernGalaxy = "UCGISiGs_RL7Z1qfs1-GD6PA"
socofilms = "UCyDS9p6NWHpU9XbbbYLFLBw"
buriedone = "UC_lm7xXB3adOTc0T1FwyGRQ"
bitcoin = "UC-f5nPBEDyUBZz_jOPBwAfQ"
globalrashid = "UC2ldcEtbR7cFYadgrnW3B6Q"
gameofbitcoins = "UC2WKsYBxMwx7E7ENkND-fkg"
CryptoInsightsBrasil = "UC4BrnREinCBUenQZi4ZU95w"
Crypt0 = "UCdUSSt-IEUg2eq46rD7lu_g"
cryptomined = "UCGQ3XHtsH8Q9iQr9bFbgfDA"
altcoinbuzz = "UCGyqEtcGQQtXyUwvcy7Gmyg"
crypto = "UCiMgF08KQ4z-Gnu8o2BLOxA"
Crypto99 = "UCjsrdOJCAKcuBqyyqjg1cCQ"
TheCryptoLark = "UCl2oCaw8hdR_kbqyqd2klIA"
NuggetsNews = "UCLo66QVfEod0nNM_GzKNxmQ"
btckyle = "UCNCGCxxTT10aeTgUMHW5FfQ"
LouisThomas = "UCpceefaJ9vs4RYUTsO9Y3FA"
BitsBeTrippin = "UCVVWXoQfMfQVuzxcLylq9aA"
cryptocrow = "UCwsRWmIL5XKqFtdytBfeX0g"
KouSuccessLeeFX = "UC0YkP4Fg_d8y6JFwLj3MLdg"
TheSchiffReport = "UCIjuLiLHdFxYtFmWlbTGQRQ"
MRU = "UCnkEhPBMZcEO0QGu51fDFDg"
Vidello = "UCwMaWqZ6SdDpTaYOW0huELw"
kaccreative = "UC02O9ICMuwrfULSa0N6SiSw"
shecooks = "UChZYqTJkeYV2r7WETcytdPQ"
vegsource = "UClEsPxvotpTJ1Z8eu2Y97rg"
VeganGains = "UCr2eKhGzPhN5RPVk5dd5o3g"
AwesomeKnowledge = "UC_pC4T8vr-caiNDYU6PYTTA"
luckyloush = "UCbO0Oomf_jr20Wb4V2ap5Uw"
MichaelLuzzi = "UCDwLrj4DSGrw3gFvANbl_Cw"
NakedApe = "UCMOaRU-YsXVgU-WahBkZqWQ"
ParaReact = "UCuJKELjsmWTlJG0X7T4ZD_A"
dullytv = "UCv1J91Nhn7KsxMFaxKChT3w"
REDONKULAS = "UCwd_sSDZ8EQt6SEeOO2tBRA"
RedactedTonight = "UCyvaZ2RHEDrgKXz43gz7CbQ"
TonyTornado = "UCZu9AV3mrCCDpK_dy1qJRAQ"
hoodlumscrafty = "UC0cTVjYKgAnBrXQKcICyNmA"
Karmakut = "UC2B8TOklu2rSDULqAzwn5GQ"
Draxr = "UC72o3j23E1wKskBDEKlmTOw"
BigfryTV = "UC7FVdUA3SxDMfl4fLBGlADg"
CSGONews = "UC7L6NRLyldvxWukhOHABazQ"
ZedGaming = "UC7Q15H71DmB7F1iSFE1Z8LQ"
KamiVS = "UC9fh15yUcGAr8iUfQaoRRpQ"
grabthegames = "UCaJFEgY6ij05Fxgn6qtcX_g"
blueplays = "UCbjMsFlYb2NLpjS3uDzm9ow"
ImNotBonkers = "UCCyN0G77B7wnf-1AOTI3gWQ"
GunslingerMedia = "UCDjs4JoXmMzvaZyPVsvjKXw"
LeagueVoices = "UCdkonRjBLzr1Adf7Jhu1bWQ"
SirPugger = "UCelqWKTcCvBZP_1_1iYZAWA"
gamesoup = "UCGPMrF9AN_D9BrmSmMeV3hA"
ProHenis = "UCIjFoXSQ9HYbcWmRVIsH-Ow"
Dota2Divine = "UCiR9IHCurqVHpC821NVcW6g"
CrazyFoxMovies = "UCjewtQLpJEENPLbrCtb6YpA"
Larry = "UCJVdNvvuvOnthuWVQjYff2w"
hottake = "UCK24784Oqb4oYQfHL9XePNg"
Breezy = "UCKWRpZpcLKriWd1am9SHf8A"
retrorgb = "UCLPIbBCKVH2uKGm5C4sOkew"
Zer0Gamer = "UCmII34jN4rqCIsGqWkK5JEg"
nickatnyte = "UCMxYQX1zaepCgmiSmwbT39w"
DyllonStejGaming = "UCngaLL0QDbsAGYj7zsB8o_Q"
bluedrake42 = "UCNSwcDEUfIEzYdAPscXo6ZA"
Rerez = "UCoFpRCAsKfWAshvLE1bYzdw"
Op = "UCowi5kFfvGXR8NqhyE6jneQ"
kidsgamesfun = "UCqjGzmb2pMWSFYm0GzKeTEA"
GamesGlitches = "UCRj3Q06KOxAZWHaHScrkAOw"
Musikage = "UCsej4tgCoXDgVH3J7M3NMgw"
JeffyGaming = "UCTZzSNnZ43XQslejT5wFdRw"
oniblackmage = "UCUEF9XL3o8dZ6hvVf8jAi8Q"
nickatnyte2 = "UCuoTqrobMyZj0ge8LOCkiSw"
KazeEmanuar = "UCuvSqzfO_LV_QzHdmEj84SQ"
TheLinuxGamer = "UCv1Kcz-CuGM6mxzL3B1_Eiw"
GaminGHD = "UCW-thz5HxE-goYq8yPds1Gw"
BHGaming = "UCX4N3DioqqrugFeilxTkSIw"
Potato = "UCxPPTDNH85HZWxrgZ3FQBYA"
MikeyTaylorGaming = "UCycXj6lRWtsSqo-bZOIZePw"
Acituanbus = "UCzfx1QvKjn-BxLMLBdBGgMw"
juggling = "UC2fhTIbnQlFYaFzyTcmPkXg"
KhanAcademy = "UC4a-Gbdw7vOaccHmFo40b9g"
DON = "UCAYrPk70AePJZSaVLKrWdfQ"
shogogarcia = "UCE3yZjxDg3iI91bcNJDFnsg"
alphalifestyleacademy = "UCeggEaXtJu2domMahYMD_ig"
NileRed = "UCFhXFikryT4aFcLkLw2LBLA"
veritasium = "UCHnyfMqiRRG1u-2MsSQLbXA"
stevecronin = "UCJYawZQYwjZ76mrF_US9eNg"
jeranism = "UCS_FY5mR4g22L_E9t1D_ExQ"
MinutePhysics = "UCUHW94eEFW7hkUMVaZz4eDg"
_3Blue1Brown = "UCYO_jab_esuFRV4b17AJtAw"
Itsrucka = "UC-B2LyEZcl3avG0coKeohGQ"
unitedtaps = "UC5Q8e9-uutVZmiwAcEHKuzA"
srodalmenara = "UC9kc1DaOy2kSzXHZgpn9kfQ"
ShutupAndPlay = "UCAwuvzhah0KUw5QNihSkEwQ"
sanx2 = "UCJ_waKl9kjbhfXI3LJOfHvA"
DerickWattsAndTheSundayBlues = "UCmZhiZq7M7d73Kbey4yna_Q"
akirathedon = "UCsoiSpBvkr4Y-78Pj3recUw"
Musicoterapia = "UCsoSK8K4OpdMV1tqJFwO5QA"
daydreaming = "UCtbuGylbRXc42pIxWey19Dg"
RemixHolicRecords = "UCUW5GjwcXgbPcfk00t-GQZA"
EDMBot = "UCvmUdL2NHWlj1NRiNJPI-TQ"
TrioTravels = "UCdAPAmdnkdFsH5R2Hxevucg"
caosonnguyen = "UCwPeW9kFId5-VbQ2LQEjVhg"
TheAlmonteFilms = "UC4C_SF5koS4Q5om50b9NMTw"
timcast = "UCG749Dj4V2fKa143f8sE60Q"
SeekingTheTruth = "UCHrDpTVL9S0h91u9UCPgVbA"
davidpakman = "UCvixJtaXuNdMPUGdOPcY8Ag"
mikenayna = "UCzk08fzh5c_BhjQa1w35wtA"
DoorMonster = "UC-to_wlckb-bFDtQfUZL3Kw"
barnacules = "UC1MwJy1R0nGQkXxRD9p-zTQ"
brightsunfilms = "UC5k3Kc0avyDJ2nG9Kxm9JmQ"
Onision = "UC5OxQNCgW88FDBxeZCnrBbg"
top10archive = "UCa03bf8gAS2EtffptV-_jfA"
thought = "UCb0yiUQhhLV_jpY3BayJaLA"
MothersBasement = "UCBs2Y3i14e1NWQxOGliatmg"
VlogsOfKnowledge = "UCDUPGR6uL5uz0_hiRAQjZMQ"
GorTheMovieGod = "UCHdTVw89QU6coU1MgN-9RHA"
iamalexoconnor = "UCKAQLEk1GGqnPtov9EW0huQ"
ADedits = "UCsX-zRuq3ovMsgFqEQLw2Bw"
SynthCool = "UCxGTHsD0pLSFlFI7M7jYmBQ"
JordanBPeterson = "UCL_f53ZEJxp8TtlOkHwMV9Q"
Sciencedocumentinhindi = "UC9SpfUF3rm-MGep5WE6FSCA"
NurdRage = "UCIgKGGJkt1MrNmhq3vRibYA"
MINDBLASTER = "UC_ZMqbRu44jK-EogjYyHz8g"
DaminousPurity = "UCdKRWvz50QoioZFgu6Nf9og"
ScammerRevolts = "UC0uJKUXiU5T41Fzawy5H6mw"
eevblog = "UC2DjFE7Xf11URZqWBigcVOQ"
Luke = "UC2eYFnH61tmytImy1mTYvhA"
AppGirl = "UC389S4_2Yt9cei1qNDwmBrA"
thecryptodad = "UC68x_TIzqCtF69fYl2_kl3w"
ChrisWereDigital = "UCAPR27YUyxmgwm3Wc2WSHLw"
alecaddd = "UCbmBY_XYZqCa2G0XmFA7ZWg"
EliTheComputerGuy = "UCD4EOyXKjfDUhCI6jlOZZYQ"
archetapp = "UCDIBBmkZIB2hjBsk1hUImdA"
PCPlaceNZ = "UCf5ZTSZAKbinY03jOwylfOg"
NaomiSexyCyborgWu = "UCh_ugKacslKhsGGdXP0cRRA"
NibiruWatcher = "UCi62JvN-lUn7hVL3ffofADA"
imineblocks = "UCjYHcWGAjUVqU49D2JOKD3w"
Lunduke = "UCkK9UDm_ZNrq_rIXCz3xCGA"
CooLoserTech = "UCl97rZ2Tc7KV9lktmmHNFDQ"
TechHD = "UCN3bPy04Jkp3ADRtyYvXomQ"
MiketheScrapper = "UCqtlJpkH_llXS_vuDExGVvw"
eevblog2 = "UCr-cm90DwFJC0W3f9jBs5jA"
thecreativeone = "UCTikFhzCiIXfOMS7D29dvYg"
weekendtricks = "UCYtAJXx0ymGPpCndn2Gt6-w"
GBGuns = "UC2VOURrALs1CwVmbGlXJOPQ"
Matsimus = "UCFWjEwhX6cSAKBQ28pufG3w"
TheLateBoyScout = "UCZjvj5MN3BMxPFfdEKIrvxQ"
BravoCinematografica = "UC2ruSXQoKMgr7JXzwO0H0KA"
PyroNation = "UC4ffy3n1hE7Z8q-2KVq_91Q"
dramatuber = "UC4Y8mImty3gFEG9grsR6T-w"
BarnabasNagy = "UC8TRZRK1sJfxKJ0tXMmGTow"
avery = "UCcfjIZLDCuSqkIlkH8i4DDg"
Tingledove = "UCfI7wtV6K64gVbzjH7DOA_Q"
crmjewelers = "UChpFWeF84jA5JeV3YyIo3pQ"
NerfNerd18 = "UCIgmaEJNqvH9bU9zGMapZGg"
SEIJIHITO = "UCNqUrLE6dI8fWw_u3HQkpXA"
anvithavlogs = "UCsP9pYat2DEBvnvF_iFGG_w"
YoelRekts = "UCZ_BcFyhIo6GdtTSrqXXepg"
TechFox = "UCIp-oTSdFO7BhAJpW2d5HMQ"
)
var channelWideTags = map[string][]string{
Lunduke: {"linux", "technology"},
SwissExperiments: {"science & technology", "experiments", "switzerland"},
Juggling: {"juggling", "circus arts", "malabares"},
JustJuggling: {"juggling", "circus arts", "malabares"},
JordanBPeterson: {"postmodernism", "psychology", "news"},
DavidPakman: {"news"},
TheLinuxGamer: {"gaming", "technology", "linux"},
JustJuggling: {"juggling", "circus arts", "malabares"},
SwissExperiments: {"science & technology", "experiments", "switzerland"},
TechFox: {"technology", "reviews"},
misszizi: {"art", "pop culture"},
creationshub: {"art"},
AlicePandora: {"art"},
Akito0405: {"art"},
dashcambristol: {"automotive"},
adrianbonillap: {"automotive"},
SubhamVlogs: {"automotive"},
DisciplesofJesusChristJeremiahPayne: {"beliefs"},
ModernGalaxy: {"beliefs"},
socofilms: {"beliefs"},
buriedone: {"blockchain"},
bitcoin: {"blockchain"},
globalrashid: {"blockchain"},
gameofbitcoins: {"blockchain"},
CryptoInsightsBrasil: {"blockchain"},
Crypt0: {"blockchain"},
cryptomined: {"blockchain"},
altcoinbuzz: {"blockchain", "technology"},
crypto: {"blockchain"},
Crypto99: {"blockchain"},
TheCryptoLark: {"blockchain", "technology"},
NuggetsNews: {"blockchain", "learning"},
btckyle: {"blockchain"},
LouisThomas: {"blockchain"},
BitsBeTrippin: {"blockchain"},
cryptocrow: {"blockchain"},
KouSuccessLeeFX: {"economics", "learning"},
TheSchiffReport: {"economics"},
MRU: {"economics"},
Vidello: {"economics", "pop culture"},
kaccreative: {"food", "art"},
shecooks: {"food"},
vegsource: {"food"},
VeganGains: {"food"},
AwesomeKnowledge: {"funny"},
luckyloush: {"funny"},
MichaelLuzzi: {"funny", "pop culture"},
NakedApe: {"funny"},
ParaReact: {"funny"},
dullytv: {"funny"},
REDONKULAS: {"funny"},
RedactedTonight: {"funny", "news"},
TonyTornado: {"funny"},
hoodlumscrafty: {"gaming", "pop culture"},
Karmakut: {"gaming"},
Draxr: {"gaming", "pop culture"},
BigfryTV: {"gaming"},
CSGONews: {"gaming", "funny"},
ZedGaming: {"gaming"},
KamiVS: {"gaming"},
grabthegames: {"gaming"},
blueplays: {"gaming"},
ImNotBonkers: {"gaming", "funny"},
GunslingerMedia: {"gaming"},
LeagueVoices: {"gaming"},
SirPugger: {"gaming"},
gamesoup: {"gaming"},
ProHenis: {"gaming"},
Dota2Divine: {"gaming"},
CrazyFoxMovies: {"gaming", "pop culture", "technology"},
Larry: {"gaming"},
hottake: {"gaming", "funny"},
Breezy: {"gaming", "funny"},
retrorgb: {"gaming"},
Zer0Gamer: {"gaming"},
nickatnyte: {"gaming"},
DyllonStejGaming: {"gaming"},
bluedrake42: {"gaming"},
Rerez: {"gaming"},
Op: {"gaming"},
kidsgamesfun: {"gaming", "pop culture"},
GamesGlitches: {"gaming"},
Musikage: {"gaming", "pop culture"},
JeffyGaming: {"gaming", "funny"},
oniblackmage: {"gaming"},
nickatnyte2: {"gaming"},
KazeEmanuar: {"gaming"},
TheLinuxGamer: {"gaming", "technology", "linux"},
GaminGHD: {"gaming", "pop culture"},
BHGaming: {"gaming"},
Potato: {"gaming"},
MikeyTaylorGaming: {"gaming"},
Acituanbus: {"gaming"},
juggling: {"juggling", "circus art", "malabares"},
KhanAcademy: {"learning", "science"},
DON: {"learning", "pop culture"},
shogogarcia: {"learning"},
alphalifestyleacademy: {"learning"},
NileRed: {"learning", "science"},
veritasium: {"learning", "science"},
stevecronin: {"learning"},
jeranism: {"learning"},
MinutePhysics: {"learning", "science"},
_3Blue1Brown: {"learning"},
Itsrucka: {"music"},
unitedtaps: {"music"},
srodalmenara: {"music"},
ShutupAndPlay: {"music", "learning"},
sanx2: {"music"},
DerickWattsAndTheSundayBlues: {"music", "funny"},
akirathedon: {"music"},
Musicoterapia: {"music"},
daydreaming: {"music"},
RemixHolicRecords: {"music"},
EDMBot: {"music"},
TrioTravels: {"nature"},
caosonnguyen: {"nature"},
TheAlmonteFilms: {"news"},
timcast: {"news", "technology"},
SeekingTheTruth: {"news"},
davidpakman: {"news"},
mikenayna: {"news"},
DoorMonster: {"pop culture", "funny"},
barnacules: {"pop culture", "gaming"},
brightsunfilms: {"pop culture"},
Onision: {"pop culture", "funny"},
top10archive: {"pop culture"},
thought: {"pop culture", "learning"},
MothersBasement: {"pop culture", "gaming"},
VlogsOfKnowledge: {"pop culture", "gaming"},
GorTheMovieGod: {"pop culture"},
iamalexoconnor: {"pop culture"},
ADedits: {"pop culture"},
SynthCool: {"pop culture", "funny"},
JordanBPeterson: {"psychology", "postmodernism", "news"},
Sciencedocumentinhindi: {"science"},
NurdRage: {"science", "learning"},
MINDBLASTER: {"sports", "funny"},
DaminousPurity: {"sports", "gaming"},
ScammerRevolts: {"technology"},
eevblog: {"technology", "learning"},
Luke: {"technology", "funny"},
AppGirl: {"technology"},
thecryptodad: {"technology", "blockchain"},
ChrisWereDigital: {"technology"},
alecaddd: {"technology", "learning"},
EliTheComputerGuy: {"technology"},
archetapp: {"technology", "learning"},
PCPlaceNZ: {"technology"},
NaomiSexyCyborgWu: {"technology"},
NibiruWatcher: {"technology", "learning"},
imineblocks: {"technology", "blockchain"},
Lunduke: {"technology"},
CooLoserTech: {"technology"},
TechHD: {"technology", "learning"},
MiketheScrapper: {"technology"},
eevblog2: {"technology"},
thecreativeone: {"technology", "gaming"},
weekendtricks: {"technology"},
GBGuns: {"weapons"},
Matsimus: {"weapons", "gaming"},
TheLateBoyScout: {"weapons"},
}
var tagsToSkip = map[string]*struct{}{
"#hangoutsonair": nil,
+5 -4
View File
@@ -61,10 +61,11 @@ func (u *thumbnailUploader) uploadThumbnail() error {
uploader := s3manager.NewUploader(s3Session)
_, err = uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String("thumbnails.lbry.com"),
Key: key,
Body: thumb,
ACL: aws.String("public-read"),
Bucket: aws.String("thumbnails.lbry.com"),
Key: key,
Body: thumb,
ACL: aws.String("public-read"),
ContentType: aws.String("image/jpeg"),
})
u.mirroredUrl = ThumbnailEndpoint + u.name
return errors.Err(err)
+34
View File
@@ -0,0 +1,34 @@
package util
import (
"fmt"
"github.com/lbryio/lbry.go/extras/util"
log "github.com/sirupsen/logrus"
)
// SendErrorToSlack Sends an error message to the default channel and to the process log.
func SendErrorToSlack(format string, a ...interface{}) {
message := format
if len(a) > 0 {
message = fmt.Sprintf(format, a...)
}
log.Errorln(message)
err := util.SendToSlack(":sos: " + message)
if err != nil {
log.Errorln(err)
}
}
// SendInfoToSlack Sends an info message to the default channel and to the process log.
func SendInfoToSlack(format string, a ...interface{}) {
message := format
if len(a) > 0 {
message = fmt.Sprintf(format, a...)
}
log.Infoln(message)
err := util.SendToSlack(":information_source: " + message)
if err != nil {
log.Errorln(err)
}
}