Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1760935edf | ||
|
|
60f2585f33 | ||
|
|
df08d42d9b | ||
|
|
1b55033bb8 | ||
|
|
1331e5769f | ||
|
|
43f745ae20 | ||
|
|
8c2a8262e6 | ||
|
|
6f486717da | ||
|
|
a3fcd67611 | ||
|
|
ae1ffb60c5 | ||
|
|
fa24d83ae9 | ||
|
|
2f11d99a97 | ||
|
|
bab01a6354 | ||
|
|
89fe1e99d6 | ||
|
|
ec81af0857 | ||
|
|
51cb7f5e34 | ||
|
|
8ad2acc936 | ||
|
|
b8caf3f82b | ||
|
|
b34d328cba | ||
|
|
a0dcb2216d | ||
|
|
9d6a7e03cd | ||
|
|
e44e272689 | ||
|
|
87f7d4c9f9 | ||
|
|
173e62d286 | ||
|
|
013760d48d | ||
|
|
5dbad2d2e4 | ||
|
|
545ab07013 | ||
|
|
c402a1cdef | ||
|
|
a4c6b2da30 | ||
|
|
7f48220a80 | ||
|
|
9e49110295 | ||
|
|
4506bdb638 | ||
|
|
a8551bc9ee | ||
|
|
b5c9806dd0 | ||
|
|
8c349cf7ac | ||
|
|
f3f40c41ef | ||
|
|
05b65c4cb5 | ||
|
|
69ad13848b | ||
|
|
e496ad670a | ||
|
|
9d8eb32f9f | ||
|
|
6668840b11 | ||
|
|
e893f68bc1 | ||
|
|
8da631eac3 | ||
|
|
1729af0ae5 | ||
|
|
dce71d2451 | ||
|
|
f4e75cf221 | ||
|
|
ae3dad944d | ||
|
|
f7f0231c80 | ||
|
|
0c42b96332 | ||
|
|
52cd483f47 | ||
|
|
79dd6d1458 | ||
|
|
10ad46f464 | ||
|
|
ccebab35da | ||
|
|
8dce052fe6 | ||
|
|
76e653fb9b | ||
|
|
cfe8ff0879 | ||
|
|
3389a8be93 | ||
|
|
68c164ed5d | ||
|
|
913c31a9f7 | ||
|
|
b184f033ab | ||
|
|
6767fdcc40 | ||
|
|
e73085b6bc | ||
|
|
7df4137445 | ||
|
|
74d5e2606d | ||
|
|
b8d2b9335b | ||
|
|
637785f4ac | ||
|
|
c9dffb0a82 | ||
|
|
6ea0bd9834 | ||
|
|
93986c53e0 | ||
|
|
843575c4ba | ||
|
|
aecfc67f84 | ||
|
|
1f7017f3d0 | ||
|
|
75364805a5 | ||
|
|
07f66a089f | ||
|
|
36abd1e471 | ||
|
|
502bfa924b | ||
|
|
23b8fb4e54 | ||
|
|
a648287e1c | ||
|
|
af13ec5684 |
No files matched your search
+1
-1
@@ -9,7 +9,7 @@ env:
|
||||
global:
|
||||
- GO111MODULE=on
|
||||
#GITHUB_TOKEN
|
||||
- secure: "+G0TxyGyRYJb0L0qCCn6NPregigwhalkS9+/3mkV41xoAwUlEU9aclSJQVKKqeMrV0PkmJ6KrzYD78tnRj8Q5ZDe93LBDbc/WhXZ91JbRmIXIsVktz/78R2a2Q0DBrXMlyWXS84QouRfI3eyxf2BusERSYdr3H6BL4r15PIHJEmLGOchHsHwSI46PRGag+IFnbUb+fyUSs7er6gKeSp4+ZK1bSUgxiMUQrZfImYI9LS+cgqpDh67pX5jRbDNBihZZUwfreToYw3RbIAom4i2oix1rBBD/jpcTwO+WdyXFtur8RYGnsknePU88KiAB9bcXU4T2xVxR9wusK6jX5G2BChEOG9PjuVfeDqXBnskOEtAevwAKJRYW6ATmXqQZ+TAy1DFxsUWPVS7ykIA+69n7HzE7xu50AB1U2n6myT977i4jhBzw3VrB9NQz//4b2j6MCG1PhIgB38kffQWVsa/CF4RiiWi6a7ePRsPP0HdRUbXFQJWPevKgVJ9ve/ArQKiea1rP9zy9KdFAP30vESBkn4WMOCjfg7F6JbfV3e1ywBq9XMznj14iAZHwXR1tpPp0GekuMlVJHJWa0Ea/dfDGWshnr1Be71vc9T118w7sd0v1Dp4EQCFa4B0Hx7Ual7uWgeuweL/JLr1Am3BgWVcXOWrhzMT0b1tGFa9ZNz2Wwk="
|
||||
- secure: "Ps3KocRP5xnM3/uA99CeYhDTVxRIuW7fGyrtqBeRWZW0cXzeA4XCTKxqcFbrPUPw67XkrBVgE58JDdWoQEJ7tm67PjMm/ltp5Evhx/QAJDh+YSofXyGDVpG1mrTZFI66R3NVVJLkSGALMkuWWXvfYZeU//AworJbyRoaIK/CVt5OP23i5N4tdd5UXc5dfLuYqnKRynyMmCkz9c3yEIQMXoPhG2hx7l7L2BeMJvcKmVhkSN7nQayjnrbUXGm/IRqrb88lvkyBevN5E3IB2V5IKEieIPZjbD/N0IfcnAt89Z96tgDhtIbx3ZvXm92lsvHA8buqQpG9d2AmSi6GKs64lQcnGeM5o0wER2JHWl1OSa1Nr/UAo5Xb/PM65Yt3yZE8AuMKHBmbfDSBzdkTXx58AeDzFUd3kMXD/fFjeQQWyXFlOss3ygH9SObl827Txmz9OJqZaxabs5Q3AP6m3EjKjz7zfLfrgpcxJM2WBiU1bN0ZxUgZkImy/CHk5gCZ7vhcnaLiDO4HZnzY/aRJwKYQPE5i0O2nHpIfovqkc0DFBA7U/7Cjin7e1E0UZvF3meLOxMqkfc6X7QTxqQpt2Tej6jlpdxw4CTLwGUhGkAw9IAPkUB3L0EbZ1/ksGhNvGDvUeSTq8hYdMAPmA+k9jS6653V4SQ+qBMy5++tbr5AeZQI="
|
||||
deploy:
|
||||
provider: script
|
||||
skip_cleanup: true
|
||||
|
||||
@@ -55,7 +55,7 @@ Usage:
|
||||
|
||||
Flags:
|
||||
--after int Specify from when to pull jobs [Unix time](Default: 0)
|
||||
--before int Specify until when to pull jobs [Unix time](Default: current Unix time) (default 1582834707)
|
||||
--before int Specify until when to pull jobs [Unix time](Default: current Unix time) (default current timestamp)
|
||||
--channelID string If specified, only this channel will be synced.
|
||||
--concurrent-jobs int how many jobs to process concurrently (default 1)
|
||||
-h, --help help for ytsync
|
||||
@@ -63,12 +63,14 @@ Flags:
|
||||
--max-length float Maximum video length to process (in hours) (default 2)
|
||||
--max-size int Maximum video size to process (in MB) (default 2048)
|
||||
--max-tries int Number of times to try a publish that fails (default 3)
|
||||
--remove-db-unpublished Remove videos from the database that are marked as published but aren't really published
|
||||
--run-once Whether the process should be stopped after one cycle or not
|
||||
--skip-space-check Do not perform free space check on startup
|
||||
--status string Specify which queue to pull from. Overrides --update
|
||||
--stop-on-error If a publish fails, stop all publishing and exit
|
||||
--takeover-existing-channel If channel exists and we don't own it, take over the channel
|
||||
--update Update previously synced channels instead of syncing new ones
|
||||
--upgrade-metadata Upgrade videos if they're on the old metadata version
|
||||
--videos-limit int how many videos to process per channel (default 1000)
|
||||
```
|
||||
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
package blobs_reflector
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
"github.com/lbryio/reflector.go/cmd"
|
||||
"github.com/lbryio/reflector.go/db"
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
"github.com/mitchellh/go-ps"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/user"
|
||||
"path/filepath"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func ReflectAndClean() error {
|
||||
err := reflectBlobs()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return cleanupLbrynet()
|
||||
}
|
||||
|
||||
func loadConfig(path string) (cmd.Config, error) {
|
||||
var c cmd.Config
|
||||
|
||||
raw, err := ioutil.ReadFile(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return c, errors.Err("config file not found")
|
||||
}
|
||||
return c, err
|
||||
}
|
||||
|
||||
err = json.Unmarshal(raw, &c)
|
||||
return c, err
|
||||
}
|
||||
|
||||
func reflectBlobs() error {
|
||||
//make sure lbrynet is off
|
||||
running, err := isLbrynetRunning()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if running {
|
||||
return errors.Prefix("cannot reflect blobs as the daemon is running", err)
|
||||
}
|
||||
|
||||
dbHandle := new(db.SQL)
|
||||
ex, err := os.Executable()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
exPath := filepath.Dir(ex)
|
||||
config, err := loadConfig(exPath + "/prism_config.json")
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
err = dbHandle.Connect(config.DBConn)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer func() {
|
||||
err := dbHandle.CloseDB()
|
||||
if err != nil {
|
||||
log.Errorf("failed to close db handle: %s", err.Error())
|
||||
}
|
||||
|
||||
}()
|
||||
st := store.NewDBBackedS3Store(
|
||||
store.NewS3BlobStore(config.AwsID, config.AwsSecret, config.BucketRegion, config.BucketName),
|
||||
dbHandle)
|
||||
|
||||
uploadWorkers := 10
|
||||
uploader := reflector.NewUploader(dbHandle, st, uploadWorkers, false)
|
||||
usr, err := user.Current()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
blobsDir := usr.HomeDir + "/.lbrynet/blobfiles/"
|
||||
err = uploader.Upload(blobsDir)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
if uploader.GetSummary().Err > 0 {
|
||||
return errors.Err("not al blobs were reflected. Errors: %d", uploader.GetSummary().Err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func cleanupLbrynet() error {
|
||||
//make sure lbrynet is off
|
||||
running, err := isLbrynetRunning()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if running {
|
||||
return errors.Prefix("cannot cleanup lbrynet as the daemon is running", err)
|
||||
}
|
||||
usr, err := user.Current()
|
||||
if err != nil {
|
||||
log.Errorln(err.Error())
|
||||
return errors.Err(err)
|
||||
}
|
||||
lbrynetDir := usr.HomeDir + "/.lbrynet/"
|
||||
files, err := filepath.Glob(lbrynetDir + "lbrynet.sqlite*")
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
for _, f := range files {
|
||||
err = os.Remove(f)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
}
|
||||
blobsDir := lbrynetDir + "/blobfiles/"
|
||||
err = os.RemoveAll(blobsDir)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
err = os.Mkdir(blobsDir, 0755)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isLbrynetRunning() (bool, error) {
|
||||
processes, err := ps.Processes()
|
||||
if err != nil {
|
||||
return true, errors.Err(err)
|
||||
}
|
||||
var daemonProcessId = -1
|
||||
for _, p := range processes {
|
||||
if p.Executable() == "lbrynet" {
|
||||
daemonProcessId = p.Pid()
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
running := daemonProcessId != -1
|
||||
return running, nil
|
||||
}
|
||||
@@ -3,28 +3,27 @@ module github.com/lbryio/ytsync
|
||||
require (
|
||||
cloud.google.com/go v0.37.4 // indirect
|
||||
github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61
|
||||
github.com/PuerkitoBio/goquery v1.5.0 // indirect
|
||||
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
|
||||
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf
|
||||
github.com/aws/aws-sdk-go v1.17.3
|
||||
github.com/btcsuite/btcd v0.0.0-20190410025418-9bfb2ca0346b // indirect
|
||||
github.com/btcsuite/btcutil v0.0.0-20190316010144-3ac1210f4b38 // indirect
|
||||
github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 // indirect
|
||||
github.com/go-ini/ini v1.42.0 // indirect
|
||||
github.com/golang/protobuf v1.3.1 // indirect
|
||||
github.com/go-sql-driver/mysql v1.4.1 // indirect
|
||||
github.com/gopherjs/gopherjs v0.0.0-20190411002643-bd77b112433e // indirect
|
||||
github.com/gorilla/websocket v1.4.0 // indirect
|
||||
github.com/hashicorp/go-msgpack v0.5.5 // indirect
|
||||
github.com/hashicorp/golang-lru v0.5.1 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.0.0 // indirect
|
||||
github.com/hashicorp/memberlist v0.1.4 // indirect
|
||||
github.com/hashicorp/serf v0.8.2 // indirect
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
|
||||
github.com/kr/pretty v0.1.0 // indirect
|
||||
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c
|
||||
github.com/lbryio/lbry.go v0.0.0-20190419005332-80b25b225e18
|
||||
github.com/lbryio/types v0.0.0-20190415181811-35ddf1afe731 // indirect
|
||||
github.com/lbryio/lbry.go v1.0.15
|
||||
github.com/lbryio/reflector.go v1.0.6-0.20190710201919-7c7ed9da72ce
|
||||
github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2 // indirect
|
||||
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936
|
||||
github.com/mitchellh/mapstructure v1.1.2 // indirect
|
||||
github.com/nikooo777/ytdl v0.0.0-20190215151411-9c7832eaf457
|
||||
github.com/onsi/ginkgo v1.8.0 // indirect
|
||||
github.com/onsi/gomega v1.5.0 // indirect
|
||||
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 // indirect
|
||||
github.com/prometheus/common v0.3.0
|
||||
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24
|
||||
github.com/sirupsen/logrus v1.4.1
|
||||
@@ -35,14 +34,13 @@ require (
|
||||
github.com/ybbus/jsonrpc v2.1.2+incompatible // indirect
|
||||
go.opencensus.io v0.20.2 // indirect
|
||||
golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480 // indirect
|
||||
golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2 // indirect
|
||||
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c // indirect
|
||||
golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a // indirect
|
||||
golang.org/x/sys v0.0.0-20190418153312-f0ce4c0180be // indirect
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58 // indirect
|
||||
golang.org/x/text v0.3.2 // indirect
|
||||
google.golang.org/api v0.3.2
|
||||
google.golang.org/appengine v1.5.0 // indirect
|
||||
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7 // indirect
|
||||
google.golang.org/grpc v1.20.0 // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
|
||||
gopkg.in/ini.v1 v1.42.0 // indirect
|
||||
gopkg.in/yaml.v2 v2.2.2 // indirect
|
||||
)
|
||||
@@ -5,8 +5,7 @@ cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7h
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 h1:N5Vqww5QISEHsWHOWDEx4PzdIay3Cg0Jp7zItq2ZAro=
|
||||
github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61/go.mod h1:GnKXcK+7DYNy/8w2Ex//Uql4IgfaU82Cd5rWKb7ah00=
|
||||
github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP7EJk=
|
||||
github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg=
|
||||
github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
|
||||
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
|
||||
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
|
||||
github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
|
||||
@@ -14,32 +13,44 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5Vpd
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
|
||||
github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
|
||||
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
|
||||
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
|
||||
github.com/armon/go-metrics v0.0.0-20180713145231-3c58d8115a78 h1:mdRSArcFLfW0VoL34LZAKSz6LkkK4jFxVx2xYavACMg=
|
||||
github.com/armon/go-metrics v0.0.0-20180713145231-3c58d8115a78/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
|
||||
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
|
||||
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM=
|
||||
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg=
|
||||
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
|
||||
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf h1:eg0MeVzsP1G42dRafH3vf+al2vQIJU0YHX+1Tw87oco=
|
||||
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
|
||||
github.com/aws/aws-sdk-go v1.16.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
||||
github.com/aws/aws-sdk-go v1.17.3 h1:KBXxg7Jh0TxE5zmpNB2DwKmJeDUqh0O6jhy25TuYOmc=
|
||||
github.com/aws/aws-sdk-go v1.17.3/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
|
||||
github.com/btcsuite/btcd v0.0.0-20180531025944-86fed781132a/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ=
|
||||
github.com/btcsuite/btcd v0.0.0-20190410025418-9bfb2ca0346b h1:7J7sEce3LgtbMgs7PKcN61gF3b4txM6SjaRoJTSk640=
|
||||
github.com/btcsuite/btcd v0.0.0-20190410025418-9bfb2ca0346b/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8=
|
||||
github.com/btcsuite/btcd v0.0.0-20190109040709-5bda5314ca95/go.mod h1:d3C0AkH6BRcvO8T0UEPu53cnw4IbV63x1bEjildYhO0=
|
||||
github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32 h1:qkOC5Gd33k54tobS36cXdAzJbeHaduLtnLQQwNoIi78=
|
||||
github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8=
|
||||
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo=
|
||||
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
|
||||
github.com/btcsuite/btcutil v0.0.0-20180524032703-d4cc87b86016/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
|
||||
github.com/btcsuite/btcutil v0.0.0-20180706230648-ab6388e0c60a/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
|
||||
github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803 h1:j3AgPKKZtZStM2nyhrDSLSYgT7YHrZKdSkq1OYeLjvM=
|
||||
github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
|
||||
github.com/btcsuite/btcutil v0.0.0-20190316010144-3ac1210f4b38 h1:GbQHMJ2u/geMPV1tbN7i7zARSoPAPuXWa44V0KYvJXU=
|
||||
github.com/btcsuite/btcutil v0.0.0-20190316010144-3ac1210f4b38/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
|
||||
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd h1:R/opQEbFEy9JGkIguV40SvRY1uliPX8ifOvi6ICsFCw=
|
||||
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg=
|
||||
github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY=
|
||||
github.com/btcsuite/goleveldb v1.0.0/go.mod h1:QiK9vBlgftBg6rWQIj6wFzbPfRjiykIEhBH4obrXJ/I=
|
||||
github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
|
||||
github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
|
||||
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3SkEwmHoWBmX1DNXhXZqlTpq6s4tyJGc=
|
||||
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
|
||||
github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
|
||||
github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 h1:o64h9XF42kVEUuhuer2ehqrlX8rZmvQSU0+Vpj1rF6Q=
|
||||
github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61/go.mod h1:Rp8e0DCtEKwXFOC6JPJQVTz8tuGoGvw6Xfexggh/ed0=
|
||||
github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
|
||||
github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
@@ -49,6 +60,7 @@ github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5m
|
||||
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
|
||||
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
|
||||
github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g=
|
||||
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
|
||||
github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
|
||||
github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
|
||||
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
|
||||
@@ -56,26 +68,30 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
|
||||
github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
|
||||
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
|
||||
github.com/go-ini/ini v1.38.2/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-ini/ini v1.41.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-ini/ini v1.42.0 h1:TWr1wGj35+UiWHlBA8er89seFXxzwFn11spilrrj+38=
|
||||
github.com/go-ini/ini v1.42.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
|
||||
github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
|
||||
github.com/go-sql-driver/mysql v0.0.0-20180719071942-99ff426eb706/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
||||
github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
|
||||
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
|
||||
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.0 h1:kbxbvI4Un1LUWKxufD+BiE6AEExYYgkQLQmLFqA1LFk=
|
||||
github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
|
||||
github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
|
||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw=
|
||||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
@@ -83,37 +99,84 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi
|
||||
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20190411002643-bd77b112433e h1:XWcjeEtTFTOVA9Fs1w7n2XBftk5ib4oZrhzWk0B+3eA=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20190411002643-bd77b112433e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8=
|
||||
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
|
||||
github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk=
|
||||
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
|
||||
github.com/gorilla/rpc v1.1.0 h1:marKfvVP0Gpd/jHlVBKCQ8RAoUPdX7K1Nuh6l1BNh7A=
|
||||
github.com/gorilla/rpc v1.1.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ=
|
||||
github.com/gorilla/websocket v1.2.0 h1:VJtLvh6VQym50czpZzx07z/kw9EgAxI3x1ZB8taTMQQ=
|
||||
github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
|
||||
github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
|
||||
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
|
||||
github.com/hashicorp/errwrap v0.0.0-20180715044906-d6c0cd880357 h1:Rem2+U35z1QtPQc6r+WolF7yXiefXqDKyk+lN2pE164=
|
||||
github.com/hashicorp/errwrap v0.0.0-20180715044906-d6c0cd880357/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
|
||||
github.com/hashicorp/go-immutable-radix v0.0.0-20180129170900-7f3cd4390caa h1:0nA8i+6Rwqaq9xlpmVxxTwk6rxiEhX+E6Wh4vPNHiS8=
|
||||
github.com/hashicorp/go-immutable-radix v0.0.0-20180129170900-7f3cd4390caa/go.mod h1:6ij3Z20p+OhOkCSrA0gImAWoHYQRGbnlcuk6XYTiaRw=
|
||||
github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0=
|
||||
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
|
||||
github.com/hashicorp/go-msgpack v0.0.0-20150518234257-fa3f63826f7c h1:BTAbnbegUIMB6xmQCwWE8yRzbA4XSpnZY5hvRJC188I=
|
||||
github.com/hashicorp/go-msgpack v0.0.0-20150518234257-fa3f63826f7c/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
|
||||
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
|
||||
github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI=
|
||||
github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
|
||||
github.com/hashicorp/go-multierror v0.0.0-20180717150148-3d5d8f294aa0 h1:j30noezaCfvNLcdMYSvHLv81DxYRSt1grlpseG67vhU=
|
||||
github.com/hashicorp/go-multierror v0.0.0-20180717150148-3d5d8f294aa0/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=
|
||||
github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
|
||||
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
|
||||
github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
|
||||
github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86 h1:7YOlAIO2YWnJZkQp7B5eFykaIY7C9JndqAFQyVV5BhM=
|
||||
github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
|
||||
github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs=
|
||||
github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
|
||||
github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
|
||||
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE=
|
||||
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
|
||||
github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
|
||||
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
|
||||
github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
|
||||
github.com/hashicorp/memberlist v0.1.0 h1:qSsCiC0WYD39lbSitKNt40e30uorm2Ss/d4JGU1hzH8=
|
||||
github.com/hashicorp/memberlist v0.1.0/go.mod h1:ncdBp14cuox2iFOq3kDiquKU6fqsTBc3W6JvZwjxxsE=
|
||||
github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
|
||||
github.com/hashicorp/memberlist v0.1.4 h1:gkyML/r71w3FL8gUi74Vk76avkj/9lYAY9lvg0OcoGs=
|
||||
github.com/hashicorp/memberlist v0.1.4/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
|
||||
github.com/hashicorp/serf v0.0.0-20180530155958-984a73625de3 h1:NUr1hG6WO9sI1x8ofSimmpqfJ+rEHiHP/PLEA33rcfQ=
|
||||
github.com/hashicorp/serf v0.0.0-20180530155958-984a73625de3/go.mod h1:h/Ru6tmZazX7WO/GDmwdpS975F019L4t5ng5IgwbNrE=
|
||||
github.com/hashicorp/serf v0.8.2 h1:YZ7UKsJv+hKjqGVUUbtE3HNj79Eln2oQ75tniF6iPt0=
|
||||
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
|
||||
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf h1:WfD7VjIE6z8dIvMsI4/s+1qr5EL+zoIGev1BQj1eoJ8=
|
||||
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h1:hyb9oH7vZsitZCiBt0ZvifOrB+qc8PS5IiilCIb87rg=
|
||||
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
|
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
|
||||
github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22 h1:jKUP9TQ0c7X3w6+IPyMit07RE42MtTWNd77sN2cHngQ=
|
||||
github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22/go.mod h1:u0Jo4f2dNlTJeeOywkM6bLwxq6gC3pZ9rEFHn3AhTdk=
|
||||
github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07 h1:+kBG/8rjCa6vxJZbUjAiE4MQmBEBYc8nLEb51frnvBY=
|
||||
github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07/go.mod h1:j1kV/8f3jowErEq4XyeypkCdvg5EeHkf0YCKCcq5Ybo=
|
||||
github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE=
|
||||
github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
|
||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
|
||||
github.com/kkdai/bstream v0.0.0-20181106074824-b3251f7901ec/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
@@ -125,65 +188,94 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M=
|
||||
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8=
|
||||
github.com/lbryio/lbry.go v0.0.0-20190419005332-80b25b225e18 h1:ZhWjtvaq5r7julhcF9OSgx4bLv9UsdIx27zH1/fbrEc=
|
||||
github.com/lbryio/lbry.go v0.0.0-20190419005332-80b25b225e18/go.mod h1:kd08aOMCuBVYJ3EafY4Kx3dRAWWQYhobJ9beREgsaRI=
|
||||
github.com/lbryio/lbry.go v0.0.0-20190109223729-30c312501602/go.mod h1:YEuFJD/oHNra6BFy+NfuvS84Wg6RMWJFGtiCCCc6MmQ=
|
||||
github.com/lbryio/lbry.go v1.0.15 h1:g4g9cDDUsobmWgTA+1jEIb5k2fRCP0/NvPOMXduP8xY=
|
||||
github.com/lbryio/lbry.go v1.0.15/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002 h1:urfYK5ElpUrAv90auPLldoVC60LwiGAcY0OE6HJB9KI=
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
|
||||
github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4=
|
||||
github.com/lbryio/types v0.0.0-20190405005919-54c3c28f676a h1:twWvrsBDvSb+qnmpSq3nvFrodgC5PpXUipyo4T/W790=
|
||||
github.com/lbryio/types v0.0.0-20190405005919-54c3c28f676a/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
|
||||
github.com/lbryio/types v0.0.0-20190415181811-35ddf1afe731 h1:iERWR8Dkng30eRInI7gzolTEJBW9nBSK/sT+Z9aSipI=
|
||||
github.com/lbryio/types v0.0.0-20190415181811-35ddf1afe731/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
|
||||
github.com/lbryio/reflector.go v1.0.6-0.20190710201919-7c7ed9da72ce h1:ixsKssly++rkh0DWxNDrZyiRp00NieKZlndkqtpgj/c=
|
||||
github.com/lbryio/reflector.go v1.0.6-0.20190710201919-7c7ed9da72ce/go.mod h1:7Y3YYeAKS6egH2WzwfU8f6+uNGjVHfLzKvwn+Nv3VMY=
|
||||
github.com/lbryio/types v0.0.0-20181001180206-594241d24e00/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
|
||||
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c h1:m3O7561xBQ00lfUVayW4c6SnpVbUDQtPUwGcGYSUYQA=
|
||||
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
|
||||
github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5 h1:AsEBgzv3DhuYHI/GiQh2HxvTP71HCCE9E/tzGUzGdtU=
|
||||
github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0=
|
||||
github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6 h1:iOAVXzZyXtW408TMYejlUPo6BIn92HmOacWtIfNyYns=
|
||||
github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg=
|
||||
github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2 h1:CTzpAplpuXay3gDDqbErv/OSV6SoKZ9UTsGxLKfxB8M=
|
||||
github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg=
|
||||
github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
|
||||
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 h1:mG83tLXWSRdcXMWfkoumVwhcCbf3jHF9QKv/m37BkM0=
|
||||
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCWffGOaDcjkw1iB7W9DVLp6GXmfcJY/7YZCWPA4=
|
||||
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
|
||||
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
github.com/miekg/dns v1.0.8 h1:Zi8HNpze3NeRWH1PQV6O71YcvJRQ6j0lORO6DAEmAAI=
|
||||
github.com/miekg/dns v1.0.8/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
|
||||
github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA=
|
||||
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
|
||||
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
|
||||
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936 h1:kw1v0NlnN+GZcU8Ma8CLF2Zzgjfx95gs3/GN3vYAPpo=
|
||||
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk=
|
||||
github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
|
||||
github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
|
||||
github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/mitchellh/mapstructure v0.0.0-20180511142126-bb74f1db0675/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
|
||||
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/nikooo777/ytdl v0.0.0-20190215151411-9c7832eaf457 h1:S6VzSeZkOv8pKevvKF1isVp9HF294LxVsYw5JrV0FdQ=
|
||||
github.com/nikooo777/ytdl v0.0.0-20190215151411-9c7832eaf457/go.mod h1:Z/EmaCtMz3DxZsLaTnbHjqjZrOH4MHQBvvxtMObRGAY=
|
||||
github.com/nlopes/slack v0.2.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM=
|
||||
github.com/nlopes/slack v0.4.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM=
|
||||
github.com/nlopes/slack v0.5.0 h1:NbIae8Kd0NpqaEI3iUrsuS0KbcEDhzhc939jLW5fNm0=
|
||||
github.com/nlopes/slack v0.5.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
|
||||
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
|
||||
github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
|
||||
github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
|
||||
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
|
||||
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
|
||||
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
||||
github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
|
||||
github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
||||
github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
|
||||
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc=
|
||||
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
|
||||
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
|
||||
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
|
||||
github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
|
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
||||
github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
||||
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
|
||||
github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
|
||||
github.com/prometheus/common v0.3.0 h1:taZ4h8Tkxv2kNyoSctBvfXEHmBmxrwmIidZTIaHons4=
|
||||
github.com/prometheus/common v0.3.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
|
||||
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
||||
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
|
||||
github.com/sebdah/goldie v0.0.0-20180424091453-8784dd1ab561 h1:IY+sDBJR/wRtsxq+626xJnt4Tw7/ROA9cDIR8MMhWyg=
|
||||
github.com/sebdah/goldie v0.0.0-20180424091453-8784dd1ab561/go.mod h1:lvjGftC8oe7XPtyrOidaMi0rp5B9+XY/ZRUynGnuaxQ=
|
||||
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/shopspring/decimal v0.0.0-20180607144847-19e3cb6c2930/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
|
||||
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 h1:pntxY8Ary0t43dCZ5dqY4YTJCObLY1kIXl0uzMv+7DE=
|
||||
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
|
||||
github.com/sirupsen/logrus v0.0.0-20180523074243-ea8897e79973/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k=
|
||||
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
@@ -192,9 +284,14 @@ github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3/go.mod h1
|
||||
github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
|
||||
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs=
|
||||
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||
github.com/spf13/cast v1.2.0 h1:HHl1DSRbEQN2i8tJmtS6ViPyHx35+p51amrdsiTCrkg=
|
||||
github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
|
||||
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
|
||||
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
|
||||
github.com/spf13/cobra v0.0.0-20180722215644-7c4570c3ebeb/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
|
||||
github.com/spf13/cobra v0.0.0-20190109003409-7547e83b2d85 h1:UQHWkFUuJBy5rWN1DxosG/efssLu7u0fXXSTC2HHKfQ=
|
||||
github.com/spf13/cobra v0.0.0-20190109003409-7547e83b2d85/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
|
||||
github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
|
||||
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
@@ -202,6 +299,8 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
|
||||
github.com/uber-go/atomic v1.3.2 h1:Azu9lPBWRNKzYXSIwRfgRuDuS0YKsK4NFhiQv98gkxo=
|
||||
github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
|
||||
github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE=
|
||||
github.com/ybbus/jsonrpc v2.1.2+incompatible h1:V4mkE9qhbDQ92/MLMIhlhMSbz8jNXdagC3xBR5NDwaQ=
|
||||
@@ -209,53 +308,71 @@ github.com/ybbus/jsonrpc v2.1.2+incompatible/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G
|
||||
go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
|
||||
go.opencensus.io v0.20.2 h1:NAfh7zF0/3/HqtMvJNZ/RFrSlCE6ZTlHmKfhL/Dm1Jk=
|
||||
go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
|
||||
go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4=
|
||||
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480 h1:O5YqonU5IWby+w98jVUG9h7zlCWCcH4RHyPVReBmhzk=
|
||||
golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20181217174547-8f45f776aaf1/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd h1:HuTn7WObtcDo9uEEU7rEqL0jYthdXAmZ6PP+meazmaU=
|
||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2 h1:iC0Y6EDq+rhnAePxGvJs2kzUAYcwESqdcGRPzEUfzTU=
|
||||
golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c h1:uOCk1iQW6Vc18bnC13MfzScl+wdKBmM9Y9kU7Z83/lw=
|
||||
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a h1:tImsplftrFpALCYumobsd0K86vlAs/eXGFms2txfJfA=
|
||||
golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI=
|
||||
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190418153312-f0ce4c0180be h1:mI+jhqkn68ybP0ORJqunXn+fq+Eeb4hHKqLQcFICjAc=
|
||||
golang.org/x/sys v0.0.0-20190418153312-f0ce4c0180be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190520201301-c432e742b0af h1:NXfmMfXz6JqGfG3ikSxcz2N93j6DgScr19Oo2uwFu88=
|
||||
golang.org/x/sys v0.0.0-20190520201301-c432e742b0af/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190109165630-d30e00c24034/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
@@ -271,10 +388,12 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190108161440-ae2f86662275/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
|
||||
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7 h1:ZUjXAXmrAyrmmCPHgCA/vChHcpsX27MZ3yBonD/z1KE=
|
||||
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
|
||||
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.0 h1:DlsSIrgEBuZAUFJcta2B5i/lzeHHbnfkNFAfFXLVFYQ=
|
||||
@@ -298,6 +417,9 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
|
||||
gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
|
||||
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190109154334-5bcec433c8ea/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
@@ -0,0 +1,122 @@
|
||||
package ipManager
|
||||
|
||||
import (
|
||||
"github.com/asaskevich/govalidator"
|
||||
"github.com/lbryio/ytsync/util"
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
"github.com/lbryio/lbry.go/extras/stop"
|
||||
|
||||
"net"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const IPCooldownPeriod = 25 * time.Second
|
||||
const unbanTimeout = 3 * time.Hour
|
||||
|
||||
var ipv6Pool []string
|
||||
var ipv4Pool []string
|
||||
var throttledIPs map[string]bool
|
||||
var ipInUse map[string]bool
|
||||
var ipLastUsed map[string]time.Time
|
||||
var ipMutex sync.Mutex
|
||||
var stopper = stop.New()
|
||||
|
||||
func GetNextIP(ipv6 bool) (string, error) {
|
||||
ipMutex.Lock()
|
||||
defer ipMutex.Unlock()
|
||||
if len(ipv4Pool) < 1 || len(ipv6Pool) < 1 {
|
||||
throttledIPs = make(map[string]bool)
|
||||
ipInUse = make(map[string]bool)
|
||||
ipLastUsed = make(map[string]time.Time)
|
||||
addrs, err := net.InterfaceAddrs()
|
||||
if err != nil {
|
||||
return "", errors.Err(err)
|
||||
}
|
||||
|
||||
for _, address := range addrs {
|
||||
if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
|
||||
if ipnet.IP.To16() != nil && govalidator.IsIPv6(ipnet.IP.String()) {
|
||||
ipv6Pool = append(ipv6Pool, ipnet.IP.String())
|
||||
ipLastUsed[ipnet.IP.String()] = time.Now().Add(-IPCooldownPeriod)
|
||||
} else if ipnet.IP.To4() != nil && govalidator.IsIPv4(ipnet.IP.String()) {
|
||||
ipv4Pool = append(ipv4Pool, ipnet.IP.String())
|
||||
ipLastUsed[ipnet.IP.String()] = time.Now().Add(-IPCooldownPeriod)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
nextIP := ""
|
||||
if ipv6 {
|
||||
nextIP = getLeastUsedIP(ipv6Pool)
|
||||
} else {
|
||||
nextIP = getLeastUsedIP(ipv4Pool)
|
||||
}
|
||||
if nextIP == "" {
|
||||
return "throttled", errors.Err("all IPs are throttled")
|
||||
}
|
||||
lastUse := ipLastUsed[nextIP]
|
||||
if time.Since(lastUse) < IPCooldownPeriod {
|
||||
log.Debugf("The IP %s is too hot, waiting for %.1f seconds before continuing", nextIP, (IPCooldownPeriod - time.Since(lastUse)).Seconds())
|
||||
time.Sleep(IPCooldownPeriod - time.Since(lastUse))
|
||||
}
|
||||
|
||||
ipInUse[nextIP] = true
|
||||
return nextIP, nil
|
||||
}
|
||||
|
||||
func ReleaseIP(ip string) {
|
||||
ipMutex.Lock()
|
||||
defer ipMutex.Unlock()
|
||||
ipLastUsed[ip] = time.Now()
|
||||
ipInUse[ip] = false
|
||||
}
|
||||
|
||||
func getLeastUsedIP(ipPool []string) string {
|
||||
nextIP := ""
|
||||
veryLastUse := time.Now()
|
||||
for _, ip := range ipPool {
|
||||
isThrottled := throttledIPs[ip]
|
||||
if isThrottled {
|
||||
continue
|
||||
}
|
||||
inUse := ipInUse[ip]
|
||||
if inUse {
|
||||
continue
|
||||
}
|
||||
lastUse := ipLastUsed[ip]
|
||||
if lastUse.Before(veryLastUse) {
|
||||
nextIP = ip
|
||||
veryLastUse = lastUse
|
||||
}
|
||||
}
|
||||
return nextIP
|
||||
}
|
||||
|
||||
func SetIpThrottled(ip string, stopGrp *stop.Group) {
|
||||
ipMutex.Lock()
|
||||
isThrottled := throttledIPs[ip]
|
||||
if isThrottled {
|
||||
return
|
||||
}
|
||||
throttledIPs[ip] = true
|
||||
ipMutex.Unlock()
|
||||
util.SendErrorToSlack("%s set to throttled", ip)
|
||||
|
||||
stopper.Add(1)
|
||||
go func() {
|
||||
defer stopper.Done()
|
||||
unbanTimer := time.NewTimer(unbanTimeout)
|
||||
select {
|
||||
case <-unbanTimer.C:
|
||||
ipMutex.Lock()
|
||||
throttledIPs[ip] = false
|
||||
ipMutex.Unlock()
|
||||
util.SendInfoToSlack("%s set back to not throttled", ip)
|
||||
case <-stopGrp.Ch():
|
||||
unbanTimer.Stop()
|
||||
}
|
||||
}()
|
||||
}
|
||||
@@ -9,10 +9,11 @@ import (
|
||||
|
||||
"github.com/lbryio/lbry.go/extras/util"
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"github.com/lbryio/ytsync/manager"
|
||||
log "github.com/sirupsen/logrus"
|
||||
logUtils "github.com/lbryio/ytsync/util"
|
||||
)
|
||||
|
||||
var Version string
|
||||
@@ -36,6 +37,8 @@ var (
|
||||
videosLimit int
|
||||
maxVideoSize int
|
||||
maxVideoLength float64
|
||||
removeDBUnpublished bool
|
||||
upgradeMetadata bool
|
||||
)
|
||||
|
||||
func main() {
|
||||
@@ -56,6 +59,8 @@ func main() {
|
||||
cmd.Flags().BoolVar(&skipSpaceCheck, "skip-space-check", false, "Do not perform free space check on startup")
|
||||
cmd.Flags().BoolVar(&syncUpdate, "update", false, "Update previously synced channels instead of syncing new ones")
|
||||
cmd.Flags().BoolVar(&singleRun, "run-once", false, "Whether the process should be stopped after one cycle or not")
|
||||
cmd.Flags().BoolVar(&removeDBUnpublished, "remove-db-unpublished", false, "Remove videos from the database that are marked as published but aren't really published")
|
||||
cmd.Flags().BoolVar(&upgradeMetadata, "upgrade-metadata", false, "Upgrade videos if they're on the old metadata version")
|
||||
cmd.Flags().StringVar(&syncStatus, "status", "", "Specify which queue to pull from. Overrides --update")
|
||||
cmd.Flags().StringVar(&channelID, "channelID", "", "If specified, only this channel will be synced.")
|
||||
cmd.Flags().Int64Var(&syncFrom, "after", time.Unix(0, 0).Unix(), "Specify from when to pull jobs [Unix time](Default: 0)")
|
||||
@@ -188,10 +193,12 @@ func ytSync(cmd *cobra.Command, args []string) {
|
||||
syncProperties,
|
||||
apiConfig,
|
||||
maxVideoLength,
|
||||
removeDBUnpublished,
|
||||
upgradeMetadata,
|
||||
)
|
||||
err := sm.Start()
|
||||
if err != nil {
|
||||
manager.SendErrorToSlack(err.Error())
|
||||
logUtils.SendErrorToSlack(err.Error())
|
||||
}
|
||||
manager.SendInfoToSlack("Syncing process terminated!")
|
||||
logUtils.SendInfoToSlack("Syncing process terminated!")
|
||||
}
|
||||
+45
-18
@@ -6,11 +6,16 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/ytsync/blobs_reflector"
|
||||
"github.com/lbryio/ytsync/namer"
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
logUtils "github.com/lbryio/ytsync/util"
|
||||
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
"github.com/lbryio/lbry.go/extras/util"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
@@ -37,12 +42,14 @@ type SyncManager struct {
|
||||
singleRun bool
|
||||
syncProperties *sdk.SyncProperties
|
||||
apiConfig *sdk.APIConfig
|
||||
removeDBUnpublished bool
|
||||
upgradeMetadata bool
|
||||
}
|
||||
|
||||
func NewSyncManager(stopOnError bool, maxTries int, takeOverExistingChannel bool, refill int, limit int,
|
||||
skipSpaceCheck bool, syncUpdate bool, concurrentJobs int, concurrentVideos int, blobsDir string, videosLimit int,
|
||||
maxVideoSize int, lbrycrdString string, awsS3ID string, awsS3Secret string, awsS3Region string, awsS3Bucket string,
|
||||
syncStatus string, singleRun bool, syncProperties *sdk.SyncProperties, apiConfig *sdk.APIConfig, maxVideoLength float64) *SyncManager {
|
||||
syncStatus string, singleRun bool, syncProperties *sdk.SyncProperties, apiConfig *sdk.APIConfig, maxVideoLength float64, removeDBUnpublished bool, upgradeMetadata bool) *SyncManager {
|
||||
return &SyncManager{
|
||||
stopOnError: stopOnError,
|
||||
maxTries: maxTries,
|
||||
@@ -66,25 +73,30 @@ func NewSyncManager(stopOnError bool, maxTries int, takeOverExistingChannel bool
|
||||
singleRun: singleRun,
|
||||
syncProperties: syncProperties,
|
||||
apiConfig: apiConfig,
|
||||
removeDBUnpublished: removeDBUnpublished,
|
||||
upgradeMetadata: upgradeMetadata,
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
StatusPending = "pending" // waiting for permission to sync
|
||||
StatusPendingEmail = "pendingemail" // permission granted but missing email
|
||||
StatusQueued = "queued" // in sync queue. will be synced soon
|
||||
StatusSyncing = "syncing" // syncing now
|
||||
StatusSynced = "synced" // done
|
||||
StatusFailed = "failed"
|
||||
StatusFinalized = "finalized" // no more changes allowed
|
||||
StatusAbandoned = "abandoned" // deleted on youtube or banned
|
||||
StatusPending = "pending" // waiting for permission to sync
|
||||
StatusPendingEmail = "pendingemail" // permission granted but missing email
|
||||
StatusQueued = "queued" // in sync queue. will be synced soon
|
||||
StatusPendingUpgrade = "pendingupgrade" // in sync queue. will be synced soon
|
||||
StatusSyncing = "syncing" // syncing now
|
||||
StatusSynced = "synced" // done
|
||||
StatusFailed = "failed"
|
||||
StatusFinalized = "finalized" // no more changes allowed
|
||||
StatusAbandoned = "abandoned" // deleted on youtube or banned
|
||||
)
|
||||
|
||||
var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned}
|
||||
var SyncStatuses = []string{StatusPending, StatusPendingEmail, StatusPendingUpgrade, StatusQueued, StatusSyncing, StatusSynced, StatusFailed, StatusFinalized, StatusAbandoned}
|
||||
|
||||
const (
|
||||
VideoStatusPublished = "published"
|
||||
VideoStatusFailed = "failed"
|
||||
VideoStatusPublished = "published"
|
||||
VideoStatusFailed = "failed"
|
||||
VideoStatusUpgradeFailed = "upgradefailed"
|
||||
VideoStatusUnpublished = "unpublished"
|
||||
)
|
||||
|
||||
func (s *SyncManager) Start() error {
|
||||
@@ -127,10 +139,12 @@ func (s *SyncManager) Start() error {
|
||||
AwsS3Region: s.awsS3Region,
|
||||
AwsS3Bucket: s.awsS3Bucket,
|
||||
namer: namer.NewNamer(),
|
||||
Fee: channels[0].Fee,
|
||||
}
|
||||
shouldInterruptLoop = true
|
||||
} else {
|
||||
var queuesToSync []string
|
||||
//TODO: implement scrambling to avoid starvation of queues
|
||||
if s.syncStatus != "" {
|
||||
queuesToSync = append(queuesToSync, s.syncStatus)
|
||||
} else if s.syncUpdate {
|
||||
@@ -143,7 +157,9 @@ func (s *SyncManager) Start() error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, c := range channels {
|
||||
log.Infof("There are %d channels in the \"%s\" queue", len(channels), q)
|
||||
if len(channels) > 0 {
|
||||
c := channels[0]
|
||||
syncs = append(syncs, Sync{
|
||||
APIConfig: s.apiConfig,
|
||||
YoutubeChannelID: c.ChannelId,
|
||||
@@ -161,7 +177,9 @@ func (s *SyncManager) Start() error {
|
||||
AwsS3Region: s.awsS3Region,
|
||||
AwsS3Bucket: s.awsS3Bucket,
|
||||
namer: namer.NewNamer(),
|
||||
Fee: c.Fee,
|
||||
})
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -169,9 +187,9 @@ func (s *SyncManager) Start() error {
|
||||
log.Infoln("No channels to sync. Pausing 5 minutes!")
|
||||
time.Sleep(5 * time.Minute)
|
||||
}
|
||||
for i, sync := range syncs {
|
||||
for _, sync := range syncs {
|
||||
shouldNotCount := false
|
||||
SendInfoToSlack("Syncing %s (%s) to LBRY! (iteration %d/%d - total processed channels: %d)", sync.LbryChannelName, sync.YoutubeChannelID, i+1, len(syncs), syncCount+1)
|
||||
logUtils.SendInfoToSlack("Syncing %s (%s) to LBRY! total processed channels since startup: %d", sync.LbryChannelName, sync.YoutubeChannelID, syncCount+1)
|
||||
err := sync.FullCycle()
|
||||
if err != nil {
|
||||
fatalErrors := []string{
|
||||
@@ -188,10 +206,14 @@ func (s *SyncManager) Start() error {
|
||||
}
|
||||
shouldNotCount = strings.Contains(err.Error(), "this youtube channel is being managed by another server")
|
||||
if !shouldNotCount {
|
||||
SendInfoToSlack("A non fatal error was reported by the sync process. %s\nContinuing...", err.Error())
|
||||
logUtils.SendInfoToSlack("A non fatal error was reported by the sync process. %s\nContinuing...", err.Error())
|
||||
}
|
||||
}
|
||||
SendInfoToSlack("Syncing %s (%s) reached an end. (iteration %d/%d - total processed channels: %d)", sync.LbryChannelName, sync.YoutubeChannelID, i+1, len(syncs), syncCount+1)
|
||||
err = blobs_reflector.ReflectAndClean()
|
||||
if err != nil {
|
||||
return errors.Prefix("@Nikooo777 something went wrong while reflecting blobs", err)
|
||||
}
|
||||
logUtils.SendInfoToSlack("Syncing %s (%s) reached an end. total processed channels since startup: %d", sync.LbryChannelName, sync.YoutubeChannelID, syncCount+1)
|
||||
if !shouldNotCount {
|
||||
syncCount++
|
||||
}
|
||||
@@ -206,7 +228,12 @@ func (s *SyncManager) Start() error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *SyncManager) GetS3AWSConfig() aws.Config {
|
||||
return aws.Config{
|
||||
Credentials: credentials.NewStaticCredentials(s.awsS3ID, s.awsS3Secret, ""),
|
||||
Region: &s.awsS3Region,
|
||||
}
|
||||
}
|
||||
func (s *SyncManager) checkUsedSpace() error {
|
||||
usedPctile, err := GetUsedSpace(s.blobsDir)
|
||||
if err != nil {
|
||||
|
||||
+130
-76
@@ -2,18 +2,20 @@ package manager
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
"github.com/lbryio/lbry.go/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/extras/util"
|
||||
"github.com/lbryio/lbry.go/lbrycrd"
|
||||
|
||||
"github.com/lbryio/ytsync/tagsManager"
|
||||
"github.com/lbryio/ytsync/thumbs"
|
||||
logUtils "github.com/lbryio/ytsync/util"
|
||||
|
||||
"github.com/shopspring/decimal"
|
||||
log "github.com/sirupsen/logrus"
|
||||
@@ -21,8 +23,28 @@ import (
|
||||
"google.golang.org/api/youtube/v3"
|
||||
)
|
||||
|
||||
func (s *Sync) enableAddressReuse() error {
|
||||
accountsResponse, err := s.daemon.AccountList()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
accounts := accountsResponse.LBCMainnet
|
||||
if os.Getenv("REGTEST") == "true" {
|
||||
accounts = accountsResponse.LBCRegtest
|
||||
}
|
||||
for _, a := range accounts {
|
||||
_, err = s.daemon.AccountSet(a.ID, jsonrpc.AccountSettings{
|
||||
ChangeMaxUses: util.PtrToInt(1000),
|
||||
ReceivingMaxUses: util.PtrToInt(100),
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (s *Sync) walletSetup() error {
|
||||
//prevent unnecessary concurrent execution
|
||||
//prevent unnecessary concurrent execution and publishing while refilling/reallocating UTXOs
|
||||
s.walletMux.Lock()
|
||||
defer s.walletMux.Unlock()
|
||||
err := s.ensureChannelOwnership()
|
||||
@@ -42,51 +64,55 @@ func (s *Sync) walletSetup() error {
|
||||
}
|
||||
log.Debugf("Starting balance is %.4f", balance)
|
||||
|
||||
var numOnSource int
|
||||
if s.LbryChannelName == "@UCBerkeley" {
|
||||
numOnSource = 10104
|
||||
} else {
|
||||
n, err := s.CountVideos()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
numOnSource = int(n)
|
||||
n, err := s.CountVideos()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
videosOnYoutube := int(n)
|
||||
|
||||
log.Debugf("Source channel has %d videos", numOnSource)
|
||||
if numOnSource == 0 {
|
||||
log.Debugf("Source channel has %d videos", videosOnYoutube)
|
||||
if videosOnYoutube == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
s.syncedVideosMux.RLock()
|
||||
numPublished := len(s.syncedVideos) //should we only count published videos? Credits are allocated even for failed ones...
|
||||
publishedCount := 0
|
||||
notUpgradedCount := 0
|
||||
failedCount := 0
|
||||
for _, sv := range s.syncedVideos {
|
||||
if sv.Published {
|
||||
publishedCount++
|
||||
if sv.MetadataVersion < 2 {
|
||||
notUpgradedCount++
|
||||
}
|
||||
} else {
|
||||
failedCount++
|
||||
}
|
||||
}
|
||||
s.syncedVideosMux.RUnlock()
|
||||
log.Debugf("We already allocated credits for %d videos", numPublished)
|
||||
|
||||
if numOnSource-numPublished > s.Manager.videosLimit {
|
||||
numOnSource = s.Manager.videosLimit
|
||||
log.Debugf("We already allocated credits for %d published videos and %d failed videos", publishedCount, failedCount)
|
||||
|
||||
if videosOnYoutube > s.Manager.videosLimit {
|
||||
videosOnYoutube = s.Manager.videosLimit
|
||||
}
|
||||
unallocatedVideos := videosOnYoutube - (publishedCount + failedCount)
|
||||
requiredBalance := float64(unallocatedVideos)*(publishAmount+estimatedMaxTxFee) + channelClaimAmount
|
||||
if s.Manager.upgradeMetadata {
|
||||
requiredBalance += float64(notUpgradedCount) * 0.001
|
||||
}
|
||||
|
||||
minBalance := (float64(numOnSource)-float64(numPublished))*(publishAmount+0.1) + channelClaimAmount
|
||||
if numPublished > numOnSource && balance < 1 {
|
||||
SendErrorToSlack("something is going on as we published more videos than those available on source: %d/%d", numPublished, numOnSource)
|
||||
minBalance = 1 //since we ended up in this function it means some juice is still needed
|
||||
refillAmount := 0.0
|
||||
if balance < requiredBalance || balance < minimumAccountBalance {
|
||||
refillAmount = math.Max(requiredBalance-balance, minimumRefillAmount)
|
||||
}
|
||||
amountToAdd := minBalance - balance
|
||||
|
||||
if s.Refill > 0 {
|
||||
if amountToAdd < 0 {
|
||||
amountToAdd = float64(s.Refill)
|
||||
} else {
|
||||
amountToAdd += float64(s.Refill)
|
||||
}
|
||||
refillAmount += float64(s.Refill)
|
||||
}
|
||||
|
||||
if amountToAdd > 0 {
|
||||
if amountToAdd < 1 {
|
||||
amountToAdd = 1 // no reason to bother adding less than 1 credit
|
||||
}
|
||||
err := s.addCredits(amountToAdd)
|
||||
if refillAmount > 0 {
|
||||
err := s.addCredits(refillAmount)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
@@ -98,7 +124,7 @@ func (s *Sync) walletSetup() error {
|
||||
} else if claimAddress == nil {
|
||||
return errors.Err("could not get unused address")
|
||||
}
|
||||
s.claimAddress = string((*claimAddress)[0])
|
||||
s.claimAddress = string((*claimAddress)[0]) //TODO: remove claimAddress completely
|
||||
if s.claimAddress == "" {
|
||||
return errors.Err("found blank claim address")
|
||||
}
|
||||
@@ -122,7 +148,7 @@ func (s *Sync) ensureEnoughUTXOs() error {
|
||||
}
|
||||
defaultAccount := ""
|
||||
for _, account := range accountsNet {
|
||||
if account.IsDefaultAccount {
|
||||
if account.IsDefault {
|
||||
defaultAccount = account.ID
|
||||
break
|
||||
}
|
||||
@@ -141,14 +167,19 @@ func (s *Sync) ensureEnoughUTXOs() error {
|
||||
target := 40
|
||||
slack := int(float32(0.1) * float32(target))
|
||||
count := 0
|
||||
confirmedCount := 0
|
||||
|
||||
for _, utxo := range *utxolist {
|
||||
amount, _ := strconv.ParseFloat(utxo.Amount, 64)
|
||||
if !utxo.IsClaim && !utxo.IsSupport && !utxo.IsUpdate && amount != 0.0 {
|
||||
if utxo.IsMine && utxo.Type == "payment" && amount > 0.001 {
|
||||
if utxo.Confirmations > 0 {
|
||||
confirmedCount++
|
||||
}
|
||||
count++
|
||||
}
|
||||
}
|
||||
log.Infof("utxo count: %d", count)
|
||||
log.Infof("utxo count: %d (%d confirmed)", count, confirmedCount)
|
||||
UTXOWaitThreshold := 16
|
||||
if count < target-slack {
|
||||
balance, err := s.daemon.AccountBalance(&defaultAccount)
|
||||
if err != nil {
|
||||
@@ -161,23 +192,27 @@ func (s *Sync) ensureEnoughUTXOs() error {
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
broadcastFee := 0.01
|
||||
amountToSplit := fmt.Sprintf("%.6f", balanceAmount-broadcastFee)
|
||||
maxUTXOs := uint64(500)
|
||||
desiredUTXOCount := uint64(math.Floor((balanceAmount) / 0.1))
|
||||
if desiredUTXOCount > maxUTXOs {
|
||||
desiredUTXOCount = maxUTXOs
|
||||
}
|
||||
log.Infof("Splitting balance of %s evenly between %d UTXOs", *balance, desiredUTXOCount)
|
||||
|
||||
log.Infof("Splitting balance of %s evenly between 40 UTXOs", *balance)
|
||||
|
||||
prefillTx, err := s.daemon.AccountFund(defaultAccount, defaultAccount, amountToSplit, uint64(target))
|
||||
broadcastFee := 0.1
|
||||
prefillTx, err := s.daemon.AccountFund(defaultAccount, defaultAccount, fmt.Sprintf("%.4f", balanceAmount-broadcastFee), desiredUTXOCount, false)
|
||||
if err != nil {
|
||||
return err
|
||||
} else if prefillTx == nil {
|
||||
return errors.Err("no response")
|
||||
}
|
||||
|
||||
err = s.waitForNewBlock()
|
||||
if err != nil {
|
||||
return err
|
||||
if confirmedCount < UTXOWaitThreshold {
|
||||
err = s.waitForNewBlock()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
} else if !allUTXOsConfirmed(utxolist) {
|
||||
} else if confirmedCount < UTXOWaitThreshold {
|
||||
log.Println("Waiting for previous txns to confirm")
|
||||
err := s.waitForNewBlock()
|
||||
if err != nil {
|
||||
@@ -220,7 +255,7 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
return errors.Err("no channel name set")
|
||||
}
|
||||
//@TODO: get rid of this when imported channels are supported
|
||||
if s.YoutubeChannelID == "UCkK9UDm_ZNrq_rIXCz3xCGA" || s.YoutubeChannelID == "UCW-thz5HxE-goYq8yPds1Gw" {
|
||||
if s.YoutubeChannelID == "UCW-thz5HxE-goYq8yPds1Gw" {
|
||||
return nil
|
||||
}
|
||||
channels, err := s.daemon.ChannelList(nil, 1, 50)
|
||||
@@ -233,7 +268,7 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
if len((*channels).Items) > 1 {
|
||||
// This wallet is probably not under our control anymore but we still want to publish to it
|
||||
// here we shall check if within all the channels there is one that was created by ytsync
|
||||
SendInfoToSlack("we are dealing with a wallet that has multiple channels. This indicates that the wallet was probably transferred but we still want to sync their content. YoutubeID: %s", s.YoutubeChannelID)
|
||||
logUtils.SendInfoToSlack("we are dealing with a wallet that has multiple channels. This indicates that the wallet was probably transferred but we still want to sync their content. YoutubeID: %s", s.YoutubeChannelID)
|
||||
if s.lbryChannelID == "" {
|
||||
return errors.Err("this channel does not have a recorded claimID in the database. To prevent failures, updates are not supported until an entry is manually added in the database")
|
||||
}
|
||||
@@ -246,9 +281,11 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
}
|
||||
}
|
||||
}
|
||||
channelUsesOldMetadata := false
|
||||
if len((*channels).Items) == 1 {
|
||||
channel := ((*channels).Items)[0]
|
||||
if channel.Name == s.LbryChannelName {
|
||||
channelUsesOldMetadata = channel.Value.GetThumbnail() == nil
|
||||
//TODO: eventually get rid of this when the whole db is filled
|
||||
if s.lbryChannelID == "" {
|
||||
err = s.Manager.apiConfig.SetChannelClaimID(s.YoutubeChannelID, channel.ClaimID)
|
||||
@@ -256,7 +293,9 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
return errors.Err("the channel in the wallet is different than the channel in the database")
|
||||
}
|
||||
s.lbryChannelID = channel.ClaimID
|
||||
return err
|
||||
if !channelUsesOldMetadata {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
return errors.Err("this channel does not belong to this wallet! Expected: %s, found: %s", s.LbryChannelName, channel.Name)
|
||||
}
|
||||
@@ -290,7 +329,7 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
return errors.Prefix("error creating YouTube service", err)
|
||||
}
|
||||
|
||||
response, err := service.Channels.List("snippet").Id(s.YoutubeChannelID).Do()
|
||||
response, err := service.Channels.List("snippet,brandingSettings").Id(s.YoutubeChannelID).Do()
|
||||
if err != nil {
|
||||
return errors.Prefix("error getting channel details", err)
|
||||
}
|
||||
@@ -300,24 +339,23 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
}
|
||||
|
||||
channelInfo := response.Items[0].Snippet
|
||||
channelBranding := response.Items[0].BrandingSettings
|
||||
|
||||
thumbnail := channelInfo.Thumbnails.Default
|
||||
if channelInfo.Thumbnails.Maxres != nil {
|
||||
thumbnail = channelInfo.Thumbnails.Maxres
|
||||
} else if channelInfo.Thumbnails.High != nil {
|
||||
thumbnail = channelInfo.Thumbnails.High
|
||||
} else if channelInfo.Thumbnails.Medium != nil {
|
||||
thumbnail = channelInfo.Thumbnails.Medium
|
||||
} else if channelInfo.Thumbnails.Standard != nil {
|
||||
thumbnail = channelInfo.Thumbnails.Standard
|
||||
}
|
||||
thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail.Url, s.YoutubeChannelID, aws.Config{
|
||||
Credentials: credentials.NewStaticCredentials(s.AwsS3ID, s.AwsS3Secret, ""),
|
||||
Region: &s.AwsS3Region,
|
||||
})
|
||||
thumbnail := thumbs.GetBestThumbnail(channelInfo.Thumbnails)
|
||||
thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail.Url, s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var bannerURL *string
|
||||
if channelBranding.Image != nil && channelBranding.Image.BannerImageUrl != "" {
|
||||
bURL, err := thumbs.MirrorThumbnail(channelBranding.Image.BannerImageUrl, "banner-"+s.YoutubeChannelID, s.Manager.GetS3AWSConfig())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bannerURL = &bURL
|
||||
}
|
||||
|
||||
var languages []string = nil
|
||||
if channelInfo.DefaultLanguage != "" {
|
||||
languages = []string{channelInfo.DefaultLanguage}
|
||||
@@ -326,16 +364,32 @@ func (s *Sync) ensureChannelOwnership() error {
|
||||
if channelInfo.Country != "" {
|
||||
locations = []jsonrpc.Location{{Country: util.PtrToString(channelInfo.Country)}}
|
||||
}
|
||||
c, err := s.daemon.ChannelCreate(s.LbryChannelName, channelBidAmount, jsonrpc.ChannelCreateOptions{
|
||||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
|
||||
Title: channelInfo.Title,
|
||||
Description: channelInfo.Description,
|
||||
Tags: nil,
|
||||
Languages: languages,
|
||||
Locations: locations,
|
||||
ThumbnailURL: &thumbnailURL,
|
||||
},
|
||||
})
|
||||
var c *jsonrpc.TransactionSummary
|
||||
claimCreateOptions := jsonrpc.ClaimCreateOptions{
|
||||
Title: &channelInfo.Title,
|
||||
Description: &channelInfo.Description,
|
||||
Tags: tagsManager.GetTagsForChannel(s.YoutubeChannelID),
|
||||
Languages: languages,
|
||||
Locations: locations,
|
||||
ThumbnailURL: &thumbnailURL,
|
||||
}
|
||||
if channelUsesOldMetadata {
|
||||
c, err = s.daemon.ChannelUpdate(s.lbryChannelID, jsonrpc.ChannelUpdateOptions{
|
||||
ClearTags: util.PtrToBool(true),
|
||||
ClearLocations: util.PtrToBool(true),
|
||||
ClearLanguages: util.PtrToBool(true),
|
||||
ChannelCreateOptions: jsonrpc.ChannelCreateOptions{
|
||||
ClaimCreateOptions: claimCreateOptions,
|
||||
CoverURL: bannerURL,
|
||||
},
|
||||
})
|
||||
} else {
|
||||
c, err = s.daemon.ChannelCreate(s.LbryChannelName, channelBidAmount, jsonrpc.ChannelCreateOptions{
|
||||
ClaimCreateOptions: claimCreateOptions,
|
||||
CoverURL: bannerURL,
|
||||
})
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -353,7 +407,7 @@ func allUTXOsConfirmed(utxolist *jsonrpc.UTXOListResponse) bool {
|
||||
}
|
||||
|
||||
for _, utxo := range *utxolist {
|
||||
if utxo.Height == 0 {
|
||||
if utxo.Confirmations <= 0 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
+202
-159
@@ -1,11 +1,7 @@
|
||||
package manager
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/csv"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
@@ -21,6 +17,8 @@ import (
|
||||
"github.com/lbryio/ytsync/namer"
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
"github.com/lbryio/ytsync/sources"
|
||||
"github.com/lbryio/ytsync/thumbs"
|
||||
logUtils "github.com/lbryio/ytsync/util"
|
||||
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
"github.com/lbryio/lbry.go/extras/jsonrpc"
|
||||
@@ -40,9 +38,12 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
channelClaimAmount = 0.01
|
||||
publishAmount = 0.01
|
||||
maxReasonLength = 500
|
||||
channelClaimAmount = 0.01
|
||||
estimatedMaxTxFee = 0.1
|
||||
minimumAccountBalance = 4.0
|
||||
minimumRefillAmount = 1
|
||||
publishAmount = 0.01
|
||||
maxReasonLength = 500
|
||||
)
|
||||
|
||||
type video interface {
|
||||
@@ -51,7 +52,7 @@ type video interface {
|
||||
IDAndNum() string
|
||||
PlaylistPosition() int
|
||||
PublishedAt() time.Time
|
||||
Sync(*jsonrpc.Client, string, float64, string, int, *namer.Namer, float64) (*sources.SyncSummary, error)
|
||||
Sync(*jsonrpc.Client, sources.SyncParams, *sdk.SyncedVideo, bool, *sync.RWMutex) (*sources.SyncSummary, error)
|
||||
}
|
||||
|
||||
// sorting videos
|
||||
@@ -77,53 +78,30 @@ type Sync struct {
|
||||
AwsS3Secret string
|
||||
AwsS3Region string
|
||||
AwsS3Bucket string
|
||||
|
||||
daemon *jsonrpc.Client
|
||||
claimAddress string
|
||||
videoDirectory string
|
||||
syncedVideosMux *sync.RWMutex
|
||||
syncedVideos map[string]sdk.SyncedVideo
|
||||
grp *stop.Group
|
||||
lbryChannelID string
|
||||
namer *namer.Namer
|
||||
|
||||
walletMux *sync.Mutex
|
||||
queue chan video
|
||||
Fee *sdk.Fee
|
||||
daemon *jsonrpc.Client
|
||||
claimAddress string
|
||||
videoDirectory string
|
||||
syncedVideosMux *sync.RWMutex
|
||||
syncedVideos map[string]sdk.SyncedVideo
|
||||
grp *stop.Group
|
||||
lbryChannelID string
|
||||
namer *namer.Namer
|
||||
walletMux *sync.RWMutex
|
||||
queue chan video
|
||||
}
|
||||
|
||||
func (s *Sync) AppendSyncedVideo(videoID string, published bool, failureReason string, claimName string) {
|
||||
func (s *Sync) AppendSyncedVideo(videoID string, published bool, failureReason string, claimName string, claimID string, metadataVersion int8, size int64) {
|
||||
s.syncedVideosMux.Lock()
|
||||
defer s.syncedVideosMux.Unlock()
|
||||
s.syncedVideos[videoID] = sdk.SyncedVideo{
|
||||
VideoID: videoID,
|
||||
Published: published,
|
||||
FailureReason: failureReason,
|
||||
}
|
||||
}
|
||||
|
||||
// SendErrorToSlack Sends an error message to the default channel and to the process log.
|
||||
func SendErrorToSlack(format string, a ...interface{}) {
|
||||
message := format
|
||||
if len(a) > 0 {
|
||||
message = fmt.Sprintf(format, a...)
|
||||
}
|
||||
log.Errorln(message)
|
||||
err := util.SendToSlack(":sos: " + message)
|
||||
if err != nil {
|
||||
log.Errorln(err)
|
||||
}
|
||||
}
|
||||
|
||||
// SendInfoToSlack Sends an info message to the default channel and to the process log.
|
||||
func SendInfoToSlack(format string, a ...interface{}) {
|
||||
message := format
|
||||
if len(a) > 0 {
|
||||
message = fmt.Sprintf(format, a...)
|
||||
}
|
||||
log.Infoln(message)
|
||||
err := util.SendToSlack(":information_source: " + message)
|
||||
if err != nil {
|
||||
log.Errorln(err)
|
||||
VideoID: videoID,
|
||||
Published: published,
|
||||
FailureReason: failureReason,
|
||||
ClaimID: claimID,
|
||||
ClaimName: claimName,
|
||||
MetadataVersion: metadataVersion,
|
||||
Size: size,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -258,7 +236,7 @@ func (s *Sync) FullCycle() (e error) {
|
||||
s.setExceptions()
|
||||
|
||||
s.syncedVideosMux = &sync.RWMutex{}
|
||||
s.walletMux = &sync.Mutex{}
|
||||
s.walletMux = &sync.RWMutex{}
|
||||
s.grp = stop.New()
|
||||
s.queue = make(chan video)
|
||||
interruptChan := make(chan os.Signal, 1)
|
||||
@@ -292,6 +270,7 @@ func (s *Sync) FullCycle() (e error) {
|
||||
return errors.Wrap(err, 0)
|
||||
}
|
||||
|
||||
defer deleteSyncFolder(s.videoDirectory)
|
||||
log.Printf("Starting daemon")
|
||||
err = startDaemonViaSystemd()
|
||||
if err != nil {
|
||||
@@ -307,18 +286,19 @@ func (s *Sync) FullCycle() (e error) {
|
||||
return err
|
||||
}
|
||||
|
||||
err = s.doSync()
|
||||
if err != nil {
|
||||
return err
|
||||
} else {
|
||||
// wait for reflection to finish???
|
||||
wait := 15 * time.Second // should bump this up to a few min, but keeping it low for testing
|
||||
log.Println("Waiting " + wait.String() + " to finish reflecting everything")
|
||||
time.Sleep(wait)
|
||||
}
|
||||
|
||||
return nil
|
||||
return s.doSync()
|
||||
}
|
||||
|
||||
func deleteSyncFolder(videoDirectory string) {
|
||||
if !strings.Contains(videoDirectory, "/tmp/ytsync") {
|
||||
_ = util.SendToSlack(errors.Err("Trying to delete an unexpected directory: %s", videoDirectory).Error())
|
||||
}
|
||||
err := os.RemoveAll(videoDirectory)
|
||||
if err != nil {
|
||||
_ = util.SendToSlack(err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Sync) setChannelTerminationStatus(e *error) {
|
||||
if *e != nil {
|
||||
//conditions for which a channel shouldn't be marked as failed
|
||||
@@ -384,26 +364,30 @@ func (s *Sync) stopAndUploadWallet(e *error) {
|
||||
}
|
||||
}
|
||||
func logShutdownError(shutdownErr error) {
|
||||
SendErrorToSlack("error shutting down daemon: %v", shutdownErr)
|
||||
SendErrorToSlack("WALLET HAS NOT BEEN MOVED TO THE WALLET BACKUP DIR")
|
||||
logUtils.SendErrorToSlack("error shutting down daemon: %v", shutdownErr)
|
||||
logUtils.SendErrorToSlack("WALLET HAS NOT BEEN MOVED TO THE WALLET BACKUP DIR")
|
||||
}
|
||||
|
||||
var thumbnailHosts = []string{
|
||||
"berk.ninja/thumbnails/",
|
||||
"https://thumbnails.lbry.com/",
|
||||
thumbs.ThumbnailEndpoint,
|
||||
}
|
||||
|
||||
func isYtsyncClaim(c jsonrpc.Claim) bool {
|
||||
|
||||
if !util.InSlice(c.Type, []string{"claim", "update"}) || c.Value.GetStream() == nil {
|
||||
return false
|
||||
}
|
||||
if c.Value.GetStream().GetThumbnailUrl() == "" {
|
||||
if c.Value.GetThumbnail() == nil || c.Value.GetThumbnail().GetUrl() == "" {
|
||||
//most likely a claim created outside of ytsync, ignore!
|
||||
return false
|
||||
}
|
||||
|
||||
return util.InSlice(c.Value.GetStream().GetThumbnailUrl(), thumbnailHosts)
|
||||
for _, th := range thumbnailHosts {
|
||||
if strings.Contains(c.Value.GetThumbnail().GetUrl(), th) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// fixDupes abandons duplicate claims
|
||||
@@ -414,7 +398,7 @@ func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) {
|
||||
if !isYtsyncClaim(c) {
|
||||
continue
|
||||
}
|
||||
tn := c.Value.GetStream().GetThumbnailUrl()
|
||||
tn := c.Value.GetThumbnail().GetUrl()
|
||||
videoID := tn[strings.LastIndex(tn, "/")+1:]
|
||||
|
||||
log.Infof("claimid: %s, claimName: %s, videoID: %s", c.ClaimID, c.Name, videoID)
|
||||
@@ -442,28 +426,74 @@ func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) {
|
||||
}
|
||||
|
||||
//updateRemoteDB counts the amount of videos published so far and updates the remote db if some videos weren't marked as published
|
||||
func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total int, fixed int, err error) {
|
||||
//additionally it removes all entries in the database indicating that a video is published when it's actually not
|
||||
func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total, fixed, removed int, err error) {
|
||||
count := 0
|
||||
videoIDMap := make(map[string]string, len(claims))
|
||||
for _, c := range claims {
|
||||
if !isYtsyncClaim(c) {
|
||||
continue
|
||||
}
|
||||
count++
|
||||
//check if claimID is in remote db
|
||||
tn := c.Value.GetStream().GetThumbnailUrl()
|
||||
tn := c.Value.GetThumbnail().GetUrl()
|
||||
videoID := tn[strings.LastIndex(tn, "/")+1:]
|
||||
pv, ok := s.syncedVideos[videoID]
|
||||
if !ok || pv.ClaimName != c.Name {
|
||||
fixed++
|
||||
log.Debugf("adding %s to the database", c.Name)
|
||||
videoIDMap[videoID] = c.ClaimID
|
||||
s.syncedVideosMux.RLock()
|
||||
pv, claimInDatabase := s.syncedVideos[videoID]
|
||||
s.syncedVideosMux.RUnlock()
|
||||
claimMetadataVersion := uint(1)
|
||||
if strings.Contains(tn, thumbs.ThumbnailEndpoint) {
|
||||
claimMetadataVersion = 2
|
||||
}
|
||||
|
||||
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, videoID, VideoStatusPublished, c.ClaimID, c.Name, "", nil)
|
||||
metadataDiffers := claimInDatabase && pv.MetadataVersion != int8(claimMetadataVersion)
|
||||
claimIDDiffers := claimInDatabase && pv.ClaimID != c.ClaimID
|
||||
claimNameDiffers := claimInDatabase && pv.ClaimName != c.Name
|
||||
claimMarkedUnpublished := claimInDatabase && !pv.Published
|
||||
if metadataDiffers {
|
||||
log.Debugf("%s: Mismatch in database for metadata. DB: %d - Blockchain: %d", videoID, pv.MetadataVersion, claimMetadataVersion)
|
||||
}
|
||||
if claimIDDiffers {
|
||||
log.Debugf("%s: Mismatch in database for claimID. DB: %s - Blockchain: %s", videoID, pv.ClaimID, c.ClaimID)
|
||||
}
|
||||
if claimNameDiffers {
|
||||
log.Debugf("%s: Mismatch in database for claimName. DB: %s - Blockchain: %s", videoID, pv.ClaimName, c.Name)
|
||||
}
|
||||
if claimMarkedUnpublished {
|
||||
log.Debugf("%s: Mismatch in database: published but marked as unpublished", videoID)
|
||||
}
|
||||
if !claimInDatabase {
|
||||
log.Debugf("%s: Published but is not in database (%s - %s)", videoID, c.Name, c.ClaimID)
|
||||
}
|
||||
if !claimInDatabase || metadataDiffers || claimIDDiffers || claimNameDiffers || claimMarkedUnpublished {
|
||||
claimSize, err := c.GetStreamSizeByMagic()
|
||||
if err != nil {
|
||||
return count, fixed, err
|
||||
claimSize = 0
|
||||
}
|
||||
fixed++
|
||||
log.Debugf("updating %s in the database", videoID)
|
||||
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, videoID, VideoStatusPublished, c.ClaimID, c.Name, "", util.PtrToInt64(int64(claimSize)), claimMetadataVersion)
|
||||
if err != nil {
|
||||
return count, fixed, 0, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return count, fixed, nil
|
||||
idsToRemove := make([]string, 0, len(videoIDMap))
|
||||
for vID, sv := range s.syncedVideos {
|
||||
_, ok := videoIDMap[vID]
|
||||
if !ok && sv.Published {
|
||||
log.Debugf("%s: claims to be published but wasn't found in the list of claims and will be removed if --remove-db-unpublished was specified", vID)
|
||||
idsToRemove = append(idsToRemove, vID)
|
||||
}
|
||||
}
|
||||
if s.Manager.removeDBUnpublished && len(idsToRemove) > 0 {
|
||||
err := s.Manager.apiConfig.DeleteVideos(idsToRemove)
|
||||
if err != nil {
|
||||
return count, fixed, 0, err
|
||||
}
|
||||
}
|
||||
return count, fixed, len(idsToRemove), nil
|
||||
}
|
||||
|
||||
func (s *Sync) getClaims() ([]jsonrpc.Claim, error) {
|
||||
@@ -481,7 +511,10 @@ func (s *Sync) getClaims() ([]jsonrpc.Claim, error) {
|
||||
}
|
||||
|
||||
func (s *Sync) doSync() error {
|
||||
var err error
|
||||
err := s.enableAddressReuse()
|
||||
if err != nil {
|
||||
return errors.Prefix("could not set address reuse policy", err)
|
||||
}
|
||||
err = s.walletSetup()
|
||||
if err != nil {
|
||||
return errors.Prefix("Initial wallet setup failed! Manual Intervention is required.", err)
|
||||
@@ -495,7 +528,7 @@ func (s *Sync) doSync() error {
|
||||
return errors.Prefix("error checking for duplicates", err)
|
||||
}
|
||||
if hasDupes {
|
||||
SendInfoToSlack("Channel had dupes and was fixed!")
|
||||
logUtils.SendInfoToSlack("Channel had dupes and was fixed!")
|
||||
err = s.waitForNewBlock()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -506,16 +539,21 @@ func (s *Sync) doSync() error {
|
||||
}
|
||||
}
|
||||
|
||||
pubsOnWallet, nFixed, err := s.updateRemoteDB(allClaims)
|
||||
pubsOnWallet, nFixed, nRemoved, err := s.updateRemoteDB(allClaims)
|
||||
if err != nil {
|
||||
return errors.Prefix("error counting claims", err)
|
||||
return errors.Prefix("error updating remote database", err)
|
||||
}
|
||||
if nFixed > 0 {
|
||||
if nFixed > 0 || nRemoved > 0 {
|
||||
err := s.setStatusSyncing()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
SendInfoToSlack("%d claims were not on the remote database and were fixed", nFixed)
|
||||
if nFixed > 0 {
|
||||
logUtils.SendInfoToSlack("%d claims had mismatched database info or were completely missing and were fixed", nFixed)
|
||||
}
|
||||
if nRemoved > 0 {
|
||||
logUtils.SendInfoToSlack("%d were marked as published but weren't actually published and thus removed from the database", nRemoved)
|
||||
}
|
||||
}
|
||||
pubsOnDB := 0
|
||||
for _, sv := range s.syncedVideos {
|
||||
@@ -525,11 +563,11 @@ func (s *Sync) doSync() error {
|
||||
}
|
||||
|
||||
if pubsOnWallet > pubsOnDB { //This case should never happen
|
||||
SendInfoToSlack("We're claiming to have published %d videos but in reality we published %d (%s)", pubsOnDB, pubsOnWallet, s.YoutubeChannelID)
|
||||
logUtils.SendInfoToSlack("We're claiming to have published %d videos but in reality we published %d (%s)", pubsOnDB, pubsOnWallet, s.YoutubeChannelID)
|
||||
return errors.Err("not all published videos are in the database")
|
||||
}
|
||||
if pubsOnWallet < pubsOnDB {
|
||||
SendInfoToSlack("we're claiming to have published %d videos but we only published %d (%s)", pubsOnDB, pubsOnWallet, s.YoutubeChannelID)
|
||||
logUtils.SendInfoToSlack("we're claiming to have published %d videos but we only published %d (%s)", pubsOnDB, pubsOnWallet, s.YoutubeChannelID)
|
||||
}
|
||||
|
||||
if s.StopOnError {
|
||||
@@ -545,7 +583,7 @@ func (s *Sync) doSync() error {
|
||||
}
|
||||
|
||||
if s.LbryChannelName == "@UCBerkeley" {
|
||||
err = s.enqueueUCBVideos()
|
||||
err = errors.Err("UCB is not supported in this version of YTSYNC")
|
||||
} else {
|
||||
err = s.enqueueYoutubeVideos()
|
||||
}
|
||||
@@ -584,8 +622,11 @@ func (s *Sync) startWorker(workerNum int) {
|
||||
err := s.processVideo(v)
|
||||
|
||||
if err != nil {
|
||||
logMsg := "error processing video: " + err.Error()
|
||||
logMsg := fmt.Sprintf("error processing video %s: %s", v.ID(), err.Error())
|
||||
log.Errorln(logMsg)
|
||||
if strings.Contains(strings.ToLower(err.Error()), "interrupted by user") {
|
||||
return
|
||||
}
|
||||
fatalErrors := []string{
|
||||
":5279: read: connection reset by peer",
|
||||
"no space left on device",
|
||||
@@ -593,13 +634,14 @@ func (s *Sync) startWorker(workerNum int) {
|
||||
"Cannot publish using channel",
|
||||
"cannot concatenate 'str' and 'NoneType' objects",
|
||||
"more than 90% of the space has been used.",
|
||||
"Couldn't find private key for id",
|
||||
}
|
||||
if util.SubstringInSlice(err.Error(), fatalErrors) || s.StopOnError {
|
||||
s.grp.Stop()
|
||||
} else if s.MaxTries > 1 {
|
||||
errorsNoRetry := []string{
|
||||
"non 200 status code received",
|
||||
" reason: 'This video contains content from",
|
||||
"This video contains content from",
|
||||
"dont know which claim to update",
|
||||
"uploader has not made this video available in your country",
|
||||
"download error: AccessDenied: Access Denied",
|
||||
@@ -613,6 +655,8 @@ func (s *Sync) startWorker(workerNum int) {
|
||||
"no compatible format available for this video",
|
||||
"Watch this video on YouTube.",
|
||||
"have blocked it on copyright grounds",
|
||||
"the video must be republished as we can't get the right size",
|
||||
"HTTP Error 403",
|
||||
}
|
||||
if util.SubstringInSlice(err.Error(), errorsNoRetry) {
|
||||
log.Println("This error should not be retried at all")
|
||||
@@ -620,33 +664,57 @@ func (s *Sync) startWorker(workerNum int) {
|
||||
if strings.Contains(err.Error(), "txn-mempool-conflict") ||
|
||||
strings.Contains(err.Error(), "too-long-mempool-chain") {
|
||||
log.Println("waiting for a block before retrying")
|
||||
err = s.waitForNewBlock()
|
||||
err := s.waitForNewBlock()
|
||||
if err != nil {
|
||||
s.grp.Stop()
|
||||
SendErrorToSlack("something went wrong while waiting for a block: %v", err)
|
||||
logUtils.SendErrorToSlack("something went wrong while waiting for a block: %v", err)
|
||||
break
|
||||
}
|
||||
} else if util.SubstringInSlice(err.Error(), []string{
|
||||
"Not enough funds to cover this transaction",
|
||||
"failed: Not enough funds",
|
||||
"Error in daemon: Insufficient funds, please deposit additional LBC"}) {
|
||||
log.Println("refilling addresses before retrying")
|
||||
err = s.walletSetup()
|
||||
"Error in daemon: Insufficient funds, please deposit additional LBC",
|
||||
}) {
|
||||
log.Println("checking funds and UTXOs before retrying...")
|
||||
err := s.walletSetup()
|
||||
if err != nil {
|
||||
s.grp.Stop()
|
||||
SendErrorToSlack("failed to setup the wallet for a refill: %v", err)
|
||||
logUtils.SendErrorToSlack("failed to setup the wallet for a refill: %v", err)
|
||||
break
|
||||
}
|
||||
} else if strings.Contains(err.Error(), "Error in daemon: 'str' object has no attribute 'get'") {
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
log.Println("Retrying")
|
||||
continue
|
||||
}
|
||||
SendErrorToSlack("Video failed after %d retries, skipping. Stack: %s", tryCount, logMsg)
|
||||
logUtils.SendErrorToSlack("Video failed after %d retries, skipping. Stack: %s", tryCount, logMsg)
|
||||
}
|
||||
s.AppendSyncedVideo(v.ID(), false, err.Error(), "")
|
||||
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusFailed, "", "", err.Error(), v.Size())
|
||||
s.syncedVideosMux.RLock()
|
||||
existingClaim, ok := s.syncedVideos[v.ID()]
|
||||
s.syncedVideosMux.RUnlock()
|
||||
existingClaimID := ""
|
||||
existingClaimName := ""
|
||||
existingClaimSize := int64(0)
|
||||
if v.Size() != nil {
|
||||
existingClaimSize = *v.Size()
|
||||
}
|
||||
if ok {
|
||||
existingClaimID = existingClaim.ClaimID
|
||||
existingClaimName = existingClaim.ClaimName
|
||||
if existingClaim.Size > 0 {
|
||||
existingClaimSize = existingClaim.Size
|
||||
}
|
||||
}
|
||||
videoStatus := VideoStatusFailed
|
||||
if strings.Contains(err.Error(), "upgrade failed") {
|
||||
videoStatus = VideoStatusUpgradeFailed
|
||||
} else {
|
||||
s.AppendSyncedVideo(v.ID(), false, err.Error(), existingClaimName, existingClaimID, 0, existingClaimSize)
|
||||
}
|
||||
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), videoStatus, existingClaimID, existingClaimName, err.Error(), &existingClaimSize, 0)
|
||||
if err != nil {
|
||||
SendErrorToSlack("Failed to mark video on the database: %s", err.Error())
|
||||
logUtils.SendErrorToSlack("Failed to mark video on the database: %s", err.Error())
|
||||
}
|
||||
}
|
||||
break
|
||||
@@ -683,7 +751,7 @@ func (s *Sync) enqueueYoutubeVideos() error {
|
||||
}
|
||||
|
||||
var videos []video
|
||||
|
||||
playlistMap := make(map[string]*youtube.PlaylistItemSnippet, 50)
|
||||
nextPageToken := ""
|
||||
for {
|
||||
req := service.PlaylistItems.List("snippet").
|
||||
@@ -705,7 +773,7 @@ func (s *Sync) enqueueYoutubeVideos() error {
|
||||
}
|
||||
return errors.Err("playlist items not found")
|
||||
}
|
||||
playlistMap := make(map[string]*youtube.PlaylistItemSnippet, 50)
|
||||
//playlistMap := make(map[string]*youtube.PlaylistItemSnippet, 50)
|
||||
videoIDs := make([]string, 50)
|
||||
for i, item := range playlistResponse.Items {
|
||||
// normally we'd send the video into the channel here, but youtube api doesn't have sorting
|
||||
@@ -713,14 +781,14 @@ func (s *Sync) enqueueYoutubeVideos() error {
|
||||
playlistMap[item.Snippet.ResourceId.VideoId] = item.Snippet
|
||||
videoIDs[i] = item.Snippet.ResourceId.VideoId
|
||||
}
|
||||
req2 := service.Videos.List("snippet,contentDetails").Id(strings.Join(videoIDs[:], ","))
|
||||
req2 := service.Videos.List("snippet,contentDetails,recordingDetails").Id(strings.Join(videoIDs[:], ","))
|
||||
|
||||
videosListResponse, err := req2.Do()
|
||||
if err != nil {
|
||||
return errors.Prefix("error getting videos info", err)
|
||||
}
|
||||
for _, item := range videosListResponse.Items {
|
||||
videos = append(videos, sources.NewYoutubeVideo(s.videoDirectory, item, playlistMap[item.Id].Position))
|
||||
videos = append(videos, sources.NewYoutubeVideo(s.videoDirectory, item, playlistMap[item.Id].Position, s.Manager.GetS3AWSConfig(), s.grp))
|
||||
}
|
||||
|
||||
log.Infof("Got info for %d videos from youtube API", len(videos))
|
||||
@@ -730,7 +798,16 @@ func (s *Sync) enqueueYoutubeVideos() error {
|
||||
break
|
||||
}
|
||||
}
|
||||
for k, v := range s.syncedVideos {
|
||||
if !v.Published {
|
||||
continue
|
||||
}
|
||||
_, ok := playlistMap[k]
|
||||
if !ok {
|
||||
videos = append(videos, sources.NewMockedVideo(s.videoDirectory, k, s.YoutubeChannelID, s.Manager.GetS3AWSConfig(), s.grp))
|
||||
}
|
||||
|
||||
}
|
||||
sort.Sort(byPublishedAt(videos))
|
||||
//or sort.Sort(sort.Reverse(byPlaylistPosition(videos)))
|
||||
|
||||
@@ -752,55 +829,6 @@ Enqueue:
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Sync) enqueueUCBVideos() error {
|
||||
var videos []video
|
||||
|
||||
csvFile, err := os.Open("ucb.csv")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
reader := csv.NewReader(bufio.NewReader(csvFile))
|
||||
for {
|
||||
line, err := reader.Read()
|
||||
if err == io.EOF {
|
||||
break
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
data := struct {
|
||||
PublishedAt string `json:"publishedAt"`
|
||||
}{}
|
||||
err = json.Unmarshal([]byte(line[4]), &data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
videos = append(videos, sources.NewUCBVideo(line[0], line[2], line[1], line[3], data.PublishedAt, s.videoDirectory))
|
||||
}
|
||||
|
||||
log.Printf("Publishing %d videos\n", len(videos))
|
||||
|
||||
sort.Sort(byPublishedAt(videos))
|
||||
|
||||
Enqueue:
|
||||
for _, v := range videos {
|
||||
select {
|
||||
case <-s.grp.Ch():
|
||||
break Enqueue
|
||||
default:
|
||||
}
|
||||
|
||||
select {
|
||||
case s.queue <- v:
|
||||
case <-s.grp.Ch():
|
||||
break Enqueue
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Sync) processVideo(v video) (err error) {
|
||||
defer func() {
|
||||
if p := recover(); p != nil {
|
||||
@@ -822,13 +850,16 @@ func (s *Sync) processVideo(v video) (err error) {
|
||||
s.syncedVideosMux.RLock()
|
||||
sv, ok := s.syncedVideos[v.ID()]
|
||||
s.syncedVideosMux.RUnlock()
|
||||
newMetadataVersion := int8(2)
|
||||
alreadyPublished := ok && sv.Published
|
||||
videoRequiresUpgrade := ok && s.Manager.upgradeMetadata && sv.MetadataVersion < newMetadataVersion
|
||||
|
||||
neverRetryFailures := []string{
|
||||
"Error extracting sts from embedded url response",
|
||||
"Unable to extract signature tokens",
|
||||
"the video is too big to sync, skipping for now",
|
||||
"video is too long to process",
|
||||
"This video contains content from",
|
||||
"no compatible format available for this video",
|
||||
"Watch this video on YouTube.",
|
||||
"have blocked it on copyright grounds",
|
||||
@@ -838,12 +869,16 @@ func (s *Sync) processVideo(v video) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if alreadyPublished {
|
||||
if alreadyPublished && !videoRequiresUpgrade {
|
||||
log.Println(v.ID() + " already published")
|
||||
return nil
|
||||
}
|
||||
if ok && sv.MetadataVersion >= newMetadataVersion {
|
||||
log.Println(v.ID() + " upgraded to the new metadata")
|
||||
return nil
|
||||
}
|
||||
|
||||
if v.PlaylistPosition() > s.Manager.videosLimit {
|
||||
if !videoRequiresUpgrade && v.PlaylistPosition() > s.Manager.videosLimit {
|
||||
log.Println(v.ID() + " is old: skipping")
|
||||
return nil
|
||||
}
|
||||
@@ -851,19 +886,27 @@ func (s *Sync) processVideo(v video) (err error) {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sp := sources.SyncParams{
|
||||
ClaimAddress: s.claimAddress,
|
||||
Amount: publishAmount,
|
||||
ChannelID: s.lbryChannelID,
|
||||
MaxVideoSize: s.Manager.maxVideoSize,
|
||||
Namer: s.namer,
|
||||
MaxVideoLength: s.Manager.maxVideoLength,
|
||||
Fee: s.Fee,
|
||||
}
|
||||
|
||||
summary, err := v.Sync(s.daemon, s.claimAddress, publishAmount, s.lbryChannelID, s.Manager.maxVideoSize, s.namer, s.Manager.maxVideoLength)
|
||||
summary, err := v.Sync(s.daemon, sp, &sv, videoRequiresUpgrade, s.walletMux)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusPublished, summary.ClaimID, summary.ClaimName, "", v.Size())
|
||||
s.AppendSyncedVideo(v.ID(), true, "", summary.ClaimName, summary.ClaimID, newMetadataVersion, *v.Size())
|
||||
err = s.Manager.apiConfig.MarkVideoStatus(s.YoutubeChannelID, v.ID(), VideoStatusPublished, summary.ClaimID, summary.ClaimName, "", v.Size(), 2)
|
||||
if err != nil {
|
||||
SendErrorToSlack("Failed to mark video on the database: %s", err.Error())
|
||||
logUtils.SendErrorToSlack("Failed to mark video on the database: %s", err.Error())
|
||||
}
|
||||
|
||||
s.AppendSyncedVideo(v.ID(), true, "", summary.ClaimName)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
+61
-18
@@ -2,6 +2,7 @@ package sdk
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@@ -33,16 +34,17 @@ type SyncProperties struct {
|
||||
YoutubeChannelID string
|
||||
}
|
||||
|
||||
type Fee struct {
|
||||
Amount string `json:"amount"`
|
||||
Address string `json:"address"`
|
||||
Currency string `json:"currency"`
|
||||
}
|
||||
type YoutubeChannel struct {
|
||||
ChannelId string `json:"channel_id"`
|
||||
TotalVideos uint `json:"total_videos"`
|
||||
DesiredChannelName string `json:"desired_channel_name"`
|
||||
Fee *struct {
|
||||
Amount string `json:"amount"`
|
||||
Address string `json:"address"`
|
||||
Currency string `json:"currency"`
|
||||
} `json:"fee"`
|
||||
ChannelClaimID string `json:"channel_claim_id"`
|
||||
Fee *Fee `json:"fee"`
|
||||
ChannelClaimID string `json:"channel_claim_id"`
|
||||
}
|
||||
|
||||
func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeChannel, error) {
|
||||
@@ -52,7 +54,7 @@ func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeC
|
||||
Data []YoutubeChannel `json:"data"`
|
||||
}
|
||||
endpoint := a.ApiURL + "/yt/jobs"
|
||||
res, _ := http.PostForm(endpoint, url.Values{
|
||||
res, err := http.PostForm(endpoint, url.Values{
|
||||
"auth_token": {a.ApiToken},
|
||||
"sync_status": {status},
|
||||
"min_videos": {strconv.Itoa(1)},
|
||||
@@ -61,10 +63,13 @@ func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeC
|
||||
"sync_server": {a.HostName},
|
||||
"channel_id": {cp.YoutubeChannelID},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
var response apiJobsResponse
|
||||
err := json.Unmarshal(body, &response)
|
||||
err = json.Unmarshal(body, &response)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -76,10 +81,13 @@ func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeC
|
||||
}
|
||||
|
||||
type SyncedVideo struct {
|
||||
VideoID string `json:"video_id"`
|
||||
Published bool `json:"published"`
|
||||
FailureReason string `json:"failure_reason"`
|
||||
ClaimName string `json:"claim_name"`
|
||||
VideoID string `json:"video_id"`
|
||||
Published bool `json:"published"`
|
||||
FailureReason string `json:"failure_reason"`
|
||||
ClaimName string `json:"claim_name"`
|
||||
ClaimID string `json:"claim_id"`
|
||||
Size int64 `json:"size"`
|
||||
MetadataVersion int8 `json:"metadata_version"`
|
||||
}
|
||||
|
||||
func sanitizeFailureReason(s *string) {
|
||||
@@ -121,7 +129,9 @@ func (a *APIConfig) SetChannelStatus(channelID string, status string, failureRea
|
||||
claimNames := make(map[string]bool)
|
||||
for _, v := range response.Data {
|
||||
svs[v.VideoID] = v
|
||||
claimNames[v.ClaimName] = v.Published
|
||||
if v.ClaimName != "" {
|
||||
claimNames[v.ClaimName] = v.Published
|
||||
}
|
||||
}
|
||||
return svs, claimNames, nil
|
||||
}
|
||||
@@ -157,11 +167,41 @@ func (a *APIConfig) SetChannelClaimID(channelID string, channelClaimID string) e
|
||||
}
|
||||
|
||||
const (
|
||||
VideoStatusPublished = "published"
|
||||
VideoStatusFailed = "failed"
|
||||
VideoStatusPublished = "published"
|
||||
VideoStatusUpgradeFailed = "upgradefailed"
|
||||
VideoStatusFailed = "failed"
|
||||
)
|
||||
|
||||
func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status string, claimID string, claimName string, failureReason string, size *int64) error {
|
||||
func (a *APIConfig) DeleteVideos(videos []string) error {
|
||||
endpoint := a.ApiURL + "/yt/video_delete"
|
||||
videoIDs := strings.Join(videos, ",")
|
||||
vals := url.Values{
|
||||
"video_ids": {videoIDs},
|
||||
"auth_token": {a.ApiToken},
|
||||
}
|
||||
res, _ := http.PostForm(endpoint, vals)
|
||||
defer res.Body.Close()
|
||||
body, _ := ioutil.ReadAll(res.Body)
|
||||
var response struct {
|
||||
Success bool `json:"success"`
|
||||
Error null.String `json:"error"`
|
||||
Data null.String `json:"data"`
|
||||
}
|
||||
err := json.Unmarshal(body, &response)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
if !response.Error.IsNull() {
|
||||
return errors.Err(response.Error.String)
|
||||
}
|
||||
|
||||
if !response.Data.IsNull() && response.Data.String == "ok" {
|
||||
return nil
|
||||
}
|
||||
return errors.Err("invalid API response. Status code: %d", res.StatusCode)
|
||||
}
|
||||
|
||||
func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status string, claimID string, claimName string, failureReason string, size *int64, metadataVersion uint) error {
|
||||
endpoint := a.ApiURL + "/yt/video_status"
|
||||
|
||||
sanitizeFailureReason(&failureReason)
|
||||
@@ -171,13 +211,16 @@ func (a *APIConfig) MarkVideoStatus(channelID string, videoID string, status str
|
||||
"status": {status},
|
||||
"auth_token": {a.ApiToken},
|
||||
}
|
||||
if status == VideoStatusPublished {
|
||||
if status == VideoStatusPublished || status == VideoStatusUpgradeFailed {
|
||||
if claimID == "" || claimName == "" {
|
||||
return errors.Err("claimID or claimName missing")
|
||||
return errors.Err("claimID (%s) or claimName (%s) missing", claimID, claimName)
|
||||
}
|
||||
vals.Add("published_at", strconv.FormatInt(time.Now().Unix(), 10))
|
||||
vals.Add("claim_id", claimID)
|
||||
vals.Add("claim_name", claimName)
|
||||
if metadataVersion > 0 {
|
||||
vals.Add("metadata_version", fmt.Sprintf("%d", metadataVersion))
|
||||
}
|
||||
if size != nil {
|
||||
vals.Add("size", strconv.FormatInt(*size, 10))
|
||||
}
|
||||
|
||||
+4
-1
@@ -2,6 +2,7 @@ package sources
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/lbryio/lbry.go/extras/jsonrpc"
|
||||
"github.com/lbryio/ytsync/namer"
|
||||
@@ -12,7 +13,9 @@ type SyncSummary struct {
|
||||
ClaimName string
|
||||
}
|
||||
|
||||
func publishAndRetryExistingNames(daemon *jsonrpc.Client, title, filename string, amount float64, options jsonrpc.StreamCreateOptions, namer *namer.Namer) (*SyncSummary, error) {
|
||||
func publishAndRetryExistingNames(daemon *jsonrpc.Client, title, filename string, amount float64, options jsonrpc.StreamCreateOptions, namer *namer.Namer, walletLock *sync.RWMutex) (*SyncSummary, error) {
|
||||
walletLock.RLock()
|
||||
defer walletLock.RUnlock()
|
||||
for {
|
||||
name := namer.GetNextName(title)
|
||||
response, err := daemon.StreamCreate(name, filename, amount, options)
|
||||
|
||||
@@ -1,219 +0,0 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"os"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
"github.com/lbryio/lbry.go/extras/jsonrpc"
|
||||
"github.com/lbryio/ytsync/namer"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type ucbVideo struct {
|
||||
id string
|
||||
title string
|
||||
channel string
|
||||
description string
|
||||
publishedAt time.Time
|
||||
dir string
|
||||
claimNames map[string]bool
|
||||
syncedVideosMux *sync.RWMutex
|
||||
}
|
||||
|
||||
func NewUCBVideo(id, title, channel, description, publishedAt, dir string) *ucbVideo {
|
||||
p, _ := time.Parse(time.RFC3339Nano, publishedAt) // ignore parse errors
|
||||
return &ucbVideo{
|
||||
id: id,
|
||||
title: title,
|
||||
description: description,
|
||||
channel: channel,
|
||||
dir: dir,
|
||||
publishedAt: p,
|
||||
}
|
||||
}
|
||||
|
||||
func (v *ucbVideo) ID() string {
|
||||
return v.id
|
||||
}
|
||||
|
||||
func (v *ucbVideo) PlaylistPosition() int {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (v *ucbVideo) IDAndNum() string {
|
||||
return v.ID() + " (?)"
|
||||
}
|
||||
|
||||
func (v *ucbVideo) PublishedAt() time.Time {
|
||||
return v.publishedAt
|
||||
//r := regexp.MustCompile(`(\d\d\d\d)-(\d\d)-(\d\d)`)
|
||||
//matches := r.FindStringSubmatch(v.title)
|
||||
//if len(matches) > 0 {
|
||||
// year, _ := strconv.Atoi(matches[1])
|
||||
// month, _ := strconv.Atoi(matches[2])
|
||||
// day, _ := strconv.Atoi(matches[3])
|
||||
// return time.Date(year, time.Month(month), day, 0, 0, 0, 0, time.UTC)
|
||||
//}
|
||||
//return time.Now()
|
||||
}
|
||||
|
||||
func (v *ucbVideo) getFilename() string {
|
||||
return v.dir + "/" + v.id + ".mp4"
|
||||
}
|
||||
|
||||
func (v *ucbVideo) getClaimName(attempt int) string {
|
||||
reg := regexp.MustCompile(`[^a-zA-Z0-9]+`)
|
||||
suffix := ""
|
||||
if attempt > 1 {
|
||||
suffix = "-" + strconv.Itoa(attempt)
|
||||
}
|
||||
maxLen := 40 - len(suffix)
|
||||
|
||||
chunks := strings.Split(strings.ToLower(strings.Trim(reg.ReplaceAllString(v.title, "-"), "-")), "-")
|
||||
|
||||
name := chunks[0]
|
||||
if len(name) > maxLen {
|
||||
return name[:maxLen]
|
||||
}
|
||||
|
||||
for _, chunk := range chunks[1:] {
|
||||
tmpName := name + "-" + chunk
|
||||
if len(tmpName) > maxLen {
|
||||
if len(name) < 20 {
|
||||
name = tmpName[:maxLen]
|
||||
}
|
||||
break
|
||||
}
|
||||
name = tmpName
|
||||
}
|
||||
|
||||
return name + suffix
|
||||
}
|
||||
|
||||
func (v *ucbVideo) getAbbrevDescription() string {
|
||||
maxLines := 10
|
||||
description := strings.TrimSpace(v.description)
|
||||
if strings.Count(description, "\n") < maxLines {
|
||||
return description
|
||||
}
|
||||
return strings.Join(strings.Split(description, "\n")[:maxLines], "\n") + "\n..."
|
||||
}
|
||||
|
||||
func (v *ucbVideo) download() error {
|
||||
videoPath := v.getFilename()
|
||||
|
||||
_, err := os.Stat(videoPath)
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
} else if err == nil {
|
||||
log.Debugln(v.id + " already exists at " + videoPath)
|
||||
return nil
|
||||
}
|
||||
|
||||
creds := credentials.NewStaticCredentials("ID-GOES-HERE", "SECRET-GOES-HERE", "")
|
||||
s, err := session.NewSession(&aws.Config{Region: aws.String("us-east-2"), Credentials: creds})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
downloader := s3manager.NewDownloader(s)
|
||||
|
||||
out, err := os.Create(videoPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
log.Println("lbry-niko2/videos/" + v.channel + "/" + v.id)
|
||||
|
||||
bytesWritten, err := downloader.Download(out, &s3.GetObjectInput{
|
||||
Bucket: aws.String("lbry-niko2"),
|
||||
Key: aws.String("/videos/" + v.channel + "/" + v.id + ".mp4"),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
} else if bytesWritten == 0 {
|
||||
return errors.Err("zero bytes written")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (v *ucbVideo) saveThumbnail() error {
|
||||
resp, err := http.Get("https://s3.us-east-2.amazonaws.com/lbry-niko2/thumbnails/" + v.id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
creds := credentials.NewStaticCredentials("ID-GOES-HERE", "SECRET-GOES-HERE", "")
|
||||
s, err := session.NewSession(&aws.Config{Region: aws.String("us-east-2"), Credentials: creds})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
uploader := s3manager.NewUploader(s)
|
||||
|
||||
_, err = uploader.Upload(&s3manager.UploadInput{
|
||||
Bucket: aws.String("berk.ninja"),
|
||||
Key: aws.String("thumbnails/" + v.id),
|
||||
ContentType: aws.String("image/jpeg"),
|
||||
Body: resp.Body,
|
||||
})
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (v *ucbVideo) publish(daemon *jsonrpc.Client, claimAddress string, amount float64, channelID string, namer *namer.Namer) (*SyncSummary, error) {
|
||||
options := jsonrpc.StreamCreateOptions{
|
||||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
|
||||
Title: v.title,
|
||||
Description: v.getAbbrevDescription(),
|
||||
ClaimAddress: &claimAddress,
|
||||
Languages: []string{"en"},
|
||||
ThumbnailURL: strPtr("https://berk.ninja/thumbnails/" + v.id),
|
||||
Tags: []string{},
|
||||
},
|
||||
Author: strPtr("UC Berkeley"),
|
||||
License: strPtr("see description"),
|
||||
StreamType: &jsonrpc.StreamTypeVideo,
|
||||
ChannelID: &channelID,
|
||||
}
|
||||
return publishAndRetryExistingNames(daemon, v.title, v.getFilename(), amount, options, namer)
|
||||
}
|
||||
|
||||
func (v *ucbVideo) Size() *int64 {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (v *ucbVideo) Sync(daemon *jsonrpc.Client, claimAddress string, amount float64, channelID string, maxVideoSize int, namer *namer.Namer, maxVideoLength float64) (*SyncSummary, error) {
|
||||
//download and thumbnail can be done in parallel
|
||||
err := v.download()
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("download error", err)
|
||||
}
|
||||
log.Debugln("Downloaded " + v.id)
|
||||
|
||||
//err = v.SaveThumbnail()
|
||||
//if err != nil {
|
||||
// return errors.WrapPrefix(err, "thumbnail error", 0)
|
||||
//}
|
||||
//log.Debugln("Created thumbnail for " + v.id)
|
||||
|
||||
summary, err := v.publish(daemon, claimAddress, amount, channelID, namer)
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("publish error", err)
|
||||
}
|
||||
|
||||
return summary, nil
|
||||
}
|
||||
+412
-140
@@ -1,31 +1,38 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
"github.com/lbryio/lbry.go/extras/jsonrpc"
|
||||
"github.com/lbryio/lbry.go/extras/stop"
|
||||
"github.com/lbryio/lbry.go/extras/util"
|
||||
|
||||
"github.com/lbryio/ytsync/ipManager"
|
||||
"github.com/lbryio/ytsync/namer"
|
||||
"github.com/lbryio/ytsync/sdk"
|
||||
"github.com/lbryio/ytsync/tagsManager"
|
||||
"github.com/lbryio/ytsync/thumbs"
|
||||
|
||||
"github.com/ChannelMeter/iso8601duration"
|
||||
"github.com/nikooo777/ytdl"
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/shopspring/decimal"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"google.golang.org/api/youtube/v3"
|
||||
)
|
||||
|
||||
type YoutubeVideo struct {
|
||||
id string
|
||||
channelTitle string
|
||||
title string
|
||||
description string
|
||||
playlistPosition int64
|
||||
@@ -35,20 +42,76 @@ type YoutubeVideo struct {
|
||||
publishedAt time.Time
|
||||
dir string
|
||||
youtubeInfo *youtube.Video
|
||||
youtubeChannelID string
|
||||
tags []string
|
||||
awsConfig aws.Config
|
||||
thumbnailURL string
|
||||
lbryChannelID string
|
||||
mocked bool
|
||||
walletLock *sync.RWMutex
|
||||
stopGroup *stop.Group
|
||||
}
|
||||
|
||||
func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64) *YoutubeVideo {
|
||||
var youtubeCategories = map[string]string{
|
||||
"1": "film & animation",
|
||||
"2": "autos & vehicles",
|
||||
"10": "music",
|
||||
"15": "pets & animals",
|
||||
"17": "sports",
|
||||
"18": "short movies",
|
||||
"19": "travel & events",
|
||||
"20": "gaming",
|
||||
"21": "videoblogging",
|
||||
"22": "people & blogs",
|
||||
"23": "comedy",
|
||||
"24": "entertainment",
|
||||
"25": "news & politics",
|
||||
"26": "howto & style",
|
||||
"27": "education",
|
||||
"28": "science & technology",
|
||||
"29": "nonprofits & activism",
|
||||
"30": "movies",
|
||||
"31": "anime/animation",
|
||||
"32": "action/adventure",
|
||||
"33": "classics",
|
||||
"34": "comedy",
|
||||
"35": "documentary",
|
||||
"36": "drama",
|
||||
"37": "family",
|
||||
"38": "foreign",
|
||||
"39": "horror",
|
||||
"40": "sci-fi/fantasy",
|
||||
"41": "thriller",
|
||||
"42": "shorts",
|
||||
"43": "shows",
|
||||
"44": "trailers",
|
||||
}
|
||||
|
||||
func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group) *YoutubeVideo {
|
||||
publishedAt, _ := time.Parse(time.RFC3339Nano, videoData.Snippet.PublishedAt) // ignore parse errors
|
||||
return &YoutubeVideo{
|
||||
id: videoData.Id,
|
||||
title: videoData.Snippet.Title,
|
||||
description: videoData.Snippet.Description,
|
||||
channelTitle: videoData.Snippet.ChannelTitle,
|
||||
playlistPosition: playlistPosition,
|
||||
publishedAt: publishedAt,
|
||||
dir: directory,
|
||||
youtubeInfo: videoData,
|
||||
awsConfig: awsConfig,
|
||||
mocked: false,
|
||||
youtubeChannelID: videoData.Snippet.ChannelId,
|
||||
stopGroup: stopGroup,
|
||||
}
|
||||
}
|
||||
func NewMockedVideo(directory string, videoID string, youtubeChannelID string, awsConfig aws.Config, stopGroup *stop.Group) *YoutubeVideo {
|
||||
return &YoutubeVideo{
|
||||
id: videoID,
|
||||
playlistPosition: 0,
|
||||
dir: directory,
|
||||
awsConfig: awsConfig,
|
||||
mocked: true,
|
||||
youtubeChannelID: youtubeChannelID,
|
||||
stopGroup: stopGroup,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,6 +128,9 @@ func (v *YoutubeVideo) IDAndNum() string {
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) PublishedAt() time.Time {
|
||||
if v.mocked {
|
||||
return time.Unix(0, 0)
|
||||
}
|
||||
return v.publishedAt
|
||||
}
|
||||
|
||||
@@ -98,13 +164,18 @@ func (v *YoutubeVideo) getFullPath() string {
|
||||
func (v *YoutubeVideo) getAbbrevDescription() string {
|
||||
maxLines := 10
|
||||
description := strings.TrimSpace(v.description)
|
||||
if strings.Count(description, "\n") < maxLines {
|
||||
return description
|
||||
additionalDescription := "\nhttps://www.youtube.com/watch?v=" + v.id
|
||||
khanAcademyClaimID := "5fc52291980268b82413ca4c0ace1b8d749f3ffb"
|
||||
if v.lbryChannelID == khanAcademyClaimID {
|
||||
additionalDescription = additionalDescription + "\nNote: All Khan Academy content is available for free at (www.khanacademy.org)"
|
||||
}
|
||||
return strings.Join(strings.Split(description, "\n")[:maxLines], "\n") + "\n..."
|
||||
if strings.Count(description, "\n") < maxLines {
|
||||
return description + "\n..." + additionalDescription
|
||||
}
|
||||
return strings.Join(strings.Split(description, "\n")[:maxLines], "\n") + "\n..." + additionalDescription
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) download() error {
|
||||
func (v *YoutubeVideo) download(useIPv6 bool) error {
|
||||
videoPath := v.getFullPath()
|
||||
|
||||
err := os.Mkdir(v.videoDir(), 0750)
|
||||
@@ -119,179 +190,246 @@ func (v *YoutubeVideo) download() error {
|
||||
log.Debugln(v.id + " already exists at " + videoPath)
|
||||
return nil
|
||||
}
|
||||
qualities := []string{
|
||||
//"1080",
|
||||
"720",
|
||||
"480",
|
||||
"320",
|
||||
}
|
||||
qualityIndex := 0
|
||||
ytdlArgs := []string{
|
||||
"--no-progress",
|
||||
"--max-filesize",
|
||||
fmt.Sprintf("%dM", v.maxVideoSize),
|
||||
"--match-filter",
|
||||
fmt.Sprintf("duration <= %d", int(math.Round(v.maxVideoLength*3600))),
|
||||
"-fbestvideo[ext=mp4][height<=" + qualities[qualityIndex] + "]+bestaudio[ext!=webm]",
|
||||
"-o" + strings.TrimSuffix(v.getFullPath(), ".mp4"),
|
||||
"--merge-output-format",
|
||||
"--abort-on-unavailable-fragment",
|
||||
"--fragment-retries",
|
||||
"0",
|
||||
"mp4",
|
||||
}
|
||||
sourceAddress, err := ipManager.GetNextIP(useIPv6)
|
||||
if err != nil {
|
||||
if sourceAddress == "throttled" {
|
||||
for {
|
||||
select {
|
||||
case <-v.stopGroup.Ch():
|
||||
return errors.Err("interrupted by user")
|
||||
default:
|
||||
}
|
||||
|
||||
videoUrl := "https://www.youtube.com/watch?v=" + v.id
|
||||
videoInfo, err := ytdl.GetVideoInfo(videoUrl)
|
||||
time.Sleep(ipManager.IPCooldownPeriod)
|
||||
sourceAddress, err = ipManager.GetNextIP(useIPv6)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return errors.Err(err)
|
||||
}
|
||||
}
|
||||
defer ipManager.ReleaseIP(sourceAddress)
|
||||
if useIPv6 {
|
||||
log.Infof("using IPv6: %s", sourceAddress)
|
||||
ytdlArgs = append(ytdlArgs,
|
||||
"-6",
|
||||
"--source-address",
|
||||
sourceAddress,
|
||||
)
|
||||
} else {
|
||||
log.Infof("using IPv4: %s", sourceAddress)
|
||||
ytdlArgs = append(ytdlArgs,
|
||||
"-4",
|
||||
"--source-address",
|
||||
sourceAddress,
|
||||
)
|
||||
}
|
||||
ytdlArgs = append(ytdlArgs, "https://www.youtube.com/watch?v="+v.ID())
|
||||
runcmd:
|
||||
cmd := exec.Command("youtube-dl", ytdlArgs...)
|
||||
|
||||
log.Printf("Running command and waiting for it to finish...")
|
||||
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
codec := []string{"H.264"}
|
||||
ext := []string{"mp4"}
|
||||
|
||||
//Filter requires a [] interface{}
|
||||
codecFilter := make([]interface{}, len(codec))
|
||||
for i, v := range codec {
|
||||
codecFilter[i] = v
|
||||
if err := cmd.Start(); err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
//Filter requires a [] interface{}
|
||||
extFilter := make([]interface{}, len(ext))
|
||||
for i, v := range ext {
|
||||
extFilter[i] = v
|
||||
}
|
||||
errorLog, _ := ioutil.ReadAll(stderr)
|
||||
outLog, _ := ioutil.ReadAll(stdout)
|
||||
|
||||
formats := videoInfo.Formats.Filter(ytdl.FormatVideoEncodingKey, codecFilter).Filter(ytdl.FormatExtensionKey, extFilter)
|
||||
if len(formats) == 0 {
|
||||
return errors.Err("no compatible format available for this video")
|
||||
if err = cmd.Wait(); err != nil {
|
||||
if strings.Contains(err.Error(), "exit status 1") {
|
||||
if strings.Contains(string(errorLog), "HTTP Error 429") {
|
||||
ipManager.SetIpThrottled(sourceAddress, v.stopGroup)
|
||||
}
|
||||
return errors.Err(string(errorLog))
|
||||
} else if strings.Contains(string(errorLog), "giving up after 0 fragment retries") && qualityIndex < len(qualities)-1 {
|
||||
qualityIndex++
|
||||
goto runcmd
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
maxRetryAttempts := 5
|
||||
isLengthLimitSet := v.maxVideoLength > 0.01
|
||||
if isLengthLimitSet && videoInfo.Duration.Hours() > v.maxVideoLength {
|
||||
log.Debugln(string(outLog))
|
||||
|
||||
if strings.Contains(string(outLog), "does not pass filter duration") {
|
||||
_ = v.delete()
|
||||
return errors.Err("video is too long to process")
|
||||
}
|
||||
|
||||
for i := 0; i < len(formats) && i < maxRetryAttempts; i++ {
|
||||
formatIndex := i
|
||||
if i == maxRetryAttempts-1 {
|
||||
formatIndex = len(formats) - 1
|
||||
}
|
||||
var downloadedFile *os.File
|
||||
downloadedFile, err = os.Create(videoPath)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
err = videoInfo.Download(formats[formatIndex], downloadedFile)
|
||||
_ = downloadedFile.Close()
|
||||
if err != nil {
|
||||
//delete the video and ignore the error
|
||||
_ = v.delete()
|
||||
return errors.Err(err.Error())
|
||||
}
|
||||
fi, err := os.Stat(v.getFullPath())
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
videoSize := fi.Size()
|
||||
v.size = &videoSize
|
||||
|
||||
isVideoSizeLimitSet := v.maxVideoSize > 0
|
||||
if isVideoSizeLimitSet && videoSize > v.maxVideoSize {
|
||||
//delete the video and ignore the error
|
||||
_ = v.delete()
|
||||
err = errors.Err("file is too big and there is no other format available")
|
||||
} else {
|
||||
break
|
||||
}
|
||||
if strings.Contains(string(outLog), "File is larger than max-filesize") {
|
||||
_ = v.delete()
|
||||
return errors.Err("the video is too big to sync, skipping for now")
|
||||
}
|
||||
return errors.Err(err)
|
||||
if string(errorLog) != "" {
|
||||
log.Printf("Command finished with error: %v", errors.Err(string(errorLog)))
|
||||
_ = v.delete()
|
||||
return errors.Err(string(errorLog))
|
||||
}
|
||||
fi, err := os.Stat(v.getFullPath())
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
videoSize := fi.Size()
|
||||
v.size = &videoSize
|
||||
return nil
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) videoDir() string {
|
||||
return v.dir + "/" + v.id
|
||||
}
|
||||
func (v *YoutubeVideo) getDownloadedPath() (string, error) {
|
||||
files, err := ioutil.ReadDir(v.videoDir())
|
||||
log.Infoln(v.videoDir())
|
||||
if err != nil {
|
||||
err = errors.Prefix("list error", err)
|
||||
log.Errorln(err)
|
||||
return "", err
|
||||
}
|
||||
|
||||
for _, f := range files {
|
||||
if f.IsDir() {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(v.getFullPath(), strings.TrimSuffix(f.Name(), filepath.Ext(f.Name()))) {
|
||||
return v.videoDir() + "/" + f.Name(), nil
|
||||
}
|
||||
}
|
||||
return "", errors.Err("could not find any downloaded videos")
|
||||
|
||||
}
|
||||
func (v *YoutubeVideo) delete() error {
|
||||
videoPath := v.getFullPath()
|
||||
err := os.Remove(videoPath)
|
||||
videoPath, err := v.getDownloadedPath()
|
||||
if err != nil {
|
||||
log.Errorln(errors.Prefix("delete error", err))
|
||||
log.Errorln(err)
|
||||
return err
|
||||
}
|
||||
log.Debugln(v.id + " deleted from disk (" + videoPath + ")")
|
||||
return nil
|
||||
}
|
||||
err = os.Remove(videoPath)
|
||||
log.Debugf("%s deleted from disk (%s)", v.id, videoPath)
|
||||
|
||||
func (v *YoutubeVideo) triggerThumbnailSave() error {
|
||||
client := &http.Client{Timeout: 30 * time.Second}
|
||||
|
||||
params, err := json.Marshal(map[string]string{"videoid": v.id})
|
||||
if err != nil {
|
||||
err = errors.Prefix("delete error", err)
|
||||
log.Errorln(err)
|
||||
return err
|
||||
}
|
||||
|
||||
request, err := http.NewRequest(http.MethodPut, "https://jgp4g1qoud.execute-api.us-east-1.amazonaws.com/prod/thumbnail", bytes.NewBuffer(params))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
|
||||
contents, err := ioutil.ReadAll(response.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var decoded struct {
|
||||
Error int `json:"error"`
|
||||
Url string `json:"url,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
}
|
||||
err = json.Unmarshal(contents, &decoded)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if decoded.Error != 0 {
|
||||
return errors.Err("error creating thumbnail: " + decoded.Message)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func strPtr(s string) *string { return &s }
|
||||
func (v *YoutubeVideo) triggerThumbnailSave() (err error) {
|
||||
thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Snippet.Thumbnails)
|
||||
v.thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.Url, v.ID(), v.awsConfig)
|
||||
return err
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, claimAddress string, amount float64, channelID string, namer *namer.Namer) (*SyncSummary, error) {
|
||||
additionalDescription := "\nhttps://www.youtube.com/watch?v=" + v.id
|
||||
khanAcademyClaimID := "5fc52291980268b82413ca4c0ace1b8d749f3ffb"
|
||||
if channelID == khanAcademyClaimID {
|
||||
additionalDescription = additionalDescription + "\nNote: All Khan Academy content is available for free at (www.khanacademy.org)"
|
||||
}
|
||||
var languages []string = nil
|
||||
if v.youtubeInfo.Snippet.DefaultLanguage != "" {
|
||||
languages = []string{v.youtubeInfo.Snippet.DefaultLanguage}
|
||||
func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
|
||||
languages, locations, tags := v.getMetadata()
|
||||
|
||||
var fee *jsonrpc.Fee
|
||||
if params.Fee != nil {
|
||||
feeAmount, err := decimal.NewFromString(params.Fee.Amount)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
fee = &jsonrpc.Fee{
|
||||
FeeAddress: ¶ms.Fee.Address,
|
||||
FeeAmount: feeAmount,
|
||||
FeeCurrency: jsonrpc.Currency(params.Fee.Currency),
|
||||
}
|
||||
}
|
||||
|
||||
videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration)
|
||||
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
options := jsonrpc.StreamCreateOptions{
|
||||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
|
||||
Title: v.title,
|
||||
Description: v.getAbbrevDescription() + additionalDescription,
|
||||
ClaimAddress: &claimAddress,
|
||||
Title: &v.title,
|
||||
Description: util.PtrToString(v.getAbbrevDescription()),
|
||||
ClaimAddress: ¶ms.ClaimAddress,
|
||||
Languages: languages,
|
||||
ThumbnailURL: strPtr("https://thumbnails.lbry.com/" + v.id),
|
||||
Tags: v.youtubeInfo.Snippet.Tags,
|
||||
ThumbnailURL: &v.thumbnailURL,
|
||||
Tags: tags,
|
||||
Locations: locations,
|
||||
},
|
||||
Author: strPtr(v.channelTitle),
|
||||
License: strPtr("Copyrighted (contact author)"),
|
||||
StreamType: &jsonrpc.StreamTypeVideo,
|
||||
ReleaseTime: util.PtrToInt64(v.publishedAt.Unix()),
|
||||
VideoDuration: util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds()))),
|
||||
ChannelID: &channelID,
|
||||
Fee: fee,
|
||||
License: util.PtrToString("Copyrighted (contact publisher)"),
|
||||
ReleaseTime: util.PtrToInt64(v.publishedAt.Unix()),
|
||||
ChannelID: &v.lbryChannelID,
|
||||
}
|
||||
|
||||
return publishAndRetryExistingNames(daemon, v.title, v.getFullPath(), amount, options, namer)
|
||||
downloadPath, err := v.getDownloadedPath()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return publishAndRetryExistingNames(daemon, v.title, downloadPath, params.Amount, options, params.Namer, v.walletLock)
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) Size() *int64 {
|
||||
return v.size
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, claimAddress string, amount float64, channelID string, maxVideoSize int, namer *namer.Namer, maxVideoLength float64) (*SyncSummary, error) {
|
||||
v.maxVideoSize = int64(maxVideoSize) * 1024 * 1024
|
||||
v.maxVideoLength = maxVideoLength
|
||||
//download and thumbnail can be done in parallel
|
||||
err := v.download()
|
||||
type SyncParams struct {
|
||||
ClaimAddress string
|
||||
Amount float64
|
||||
ChannelID string
|
||||
MaxVideoSize int
|
||||
Namer *namer.Namer
|
||||
MaxVideoLength float64
|
||||
Fee *sdk.Fee
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo, reprocess bool, walletLock *sync.RWMutex) (*SyncSummary, error) {
|
||||
v.maxVideoSize = int64(params.MaxVideoSize)
|
||||
v.maxVideoLength = params.MaxVideoLength
|
||||
v.lbryChannelID = params.ChannelID
|
||||
v.walletLock = walletLock
|
||||
if reprocess && existingVideoData != nil && existingVideoData.Published {
|
||||
summary, err := v.reprocess(daemon, params, existingVideoData)
|
||||
return summary, errors.Prefix("upgrade failed", err)
|
||||
}
|
||||
return v.downloadAndPublish(daemon, params)
|
||||
}
|
||||
|
||||
var isThrottled bool
|
||||
|
||||
func (v *YoutubeVideo) downloadAndPublish(daemon *jsonrpc.Client, params SyncParams) (*SyncSummary, error) {
|
||||
err := v.download(isThrottled)
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("download error", err)
|
||||
if strings.Contains(err.Error(), "HTTP Error 429") && !isThrottled {
|
||||
isThrottled = true
|
||||
err = v.download(isThrottled)
|
||||
if err != nil {
|
||||
return nil, errors.Prefix("download error", err)
|
||||
}
|
||||
} else {
|
||||
return nil, errors.Prefix("download error", err)
|
||||
}
|
||||
}
|
||||
log.Debugln("Downloaded " + v.id)
|
||||
|
||||
@@ -301,9 +439,143 @@ func (v *YoutubeVideo) Sync(daemon *jsonrpc.Client, claimAddress string, amount
|
||||
}
|
||||
log.Debugln("Created thumbnail for " + v.id)
|
||||
|
||||
summary, err := v.publish(daemon, claimAddress, amount, channelID, namer)
|
||||
summary, err := v.publish(daemon, params)
|
||||
//delete the video in all cases (and ignore the error)
|
||||
_ = v.delete()
|
||||
|
||||
return summary, errors.Prefix("publish error", err)
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) getMetadata() (languages []string, locations []jsonrpc.Location, tags []string) {
|
||||
languages = nil
|
||||
locations = nil
|
||||
tags = nil
|
||||
if !v.mocked {
|
||||
if v.youtubeInfo.Snippet.DefaultLanguage != "" {
|
||||
languages = []string{v.youtubeInfo.Snippet.DefaultLanguage}
|
||||
}
|
||||
|
||||
if v.youtubeInfo.RecordingDetails != nil && v.youtubeInfo.RecordingDetails.Location != nil {
|
||||
locations = []jsonrpc.Location{{
|
||||
Latitude: util.PtrToString(fmt.Sprintf("%.7f", v.youtubeInfo.RecordingDetails.Location.Latitude)),
|
||||
Longitude: util.PtrToString(fmt.Sprintf("%.7f", v.youtubeInfo.RecordingDetails.Location.Longitude)),
|
||||
}}
|
||||
}
|
||||
tags = v.youtubeInfo.Snippet.Tags
|
||||
}
|
||||
tags, err := tagsManager.SanitizeTags(tags, v.youtubeChannelID)
|
||||
if err != nil {
|
||||
log.Errorln(err.Error())
|
||||
}
|
||||
if !v.mocked {
|
||||
tags = append(tags, youtubeCategories[v.youtubeInfo.Snippet.CategoryId])
|
||||
}
|
||||
|
||||
return languages, locations, tags
|
||||
}
|
||||
|
||||
func (v *YoutubeVideo) reprocess(daemon *jsonrpc.Client, params SyncParams, existingVideoData *sdk.SyncedVideo) (*SyncSummary, error) {
|
||||
c, err := daemon.ClaimSearch(nil, &existingVideoData.ClaimID, nil, nil)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
if len(c.Claims) == 0 {
|
||||
return nil, errors.Err("cannot reprocess: no claim found for this video")
|
||||
} else if len(c.Claims) > 1 {
|
||||
return nil, errors.Err("cannot reprocess: too many claims. claimID: %s", existingVideoData.ClaimID)
|
||||
}
|
||||
|
||||
currentClaim := c.Claims[0]
|
||||
languages, locations, tags := v.getMetadata()
|
||||
|
||||
thumbnailURL := ""
|
||||
if currentClaim.Value.GetThumbnail() == nil {
|
||||
if v.mocked {
|
||||
return nil, errors.Err("could not find thumbnail for mocked video")
|
||||
}
|
||||
thumbnail := thumbs.GetBestThumbnail(v.youtubeInfo.Snippet.Thumbnails)
|
||||
thumbnailURL, err = thumbs.MirrorThumbnail(thumbnail.Url, v.ID(), v.awsConfig)
|
||||
} else {
|
||||
thumbnailURL = thumbs.ThumbnailEndpoint + v.ID()
|
||||
}
|
||||
|
||||
videoSize, err := currentClaim.GetStreamSizeByMagic()
|
||||
if err != nil {
|
||||
if existingVideoData.Size > 0 {
|
||||
videoSize = uint64(existingVideoData.Size)
|
||||
} else {
|
||||
log.Infof("%s: the video must be republished as we can't get the right size", v.ID())
|
||||
//return v.downloadAndPublish(daemon, params) //TODO: actually republish the video. NB: the current claim should be abandoned first
|
||||
return nil, errors.Err("the video must be republished as we can't get the right size")
|
||||
}
|
||||
}
|
||||
v.size = util.PtrToInt64(int64(videoSize))
|
||||
var fee *jsonrpc.Fee
|
||||
if params.Fee != nil {
|
||||
feeAmount, err := decimal.NewFromString(params.Fee.Amount)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
fee = &jsonrpc.Fee{
|
||||
FeeAddress: ¶ms.Fee.Address,
|
||||
FeeAmount: feeAmount,
|
||||
FeeCurrency: jsonrpc.Currency(params.Fee.Currency),
|
||||
}
|
||||
}
|
||||
streamCreateOptions := &jsonrpc.StreamCreateOptions{
|
||||
ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
|
||||
Tags: tags,
|
||||
ThumbnailURL: &thumbnailURL,
|
||||
Languages: languages,
|
||||
Locations: locations,
|
||||
},
|
||||
Author: util.PtrToString(""),
|
||||
License: util.PtrToString("Copyrighted (contact publisher)"),
|
||||
ChannelID: &v.lbryChannelID,
|
||||
Height: util.PtrToUint(720),
|
||||
Width: util.PtrToUint(1280),
|
||||
Fee: fee,
|
||||
}
|
||||
|
||||
v.walletLock.RLock()
|
||||
defer v.walletLock.RUnlock()
|
||||
if v.mocked {
|
||||
pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
|
||||
StreamCreateOptions: streamCreateOptions,
|
||||
FileSize: &videoSize,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &SyncSummary{
|
||||
ClaimID: pr.Outputs[0].ClaimID,
|
||||
ClaimName: pr.Outputs[0].Name,
|
||||
}, nil
|
||||
}
|
||||
|
||||
videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
streamCreateOptions.ClaimCreateOptions.Title = &v.title
|
||||
streamCreateOptions.ClaimCreateOptions.Description = util.PtrToString(v.getAbbrevDescription())
|
||||
streamCreateOptions.Duration = util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds())))
|
||||
streamCreateOptions.ReleaseTime = util.PtrToInt64(v.publishedAt.Unix())
|
||||
pr, err := daemon.StreamUpdate(existingVideoData.ClaimID, jsonrpc.StreamUpdateOptions{
|
||||
ClearLanguages: util.PtrToBool(true),
|
||||
ClearLocations: util.PtrToBool(true),
|
||||
ClearTags: util.PtrToBool(true),
|
||||
StreamCreateOptions: streamCreateOptions,
|
||||
FileSize: &videoSize,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &SyncSummary{
|
||||
ClaimID: pr.Outputs[0].ClaimID,
|
||||
ClaimName: pr.Outputs[0].Name,
|
||||
}, nil
|
||||
}
|
||||
-152
@@ -1,152 +0,0 @@
|
||||
import os
|
||||
import sys
|
||||
from decimal import Decimal
|
||||
from bitcoinrpc.authproxy import AuthServiceProxy
|
||||
|
||||
from lbryum.wallet import Wallet, WalletStorage
|
||||
from lbryum.commands import known_commands, Commands
|
||||
from lbryum.simple_config import SimpleConfig
|
||||
from lbryum.blockchain import get_blockchain
|
||||
from lbryum.network import Network
|
||||
|
||||
|
||||
def get_lbrycrdd_connection_string(wallet_conf):
|
||||
settings = {"username": "rpcuser",
|
||||
"password": "rpcpassword",
|
||||
"rpc_port": 9245}
|
||||
if wallet_conf and os.path.exists(wallet_conf):
|
||||
with open(wallet_conf, "r") as conf:
|
||||
conf_lines = conf.readlines()
|
||||
for l in conf_lines:
|
||||
if l.startswith("rpcuser="):
|
||||
settings["username"] = l[8:].rstrip('\n')
|
||||
if l.startswith("rpcpassword="):
|
||||
settings["password"] = l[12:].rstrip('\n')
|
||||
if l.startswith("rpcport="):
|
||||
settings["rpc_port"] = int(l[8:].rstrip('\n'))
|
||||
|
||||
rpc_user = settings["username"]
|
||||
rpc_pass = settings["password"]
|
||||
rpc_port = settings["rpc_port"]
|
||||
rpc_url = "127.0.0.1"
|
||||
return "http://%s:%s@%s:%i" % (rpc_user, rpc_pass, rpc_url, rpc_port)
|
||||
|
||||
|
||||
class LBRYumWallet(object):
|
||||
def __init__(self, lbryum_path):
|
||||
self.config = SimpleConfig()
|
||||
self.config.set_key('chain', 'lbrycrd_main')
|
||||
self.storage = WalletStorage(lbryum_path)
|
||||
self.wallet = Wallet(self.storage)
|
||||
self.cmd_runner = Commands(self.config, self.wallet, None)
|
||||
if not self.wallet.has_seed():
|
||||
seed = self.wallet.make_seed()
|
||||
self.wallet.add_seed(seed, "derp")
|
||||
self.wallet.create_master_keys("derp")
|
||||
self.wallet.create_main_account()
|
||||
self.wallet.update_password("derp", "")
|
||||
self.network = Network(self.config)
|
||||
self.blockchain = get_blockchain(self.config, self.network)
|
||||
print self.config.get('chain'), self.blockchain
|
||||
self.wallet.storage.write()
|
||||
|
||||
def command(self, command_name, *args, **kwargs):
|
||||
cmd_runner = Commands(self.config, self.wallet, None)
|
||||
cmd = known_commands[command_name]
|
||||
func = getattr(cmd_runner, cmd.name)
|
||||
return func(*args, **kwargs)
|
||||
|
||||
def generate_address(self):
|
||||
address = self.wallet.create_new_address()
|
||||
self.wallet.storage.write()
|
||||
return address
|
||||
|
||||
|
||||
class LBRYcrd(object):
|
||||
def __init__(self, lbrycrdd_path):
|
||||
self.lbrycrdd_conn_str = get_lbrycrdd_connection_string(lbrycrdd_path)
|
||||
|
||||
def __call__(self, method, *args, **kwargs):
|
||||
return self.rpc(method)(*args, **kwargs)
|
||||
|
||||
def rpc(self, method):
|
||||
return AuthServiceProxy(self.lbrycrdd_conn_str, service_name=method)
|
||||
|
||||
|
||||
def get_wallet_path():
|
||||
cwd = os.getcwd()
|
||||
wallet_path = os.path.join(cwd, "wallet.json")
|
||||
if not os.path.exists(wallet_path):
|
||||
return wallet_path
|
||||
i = 1
|
||||
while True:
|
||||
wallet_path = os.path.join(cwd, "wallet_%i.json" % i)
|
||||
if not os.path.exists(wallet_path):
|
||||
return wallet_path
|
||||
i += 1
|
||||
|
||||
|
||||
def coin_chooser(lbrycrdd, amount, fee=0.001):
|
||||
def iter_txis():
|
||||
unspent = lbrycrdd("listunspent")
|
||||
unspent = sorted(unspent, key=lambda x: x['amount'], reverse=True)
|
||||
spendable = Decimal(0.0)
|
||||
for txi in unspent:
|
||||
if spendable >= amount:
|
||||
break
|
||||
else:
|
||||
spendable += txi['amount']
|
||||
yield txi
|
||||
if spendable < amount:
|
||||
print spendable, amount
|
||||
raise Exception("Not enough funds")
|
||||
|
||||
coins = list(iter(iter_txis()))
|
||||
total = sum(c['amount'] for c in coins)
|
||||
change = Decimal(total) - Decimal(amount) - Decimal(fee)
|
||||
|
||||
if change < 0:
|
||||
raise Exception("Not enough funds")
|
||||
if change:
|
||||
change_address = lbrycrdd("getnewaddress")
|
||||
else:
|
||||
change_address = None
|
||||
|
||||
print "Total: %f, amount: %f, change: %f" % (total, amount, change)
|
||||
|
||||
return coins, change, change_address
|
||||
|
||||
|
||||
def get_raw_tx(lbrycrdd, addresses, coins, amount, change, change_address):
|
||||
txi = [{'txid': c['txid'], 'vout': c['vout']} for c in coins]
|
||||
txo = {address: float(amount) for address in addresses}
|
||||
if change_address:
|
||||
txo[change_address] = float(change)
|
||||
return lbrycrdd("createrawtransaction", txi, txo)
|
||||
|
||||
|
||||
def main(count, value=None, lbryum_path=None, lbrycrdd_path=None):
|
||||
count = int(count)
|
||||
lbryum_path = lbryum_path or get_wallet_path()
|
||||
if sys.platform == "darwin":
|
||||
default_lbrycrdd = os.path.join(os.path.expanduser("~"),
|
||||
"Library/Application Support/lbrycrd/lbrycrd.conf")
|
||||
else:
|
||||
default_lbrycrdd = os.path.join(os.path.expanduser("~"), ".lbrycrd/lbrycrd.conf")
|
||||
lbrycrdd_path = lbrycrdd_path or default_lbrycrdd
|
||||
l = LBRYcrd(lbrycrdd_path=lbrycrdd_path)
|
||||
s = LBRYumWallet(lbryum_path)
|
||||
value = value or 1.0
|
||||
value = Decimal(value)
|
||||
|
||||
coins, change, change_address = coin_chooser(l, count * value)
|
||||
addresses = [s.generate_address() for i in range(count)]
|
||||
raw_tx = get_raw_tx(l, addresses, coins, value, change, change_address)
|
||||
signed = l("signrawtransaction", raw_tx)['hex']
|
||||
txid = l("sendrawtransaction", signed)
|
||||
print txid
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = sys.argv[1:]
|
||||
main(*args)
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,144 @@
|
||||
package tagsManager
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSanitizeTags(t *testing.T) {
|
||||
got, err := SanitizeTags([]string{"this", "super", "expensive", "test", "has", "a lot of", "crypto", "currency", "in it", "trump", "will build the", "wall"}, "UCNQfQvFMPnInwsU_iGYArJQ")
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
expectedTags := []string{
|
||||
"blockchain",
|
||||
"switzerland",
|
||||
"news",
|
||||
"science & technology",
|
||||
"economics",
|
||||
"experiments",
|
||||
"this",
|
||||
"in it",
|
||||
"will build the",
|
||||
"has",
|
||||
"crypto",
|
||||
"trump",
|
||||
"wall",
|
||||
"expensive",
|
||||
"currency",
|
||||
"a lot of",
|
||||
}
|
||||
if len(expectedTags) != len(got) {
|
||||
t.Error("number of tags differ")
|
||||
return
|
||||
}
|
||||
outer:
|
||||
for _, et := range expectedTags {
|
||||
for _, t := range got {
|
||||
if et == t {
|
||||
continue outer
|
||||
}
|
||||
}
|
||||
t.Error("tag not found")
|
||||
return
|
||||
}
|
||||
|
||||
}
|
||||
func TestNormalizeTag(t *testing.T) {
|
||||
tags := []string{
|
||||
"blockchain",
|
||||
"Switzerland",
|
||||
"news ",
|
||||
" science & Technology ",
|
||||
"economics",
|
||||
"experiments",
|
||||
"this",
|
||||
"in it",
|
||||
"will build the (WOOPS)",
|
||||
"~has",
|
||||
"crypto",
|
||||
"trump",
|
||||
"wall",
|
||||
"expensive",
|
||||
"!currency",
|
||||
" a lot of ",
|
||||
"#",
|
||||
"#whatever",
|
||||
"#123",
|
||||
"#123 Something else",
|
||||
"#123aaa",
|
||||
"!asdasd",
|
||||
"CASA BLANCA",
|
||||
"wwe 2k18 Elimination chamber!",
|
||||
"pero'",
|
||||
"però",
|
||||
"è proprio",
|
||||
"Ep 29",
|
||||
"sctest29 Keddr",
|
||||
"mortal kombat 11 shang tsung",
|
||||
"!asdasd!",
|
||||
}
|
||||
normalizedTags := make([]string, 0, len(tags))
|
||||
for _, tag := range tags {
|
||||
got, err := normalizeTag(tag)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
if got != "" {
|
||||
normalizedTags = append(normalizedTags, got)
|
||||
}
|
||||
fmt.Printf("Got tag: '%s'\n", got)
|
||||
}
|
||||
expected := []string{
|
||||
"blockchain",
|
||||
"switzerland",
|
||||
"news",
|
||||
"science & technology",
|
||||
"economics",
|
||||
"experiments",
|
||||
"this",
|
||||
"in it",
|
||||
"will build the",
|
||||
"has",
|
||||
"crypto",
|
||||
"trump",
|
||||
"wall",
|
||||
"expensive",
|
||||
"currency",
|
||||
"a lot of",
|
||||
"whatever",
|
||||
"123",
|
||||
"something else",
|
||||
"123aaa",
|
||||
"asdasd",
|
||||
"casa blanca",
|
||||
"wwe 2k18 elimination chamber",
|
||||
"pero",
|
||||
"però",
|
||||
"è proprio",
|
||||
"ep 29",
|
||||
"sctest29 keddr",
|
||||
"mortal kombat 11 shang tsung",
|
||||
"asdasd",
|
||||
}
|
||||
if !Equal(normalizedTags, expected) {
|
||||
t.Error("result not as expected")
|
||||
return
|
||||
}
|
||||
|
||||
}
|
||||
func Equal(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
fmt.Printf("expected length %d but got %d", len(b), len(a))
|
||||
return false
|
||||
}
|
||||
for i, v := range a {
|
||||
if v != b[i] {
|
||||
fmt.Printf("expected %s but bot %s\n", b[i], v)
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
+22
-5
@@ -1,6 +1,7 @@
|
||||
package thumbs
|
||||
|
||||
import (
|
||||
"google.golang.org/api/youtube/v3"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
@@ -21,6 +22,7 @@ type thumbnailUploader struct {
|
||||
}
|
||||
|
||||
const thumbnailPath = "/tmp/ytsync_thumbnails/"
|
||||
const ThumbnailEndpoint = "https://thumbnails.lbry.com/"
|
||||
|
||||
func (u *thumbnailUploader) downloadThumbnail() error {
|
||||
_ = os.Mkdir(thumbnailPath, 0750)
|
||||
@@ -44,7 +46,7 @@ func (u *thumbnailUploader) downloadThumbnail() error {
|
||||
}
|
||||
|
||||
func (u *thumbnailUploader) uploadThumbnail() error {
|
||||
key := aws.String("/thumbnails/" + u.name)
|
||||
key := &u.name
|
||||
thumb, err := os.Open("/tmp/ytsync_thumbnails/" + u.name)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
@@ -59,11 +61,13 @@ func (u *thumbnailUploader) uploadThumbnail() error {
|
||||
uploader := s3manager.NewUploader(s3Session)
|
||||
|
||||
_, err = uploader.Upload(&s3manager.UploadInput{
|
||||
Bucket: aws.String("thumbnails.lbry.com"),
|
||||
Key: key,
|
||||
Body: thumb,
|
||||
Bucket: aws.String("thumbnails.lbry.com"),
|
||||
Key: key,
|
||||
Body: thumb,
|
||||
ACL: aws.String("public-read"),
|
||||
ContentType: aws.String("image/jpeg"),
|
||||
})
|
||||
u.mirroredUrl = "https://thumbnails.lbry.com/" + u.name
|
||||
u.mirroredUrl = ThumbnailEndpoint + u.name
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
@@ -92,3 +96,16 @@ func MirrorThumbnail(url string, name string, s3Config aws.Config) (string, erro
|
||||
|
||||
return tu.mirroredUrl, nil
|
||||
}
|
||||
|
||||
func GetBestThumbnail(thumbnails *youtube.ThumbnailDetails) *youtube.Thumbnail {
|
||||
if thumbnails.Maxres != nil {
|
||||
return thumbnails.Maxres
|
||||
} else if thumbnails.High != nil {
|
||||
return thumbnails.High
|
||||
} else if thumbnails.Medium != nil {
|
||||
return thumbnails.Medium
|
||||
} else if thumbnails.Standard != nil {
|
||||
return thumbnails.Standard
|
||||
}
|
||||
return thumbnails.Default
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/lbryio/lbry.go/extras/util"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// SendErrorToSlack Sends an error message to the default channel and to the process log.
|
||||
func SendErrorToSlack(format string, a ...interface{}) {
|
||||
message := format
|
||||
if len(a) > 0 {
|
||||
message = fmt.Sprintf(format, a...)
|
||||
}
|
||||
log.Errorln(message)
|
||||
err := util.SendToSlack(":sos: " + message)
|
||||
if err != nil {
|
||||
log.Errorln(err)
|
||||
}
|
||||
}
|
||||
|
||||
// SendInfoToSlack Sends an info message to the default channel and to the process log.
|
||||
func SendInfoToSlack(format string, a ...interface{}) {
|
||||
message := format
|
||||
if len(a) > 0 {
|
||||
message = fmt.Sprintf(format, a...)
|
||||
}
|
||||
log.Infoln(message)
|
||||
err := util.SendToSlack(":information_source: " + message)
|
||||
if err != nil {
|
||||
log.Errorln(err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user