Compare commits
91
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6c071ca9bb | ||
|
|
7bddcf01b8 | ||
|
|
659a6e73cc | ||
|
|
aaae3ffa5b | ||
|
|
131fed28d2 | ||
|
|
72571236ab | ||
|
|
560e180e36 | ||
|
|
070c378dfd | ||
|
|
124d4065c2 | ||
|
|
f131c1f35b | ||
|
|
7a3225434e | ||
|
|
3608971f0b | ||
|
|
c9fa04043c | ||
|
|
c6b53792c8 | ||
|
|
69fa06420b | ||
|
|
5cb1365903 | ||
|
|
e430c2fd40 | ||
|
|
66024716ac | ||
|
|
f043516a14 | ||
|
|
c3db95a6c1 | ||
|
|
08c93d44fd | ||
|
|
b02e80d472 | ||
|
|
f1875454cc | ||
|
|
a80e0f5b0d | ||
|
|
2b3581a692 | ||
|
|
a0f78028cc | ||
|
|
a084330055 | ||
|
|
ea80ed6506 | ||
|
|
4b335ed692 | ||
|
|
6118dde36c | ||
|
|
8364d3fc54 | ||
|
|
de0ccd4da7 | ||
|
|
31f9346027 | ||
|
|
5658fe4607 | ||
|
|
fc5f5ff7d3 | ||
|
|
e8b98bc862 | ||
|
|
f458529c74 | ||
|
|
150b1f6f1f | ||
|
|
47f28002ff | ||
|
|
beff466d18 | ||
|
|
694bda105c | ||
|
|
34ca7847d0 | ||
|
|
5c91051b78 | ||
|
|
df4f42db82 | ||
|
|
41d758ef5c | ||
|
|
264390a2b2 | ||
|
|
fdcc41829a | ||
|
|
09c7718f30 | ||
|
|
8a5f57b14f | ||
|
|
90997b9918 | ||
|
|
e0da2674a1 | ||
|
|
a80599413c | ||
|
|
3ffe7a10c7 | ||
|
|
1bf3cb81b3 | ||
|
|
4a9f127ecc | ||
|
|
de1fb63a1c | ||
|
|
e98794e125 | ||
|
|
fb0004bac4 | ||
|
|
1f12b1d5d8 | ||
|
|
bc95ca61d5 | ||
|
|
be69c2f05c | ||
|
|
4a902597df | ||
|
|
dd528f03f6 | ||
|
|
e6ba61fce2 | ||
|
|
e03abad012 | ||
|
|
dde93a1fe6 | ||
|
|
4a5a148843 | ||
|
|
cb6443fa1e | ||
|
|
1a1f991e48 | ||
|
|
d1063bd54e | ||
|
|
d291c063ec | ||
|
|
a939529d19 | ||
|
|
cf34547b8a | ||
|
|
6631ad325f | ||
|
|
df266f6194 | ||
|
|
b3b581c00e | ||
|
|
18b15674d9 | ||
|
|
780e899e90 | ||
|
|
5c5643749e | ||
|
|
11e50a6022 | ||
|
|
1603b3bb22 | ||
|
|
5465527faf | ||
|
|
5d8a2d697c | ||
|
|
50089481fb | ||
|
|
d6cf9e9e63 | ||
|
|
dc6dd8d12b | ||
|
|
ad8d623863 | ||
|
|
d89b58c52a | ||
|
|
2aa40850b0 | ||
|
|
ca24c2be28 | ||
|
|
52127eee7c |
No files matched your search
+2
-2
@@ -6,7 +6,7 @@ env:
|
||||
- GO111MODULE=on
|
||||
|
||||
go:
|
||||
- 1.13.x
|
||||
- 1.15.x
|
||||
|
||||
cache:
|
||||
directories:
|
||||
@@ -24,7 +24,7 @@ install: true
|
||||
# set -e enabled in bash.
|
||||
before_script:
|
||||
# All the .go files, excluding vendor/ and model (auto generated)
|
||||
- GO_FILES=$(find . -iname '*.go' ! -iname '*_test.go' -type f | grep -v /vendor/ )
|
||||
- GO_FILES=$(find . -iname '*.go' ! -iname '*_test.go' -type f | grep -v /vendor/ ) #i wish we were this crazy :p
|
||||
- go get golang.org/x/tools/cmd/goimports # Used in build script for generated files
|
||||
# - go get github.com/golang/lint/golint # Linter
|
||||
# - go get honnef.co/go/tools/cmd/megacheck # Badass static analyzer/linter
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
FROM alpine
|
||||
EXPOSE 8080
|
||||
|
||||
RUN mkdir /app
|
||||
WORKDIR /app
|
||||
COPY bin/prism-bin ./prism
|
||||
RUN chmod +x prism
|
||||
|
||||
ENTRYPOINT [ "/app/prism" ]
|
||||
@@ -1,6 +1,6 @@
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2016-2019 LBRY Inc
|
||||
Copyright (c) 2016-2020 LBRY Inc
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish,distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||
|
||||
|
||||
+1
-1
@@ -4,7 +4,7 @@ import (
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
|
||||
"github.com/lbryio/lbryschema.go/claim"
|
||||
"github.com/lbryio/lbry.go/v2/schema/claim"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/golang/protobuf/jsonpb"
|
||||
|
||||
+10
-3
@@ -3,6 +3,7 @@ package cmd
|
||||
import (
|
||||
"encoding/hex"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/peer"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
@@ -27,9 +28,10 @@ func getStreamCmd(cmd *cobra.Command, args []string) {
|
||||
addr := args[0]
|
||||
sdHash := args[1]
|
||||
|
||||
s := store.NewCachingBlobStore(
|
||||
s := store.NewCachingStore(
|
||||
"getstream",
|
||||
peer.NewStore(peer.StoreOpts{Address: addr}),
|
||||
store.NewDiskBlobStore("/tmp/lbry_downloaded_blobs", 2),
|
||||
store.NewDiskStore("/tmp/lbry_downloaded_blobs", 2),
|
||||
)
|
||||
|
||||
wd, err := os.Getwd()
|
||||
@@ -49,7 +51,12 @@ func getStreamCmd(cmd *cobra.Command, args []string) {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
f, err := os.Create(wd + "/" + sd.SuggestedFileName)
|
||||
filename := sd.SuggestedFileName
|
||||
if filename == "" {
|
||||
filename = "stream_" + time.Now().Format("20060102_150405")
|
||||
}
|
||||
|
||||
f, err := os.Create(wd + "/" + filename)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
+1
-1
@@ -29,7 +29,7 @@ func init() {
|
||||
func peerCmd(cmd *cobra.Command, args []string) {
|
||||
var err error
|
||||
|
||||
s3 := store.NewS3BlobStore(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
|
||||
s3 := store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
|
||||
peerServer := peer.NewServer(s3)
|
||||
|
||||
if !peerNoDB {
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/lbryio/reflector.go/publish"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/lbrycrd"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func init() {
|
||||
var cmd = &cobra.Command{
|
||||
Use: "publish FILE",
|
||||
Short: "Publish a file",
|
||||
Args: cobra.ExactArgs(1),
|
||||
Run: publishCmd,
|
||||
}
|
||||
cmd.Flags().String("name", "", "Claim name")
|
||||
cmd.Flags().String("title", "", "Title of the content")
|
||||
cmd.Flags().String("description", "", "Description of the content")
|
||||
cmd.Flags().String("author", "", "Content author")
|
||||
cmd.Flags().String("tags", "", "Comma-separated list of tags")
|
||||
cmd.Flags().Int64("release-time", 0, "original public release of content, seconds since UNIX epoch")
|
||||
rootCmd.AddCommand(cmd)
|
||||
}
|
||||
|
||||
func publishCmd(cmd *cobra.Command, args []string) {
|
||||
var err error
|
||||
|
||||
claimName := mustGetFlagString(cmd, "name")
|
||||
if claimName == "" {
|
||||
log.Errorln("--name required")
|
||||
return
|
||||
}
|
||||
|
||||
path := args[0]
|
||||
|
||||
client, err := lbrycrd.NewWithDefaultURL(nil)
|
||||
checkErr(err)
|
||||
|
||||
tx, txid, err := publish.Publish(
|
||||
client,
|
||||
path,
|
||||
claimName,
|
||||
"bSzpgkTnAoiT2YAhUShPpfpajPESfNXVTu",
|
||||
publish.Details{
|
||||
Title: mustGetFlagString(cmd, "title"),
|
||||
Description: mustGetFlagString(cmd, "description"),
|
||||
Author: mustGetFlagString(cmd, "author"),
|
||||
Tags: nil,
|
||||
ReleaseTime: mustGetFlagInt64(cmd, "release-time"),
|
||||
},
|
||||
"reflector.lbry.com:5566",
|
||||
)
|
||||
checkErr(err)
|
||||
|
||||
decoded, err := publish.Decode(client, tx)
|
||||
checkErr(err)
|
||||
|
||||
fmt.Printf("TX: %s\n\n", decoded)
|
||||
fmt.Printf("TXID: %s\n", txid.String())
|
||||
}
|
||||
+149
-38
@@ -4,81 +4,192 @@ import (
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
|
||||
"github.com/lbryio/reflector.go/db"
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
"github.com/lbryio/reflector.go/meta"
|
||||
"github.com/lbryio/reflector.go/peer"
|
||||
"github.com/lbryio/reflector.go/peer/http3"
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cast"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var (
|
||||
tcpPeerPort int
|
||||
http3PeerPort int
|
||||
receiverPort int
|
||||
metricsPort int
|
||||
disableUploads bool
|
||||
disableBlocklist bool
|
||||
proxyAddress string
|
||||
proxyPort string
|
||||
proxyProtocol string
|
||||
useDB bool
|
||||
cloudFrontEndpoint string
|
||||
reflectorCmdDiskCache string
|
||||
reflectorCmdMemCache int
|
||||
)
|
||||
|
||||
func init() {
|
||||
var cmd = &cobra.Command{
|
||||
Use: "reflector",
|
||||
Short: "Run reflector server",
|
||||
Run: reflectorCmd,
|
||||
}
|
||||
cmd.Flags().StringVar(&proxyAddress, "proxy-address", "", "address of another reflector server where blobs are fetched from")
|
||||
cmd.Flags().StringVar(&proxyPort, "proxy-port", "5567", "port of another reflector server where blobs are fetched from")
|
||||
cmd.Flags().StringVar(&proxyProtocol, "proxy-protocol", "http3", "protocol used to fetch blobs from another reflector server (tcp/http3)")
|
||||
cmd.Flags().StringVar(&cloudFrontEndpoint, "cloudfront-endpoint", "", "CloudFront edge endpoint for standard HTTP retrieval")
|
||||
cmd.Flags().IntVar(&tcpPeerPort, "tcp-peer-port", 5567, "The port reflector will distribute content from")
|
||||
cmd.Flags().IntVar(&http3PeerPort, "http3-peer-port", 5568, "The port reflector will distribute content from over HTTP3 protocol")
|
||||
cmd.Flags().IntVar(&receiverPort, "receiver-port", 5566, "The port reflector will receive content from")
|
||||
cmd.Flags().IntVar(&metricsPort, "metrics-port", 2112, "The port reflector will use for metrics")
|
||||
cmd.Flags().BoolVar(&disableUploads, "disable-uploads", false, "Disable uploads to this reflector server")
|
||||
cmd.Flags().BoolVar(&disableBlocklist, "disable-blocklist", false, "Disable blocklist watching/updating")
|
||||
cmd.Flags().BoolVar(&useDB, "use-db", true, "whether to connect to the reflector db or not")
|
||||
cmd.Flags().StringVar(&reflectorCmdDiskCache, "disk-cache", "",
|
||||
"enable disk cache, setting max size and path where to store blobs. format is 'MAX_BLOBS:CACHE_PATH'")
|
||||
cmd.Flags().IntVar(&reflectorCmdMemCache, "mem-cache", 0, "enable in-memory cache with a max size of this many blobs")
|
||||
rootCmd.AddCommand(cmd)
|
||||
}
|
||||
|
||||
func reflectorCmd(cmd *cobra.Command, args []string) {
|
||||
log.Printf("reflector version %s, built %s", meta.Version, meta.BuildTime.Format(time.RFC3339))
|
||||
log.Printf("reflector %s", meta.VersionString())
|
||||
|
||||
// flip this flag to false when doing db maintenance. uploads will not work (as reflector server wont be running)
|
||||
// but downloads will still work straight from s3
|
||||
useDB := true
|
||||
// the blocklist logic requires the db backed store to be the outer-most store
|
||||
underlyingStore := setupStore()
|
||||
outerStore := wrapWithCache(underlyingStore)
|
||||
|
||||
s3 := store.NewS3BlobStore(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
|
||||
|
||||
var err error
|
||||
|
||||
var blobStore store.BlobStore = s3
|
||||
var reflectorServer *reflector.Server
|
||||
|
||||
if useDB {
|
||||
db := new(db.SQL)
|
||||
err = db.Connect(globalConfig.DBConn)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
blobStore = store.NewDBBackedStore(s3, db)
|
||||
|
||||
reflectorServer = reflector.NewServer(blobStore)
|
||||
if !disableUploads {
|
||||
reflectorServer := reflector.NewServer(underlyingStore)
|
||||
reflectorServer.Timeout = 3 * time.Minute
|
||||
if globalConfig.SlackHookURL != "" {
|
||||
reflectorServer.StatLogger = log.StandardLogger()
|
||||
reflectorServer.StatReportFrequency = 1 * time.Hour
|
||||
}
|
||||
reflectorServer.EnableBlocklist = true
|
||||
reflectorServer.EnableBlocklist = !disableBlocklist
|
||||
|
||||
err = reflectorServer.Start(":" + strconv.Itoa(reflector.DefaultPort))
|
||||
err := reflectorServer.Start(":" + strconv.Itoa(receiverPort))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer reflectorServer.Shutdown()
|
||||
}
|
||||
|
||||
peerServer := peer.NewServer(blobStore)
|
||||
if globalConfig.SlackHookURL != "" {
|
||||
peerServer.StatLogger = log.StandardLogger()
|
||||
peerServer.StatReportFrequency = 1 * time.Hour
|
||||
}
|
||||
err = peerServer.Start(":5567")
|
||||
peerServer := peer.NewServer(outerStore)
|
||||
err := peerServer.Start(":" + strconv.Itoa(tcpPeerPort))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer peerServer.Shutdown()
|
||||
|
||||
http3PeerServer := http3.NewServer(outerStore)
|
||||
err = http3PeerServer.Start(":" + strconv.Itoa(http3PeerPort))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer http3PeerServer.Shutdown()
|
||||
|
||||
metricsServer := metrics.NewServer(":"+strconv.Itoa(metricsPort), "/metrics")
|
||||
metricsServer.Start()
|
||||
defer metricsServer.Shutdown()
|
||||
|
||||
interruptChan := make(chan os.Signal, 1)
|
||||
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
|
||||
<-interruptChan
|
||||
peerServer.Shutdown()
|
||||
if reflectorServer != nil {
|
||||
reflectorServer.Shutdown()
|
||||
// deferred shutdowns happen now
|
||||
}
|
||||
|
||||
func setupStore() store.BlobStore {
|
||||
var s store.BlobStore
|
||||
|
||||
if proxyAddress != "" {
|
||||
switch proxyProtocol {
|
||||
case "tcp":
|
||||
s = peer.NewStore(peer.StoreOpts{
|
||||
Address: proxyAddress + ":" + proxyPort,
|
||||
Timeout: 30 * time.Second,
|
||||
})
|
||||
case "http3":
|
||||
s = http3.NewStore(http3.StoreOpts{
|
||||
Address: proxyAddress + ":" + proxyPort,
|
||||
Timeout: 30 * time.Second,
|
||||
})
|
||||
default:
|
||||
log.Fatalf("protocol is not recognized: %s", proxyProtocol)
|
||||
}
|
||||
} else {
|
||||
s3Store := store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
|
||||
if cloudFrontEndpoint != "" {
|
||||
s = store.NewCloudFrontRWStore(store.NewCloudFrontROStore(cloudFrontEndpoint), s3Store)
|
||||
} else {
|
||||
s = s3Store
|
||||
}
|
||||
}
|
||||
|
||||
if useDB {
|
||||
db := new(db.SQL)
|
||||
db.TrackAccessTime = true
|
||||
err := db.Connect(globalConfig.DBConn)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
s = store.NewDBBackedStore(s, db)
|
||||
}
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
func wrapWithCache(s store.BlobStore) store.BlobStore {
|
||||
wrapped := s
|
||||
|
||||
diskCacheMaxSize, diskCachePath := diskCacheParams()
|
||||
if diskCacheMaxSize > 0 {
|
||||
err := os.MkdirAll(diskCachePath, os.ModePerm)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
wrapped = store.NewCachingStore(
|
||||
"reflector",
|
||||
wrapped,
|
||||
store.NewLRUStore("peer_server", store.NewDiskStore(diskCachePath, 2), diskCacheMaxSize),
|
||||
)
|
||||
}
|
||||
|
||||
if reflectorCmdMemCache > 0 {
|
||||
wrapped = store.NewCachingStore(
|
||||
"reflector",
|
||||
wrapped,
|
||||
store.NewLRUStore("peer_server", store.NewMemStore(), reflectorCmdMemCache),
|
||||
)
|
||||
}
|
||||
|
||||
return wrapped
|
||||
}
|
||||
|
||||
func diskCacheParams() (int, string) {
|
||||
if reflectorCmdDiskCache == "" {
|
||||
return 0, ""
|
||||
}
|
||||
|
||||
parts := strings.Split(reflectorCmdDiskCache, ":")
|
||||
if len(parts) != 2 {
|
||||
log.Fatalf("--disk-cache must be a number, followed by ':', followed by a string")
|
||||
}
|
||||
|
||||
maxSize := cast.ToInt(parts[0])
|
||||
if maxSize <= 0 {
|
||||
log.Fatalf("--disk-cache max size must be more than 0")
|
||||
}
|
||||
|
||||
path := parts[1]
|
||||
if len(path) == 0 || path[0] != '/' {
|
||||
log.Fatalf("--disk-cache path must start with '/'")
|
||||
}
|
||||
|
||||
return maxSize, path
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/lbryio/reflector.go/wallet"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func init() {
|
||||
var cmd = &cobra.Command{
|
||||
Use: "resolve ADDRESS:PORT URL",
|
||||
Short: "Resolve a URL",
|
||||
Args: cobra.ExactArgs(2),
|
||||
Run: resolveCmd,
|
||||
}
|
||||
rootCmd.AddCommand(cmd)
|
||||
}
|
||||
|
||||
func resolveCmd(cmd *cobra.Command, args []string) {
|
||||
addr := args[0]
|
||||
url := args[1]
|
||||
|
||||
node := wallet.NewNode()
|
||||
defer node.Shutdown()
|
||||
err := node.Connect([]string{addr}, nil)
|
||||
checkErr(err)
|
||||
|
||||
claim, _, err := node.ResolveToClaim(url)
|
||||
checkErr(err)
|
||||
|
||||
jsonClaim, err := json.MarshalIndent(claim, "", " ")
|
||||
checkErr(err)
|
||||
|
||||
fmt.Println(string(jsonClaim))
|
||||
}
|
||||
+19
-3
@@ -4,6 +4,7 @@ import (
|
||||
"encoding/json"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/dht"
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
@@ -49,7 +50,7 @@ var rootCmd = &cobra.Command{
|
||||
|
||||
func init() {
|
||||
rootCmd.PersistentFlags().StringSliceVarP(&verbose, "verbose", "v", []string{}, "Verbose logging for specific components")
|
||||
rootCmd.PersistentFlags().StringVar(&conf, "conf", "config.json", "Path to config")
|
||||
rootCmd.PersistentFlags().StringVar(&conf, "conf", "config.json", "Path to config. Use 'none' to disable")
|
||||
}
|
||||
|
||||
// Execute adds all child commands to the root command and sets flags appropriately.
|
||||
@@ -68,8 +69,11 @@ func preRun(cmd *cobra.Command, args []string) {
|
||||
debugLogger.SetOutput(os.Stderr)
|
||||
|
||||
if util.InSlice(verboseAll, verbose) {
|
||||
logrus.Info("global verbose logging enabled")
|
||||
logrus.SetLevel(logrus.DebugLevel)
|
||||
verbose = []string{verboseDHT, verboseNodeFinder}
|
||||
} else if len(verbose) > 0 {
|
||||
logrus.Infof("verbose logging enabled for: %s", strings.Join(verbose, ", "))
|
||||
}
|
||||
|
||||
for _, debugType := range verbose {
|
||||
@@ -141,9 +145,21 @@ func loadConfig(path string) (Config, error) {
|
||||
if os.IsNotExist(err) {
|
||||
return c, errors.Err("config file not found")
|
||||
}
|
||||
return c, err
|
||||
return c, errors.Err(err)
|
||||
}
|
||||
|
||||
err = json.Unmarshal(raw, &c)
|
||||
return c, err
|
||||
return c, errors.Err(err)
|
||||
}
|
||||
|
||||
func mustGetFlagString(cmd *cobra.Command, name string) string {
|
||||
v, err := cmd.Flags().GetString(name)
|
||||
checkErr(err)
|
||||
return v
|
||||
}
|
||||
|
||||
func mustGetFlagInt64(cmd *cobra.Command, name string) int64 {
|
||||
v, err := cmd.Flags().GetInt64(name)
|
||||
checkErr(err)
|
||||
return v
|
||||
}
|
||||
+38
-9
@@ -2,6 +2,8 @@ package cmd
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
|
||||
@@ -13,9 +15,9 @@ import (
|
||||
|
||||
func init() {
|
||||
var cmd = &cobra.Command{
|
||||
Use: "sendblob ADDRESS:PORT",
|
||||
Use: "sendblob ADDRESS:PORT [PATH]",
|
||||
Short: "Send a random blob to a reflector server",
|
||||
Args: cobra.ExactArgs(1),
|
||||
Args: cobra.RangeArgs(1, 2),
|
||||
Run: sendBlobCmd,
|
||||
}
|
||||
rootCmd.AddCommand(cmd)
|
||||
@@ -23,6 +25,10 @@ func init() {
|
||||
|
||||
func sendBlobCmd(cmd *cobra.Command, args []string) {
|
||||
addr := args[0]
|
||||
var path string
|
||||
if len(args) >= 2 {
|
||||
path = args[1]
|
||||
}
|
||||
|
||||
c := reflector.Client{}
|
||||
err := c.Connect(addr)
|
||||
@@ -30,14 +36,37 @@ func sendBlobCmd(cmd *cobra.Command, args []string) {
|
||||
log.Fatal("error connecting client to server: ", err)
|
||||
}
|
||||
|
||||
blob := make(stream.Blob, 1024)
|
||||
_, err = rand.Read(blob)
|
||||
if err != nil {
|
||||
log.Fatal("failed to make random blob: ", err)
|
||||
if path == "" {
|
||||
blob := make(stream.Blob, 1024)
|
||||
_, err = rand.Read(blob)
|
||||
if err != nil {
|
||||
log.Fatal("failed to make random blob: ", err)
|
||||
}
|
||||
|
||||
err = c.SendBlob(blob)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
err = c.SendBlob(blob)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
file, err := os.Open(path)
|
||||
checkErr(err)
|
||||
data, err := ioutil.ReadAll(file)
|
||||
checkErr(err)
|
||||
s, err := stream.New(data)
|
||||
checkErr(err)
|
||||
|
||||
sdBlob := &stream.SDBlob{}
|
||||
err = sdBlob.FromBlob(s[0])
|
||||
checkErr(err)
|
||||
|
||||
for i, b := range s {
|
||||
if i == 0 {
|
||||
err = c.SendSDBlob(b)
|
||||
} else {
|
||||
err = c.SendBlob(b)
|
||||
}
|
||||
checkErr(err)
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -55,7 +55,7 @@ func startCmd(cmd *cobra.Command, args []string) {
|
||||
db := new(db.SQL)
|
||||
err := db.Connect(globalConfig.DBConn)
|
||||
checkErr(err)
|
||||
s3 := store.NewS3BlobStore(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
|
||||
s3 := store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
|
||||
comboStore := store.NewDBBackedStore(s3, db)
|
||||
|
||||
conf := prism.DefaultConf()
|
||||
|
||||
+2
-2
@@ -27,9 +27,9 @@ func init() {
|
||||
}
|
||||
|
||||
func testCmd(cmd *cobra.Command, args []string) {
|
||||
log.Printf("reflector version %s", meta.Version)
|
||||
log.Printf("reflector %s", meta.VersionString())
|
||||
|
||||
memStore := store.NewMemoryBlobStore()
|
||||
memStore := store.NewMemStore()
|
||||
|
||||
reflectorServer := reflector.NewServer(memStore)
|
||||
reflectorServer.Timeout = 3 * time.Minute
|
||||
|
||||
+4
-2
@@ -14,6 +14,7 @@ import (
|
||||
|
||||
var uploadWorkers int
|
||||
var uploadSkipExistsCheck bool
|
||||
var uploadDeleteBlobsAfterUpload bool
|
||||
|
||||
func init() {
|
||||
var cmd = &cobra.Command{
|
||||
@@ -24,6 +25,7 @@ func init() {
|
||||
}
|
||||
cmd.PersistentFlags().IntVar(&uploadWorkers, "workers", 1, "How many worker threads to run at once")
|
||||
cmd.PersistentFlags().BoolVar(&uploadSkipExistsCheck, "skipExistsCheck", false, "Dont check if blobs exist before uploading")
|
||||
cmd.PersistentFlags().BoolVar(&uploadDeleteBlobsAfterUpload, "deleteBlobsAfterUpload", false, "Delete blobs after uploading them")
|
||||
rootCmd.AddCommand(cmd)
|
||||
}
|
||||
|
||||
@@ -33,10 +35,10 @@ func uploadCmd(cmd *cobra.Command, args []string) {
|
||||
checkErr(err)
|
||||
|
||||
st := store.NewDBBackedStore(
|
||||
store.NewS3BlobStore(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName),
|
||||
store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName),
|
||||
db)
|
||||
|
||||
uploader := reflector.NewUploader(db, st, uploadWorkers, uploadSkipExistsCheck)
|
||||
uploader := reflector.NewUploader(db, st, uploadWorkers, uploadSkipExistsCheck, uploadDeleteBlobsAfterUpload)
|
||||
|
||||
interruptChan := make(chan os.Signal, 1)
|
||||
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
|
||||
|
||||
+1
-2
@@ -2,7 +2,6 @@ package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/meta"
|
||||
"github.com/spf13/cobra"
|
||||
@@ -18,5 +17,5 @@ func init() {
|
||||
}
|
||||
|
||||
func versionCmd(cmd *cobra.Command, args []string) {
|
||||
fmt.Printf("version %s (built %s)\n", meta.Version, meta.BuildTime.Format(time.RFC3339))
|
||||
fmt.Println(meta.VersionString())
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/go-sql-driver/mysql"
|
||||
_ "github.com/go-sql-driver/mysql" // blank import for db driver ensures its imported even if its not used
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/volatiletech/null"
|
||||
)
|
||||
|
||||
// SdBlob is a special blob that contains information on the rest of the blobs in the stream
|
||||
@@ -32,6 +33,8 @@ type SdBlob struct {
|
||||
// SQL implements the DB interface
|
||||
type SQL struct {
|
||||
conn *sql.DB
|
||||
|
||||
TrackAccessTime bool
|
||||
}
|
||||
|
||||
func logQuery(query string, args ...interface{}) {
|
||||
@@ -75,9 +78,10 @@ func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error)
|
||||
return 0, errors.Err("length must be positive")
|
||||
}
|
||||
|
||||
args := []interface{}{hash, isStored, length}
|
||||
blobID, err := s.exec(
|
||||
"INSERT INTO blob_ (hash, is_stored, length) VALUES (?,?,?) ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored))",
|
||||
hash, isStored, length,
|
||||
"INSERT INTO blob_ (hash, is_stored, length) VALUES ("+qt.Qs(len(args))+") ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored))",
|
||||
args...,
|
||||
)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
@@ -97,9 +101,10 @@ func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error)
|
||||
}
|
||||
|
||||
func (s *SQL) insertStream(hash string, sdBlobID int64) (int64, error) {
|
||||
args := []interface{}{hash, sdBlobID, time.Now()}
|
||||
streamID, err := s.exec(
|
||||
"INSERT IGNORE INTO stream (hash, sd_blob_id) VALUES (?,?)",
|
||||
hash, sdBlobID,
|
||||
"INSERT IGNORE INTO stream (hash, sd_blob_id, last_accessed_at) VALUES ("+qt.Qs(len(args))+")",
|
||||
args...,
|
||||
)
|
||||
if err != nil {
|
||||
return 0, errors.Err(err)
|
||||
@@ -113,6 +118,13 @@ func (s *SQL) insertStream(hash string, sdBlobID int64) (int64, error) {
|
||||
if streamID == 0 {
|
||||
return 0, errors.Err("stream ID is 0 even after INSERTing and SELECTing")
|
||||
}
|
||||
|
||||
if s.TrackAccessTime {
|
||||
err := s.touch([]uint64{uint64(streamID)})
|
||||
if err != nil {
|
||||
return 0, errors.Err(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return streamID, nil
|
||||
}
|
||||
@@ -128,12 +140,44 @@ func (s *SQL) HasBlob(hash string) (bool, error) {
|
||||
|
||||
// HasBlobs checks if the database contains the set of blobs and returns a bool map.
|
||||
func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
|
||||
if s.conn == nil {
|
||||
return nil, errors.Err("not connected")
|
||||
exists, streamsNeedingTouch, err := s.hasBlobs(hashes)
|
||||
s.touch(streamsNeedingTouch)
|
||||
return exists, err
|
||||
}
|
||||
|
||||
func (s *SQL) touch(streamIDs []uint64) error {
|
||||
if len(streamIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var hash string
|
||||
query := "UPDATE stream SET last_accessed_at = ? WHERE id IN (" + qt.Qs(len(streamIDs)) + ")"
|
||||
args := make([]interface{}, len(streamIDs)+1)
|
||||
args[0] = time.Now()
|
||||
for i := range streamIDs {
|
||||
args[i+1] = streamIDs[i]
|
||||
}
|
||||
|
||||
startTime := time.Now()
|
||||
_, err := s.exec(query, args...)
|
||||
log.Debugf("stream access query touched %d streams and took %s", len(streamIDs), time.Since(startTime))
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
func (s *SQL) hasBlobs(hashes []string) (map[string]bool, []uint64, error) {
|
||||
if s.conn == nil {
|
||||
return nil, nil, errors.Err("not connected")
|
||||
}
|
||||
|
||||
var (
|
||||
hash string
|
||||
streamID uint64
|
||||
lastAccessedAt null.Time
|
||||
)
|
||||
|
||||
var needsTouch []uint64
|
||||
exists := make(map[string]bool)
|
||||
|
||||
touchDeadline := time.Now().AddDate(0, 0, -1) // touch blob if last accessed before this time
|
||||
maxBatchSize := 10000
|
||||
doneIndex := 0
|
||||
|
||||
@@ -145,7 +189,13 @@ func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
|
||||
log.Debugf("getting hashes[%d:%d] of %d", doneIndex, sliceEnd, len(hashes))
|
||||
batch := hashes[doneIndex:sliceEnd]
|
||||
|
||||
query := "SELECT hash FROM blob_ WHERE is_stored = ? && hash IN (" + qt.Qs(len(batch)) + ")"
|
||||
// TODO: this query doesn't work for SD blobs, which are not in the stream_blob table
|
||||
|
||||
query := `SELECT b.hash, s.id, s.last_accessed_at
|
||||
FROM blob_ b
|
||||
LEFT JOIN stream_blob sb ON b.id = sb.blob_id
|
||||
INNER JOIN stream s on (sb.stream_id = s.id or s.sd_blob_id = b.id)
|
||||
WHERE b.is_stored = ? and b.hash IN (` + qt.Qs(len(batch)) + `)`
|
||||
args := make([]interface{}, len(batch)+1)
|
||||
args[0] = true
|
||||
for i := range batch {
|
||||
@@ -164,11 +214,14 @@ func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
|
||||
defer closeRows(rows)
|
||||
|
||||
for rows.Next() {
|
||||
err := rows.Scan(&hash)
|
||||
err := rows.Scan(&hash, &streamID, &lastAccessedAt)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
exists[hash] = true
|
||||
if s.TrackAccessTime && (!lastAccessedAt.Valid || lastAccessedAt.Time.Before(touchDeadline)) {
|
||||
needsTouch = append(needsTouch, streamID)
|
||||
}
|
||||
}
|
||||
|
||||
err = rows.Err()
|
||||
@@ -180,11 +233,11 @@ func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
|
||||
return nil
|
||||
}()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return exists, nil
|
||||
return exists, needsTouch, nil
|
||||
}
|
||||
|
||||
// Delete will remove the blob from the db
|
||||
@@ -309,9 +362,10 @@ func (s *SQL) AddSDBlob(sdHash string, sdBlobLength int, sdBlob SdBlob) error {
|
||||
return err
|
||||
}
|
||||
|
||||
args := []interface{}{streamID, blobID, contentBlob.BlobNum}
|
||||
_, err = s.exec(
|
||||
"INSERT IGNORE INTO stream_blob (stream_id, blob_id, num) VALUES (?,?,?)",
|
||||
streamID, blobID, contentBlob.BlobNum,
|
||||
"INSERT IGNORE INTO stream_blob (stream_id, blob_id, num) VALUES ("+qt.Qs(len(args))+")",
|
||||
args...,
|
||||
)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
@@ -482,9 +536,11 @@ CREATE TABLE stream (
|
||||
id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT UNIQUE,
|
||||
hash char(96) NOT NULL,
|
||||
sd_blob_id BIGINT UNSIGNED NOT NULL,
|
||||
last_accessed_at TIMESTAMP NULL DEFAULT NULL,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY stream_hash_idx (hash),
|
||||
KEY stream_sd_blob_id_idx (sd_blob_id),
|
||||
KEY last_accessed_at_idx (last_accessed_at),
|
||||
FOREIGN KEY (sd_blob_id) REFERENCES blob_ (id) ON DELETE RESTRICT ON UPDATE CASCADE
|
||||
);
|
||||
|
||||
|
||||
@@ -1,31 +1,46 @@
|
||||
module github.com/lbryio/reflector.go
|
||||
|
||||
replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19
|
||||
|
||||
require (
|
||||
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
|
||||
github.com/aws/aws-sdk-go v1.16.11
|
||||
github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3 // indirect
|
||||
github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3
|
||||
github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d
|
||||
github.com/davecgh/go-spew v1.1.1
|
||||
github.com/go-sql-driver/mysql v0.0.0-20180719071942-99ff426eb706
|
||||
github.com/golang/protobuf v1.3.2
|
||||
github.com/go-sql-driver/mysql v1.4.1
|
||||
github.com/golang/protobuf v1.4.2
|
||||
github.com/google/btree v1.0.0 // indirect
|
||||
github.com/google/gops v0.3.7
|
||||
github.com/gorilla/mux v1.7.4
|
||||
github.com/hashicorp/go-msgpack v0.5.5 // indirect
|
||||
github.com/hashicorp/golang-lru v0.5.3 // indirect
|
||||
github.com/hashicorp/golang-lru v0.5.4
|
||||
github.com/hashicorp/memberlist v0.1.4 // indirect
|
||||
github.com/hashicorp/serf v0.8.2
|
||||
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf
|
||||
github.com/inconshreveable/mousetrap v1.0.0 // indirect
|
||||
github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22 // indirect
|
||||
github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07
|
||||
github.com/lbryio/lbry.go v1.1.2
|
||||
github.com/lbryio/lbry.go/v2 v2.4.0
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46
|
||||
github.com/lbryio/types v0.0.0-20191009145016-1bb8107e04f8
|
||||
github.com/karrick/godirwalk v1.16.1
|
||||
github.com/lbryio/chainquery v1.9.0
|
||||
github.com/lbryio/lbry.go v1.1.2 // indirect
|
||||
github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128
|
||||
github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec
|
||||
github.com/lucas-clemente/quic-go v0.18.1
|
||||
github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6
|
||||
github.com/prometheus/client_golang v0.9.2
|
||||
github.com/sirupsen/logrus v1.4.2
|
||||
github.com/spf13/cobra v0.0.0-20180722215644-7c4570c3ebeb
|
||||
github.com/spf13/afero v1.4.1
|
||||
github.com/spf13/cast v1.3.0
|
||||
github.com/spf13/cobra v0.0.3
|
||||
github.com/spf13/pflag v1.0.3 // indirect
|
||||
github.com/uber-go/atomic v1.4.0
|
||||
github.com/stretchr/testify v1.4.0
|
||||
github.com/volatiletech/null v8.0.0+incompatible
|
||||
go.uber.org/atomic v1.5.1
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
|
||||
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
|
||||
golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4 // indirect
|
||||
google.golang.org/appengine v1.6.2 // indirect
|
||||
gotest.tools v2.2.0+incompatible
|
||||
)
|
||||
|
||||
go 1.13
|
||||
go 1.15
|
||||
@@ -1,7 +1,18 @@
|
||||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo=
|
||||
dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU=
|
||||
dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU=
|
||||
dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4=
|
||||
dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU=
|
||||
git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
|
||||
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
|
||||
github.com/StackExchange/wmi v0.0.0-20170410192909-ea383cf3ba6e/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
|
||||
github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
|
||||
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
|
||||
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
|
||||
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
|
||||
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM=
|
||||
@@ -11,73 +22,115 @@ github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:l
|
||||
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
|
||||
github.com/aws/aws-sdk-go v1.16.11 h1:g/c7gJeVyHoXCxM2fddS85bPGVkBF8s2q8t3fyElegc=
|
||||
github.com/aws/aws-sdk-go v1.16.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
|
||||
github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8=
|
||||
github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3 h1:A/EVblehb75cUgXA5njHPn0kLAsykn6mJGz7rnmW5W0=
|
||||
github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI=
|
||||
github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
|
||||
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo=
|
||||
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
|
||||
github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803 h1:j3AgPKKZtZStM2nyhrDSLSYgT7YHrZKdSkq1OYeLjvM=
|
||||
github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
|
||||
github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d h1:yJzD/yFppdVCf6ApMkVy8cUxV0XrxdP9rVf6D87/Mng=
|
||||
github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
|
||||
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd h1:R/opQEbFEy9JGkIguV40SvRY1uliPX8ifOvi6ICsFCw=
|
||||
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg=
|
||||
github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY=
|
||||
github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
|
||||
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3SkEwmHoWBmX1DNXhXZqlTpq6s4tyJGc=
|
||||
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
|
||||
github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
|
||||
github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
|
||||
github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
|
||||
github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
|
||||
github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
|
||||
github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
|
||||
github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
|
||||
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
|
||||
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
|
||||
github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
|
||||
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
|
||||
github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk=
|
||||
github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
|
||||
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
|
||||
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
|
||||
github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
|
||||
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
|
||||
github.com/go-ini/ini v1.38.2/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-ini/ini v1.48.0 h1:TvO60hO/2xgaaTWp2P0wUe4CFxwdMzfbkv3+343Xzqw=
|
||||
github.com/go-ini/ini v1.48.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
|
||||
github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
|
||||
github.com/go-ozzo/ozzo-validation v3.6.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
|
||||
github.com/go-sql-driver/mysql v0.0.0-20180719071942-99ff426eb706 h1:P3NPKb7qq581SeMCB+dU1SuCX1kQh8VoQ/4HmT2ftQY=
|
||||
github.com/go-sql-driver/mysql v0.0.0-20180719071942-99ff426eb706/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
||||
github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
|
||||
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
||||
github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE=
|
||||
github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
|
||||
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
|
||||
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.4.0 h1:Rd1kQnQu0Hq3qvJppYSG0HtP+f5LPPUiDswTLiEegLg=
|
||||
github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
|
||||
github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc=
|
||||
github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
|
||||
github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
|
||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
|
||||
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw=
|
||||
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
|
||||
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
|
||||
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
|
||||
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
|
||||
github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ=
|
||||
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
|
||||
github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
|
||||
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
|
||||
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
|
||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
|
||||
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
|
||||
github.com/google/gops v0.3.7 h1:KtVAagOM0FIq+02DiQrKBTnLhYpWBMowaufcj+W1Exw=
|
||||
github.com/google/gops v0.3.7/go.mod h1:bj0cwMmX1X4XIJFTjR99R5sCxNssNJ8HebFNvoQlmgY=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
|
||||
github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f h1:TyqzGm2z1h3AGhjOoRYyeLcW4WlW81MDQkWa+rx/000=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8=
|
||||
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
|
||||
github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk=
|
||||
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
|
||||
github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
|
||||
github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
|
||||
github.com/gorilla/rpc v1.1.0 h1:marKfvVP0Gpd/jHlVBKCQ8RAoUPdX7K1Nuh6l1BNh7A=
|
||||
github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc=
|
||||
github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
|
||||
github.com/gorilla/rpc v1.1.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ=
|
||||
github.com/gorilla/rpc v1.2.0 h1:WvvdC2lNeT1SP32zrIce5l0ECBfbAlmrmSBsuc57wfk=
|
||||
github.com/gorilla/rpc v1.2.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ=
|
||||
github.com/gorilla/websocket v1.2.0 h1:VJtLvh6VQym50czpZzx07z/kw9EgAxI3x1ZB8taTMQQ=
|
||||
github.com/gorilla/schema v1.0.2/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU=
|
||||
github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
|
||||
github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM=
|
||||
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
|
||||
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
|
||||
@@ -96,10 +149,13 @@ github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b
|
||||
github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE=
|
||||
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
|
||||
github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk=
|
||||
github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
|
||||
github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
|
||||
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
|
||||
github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce h1:xdsDDbiBDQTKASoGEZ+pEmF1OnWuu8AQ9I8iNbHNeno=
|
||||
github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
|
||||
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
|
||||
github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
|
||||
github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
|
||||
@@ -107,61 +163,87 @@ github.com/hashicorp/memberlist v0.1.4 h1:gkyML/r71w3FL8gUi74Vk76avkj/9lYAY9lvg0
|
||||
github.com/hashicorp/memberlist v0.1.4/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
|
||||
github.com/hashicorp/serf v0.8.2 h1:YZ7UKsJv+hKjqGVUUbtE3HNj79Eln2oQ75tniF6iPt0=
|
||||
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
|
||||
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf h1:WfD7VjIE6z8dIvMsI4/s+1qr5EL+zoIGev1BQj1eoJ8=
|
||||
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h1:hyb9oH7vZsitZCiBt0ZvifOrB+qc8PS5IiilCIb87rg=
|
||||
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
|
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
|
||||
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
|
||||
github.com/jmoiron/sqlx v0.0.0-20170430194603-d9bd385d68c0/go.mod h1:IiEW3SEiiErVyFdH8NTuWjSifiEQKUoyK3LNqr2kCHU=
|
||||
github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22 h1:jKUP9TQ0c7X3w6+IPyMit07RE42MtTWNd77sN2cHngQ=
|
||||
github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22/go.mod h1:u0Jo4f2dNlTJeeOywkM6bLwxq6gC3pZ9rEFHn3AhTdk=
|
||||
github.com/johntdyer/slackrus v0.0.0-20170926115001-3992f319fd0a/go.mod h1:j1kV/8f3jowErEq4XyeypkCdvg5EeHkf0YCKCcq5Ybo=
|
||||
github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07 h1:+kBG/8rjCa6vxJZbUjAiE4MQmBEBYc8nLEb51frnvBY=
|
||||
github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07/go.mod h1:j1kV/8f3jowErEq4XyeypkCdvg5EeHkf0YCKCcq5Ybo=
|
||||
github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
|
||||
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
github.com/jteeuwen/go-bindata v3.0.7+incompatible/go.mod h1:JVvhzYOiGBnFSYRyV00iY8q7/0PThjIYav1p9h5dmKs=
|
||||
github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
|
||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1 h1:PJPDf8OUfOK1bb/NeTKd4f1QXZItOX389VN3B6qC8ro=
|
||||
github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
|
||||
github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw=
|
||||
github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
|
||||
github.com/keybase/go-ps v0.0.0-20161005175911-668c8856d999/go.mod h1:hY+WOq6m2FpbvyrI93sMaypsttvaIL5nhVR92dTMUcQ=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
|
||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M=
|
||||
github.com/lbryio/chainquery v1.9.0 h1:NfBZ3eKYwD3PqXU/vt+2tF3ox3WUWoW4J5YdEQ0rxw0=
|
||||
github.com/lbryio/chainquery v1.9.0/go.mod h1:7G8l7jNtANS1I7fQOvtzbiHsv6qKVmN4codXHc3C4kk=
|
||||
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8=
|
||||
github.com/lbryio/lbry.go v0.0.0-20190910204236-fd916d9eae76 h1:qH7uGPAFDnvNFC2lkS2AWwcmIyKInvW53AjL3Rx+JIQ=
|
||||
github.com/lbryio/lbry.go v0.0.0-20190910204236-fd916d9eae76/go.mod h1:qR+Ui0hYhemIU4fXqM3d1P9eiaRFlof777VJgV7KJ8w=
|
||||
github.com/lbryio/lbry.go v1.1.1-0.20190825202001-8fa28d3d656f h1:ovd2wPXzkT80vdP/FX5xcQeXu0i9RAo80SQ6qIsrAjM=
|
||||
github.com/lbryio/lbry.go v1.1.1-0.20190825202001-8fa28d3d656f/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
|
||||
github.com/lbryio/lbry.go v1.1.2 h1:Dyxc+glT/rVWJwHfIf7vjlPYYbjzrQz5ARmJd5Hp69c=
|
||||
github.com/lbryio/lbry.go v1.1.2/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
|
||||
github.com/lbryio/lbry.go/v2 v2.3.1 h1:+pe+gzdA2QYjYtdlRhoQfmRyCTe0SEsWesGgm4aw35k=
|
||||
github.com/lbryio/lbry.go/v2 v2.3.1/go.mod h1:YojiYMSD1a3HrbjoivB8In2kCyeeCkuH092t7dwlZc4=
|
||||
github.com/lbryio/lbry.go/v2 v2.4.0 h1:DVswvpn5ObG//54RoeQ/b/ZdQ4ErD6ciB7yIXt+vdwI=
|
||||
github.com/lbryio/lbry.go/v2 v2.4.0/go.mod h1:0T0rndOsKQtl18PlFD2ohgX16RCNianUgYB136sAqiE=
|
||||
github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128 h1:VL209c+AGKixMFpxT+TsOAPzBPuoUzyjXf47iNe7OzY=
|
||||
github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128/go.mod h1:RqOv4V5eWY/JGmduCPcQVdN19SEYnNY3SuF+arTKIU4=
|
||||
github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19 h1:/zWD8dVIl7bV1TdJWqPqy9tpqixzX2Qxgit48h3hQcY=
|
||||
github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46 h1:LemfR+rMxhf7nnOrzy2HqS7Me7SZ5gEwOcNFzKC8ySQ=
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
|
||||
github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4=
|
||||
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c h1:m3O7561xBQ00lfUVayW4c6SnpVbUDQtPUwGcGYSUYQA=
|
||||
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
|
||||
github.com/lbryio/types v0.0.0-20190715201353-fe180635eafe h1:6zNXyp0TtcCWcpV0yNVFgWkaVbLsjqnhKoTlnb56hM0=
|
||||
github.com/lbryio/types v0.0.0-20190715201353-fe180635eafe/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
|
||||
github.com/lbryio/types v0.0.0-20191009145016-1bb8107e04f8 h1:jSNW/rK6DQsz7Zh+iv1zR384PeQdHt0gS4hKY17tkuM=
|
||||
github.com/lbryio/types v0.0.0-20191009145016-1bb8107e04f8/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
|
||||
github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5 h1:AsEBgzv3DhuYHI/GiQh2HxvTP71HCCE9E/tzGUzGdtU=
|
||||
github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec h1:2xk/qg4VTOCJ8RzV/ED5AKqDcJ00zVb08ltf9V+sr3c=
|
||||
github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
|
||||
github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
github.com/lucas-clemente/quic-go v0.18.1 h1:DMR7guC0NtVS8zNZR3IO7NARZvZygkSC56GGtC6cyys=
|
||||
github.com/lucas-clemente/quic-go v0.18.1/go.mod h1:yXttHsSNxQi8AWijC/vLP+OJczXqzHSOcJrM5ITUlCg=
|
||||
github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
|
||||
github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0=
|
||||
github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6 h1:iOAVXzZyXtW408TMYejlUPo6BIn92HmOacWtIfNyYns=
|
||||
github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg=
|
||||
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 h1:mG83tLXWSRdcXMWfkoumVwhcCbf3jHF9QKv/m37BkM0=
|
||||
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCWffGOaDcjkw1iB7W9DVLp6GXmfcJY/7YZCWPA4=
|
||||
github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
|
||||
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
|
||||
github.com/marten-seemann/qpack v0.2.0 h1:/r1rhZoOmgxVKBqPNnYilZBDEyw+6OUHCbBzA5jc2y0=
|
||||
github.com/marten-seemann/qpack v0.2.0/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc=
|
||||
github.com/marten-seemann/qtls v0.10.0 h1:ECsuYUKalRL240rRD4Ri33ISb7kAQ3qGDlrrl55b2pc=
|
||||
github.com/marten-seemann/qtls v0.10.0/go.mod h1:UvMd1oaYDACI99/oZUYLzMCkBXQVT0aGm99sJhbT8hs=
|
||||
github.com/marten-seemann/qtls-go1-15 v0.1.0 h1:i/YPXVxz8q9umso/5y474CNcHmTpA+5DH+mFPjx6PZg=
|
||||
github.com/marten-seemann/qtls-go1-15 v0.1.0/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I=
|
||||
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
|
||||
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
|
||||
github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA=
|
||||
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
|
||||
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
|
||||
@@ -169,165 +251,319 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4
|
||||
github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
|
||||
github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/mitchellh/mapstructure v0.0.0-20180511142126-bb74f1db0675/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
|
||||
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/nlopes/slack v0.5.0 h1:NbIae8Kd0NpqaEI3iUrsuS0KbcEDhzhc939jLW5fNm0=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
|
||||
github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
|
||||
github.com/nlopes/slack v0.5.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM=
|
||||
github.com/nlopes/slack v0.6.0 h1:jt0jxVQGhssx1Ib7naAOZEZcGdtIhTzkP0nopK0AsRA=
|
||||
github.com/nlopes/slack v0.6.0/go.mod h1:JzQ9m3PMAqcpeCam7UaHSuBuupz7CmpjehYMayT6YOk=
|
||||
github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
|
||||
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
|
||||
github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA=
|
||||
github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
|
||||
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
|
||||
github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
|
||||
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
|
||||
github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
|
||||
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
||||
github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
|
||||
github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
||||
github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
|
||||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||
github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6 h1:2bae6N0SZjgzk+Zg8mzTsfmpwHXY9VBNp9UdjhaElA0=
|
||||
github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/profile v1.3.0/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
|
||||
github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
|
||||
github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740=
|
||||
github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
|
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8=
|
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
||||
github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
|
||||
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8=
|
||||
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
|
||||
github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE=
|
||||
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/rubenv/sql-migrate v0.0.0-20170330050058-38004e7a77f2/go.mod h1:WS0rl9eEliYI8DPnr3TOwz4439pay+qNgzJoVya/DmY=
|
||||
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
|
||||
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
|
||||
github.com/sebdah/goldie v0.0.0-20180424091453-8784dd1ab561 h1:IY+sDBJR/wRtsxq+626xJnt4Tw7/ROA9cDIR8MMhWyg=
|
||||
github.com/sebdah/goldie v0.0.0-20180424091453-8784dd1ab561/go.mod h1:lvjGftC8oe7XPtyrOidaMi0rp5B9+XY/ZRUynGnuaxQ=
|
||||
github.com/sebdah/goldie v0.0.0-20190531093107-d313ffb52c77 h1:Msb6XRY62jQOueNNlB5LGin1rljK7c49NLniGwJG2bg=
|
||||
github.com/sebdah/goldie v0.0.0-20190531093107-d313ffb52c77/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4=
|
||||
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/sfreiberg/gotwilio v0.0.0-20180612161623-8fb7259ba8bf/go.mod h1:60PiR0SAnAcYSiwrXB6BaxeqHdXMf172toCosHfV+Yk=
|
||||
github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
|
||||
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
|
||||
github.com/shopspring/decimal v0.0.0-20180607144847-19e3cb6c2930/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
|
||||
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
|
||||
github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5 h1:Gojs/hac/DoYEM7WEICT45+hNWczIeuL5D21e5/HPAw=
|
||||
github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
|
||||
github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY=
|
||||
github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM=
|
||||
github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0=
|
||||
github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
|
||||
github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
|
||||
github.com/shurcooL/gofontwoff v0.0.0-20180329035133-29b52fc0a18d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw=
|
||||
github.com/shurcooL/gopherjslib v0.0.0-20160914041154-feb6d3990c2c/go.mod h1:8d3azKNyqcHP1GaQE/c6dDgjkgSx2BZ4IoEi4F1reUI=
|
||||
github.com/shurcooL/highlight_diff v0.0.0-20170515013008-09bb4053de1b/go.mod h1:ZpfEhSmds4ytuByIcDnOLkTHGUI6KNqRNPDLHDk+mUU=
|
||||
github.com/shurcooL/highlight_go v0.0.0-20181028180052-98c3abbbae20/go.mod h1:UDKB5a1T23gOMUJrI+uSuH0VRDStOiUVSjBTRDVBVag=
|
||||
github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg=
|
||||
github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw=
|
||||
github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y=
|
||||
github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg=
|
||||
github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q=
|
||||
github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ=
|
||||
github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I=
|
||||
github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b5uSkrEVM1jQUspwbixRBhaIjIzL2xazXp6kntxYle0=
|
||||
github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ=
|
||||
github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk=
|
||||
github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
|
||||
github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4=
|
||||
github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w=
|
||||
github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
|
||||
github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
|
||||
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8=
|
||||
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||
github.com/spf13/cast v1.2.0 h1:HHl1DSRbEQN2i8tJmtS6ViPyHx35+p51amrdsiTCrkg=
|
||||
github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE=
|
||||
github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA=
|
||||
github.com/spf13/afero v1.1.1 h1:Lt3ihYMlE+lreX1GS4Qw4ZsNpYQLxIXKBTEOXm3nt6I=
|
||||
github.com/spf13/afero v1.1.1/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
|
||||
github.com/spf13/afero v1.4.1 h1:asw9sl74539yqavKaglDM5hFpdJVK0Y5Dr/JOgQ89nQ=
|
||||
github.com/spf13/afero v1.4.1/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
|
||||
github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
|
||||
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
|
||||
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
|
||||
github.com/spf13/cobra v0.0.0-20180722215644-7c4570c3ebeb h1:9EsYJzSlhhaP+nYmMOcptMF2VEUH52jxPzt/TX14KWM=
|
||||
github.com/spf13/cobra v0.0.0-20180722215644-7c4570c3ebeb/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
|
||||
github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8=
|
||||
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
|
||||
github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec h1:2ZXvIUGghLpdTVHR1UfvfrzoVlZaE/yOWC5LueIHZig=
|
||||
github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
|
||||
github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
|
||||
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/spf13/viper v1.0.2 h1:Ncr3ZIuJn322w2k1qmzXDnkLAdQMlJqBa9kfAH+irso=
|
||||
github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
|
||||
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
|
||||
github.com/uber-go/atomic v1.3.2 h1:Azu9lPBWRNKzYXSIwRfgRuDuS0YKsK4NFhiQv98gkxo=
|
||||
github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
|
||||
github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o=
|
||||
github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
|
||||
github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
|
||||
github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
|
||||
github.com/volatiletech/inflect v0.0.0-20170731032912-e7201282ae8d h1:gI4/tqP6lCY5k6Sg+4k9qSoBXmPwG+xXgMpK7jivD4M=
|
||||
github.com/volatiletech/inflect v0.0.0-20170731032912-e7201282ae8d/go.mod h1:jspfvgf53t5NLUT4o9L1IX0kIBNKamGq1tWc/MgWK9Q=
|
||||
github.com/volatiletech/null v8.0.0+incompatible h1:7wP8m5d/gZ6kW/9GnrLtMCRre2dlEnaQ9Km5OXlK4zg=
|
||||
github.com/volatiletech/null v8.0.0+incompatible/go.mod h1:0wD98JzdqB+rLyZ70fN05VDbXbafIb0KU0MdVhCzmOQ=
|
||||
github.com/volatiletech/sqlboiler v3.4.0+incompatible h1:saQ6WxZ9wEJp33q3w/DHs7an7SYi1H7Yzf4/moxCbJU=
|
||||
github.com/volatiletech/sqlboiler v3.4.0+incompatible/go.mod h1:jLfDkkHWPbS2cWRLkyC20vQWaIQsASEY7gM7zSo11Yw=
|
||||
github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
|
||||
github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE=
|
||||
go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4=
|
||||
github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0=
|
||||
go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
|
||||
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
|
||||
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
go.uber.org/atomic v1.5.1 h1:rsqfU5vBkVknbhUGbAUwQKR2H4ItV8tjJ+6kJX4cxHM=
|
||||
go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
|
||||
go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
|
||||
golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw=
|
||||
golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190909091759-094676da4a83 h1:mgAKeshyNqWKdENOnQsg+8dRTwZFIwFaO3HNl52sweA=
|
||||
golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc h1:c0o/qxkaO2LF5t6fQrT4b5hzyggAkLLlCUjqfRxd8Q4=
|
||||
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE=
|
||||
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
|
||||
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||
golang.org/x/net v0.0.0-20190909003024-a7b16738d86b h1:XfVGCX+0T4WOStkaOsJRllbsiImhB2jgVBGc9L0lPGc=
|
||||
golang.org/x/net v0.0.0-20190909003024-a7b16738d86b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20191009170851-d66e71096ffb h1:TR699M2v0qoKTOHxeLgp6zPqaQNs74f01a/ob9W0qko=
|
||||
golang.org/x/net v0.0.0-20191009170851-d66e71096ffb/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
|
||||
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20171017063910-8dbc5d05d6ed/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190520201301-c432e742b0af/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190910064555-bbd175535a8b h1:3S2h5FadpNr0zUUCVZjlKIEYF+KaX/OBplTGo89CYHI=
|
||||
golang.org/x/sys v0.0.0-20190910064555-bbd175535a8b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f h1:hjzMYz/7Ea1mNKfOnFOfktR0mlA5jqhvywClCMHM/qw=
|
||||
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
|
||||
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8=
|
||||
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4=
|
||||
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=
|
||||
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 h1:xQwXv67TxFo9nC1GJFyab5eq/5B590r6RlnL/G8Sz7w=
|
||||
golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4 h1:Toz2IK7k8rbltAXwNAxKcn9OzqyNfMUhUNjz3sL0NMk=
|
||||
golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
|
||||
google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
|
||||
google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.6.2 h1:j8RI1yW0SkI+paT6uGwMlrMI/6zwYA6/CFil8rxOzGI=
|
||||
google.golang.org/appengine v1.6.2/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
|
||||
google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
|
||||
google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
|
||||
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
|
||||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
|
||||
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
|
||||
google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
|
||||
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/gorp.v1 v1.7.1/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw=
|
||||
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
|
||||
gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/ini.v1 v1.48.0 h1:URjZc+8ugRY5mL5uUeQH/a63JcHwdX9xZaWvmNWD7z8=
|
||||
gopkg.in/ini.v1 v1.48.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 h1:FpCr9V8wuOei4BAen+93HtVJ+XSi+KPbaPKm0Vj5R64=
|
||||
gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79/go.mod h1:gWkaRU7CoXpezCBWfWjm3999QqS+1pYPXGbqQCTMzo8=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
|
||||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
|
||||
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
|
||||
gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
|
||||
grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o=
|
||||
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
rsc.io/goversion v1.0.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo=
|
||||
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
|
||||
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
|
||||
sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck=
|
||||
sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
|
||||
@@ -0,0 +1,281 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
ee "github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type Server struct {
|
||||
srv *http.Server
|
||||
stop *stop.Stopper
|
||||
}
|
||||
|
||||
func NewServer(address string, path string) *Server {
|
||||
h := http.NewServeMux()
|
||||
h.Handle(path, promhttp.Handler())
|
||||
return &Server{
|
||||
srv: &http.Server{
|
||||
Addr: address,
|
||||
Handler: h,
|
||||
//https://blog.cloudflare.com/the-complete-guide-to-golang-net-http-timeouts/
|
||||
//https://blog.cloudflare.com/exposing-go-on-the-internet/
|
||||
ReadTimeout: 5 * time.Second,
|
||||
WriteTimeout: 10 * time.Second,
|
||||
IdleTimeout: 120 * time.Second,
|
||||
},
|
||||
stop: stop.New(),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) Start() {
|
||||
s.stop.Add(1)
|
||||
go func() {
|
||||
defer s.stop.Done()
|
||||
err := s.srv.ListenAndServe()
|
||||
if err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
log.Error(err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (s *Server) Shutdown() {
|
||||
_ = s.srv.Shutdown(context.Background())
|
||||
s.stop.StopAndWait()
|
||||
}
|
||||
|
||||
const (
|
||||
ns = "reflector"
|
||||
subsystemCache = "cache"
|
||||
|
||||
labelDirection = "direction"
|
||||
labelErrorType = "error_type"
|
||||
|
||||
DirectionUpload = "upload" // to reflector
|
||||
DirectionDownload = "download" // from reflector
|
||||
|
||||
LabelCacheType = "cache_type"
|
||||
LabelComponent = "component"
|
||||
LabelSource = "source"
|
||||
|
||||
errConnReset = "conn_reset"
|
||||
errReadConnReset = "read_conn_reset"
|
||||
errWriteConnReset = "write_conn_reset"
|
||||
errReadConnTimedOut = "read_conn_timed_out"
|
||||
errNoNetworkActivity = "no_network_activity"
|
||||
errWriteConnTimedOut = "write_conn_timed_out"
|
||||
errWriteBrokenPipe = "write_broken_pipe"
|
||||
errEPipe = "e_pipe"
|
||||
errETimedout = "e_timedout"
|
||||
errIOTimeout = "io_timeout"
|
||||
errUnexpectedEOF = "unexpected_eof"
|
||||
errUnexpectedEOFStr = "unexpected_eof_str"
|
||||
errJSONSyntax = "json_syntax"
|
||||
errBlobTooBig = "blob_too_big"
|
||||
errDeadlineExceeded = "deadline_exceeded"
|
||||
errHashMismatch = "hash_mismatch"
|
||||
errZeroByteBlob = "zero_byte_blob"
|
||||
errInvalidCharacter = "invalid_character"
|
||||
errBlobNotFound = "blob_not_found"
|
||||
errNoErr = "no_error"
|
||||
errQuicProto = "quic_protocol_violation"
|
||||
errOther = "other"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrorCount = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "error_total",
|
||||
Help: "Total number of errors",
|
||||
}, []string{labelDirection, labelErrorType})
|
||||
|
||||
BlobDownloadCount = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "blob_download_total",
|
||||
Help: "Total number of blobs downloaded from reflector",
|
||||
})
|
||||
PeerDownloadCount = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "peer_download_total",
|
||||
Help: "Total number of blobs downloaded from reflector through tcp protocol",
|
||||
})
|
||||
Http3DownloadCount = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "http3_blob_download_total",
|
||||
Help: "Total number of blobs downloaded from reflector through QUIC protocol",
|
||||
})
|
||||
|
||||
CacheHitCount = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Subsystem: subsystemCache,
|
||||
Name: "hit_total",
|
||||
Help: "Total number of blobs retrieved from the cache storage",
|
||||
}, []string{LabelCacheType, LabelComponent})
|
||||
CacheMissCount = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Subsystem: subsystemCache,
|
||||
Name: "miss_total",
|
||||
Help: "Total number of blobs retrieved from origin rather than cache storage",
|
||||
}, []string{LabelCacheType, LabelComponent})
|
||||
CacheOriginRequestsCount = promauto.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: ns,
|
||||
Subsystem: subsystemCache,
|
||||
Name: "origin_requests_total",
|
||||
Help: "How many Get requests are in flight from the cache to the origin",
|
||||
}, []string{LabelCacheType, LabelComponent})
|
||||
// during thundering-herd situations, the metric below should be a lot smaller than the metric above
|
||||
CacheWaitingRequestsCount = promauto.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: ns,
|
||||
Subsystem: subsystemCache,
|
||||
Name: "waiting_requests_total",
|
||||
Help: "How many cache requests are waiting for an in-flight origin request",
|
||||
}, []string{LabelCacheType, LabelComponent})
|
||||
CacheLRUEvictCount = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Subsystem: subsystemCache,
|
||||
Name: "evict_total",
|
||||
Help: "Count of blobs evicted from cache",
|
||||
}, []string{LabelCacheType, LabelComponent})
|
||||
CacheRetrievalSpeed = promauto.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: ns,
|
||||
Name: "speed_mbps",
|
||||
Help: "Speed of blob retrieval from cache or from origin",
|
||||
}, []string{LabelCacheType, LabelComponent, LabelSource})
|
||||
|
||||
BlobUploadCount = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "blob_upload_total",
|
||||
Help: "Total number of blobs uploaded to reflector",
|
||||
})
|
||||
SDBlobUploadCount = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "sdblob_upload_total",
|
||||
Help: "Total number of SD blobs (and therefore streams) uploaded to reflector",
|
||||
})
|
||||
|
||||
MtrInBytesTcp = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "tcp_in_bytes",
|
||||
Help: "Total number of bytes downloaded through TCP",
|
||||
})
|
||||
MtrOutBytesTcp = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "tcp_out_bytes",
|
||||
Help: "Total number of bytes streamed out through TCP",
|
||||
})
|
||||
MtrInBytesUdp = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "udp_in_bytes",
|
||||
Help: "Total number of bytes downloaded through UDP",
|
||||
})
|
||||
MtrOutBytesUdp = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "udp_out_bytes",
|
||||
Help: "Total number of bytes streamed out through UDP",
|
||||
})
|
||||
MtrInBytesReflector = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "reflector_in_bytes",
|
||||
Help: "Total number of incoming bytes (from users)",
|
||||
})
|
||||
MtrOutBytesReflector = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "s3_out_bytes",
|
||||
Help: "Total number of outgoing bytes (to S3)",
|
||||
})
|
||||
MtrInBytesS3 = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "s3_in_bytes",
|
||||
Help: "Total number of incoming bytes (from S3-CF)",
|
||||
})
|
||||
)
|
||||
|
||||
func CacheLabels(name, component string) prometheus.Labels {
|
||||
return prometheus.Labels{
|
||||
LabelCacheType: name,
|
||||
LabelComponent: component,
|
||||
}
|
||||
}
|
||||
|
||||
func TrackError(direction string, e error) (shouldLog bool) { // shouldLog is a hack, but whatever
|
||||
if e == nil {
|
||||
return
|
||||
}
|
||||
|
||||
err := ee.Wrap(e, 0)
|
||||
errType := errOther
|
||||
if strings.Contains(err.Error(), "i/o timeout") {
|
||||
errType = errIOTimeout
|
||||
} else if errors.Is(e, syscall.ECONNRESET) {
|
||||
// Looks like we're getting this when direction == "download", but read_conn_reset and
|
||||
// write_conn_reset when its "upload"
|
||||
errType = errConnReset
|
||||
} else if errors.Is(e, context.DeadlineExceeded) {
|
||||
errType = errDeadlineExceeded
|
||||
} else if strings.Contains(err.Error(), "read: connection reset by peer") { // the other side closed the connection using TCP reset
|
||||
errType = errReadConnReset
|
||||
} else if strings.Contains(err.Error(), "write: connection reset by peer") { // the other side closed the connection using TCP reset
|
||||
errType = errWriteConnReset
|
||||
} else if errors.Is(e, syscall.ETIMEDOUT) {
|
||||
errType = errETimedout
|
||||
} else if strings.Contains(err.Error(), "read: connection timed out") { // the other side closed the connection using TCP reset
|
||||
//log.Warnln("read conn timed out is not the same as ETIMEDOUT")
|
||||
errType = errReadConnTimedOut
|
||||
} else if strings.Contains(err.Error(), "NO_ERROR: No recent network activity") { // the other side closed the QUIC connection
|
||||
//log.Warnln("read conn timed out is not the same as ETIMEDOUT")
|
||||
errType = errNoNetworkActivity
|
||||
} else if strings.Contains(err.Error(), "write: connection timed out") {
|
||||
errType = errWriteConnTimedOut
|
||||
} else if errors.Is(e, io.ErrUnexpectedEOF) {
|
||||
errType = errUnexpectedEOF
|
||||
} else if strings.Contains(err.Error(), "unexpected EOF") { // tried to read from closed pipe or socket
|
||||
errType = errUnexpectedEOFStr
|
||||
} else if errors.Is(e, syscall.EPIPE) {
|
||||
errType = errEPipe
|
||||
} else if strings.Contains(err.Error(), "write: broken pipe") { // tried to write to a pipe or socket that was closed by the peer
|
||||
// I believe this is the same as EPipe when direction == "download", but not for upload
|
||||
errType = errWriteBrokenPipe
|
||||
//} else if errors.Is(e, reflector.ErrBlobTooBig) { # this creates a circular import
|
||||
// errType = errBlobTooBig
|
||||
} else if strings.Contains(err.Error(), "blob must be at most") {
|
||||
//log.Warnln("blob must be at most X bytes is not the same as ErrBlobTooBig")
|
||||
errType = errBlobTooBig
|
||||
} else if strings.Contains(err.Error(), "hash of received blob data does not match hash from send request") {
|
||||
errType = errHashMismatch
|
||||
} else if strings.Contains(err.Error(), "blob not found") {
|
||||
errType = errBlobNotFound
|
||||
} else if strings.Contains(err.Error(), "0-byte blob received") {
|
||||
errType = errZeroByteBlob
|
||||
} else if strings.Contains(err.Error(), "PROTOCOL_VIOLATION: tried to retire connection") {
|
||||
errType = errQuicProto
|
||||
} else if strings.Contains(err.Error(), "invalid character") {
|
||||
errType = errInvalidCharacter
|
||||
} else if _, ok := e.(*json.SyntaxError); ok {
|
||||
errType = errJSONSyntax
|
||||
} else if strings.Contains(err.Error(), "NO_ERROR") {
|
||||
errType = errNoErr
|
||||
} else {
|
||||
log.Warnf("error '%s' for direction '%s' is not being tracked", err.TypeName(), direction)
|
||||
shouldLog = true
|
||||
}
|
||||
|
||||
ErrorCount.With(map[string]string{
|
||||
labelDirection: direction,
|
||||
labelErrorType: errType,
|
||||
}).Inc()
|
||||
|
||||
return
|
||||
}
|
||||
@@ -5,9 +5,15 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/cmd"
|
||||
|
||||
"github.com/google/gops/agent"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := agent.Listen(agent.Options{}); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
cmd.Execute()
|
||||
}
|
||||
+19
-3
@@ -1,6 +1,7 @@
|
||||
package meta
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
@@ -12,9 +13,24 @@ var BuildTime time.Time
|
||||
func init() {
|
||||
if Time != "" {
|
||||
t, err := strconv.Atoi(Time)
|
||||
if err != nil {
|
||||
return
|
||||
if err == nil {
|
||||
BuildTime = time.Unix(int64(t), 0).UTC()
|
||||
}
|
||||
BuildTime = time.Unix(int64(t), 0).UTC()
|
||||
}
|
||||
}
|
||||
|
||||
func VersionString() string {
|
||||
version := Version
|
||||
if version == "" {
|
||||
version = "<unset>"
|
||||
}
|
||||
|
||||
var buildTime string
|
||||
if BuildTime.IsZero() {
|
||||
buildTime = "<now>"
|
||||
} else {
|
||||
buildTime = BuildTime.Format(time.RFC3339)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("version %s, built %s", version, buildTime)
|
||||
}
|
||||
+3
-2
@@ -8,6 +8,7 @@ import (
|
||||
"net"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
@@ -152,8 +153,8 @@ func (c *Client) GetBlob(hash string) (stream.Blob, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return stream.Blob(blob), nil
|
||||
metrics.MtrInBytesTcp.Add(float64(len(blob)))
|
||||
return blob, nil
|
||||
}
|
||||
|
||||
func (c *Client) read(v interface{}) error {
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
package http3
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lucas-clemente/quic-go/http3"
|
||||
)
|
||||
|
||||
// Client is an instance of a client connected to a server.
|
||||
type Client struct {
|
||||
Timeout time.Duration
|
||||
conn *http.Client
|
||||
roundTripper *http3.RoundTripper
|
||||
ServerAddr string
|
||||
}
|
||||
|
||||
// Close closes the connection with the client.
|
||||
func (c *Client) Close() error {
|
||||
c.conn.CloseIdleConnections()
|
||||
return c.roundTripper.Close()
|
||||
}
|
||||
|
||||
// GetStream gets a stream
|
||||
func (c *Client) GetStream(sdHash string, blobCache store.BlobStore) (stream.Stream, error) {
|
||||
var sd stream.SDBlob
|
||||
|
||||
b, err := c.GetBlob(sdHash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = sd.FromBlob(b)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s := make(stream.Stream, len(sd.BlobInfos)+1-1) // +1 for sd blob, -1 for last null blob
|
||||
s[0] = b
|
||||
|
||||
for i := 0; i < len(sd.BlobInfos)-1; i++ {
|
||||
s[i+1], err = c.GetBlob(hex.EncodeToString(sd.BlobInfos[i].BlobHash))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// HasBlob checks if the blob is available
|
||||
func (c *Client) HasBlob(hash string) (bool, error) {
|
||||
resp, err := c.conn.Get(fmt.Sprintf("https://%s/has/%s", c.ServerAddr, hash))
|
||||
if err != nil {
|
||||
return false, errors.Err(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusOK {
|
||||
return true, nil
|
||||
}
|
||||
if resp.StatusCode == http.StatusNotFound {
|
||||
return false, nil
|
||||
}
|
||||
return false, errors.Err("non 200 status code returned: %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
// GetBlob gets a blob
|
||||
func (c *Client) GetBlob(hash string) (stream.Blob, error) {
|
||||
resp, err := c.conn.Get(fmt.Sprintf("https://%s/get/%s", c.ServerAddr, hash))
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusNotFound {
|
||||
fmt.Printf("%s blob not found %d\n", hash, resp.StatusCode)
|
||||
return nil, errors.Err(store.ErrBlobNotFound)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, errors.Err("non 200 status code returned: %d", resp.StatusCode)
|
||||
}
|
||||
body := &bytes.Buffer{}
|
||||
_, err = io.Copy(body, resp.Body)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
metrics.MtrInBytesUdp.Add(float64(len(body.Bytes())))
|
||||
return body.Bytes(), nil
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
package http3
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/lucas-clemente/quic-go"
|
||||
"github.com/lucas-clemente/quic-go/http3"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// Server is an instance of a peer server that houses the listener and store.
|
||||
type Server struct {
|
||||
store store.BlobStore
|
||||
grp *stop.Group
|
||||
}
|
||||
|
||||
// NewServer returns an initialized Server pointer.
|
||||
func NewServer(store store.BlobStore) *Server {
|
||||
return &Server{
|
||||
store: store,
|
||||
grp: stop.New(),
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown gracefully shuts down the peer server.
|
||||
func (s *Server) Shutdown() {
|
||||
log.Debug("shutting down http3 peer server")
|
||||
s.grp.StopAndWait()
|
||||
log.Debug("http3 peer server stopped")
|
||||
}
|
||||
|
||||
func (s *Server) logError(e error) {
|
||||
if e == nil {
|
||||
return
|
||||
}
|
||||
shouldLog := metrics.TrackError(metrics.DirectionDownload, e)
|
||||
if shouldLog {
|
||||
log.Errorln(errors.FullTrace(e))
|
||||
}
|
||||
}
|
||||
|
||||
type availabilityResponse struct {
|
||||
LbrycrdAddress string `json:"lbrycrd_address"`
|
||||
IsAvailable bool `json:"is_available"`
|
||||
}
|
||||
|
||||
// Start starts the server listener to handle connections.
|
||||
func (s *Server) Start(address string) error {
|
||||
log.Println("HTTP3 peer listening on " + address)
|
||||
quicConf := &quic.Config{
|
||||
HandshakeTimeout: 4 * time.Second,
|
||||
MaxIdleTimeout: 10 * time.Second,
|
||||
}
|
||||
r := mux.NewRouter()
|
||||
r.HandleFunc("/get/{hash}", func(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
requestedBlob := vars["hash"]
|
||||
blob, err := s.store.Get(requestedBlob)
|
||||
if err != nil {
|
||||
if errors.Is(err, store.ErrBlobNotFound) {
|
||||
http.Error(w, err.Error(), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
fmt.Printf("%s: %s", requestedBlob, errors.FullTrace(err))
|
||||
s.logError(err)
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
_, err = w.Write(blob)
|
||||
if err != nil {
|
||||
s.logError(err)
|
||||
}
|
||||
metrics.MtrOutBytesUdp.Add(float64(len(blob)))
|
||||
metrics.BlobDownloadCount.Inc()
|
||||
metrics.Http3DownloadCount.Inc()
|
||||
})
|
||||
r.HandleFunc("/has/{hash}", func(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
requestedBlob := vars["hash"]
|
||||
blobExists, err := s.store.Has(requestedBlob)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
s.logError(err)
|
||||
return
|
||||
}
|
||||
if !blobExists {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
// LbrycrdAddress to be used when paying for data. Not implemented yet.
|
||||
const LbrycrdAddress = "bJxKvpD96kaJLriqVajZ7SaQTsWWyrGQct"
|
||||
resp, err := json.Marshal(availabilityResponse{
|
||||
LbrycrdAddress: LbrycrdAddress,
|
||||
IsAvailable: blobExists,
|
||||
})
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
s.logError(err)
|
||||
return
|
||||
}
|
||||
_, err = w.Write(resp)
|
||||
if err != nil {
|
||||
s.logError(err)
|
||||
}
|
||||
})
|
||||
server := http3.Server{
|
||||
Server: &http.Server{
|
||||
Handler: r,
|
||||
Addr: address,
|
||||
TLSConfig: generateTLSConfig(),
|
||||
},
|
||||
QuicConfig: quicConf,
|
||||
}
|
||||
|
||||
go s.listenForShutdown(&server)
|
||||
s.grp.Add(1)
|
||||
go func() {
|
||||
s.listenAndServe(&server)
|
||||
s.grp.Done()
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Setup a bare-bones TLS config for the server
|
||||
func generateTLSConfig() *tls.Config {
|
||||
key, err := rsa.GenerateKey(rand.Reader, 1024)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
template := x509.Certificate{SerialNumber: big.NewInt(1)}
|
||||
certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})
|
||||
certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
|
||||
|
||||
tlsCert, err := tls.X509KeyPair(certPEM, keyPEM)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return &tls.Config{
|
||||
Certificates: []tls.Certificate{tlsCert},
|
||||
NextProtos: []string{"http3-reflector-server"},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) listenAndServe(server *http3.Server) {
|
||||
err := server.ListenAndServe()
|
||||
if err != nil && err.Error() != "server closed" {
|
||||
log.Errorln(errors.FullTrace(err))
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) listenForShutdown(listener *http3.Server) {
|
||||
<-s.grp.Ch()
|
||||
err := listener.Close()
|
||||
if err != nil {
|
||||
log.Error("error closing listener for peer server - ", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package http3
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
"github.com/lucas-clemente/quic-go"
|
||||
"github.com/lucas-clemente/quic-go/http3"
|
||||
)
|
||||
|
||||
// Store is a blob store that gets blobs from a peer.
|
||||
// It satisfies the store.BlobStore interface but cannot put or delete blobs.
|
||||
type Store struct {
|
||||
opts StoreOpts
|
||||
}
|
||||
|
||||
// StoreOpts allows to set options for a new Store.
|
||||
type StoreOpts struct {
|
||||
Address string
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
// NewStore makes a new peer store.
|
||||
func NewStore(opts StoreOpts) *Store {
|
||||
return &Store{opts: opts}
|
||||
}
|
||||
|
||||
func (p *Store) getClient() (*Client, error) {
|
||||
var qconf quic.Config
|
||||
qconf.HandshakeTimeout = 4 * time.Second
|
||||
qconf.MaxIdleTimeout = 10 * time.Second
|
||||
pool, err := x509.SystemCertPool()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
roundTripper := &http3.RoundTripper{
|
||||
TLSClientConfig: &tls.Config{
|
||||
RootCAs: pool,
|
||||
InsecureSkipVerify: true,
|
||||
},
|
||||
QuicConfig: &qconf,
|
||||
}
|
||||
connection := &http.Client{
|
||||
Transport: roundTripper,
|
||||
}
|
||||
c := &Client{
|
||||
conn: connection,
|
||||
roundTripper: roundTripper,
|
||||
ServerAddr: p.opts.Address,
|
||||
}
|
||||
return c, errors.Prefix("connection error", err)
|
||||
}
|
||||
|
||||
func (p *Store) Name() string { return "http3" }
|
||||
|
||||
// Has asks the peer if they have a hash
|
||||
func (p *Store) Has(hash string) (bool, error) {
|
||||
c, err := p.getClient()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer c.Close()
|
||||
return c.HasBlob(hash)
|
||||
}
|
||||
|
||||
// Get downloads the blob from the peer
|
||||
func (p *Store) Get(hash string) (stream.Blob, error) {
|
||||
c, err := p.getClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer c.Close()
|
||||
return c.GetBlob(hash)
|
||||
}
|
||||
|
||||
// Put is not supported
|
||||
func (p *Store) Put(hash string, blob stream.Blob) error {
|
||||
panic("http3Store cannot put or delete blobs")
|
||||
}
|
||||
|
||||
// PutSD is not supported
|
||||
func (p *Store) PutSD(hash string, blob stream.Blob) error {
|
||||
panic("http3Store cannot put or delete blobs")
|
||||
}
|
||||
|
||||
// Delete is not supported
|
||||
func (p *Store) Delete(hash string) error {
|
||||
panic("http3Store cannot put or delete blobs")
|
||||
}
|
||||
+16
-34
@@ -2,12 +2,15 @@ package peer
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
ee "errors"
|
||||
"io"
|
||||
"net"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
@@ -27,14 +30,10 @@ const (
|
||||
|
||||
// Server is an instance of a peer server that houses the listener and store.
|
||||
type Server struct {
|
||||
StatLogger *log.Logger // logger to log stats
|
||||
StatReportFrequency time.Duration // how often to log stats
|
||||
|
||||
store store.BlobStore
|
||||
closed bool
|
||||
|
||||
grp *stop.Group
|
||||
stats *reflector.Stats
|
||||
grp *stop.Group
|
||||
}
|
||||
|
||||
// NewServer returns an initialized Server pointer.
|
||||
@@ -47,8 +46,7 @@ func NewServer(store store.BlobStore) *Server {
|
||||
|
||||
// Shutdown gracefully shuts down the peer server.
|
||||
func (s *Server) Shutdown() {
|
||||
log.Debug("shutting down peer server...")
|
||||
s.stats.Shutdown()
|
||||
log.Debug("shutting down peer server")
|
||||
s.grp.StopAndWait()
|
||||
log.Debug("peer server stopped")
|
||||
}
|
||||
@@ -68,11 +66,6 @@ func (s *Server) Start(address string) error {
|
||||
s.grp.Done()
|
||||
}()
|
||||
|
||||
s.stats = reflector.NewStatLogger("DOWNLOAD", s.StatLogger, s.StatReportFrequency, s.grp.Child())
|
||||
if s.StatLogger != nil && s.StatReportFrequency > 0 {
|
||||
s.stats.Start()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -137,7 +130,7 @@ func (s *Server) handleConnection(conn net.Conn) {
|
||||
|
||||
response, err = s.handleCompositeRequest(request)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
log.Error(errors.FullTrace(err))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -168,7 +161,7 @@ func (s *Server) handleAvailabilityRequest(data []byte) ([]byte, error) {
|
||||
var request availabilityRequest
|
||||
err := json.Unmarshal(data, &request)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
availableBlobs := []string{}
|
||||
@@ -229,7 +222,11 @@ func (s *Server) handleCompositeRequest(data []byte) ([]byte, error) {
|
||||
var request compositeRequest
|
||||
err := json.Unmarshal(data, &request)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
var je *json.SyntaxError
|
||||
if ee.As(err, &je) {
|
||||
return nil, errors.Err("invalid json at offset %d in data %s", je.Offset, hex.EncodeToString(data))
|
||||
}
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
response := compositeResponse{
|
||||
@@ -275,7 +272,9 @@ func (s *Server) handleCompositeRequest(data []byte) ([]byte, error) {
|
||||
BlobHash: reflector.BlobHash(blob),
|
||||
Length: len(blob),
|
||||
}
|
||||
s.stats.AddBlob()
|
||||
metrics.MtrOutBytesTcp.Add(float64(len(blob)))
|
||||
metrics.BlobDownloadCount.Inc()
|
||||
metrics.PeerDownloadCount.Inc()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -291,27 +290,10 @@ func (s *Server) logError(e error) {
|
||||
if e == nil {
|
||||
return
|
||||
}
|
||||
shouldLog := s.stats.AddError(e)
|
||||
shouldLog := metrics.TrackError(metrics.DirectionDownload, e)
|
||||
if shouldLog {
|
||||
log.Errorln(errors.FullTrace(e))
|
||||
}
|
||||
|
||||
return
|
||||
|
||||
// old stuff below. its here for posterity, because we're gonna have to deal with these errors someday for real
|
||||
|
||||
//err := errors.Wrap(e, 0)
|
||||
|
||||
// these happen because the peer protocol does not have a way to cancel blob downloads
|
||||
// so the client will just close the connection if its in the middle of downloading a blob
|
||||
// but receives the blob from a different peer first or simply goes offline (timeout)
|
||||
//if strings.Contains(err.Error(), "connection reset by peer") ||
|
||||
// strings.Contains(err.Error(), "i/o timeout") ||
|
||||
// strings.Contains(err.Error(), "broken pipe") {
|
||||
// return
|
||||
//}
|
||||
//
|
||||
//log.Error(errors.FullTrace(e))
|
||||
}
|
||||
|
||||
func readNextMessage(buf *bufio.Reader) ([]byte, error) {
|
||||
|
||||
+1
-1
@@ -34,7 +34,7 @@ var availabilityRequests = []pair{
|
||||
}
|
||||
|
||||
func getServer(t *testing.T, withBlobs bool) *Server {
|
||||
st := store.NewMemoryBlobStore()
|
||||
st := store.NewMemStore()
|
||||
if withBlobs {
|
||||
for k, v := range blobs {
|
||||
err := st.Put(k, v)
|
||||
|
||||
+20
-16
@@ -10,8 +10,7 @@ import (
|
||||
// Store is a blob store that gets blobs from a peer.
|
||||
// It satisfies the store.BlobStore interface but cannot put or delete blobs.
|
||||
type Store struct {
|
||||
client *Client
|
||||
connErr error
|
||||
opts StoreOpts
|
||||
}
|
||||
|
||||
// StoreOpts allows to set options for a new Store.
|
||||
@@ -22,30 +21,35 @@ type StoreOpts struct {
|
||||
|
||||
// NewStore makes a new peer store.
|
||||
func NewStore(opts StoreOpts) *Store {
|
||||
if opts.Timeout == 0 {
|
||||
opts.Timeout = time.Second * 5
|
||||
}
|
||||
c := &Client{Timeout: opts.Timeout}
|
||||
err := c.Connect(opts.Address)
|
||||
return &Store{client: c, connErr: err}
|
||||
return &Store{opts: opts}
|
||||
}
|
||||
|
||||
func (p *Store) getClient() (*Client, error) {
|
||||
c := &Client{Timeout: p.opts.Timeout}
|
||||
err := c.Connect(p.opts.Address)
|
||||
return c, errors.Prefix("connection error", err)
|
||||
}
|
||||
|
||||
func (p *Store) Name() string { return "peer" }
|
||||
|
||||
// Has asks the peer if they have a hash
|
||||
func (p *Store) Has(hash string) (bool, error) {
|
||||
if p.connErr != nil {
|
||||
return false, errors.Prefix("connection error", p.connErr)
|
||||
c, err := p.getClient()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
return p.client.HasBlob(hash)
|
||||
defer c.Close()
|
||||
return c.HasBlob(hash)
|
||||
}
|
||||
|
||||
// Get downloads the blob from the peer
|
||||
func (p *Store) Get(hash string) (stream.Blob, error) {
|
||||
if p.connErr != nil {
|
||||
return nil, errors.Prefix("connection error", p.connErr)
|
||||
c, err := p.getClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return p.client.GetBlob(hash)
|
||||
defer c.Close()
|
||||
return c.GetBlob(hash)
|
||||
}
|
||||
|
||||
// Put is not supported
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
package publish
|
||||
|
||||
import "strings"
|
||||
|
||||
func guessMimeType(ext string) (string, string) {
|
||||
if ext == "" {
|
||||
return "application/octet-stream", "binary"
|
||||
}
|
||||
|
||||
ext = strings.ToLower(strings.TrimLeft(strings.TrimSpace(ext), "."))
|
||||
|
||||
types := map[string]struct{ mime, t string }{
|
||||
"a": {"application/octet-stream", "binary"},
|
||||
"ai": {"application/postscript", "image"},
|
||||
"aif": {"audio/x-aiff", "audio"},
|
||||
"aifc": {"audio/x-aiff", "audio"},
|
||||
"aiff": {"audio/x-aiff", "audio"},
|
||||
"au": {"audio/basic", "audio"},
|
||||
"avi": {"video/x-msvideo", "video"},
|
||||
"bat": {"text/plain", "document"},
|
||||
"bcpio": {"application/x-bcpio", "binary"},
|
||||
"bin": {"application/octet-stream", "binary"},
|
||||
"bmp": {"image/bmp", "image"},
|
||||
"c": {"text/plain", "document"},
|
||||
"cdf": {"application/x-netcdf", "binary"},
|
||||
"cpio": {"application/x-cpio", "binary"},
|
||||
"csh": {"application/x-csh", "binary"},
|
||||
"css": {"text/css", "document"},
|
||||
"csv": {"text/csv", "document"},
|
||||
"dll": {"application/octet-stream", "binary"},
|
||||
"doc": {"application/msword", "document"},
|
||||
"dot": {"application/msword", "document"},
|
||||
"dvi": {"application/x-dvi", "binary"},
|
||||
"eml": {"message/rfc822", "document"},
|
||||
"eps": {"application/postscript", "document"},
|
||||
"epub": {"application/epub+zip", "document"},
|
||||
"etx": {"text/x-setext", "document"},
|
||||
"exe": {"application/octet-stream", "binary"},
|
||||
"gif": {"image/gif", "image"},
|
||||
"gtar": {"application/x-gtar", "binary"},
|
||||
"h": {"text/plain", "document"},
|
||||
"hdf": {"application/x-hdf", "binary"},
|
||||
"htm": {"text/html", "document"},
|
||||
"html": {"text/html", "document"},
|
||||
"ico": {"image/vnd.microsoft.icon", "image"},
|
||||
"ief": {"image/ief", "image"},
|
||||
"iges": {"model/iges", "model"},
|
||||
"jpe": {"image/jpeg", "image"},
|
||||
"jpeg": {"image/jpeg", "image"},
|
||||
"jpg": {"image/jpeg", "image"},
|
||||
"js": {"application/javascript", "document"},
|
||||
"json": {"application/json", "document"},
|
||||
"ksh": {"text/plain", "document"},
|
||||
"latex": {"application/x-latex", "binary"},
|
||||
"m1v": {"video/mpeg", "video"},
|
||||
"m3u": {"application/vnd.apple.mpegurl", "audio"},
|
||||
"m3u8": {"application/vnd.apple.mpegurl", "audio"},
|
||||
"man": {"application/x-troff-man", "document"},
|
||||
"markdown": {"text/markdown", "document"},
|
||||
"md": {"text/markdown", "document"},
|
||||
"me": {"application/x-troff-me", "binary"},
|
||||
"mht": {"message/rfc822", "document"},
|
||||
"mhtml": {"message/rfc822", "document"},
|
||||
"mif": {"application/x-mif", "binary"},
|
||||
"mov": {"video/quicktime", "video"},
|
||||
"movie": {"video/x-sgi-movie", "video"},
|
||||
"mp2": {"audio/mpeg", "audio"},
|
||||
"mp3": {"audio/mpeg", "audio"},
|
||||
"mp4": {"video/mp4", "video"},
|
||||
"mpa": {"video/mpeg", "video"},
|
||||
"mpe": {"video/mpeg", "video"},
|
||||
"mpeg": {"video/mpeg", "video"},
|
||||
"mpg": {"video/mpeg", "video"},
|
||||
"ms": {"application/x-troff-ms", "binary"},
|
||||
"nc": {"application/x-netcdf", "binary"},
|
||||
"nws": {"message/rfc822", "document"},
|
||||
"o": {"application/octet-stream", "binary"},
|
||||
"obj": {"application/octet-stream", "model"},
|
||||
"oda": {"application/oda", "binary"},
|
||||
"p12": {"application/x-pkcs12", "binary"},
|
||||
"p7c": {"application/pkcs7-mime", "binary"},
|
||||
"pbm": {"image/x-portable-bitmap", "image"},
|
||||
"pdf": {"application/pdf", "document"},
|
||||
"pfx": {"application/x-pkcs12", "binary"},
|
||||
"pgm": {"image/x-portable-graymap", "image"},
|
||||
"pl": {"text/plain", "document"},
|
||||
"png": {"image/png", "image"},
|
||||
"pnm": {"image/x-portable-anymap", "image"},
|
||||
"pot": {"application/vnd.ms-powerpoint", "document"},
|
||||
"ppa": {"application/vnd.ms-powerpoint", "document"},
|
||||
"ppm": {"image/x-portable-pixmap", "image"},
|
||||
"pps": {"application/vnd.ms-powerpoint", "document"},
|
||||
"ppt": {"application/vnd.ms-powerpoint", "document"},
|
||||
"ps": {"application/postscript", "document"},
|
||||
"pwz": {"application/vnd.ms-powerpoint", "document"},
|
||||
"py": {"text/x-python", "document"},
|
||||
"pyc": {"application/x-python-code", "binary"},
|
||||
"pyo": {"application/x-python-code", "binary"},
|
||||
"qt": {"video/quicktime", "video"},
|
||||
"ra": {"audio/x-pn-realaudio", "audio"},
|
||||
"ram": {"application/x-pn-realaudio", "audio"},
|
||||
"ras": {"image/x-cmu-raster", "image"},
|
||||
"rdf": {"application/xml", "binary"},
|
||||
"rgb": {"image/x-rgb", "image"},
|
||||
"roff": {"application/x-troff", "binary"},
|
||||
"rtx": {"text/richtext", "document"},
|
||||
"sgm": {"text/x-sgml", "document"},
|
||||
"sgml": {"text/x-sgml", "document"},
|
||||
"sh": {"application/x-sh", "document"},
|
||||
"shar": {"application/x-shar", "binary"},
|
||||
"snd": {"audio/basic", "audio"},
|
||||
"so": {"application/octet-stream", "binary"},
|
||||
"src": {"application/x-wais-source", "binary"},
|
||||
"stl": {"model/stl", "model"},
|
||||
"sv4cpio": {"application/x-sv4cpio", "binary"},
|
||||
"sv4crc": {"application/x-sv4crc", "binary"},
|
||||
"svg": {"image/svg+xml", "image"},
|
||||
"swf": {"application/x-shockwave-flash", "binary"},
|
||||
"t": {"application/x-troff", "binary"},
|
||||
"tar": {"application/x-tar", "binary"},
|
||||
"tcl": {"application/x-tcl", "binary"},
|
||||
"tex": {"application/x-tex", "binary"},
|
||||
"texi": {"application/x-texinfo", "binary"},
|
||||
"texinfo": {"application/x-texinfo", "binary"},
|
||||
"tif": {"image/tiff", "image"},
|
||||
"tiff": {"image/tiff", "image"},
|
||||
"tr": {"application/x-troff", "binary"},
|
||||
"tsv": {"text/tab-separated-values", "document"},
|
||||
"txt": {"text/plain", "document"},
|
||||
"ustar": {"application/x-ustar", "binary"},
|
||||
"vcf": {"text/x-vcard", "document"},
|
||||
"wav": {"audio/x-wav", "audio"},
|
||||
"webm": {"video/webm", "video"},
|
||||
"wiz": {"application/msword", "document"},
|
||||
"wsdl": {"application/xml", "document"},
|
||||
"xbm": {"image/x-xbitmap", "image"},
|
||||
"xlb": {"application/vnd.ms-excel", "document"},
|
||||
"xls": {"application/vnd.ms-excel", "document"},
|
||||
"xml": {"text/xml", "document"},
|
||||
"xpdl": {"application/xml", "document"},
|
||||
"xpm": {"image/x-xpixmap", "image"},
|
||||
"xsl": {"application/xml", "document"},
|
||||
"xwd": {"image/x-xwindowdump", "image"},
|
||||
"zip": {"application/zip", "binary"},
|
||||
|
||||
// These are non-standard types, commonly found in the wild.
|
||||
"cbr": {"application/vnd.comicbook-rar", "document"},
|
||||
"cbz": {"application/vnd.comicbook+zip", "document"},
|
||||
"flac": {"audio/flac", "audio"},
|
||||
"lbry": {"application/x-ext-lbry", "document"},
|
||||
"m4v": {"video/m4v", "video"},
|
||||
"mid": {"audio/midi", "audio"},
|
||||
"midi": {"audio/midi", "audio"},
|
||||
"mkv": {"video/x-matroska", "video"},
|
||||
"mobi": {"application/x-mobipocket-ebook", "document"},
|
||||
"oga": {"audio/ogg", "audio"},
|
||||
"ogv": {"video/ogg", "video"},
|
||||
"pct": {"image/pict", "image"},
|
||||
"pic": {"image/pict", "image"},
|
||||
"pict": {"image/pict", "image"},
|
||||
"prc": {"application/x-mobipocket-ebook", "document"},
|
||||
"rtf": {"application/rtf", "document"},
|
||||
"xul": {"text/xul", "document"},
|
||||
|
||||
// microsoft is special and has its own "standard"
|
||||
// https://docs.microsoft.com/en-us/windows/desktop/wmp/file-name-extensions
|
||||
"wmv": {"video/x-ms-wmv", "video"},
|
||||
}
|
||||
|
||||
if data, ok := types[ext]; ok {
|
||||
return data.mime, data.t
|
||||
}
|
||||
return "application/x-ext-" + ext, "binary"
|
||||
}
|
||||
@@ -0,0 +1,303 @@
|
||||
package publish
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/lbrycrd"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
pb "github.com/lbryio/types/v2/go"
|
||||
|
||||
"github.com/btcsuite/btcd/btcjson"
|
||||
"github.com/btcsuite/btcd/chaincfg/chainhash"
|
||||
"github.com/btcsuite/btcd/txscript"
|
||||
"github.com/btcsuite/btcd/wire"
|
||||
"github.com/btcsuite/btcutil"
|
||||
"github.com/golang/protobuf/proto"
|
||||
"golang.org/x/crypto/sha3"
|
||||
)
|
||||
|
||||
var TODO = `
|
||||
import cert from wallet
|
||||
get all utxos from chainquery
|
||||
create transaction
|
||||
sign it with the channel
|
||||
track state of utxos across publishes from this channel so that we can just do one query to get utxos
|
||||
prioritize only confirmed utxos
|
||||
|
||||
Handling all the issues we handle currently with lbrynet:
|
||||
"Couldn't find private key for id",
|
||||
"You already have a stream claim published under the name",
|
||||
"Cannot publish using channel",
|
||||
"txn-mempool-conflict",
|
||||
"too-long-mempool-chain",
|
||||
"Missing inputs",
|
||||
"Not enough funds to cover this transaction",
|
||||
}
|
||||
`
|
||||
|
||||
func Publish(client *lbrycrd.Client, path, name, address string, details Details, reflectorAddress string) (*wire.MsgTx, *chainhash.Hash, error) {
|
||||
if name == "" {
|
||||
return nil, nil, errors.Err("name required")
|
||||
}
|
||||
|
||||
//TODO: sign claim if publishing into channel
|
||||
|
||||
addr, err := btcutil.DecodeAddress(address, &lbrycrd.MainNetParams)
|
||||
if errors.Is(err, btcutil.ErrUnknownAddressType) {
|
||||
return nil, nil, errors.Err(`unknown address type. here's what you need to make this work:
|
||||
- deprecatedrpc=validateaddress" and "deprecatedrpc=signrawtransaction" in your lbrycrd.conf
|
||||
- github.com/btcsuite/btcd pinned to hash 306aecffea32
|
||||
- github.com/btcsuite/btcutil pinned to 4c204d697803
|
||||
- github.com/lbryio/lbry.go/v2 (make sure you have v2 at the end)`)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
amount := 0.01
|
||||
changeAddr := addr // TODO: fix this? or maybe its fine?
|
||||
tx, err := baseTx(client, amount, changeAddr)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
claim, st, err := makeClaimAndStream(path, details)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
err = addClaimToTx(tx, claim, name, amount, addr)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// sign and send
|
||||
signedTx, allInputsSigned, err := client.SignRawTransaction(tx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if !allInputsSigned {
|
||||
return nil, nil, errors.Err("not all inputs for the tx could be signed")
|
||||
}
|
||||
|
||||
err = reflect(st, reflectorAddress)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
txid, err := client.SendRawTransaction(signedTx, false)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return signedTx, txid, nil
|
||||
}
|
||||
|
||||
//TODO: lots of assumptions. hardcoded values need to be passed in or calculated
|
||||
func baseTx(client *lbrycrd.Client, amount float64, changeAddress btcutil.Address) (*wire.MsgTx, error) {
|
||||
txFee := 0.0002 // TODO: estimate this better?
|
||||
|
||||
inputs, total, err := coinChooser(client, amount+txFee)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
change := total - amount - txFee
|
||||
|
||||
// create base raw tx
|
||||
addresses := make(map[btcutil.Address]btcutil.Amount)
|
||||
//changeAddr, err := client.GetNewAddress("")
|
||||
changeAmount, err := btcutil.NewAmount(change)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
addresses[changeAddress] = changeAmount
|
||||
lockTime := int64(0)
|
||||
return client.CreateRawTransaction(inputs, addresses, &lockTime)
|
||||
}
|
||||
|
||||
func coinChooser(client *lbrycrd.Client, amount float64) ([]btcjson.TransactionInput, float64, error) {
|
||||
utxos, err := client.ListUnspentMin(1)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
sort.Slice(utxos, func(i, j int) bool { return utxos[i].Amount < utxos[j].Amount })
|
||||
|
||||
var utxo btcjson.ListUnspentResult
|
||||
for _, u := range utxos {
|
||||
if u.Spendable && u.Amount >= amount {
|
||||
utxo = u
|
||||
break
|
||||
}
|
||||
}
|
||||
if utxo.TxID == "" {
|
||||
return nil, 0, errors.Err("not enough utxos to create tx")
|
||||
}
|
||||
|
||||
return []btcjson.TransactionInput{{Txid: utxo.TxID, Vout: utxo.Vout}}, utxo.Amount, nil
|
||||
}
|
||||
|
||||
func addClaimToTx(tx *wire.MsgTx, claim *pb.Claim, name string, amount float64, claimAddress btcutil.Address) error {
|
||||
claimBytes, err := proto.Marshal(claim)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
claimBytes = append([]byte{0}, claimBytes...) // version 0 = no channel sig
|
||||
|
||||
amt, err := btcutil.NewAmount(amount)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
script, err := getClaimPayoutScript(name, claimBytes, claimAddress)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tx.AddTxOut(wire.NewTxOut(int64(amt), script))
|
||||
return nil
|
||||
}
|
||||
|
||||
func Decode(client *lbrycrd.Client, tx *wire.MsgTx) (string, error) {
|
||||
buf := bytes.NewBuffer(make([]byte, 0, tx.SerializeSize()))
|
||||
if err := tx.Serialize(buf); err != nil {
|
||||
return "", errors.Err(err)
|
||||
}
|
||||
//txHex := hex.EncodeToString(buf.Bytes())
|
||||
//spew.Dump(txHex)
|
||||
decoded, err := client.DecodeRawTransaction(buf.Bytes())
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
data, err := json.MarshalIndent(decoded, "", " ")
|
||||
return string(data), err
|
||||
}
|
||||
|
||||
func reflect(st stream.Stream, reflectorAddress string) error {
|
||||
// upload blobs to reflector
|
||||
c := reflector.Client{}
|
||||
err := c.Connect(reflectorAddress)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
for i, b := range st {
|
||||
if i == 0 {
|
||||
err = c.SendSDBlob(b)
|
||||
} else {
|
||||
err = c.SendBlob(b)
|
||||
}
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type Details struct {
|
||||
Title string
|
||||
Description string
|
||||
Author string
|
||||
Tags []string
|
||||
ReleaseTime int64
|
||||
}
|
||||
|
||||
func makeClaimAndStream(path string, details Details) (*pb.Claim, stream.Stream, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, nil, errors.Err(err)
|
||||
}
|
||||
data, err := ioutil.ReadAll(file)
|
||||
if err != nil {
|
||||
return nil, nil, errors.Err(err)
|
||||
}
|
||||
s, err := stream.New(data)
|
||||
if err != nil {
|
||||
return nil, nil, errors.Err(err)
|
||||
}
|
||||
|
||||
// make the claim
|
||||
sdBlob := &stream.SDBlob{}
|
||||
err = sdBlob.FromBlob(s[0])
|
||||
if err != nil {
|
||||
return nil, nil, errors.Err(err)
|
||||
}
|
||||
|
||||
filehash := sha3.Sum384(data)
|
||||
|
||||
streamPB := &pb.Stream{
|
||||
Author: details.Author,
|
||||
ReleaseTime: details.ReleaseTime,
|
||||
Source: &pb.Source{
|
||||
SdHash: s[0].Hash(),
|
||||
Name: filepath.Base(file.Name()),
|
||||
Size: uint64(len(data)),
|
||||
Hash: filehash[:],
|
||||
},
|
||||
}
|
||||
|
||||
mimeType, category := guessMimeType(filepath.Ext(file.Name()))
|
||||
streamPB.Source.MediaType = mimeType
|
||||
|
||||
switch category {
|
||||
case "video":
|
||||
//t, err := streamVideoMetadata(path)
|
||||
//if err != nil {
|
||||
// return nil, nil, err
|
||||
//}
|
||||
streamPB.Type = &pb.Stream_Video{}
|
||||
case "audio":
|
||||
streamPB.Type = &pb.Stream_Audio{}
|
||||
case "image":
|
||||
streamPB.Type = &pb.Stream_Image{}
|
||||
}
|
||||
|
||||
claim := &pb.Claim{
|
||||
Title: details.Title,
|
||||
Description: details.Description,
|
||||
Type: &pb.Claim_Stream{Stream: streamPB},
|
||||
}
|
||||
|
||||
return claim, s, nil
|
||||
}
|
||||
|
||||
func getClaimPayoutScript(name string, value []byte, address btcutil.Address) ([]byte, error) {
|
||||
//OP_CLAIM_NAME <name> <value> OP_2DROP OP_DROP OP_DUP OP_HASH160 <address> OP_EQUALVERIFY OP_CHECKSIG
|
||||
|
||||
pkscript, err := txscript.PayToAddrScript(address)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
return txscript.NewScriptBuilder().
|
||||
AddOp(txscript.OP_NOP6). //OP_CLAIM_NAME
|
||||
AddData([]byte(name)). //<name>
|
||||
AddData(value). //<value>
|
||||
AddOp(txscript.OP_2DROP). //OP_2DROP
|
||||
AddOp(txscript.OP_DROP). //OP_DROP
|
||||
AddOps(pkscript). //OP_DUP OP_HASH160 <address> OP_EQUALVERIFY OP_CHECKSIG
|
||||
Script()
|
||||
}
|
||||
|
||||
//func streamVideoMetadata(path string) (*pb.Stream_Video, error) {
|
||||
// mi, err := mediainfo.GetMediaInfo(path)
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
// return &pb.Stream_Video{
|
||||
// Video: &pb.Video{
|
||||
// Duration: uint32(mi.General.Duration / 1000),
|
||||
// Height: uint32(mi.Video.Height),
|
||||
// Width: uint32(mi.Video.Width),
|
||||
// },
|
||||
// }, nil
|
||||
//}
|
||||
@@ -0,0 +1,59 @@
|
||||
package publish
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
)
|
||||
|
||||
func LoadWallet(r io.Reader) (WalletFile, error) {
|
||||
var w WalletFile
|
||||
err := json.NewDecoder(r).Decode(&w)
|
||||
return w, err
|
||||
}
|
||||
|
||||
type WalletFile struct {
|
||||
Name string `json:"name"`
|
||||
Version int `json:"version"`
|
||||
Preferences WalletPrefs `json:"preferences"`
|
||||
Accounts []Account `json:"accounts"`
|
||||
}
|
||||
|
||||
type Account struct {
|
||||
AddressGenerator AddressGenerator `json:"address_generator"`
|
||||
Certificates map[string]string `json:"certificates"`
|
||||
Encrypted bool `json:"encrypted"`
|
||||
Ledger string `json:"ledger"`
|
||||
ModifiedOn float64 `json:"modified_on"`
|
||||
Name string `json:"name"`
|
||||
PrivateKey string `json:"private_key"`
|
||||
PublicKey string `json:"public_key"`
|
||||
Seed string `json:"seed"`
|
||||
}
|
||||
|
||||
type AddressGenerator struct {
|
||||
Name string `json:"name"`
|
||||
Change AddressGenParams `json:"change"` // should "change" and "receiving" be replaced with a map[string]AddressGenParams?
|
||||
Receiving AddressGenParams `json:"receiving"`
|
||||
}
|
||||
|
||||
type AddressGenParams struct {
|
||||
Gap int `json:"gap"`
|
||||
MaximumUsesPerAddress int `json:"maximum_uses_per_address"`
|
||||
}
|
||||
|
||||
type WalletPrefs struct {
|
||||
Shared struct {
|
||||
Ts float64 `json:"ts"`
|
||||
Value struct {
|
||||
Type string `json:"type"`
|
||||
Value struct {
|
||||
AppWelcomeVersion int `json:"app_welcome_version"`
|
||||
Blocked []interface{} `json:"blocked"`
|
||||
Sharing3P bool `json:"sharing_3P"`
|
||||
Subscriptions []string `json:"subscriptions"`
|
||||
Tags []string `json:"tags"`
|
||||
} `json:"value"`
|
||||
Version string `json:"version"`
|
||||
} `json:"value"`
|
||||
} `json:"shared"`
|
||||
}
|
||||
+48
-67
@@ -12,38 +12,44 @@ import (
|
||||
"github.com/lbryio/reflector.go/wallet"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
types1 "github.com/lbryio/types/v1/go"
|
||||
types2 "github.com/lbryio/types/v2/go"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
|
||||
"github.com/golang/protobuf/proto"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const blocklistURL = "https://api.lbry.com/file/list_blocked"
|
||||
|
||||
func (s *Server) enableBlocklist(b store.Blocklister) {
|
||||
// TODO: updateBlocklist should be killed when server is shutting down
|
||||
updateBlocklist(b)
|
||||
walletServers := []string{
|
||||
"spv25.lbry.com:50001",
|
||||
"spv26.lbry.com:50001",
|
||||
"spv19.lbry.com:50001",
|
||||
"spv14.lbry.com:50001",
|
||||
}
|
||||
|
||||
updateBlocklist(b, walletServers, s.grp.Ch())
|
||||
t := time.NewTicker(12 * time.Hour)
|
||||
for {
|
||||
select {
|
||||
case <-s.grp.Ch():
|
||||
return
|
||||
case <-t.C:
|
||||
updateBlocklist(b)
|
||||
updateBlocklist(b, walletServers, s.grp.Ch())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func updateBlocklist(b store.Blocklister) {
|
||||
values, err := blockedSdHashes()
|
||||
func updateBlocklist(b store.Blocklister, walletServers []string, stopper stop.Chan) {
|
||||
log.Debugf("blocklist update starting")
|
||||
values, err := blockedSdHashes(walletServers, stopper)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, v := range values {
|
||||
for name, v := range values {
|
||||
if v.Err != nil {
|
||||
log.Error(errors.FullTrace(errors.Err("blocklist: %s: %s", name, v.Err)))
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -52,17 +58,19 @@ func updateBlocklist(b store.Blocklister) {
|
||||
log.Error(err)
|
||||
}
|
||||
}
|
||||
log.Debugf("blocklist update done")
|
||||
}
|
||||
|
||||
func blockedSdHashes() (map[string]valOrErr, error) {
|
||||
resp, err := http.Get(blocklistURL)
|
||||
func blockedSdHashes(walletServers []string, stopper stop.Chan) (map[string]valOrErr, error) {
|
||||
client := http.Client{Timeout: 1 * time.Second}
|
||||
resp, err := client.Get(blocklistURL)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
defer func() {
|
||||
err := resp.Body.Close()
|
||||
if err != nil {
|
||||
log.Errorln(err)
|
||||
log.Errorln(errors.Err(err))
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -82,7 +90,7 @@ func blockedSdHashes() (map[string]valOrErr, error) {
|
||||
return nil, errors.Prefix("list_blocked API call", r.Error)
|
||||
}
|
||||
|
||||
return sdHashesForOutpoints(r.Data.Outpoints)
|
||||
return sdHashesForOutpoints(walletServers, r.Data.Outpoints, stopper)
|
||||
}
|
||||
|
||||
type valOrErr struct {
|
||||
@@ -91,23 +99,33 @@ type valOrErr struct {
|
||||
}
|
||||
|
||||
// sdHashesForOutpoints queries wallet server for the sd hashes in a given outpoints
|
||||
func sdHashesForOutpoints(outpoints []string) (map[string]valOrErr, error) {
|
||||
func sdHashesForOutpoints(walletServers, outpoints []string, stopper stop.Chan) (map[string]valOrErr, error) {
|
||||
values := make(map[string]valOrErr)
|
||||
|
||||
node := wallet.NewNode()
|
||||
defer node.Shutdown()
|
||||
err := node.Connect([]string{
|
||||
"spv1.lbry.com:50001",
|
||||
"spv2.lbry.com:50001",
|
||||
"spv3.lbry.com:50001",
|
||||
"spv4.lbry.com:50001",
|
||||
"spv5.lbry.com:50001",
|
||||
}, nil)
|
||||
err := node.Connect(walletServers, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
done := make(chan bool)
|
||||
|
||||
go func() {
|
||||
select {
|
||||
case <-done:
|
||||
case <-stopper:
|
||||
}
|
||||
node.Shutdown()
|
||||
}()
|
||||
|
||||
OutpointLoop:
|
||||
for _, outpoint := range outpoints {
|
||||
select {
|
||||
case <-stopper:
|
||||
break OutpointLoop
|
||||
default:
|
||||
}
|
||||
|
||||
parts := strings.Split(outpoint, ":")
|
||||
if len(parts) != 2 {
|
||||
values[outpoint] = valOrErr{Err: errors.Err("invalid outpoint format")}
|
||||
@@ -120,57 +138,20 @@ func sdHashesForOutpoints(outpoints []string) (map[string]valOrErr, error) {
|
||||
continue
|
||||
}
|
||||
|
||||
resp, err := node.GetClaimsInTx(parts[0])
|
||||
claim, err := node.GetClaimInTx(parts[0], nout)
|
||||
if err != nil {
|
||||
values[outpoint] = valOrErr{Err: err}
|
||||
continue
|
||||
}
|
||||
|
||||
var value []byte
|
||||
for _, tx := range resp.Result {
|
||||
if tx.Nout != nout {
|
||||
continue
|
||||
}
|
||||
hash := hex.EncodeToString(claim.GetStream().GetSource().GetSdHash())
|
||||
values[outpoint] = valOrErr{Value: hash, Err: nil}
|
||||
}
|
||||
|
||||
value, err = hex.DecodeString(tx.Value)
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
values[outpoint] = valOrErr{Err: err}
|
||||
continue
|
||||
} else if value == nil {
|
||||
values[outpoint] = valOrErr{Err: errors.Err("outpoint not found")}
|
||||
continue
|
||||
}
|
||||
|
||||
hash, err := hashFromClaim(value)
|
||||
values[outpoint] = valOrErr{Value: hash, Err: err}
|
||||
select {
|
||||
case done <- true:
|
||||
default: // in case of race where stopper got stopped right after loop finished
|
||||
}
|
||||
|
||||
return values, nil
|
||||
}
|
||||
|
||||
func hashFromClaim(value []byte) (string, error) {
|
||||
claim := &types1.Claim{}
|
||||
err := proto.Unmarshal(value, claim)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if claim.GetStream().GetSource().GetSourceType() == types1.Source_lbry_sd_hash && claim.GetStream().GetSource().GetSource() != nil {
|
||||
return hex.EncodeToString(claim.GetStream().GetSource().GetSource()), nil
|
||||
}
|
||||
|
||||
claim2 := &types2.Claim{}
|
||||
err = proto.Unmarshal(value, claim2)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
stream, ok := claim2.GetType().(*types2.Claim_Stream)
|
||||
if !ok || stream == nil {
|
||||
return "", errors.Err("not a stream claim")
|
||||
}
|
||||
|
||||
return hex.EncodeToString(claim2.GetStream().GetSource().GetSdHash()), nil
|
||||
}
|
||||
+59
-24
@@ -2,12 +2,11 @@ package reflector
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log"
|
||||
"net"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// ErrBlobExists is a default error for when a blob already exists on the reflector server.
|
||||
@@ -36,8 +35,18 @@ func (c *Client) Close() error {
|
||||
return c.conn.Close()
|
||||
}
|
||||
|
||||
// SendBlob sends a send blob request to the client.
|
||||
// SendBlob sends a blob to the server.
|
||||
func (c *Client) SendBlob(blob stream.Blob) error {
|
||||
return c.sendBlob(blob, false)
|
||||
}
|
||||
|
||||
// SendSDBlob sends an SD blob request to the server.
|
||||
func (c *Client) SendSDBlob(blob stream.Blob) error {
|
||||
return c.sendBlob(blob, true)
|
||||
}
|
||||
|
||||
// sendBlob does the actual blob sending
|
||||
func (c *Client) sendBlob(blob stream.Blob, isSDBlob bool) error {
|
||||
if !c.connected {
|
||||
return errors.Err("not connected")
|
||||
}
|
||||
@@ -47,10 +56,15 @@ func (c *Client) SendBlob(blob stream.Blob) error {
|
||||
}
|
||||
|
||||
blobHash := blob.HashHex()
|
||||
sendRequest, err := json.Marshal(sendBlobRequest{
|
||||
BlobSize: blob.Size(),
|
||||
BlobHash: blobHash,
|
||||
})
|
||||
var req sendBlobRequest
|
||||
if isSDBlob {
|
||||
req.SdBlobSize = blob.Size()
|
||||
req.SdBlobHash = blobHash
|
||||
} else {
|
||||
req.BlobSize = blob.Size()
|
||||
req.BlobHash = blobHash
|
||||
}
|
||||
sendRequest, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -62,30 +76,51 @@ func (c *Client) SendBlob(blob stream.Blob) error {
|
||||
|
||||
dec := json.NewDecoder(c.conn)
|
||||
|
||||
var sendResp sendBlobResponse
|
||||
err = dec.Decode(&sendResp)
|
||||
if err != nil {
|
||||
return err
|
||||
if isSDBlob {
|
||||
var sendResp sendSdBlobResponse
|
||||
err = dec.Decode(&sendResp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !sendResp.SendSdBlob {
|
||||
return errors.Prefix(blobHash[:8], ErrBlobExists)
|
||||
}
|
||||
log.Println("Sending SD blob " + blobHash[:8])
|
||||
} else {
|
||||
var sendResp sendBlobResponse
|
||||
err = dec.Decode(&sendResp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !sendResp.SendBlob {
|
||||
return errors.Prefix(blobHash[:8], ErrBlobExists)
|
||||
}
|
||||
log.Println("Sending blob " + blobHash[:8])
|
||||
}
|
||||
|
||||
if !sendResp.SendBlob {
|
||||
return errors.Prefix(blobHash[:8], ErrBlobExists)
|
||||
}
|
||||
|
||||
log.Println("Sending blob " + blobHash[:8])
|
||||
|
||||
_, err = c.conn.Write(blob)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var transferResp blobTransferResponse
|
||||
err = dec.Decode(&transferResp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !transferResp.ReceivedBlob {
|
||||
return errors.Err("server did not received blob")
|
||||
if isSDBlob {
|
||||
var transferResp sdBlobTransferResponse
|
||||
err = dec.Decode(&transferResp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !transferResp.ReceivedSdBlob {
|
||||
return errors.Err("server did not received SD blob")
|
||||
}
|
||||
} else {
|
||||
var transferResp blobTransferResponse
|
||||
err = dec.Decode(&transferResp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !transferResp.ReceivedBlob {
|
||||
return errors.Err("server did not received blob")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
+11
-16
@@ -8,9 +8,9 @@ import (
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
@@ -32,18 +32,16 @@ const (
|
||||
maxBlobSize = stream.MaxBlobSize
|
||||
)
|
||||
|
||||
var ErrBlobTooBig = errors.Base("blob must be at most %d bytes", maxBlobSize)
|
||||
|
||||
// Server is and instance of the reflector server. It houses the blob store and listener.
|
||||
type Server struct {
|
||||
Timeout time.Duration // timeout to read or write next message
|
||||
|
||||
StatLogger *log.Logger // logger to log stats
|
||||
StatReportFrequency time.Duration // how often to log stats
|
||||
|
||||
EnableBlocklist bool // if true, blocklist checking and blob deletion will be enabled
|
||||
|
||||
store store.BlobStore
|
||||
grp *stop.Group
|
||||
stats *Stats
|
||||
}
|
||||
|
||||
// NewServer returns an initialized reflector server pointer.
|
||||
@@ -58,7 +56,6 @@ func NewServer(store store.BlobStore) *Server {
|
||||
// Shutdown shuts down the reflector server gracefully.
|
||||
func (s *Server) Shutdown() {
|
||||
log.Println("shutting down reflector server...")
|
||||
s.stats.Shutdown()
|
||||
s.grp.StopAndWait()
|
||||
log.Println("reflector server stopped")
|
||||
}
|
||||
@@ -87,11 +84,6 @@ func (s *Server) Start(address string) error {
|
||||
s.grp.Done()
|
||||
}()
|
||||
|
||||
s.stats = NewStatLogger("UPLOAD", s.StatLogger, s.StatReportFrequency, s.grp.Child())
|
||||
if s.StatLogger != nil && s.StatReportFrequency > 0 {
|
||||
s.stats.Start()
|
||||
}
|
||||
|
||||
if s.EnableBlocklist {
|
||||
if b, ok := s.store.(store.Blocklister); ok {
|
||||
s.grp.Add(1)
|
||||
@@ -173,7 +165,10 @@ func (s *Server) handleConn(conn net.Conn) {
|
||||
}
|
||||
|
||||
func (s *Server) doError(conn net.Conn, err error) error {
|
||||
shouldLog := s.stats.AddError(err)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
shouldLog := metrics.TrackError(metrics.DirectionUpload, err)
|
||||
if shouldLog {
|
||||
log.Errorln(errors.FullTrace(err))
|
||||
}
|
||||
@@ -261,10 +256,10 @@ func (s *Server) receiveBlob(conn net.Conn) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s.stats.AddBlob()
|
||||
metrics.MtrInBytesReflector.Add(float64(len(blob)))
|
||||
metrics.BlobUploadCount.Inc()
|
||||
if isSdBlob {
|
||||
s.stats.AddStream()
|
||||
metrics.SDBlobUploadCount.Inc()
|
||||
}
|
||||
return s.sendTransferResponse(conn, true, isSdBlob)
|
||||
}
|
||||
@@ -311,7 +306,7 @@ func (s *Server) readBlobRequest(conn net.Conn) (int, string, bool, error) {
|
||||
return blobSize, blobHash, isSdBlob, errors.Err("blob hash is empty")
|
||||
}
|
||||
if blobSize > maxBlobSize {
|
||||
return blobSize, blobHash, isSdBlob, errors.Err("blob must be at most " + strconv.Itoa(maxBlobSize) + " bytes")
|
||||
return blobSize, blobHash, isSdBlob, errors.Err(ErrBlobTooBig)
|
||||
}
|
||||
if blobSize == 0 {
|
||||
return blobSize, blobHash, isSdBlob, errors.Err("0-byte blob received")
|
||||
|
||||
@@ -22,7 +22,7 @@ func startServerOnRandomPort(t *testing.T) (*Server, int) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
srv := NewServer(store.NewMemoryBlobStore())
|
||||
srv := NewServer(store.NewMemStore())
|
||||
err = srv.Start("127.0.0.1:" + strconv.Itoa(port))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -119,7 +119,7 @@ func TestServer_Timeout(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
srv := NewServer(store.NewMemoryBlobStore())
|
||||
srv := NewServer(store.NewMemStore())
|
||||
srv.Timeout = testTimeout
|
||||
err = srv.Start("127.0.0.1:" + strconv.Itoa(port))
|
||||
if err != nil {
|
||||
@@ -161,7 +161,7 @@ func TestServer_Timeout(t *testing.T) {
|
||||
//}
|
||||
|
||||
type mockPartialStore struct {
|
||||
*store.MemoryBlobStore
|
||||
*store.MemStore
|
||||
missing []string
|
||||
}
|
||||
|
||||
@@ -181,7 +181,7 @@ func TestServer_PartialUpload(t *testing.T) {
|
||||
missing[i] = bits.Rand().String()
|
||||
}
|
||||
|
||||
st := store.BlobStore(&mockPartialStore{MemoryBlobStore: store.NewMemoryBlobStore(), missing: missing})
|
||||
st := store.BlobStore(&mockPartialStore{MemStore: store.NewMemStore(), missing: missing})
|
||||
if _, ok := st.(neededBlobChecker); !ok {
|
||||
t.Fatal("mock does not implement the relevant interface")
|
||||
}
|
||||
|
||||
@@ -1,122 +0,0 @@
|
||||
package reflector
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// TODO: store daily stats too. and maybe other intervals
|
||||
|
||||
type Stats struct {
|
||||
mu *sync.Mutex
|
||||
blobs int
|
||||
streams int
|
||||
errors map[string]int
|
||||
started bool
|
||||
|
||||
name string
|
||||
logger *log.Logger
|
||||
logFreq time.Duration
|
||||
grp *stop.Group
|
||||
}
|
||||
|
||||
func NewStatLogger(name string, logger *log.Logger, logFreq time.Duration, parentGrp *stop.Group) *Stats {
|
||||
return &Stats{
|
||||
mu: &sync.Mutex{},
|
||||
grp: stop.New(parentGrp),
|
||||
logger: logger,
|
||||
logFreq: logFreq,
|
||||
errors: make(map[string]int),
|
||||
name: name,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Stats) Start() {
|
||||
s.started = true
|
||||
s.grp.Add(1)
|
||||
go func() {
|
||||
defer s.grp.Done()
|
||||
s.runSlackLogger()
|
||||
}()
|
||||
}
|
||||
|
||||
func (s *Stats) Shutdown() {
|
||||
if !s.started {
|
||||
return
|
||||
}
|
||||
s.log()
|
||||
s.grp.StopAndWait()
|
||||
s.started = false
|
||||
}
|
||||
|
||||
func (s *Stats) AddBlob() {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.blobs++
|
||||
}
|
||||
func (s *Stats) AddStream() {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.streams++
|
||||
}
|
||||
|
||||
func (s *Stats) AddError(e error) (shouldLog bool) { // shouldLog is a hack, but whatever
|
||||
if e == nil {
|
||||
return
|
||||
}
|
||||
err := errors.Wrap(e, 0)
|
||||
name := err.TypeName()
|
||||
if strings.Contains(err.Error(), "i/o timeout") { // hit a read or write deadline
|
||||
name = "i/o timeout"
|
||||
} else if strings.Contains(err.Error(), "read: connection reset by peer") { // the other side closed the connection using TCP reset
|
||||
name = "read conn reset"
|
||||
} else if strings.Contains(err.Error(), "unexpected EOF") { // tried to read from closed pipe or socket
|
||||
name = "unexpected EOF"
|
||||
} else if strings.Contains(err.Error(), "write: broken pipe") { // tried to write to a pipe or socket that was closed by the peer
|
||||
name = "write broken pipe"
|
||||
} else {
|
||||
shouldLog = true
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.errors[name]++
|
||||
return
|
||||
}
|
||||
|
||||
func (s *Stats) runSlackLogger() {
|
||||
t := time.NewTicker(s.logFreq)
|
||||
for {
|
||||
select {
|
||||
case <-s.grp.Ch():
|
||||
return
|
||||
case <-t.C:
|
||||
s.log()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Stats) log() {
|
||||
s.mu.Lock()
|
||||
blobs, streams := s.blobs, s.streams
|
||||
s.blobs, s.streams = 0, 0
|
||||
errStr := ""
|
||||
for name, count := range s.errors {
|
||||
errStr += fmt.Sprintf("%d %s, ", count, name)
|
||||
delete(s.errors, name)
|
||||
}
|
||||
s.mu.Unlock()
|
||||
|
||||
if len(errStr) > 2 {
|
||||
errStr = errStr[:len(errStr)-2] // trim last comma and space
|
||||
}
|
||||
|
||||
s.logger.Debugf("%s stats: %d blobs, %d streams, errors: %s", s.name, blobs, streams, errStr)
|
||||
}
|
||||
+22
-15
@@ -29,24 +29,26 @@ type Summary struct {
|
||||
}
|
||||
|
||||
type Uploader struct {
|
||||
db *db.SQL
|
||||
store *store.DBBackedStore // could just be store.BlobStore interface
|
||||
workers int
|
||||
skipExistsCheck bool
|
||||
stopper *stop.Group
|
||||
countChan chan increment
|
||||
db *db.SQL
|
||||
store *store.DBBackedStore // could just be store.BlobStore interface
|
||||
workers int
|
||||
skipExistsCheck bool
|
||||
deleteBlobsAfterUpload bool
|
||||
stopper *stop.Group
|
||||
countChan chan increment
|
||||
|
||||
count Summary
|
||||
}
|
||||
|
||||
func NewUploader(db *db.SQL, store *store.DBBackedStore, workers int, skipExistsCheck bool) *Uploader {
|
||||
func NewUploader(db *db.SQL, store *store.DBBackedStore, workers int, skipExistsCheck, deleteBlobsAfterUpload bool) *Uploader {
|
||||
return &Uploader{
|
||||
db: db,
|
||||
store: store,
|
||||
workers: workers,
|
||||
skipExistsCheck: skipExistsCheck,
|
||||
stopper: stop.New(),
|
||||
countChan: make(chan increment),
|
||||
db: db,
|
||||
store: store,
|
||||
workers: workers,
|
||||
skipExistsCheck: skipExistsCheck,
|
||||
deleteBlobsAfterUpload: deleteBlobsAfterUpload,
|
||||
stopper: stop.New(),
|
||||
countChan: make(chan increment),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -126,7 +128,7 @@ Upload:
|
||||
return nil
|
||||
}
|
||||
|
||||
// worker reads paths from a channel and uploads them
|
||||
// worker reads paths from a channel, uploads them, and optionally deletes them
|
||||
func (u *Uploader) worker(pathChan chan string) {
|
||||
for {
|
||||
select {
|
||||
@@ -140,6 +142,11 @@ func (u *Uploader) worker(pathChan chan string) {
|
||||
err := u.uploadBlob(filepath)
|
||||
if err != nil {
|
||||
log.Errorln(err)
|
||||
} else if u.deleteBlobsAfterUpload {
|
||||
err = os.Remove(filepath)
|
||||
if err != nil {
|
||||
log.Errorln(errors.Prefix("deleting blob", err))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -155,7 +162,7 @@ func (u *Uploader) uploadBlob(filepath string) (err error) {
|
||||
|
||||
blob, err := ioutil.ReadFile(filepath)
|
||||
if err != nil {
|
||||
return err
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
hash := BlobHash(blob)
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
// +build linux
|
||||
|
||||
package store
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
func timespecToTime(ts syscall.Timespec) time.Time {
|
||||
return time.Unix(int64(ts.Sec), int64(ts.Nsec))
|
||||
}
|
||||
|
||||
func atime(fi os.FileInfo) time.Time {
|
||||
return timespecToTime(fi.Sys().(*syscall.Stat_t).Atim)
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
// +build !linux
|
||||
|
||||
package store
|
||||
|
||||
import (
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
func atime(fi os.FileInfo) time.Time {
|
||||
return fi.ModTime()
|
||||
}
|
||||
+34
-11
@@ -1,24 +1,38 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
)
|
||||
|
||||
// CachingBlobStore combines two stores, typically a local and a remote store, to improve performance.
|
||||
// CachingStore combines two stores, typically a local and a remote store, to improve performance.
|
||||
// Accessed blobs are stored in and retrieved from the cache. If they are not in the cache, they
|
||||
// are retrieved from the origin and cached. Puts are cached and also forwarded to the origin.
|
||||
type CachingBlobStore struct {
|
||||
type CachingStore struct {
|
||||
origin, cache BlobStore
|
||||
component string
|
||||
}
|
||||
|
||||
// NewCachingBlobStore makes a new caching disk store and returns a pointer to it.
|
||||
func NewCachingBlobStore(origin, cache BlobStore) *CachingBlobStore {
|
||||
return &CachingBlobStore{origin: origin, cache: cache}
|
||||
// NewCachingStore makes a new caching disk store and returns a pointer to it.
|
||||
func NewCachingStore(component string, origin, cache BlobStore) *CachingStore {
|
||||
return &CachingStore{
|
||||
component: component,
|
||||
origin: WithSingleFlight(component, origin),
|
||||
cache: cache,
|
||||
}
|
||||
}
|
||||
|
||||
const nameCaching = "caching"
|
||||
|
||||
// Name is the cache type name
|
||||
func (c *CachingStore) Name() string { return nameCaching }
|
||||
|
||||
// Has checks the cache and then the origin for a hash. It returns true if either store has it.
|
||||
func (c *CachingBlobStore) Has(hash string) (bool, error) {
|
||||
func (c *CachingStore) Has(hash string) (bool, error) {
|
||||
has, err := c.cache.Has(hash)
|
||||
if has || err != nil {
|
||||
return has, err
|
||||
@@ -28,24 +42,33 @@ func (c *CachingBlobStore) Has(hash string) (bool, error) {
|
||||
|
||||
// Get tries to get the blob from the cache first, falling back to the origin. If the blob comes
|
||||
// from the origin, it is also stored in the cache.
|
||||
func (c *CachingBlobStore) Get(hash string) (stream.Blob, error) {
|
||||
func (c *CachingStore) Get(hash string) (stream.Blob, error) {
|
||||
start := time.Now()
|
||||
blob, err := c.cache.Get(hash)
|
||||
if err == nil || !errors.Is(err, ErrBlobNotFound) {
|
||||
metrics.CacheHitCount.With(metrics.CacheLabels(c.cache.Name(), c.component)).Inc()
|
||||
rate := float64(len(blob)) / 1024 / 1024 / time.Since(start).Seconds()
|
||||
metrics.CacheRetrievalSpeed.With(map[string]string{
|
||||
metrics.LabelCacheType: c.cache.Name(),
|
||||
metrics.LabelComponent: c.component,
|
||||
metrics.LabelSource: "cache",
|
||||
}).Set(rate)
|
||||
return blob, err
|
||||
}
|
||||
|
||||
metrics.CacheMissCount.With(metrics.CacheLabels(c.cache.Name(), c.component)).Inc()
|
||||
|
||||
blob, err = c.origin.Get(hash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = c.cache.Put(hash, blob)
|
||||
|
||||
return blob, err
|
||||
}
|
||||
|
||||
// Put stores the blob in the origin and the cache
|
||||
func (c *CachingBlobStore) Put(hash string, blob stream.Blob) error {
|
||||
func (c *CachingStore) Put(hash string, blob stream.Blob) error {
|
||||
err := c.origin.Put(hash, blob)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -54,7 +77,7 @@ func (c *CachingBlobStore) Put(hash string, blob stream.Blob) error {
|
||||
}
|
||||
|
||||
// PutSD stores the sd blob in the origin and the cache
|
||||
func (c *CachingBlobStore) PutSD(hash string, blob stream.Blob) error {
|
||||
func (c *CachingStore) PutSD(hash string, blob stream.Blob) error {
|
||||
err := c.origin.PutSD(hash, blob)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -63,7 +86,7 @@ func (c *CachingBlobStore) PutSD(hash string, blob stream.Blob) error {
|
||||
}
|
||||
|
||||
// Delete deletes the blob from the origin and the cache
|
||||
func (c *CachingBlobStore) Delete(hash string) error {
|
||||
func (c *CachingStore) Delete(hash string) error {
|
||||
err := c.origin.Delete(hash)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
+104
-8
@@ -2,13 +2,17 @@ package store
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
)
|
||||
|
||||
func TestCachingBlobStore_Put(t *testing.T) {
|
||||
origin := NewMemoryBlobStore()
|
||||
cache := NewMemoryBlobStore()
|
||||
s := NewCachingBlobStore(origin, cache)
|
||||
func TestCachingStore_Put(t *testing.T) {
|
||||
origin := NewMemStore()
|
||||
cache := NewMemStore()
|
||||
s := NewCachingStore("test", origin, cache)
|
||||
|
||||
b := []byte("this is a blob of stuff")
|
||||
hash := "hash"
|
||||
@@ -35,10 +39,10 @@ func TestCachingBlobStore_Put(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCachingBlobStore_CacheMiss(t *testing.T) {
|
||||
origin := NewMemoryBlobStore()
|
||||
cache := NewMemoryBlobStore()
|
||||
s := NewCachingBlobStore(origin, cache)
|
||||
func TestCachingStore_CacheMiss(t *testing.T) {
|
||||
origin := NewMemStore()
|
||||
cache := NewMemStore()
|
||||
s := NewCachingStore("test", origin, cache)
|
||||
|
||||
b := []byte("this is a blob of stuff")
|
||||
hash := "hash"
|
||||
@@ -71,3 +75,95 @@ func TestCachingBlobStore_CacheMiss(t *testing.T) {
|
||||
t.Errorf("expected cached Get() to return %s, got %s", string(b), string(res))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCachingStore_ThunderingHerd(t *testing.T) {
|
||||
storeDelay := 100 * time.Millisecond
|
||||
origin := NewSlowBlobStore(storeDelay)
|
||||
cache := NewMemStore()
|
||||
s := NewCachingStore("test", origin, cache)
|
||||
|
||||
b := []byte("this is a blob of stuff")
|
||||
hash := "hash"
|
||||
err := origin.Put(hash, b)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
wg := &sync.WaitGroup{}
|
||||
|
||||
getNoErr := func() {
|
||||
res, err := s.Get(hash)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(b, res) {
|
||||
t.Errorf("expected Get() to return %s, got %s", string(b), string(res))
|
||||
}
|
||||
wg.Done()
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
|
||||
wg.Add(4)
|
||||
go func() {
|
||||
go getNoErr()
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
go getNoErr()
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
go getNoErr()
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
go getNoErr()
|
||||
}()
|
||||
|
||||
wg.Wait()
|
||||
duration := time.Since(start)
|
||||
|
||||
// only the first getNoErr() should hit the origin. the rest should wait for the first request to return
|
||||
// once the first returns, the others should return immediately
|
||||
// therefore, if the delay much longer than 100ms, it means subsequent requests also went to the origin
|
||||
expectedMaxDelay := storeDelay + 5*time.Millisecond // a bit of extra time to let requests finish
|
||||
if duration > expectedMaxDelay {
|
||||
t.Errorf("Expected delay of at most %s, got %s", expectedMaxDelay, duration)
|
||||
}
|
||||
}
|
||||
|
||||
// SlowBlobStore adds a delay to each request
|
||||
type SlowBlobStore struct {
|
||||
mem *MemStore
|
||||
delay time.Duration
|
||||
}
|
||||
|
||||
func NewSlowBlobStore(delay time.Duration) *SlowBlobStore {
|
||||
return &SlowBlobStore{
|
||||
mem: NewMemStore(),
|
||||
delay: delay,
|
||||
}
|
||||
}
|
||||
func (s *SlowBlobStore) Name() string {
|
||||
return "slow"
|
||||
}
|
||||
|
||||
func (s *SlowBlobStore) Has(hash string) (bool, error) {
|
||||
time.Sleep(s.delay)
|
||||
return s.mem.Has(hash)
|
||||
}
|
||||
|
||||
func (s *SlowBlobStore) Get(hash string) (stream.Blob, error) {
|
||||
time.Sleep(s.delay)
|
||||
return s.mem.Get(hash)
|
||||
}
|
||||
|
||||
func (s *SlowBlobStore) Put(hash string, blob stream.Blob) error {
|
||||
time.Sleep(s.delay)
|
||||
return s.mem.Put(hash, blob)
|
||||
}
|
||||
|
||||
func (s *SlowBlobStore) PutSD(hash string, blob stream.Blob) error {
|
||||
time.Sleep(s.delay)
|
||||
return s.mem.PutSD(hash, blob)
|
||||
}
|
||||
|
||||
func (s *SlowBlobStore) Delete(hash string) error {
|
||||
time.Sleep(s.delay)
|
||||
return s.mem.Delete(hash)
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
"github.com/lbryio/reflector.go/meta"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// CloudFrontROStore reads from cloudfront. All writes panic.
|
||||
type CloudFrontROStore struct {
|
||||
endpoint string // cloudflare endpoint
|
||||
}
|
||||
|
||||
// NewCloudFrontROStore returns an initialized CloudFrontROStore store pointer.
|
||||
func NewCloudFrontROStore(endpoint string) *CloudFrontROStore {
|
||||
return &CloudFrontROStore{endpoint: endpoint}
|
||||
}
|
||||
|
||||
const nameCloudFrontRO = "cloudfront_ro"
|
||||
|
||||
// Name is the cache type name
|
||||
func (c *CloudFrontROStore) Name() string { return nameCloudFrontRO }
|
||||
|
||||
// Has checks if the hash is in the store.
|
||||
func (c *CloudFrontROStore) Has(hash string) (bool, error) {
|
||||
status, body, err := c.cfRequest(http.MethodHead, hash)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer body.Close()
|
||||
|
||||
switch status {
|
||||
case http.StatusNotFound, http.StatusForbidden:
|
||||
return false, nil
|
||||
case http.StatusOK:
|
||||
return true, nil
|
||||
default:
|
||||
return false, errors.Err("unexpected status %d", status)
|
||||
}
|
||||
}
|
||||
|
||||
// Get gets the blob from Cloudfront.
|
||||
func (c *CloudFrontROStore) Get(hash string) (stream.Blob, error) {
|
||||
log.Debugf("Getting %s from S3", hash[:8])
|
||||
defer func(t time.Time) {
|
||||
log.Debugf("Getting %s from S3 took %s", hash[:8], time.Since(t).String())
|
||||
}(time.Now())
|
||||
|
||||
status, body, err := c.cfRequest(http.MethodGet, hash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer body.Close()
|
||||
|
||||
switch status {
|
||||
case http.StatusNotFound, http.StatusForbidden:
|
||||
return nil, errors.Err(ErrBlobNotFound)
|
||||
case http.StatusOK:
|
||||
b, err := ioutil.ReadAll(body)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
metrics.MtrInBytesS3.Add(float64(len(b)))
|
||||
return b, nil
|
||||
default:
|
||||
return nil, errors.Err("unexpected status %d", status)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *CloudFrontROStore) cfRequest(method, hash string) (int, io.ReadCloser, error) {
|
||||
url := c.endpoint + hash
|
||||
req, err := http.NewRequest(method, url, nil)
|
||||
if err != nil {
|
||||
return 0, nil, errors.Err(err)
|
||||
}
|
||||
req.Header.Add("User-Agent", "reflector.go/"+meta.Version)
|
||||
|
||||
res, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return 0, nil, errors.Err(err)
|
||||
}
|
||||
|
||||
return res.StatusCode, res.Body, nil
|
||||
}
|
||||
|
||||
func (c *CloudFrontROStore) Put(_ string, _ stream.Blob) error {
|
||||
panic("CloudFrontROStore cannot do writes. Use CloudFrontRWStore")
|
||||
}
|
||||
|
||||
func (c *CloudFrontROStore) PutSD(_ string, _ stream.Blob) error {
|
||||
panic("CloudFrontROStore cannot do writes. Use CloudFrontRWStore")
|
||||
}
|
||||
|
||||
func (c *CloudFrontROStore) Delete(_ string) error {
|
||||
panic("CloudFrontROStore cannot do writes. Use CloudFrontRWStore")
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
)
|
||||
|
||||
// CloudFrontRWStore combines a Cloudfront and an S3 store. Reads go to Cloudfront, writes go to S3.
|
||||
type CloudFrontRWStore struct {
|
||||
cf *CloudFrontROStore
|
||||
s3 *S3Store
|
||||
}
|
||||
|
||||
// NewCloudFrontRWStore returns an initialized CloudFrontRWStore store pointer.
|
||||
// NOTE: It panics if either argument is nil.
|
||||
func NewCloudFrontRWStore(cf *CloudFrontROStore, s3 *S3Store) *CloudFrontRWStore {
|
||||
if cf == nil || s3 == nil {
|
||||
panic("both stores must be set")
|
||||
}
|
||||
return &CloudFrontRWStore{cf: cf, s3: s3}
|
||||
}
|
||||
|
||||
const nameCloudFrontRW = "cloudfront_rw"
|
||||
|
||||
// Name is the cache type name
|
||||
func (c *CloudFrontRWStore) Name() string { return nameCloudFrontRW }
|
||||
|
||||
// Has checks if the hash is in the store.
|
||||
func (c *CloudFrontRWStore) Has(hash string) (bool, error) {
|
||||
return c.cf.Has(hash)
|
||||
}
|
||||
|
||||
// Get gets the blob from Cloudfront.
|
||||
func (c *CloudFrontRWStore) Get(hash string) (stream.Blob, error) {
|
||||
return c.cf.Get(hash)
|
||||
}
|
||||
|
||||
// Put stores the blob on S3
|
||||
func (c *CloudFrontRWStore) Put(hash string, blob stream.Blob) error {
|
||||
return c.s3.Put(hash, blob)
|
||||
}
|
||||
|
||||
// PutSD stores the sd blob on S3
|
||||
func (c *CloudFrontRWStore) PutSD(hash string, blob stream.Blob) error {
|
||||
return c.s3.PutSD(hash, blob)
|
||||
}
|
||||
|
||||
// Delete deletes the blob from S3
|
||||
func (c *CloudFrontRWStore) Delete(hash string) error {
|
||||
return c.s3.Delete(hash)
|
||||
}
|
||||
+14
-1
@@ -25,6 +25,11 @@ func NewDBBackedStore(blobs BlobStore, db *db.SQL) *DBBackedStore {
|
||||
return &DBBackedStore{blobs: blobs, db: db}
|
||||
}
|
||||
|
||||
const nameDBBacked = "db-backed"
|
||||
|
||||
// Name is the cache type name
|
||||
func (d *DBBackedStore) Name() string { return nameDBBacked }
|
||||
|
||||
// Has returns true if the blob is in the store
|
||||
func (d *DBBackedStore) Has(hash string) (bool, error) {
|
||||
return d.db.HasBlob(hash)
|
||||
@@ -32,6 +37,14 @@ func (d *DBBackedStore) Has(hash string) (bool, error) {
|
||||
|
||||
// Get gets the blob
|
||||
func (d *DBBackedStore) Get(hash string) (stream.Blob, error) {
|
||||
has, err := d.db.HasBlob(hash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !has {
|
||||
return nil, ErrBlobNotFound
|
||||
}
|
||||
|
||||
return d.blobs.Get(hash)
|
||||
}
|
||||
|
||||
@@ -51,7 +64,7 @@ func (d *DBBackedStore) PutSD(hash string, blob stream.Blob) error {
|
||||
var blobContents db.SdBlob
|
||||
err := json.Unmarshal(blob, &blobContents)
|
||||
if err != nil {
|
||||
return err
|
||||
return errors.Err(err)
|
||||
}
|
||||
if blobContents.StreamHash == "" {
|
||||
return errors.Err("sd blob is missing stream hash")
|
||||
|
||||
+64
-40
@@ -7,54 +7,35 @@ import (
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
"github.com/lbryio/reflector.go/store/speedwalk"
|
||||
)
|
||||
|
||||
// DiskBlobStore stores blobs on a local disk
|
||||
type DiskBlobStore struct {
|
||||
// DiskStore stores blobs on a local disk
|
||||
type DiskStore struct {
|
||||
// the location of blobs on disk
|
||||
blobDir string
|
||||
// store files in subdirectories based on the first N chars in the filename. 0 = don't create subdirectories.
|
||||
prefixLength int
|
||||
|
||||
// true if initOnce ran, false otherwise
|
||||
initialized bool
|
||||
}
|
||||
|
||||
// NewDiskBlobStore returns an initialized file disk store pointer.
|
||||
func NewDiskBlobStore(dir string, prefixLength int) *DiskBlobStore {
|
||||
return &DiskBlobStore{blobDir: dir, prefixLength: prefixLength}
|
||||
}
|
||||
|
||||
func (d *DiskBlobStore) dir(hash string) string {
|
||||
if d.prefixLength <= 0 || len(hash) < d.prefixLength {
|
||||
return d.blobDir
|
||||
// NewDiskStore returns an initialized file disk store pointer.
|
||||
func NewDiskStore(dir string, prefixLength int) *DiskStore {
|
||||
return &DiskStore{
|
||||
blobDir: dir,
|
||||
prefixLength: prefixLength,
|
||||
}
|
||||
return path.Join(d.blobDir, hash[:d.prefixLength])
|
||||
}
|
||||
|
||||
func (d *DiskBlobStore) path(hash string) string {
|
||||
return path.Join(d.dir(hash), hash)
|
||||
}
|
||||
const nameDisk = "disk"
|
||||
|
||||
func (d *DiskBlobStore) ensureDirExists(dir string) error {
|
||||
return errors.Err(os.MkdirAll(dir, 0755))
|
||||
}
|
||||
|
||||
func (d *DiskBlobStore) initOnce() error {
|
||||
if d.initialized {
|
||||
return nil
|
||||
}
|
||||
|
||||
err := d.ensureDirExists(d.blobDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
d.initialized = true
|
||||
return nil
|
||||
}
|
||||
// Name is the cache type name
|
||||
func (d *DiskStore) Name() string { return nameDisk }
|
||||
|
||||
// Has returns T/F or Error if it the blob stored already. It will error with any IO disk error.
|
||||
func (d *DiskBlobStore) Has(hash string) (bool, error) {
|
||||
func (d *DiskStore) Has(hash string) (bool, error) {
|
||||
err := d.initOnce()
|
||||
if err != nil {
|
||||
return false, err
|
||||
@@ -65,13 +46,13 @@ func (d *DiskBlobStore) Has(hash string) (bool, error) {
|
||||
if os.IsNotExist(err) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
return false, errors.Err(err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// Get returns the blob or an error if the blob doesn't exist.
|
||||
func (d *DiskBlobStore) Get(hash string) (stream.Blob, error) {
|
||||
func (d *DiskStore) Get(hash string) (stream.Blob, error) {
|
||||
err := d.initOnce()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -84,12 +65,14 @@ func (d *DiskBlobStore) Get(hash string) (stream.Blob, error) {
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
return ioutil.ReadAll(file)
|
||||
blob, err := ioutil.ReadAll(file)
|
||||
return blob, errors.Err(err)
|
||||
}
|
||||
|
||||
// Put stores the blob on disk
|
||||
func (d *DiskBlobStore) Put(hash string, blob stream.Blob) error {
|
||||
func (d *DiskStore) Put(hash string, blob stream.Blob) error {
|
||||
err := d.initOnce()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -100,16 +83,17 @@ func (d *DiskBlobStore) Put(hash string, blob stream.Blob) error {
|
||||
return err
|
||||
}
|
||||
|
||||
return ioutil.WriteFile(d.path(hash), blob, 0644)
|
||||
err = ioutil.WriteFile(d.path(hash), blob, 0644)
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
// PutSD stores the sd blob on the disk
|
||||
func (d *DiskBlobStore) PutSD(hash string, blob stream.Blob) error {
|
||||
func (d *DiskStore) PutSD(hash string, blob stream.Blob) error {
|
||||
return d.Put(hash, blob)
|
||||
}
|
||||
|
||||
// Delete deletes the blob from the store
|
||||
func (d *DiskBlobStore) Delete(hash string) error {
|
||||
func (d *DiskStore) Delete(hash string) error {
|
||||
err := d.initOnce()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -123,5 +107,45 @@ func (d *DiskBlobStore) Delete(hash string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
return os.Remove(d.path(hash))
|
||||
err = os.Remove(d.path(hash))
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
// list returns the hashes of blobs that already exist in the blobDir
|
||||
func (d *DiskStore) list() ([]string, error) {
|
||||
err := d.initOnce()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return speedwalk.AllFiles(d.blobDir, true)
|
||||
}
|
||||
|
||||
func (d *DiskStore) dir(hash string) string {
|
||||
if d.prefixLength <= 0 || len(hash) < d.prefixLength {
|
||||
return d.blobDir
|
||||
}
|
||||
return path.Join(d.blobDir, hash[:d.prefixLength])
|
||||
}
|
||||
|
||||
func (d *DiskStore) path(hash string) string {
|
||||
return path.Join(d.dir(hash), hash)
|
||||
}
|
||||
|
||||
func (d *DiskStore) ensureDirExists(dir string) error {
|
||||
return errors.Err(os.MkdirAll(dir, 0755))
|
||||
}
|
||||
|
||||
func (d *DiskStore) initOnce() error {
|
||||
if d.initialized {
|
||||
return nil
|
||||
}
|
||||
|
||||
err := d.ensureDirExists(d.blobDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
d.initialized = true
|
||||
return nil
|
||||
}
|
||||
+120
@@ -0,0 +1,120 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
|
||||
golru "github.com/hashicorp/golang-lru"
|
||||
)
|
||||
|
||||
// LRUStore adds a max cache size and LRU eviction to a BlobStore
|
||||
type LRUStore struct {
|
||||
// underlying store
|
||||
store BlobStore
|
||||
// lru implementation
|
||||
lru *golru.Cache
|
||||
}
|
||||
|
||||
// NewLRUStore initialize a new LRUStore
|
||||
func NewLRUStore(component string, store BlobStore, maxItems int) *LRUStore {
|
||||
lru, err := golru.NewWithEvict(maxItems, func(key interface{}, value interface{}) {
|
||||
metrics.CacheLRUEvictCount.With(metrics.CacheLabels(store.Name(), component)).Inc()
|
||||
_ = store.Delete(key.(string)) // TODO: log this error. may happen if underlying entry is gone but cache entry still there
|
||||
})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
l := &LRUStore{
|
||||
store: store,
|
||||
lru: lru,
|
||||
}
|
||||
|
||||
if lstr, ok := store.(lister); ok {
|
||||
err = l.loadExisting(lstr, maxItems)
|
||||
if err != nil {
|
||||
panic(err) // TODO: what should happen here? panic? return nil? just keep going?
|
||||
}
|
||||
}
|
||||
|
||||
return l
|
||||
}
|
||||
|
||||
const nameLRU = "lru"
|
||||
|
||||
// Name is the cache type name
|
||||
func (l *LRUStore) Name() string { return nameLRU }
|
||||
|
||||
// Has returns whether the blob is in the store, without updating the recent-ness.
|
||||
func (l *LRUStore) Has(hash string) (bool, error) {
|
||||
return l.lru.Contains(hash), nil
|
||||
}
|
||||
|
||||
// Get returns the blob or an error if the blob doesn't exist.
|
||||
func (l *LRUStore) Get(hash string) (stream.Blob, error) {
|
||||
_, has := l.lru.Get(hash)
|
||||
if !has {
|
||||
return nil, errors.Err(ErrBlobNotFound)
|
||||
}
|
||||
blob, err := l.store.Get(hash)
|
||||
if errors.Is(err, ErrBlobNotFound) {
|
||||
// Blob disappeared from underlying store
|
||||
l.lru.Remove(hash)
|
||||
}
|
||||
return blob, err
|
||||
}
|
||||
|
||||
// Put stores the blob
|
||||
func (l *LRUStore) Put(hash string, blob stream.Blob) error {
|
||||
err := l.store.Put(hash, blob)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
l.lru.Add(hash, true)
|
||||
return nil
|
||||
}
|
||||
|
||||
// PutSD stores the sd blob
|
||||
func (l *LRUStore) PutSD(hash string, blob stream.Blob) error {
|
||||
err := l.store.PutSD(hash, blob)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
l.lru.Add(hash, true)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete deletes the blob from the store
|
||||
func (l *LRUStore) Delete(hash string) error {
|
||||
err := l.store.Delete(hash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// This must come after store.Delete()
|
||||
// Remove triggers onEvict function, which also tries to delete blob from store
|
||||
// We need to delete it manually first so any errors can be propagated up
|
||||
l.lru.Remove(hash)
|
||||
return nil
|
||||
}
|
||||
|
||||
// loadExisting imports existing blobs from the underlying store into the LRU cache
|
||||
func (l *LRUStore) loadExisting(store lister, maxItems int) error {
|
||||
existing, err := store.list()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
added := 0
|
||||
for _, h := range existing {
|
||||
l.lru.Add(h, true)
|
||||
added++
|
||||
if maxItems > 0 && added >= maxItems { // underlying cache is bigger than LRU cache
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const cacheMaxBlobs = 3
|
||||
|
||||
func getTestLRUStore() (*LRUStore, *MemStore) {
|
||||
m := NewMemStore()
|
||||
return NewLRUStore("test", m, 3), m
|
||||
}
|
||||
|
||||
func TestLRUStore_Eviction(t *testing.T) {
|
||||
lru, mem := getTestLRUStore()
|
||||
b := []byte("x")
|
||||
err := lru.Put("one", b)
|
||||
require.NoError(t, err)
|
||||
err = lru.Put("two", b)
|
||||
require.NoError(t, err)
|
||||
err = lru.Put("three", b)
|
||||
require.NoError(t, err)
|
||||
err = lru.Put("four", b)
|
||||
require.NoError(t, err)
|
||||
err = lru.Put("five", b)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, cacheMaxBlobs, len(mem.Debug()))
|
||||
|
||||
for k, v := range map[string]bool{
|
||||
"one": false,
|
||||
"two": false,
|
||||
"three": true,
|
||||
"four": true,
|
||||
"five": true,
|
||||
"six": false,
|
||||
} {
|
||||
has, err := lru.Has(k)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, v, has)
|
||||
}
|
||||
|
||||
lru.Get("three") // touch so it stays in cache
|
||||
lru.Put("six", b)
|
||||
|
||||
assert.Equal(t, cacheMaxBlobs, len(mem.Debug()))
|
||||
|
||||
for k, v := range map[string]bool{
|
||||
"one": false,
|
||||
"two": false,
|
||||
"three": true,
|
||||
"four": false,
|
||||
"five": true,
|
||||
"six": true,
|
||||
} {
|
||||
has, err := lru.Has(k)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, v, has)
|
||||
}
|
||||
|
||||
err = lru.Delete("three")
|
||||
assert.NoError(t, err)
|
||||
err = lru.Delete("five")
|
||||
assert.NoError(t, err)
|
||||
err = lru.Delete("six")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 0, len(mem.Debug()))
|
||||
}
|
||||
|
||||
func TestLRUStore_UnderlyingBlobMissing(t *testing.T) {
|
||||
lru, mem := getTestLRUStore()
|
||||
hash := "hash"
|
||||
b := []byte("this is a blob of stuff")
|
||||
err := lru.Put(hash, b)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = mem.Delete(hash)
|
||||
require.NoError(t, err)
|
||||
|
||||
// hash still exists in lru
|
||||
assert.True(t, lru.lru.Contains(hash))
|
||||
|
||||
blob, err := lru.Get(hash)
|
||||
assert.Nil(t, blob)
|
||||
assert.True(t, errors.Is(err, ErrBlobNotFound), "expected (%s) %s, got (%s) %s",
|
||||
reflect.TypeOf(ErrBlobNotFound).String(), ErrBlobNotFound.Error(),
|
||||
reflect.TypeOf(err).String(), err.Error())
|
||||
|
||||
// lru.Get() removes hash if underlying store doesn't have it
|
||||
assert.False(t, lru.lru.Contains(hash))
|
||||
}
|
||||
|
||||
func TestLRUStore_loadExisting(t *testing.T) {
|
||||
tmpDir, err := ioutil.TempDir("", "reflector_test_*")
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(tmpDir)
|
||||
d := NewDiskStore(tmpDir, 2)
|
||||
|
||||
hash := "hash"
|
||||
b := []byte("this is a blob of stuff")
|
||||
err = d.Put(hash, b)
|
||||
require.NoError(t, err)
|
||||
|
||||
existing, err := d.list()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, len(existing), "blob should exist in cache")
|
||||
assert.Equal(t, hash, existing[0])
|
||||
|
||||
lru := NewLRUStore("test", d, 3) // lru should load existing blobs when it's created
|
||||
has, err := lru.Has(hash)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, has, "hash should be loaded from disk store but it's not")
|
||||
}
|
||||
+29
-10
@@ -1,29 +1,42 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
)
|
||||
|
||||
// MemoryBlobStore is an in memory only blob store with no persistence.
|
||||
type MemoryBlobStore struct {
|
||||
// MemStore is an in memory only blob store with no persistence.
|
||||
type MemStore struct {
|
||||
blobs map[string]stream.Blob
|
||||
mu *sync.RWMutex
|
||||
}
|
||||
|
||||
func NewMemoryBlobStore() *MemoryBlobStore {
|
||||
return &MemoryBlobStore{
|
||||
func NewMemStore() *MemStore {
|
||||
return &MemStore{
|
||||
blobs: make(map[string]stream.Blob),
|
||||
mu: &sync.RWMutex{},
|
||||
}
|
||||
}
|
||||
|
||||
const nameMem = "mem"
|
||||
|
||||
// Name is the cache type name
|
||||
func (m *MemStore) Name() string { return nameMem }
|
||||
|
||||
// Has returns T/F if the blob is currently stored. It will never error.
|
||||
func (m *MemoryBlobStore) Has(hash string) (bool, error) {
|
||||
func (m *MemStore) Has(hash string) (bool, error) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
_, ok := m.blobs[hash]
|
||||
return ok, nil
|
||||
}
|
||||
|
||||
// Get returns the blob byte slice if present and errors if the blob is not found.
|
||||
func (m *MemoryBlobStore) Get(hash string) (stream.Blob, error) {
|
||||
func (m *MemStore) Get(hash string) (stream.Blob, error) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
blob, ok := m.blobs[hash]
|
||||
if !ok {
|
||||
return nil, errors.Err(ErrBlobNotFound)
|
||||
@@ -32,23 +45,29 @@ func (m *MemoryBlobStore) Get(hash string) (stream.Blob, error) {
|
||||
}
|
||||
|
||||
// Put stores the blob in memory
|
||||
func (m *MemoryBlobStore) Put(hash string, blob stream.Blob) error {
|
||||
func (m *MemStore) Put(hash string, blob stream.Blob) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.blobs[hash] = blob
|
||||
return nil
|
||||
}
|
||||
|
||||
// PutSD stores the sd blob in memory
|
||||
func (m *MemoryBlobStore) PutSD(hash string, blob stream.Blob) error {
|
||||
func (m *MemStore) PutSD(hash string, blob stream.Blob) error {
|
||||
return m.Put(hash, blob)
|
||||
}
|
||||
|
||||
// Delete deletes the blob from the store
|
||||
func (m *MemoryBlobStore) Delete(hash string) error {
|
||||
func (m *MemStore) Delete(hash string) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
delete(m.blobs, hash)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Debug returns the blobs in memory. It's useful for testing and debugging.
|
||||
func (m *MemoryBlobStore) Debug() map[string]stream.Blob {
|
||||
func (m *MemStore) Debug() map[string]stream.Blob {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
return m.blobs
|
||||
}
|
||||
@@ -7,8 +7,8 @@ import (
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
)
|
||||
|
||||
func TestMemoryBlobStore_Put(t *testing.T) {
|
||||
s := NewMemoryBlobStore()
|
||||
func TestMemStore_Put(t *testing.T) {
|
||||
s := NewMemStore()
|
||||
blob := []byte("abcdefg")
|
||||
err := s.Put("abc", blob)
|
||||
if err != nil {
|
||||
@@ -16,8 +16,8 @@ func TestMemoryBlobStore_Put(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMemoryBlobStore_Get(t *testing.T) {
|
||||
s := NewMemoryBlobStore()
|
||||
func TestMemStore_Get(t *testing.T) {
|
||||
s := NewMemStore()
|
||||
hash := "abc"
|
||||
blob := []byte("abcdefg")
|
||||
err := s.Put(hash, blob)
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
package store
|
||||
|
||||
import "github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
// NoopStore is a store that does nothing
|
||||
type NoopStore struct{}
|
||||
|
||||
const nameNoop = "noop"
|
||||
|
||||
func (n *NoopStore) Name() string { return nameNoop }
|
||||
func (n *NoopStore) Has(_ string) (bool, error) { return false, nil }
|
||||
func (n *NoopStore) Get(_ string) (stream.Blob, error) { return nil, nil }
|
||||
func (n *NoopStore) Put(_ string, _ stream.Blob) error { return nil }
|
||||
func (n *NoopStore) PutSD(_ string, _ stream.Blob) error { return nil }
|
||||
func (n *NoopStore) Delete(_ string) error { return nil }
|
||||
+32
-25
@@ -7,6 +7,7 @@ import (
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
@@ -17,8 +18,8 @@ import (
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// S3BlobStore is an S3 store
|
||||
type S3BlobStore struct {
|
||||
// S3Store is an S3 store
|
||||
type S3Store struct {
|
||||
awsID string
|
||||
awsSecret string
|
||||
region string
|
||||
@@ -27,9 +28,9 @@ type S3BlobStore struct {
|
||||
session *session.Session
|
||||
}
|
||||
|
||||
// NewS3BlobStore returns an initialized S3 store pointer.
|
||||
func NewS3BlobStore(awsID, awsSecret, region, bucket string) *S3BlobStore {
|
||||
return &S3BlobStore{
|
||||
// NewS3Store returns an initialized S3 store pointer.
|
||||
func NewS3Store(awsID, awsSecret, region, bucket string) *S3Store {
|
||||
return &S3Store{
|
||||
awsID: awsID,
|
||||
awsSecret: awsSecret,
|
||||
region: region,
|
||||
@@ -37,25 +38,13 @@ func NewS3BlobStore(awsID, awsSecret, region, bucket string) *S3BlobStore {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *S3BlobStore) initOnce() error {
|
||||
if s.session != nil {
|
||||
return nil
|
||||
}
|
||||
const nameS3 = "s3"
|
||||
|
||||
sess, err := session.NewSession(&aws.Config{
|
||||
Credentials: credentials.NewStaticCredentials(s.awsID, s.awsSecret, ""),
|
||||
Region: aws.String(s.region),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s.session = sess
|
||||
return nil
|
||||
}
|
||||
// Name is the cache type name
|
||||
func (s *S3Store) Name() string { return nameS3 }
|
||||
|
||||
// Has returns T/F or Error ( from S3 ) if the store contains the blob.
|
||||
func (s *S3BlobStore) Has(hash string) (bool, error) {
|
||||
func (s *S3Store) Has(hash string) (bool, error) {
|
||||
err := s.initOnce()
|
||||
if err != nil {
|
||||
return false, err
|
||||
@@ -76,7 +65,7 @@ func (s *S3BlobStore) Has(hash string) (bool, error) {
|
||||
}
|
||||
|
||||
// Get returns the blob slice if present or errors on S3.
|
||||
func (s *S3BlobStore) Get(hash string) (stream.Blob, error) {
|
||||
func (s *S3Store) Get(hash string) (stream.Blob, error) {
|
||||
//Todo-Need to handle error for blob doesn't exist for consistency.
|
||||
err := s.initOnce()
|
||||
if err != nil {
|
||||
@@ -109,7 +98,7 @@ func (s *S3BlobStore) Get(hash string) (stream.Blob, error) {
|
||||
}
|
||||
|
||||
// Put stores the blob on S3 or errors if S3 connection errors.
|
||||
func (s *S3BlobStore) Put(hash string, blob stream.Blob) error {
|
||||
func (s *S3Store) Put(hash string, blob stream.Blob) error {
|
||||
err := s.initOnce()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -126,17 +115,18 @@ func (s *S3BlobStore) Put(hash string, blob stream.Blob) error {
|
||||
Body: bytes.NewBuffer(blob),
|
||||
StorageClass: aws.String(s3.StorageClassIntelligentTiering),
|
||||
})
|
||||
metrics.MtrOutBytesReflector.Add(float64(blob.Size()))
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// PutSD stores the sd blob on S3 or errors if S3 connection errors.
|
||||
func (s *S3BlobStore) PutSD(hash string, blob stream.Blob) error {
|
||||
func (s *S3Store) PutSD(hash string, blob stream.Blob) error {
|
||||
//Todo - handle missing stream for consistency
|
||||
return s.Put(hash, blob)
|
||||
}
|
||||
|
||||
func (s *S3BlobStore) Delete(hash string) error {
|
||||
func (s *S3Store) Delete(hash string) error {
|
||||
err := s.initOnce()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -151,3 +141,20 @@ func (s *S3BlobStore) Delete(hash string) error {
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *S3Store) initOnce() error {
|
||||
if s.session != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
sess, err := session.NewSession(&aws.Config{
|
||||
Credentials: credentials.NewStaticCredentials(s.awsID, s.awsSecret, ""),
|
||||
Region: aws.String(s.region),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s.session = sess
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
"golang.org/x/sync/singleflight"
|
||||
)
|
||||
|
||||
func WithSingleFlight(component string, origin BlobStore) BlobStore {
|
||||
return &singleflightStore{
|
||||
BlobStore: origin,
|
||||
component: component,
|
||||
sf: new(singleflight.Group),
|
||||
}
|
||||
}
|
||||
|
||||
type singleflightStore struct {
|
||||
BlobStore
|
||||
|
||||
component string
|
||||
sf *singleflight.Group
|
||||
}
|
||||
|
||||
func (s *singleflightStore) Name() string {
|
||||
return "sf_" + s.BlobStore.Name()
|
||||
}
|
||||
|
||||
// Get ensures that only one request per hash is sent to the origin at a time,
|
||||
// thereby protecting against https://en.wikipedia.org/wiki/Thundering_herd_problem
|
||||
func (s *singleflightStore) Get(hash string) (stream.Blob, error) {
|
||||
metrics.CacheWaitingRequestsCount.With(metrics.CacheLabels(s.BlobStore.Name(), s.component)).Inc()
|
||||
defer metrics.CacheWaitingRequestsCount.With(metrics.CacheLabels(s.BlobStore.Name(), s.component)).Dec()
|
||||
|
||||
blob, err, _ := s.sf.Do(hash, s.getter(hash))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return blob.(stream.Blob), nil
|
||||
}
|
||||
|
||||
// getter returns a function that gets a blob from the origin
|
||||
// only one getter per hash will be executing at a time
|
||||
func (s *singleflightStore) getter(hash string) func() (interface{}, error) {
|
||||
return func() (interface{}, error) {
|
||||
metrics.CacheOriginRequestsCount.With(metrics.CacheLabels(s.BlobStore.Name(), s.component)).Inc()
|
||||
defer metrics.CacheOriginRequestsCount.With(metrics.CacheLabels(s.BlobStore.Name(), s.component)).Dec()
|
||||
|
||||
start := time.Now()
|
||||
blob, err := s.BlobStore.Get(hash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
rate := float64(len(blob)) / 1024 / 1024 / time.Since(start).Seconds()
|
||||
metrics.CacheRetrievalSpeed.With(map[string]string{
|
||||
metrics.LabelCacheType: s.BlobStore.Name(),
|
||||
metrics.LabelComponent: s.component,
|
||||
metrics.LabelSource: "origin",
|
||||
}).Set(rate)
|
||||
|
||||
return blob, nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
package speedwalk
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sync"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
|
||||
"github.com/karrick/godirwalk"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// AllFiles recursively lists every file in every subdirectory of a given directory
|
||||
// If basename is true, return the basename of each file. Otherwise return the full path starting at startDir.
|
||||
func AllFiles(startDir string, basename bool) ([]string, error) {
|
||||
items, err := ioutil.ReadDir(startDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
pathChan := make(chan string)
|
||||
paths := make([]string, 0, 1000)
|
||||
pathWG := &sync.WaitGroup{}
|
||||
pathWG.Add(1)
|
||||
go func() {
|
||||
defer pathWG.Done()
|
||||
for {
|
||||
path, ok := <-pathChan
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
paths = append(paths, path)
|
||||
}
|
||||
}()
|
||||
|
||||
maxThreads := runtime.NumCPU() - 1
|
||||
goroutineLimiter := make(chan struct{}, maxThreads)
|
||||
for i := 0; i < maxThreads; i++ {
|
||||
goroutineLimiter <- struct{}{}
|
||||
}
|
||||
|
||||
walkerWG := &sync.WaitGroup{}
|
||||
for _, item := range items {
|
||||
if !item.IsDir() {
|
||||
if basename {
|
||||
pathChan <- item.Name()
|
||||
} else {
|
||||
pathChan <- filepath.Join(startDir, item.Name())
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
<-goroutineLimiter
|
||||
walkerWG.Add(1)
|
||||
|
||||
go func(dir string) {
|
||||
defer func() {
|
||||
walkerWG.Done()
|
||||
goroutineLimiter <- struct{}{}
|
||||
}()
|
||||
|
||||
err = godirwalk.Walk(filepath.Join(startDir, dir), &godirwalk.Options{
|
||||
Unsorted: true, // faster this way
|
||||
Callback: func(osPathname string, de *godirwalk.Dirent) error {
|
||||
if de.IsRegular() {
|
||||
if basename {
|
||||
pathChan <- de.Name()
|
||||
} else {
|
||||
pathChan <- osPathname
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
logrus.Errorf(errors.FullTrace(err))
|
||||
}
|
||||
}(item.Name())
|
||||
}
|
||||
|
||||
walkerWG.Wait()
|
||||
|
||||
close(pathChan)
|
||||
pathWG.Wait()
|
||||
|
||||
return paths, nil
|
||||
}
|
||||
+13
-5
@@ -7,15 +7,17 @@ import (
|
||||
|
||||
// BlobStore is an interface for handling blob storage.
|
||||
type BlobStore interface {
|
||||
// Does blob exist in the store
|
||||
// Name of blob store (useful for metrics)
|
||||
Name() string
|
||||
// Does blob exist in the store.
|
||||
Has(hash string) (bool, error)
|
||||
// Get the blob from the store
|
||||
// Get the blob from the store. Must return ErrBlobNotFound if blob is not in store.
|
||||
Get(hash string) (stream.Blob, error)
|
||||
// Put the blob into the store
|
||||
// Put the blob into the store.
|
||||
Put(hash string, blob stream.Blob) error
|
||||
// Put an SD blob into the store
|
||||
// Put an SD blob into the store.
|
||||
PutSD(hash string, blob stream.Blob) error
|
||||
// Delete the blob from the store
|
||||
// Delete the blob from the store.
|
||||
Delete(hash string) error
|
||||
}
|
||||
|
||||
@@ -27,5 +29,11 @@ type Blocklister interface {
|
||||
Wants(hash string) (bool, error)
|
||||
}
|
||||
|
||||
// lister is a store that can list cached blobs. This is helpful when an overlay
|
||||
// cache needs to track blob existence.
|
||||
type lister interface {
|
||||
list() ([]string, error)
|
||||
}
|
||||
|
||||
//ErrBlobNotFound is a standard error when a blob is not found in the store.
|
||||
var ErrBlobNotFound = errors.Base("blob not found")
|
||||
@@ -1,5 +1,27 @@
|
||||
package wallet
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"strings"
|
||||
|
||||
"github.com/lbryio/chainquery/lbrycrd"
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/schema/claim"
|
||||
types "github.com/lbryio/types/v2/go"
|
||||
|
||||
"github.com/btcsuite/btcutil"
|
||||
"github.com/golang/protobuf/proto"
|
||||
"github.com/spf13/cast"
|
||||
)
|
||||
|
||||
var ErrClaimNotFound = errors.Base("claim not found")
|
||||
|
||||
// Raw makes a raw wallet server request
|
||||
func (n *Node) Raw(method string, params []string, v interface{}) error {
|
||||
return n.request(method, params, v)
|
||||
}
|
||||
|
||||
// ServerVersion returns the server's version.
|
||||
// https://electrumx.readthedocs.io/en/latest/protocol-methods.html#server-version
|
||||
func (n *Node) ServerVersion() (string, error) {
|
||||
@@ -16,6 +38,44 @@ func (n *Node) ServerVersion() (string, error) {
|
||||
return v, err
|
||||
}
|
||||
|
||||
func (n *Node) Resolve(url string) (*types.Output, error) {
|
||||
outputs := &types.Outputs{}
|
||||
resp := &struct {
|
||||
Result string `json:"result"`
|
||||
}{}
|
||||
|
||||
err := n.request("blockchain.claimtrie.resolve", []string{url}, resp)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
b, err := base64.StdEncoding.DecodeString(resp.Result)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
err = proto.Unmarshal(b, outputs)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
if len(outputs.GetTxos()) != 1 {
|
||||
return nil, errors.Err("expected 1 output, got " + cast.ToString(len(outputs.GetTxos())))
|
||||
}
|
||||
|
||||
if e := outputs.GetTxos()[0].GetError(); e != nil {
|
||||
// TODO: return these errors as real error values so callers don't have to string-match
|
||||
// https://github.com/lbryio/types/blob/master/v2/proto/result.proto#L45
|
||||
// UNKNOWN_CODE = 0;
|
||||
// NOT_FOUND = 1;
|
||||
// INVALID = 2;
|
||||
// BLOCKED = 3;
|
||||
return nil, errors.Err("%s: %s", e.GetCode(), e.GetText())
|
||||
}
|
||||
|
||||
return outputs.GetTxos()[0], nil
|
||||
}
|
||||
|
||||
type GetClaimsInTxResp struct {
|
||||
Jsonrpc string `json:"jsonrpc"`
|
||||
ID int `json:"id"`
|
||||
@@ -41,3 +101,83 @@ func (n *Node) GetClaimsInTx(txid string) (*GetClaimsInTxResp, error) {
|
||||
err := n.request("blockchain.claimtrie.getclaimsintx", []string{txid}, &resp)
|
||||
return &resp, err
|
||||
}
|
||||
|
||||
func (n *Node) GetTx(txid string) (string, error) {
|
||||
resp := &struct {
|
||||
Result string `json:"result"`
|
||||
}{}
|
||||
|
||||
err := n.request("blockchain.transaction.get", []string{txid}, resp)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return resp.Result, nil
|
||||
}
|
||||
|
||||
func (n *Node) GetClaimInTx(txid string, nout int) (*types.Claim, error) {
|
||||
hexTx, err := n.GetTx(txid)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
rawTx, err := hex.DecodeString(hexTx)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
tx, err := btcutil.NewTxFromBytes(rawTx)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
if len(tx.MsgTx().TxOut) <= nout {
|
||||
return nil, errors.Err("nout not found")
|
||||
}
|
||||
|
||||
script := tx.MsgTx().TxOut[nout].PkScript
|
||||
|
||||
var value []byte
|
||||
if lbrycrd.IsClaimNameScript(script) {
|
||||
_, value, _, err = lbrycrd.ParseClaimNameScript(script)
|
||||
} else if lbrycrd.IsClaimUpdateScript(script) {
|
||||
_, _, value, _, err = lbrycrd.ParseClaimUpdateScript(script)
|
||||
} else {
|
||||
err = errors.Err("no claim found in output")
|
||||
}
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
ch, err := claim.DecodeClaimBytes(value, "")
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
return ch.Claim, nil
|
||||
}
|
||||
|
||||
func (n *Node) ResolveToClaim(url string) (*types.Claim, *types.Output, error) {
|
||||
output, err := n.Resolve(url)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "NOT_FOUND") {
|
||||
return nil, nil, errors.Err(ErrClaimNotFound)
|
||||
}
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
claim, err := n.GetClaimInTx(hex.EncodeToString(rev(output.GetTxHash())), int(output.GetNout()))
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return claim, output, nil
|
||||
}
|
||||
|
||||
func rev(b []byte) []byte {
|
||||
r := make([]byte, len(b))
|
||||
for left, right := 0, len(b)-1; left < right; left, right = left+1, right-1 {
|
||||
r[left], r[right] = b[right], b[left]
|
||||
}
|
||||
return r
|
||||
}
|
||||
+67
-23
@@ -14,7 +14,7 @@ import (
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/uber-go/atomic"
|
||||
"go.uber.org/atomic"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -23,10 +23,10 @@ const (
|
||||
)
|
||||
|
||||
var (
|
||||
ErrNotImplemented = errors.Base("not implemented")
|
||||
ErrNodeConnected = errors.Base("node already connected")
|
||||
ErrConnectFailed = errors.Base("failed to connect")
|
||||
ErrTimeout = errors.Base("timeout")
|
||||
ErrNotImplemented = errors.Base("not implemented")
|
||||
ErrAlreadyConnected = errors.Base("node already connected")
|
||||
ErrConnectFailed = errors.Base("failed to connect")
|
||||
ErrTimeout = errors.Base("timeout")
|
||||
)
|
||||
|
||||
type response struct {
|
||||
@@ -35,6 +35,8 @@ type response struct {
|
||||
}
|
||||
|
||||
type Node struct {
|
||||
ErrFn func(error)
|
||||
|
||||
transport *TCPTransport
|
||||
nextId atomic.Uint32
|
||||
grp *stop.Group
|
||||
@@ -63,7 +65,7 @@ func NewNode() *Node {
|
||||
// Connect creates a new connection to the specified address.
|
||||
func (n *Node) Connect(addrs []string, config *tls.Config) error {
|
||||
if n.transport != nil {
|
||||
return ErrNodeConnected
|
||||
return errors.Err(ErrAlreadyConnected)
|
||||
}
|
||||
|
||||
// shuffle addresses for load balancing
|
||||
@@ -76,15 +78,18 @@ func (n *Node) Connect(addrs []string, config *tls.Config) error {
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
if errors.Is(err, ErrTimeout) {
|
||||
continue
|
||||
}
|
||||
if e, ok := err.(*net.OpError); ok && e.Err.Error() == "no such host" {
|
||||
// net.errNoSuchHost is not exported, so we have to string-match
|
||||
continue
|
||||
}
|
||||
return err
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
if n.transport == nil {
|
||||
return ErrConnectFailed
|
||||
return errors.Err(ErrConnectFailed)
|
||||
}
|
||||
|
||||
log.Debugf("wallet connected to %s", n.transport.conn.RemoteAddr())
|
||||
@@ -112,7 +117,13 @@ func (n *Node) Connect(addrs []string, config *tls.Config) error {
|
||||
}
|
||||
|
||||
func (n *Node) Shutdown() {
|
||||
var addr net.Addr
|
||||
if n.transport != nil {
|
||||
addr = n.transport.conn.RemoteAddr()
|
||||
}
|
||||
log.Debugf("shutting down wallet %s", addr)
|
||||
n.grp.StopAndWait()
|
||||
log.Debugf("wallet stopped")
|
||||
}
|
||||
|
||||
func (n *Node) handleErrors() {
|
||||
@@ -121,20 +132,29 @@ func (n *Node) handleErrors() {
|
||||
case <-n.grp.Ch():
|
||||
return
|
||||
case err := <-n.transport.Errors():
|
||||
n.err(err)
|
||||
n.err(errors.Err(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// err handles errors produced by the foreign node.
|
||||
func (n *Node) err(err error) {
|
||||
// TODO: Better error handling.
|
||||
log.Error(err)
|
||||
// err handles errors produced by the server
|
||||
func (n *Node) err(e error) {
|
||||
if n.ErrFn != nil {
|
||||
n.ErrFn(e)
|
||||
} else {
|
||||
log.Error(errors.FullTrace(e))
|
||||
}
|
||||
}
|
||||
|
||||
// listen processes messages from the server.
|
||||
func (n *Node) listen() {
|
||||
for {
|
||||
select {
|
||||
case <-n.grp.Ch():
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
select {
|
||||
case <-n.grp.Ch():
|
||||
return
|
||||
@@ -147,14 +167,36 @@ func (n *Node) listen() {
|
||||
Message string `json:"message"`
|
||||
} `json:"error"`
|
||||
}{}
|
||||
if err := json.Unmarshal(bytes, msg); err != nil {
|
||||
n.err(err)
|
||||
continue
|
||||
msg2 := &struct {
|
||||
Id uint32 `json:"id"`
|
||||
Method string `json:"method"`
|
||||
Error struct {
|
||||
Code int `json:"code"`
|
||||
Message struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
} `json:"message"`
|
||||
} `json:"error"`
|
||||
}{}
|
||||
r := response{}
|
||||
|
||||
err := json.Unmarshal(bytes, msg)
|
||||
if err != nil {
|
||||
// try msg2, a hack around the weird error-in-error response we sometimes get from wallet server
|
||||
// maybe that happens because the wallet server passes a lbrycrd error through to us?
|
||||
if err2 := json.Unmarshal(bytes, msg2); err2 == nil {
|
||||
err = nil
|
||||
msg.Id = msg2.Id
|
||||
msg.Method = msg2.Method
|
||||
msg.Error = msg2.Error.Message
|
||||
}
|
||||
}
|
||||
|
||||
r := response{}
|
||||
if len(msg.Error.Message) > 0 {
|
||||
r.err = errors.Base("%d: %s", msg.Error.Code, msg.Error.Message)
|
||||
if err != nil {
|
||||
r.err = errors.Err(err)
|
||||
n.err(r.err)
|
||||
} else if len(msg.Error.Message) > 0 {
|
||||
r.err = errors.Err("%d: %s", msg.Error.Code, msg.Error.Message)
|
||||
} else {
|
||||
r.data = bytes
|
||||
}
|
||||
@@ -206,7 +248,7 @@ func (n *Node) request(method string, params []string, v interface{}) error {
|
||||
|
||||
bytes, err := json.Marshal(msg)
|
||||
if err != nil {
|
||||
return err
|
||||
return errors.Err(err)
|
||||
}
|
||||
bytes = append(bytes, delimiter)
|
||||
|
||||
@@ -218,11 +260,13 @@ func (n *Node) request(method string, params []string, v interface{}) error {
|
||||
|
||||
err = n.transport.Send(bytes)
|
||||
if err != nil {
|
||||
return err
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
var r response
|
||||
select {
|
||||
case <-n.grp.Ch():
|
||||
return nil
|
||||
case r = <-c:
|
||||
case <-time.After(n.timeout):
|
||||
r = response{err: errors.Err(ErrTimeout)}
|
||||
@@ -233,8 +277,8 @@ func (n *Node) request(method string, params []string, v interface{}) error {
|
||||
n.handlersMu.Unlock()
|
||||
|
||||
if r.err != nil {
|
||||
return r.err
|
||||
return errors.Err(r.err)
|
||||
}
|
||||
|
||||
return json.Unmarshal(r.data, v)
|
||||
return errors.Err(json.Unmarshal(r.data, v))
|
||||
}
|
||||
+47
-11
@@ -5,10 +5,14 @@ package wallet
|
||||
import (
|
||||
"bufio"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
@@ -49,15 +53,31 @@ func NewTransport(addr string, config *tls.Config) (*TCPTransport, error) {
|
||||
|
||||
t.grp.Add(1)
|
||||
go func() {
|
||||
t.grp.Done()
|
||||
defer t.grp.Done()
|
||||
t.listen()
|
||||
}()
|
||||
|
||||
err = t.test()
|
||||
if err != nil {
|
||||
t.grp.StopAndWait()
|
||||
return nil, errors.Prefix(addr, err)
|
||||
}
|
||||
|
||||
return t, nil
|
||||
}
|
||||
|
||||
const delimiter = byte('\n')
|
||||
|
||||
func (t *TCPTransport) Send(body []byte) error {
|
||||
log.Debugf("%s <- %s", t.conn.RemoteAddr(), body)
|
||||
_, err := t.conn.Write(body)
|
||||
return err
|
||||
}
|
||||
|
||||
func (t *TCPTransport) Responses() <-chan []byte { return t.responses }
|
||||
func (t *TCPTransport) Errors() <-chan error { return t.errors }
|
||||
func (t *TCPTransport) Shutdown() { t.grp.StopAndWait() }
|
||||
|
||||
func (t *TCPTransport) listen() {
|
||||
reader := bufio.NewReader(t.conn)
|
||||
for {
|
||||
@@ -73,12 +93,6 @@ func (t *TCPTransport) listen() {
|
||||
}
|
||||
}
|
||||
|
||||
func (t *TCPTransport) Send(body []byte) error {
|
||||
log.Debugf("%s <- %s", t.conn.RemoteAddr(), body)
|
||||
_, err := t.conn.Write(body)
|
||||
return err
|
||||
}
|
||||
|
||||
func (t *TCPTransport) error(err error) {
|
||||
select {
|
||||
case t.errors <- err:
|
||||
@@ -86,11 +100,33 @@ func (t *TCPTransport) error(err error) {
|
||||
}
|
||||
}
|
||||
|
||||
func (t *TCPTransport) Responses() <-chan []byte { return t.responses }
|
||||
func (t *TCPTransport) Errors() <-chan error { return t.errors }
|
||||
func (t *TCPTransport) test() error {
|
||||
err := t.Send([]byte(`{"id":1,"method":"server.version"}` + "\n"))
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
func (t *TCPTransport) Shutdown() {
|
||||
t.grp.StopAndWait()
|
||||
var data []byte
|
||||
select {
|
||||
case data = <-t.Responses():
|
||||
case <-time.Tick(1 * time.Second):
|
||||
return errors.Err(ErrTimeout)
|
||||
}
|
||||
|
||||
var response struct {
|
||||
Error struct {
|
||||
Message string `json:"message"`
|
||||
} `json:"error"`
|
||||
}
|
||||
|
||||
err = json.Unmarshal(data, &response)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
if response.Error.Message != "" {
|
||||
return fmt.Errorf(response.Error.Message)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *TCPTransport) close() {
|
||||
|
||||
Reference in new issue
Block a user