Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
71fb06fbb8 | ||
|
|
e1b4f21e00 | ||
|
|
b4913ecedf | ||
|
|
63a574ec2f | ||
|
|
b8af3408e0 | ||
|
|
847089d0d6 | ||
|
|
170dfef3a8 | ||
|
|
2b458a6bd0 | ||
|
|
febfc51cb0 | ||
|
|
72be487262 | ||
|
|
94e7d81bd3 | ||
|
|
c6c779da39 | ||
|
|
2e101083e6 | ||
|
|
63aacd8a69 | ||
|
|
c03ae6487d | ||
|
|
0c4f455f0c | ||
|
|
af3e08c446 | ||
|
|
975bfe7fac | ||
|
|
b075d948bb | ||
|
|
2651a64dbb | ||
|
|
fa7150cf2b | ||
|
|
6c4db980c9 | ||
|
|
7adaa510fd | ||
|
|
64ed7304f6 | ||
|
|
5aefaf061e | ||
|
|
724ee47c8b | ||
|
|
caaec6fcb1 | ||
|
|
15984b8fd9 | ||
|
|
2be913b077 | ||
|
|
34c11b0a0e | ||
|
|
64acdc29c3 | ||
|
|
598773c90d | ||
|
|
766238fd7e | ||
|
|
ac5242f173 | ||
|
|
215103cb33 | ||
|
|
ed3622d0a6 | ||
|
|
848fce5afa | ||
|
|
e37eeba0c9 | ||
|
|
7da49a4ccb | ||
|
|
7b02ace5e2 | ||
|
|
5fb67b32db | ||
|
|
a0c9ed2ace | ||
|
|
998b082a06 | ||
|
|
36d4156e2a | ||
|
|
74925ebba2 | ||
|
|
6f95b3395f | ||
|
|
dff00e2317 | ||
|
|
9a5d9d7ff5 | ||
|
|
5794c57898 | ||
|
|
35c713a26e | ||
|
|
6fb0620091 | ||
|
|
03df751bc7 | ||
|
|
c902858958 | ||
|
|
84fabdd5f4 | ||
|
|
f5cad15f84 | ||
|
|
dd3d0ae42c | ||
|
|
0b565852b8 | ||
|
|
ff13d7b2f7 | ||
|
|
7f5a89fa5a | ||
|
|
704e15f8c1 | ||
|
|
5eb1f13b54 | ||
|
|
176e05714e | ||
|
|
eefd84b02d | ||
|
|
af2742c34f | ||
|
|
2cf4acdb59 | ||
|
|
044e2fe5d7 | ||
|
|
fb77bf621e | ||
|
|
e70b9af3e4 | ||
|
|
5df05cf46f | ||
|
|
7bddcf01b8 | ||
|
|
659a6e73cc | ||
|
|
aaae3ffa5b | ||
|
|
131fed28d2 | ||
|
|
72571236ab | ||
|
|
560e180e36 | ||
|
|
070c378dfd | ||
|
|
124d4065c2 | ||
|
|
f131c1f35b | ||
|
|
7a3225434e | ||
|
|
3608971f0b | ||
|
|
c9fa04043c | ||
|
|
c6b53792c8 | ||
|
|
69fa06420b | ||
|
|
5cb1365903 | ||
|
|
e430c2fd40 | ||
|
|
66024716ac | ||
|
|
f043516a14 | ||
|
|
c3db95a6c1 | ||
|
|
08c93d44fd | ||
|
|
b02e80d472 | ||
|
|
f1875454cc | ||
|
|
a80e0f5b0d | ||
|
|
2b3581a692 | ||
|
|
a0f78028cc | ||
|
|
a084330055 | ||
|
|
ea80ed6506 | ||
|
|
4b335ed692 | ||
|
|
6118dde36c | ||
|
|
8364d3fc54 | ||
|
|
de0ccd4da7 | ||
|
|
31f9346027 | ||
|
|
5658fe4607 | ||
|
|
fc5f5ff7d3 | ||
|
|
e8b98bc862 | ||
|
|
f458529c74 | ||
|
|
150b1f6f1f | ||
|
|
47f28002ff | ||
|
|
beff466d18 | ||
|
|
694bda105c | ||
|
|
34ca7847d0 | ||
|
|
5c91051b78 | ||
|
|
df4f42db82 | ||
|
|
41d758ef5c | ||
|
|
264390a2b2 | ||
|
|
fdcc41829a | ||
|
|
09c7718f30 | ||
|
|
8a5f57b14f | ||
|
|
90997b9918 | ||
|
|
e0da2674a1 | ||
|
|
a80599413c | ||
|
|
3ffe7a10c7 | ||
|
|
1bf3cb81b3 | ||
|
|
4a9f127ecc | ||
|
|
de1fb63a1c | ||
|
|
e98794e125 | ||
|
|
fb0004bac4 | ||
|
|
1f12b1d5d8 | ||
|
|
bc95ca61d5 | ||
|
|
be69c2f05c | ||
|
|
4a902597df | ||
|
|
dd528f03f6 | ||
|
|
e6ba61fce2 | ||
|
|
e03abad012 | ||
|
|
dde93a1fe6 | ||
|
|
4a5a148843 | ||
|
|
cb6443fa1e | ||
|
|
1a1f991e48 | ||
|
|
d1063bd54e | ||
|
|
d291c063ec | ||
|
|
a939529d19 | ||
|
|
cf34547b8a | ||
|
|
6631ad325f | ||
|
|
df266f6194 | ||
|
|
b3b581c00e | ||
|
|
18b15674d9 | ||
|
|
780e899e90 | ||
|
|
5c5643749e | ||
|
|
11e50a6022 | ||
|
|
1603b3bb22 | ||
|
|
5465527faf | ||
|
|
5d8a2d697c | ||
|
|
50089481fb | ||
|
|
d6cf9e9e63 | ||
|
|
dc6dd8d12b | ||
|
|
ad8d623863 | ||
|
|
d89b58c52a | ||
|
|
2aa40850b0 | ||
|
|
ca24c2be28 | ||
|
|
52127eee7c | ||
|
|
95eb94f5a7 | ||
|
|
834733b675 | ||
|
|
86a553b876 | ||
|
|
e5438713ce | ||
|
|
f6c4b7d36b | ||
|
|
661c20a21d | ||
|
|
69f1e0f4ca | ||
|
|
2ca83139df | ||
|
|
0af6d65d40 | ||
|
|
a8230db802 | ||
|
|
24f885e268 | ||
|
|
acb9840871 | ||
|
|
a745bafc58 | ||
|
|
bf9e90f14c | ||
|
|
a98d1e1217 | ||
|
|
36ee7e8d1f | ||
|
|
c1e8e7481f | ||
|
|
cb669eb1a7 | ||
|
|
08df3b167c | ||
|
|
1a6b862c96 | ||
|
|
ce3d4403db | ||
|
|
949ea2f2d8 | ||
|
|
390d2b1eec | ||
|
|
557c595e90 | ||
|
|
7d8772ed63 | ||
|
|
80f3b3437a | ||
|
|
68caed8a62 | ||
|
|
3823ee7a4e | ||
|
|
5916ab4efd | ||
|
|
227b362296 | ||
|
|
8f280bf52f | ||
|
|
f30401d7a6 | ||
|
|
83bb1791b4 | ||
|
|
3e006f1571 | ||
|
|
2497f354bd | ||
|
|
db83ebe434 | ||
|
|
a2d51cee3d | ||
|
|
47425c2c50 | ||
|
|
594a45a271 | ||
|
|
fd39b09e95 | ||
|
|
fce8beea4d | ||
|
|
96e0c2e2b2 | ||
|
|
bb5df9896a | ||
|
|
accd79c81f | ||
|
|
32dae2704f | ||
|
|
2ed1e23228 | ||
|
|
d61f7c892f | ||
|
|
027409fe40 | ||
|
|
38401bf8d6 | ||
|
|
825e699114 | ||
|
|
6509c7dc52 | ||
|
|
c088a65041 | ||
|
|
c7643fb5da | ||
|
|
c3ab5dd2a5 | ||
|
|
2047fe6c05 | ||
|
|
6166ff37cf | ||
|
|
970585c75d | ||
|
|
71549c0dea | ||
|
|
46ae5a502a | ||
|
|
fb2f807e6a | ||
|
|
ad1eea1e8e | ||
|
|
dfb77d3547 | ||
|
|
96427136ef | ||
|
|
c649636eeb | ||
|
|
2e81b1ab03 | ||
|
|
fa2ec38d3e | ||
|
|
2cacdaf22d | ||
|
|
3b269050af | ||
|
|
59ff828794 | ||
|
|
4569d04522 | ||
|
|
7a618f4228 | ||
|
|
0d996d7415 | ||
|
|
2744080c6b | ||
|
|
41b513439f | ||
|
|
31daec7054 | ||
|
|
00feaaf76c | ||
|
|
a98990f573 | ||
|
|
5788f5aa55 | ||
|
|
922a9402aa | ||
|
|
05e0a506b4 | ||
|
|
a8fe33ecf4 | ||
|
|
030ed4914d | ||
|
|
e03810903a | ||
|
|
b0b2d21a08 | ||
|
|
557988d2cb | ||
|
|
6acb99c74d | ||
|
|
d77f5d17f3 | ||
|
|
609ebdb74e | ||
|
|
5baa66f0aa | ||
|
|
75ad143d84 | ||
|
|
14d6d32a41 | ||
|
|
e3ae0ef5c9 | ||
|
|
db791e26ef | ||
|
|
dc104dd17b | ||
|
|
e67e3dbf85 | ||
|
|
e14a1f36cf | ||
|
|
11ebfb822b | ||
|
|
4293e772e2 | ||
|
|
686ac662cc | ||
|
|
747a6ec2b2 | ||
|
|
046868c4e0 | ||
|
|
53d3eea8fb | ||
|
|
61e83d86de | ||
|
|
9fb824790b | ||
|
|
6fd0526376 | ||
|
|
cd236708b0 | ||
|
|
83921b8c2d | ||
|
|
c042fe08c3 | ||
|
|
59e92ccf88 | ||
|
|
cb386de4c7 | ||
|
|
146adcf7fd | ||
|
|
5d844fc3ea | ||
|
|
4ed52e20d7 | ||
|
|
7ab2d355d5 | ||
|
|
064035f1d8 | ||
|
|
e158fa1bb2 | ||
|
|
25ec93a6b9 | ||
|
|
22f42d5559 | ||
|
|
473e7e3b07 | ||
|
|
d7a4b34d8c | ||
|
|
605afd1d6c |
No files matched your search
+1
-2
@@ -1,4 +1,3 @@
|
||||
/vendor
|
||||
/blobs
|
||||
/config.json*
|
||||
/prism-bin
|
||||
/bin
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"Disable": ["golint"],
|
||||
"Vendor": true
|
||||
}
|
||||
+36
-35
@@ -2,30 +2,21 @@ os: linux
|
||||
dist: trusty
|
||||
language: go
|
||||
|
||||
# Only the last two Go releases are supported by the Go team with security
|
||||
# updates. Any versions older than that should be considered deprecated.
|
||||
# Don't bother testing with them. tip builds your code with the latest
|
||||
# development version of Go. This can warn you that your code will break
|
||||
# in the next version of Go. Don't worry! Later we declare that test runs
|
||||
# are allowed to fail on Go tip.
|
||||
matrix:
|
||||
include:
|
||||
- go: 1.10.2
|
||||
env: DEPLOY=true # dont deploy tip build
|
||||
- go: master
|
||||
env: DEPLOY=false # dont deploy tip build
|
||||
allow_failures:
|
||||
- go: master
|
||||
# Don't wait for tip tests to finish. Mark the test run green if the
|
||||
# tests pass on the stable versions of Go.
|
||||
fast_finish: true
|
||||
env:
|
||||
- GO111MODULE=on
|
||||
|
||||
go:
|
||||
- 1.15.x
|
||||
|
||||
cache:
|
||||
directories:
|
||||
- $HOME/.cache/go-build
|
||||
- $HOME/gopath/pkg/mod
|
||||
|
||||
# Don't email me the results of the test runs.
|
||||
notifications:
|
||||
email: false
|
||||
|
||||
# Skip the install step. Don't `go get` dependencies. Only build with the
|
||||
# code in vendor/
|
||||
# Skip the install step. Don't `go get` dependencies. Only build with the code in vendor/
|
||||
install: true
|
||||
|
||||
# Anything in before_script that returns a nonzero exit code will
|
||||
@@ -33,10 +24,10 @@ install: true
|
||||
# set -e enabled in bash.
|
||||
before_script:
|
||||
# All the .go files, excluding vendor/ and model (auto generated)
|
||||
- GO_FILES=$(find . -iname '*.go' ! -iname '*_test.go' -type f | grep -v /vendor/ )
|
||||
- GO_FILES=$(find . -iname '*.go' ! -iname '*_test.go' -type f | grep -v /vendor/ ) #i wish we were this crazy :p
|
||||
- go get golang.org/x/tools/cmd/goimports # Used in build script for generated files
|
||||
- go get github.com/golang/lint/golint # Linter
|
||||
- go get honnef.co/go/tools/cmd/megacheck # Badass static analyzer/linter
|
||||
# - go get github.com/golang/lint/golint # Linter
|
||||
# - go get honnef.co/go/tools/cmd/megacheck # Badass static analyzer/linter
|
||||
- go get github.com/jgautheron/gocyclo # Check against high complexity
|
||||
- go get github.com/mdempsky/unconvert # Identifies unnecessary type conversions
|
||||
- go get github.com/kisielk/errcheck # Checks for unhandled errors
|
||||
@@ -48,12 +39,10 @@ before_script:
|
||||
# script always run to completion (set +e). All of these code checks are must haves
|
||||
# in a modern Go project.
|
||||
script:
|
||||
# Build Prism successfully
|
||||
- make
|
||||
# Fail if a .go file hasn't been formatted with gofmt
|
||||
- test -z $(gofmt -s -l $GO_FILES)
|
||||
# Run unit tests
|
||||
- go test ./...
|
||||
- make test
|
||||
# Checks for unused vars and fields on structs
|
||||
- varcheck ./...
|
||||
- structcheck ./...
|
||||
@@ -62,21 +51,33 @@ script:
|
||||
# forbid code with huge functions
|
||||
#- gocyclo -ignore "_test.go" -avg -over 19 $GO_FILES
|
||||
# checks for unhandled errors
|
||||
- errcheck ./...
|
||||
# - errcheck ./... # COMMENTED OUT UNTIL https://github.com/kisielk/errcheck/issues/155 is fixed
|
||||
# "go vet on steroids" + linter - ignore autogen code
|
||||
- megacheck -simple.exit-non-zero=true ./...
|
||||
# - megacheck -simple.exit-non-zero=true ./... # DISABLED UNTIL https://github.com/dominikh/go-tools/issues/328 is fixed
|
||||
# check for unnecessary conversions - ignore autogen code
|
||||
- unconvert ./...
|
||||
# - unconvert ./... # SEEMS TO BE BROKEN WITH GO MODULES
|
||||
# one last linter - ignore autogen code
|
||||
#- golint -set_exit_status $(go list ./... | grep -v /vendor/ )
|
||||
# Finally, build the binary
|
||||
- make
|
||||
|
||||
deploy:
|
||||
- provider: releases
|
||||
file: prism-bin
|
||||
- provider: s3
|
||||
local_dir: ./bin
|
||||
skip_cleanup: true
|
||||
on:
|
||||
repo: lbryio/reflector.go
|
||||
tags: true
|
||||
condition: "$DEPLOY = true"
|
||||
api_key:
|
||||
secure: epAlhp3SUr8hhISarJ22n6tRw2TEa4s4oNFIvJUb5HGECVp1SYN7ao0ln5NoNLmfJS60pi911i/kMhhi21/uhZ0kCYlEhhIE2pc1zsiAxK9L9ENCssJ205HfVbe5grhwskLGzgjhU9OznO8WtmyOPWXr0it8M8RCTjx6rEC0A33Id3WMYyhP938Sj9CxEYeH4KS8wFvBXkgBVtrgaYwRTCIROFddHFXOb9jyNhqQ1RbfKtllsVtQhVk5WMlomheBNSS4vr6WMS4X4+2okFqnLtiSn1wrn5I/94UQbnrI1juVnQj0K+j32EyQbAOt4T2cLW3GtG0jhaYKyNMT9ycDCdVACPSDELlHWjeyoes9bnhUFftm6kDbQxwA1UsTF1yG8tMKXxBSmYyoT7qDloi6pBifZMrFXL61uTs6yhVB9LS/2oqg4sc0Ne87bRcn4OxsBeVCe3kbBHDTR/NTyF2gNPtRvgMAWULxTVcUm9VYdO0IWvAig5g4Row0DnFzEquD6CzezbRWD9WyZyV/AFyYHeeQ2PO7jTw0/3M7aDX33Fuhh34lehzmrC03cfgD/wZW+spxozIcQCYdiJqVw+u+/NvbNr0kkFzE9zW26JEmUFTyDvKxvnza1Kwtww3EgH6zaOL8r4yVbb54rePRvLw7pl93zlfJnEB2MCPqJOY5ZpU=
|
||||
access_key_id:
|
||||
secure: "ABmoSTxTee1GubJmmi+MlyBcnqRT0ywEOtTL4hrH7T+Vj5UPcNBhjqGRiMXzO3MdrVWom19OnqMvsvAdehrB4uQJVlX/+zB26gTpEiiU5dWJgerNAYmfxDGNz/p9pyYWLvVY/cymSlJ4HkggJVg+dMTBTOP+gTNgQUhHL7pAd/Z5HMPmPW/rlNlQgDdczoY44OdiniuHny4Lr7mo8dcT4dcyJysPpHLCPGWp3kXcffIEbgO02aQZ14BbX9oU9xm1BacYnr4t8gRCU+uK0mt7TREHUzZ8bQBRXUymmpeeEQshXdifdwtk6sdRGIZiTi0NmQ7kYter6oh6623TH6rpNy/lkxKi823FbSPC1XZGH7CwcbnSIEwY++7e9PT0EA14VuDUF7+iXC/gpRkVQwUoMcoC9rHBjkXW+SiwHWoUeDKXrfVO9QGEJTi5bDRNwklwu2zkYWeRIPzfDaVayUBYwjunjT1eST997ygmJwJTJ6DeOWLx+WFlmwBJfQpBwbrlMrceiNjreMwGB6ffXEBzuM6S5inXGIChllGDpNyJeFVcH9zPpU0JsIpWoRjwLNFOiEhFsnzK5rvA2SKmlOOnSlgOCBQnFi2SmLDM790KoY8qjan2s984chQz2qsKnhYuVpJhG6LKgrI1QwE0KHr80spVFfewq11AU4Q+lfA9aAQ="
|
||||
secret_access_key:
|
||||
secure: "AxYRTy/GnjeTJKQdeJ/AEeAd+yXs783bFDKdyKNswtsHlU8sWPQgNcvTLpVqnSQMpiwkGDGi/70rvR5C+AT3SIWNw13RYrgBRpduQU0J+B2JS+3dN2DIePu25uvs++Wo22OfS8I+UjZ1mWY1SSHI2spPXvDCq5tb+Ih8nlYflEyAtxU9Oq2R3Kus2tkIlRnL25sP/2fY7RvuJFYIV63z8ZIJRzB5WzOeERqnXq2zfwos+hycAqyo/VevJnWAYTEDsvBuSODOpZF+QfKtIQ2rYSoqy8Lq1M6UOZimnC3Ulea4euBVf2ssBCnI7csGNG5UzkTiwrPDi2xIP8nM01rHW1yHJ7tQsJaghnUsfw2t6ui4ZofvbbOFTN/YCloHITifEi8Tc1/17isi3y+kX5yQ/Nk5UNry0Wbt91CP+nkL/ZmA5grkBXDL7VJMmB60TnO3ap24CtwBQartN3LoWs7h+4ov+LqbCt6IqpJVWQWlwJeb2MFPFByALtBpsqAyL1SxXlGNpPa94CuXxfQ6Bv436PtefA5FlTzR8uMmqsjWciv06bVnSvVlFEVovN2Fkplrjt7AASJ/8KJs4THDg4k61nfd8roAHx6ewQzl4wCWKCikQ0MuFd2mVHwdrbnCH1mIHuPRyvWMMIAK0ooc1/rmKiJlpgumjxoFYNE10MXtt+I="
|
||||
bucket: "lbry-reflector-binary"
|
||||
|
||||
# - provider: releases
|
||||
# file: ./bin/prism-bin
|
||||
# skip_cleanup: true
|
||||
# on:
|
||||
# repo: lbryio/reflector.go
|
||||
# tags: true
|
||||
# api_key:
|
||||
# secure: epAlhp3SUr8hhISarJ22n6tRw2TEa4s4oNFIvJUb5HGECVp1SYN7ao0ln5NoNLmfJS60pi911i/kMhhi21/uhZ0kCYlEhhIE2pc1zsiAxK9L9ENCssJ205HfVbe5grhwskLGzgjhU9OznO8WtmyOPWXr0it8M8RCTjx6rEC0A33Id3WMYyhP938Sj9CxEYeH4KS8wFvBXkgBVtrgaYwRTCIROFddHFXOb9jyNhqQ1RbfKtllsVtQhVk5WMlomheBNSS4vr6WMS4X4+2okFqnLtiSn1wrn5I/94UQbnrI1juVnQj0K+j32EyQbAOt4T2cLW3GtG0jhaYKyNMT9ycDCdVACPSDELlHWjeyoes9bnhUFftm6kDbQxwA1UsTF1yG8tMKXxBSmYyoT7qDloi6pBifZMrFXL61uTs6yhVB9LS/2oqg4sc0Ne87bRcn4OxsBeVCe3kbBHDTR/NTyF2gNPtRvgMAWULxTVcUm9VYdO0IWvAig5g4Row0DnFzEquD6CzezbRWD9WyZyV/AFyYHeeQ2PO7jTw0/3M7aDX33Fuhh34lehzmrC03cfgD/wZW+spxozIcQCYdiJqVw+u+/NvbNr0kkFzE9zW26JEmUFTyDvKxvnza1Kwtww3EgH6zaOL8r4yVbb54rePRvLw7pl93zlfJnEB2MCPqJOY5ZpU=
|
||||
@@ -0,0 +1,9 @@
|
||||
FROM alpine
|
||||
EXPOSE 8080
|
||||
|
||||
RUN mkdir /app
|
||||
WORKDIR /app
|
||||
COPY bin/prism-bin ./prism
|
||||
RUN chmod +x prism
|
||||
|
||||
ENTRYPOINT [ "/app/prism" ]
|
||||
Generated
-435
@@ -1,435 +0,0 @@
|
||||
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
|
||||
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:d64110a78451e373c5a952d2625323dbbe3bfe41c67f9652ea9668a6ceb4f645"
|
||||
name = "github.com/armon/go-metrics"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "3c58d8115a78a6879e5df75ae900846768d36895"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:c0b6dbbb56a745020d5939bdde2197241a1c6109f226cca57b16f46916be5e94"
|
||||
name = "github.com/aws/aws-sdk-go"
|
||||
packages = [
|
||||
"aws",
|
||||
"aws/awserr",
|
||||
"aws/awsutil",
|
||||
"aws/client",
|
||||
"aws/client/metadata",
|
||||
"aws/corehandlers",
|
||||
"aws/credentials",
|
||||
"aws/credentials/ec2rolecreds",
|
||||
"aws/credentials/endpointcreds",
|
||||
"aws/credentials/stscreds",
|
||||
"aws/csm",
|
||||
"aws/defaults",
|
||||
"aws/ec2metadata",
|
||||
"aws/endpoints",
|
||||
"aws/request",
|
||||
"aws/session",
|
||||
"aws/signer/v4",
|
||||
"internal/sdkio",
|
||||
"internal/sdkrand",
|
||||
"internal/sdkuri",
|
||||
"internal/shareddefaults",
|
||||
"private/protocol",
|
||||
"private/protocol/eventstream",
|
||||
"private/protocol/eventstream/eventstreamapi",
|
||||
"private/protocol/query",
|
||||
"private/protocol/query/queryutil",
|
||||
"private/protocol/rest",
|
||||
"private/protocol/restxml",
|
||||
"private/protocol/xml/xmlutil",
|
||||
"service/s3",
|
||||
"service/s3/s3iface",
|
||||
"service/s3/s3manager",
|
||||
"service/sts",
|
||||
]
|
||||
pruneopts = ""
|
||||
revision = "c0447dbaaf195bb477fd2d511b8e4665e04b9017"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:e250643be8120824556f39df6ef128fc2be490fc96e0cb64b1a8ecf96bbe3ce6"
|
||||
name = "github.com/btcsuite/btcutil"
|
||||
packages = ["base58"]
|
||||
pruneopts = ""
|
||||
revision = "ab6388e0c60ae4834a1f57511e20c17b5f78be4b"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:0a39ec8bf5629610a4bc7873a92039ee509246da3cef1a0ea60f1ed7e5f9cea5"
|
||||
name = "github.com/davecgh/go-spew"
|
||||
packages = ["spew"]
|
||||
pruneopts = ""
|
||||
revision = "346938d642f2ec3594ed81d874461961cd0faa76"
|
||||
version = "v1.1.0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:968d8903d598e3fae738325d3410f33f07ea6a2b9ee5591e9c262ee37df6845a"
|
||||
name = "github.com/go-errors/errors"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "a6af135bd4e28680facf08a3d206b454abc877a4"
|
||||
version = "v1.0.1"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:d19c78214e03e297e9e30d2eb11892f731358b2951f2a5c7374658a156373e4c"
|
||||
name = "github.com/go-ini/ini"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "358ee7663966325963d4e8b2e1fbd570c5195153"
|
||||
version = "v1.38.1"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:27b11ca1ad214ead955ff5480e8575e74c5df4e4dc02b04256a8d92131e1d3ad"
|
||||
name = "github.com/go-sql-driver/mysql"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "99ff426eb706cffe92ff3d058e168b278cabf7c7"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:dbbeb8ddb0be949954c8157ee8439c2adfd8dc1c9510eb44a6e58cb68c3dce28"
|
||||
name = "github.com/gorilla/context"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "08b5f424b9271eedf6f9f0ce86cb9396ed337a42"
|
||||
version = "v1.1.1"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:c2c8666b4836c81a1d247bdf21c6a6fc1ab586538ab56f74437c2e0df5c375e1"
|
||||
name = "github.com/gorilla/mux"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "e3702bed27f0d39777b0b37b664b6280e8ef8fbf"
|
||||
version = "v1.6.2"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:b4817bdb0da3054166de058111943ac58b315aead2fed4ee838625a4e304f74c"
|
||||
name = "github.com/gorilla/rpc"
|
||||
packages = [
|
||||
"v2",
|
||||
"v2/json",
|
||||
]
|
||||
pruneopts = ""
|
||||
revision = "22c016f3df3febe0c1f6727598b6389507e03a18"
|
||||
version = "v1.1.0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:fe1b4d4cbe48c0d55507c55f8663aa4185576cc58fa0c8be03bb8f19dfe17a9c"
|
||||
name = "github.com/gorilla/websocket"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "ea4d1f681babbce9545c9c5f3d5194a789c89f5b"
|
||||
version = "v1.2.0"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:4fe55793760295fbef367890352b720784243e0ad19b5ee242519a4682bb9ef8"
|
||||
name = "github.com/hashicorp/errwrap"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "d6c0cd88035724dd42e0f335ae30161c20575ecc"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:4423ee95d6ee30bb22f680445c58889bb5b91e1b955405bf34374a053784a8a2"
|
||||
name = "github.com/hashicorp/go-immutable-radix"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "7f3cd4390caab3250a57f30efdb2a65dd7649ecf"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:6a611e691e739173805cb54019b5c39bb9d46455526dff31e0e6fe3aaca52776"
|
||||
name = "github.com/hashicorp/go-msgpack"
|
||||
packages = ["codec"]
|
||||
pruneopts = ""
|
||||
revision = "fa3f63826f7c23912c15263591e65d54d080b458"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:deaebb4a98ca748bbad7eb653f3a675749500020823a086448ffcd7ba6b8b02d"
|
||||
name = "github.com/hashicorp/go-multierror"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "3d5d8f294aa03d8e98859feac328afbdf1ae0703"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:74f54e6ef2339f1de1e8c4b6674442118bd89e619b2fbd949ef2337330067994"
|
||||
name = "github.com/hashicorp/go-sockaddr"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "6d291a969b86c4b633730bfc6b8b9d64c3aafed9"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:9c776d7d9c54b7ed89f119e449983c3f24c0023e75001d6092442412ebca6b94"
|
||||
name = "github.com/hashicorp/golang-lru"
|
||||
packages = ["simplelru"]
|
||||
pruneopts = ""
|
||||
revision = "0fb14efe8c47ae851c0034ed7a448854d3d34cf3"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:d7ce65372f495908f80fc1f80f4dab5d763d9a1de544abd95aa719e4262d0dd5"
|
||||
name = "github.com/hashicorp/memberlist"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "ce8abaa0c60c2d6bee7219f5ddf500e0a1457b28"
|
||||
version = "v0.1.0"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:4fd01ac3766b886665cfd335cc63819ec4e4538dcc1180c05d6edc089619962c"
|
||||
name = "github.com/hashicorp/serf"
|
||||
packages = [
|
||||
"coordinate",
|
||||
"serf",
|
||||
]
|
||||
pruneopts = ""
|
||||
revision = "984a73625de3138f44deb38d00878fab39eb6447"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:870d441fe217b8e689d7949fef6e43efbc787e50f200cb1e70dbca9204a1d6be"
|
||||
name = "github.com/inconshreveable/mousetrap"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
|
||||
version = "v1.0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:4f767a115bc8e08576f6d38ab73c376fc1b1cd3bb5041171c9e8668cc7739b52"
|
||||
name = "github.com/jmespath/go-jmespath"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "0b12d6b5"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:1e4cb5c6d4e92e2fb34f1545e071fb61e81c71fc0c325445db19b9f3981cb343"
|
||||
name = "github.com/johntdyer/slack-go"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "95fac1160b220c5abcf8b0ef88e9c3cb213c09f4"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:132aae3fa5ad407b53f57cf6ebe274e414ec22d1700899023319931fa90b63e2"
|
||||
name = "github.com/johntdyer/slackrus"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "f7aae3243a0786c5a974bce71ed951c459876e64"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:65bb92382468ce7a1ee218b5f352cd0927d358569be7521dd7cb42e74589a308"
|
||||
name = "github.com/lbryio/errors.go"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "ad03d3cc6a5c27c94bbe3412cf7de0eae1a9bd7c"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:76fd7507e6014c598a01f1b3d558774d2a3114c438403bc98123870d2aecec62"
|
||||
name = "github.com/lbryio/lbry.go"
|
||||
packages = [
|
||||
"crypto",
|
||||
"errors",
|
||||
"null",
|
||||
"querytools",
|
||||
"stop",
|
||||
"util",
|
||||
]
|
||||
pruneopts = ""
|
||||
revision = "e2c96944fc485d3ab5e164da78f8439a94c5aa85"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:cabf2bf5e49edfe0c34cb9c6a256f2a99e6cc8c5e660855c8f3dafe1f81d5dcd"
|
||||
name = "github.com/lyoshenka/bencode"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "b7abd7672df533e627eddbf3a5a529786e8bda7f"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:f0bad0fece0fb73c6ea249c18d8e80ffbe86be0457715b04463068f04686cf39"
|
||||
name = "github.com/miekg/dns"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "5a2b9fab83ff0f8bfc99684bd5f43a37abe560f1"
|
||||
version = "v1.0.8"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:ba3b2eb0ae6fd3deac4386c02fe9d2279c9520738eb9db2f0667e74d5c7a0a61"
|
||||
name = "github.com/nlopes/slack"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "0db1d5eae1116bf7c8ed96c6749acfbf4daaec3e"
|
||||
version = "v0.3.0"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:af967afd3cbc6b0145937f4dcab78bcd93e7b2f2b618fb2bcaf7069ad5c638fa"
|
||||
name = "github.com/phayes/freeport"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "e27662a4a9d6b2083dfd7e7b5d0e30985daca925"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:6ee36f2cea425916d81fdaaf983469fc18f91b3cf090cfe90fa0a9d85b8bfab7"
|
||||
name = "github.com/sean-/seed"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "e2103e2c35297fb7e17febb81e49b312087a2372"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:65d0c541c450f6f8bb9654a3f1938407a835bf41faade00bb0fa1416de215d00"
|
||||
name = "github.com/sebdah/goldie"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "8784dd1ab561dcf43d141f6678e9e41f3d0dff55"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:1747c026a603e3c9f33f238e1d1390df2c8f48876b6bcb7a9c52c7b479e040f4"
|
||||
name = "github.com/sirupsen/logrus"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "d329d24db4313262a3b0a24d8aeb1dc4bd294fb0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:d0b38ba6da419a6d4380700218eeec8623841d44a856bb57369c172fbf692ab4"
|
||||
name = "github.com/spf13/cast"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "8965335b8c7107321228e3e3702cab9832751bac"
|
||||
version = "v1.2.0"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:b5212c335a490d958a6b1b5b48901b46e682b82fa9af3a238fa88df6eaa60873"
|
||||
name = "github.com/spf13/cobra"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "7c4570c3ebeb8129a1f7456d0908a8b676b6f9f1"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:8e243c568f36b09031ec18dff5f7d2769dcf5ca4d624ea511c8e3197dc3d352d"
|
||||
name = "github.com/spf13/pflag"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "583c0c0531f06d5278b7d917446061adc344b5cd"
|
||||
version = "v1.0.1"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:f05efcac20bea32e1fcefde9a3cbefb07e02053666c4a67681ad18c8efc682d3"
|
||||
name = "github.com/uber-go/atomic"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "ca680462431f8c7f96686d91b1d9e5203c4b075b"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:16b935c128f178647036048862a21e8bfd66d1e83fb19787a8b356bdcf0de899"
|
||||
name = "golang.org/x/crypto"
|
||||
packages = [
|
||||
"ed25519",
|
||||
"ed25519/internal/edwards25519",
|
||||
"ripemd160",
|
||||
"sha3",
|
||||
"ssh/terminal",
|
||||
]
|
||||
pruneopts = ""
|
||||
revision = "f027049dab0ad238e394a753dba2d14753473a04"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:15f2fc8cc79d90b0d4d712f04bd3eb3a3856ff3dd6b610a1425113ead3501610"
|
||||
name = "golang.org/x/net"
|
||||
packages = [
|
||||
"bpf",
|
||||
"context",
|
||||
"internal/iana",
|
||||
"internal/socket",
|
||||
"ipv4",
|
||||
"ipv6",
|
||||
]
|
||||
pruneopts = ""
|
||||
revision = "19491d39cadbd9cd33f26ca22cc89ba4ba38251c"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:e8f649ecfae7835a0a27ef39fd2180f6d3c12bc422e2ae55cd611f0b283b3e6e"
|
||||
name = "golang.org/x/sys"
|
||||
packages = [
|
||||
"unix",
|
||||
"windows",
|
||||
]
|
||||
pruneopts = ""
|
||||
revision = "0718ef2ef256118d53a01598f179001ec2af7626"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:55a681cb66f28755765fa5fa5104cbd8dc85c55c02d206f9f89566451e3fe1aa"
|
||||
name = "golang.org/x/time"
|
||||
packages = ["rate"]
|
||||
pruneopts = ""
|
||||
revision = "fbb02b2291d28baffd63558aa44b4b56f178d650"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:eede11c81b63c8f6fd06ef24ba0a640dc077196ec9b7a58ecde03c82eee2f151"
|
||||
name = "google.golang.org/appengine"
|
||||
packages = ["cloudsql"]
|
||||
pruneopts = ""
|
||||
revision = "b1f26356af11148e710935ed1ac8a7f5702c7612"
|
||||
version = "v1.1.0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:05eca53b271663de74078b5484b1995a8d56668a51434a698dc5d0863035d575"
|
||||
name = "gopkg.in/nullbio/null.v6"
|
||||
packages = ["convert"]
|
||||
pruneopts = ""
|
||||
revision = "40264a2e6b7972d183906cf17663983c23231c82"
|
||||
version = "v6.3"
|
||||
|
||||
[solve-meta]
|
||||
analyzer-name = "dep"
|
||||
analyzer-version = 1
|
||||
input-imports = [
|
||||
"github.com/aws/aws-sdk-go/aws",
|
||||
"github.com/aws/aws-sdk-go/aws/awserr",
|
||||
"github.com/aws/aws-sdk-go/aws/credentials",
|
||||
"github.com/aws/aws-sdk-go/aws/session",
|
||||
"github.com/aws/aws-sdk-go/service/s3",
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager",
|
||||
"github.com/davecgh/go-spew/spew",
|
||||
"github.com/go-sql-driver/mysql",
|
||||
"github.com/gorilla/mux",
|
||||
"github.com/gorilla/rpc/v2",
|
||||
"github.com/gorilla/rpc/v2/json",
|
||||
"github.com/hashicorp/serf/serf",
|
||||
"github.com/johntdyer/slackrus",
|
||||
"github.com/lbryio/errors.go",
|
||||
"github.com/lbryio/lbry.go/crypto",
|
||||
"github.com/lbryio/lbry.go/errors",
|
||||
"github.com/lbryio/lbry.go/querytools",
|
||||
"github.com/lbryio/lbry.go/stop",
|
||||
"github.com/lbryio/lbry.go/util",
|
||||
"github.com/lyoshenka/bencode",
|
||||
"github.com/phayes/freeport",
|
||||
"github.com/sebdah/goldie",
|
||||
"github.com/sirupsen/logrus",
|
||||
"github.com/spf13/cast",
|
||||
"github.com/spf13/cobra",
|
||||
"github.com/uber-go/atomic",
|
||||
"golang.org/x/time/rate",
|
||||
]
|
||||
solver-name = "gps-cdcl"
|
||||
solver-version = 1
|
||||
-39
@@ -1,39 +0,0 @@
|
||||
[[constraint]]
|
||||
name = "github.com/aws/aws-sdk-go"
|
||||
branch = "master"
|
||||
|
||||
[[constraint]]
|
||||
name = "github.com/go-sql-driver/mysql"
|
||||
branch = "master"
|
||||
|
||||
[[constraint]]
|
||||
name = "github.com/hashicorp/serf"
|
||||
branch = "master"
|
||||
|
||||
[[constraint]]
|
||||
name = "github.com/lbryio/lbry.go"
|
||||
branch = "master"
|
||||
|
||||
[[constraint]]
|
||||
name = "github.com/sirupsen/logrus"
|
||||
branch = "master"
|
||||
|
||||
[[constraint]]
|
||||
name = "github.com/spf13/cobra"
|
||||
branch = "master"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/lyoshenka/bencode"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/uber-go/atomic"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/phayes/freeport"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/johntdyer/slackrus"
|
||||
@@ -1,6 +1,6 @@
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2016-2018 LBRY Inc
|
||||
Copyright (c) 2016-2020 LBRY Inc
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish,distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||
|
||||
|
||||
@@ -1,27 +1,25 @@
|
||||
BINARY=prism-bin
|
||||
|
||||
DIR = $(shell cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)
|
||||
VENDOR_DIR = vendor
|
||||
BIN_DIR = ${DIR}/bin
|
||||
IMPORT_PATH = github.com/lbryio/reflector.go
|
||||
|
||||
VERSION = $(shell git --git-dir=${DIR}/.git describe --dirty --always --long --abbrev=7)
|
||||
LDFLAGS = -ldflags "-X ${IMPORT_PATH}/meta.Version=${VERSION}"
|
||||
LDFLAGS = -ldflags "-X ${IMPORT_PATH}/meta.Version=${VERSION} -X ${IMPORT_PATH}/meta.Time=$(shell date +%s)"
|
||||
|
||||
|
||||
.PHONY: build dep clean test
|
||||
.PHONY: build clean test lint
|
||||
.DEFAULT_GOAL: build
|
||||
|
||||
|
||||
build: dep
|
||||
CGO_ENABLED=0 go build ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${DIR}/${BINARY} main.go
|
||||
|
||||
dep: | $(VENDOR_DIR)
|
||||
|
||||
$(VENDOR_DIR):
|
||||
go get github.com/golang/dep/cmd/dep && dep ensure
|
||||
build:
|
||||
mkdir -p ${BIN_DIR} && CGO_ENABLED=0 go build ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/${BINARY} main.go
|
||||
|
||||
clean:
|
||||
if [ -f ${DIR}/${BINARY} ]; then rm ${DIR}/${BINARY}; fi
|
||||
if [ -f ${BIN_DIR}/${BINARY} ]; then rm ${BIN_DIR}/${BINARY}; fi
|
||||
|
||||
test:
|
||||
go test ./... -v -cover
|
||||
|
||||
lint:
|
||||
go get github.com/alecthomas/gometalinter && gometalinter --install && gometalinter ./...
|
||||
+3
-3
@@ -6,9 +6,9 @@ import (
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/crypto"
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/lbry.go/stop"
|
||||
"github.com/lbryio/lbry.go/v2/extras/crypto"
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
|
||||
"github.com/hashicorp/serf/serf"
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
+2
-1
@@ -6,7 +6,8 @@ import (
|
||||
"strconv"
|
||||
"syscall"
|
||||
|
||||
"github.com/lbryio/lbry.go/crypto"
|
||||
"github.com/lbryio/lbry.go/v2/extras/crypto"
|
||||
|
||||
"github.com/lbryio/reflector.go/cluster"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/schema/stake"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/gogo/protobuf/jsonpb"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func init() {
|
||||
var cmd = &cobra.Command{
|
||||
Use: "decode VALUE",
|
||||
Short: "Decode a claim value",
|
||||
Args: cobra.ExactArgs(1),
|
||||
Run: decodeCmd,
|
||||
}
|
||||
rootCmd.AddCommand(cmd)
|
||||
}
|
||||
|
||||
func decodeCmd(cmd *cobra.Command, args []string) {
|
||||
c, err := stake.DecodeClaimHex(args[0], "")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
m := jsonpb.Marshaler{Indent: " "}
|
||||
|
||||
if stream := c.Claim.GetStream(); stream != nil {
|
||||
json, err := m.MarshalToString(stream)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
fmt.Println(json)
|
||||
fmt.Printf("SD hash as hex: %s\n", hex.EncodeToString(stream.GetSource().GetSdHash()))
|
||||
} else if channel := c.Claim.GetChannel(); channel != nil {
|
||||
json, err := m.MarshalToString(channel)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
fmt.Println(json)
|
||||
} else if repost := c.Claim.GetRepost(); repost != nil {
|
||||
json, err := m.MarshalToString(repost)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
fmt.Println(json)
|
||||
} else {
|
||||
spew.Dump(c)
|
||||
}
|
||||
}
|
||||
+6
-6
@@ -1,7 +1,6 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"log"
|
||||
"net"
|
||||
"os"
|
||||
"os/signal"
|
||||
@@ -9,15 +8,16 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/dht"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
"github.com/lbryio/lbry.go/v2/dht"
|
||||
"github.com/lbryio/lbry.go/v2/dht/bits"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var dhtNodeID string
|
||||
var dhtPort int
|
||||
var dhtRpcPort int
|
||||
var dhtRPCPort int
|
||||
var dhtSeeds []string
|
||||
|
||||
func init() {
|
||||
@@ -30,7 +30,7 @@ func init() {
|
||||
}
|
||||
cmd.PersistentFlags().StringVar(&dhtNodeID, "nodeID", "", "nodeID in hex")
|
||||
cmd.PersistentFlags().IntVar(&dhtPort, "port", 4567, "Port to start DHT on")
|
||||
cmd.PersistentFlags().IntVar(&dhtRpcPort, "rpcPort", 0, "Port to listen for rpc commands on")
|
||||
cmd.PersistentFlags().IntVar(&dhtRPCPort, "rpcPort", 0, "Port to listen for rpc commands on")
|
||||
cmd.PersistentFlags().StringSliceVar(&dhtSeeds, "seeds", []string{}, "Addresses of seed nodes")
|
||||
rootCmd.AddCommand(cmd)
|
||||
}
|
||||
@@ -56,7 +56,7 @@ func dhtCmd(cmd *cobra.Command, args []string) {
|
||||
|
||||
dhtConf := dht.NewStandardConfig()
|
||||
dhtConf.Address = "0.0.0.0:" + strconv.Itoa(dhtPort)
|
||||
dhtConf.RPCPort = dhtRpcPort
|
||||
dhtConf.RPCPort = dhtRPCPort
|
||||
if len(dhtSeeds) > 0 {
|
||||
dhtConf.SeedNodes = dhtSeeds
|
||||
}
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/server/peer"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func init() {
|
||||
var cmd = &cobra.Command{
|
||||
Use: "getstream ADDRESS:PORT SDHASH",
|
||||
Short: "Get a stream from a reflector server",
|
||||
Args: cobra.ExactArgs(2),
|
||||
Run: getStreamCmd,
|
||||
}
|
||||
rootCmd.AddCommand(cmd)
|
||||
}
|
||||
|
||||
func getStreamCmd(cmd *cobra.Command, args []string) {
|
||||
addr := args[0]
|
||||
sdHash := args[1]
|
||||
|
||||
s := store.NewCachingStore(
|
||||
"getstream",
|
||||
peer.NewStore(peer.StoreOpts{Address: addr}),
|
||||
store.NewDiskStore("/tmp/lbry_downloaded_blobs", 2),
|
||||
)
|
||||
|
||||
wd, err := os.Getwd()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
var sd stream.SDBlob
|
||||
|
||||
sdb, _, err := s.Get(sdHash)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
err = sd.FromBlob(sdb)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
filename := sd.SuggestedFileName
|
||||
if filename == "" {
|
||||
filename = "stream_" + time.Now().Format("20060102_150405")
|
||||
}
|
||||
|
||||
f, err := os.Create(wd + "/" + filename)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
for i := 0; i < len(sd.BlobInfos)-1; i++ {
|
||||
b, _, err := s.Get(hex.EncodeToString(sd.BlobInfos[i].BlobHash))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
data, err := b.Plaintext(sd.Key, sd.BlobInfos[i].IV)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = f.Write(data)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"crypto/sha512"
|
||||
"encoding/hex"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path"
|
||||
"runtime"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/meta"
|
||||
"github.com/lbryio/reflector.go/store/speedwalk"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var threads int
|
||||
|
||||
func init() {
|
||||
var cmd = &cobra.Command{
|
||||
Use: "check-integrity",
|
||||
Short: "check blobs integrity for a given path",
|
||||
Run: integrityCheckCmd,
|
||||
}
|
||||
cmd.Flags().StringVar(&diskStorePath, "store-path", "", "path of the store where all blobs are cached")
|
||||
cmd.Flags().IntVar(&threads, "threads", runtime.NumCPU()-1, "number of concurrent threads to process blobs")
|
||||
rootCmd.AddCommand(cmd)
|
||||
}
|
||||
|
||||
func integrityCheckCmd(cmd *cobra.Command, args []string) {
|
||||
log.Printf("reflector %s", meta.VersionString())
|
||||
if diskStorePath == "" {
|
||||
log.Fatal("store-path must be defined")
|
||||
}
|
||||
|
||||
blobs, err := speedwalk.AllFiles(diskStorePath, true)
|
||||
if err != nil {
|
||||
log.Fatalf("error while reading blobs from disk %s", errors.FullTrace(err))
|
||||
}
|
||||
tasks := make(chan string, len(blobs))
|
||||
done := make(chan bool)
|
||||
processed := new(int32)
|
||||
go produce(tasks, blobs)
|
||||
cpus := runtime.NumCPU()
|
||||
for i := 0; i < cpus-1; i++ {
|
||||
go consume(i, tasks, done, len(blobs), processed)
|
||||
}
|
||||
<-done
|
||||
}
|
||||
|
||||
func produce(tasks chan<- string, blobs []string) {
|
||||
for _, b := range blobs {
|
||||
tasks <- b
|
||||
}
|
||||
close(tasks)
|
||||
}
|
||||
|
||||
func consume(worker int, tasks <-chan string, done chan<- bool, totalTasks int, processed *int32) {
|
||||
start := time.Now()
|
||||
|
||||
for b := range tasks {
|
||||
processedSoFar := atomic.AddInt32(processed, 1)
|
||||
if worker == 0 {
|
||||
remaining := int32(totalTasks) - processedSoFar
|
||||
timePerBlob := time.Since(start).Microseconds() / int64(processedSoFar)
|
||||
remainingTime := time.Duration(int64(remaining)*timePerBlob) * time.Microsecond
|
||||
log.Infof("[T%d] %d/%d blobs processed so far. ETA: %s", worker, processedSoFar, totalTasks, remainingTime.String())
|
||||
}
|
||||
blobPath := path.Join(diskStorePath, b[:2], b)
|
||||
blob, err := ioutil.ReadFile(blobPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
continue
|
||||
}
|
||||
log.Errorf("[Worker %d] Error looking up blob %s: %s", worker, b, err.Error())
|
||||
continue
|
||||
}
|
||||
hashBytes := sha512.Sum384(blob)
|
||||
readHash := hex.EncodeToString(hashBytes[:])
|
||||
if readHash != b {
|
||||
log.Infof("[%s] found a broken blob while reading from disk. Actual hash: %s", b, readHash)
|
||||
err := os.Remove(blobPath)
|
||||
if err != nil {
|
||||
log.Errorf("Error while deleting broken blob %s: %s", b, err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
done <- true
|
||||
}
|
||||
+18
-7
@@ -7,30 +7,41 @@ import (
|
||||
"syscall"
|
||||
|
||||
"github.com/lbryio/reflector.go/db"
|
||||
"github.com/lbryio/reflector.go/peer"
|
||||
"github.com/lbryio/reflector.go/server/peer"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var peerNoDB bool
|
||||
|
||||
func init() {
|
||||
var cmd = &cobra.Command{
|
||||
Use: "peer",
|
||||
Short: "Run peer server",
|
||||
Run: peerCmd,
|
||||
}
|
||||
cmd.Flags().BoolVar(&peerNoDB, "nodb", false, "Don't connect to a db and don't use a db-backed blob store")
|
||||
rootCmd.AddCommand(cmd)
|
||||
}
|
||||
|
||||
func peerCmd(cmd *cobra.Command, args []string) {
|
||||
db := new(db.SQL)
|
||||
err := db.Connect(globalConfig.DBConn)
|
||||
checkErr(err)
|
||||
var err error
|
||||
|
||||
s3 := store.NewS3BlobStore(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
|
||||
combo := store.NewDBBackedS3Store(s3, db)
|
||||
peerServer := peer.NewServer(combo)
|
||||
s3 := store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
|
||||
peerServer := peer.NewServer(s3)
|
||||
|
||||
if !peerNoDB {
|
||||
db := &db.SQL{
|
||||
LogQueries: log.GetLevel() == log.DebugLevel,
|
||||
}
|
||||
err = db.Connect(globalConfig.DBConn)
|
||||
checkErr(err)
|
||||
|
||||
combo := store.NewDBBackedStore(s3, db, false)
|
||||
peerServer = peer.NewServer(combo)
|
||||
}
|
||||
|
||||
err = peerServer.Start(":" + strconv.Itoa(peer.DefaultPort))
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"github.com/lbryio/reflector.go/db"
|
||||
"github.com/lbryio/reflector.go/meta"
|
||||
"github.com/lbryio/reflector.go/store/speedwalk"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var (
|
||||
diskStorePath string
|
||||
)
|
||||
|
||||
func init() {
|
||||
var cmd = &cobra.Command{
|
||||
Use: "populate-db",
|
||||
Short: "populate local database with blobs from a disk storage",
|
||||
Run: populateDbCmd,
|
||||
}
|
||||
cmd.Flags().StringVar(&diskStorePath, "store-path", "",
|
||||
"path of the store where all blobs are cached")
|
||||
rootCmd.AddCommand(cmd)
|
||||
}
|
||||
|
||||
func populateDbCmd(cmd *cobra.Command, args []string) {
|
||||
log.Printf("reflector %s", meta.VersionString())
|
||||
if diskStorePath == "" {
|
||||
log.Fatal("store-path must be defined")
|
||||
}
|
||||
localDb := &db.SQL{
|
||||
SoftDelete: true,
|
||||
TrackAccess: db.TrackAccessBlobs,
|
||||
LogQueries: log.GetLevel() == log.DebugLevel,
|
||||
}
|
||||
err := localDb.Connect("reflector:reflector@tcp(localhost:3306)/reflector")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
blobs, err := speedwalk.AllFiles(diskStorePath, true)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
err = localDb.AddBlobs(blobs)
|
||||
if err != nil {
|
||||
log.Errorf("error while storing to db: %s", errors.FullTrace(err))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/lbryio/reflector.go/publish"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/lbrycrd"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func init() {
|
||||
var cmd = &cobra.Command{
|
||||
Use: "publish FILE",
|
||||
Short: "Publish a file",
|
||||
Args: cobra.ExactArgs(1),
|
||||
Run: publishCmd,
|
||||
}
|
||||
cmd.Flags().String("name", "", "Claim name")
|
||||
cmd.Flags().String("title", "", "Title of the content")
|
||||
cmd.Flags().String("description", "", "Description of the content")
|
||||
cmd.Flags().String("author", "", "Content author")
|
||||
cmd.Flags().String("tags", "", "Comma-separated list of tags")
|
||||
cmd.Flags().Int64("release-time", 0, "original public release of content, seconds since UNIX epoch")
|
||||
rootCmd.AddCommand(cmd)
|
||||
}
|
||||
|
||||
func publishCmd(cmd *cobra.Command, args []string) {
|
||||
var err error
|
||||
|
||||
claimName := mustGetFlagString(cmd, "name")
|
||||
if claimName == "" {
|
||||
log.Errorln("--name required")
|
||||
return
|
||||
}
|
||||
|
||||
path := args[0]
|
||||
|
||||
client, err := lbrycrd.NewWithDefaultURL(nil)
|
||||
checkErr(err)
|
||||
|
||||
tx, txid, err := publish.Publish(
|
||||
client,
|
||||
path,
|
||||
claimName,
|
||||
"bSzpgkTnAoiT2YAhUShPpfpajPESfNXVTu",
|
||||
publish.Details{
|
||||
Title: mustGetFlagString(cmd, "title"),
|
||||
Description: mustGetFlagString(cmd, "description"),
|
||||
Author: mustGetFlagString(cmd, "author"),
|
||||
Tags: nil,
|
||||
ReleaseTime: mustGetFlagInt64(cmd, "release-time"),
|
||||
},
|
||||
"reflector.lbry.com:5566",
|
||||
)
|
||||
checkErr(err)
|
||||
|
||||
decoded, err := publish.Decode(client, tx)
|
||||
checkErr(err)
|
||||
|
||||
fmt.Printf("TX: %s\n\n", decoded)
|
||||
fmt.Printf("TXID: %s\n", txid.String())
|
||||
}
|
||||
+341
-11
@@ -4,46 +4,376 @@ import (
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
"github.com/lbryio/reflector.go/db"
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
"github.com/lbryio/reflector.go/meta"
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
"github.com/lbryio/reflector.go/server/http"
|
||||
http32 "github.com/lbryio/reflector.go/server/http3"
|
||||
"github.com/lbryio/reflector.go/server/peer"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
"github.com/c2h5oh/datasize"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var (
|
||||
//port configuration
|
||||
tcpPeerPort int
|
||||
http3PeerPort int
|
||||
httpPeerPort int
|
||||
receiverPort int
|
||||
metricsPort int
|
||||
|
||||
//flags configuration
|
||||
disableUploads bool
|
||||
disableBlocklist bool
|
||||
useDB bool
|
||||
|
||||
//upstream configuration
|
||||
upstreamReflector string
|
||||
upstreamProtocol string
|
||||
|
||||
//downstream configuration
|
||||
requestQueueSize int
|
||||
|
||||
//upstream edge configuration (to "cold" storage)
|
||||
originEndpoint string
|
||||
originEndpointFallback string
|
||||
|
||||
//cache configuration
|
||||
diskCache string
|
||||
secondaryDiskCache string
|
||||
memCache int
|
||||
)
|
||||
var cacheManagers = []string{"localdb", "lfu", "arc", "lru", "simple"}
|
||||
|
||||
var cacheMangerToGcache = map[string]store.EvictionStrategy{
|
||||
"lfu": store.LFU,
|
||||
"arc": store.ARC,
|
||||
"lru": store.LRU,
|
||||
"simple": store.SIMPLE,
|
||||
}
|
||||
|
||||
func init() {
|
||||
var cmd = &cobra.Command{
|
||||
Use: "reflector",
|
||||
Short: "Run reflector server",
|
||||
Run: reflectorCmd,
|
||||
}
|
||||
|
||||
cmd.Flags().IntVar(&tcpPeerPort, "tcp-peer-port", 5567, "The port reflector will distribute content from for the TCP (LBRY) protocol")
|
||||
cmd.Flags().IntVar(&http3PeerPort, "http3-peer-port", 5568, "The port reflector will distribute content from over HTTP3 protocol")
|
||||
cmd.Flags().IntVar(&httpPeerPort, "http-peer-port", 5569, "The port reflector will distribute content from over HTTP protocol")
|
||||
cmd.Flags().IntVar(&receiverPort, "receiver-port", 5566, "The port reflector will receive content from")
|
||||
cmd.Flags().IntVar(&metricsPort, "metrics-port", 2112, "The port reflector will use for prometheus metrics")
|
||||
|
||||
cmd.Flags().BoolVar(&disableUploads, "disable-uploads", false, "Disable uploads to this reflector server")
|
||||
cmd.Flags().BoolVar(&disableBlocklist, "disable-blocklist", false, "Disable blocklist watching/updating")
|
||||
cmd.Flags().BoolVar(&useDB, "use-db", true, "Whether to connect to the reflector db or not")
|
||||
|
||||
cmd.Flags().StringVar(&upstreamReflector, "upstream-reflector", "", "host:port of a reflector server where blobs are fetched from")
|
||||
cmd.Flags().StringVar(&upstreamProtocol, "upstream-protocol", "http", "protocol used to fetch blobs from another upstream reflector server (tcp/http3/http)")
|
||||
|
||||
cmd.Flags().IntVar(&requestQueueSize, "request-queue-size", 200, "How many concurrent requests from downstream should be handled at once (the rest will wait)")
|
||||
|
||||
cmd.Flags().StringVar(&originEndpoint, "origin-endpoint", "", "HTTP edge endpoint for standard HTTP retrieval")
|
||||
cmd.Flags().StringVar(&originEndpointFallback, "origin-endpoint-fallback", "", "HTTP edge endpoint for standard HTTP retrieval if first origin fails")
|
||||
|
||||
cmd.Flags().StringVar(&diskCache, "disk-cache", "100GB:/tmp/downloaded_blobs:localdb", "Where to cache blobs on the file system. format is 'sizeGB:CACHE_PATH:cachemanager' (cachemanagers: localdb/lfu/arc/lru)")
|
||||
cmd.Flags().StringVar(&secondaryDiskCache, "optional-disk-cache", "", "Optional secondary file system cache for blobs. format is 'sizeGB:CACHE_PATH:cachemanager' (cachemanagers: localdb/lfu/arc/lru) (this would get hit before the one specified in disk-cache)")
|
||||
cmd.Flags().IntVar(&memCache, "mem-cache", 0, "enable in-memory cache with a max size of this many blobs")
|
||||
|
||||
rootCmd.AddCommand(cmd)
|
||||
}
|
||||
|
||||
func reflectorCmd(cmd *cobra.Command, args []string) {
|
||||
log.Printf("reflector version %s", meta.Version)
|
||||
db := new(db.SQL)
|
||||
err := db.Connect(globalConfig.DBConn)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
log.Printf("reflector %s", meta.VersionString())
|
||||
|
||||
// the blocklist logic requires the db backed store to be the outer-most store
|
||||
underlyingStore := initStores()
|
||||
underlyingStoreWithCaches, cleanerStopper := initCaches(underlyingStore)
|
||||
|
||||
if !disableUploads {
|
||||
reflectorServer := reflector.NewServer(underlyingStore, underlyingStoreWithCaches)
|
||||
reflectorServer.Timeout = 3 * time.Minute
|
||||
reflectorServer.EnableBlocklist = !disableBlocklist
|
||||
|
||||
err := reflectorServer.Start(":" + strconv.Itoa(receiverPort))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer reflectorServer.Shutdown()
|
||||
}
|
||||
|
||||
s3 := store.NewS3BlobStore(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
|
||||
combo := store.NewDBBackedS3Store(s3, db)
|
||||
reflectorServer := reflector.NewServer(combo)
|
||||
reflectorServer.Timeout = 30 * time.Second
|
||||
err = reflectorServer.Start(":" + strconv.Itoa(reflector.DefaultPort))
|
||||
peerServer := peer.NewServer(underlyingStoreWithCaches)
|
||||
err := peerServer.Start(":" + strconv.Itoa(tcpPeerPort))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer peerServer.Shutdown()
|
||||
|
||||
http3PeerServer := http32.NewServer(underlyingStoreWithCaches, requestQueueSize)
|
||||
err = http3PeerServer.Start(":" + strconv.Itoa(http3PeerPort))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer http3PeerServer.Shutdown()
|
||||
|
||||
httpServer := http.NewServer(underlyingStoreWithCaches, requestQueueSize)
|
||||
err = httpServer.Start(":" + strconv.Itoa(httpPeerPort))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer httpServer.Shutdown()
|
||||
|
||||
metricsServer := metrics.NewServer(":"+strconv.Itoa(metricsPort), "/metrics")
|
||||
metricsServer.Start()
|
||||
defer metricsServer.Shutdown()
|
||||
defer underlyingStoreWithCaches.Shutdown()
|
||||
defer underlyingStore.Shutdown() //do we actually need this? Oo
|
||||
|
||||
interruptChan := make(chan os.Signal, 1)
|
||||
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
|
||||
<-interruptChan
|
||||
reflectorServer.Shutdown()
|
||||
// deferred shutdowns happen now
|
||||
cleanerStopper.StopAndWait()
|
||||
}
|
||||
|
||||
func initUpstreamStore() store.BlobStore {
|
||||
var s store.BlobStore
|
||||
if upstreamReflector == "" {
|
||||
return nil
|
||||
}
|
||||
switch upstreamProtocol {
|
||||
case "tcp":
|
||||
s = peer.NewStore(peer.StoreOpts{
|
||||
Address: upstreamReflector,
|
||||
Timeout: 30 * time.Second,
|
||||
})
|
||||
case "http3":
|
||||
s = http32.NewStore(http32.StoreOpts{
|
||||
Address: upstreamReflector,
|
||||
Timeout: 30 * time.Second,
|
||||
})
|
||||
case "http":
|
||||
s = store.NewHttpStore(upstreamReflector)
|
||||
default:
|
||||
log.Fatalf("protocol is not recognized: %s", upstreamProtocol)
|
||||
}
|
||||
return s
|
||||
}
|
||||
func initEdgeStore() store.BlobStore {
|
||||
var s3Store *store.S3Store
|
||||
var s store.BlobStore
|
||||
|
||||
if conf != "none" {
|
||||
s3Store = store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
|
||||
}
|
||||
if originEndpointFallback != "" && originEndpoint != "" {
|
||||
ittt := store.NewITTTStore(store.NewCloudFrontROStore(originEndpoint), store.NewCloudFrontROStore(originEndpointFallback))
|
||||
if s3Store != nil {
|
||||
s = store.NewCloudFrontRWStore(ittt, s3Store)
|
||||
} else {
|
||||
s = ittt
|
||||
}
|
||||
} else if s3Store != nil {
|
||||
s = s3Store
|
||||
} else {
|
||||
log.Fatalf("this configuration does not include a valid upstream source")
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func initDBStore(s store.BlobStore) store.BlobStore {
|
||||
if useDB {
|
||||
dbInst := &db.SQL{
|
||||
TrackAccess: db.TrackAccessStreams,
|
||||
LogQueries: log.GetLevel() == log.DebugLevel,
|
||||
}
|
||||
err := dbInst.Connect(globalConfig.DBConn)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
s = store.NewDBBackedStore(s, dbInst, false)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func initStores() store.BlobStore {
|
||||
s := initUpstreamStore()
|
||||
if s == nil {
|
||||
s = initEdgeStore()
|
||||
}
|
||||
s = initDBStore(s)
|
||||
return s
|
||||
}
|
||||
|
||||
// initCaches returns a store wrapped with caches and a stop group to execute a clean shutdown
|
||||
func initCaches(s store.BlobStore) (store.BlobStore, *stop.Group) {
|
||||
stopper := stop.New()
|
||||
diskStore := initDiskStore(s, diskCache, stopper)
|
||||
finalStore := initDiskStore(diskStore, secondaryDiskCache, stopper)
|
||||
stop.New()
|
||||
if memCache > 0 {
|
||||
finalStore = store.NewCachingStore(
|
||||
"reflector",
|
||||
finalStore,
|
||||
store.NewGcacheStore("mem", store.NewMemStore(), memCache, store.LRU),
|
||||
)
|
||||
}
|
||||
return finalStore, stopper
|
||||
}
|
||||
|
||||
func initDiskStore(upstreamStore store.BlobStore, diskParams string, stopper *stop.Group) store.BlobStore {
|
||||
diskCacheMaxSize, diskCachePath, cacheManager := diskCacheParams(diskParams)
|
||||
//we are tracking blobs in memory with a 1 byte long boolean, which means that for each 2MB (a blob) we need 1Byte
|
||||
// so if the underlying cache holds 10MB, 10MB/2MB=5Bytes which is also the exact count of objects to restore on startup
|
||||
realCacheSize := float64(diskCacheMaxSize) / float64(stream.MaxBlobSize)
|
||||
if diskCacheMaxSize == 0 {
|
||||
return upstreamStore
|
||||
}
|
||||
err := os.MkdirAll(diskCachePath, os.ModePerm)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
diskStore := store.NewDiskStore(diskCachePath, 2)
|
||||
var unwrappedStore store.BlobStore
|
||||
cleanerStopper := stop.New(stopper)
|
||||
|
||||
if cacheManager == "localdb" {
|
||||
localDb := &db.SQL{
|
||||
SoftDelete: true,
|
||||
TrackAccess: db.TrackAccessBlobs,
|
||||
LogQueries: log.GetLevel() == log.DebugLevel,
|
||||
}
|
||||
err = localDb.Connect("reflector:reflector@tcp(localhost:3306)/reflector")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
unwrappedStore = store.NewDBBackedStore(diskStore, localDb, true)
|
||||
go cleanOldestBlobs(int(realCacheSize), localDb, unwrappedStore, cleanerStopper)
|
||||
} else {
|
||||
unwrappedStore = store.NewGcacheStore("nvme", store.NewDiskStore(diskCachePath, 2), int(realCacheSize), cacheMangerToGcache[cacheManager])
|
||||
}
|
||||
|
||||
wrapped := store.NewCachingStore(
|
||||
"reflector",
|
||||
upstreamStore,
|
||||
unwrappedStore,
|
||||
)
|
||||
return wrapped
|
||||
}
|
||||
|
||||
func diskCacheParams(diskParams string) (int, string, string) {
|
||||
if diskParams == "" {
|
||||
return 0, "", ""
|
||||
}
|
||||
|
||||
parts := strings.Split(diskParams, ":")
|
||||
if len(parts) != 3 {
|
||||
log.Fatalf("%s does is formatted incorrectly. Expected format: 'sizeGB:CACHE_PATH:cachemanager' for example: '100GB:/tmp/downloaded_blobs:localdb'", diskParams)
|
||||
}
|
||||
|
||||
diskCacheSize := parts[0]
|
||||
path := parts[1]
|
||||
cacheManager := parts[2]
|
||||
|
||||
if len(path) == 0 || path[0] != '/' {
|
||||
log.Fatalf("disk cache paths must start with '/'")
|
||||
}
|
||||
|
||||
if !util.InSlice(cacheManager, cacheManagers) {
|
||||
log.Fatalf("specified cache manager '%s' is not supported. Use one of the following: %v", cacheManager, cacheManagers)
|
||||
}
|
||||
|
||||
var maxSize datasize.ByteSize
|
||||
err := maxSize.UnmarshalText([]byte(diskCacheSize))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if maxSize <= 0 {
|
||||
log.Fatal("disk cache size must be more than 0")
|
||||
}
|
||||
return int(maxSize), path, cacheManager
|
||||
}
|
||||
|
||||
func cleanOldestBlobs(maxItems int, db *db.SQL, store store.BlobStore, stopper *stop.Group) {
|
||||
// this is so that it runs on startup without having to wait for 10 minutes
|
||||
err := doClean(maxItems, db, store, stopper)
|
||||
if err != nil {
|
||||
log.Error(errors.FullTrace(err))
|
||||
}
|
||||
const cleanupInterval = 10 * time.Minute
|
||||
for {
|
||||
select {
|
||||
case <-stopper.Ch():
|
||||
log.Infoln("stopping self cleanup")
|
||||
return
|
||||
case <-time.After(cleanupInterval):
|
||||
err := doClean(maxItems, db, store, stopper)
|
||||
if err != nil {
|
||||
log.Error(errors.FullTrace(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func doClean(maxItems int, db *db.SQL, store store.BlobStore, stopper *stop.Group) error {
|
||||
blobsCount, err := db.Count()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if blobsCount >= maxItems {
|
||||
itemsToDelete := blobsCount / 10
|
||||
blobs, err := db.LeastRecentlyAccessedHashes(itemsToDelete)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
blobsChan := make(chan string, len(blobs))
|
||||
wg := &stop.Group{}
|
||||
go func() {
|
||||
for _, hash := range blobs {
|
||||
select {
|
||||
case <-stopper.Ch():
|
||||
return
|
||||
default:
|
||||
}
|
||||
blobsChan <- hash
|
||||
}
|
||||
close(blobsChan)
|
||||
}()
|
||||
for i := 0; i < 3; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for h := range blobsChan {
|
||||
select {
|
||||
case <-stopper.Ch():
|
||||
return
|
||||
default:
|
||||
}
|
||||
err = store.Delete(h)
|
||||
if err != nil {
|
||||
log.Errorf("error pruning %s: %s", h, errors.FullTrace(err))
|
||||
continue
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/lbryio/reflector.go/wallet"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func init() {
|
||||
var cmd = &cobra.Command{
|
||||
Use: "resolve ADDRESS:PORT URL",
|
||||
Short: "Resolve a URL",
|
||||
Args: cobra.ExactArgs(2),
|
||||
Run: resolveCmd,
|
||||
}
|
||||
rootCmd.AddCommand(cmd)
|
||||
}
|
||||
|
||||
func resolveCmd(cmd *cobra.Command, args []string) {
|
||||
addr := args[0]
|
||||
url := args[1]
|
||||
|
||||
node := wallet.NewNode()
|
||||
defer node.Shutdown()
|
||||
err := node.Connect([]string{addr}, nil)
|
||||
checkErr(err)
|
||||
|
||||
output, err := node.Resolve(url)
|
||||
checkErr(err)
|
||||
|
||||
claim, err := node.GetClaimInTx(hex.EncodeToString(rev(output.GetTxHash())), int(output.GetNout()))
|
||||
checkErr(err)
|
||||
|
||||
jsonClaim, err := json.MarshalIndent(claim, "", " ")
|
||||
checkErr(err)
|
||||
|
||||
fmt.Println(string(jsonClaim))
|
||||
}
|
||||
|
||||
func rev(b []byte) []byte {
|
||||
r := make([]byte, len(b))
|
||||
for left, right := 0, len(b)-1; left < right; left, right = left+1, right-1 {
|
||||
r[left], r[right] = b[right], b[left]
|
||||
}
|
||||
return r
|
||||
}
|
||||
+40
-6
@@ -4,10 +4,13 @@ import (
|
||||
"encoding/json"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/lbry.go/util"
|
||||
"github.com/lbryio/reflector.go/dht"
|
||||
"github.com/lbryio/reflector.go/updater"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/dht"
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
|
||||
"github.com/johntdyer/slackrus"
|
||||
"github.com/sirupsen/logrus"
|
||||
@@ -22,6 +25,8 @@ type Config struct {
|
||||
BucketName string `json:"bucket_name"`
|
||||
DBConn string `json:"db_conn"`
|
||||
SlackHookURL string `json:"slack_hook_url"`
|
||||
UpdateBinURL string `json:"update_bin_url"`
|
||||
UpdateCmd string `json:"update_cmd"`
|
||||
}
|
||||
|
||||
var verbose []string
|
||||
@@ -46,7 +51,7 @@ var rootCmd = &cobra.Command{
|
||||
|
||||
func init() {
|
||||
rootCmd.PersistentFlags().StringSliceVarP(&verbose, "verbose", "v", []string{}, "Verbose logging for specific components")
|
||||
rootCmd.PersistentFlags().StringVar(&conf, "conf", "config.json", "Path to config")
|
||||
rootCmd.PersistentFlags().StringVar(&conf, "conf", "config.json", "Path to config. Use 'none' to disable")
|
||||
}
|
||||
|
||||
// Execute adds all child commands to the root command and sets flags appropriately.
|
||||
@@ -65,8 +70,11 @@ func preRun(cmd *cobra.Command, args []string) {
|
||||
debugLogger.SetOutput(os.Stderr)
|
||||
|
||||
if util.InSlice(verboseAll, verbose) {
|
||||
logrus.Info("global verbose logging enabled")
|
||||
logrus.SetLevel(logrus.DebugLevel)
|
||||
verbose = []string{verboseDHT, verboseNodeFinder}
|
||||
} else if len(verbose) > 0 {
|
||||
logrus.Infof("verbose logging enabled for: %s", strings.Join(verbose, ", "))
|
||||
}
|
||||
|
||||
for _, debugType := range verbose {
|
||||
@@ -102,6 +110,14 @@ func preRun(cmd *cobra.Command, args []string) {
|
||||
logrus.AddHook(hook)
|
||||
debugLogger.AddHook(hook)
|
||||
}
|
||||
|
||||
if globalConfig.UpdateBinURL != "" {
|
||||
if globalConfig.UpdateCmd == "" {
|
||||
logrus.Warnln("update_cmd is empty in conf file")
|
||||
}
|
||||
logrus.Println("starting update checker")
|
||||
go updater.Run(globalConfig.UpdateBinURL, globalConfig.UpdateCmd)
|
||||
}
|
||||
}
|
||||
|
||||
func checkErr(err error) {
|
||||
@@ -130,9 +146,27 @@ func loadConfig(path string) (Config, error) {
|
||||
if os.IsNotExist(err) {
|
||||
return c, errors.Err("config file not found")
|
||||
}
|
||||
return c, err
|
||||
return c, errors.Err(err)
|
||||
}
|
||||
|
||||
err = json.Unmarshal(raw, &c)
|
||||
return c, err
|
||||
return c, errors.Err(err)
|
||||
}
|
||||
|
||||
func mustGetFlagString(cmd *cobra.Command, name string) string {
|
||||
v, err := cmd.Flags().GetString(name)
|
||||
checkErr(err)
|
||||
return v
|
||||
}
|
||||
|
||||
func mustGetFlagInt64(cmd *cobra.Command, name string) int64 {
|
||||
v, err := cmd.Flags().GetInt64(name)
|
||||
checkErr(err)
|
||||
return v
|
||||
}
|
||||
|
||||
//func mustGetFlagBool(cmd *cobra.Command, name string) bool {
|
||||
// v, err := cmd.Flags().GetBool(name)
|
||||
// checkErr(err)
|
||||
// return v
|
||||
//}
|
||||
+160
@@ -0,0 +1,160 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path"
|
||||
"syscall"
|
||||
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func init() {
|
||||
var cmd = &cobra.Command{
|
||||
Use: "send ADDRESS:PORT PATH",
|
||||
Short: "Send a file to a reflector",
|
||||
Args: cobra.ExactArgs(2),
|
||||
Run: sendCmd,
|
||||
}
|
||||
cmd.PersistentFlags().String("sd-cache", "", "path to dir where sd blobs will be cached")
|
||||
rootCmd.AddCommand(cmd)
|
||||
}
|
||||
|
||||
// todo: if retrying a large file is slow, we can add the ability to seek ahead in the file so we're not
|
||||
// re-uploading blobs that already exist
|
||||
|
||||
var hackyReflector reflector.Client
|
||||
|
||||
func sendCmd(cmd *cobra.Command, args []string) {
|
||||
reflectorAddress := args[0]
|
||||
err := hackyReflector.Connect(reflectorAddress)
|
||||
checkErr(err)
|
||||
defer hackyReflector.Close()
|
||||
|
||||
filePath := args[1]
|
||||
file, err := os.Open(filePath)
|
||||
checkErr(err)
|
||||
defer file.Close()
|
||||
|
||||
sdCachePath := ""
|
||||
sdCacheDir := mustGetFlagString(cmd, "sd-cache")
|
||||
if sdCacheDir != "" {
|
||||
if _, err := os.Stat(sdCacheDir); os.IsNotExist(err) {
|
||||
err = os.MkdirAll(sdCacheDir, 0777)
|
||||
checkErr(err)
|
||||
}
|
||||
sdCachePath = path.Join(sdCacheDir, filePath+".sdblob")
|
||||
}
|
||||
|
||||
var enc *stream.Encoder
|
||||
|
||||
if sdCachePath != "" {
|
||||
if _, err := os.Stat(sdCachePath); !os.IsNotExist(err) {
|
||||
sdBlob, err := ioutil.ReadFile(sdCachePath)
|
||||
checkErr(err)
|
||||
cachedSDBlob := &stream.SDBlob{}
|
||||
err = cachedSDBlob.FromBlob(sdBlob)
|
||||
checkErr(err)
|
||||
enc = stream.NewEncoderFromSD(file, cachedSDBlob)
|
||||
}
|
||||
}
|
||||
if enc == nil {
|
||||
enc = stream.NewEncoder(file)
|
||||
}
|
||||
|
||||
exitCode := 0
|
||||
|
||||
var killed bool
|
||||
interruptChan := make(chan os.Signal, 1)
|
||||
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
|
||||
go func() {
|
||||
sig := <-interruptChan
|
||||
fmt.Printf("caught %s, exiting...\n", sig.String())
|
||||
killed = true
|
||||
exitCode = 1
|
||||
}()
|
||||
|
||||
for {
|
||||
if killed {
|
||||
break
|
||||
}
|
||||
|
||||
b, err := enc.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Printf("error reading next blob: %v\n", err)
|
||||
exitCode = 1
|
||||
break
|
||||
}
|
||||
|
||||
err = hackyReflect(b, false)
|
||||
if err != nil {
|
||||
fmt.Printf("error reflecting blob %s: %v\n", b.HashHex()[:8], err)
|
||||
exitCode = 1
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
sd := enc.SDBlob()
|
||||
//sd.StreamName = filepath.Base(filePath)
|
||||
//sd.SuggestedFileName = filepath.Base(filePath)
|
||||
err = ioutil.WriteFile(sdCachePath, sd.ToBlob(), 0666)
|
||||
if err != nil {
|
||||
fmt.Printf("error saving sd blob: %v\n", err)
|
||||
fmt.Println(sd.ToJson())
|
||||
exitCode = 1
|
||||
}
|
||||
|
||||
if killed {
|
||||
os.Exit(exitCode)
|
||||
}
|
||||
|
||||
if reflectorAddress != "" {
|
||||
err = hackyReflect(sd.ToBlob(), true)
|
||||
if err != nil {
|
||||
fmt.Printf("error reflecting sd blob %s: %v\n", sd.HashHex()[:8], err)
|
||||
exitCode = 1
|
||||
}
|
||||
}
|
||||
|
||||
ret := struct {
|
||||
SDHash string `json:"sd_hash"`
|
||||
SourceHash string `json:"source_hash"`
|
||||
}{
|
||||
SDHash: sd.HashHex(),
|
||||
SourceHash: hex.EncodeToString(enc.SourceHash()),
|
||||
}
|
||||
|
||||
j, err := json.MarshalIndent(ret, "", " ")
|
||||
checkErr(err)
|
||||
fmt.Println(string(j))
|
||||
os.Exit(exitCode)
|
||||
}
|
||||
|
||||
func hackyReflect(b stream.Blob, sd bool) error {
|
||||
var err error
|
||||
if sd {
|
||||
err = hackyReflector.SendSDBlob(b)
|
||||
} else {
|
||||
err = hackyReflector.SendBlob(b)
|
||||
}
|
||||
|
||||
if errors.Is(err, reflector.ErrBlobExists) {
|
||||
//fmt.Printf("%s already reflected\n", b.HashHex()[:8])
|
||||
return nil
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
+39
-10
@@ -2,18 +2,21 @@ package cmd
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"os"
|
||||
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func init() {
|
||||
var cmd = &cobra.Command{
|
||||
Use: "sendblob ADDRESS:PORT",
|
||||
Use: "sendblob ADDRESS:PORT [PATH]",
|
||||
Short: "Send a random blob to a reflector server",
|
||||
Args: cobra.ExactArgs(1),
|
||||
Args: cobra.RangeArgs(1, 2),
|
||||
Run: sendBlobCmd,
|
||||
}
|
||||
rootCmd.AddCommand(cmd)
|
||||
@@ -21,21 +24,47 @@ func init() {
|
||||
|
||||
func sendBlobCmd(cmd *cobra.Command, args []string) {
|
||||
addr := args[0]
|
||||
var path string
|
||||
if len(args) >= 2 {
|
||||
path = args[1]
|
||||
}
|
||||
|
||||
c := reflector.Client{}
|
||||
err := c.Connect(addr)
|
||||
if err != nil {
|
||||
log.Fatal("error connecting client to server", err)
|
||||
log.Fatal("error connecting client to server: ", err)
|
||||
}
|
||||
|
||||
blob := make([]byte, 1024)
|
||||
_, err = rand.Read(blob)
|
||||
if err != nil {
|
||||
log.Fatal("failed to make random blob", err)
|
||||
if path == "" {
|
||||
blob := make(stream.Blob, 1024)
|
||||
_, err = rand.Read(blob)
|
||||
if err != nil {
|
||||
log.Fatal("failed to make random blob: ", err)
|
||||
}
|
||||
|
||||
err = c.SendBlob(blob)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
err = c.SendBlob(blob)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
file, err := os.Open(path)
|
||||
checkErr(err)
|
||||
defer file.Close()
|
||||
s, err := stream.New(file)
|
||||
checkErr(err)
|
||||
|
||||
sdBlob := &stream.SDBlob{}
|
||||
err = sdBlob.FromBlob(s[0])
|
||||
checkErr(err)
|
||||
|
||||
for i, b := range s {
|
||||
if i == 0 {
|
||||
err = c.SendSDBlob(b)
|
||||
} else {
|
||||
err = c.SendBlob(b)
|
||||
}
|
||||
checkErr(err)
|
||||
}
|
||||
}
|
||||
+9
-6
@@ -9,13 +9,14 @@ import (
|
||||
|
||||
"github.com/lbryio/reflector.go/cluster"
|
||||
"github.com/lbryio/reflector.go/db"
|
||||
"github.com/lbryio/reflector.go/dht"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
"github.com/lbryio/reflector.go/peer"
|
||||
"github.com/lbryio/reflector.go/prism"
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
"github.com/lbryio/reflector.go/server/peer"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/dht"
|
||||
"github.com/lbryio/lbry.go/v2/dht/bits"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
@@ -52,11 +53,13 @@ func init() {
|
||||
}
|
||||
|
||||
func startCmd(cmd *cobra.Command, args []string) {
|
||||
db := new(db.SQL)
|
||||
db := &db.SQL{
|
||||
LogQueries: log.GetLevel() == log.DebugLevel,
|
||||
}
|
||||
err := db.Connect(globalConfig.DBConn)
|
||||
checkErr(err)
|
||||
s3 := store.NewS3BlobStore(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
|
||||
comboStore := store.NewDBBackedS3Store(s3, db)
|
||||
s3 := store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
|
||||
comboStore := store.NewDBBackedStore(s3, db, false)
|
||||
|
||||
conf := prism.DefaultConf()
|
||||
|
||||
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/meta"
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
"github.com/lbryio/reflector.go/server/peer"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func init() {
|
||||
var cmd = &cobra.Command{
|
||||
Use: "test",
|
||||
Short: "Test things",
|
||||
Run: testCmd,
|
||||
}
|
||||
rootCmd.AddCommand(cmd)
|
||||
}
|
||||
|
||||
func testCmd(cmd *cobra.Command, args []string) {
|
||||
log.Printf("reflector %s", meta.VersionString())
|
||||
|
||||
memStore := store.NewMemStore()
|
||||
|
||||
reflectorServer := reflector.NewServer(memStore, memStore)
|
||||
reflectorServer.Timeout = 3 * time.Minute
|
||||
|
||||
err := reflectorServer.Start(":" + strconv.Itoa(reflector.DefaultPort))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
peerServer := peer.NewServer(memStore)
|
||||
err = peerServer.Start(":" + strconv.Itoa(reflector.DefaultPort+1))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
interruptChan := make(chan os.Signal, 1)
|
||||
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
|
||||
<-interruptChan
|
||||
peerServer.Shutdown()
|
||||
reflectorServer.Shutdown()
|
||||
|
||||
fmt.Println("Blobs in store")
|
||||
for hash, blob := range memStore.Debug() {
|
||||
fmt.Printf("%s: %d bytes\n", hash, len(blob))
|
||||
}
|
||||
}
|
||||
+13
-226
@@ -1,44 +1,21 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/db"
|
||||
"github.com/lbryio/reflector.go/peer"
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/stop"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var uploadWorkers int
|
||||
var uploadSkipExistsCheck bool
|
||||
|
||||
const (
|
||||
sdInc = 1
|
||||
blobInc = 2
|
||||
errInc = 3
|
||||
)
|
||||
|
||||
type uploaderParams struct {
|
||||
workerWG *sync.WaitGroup
|
||||
counterWG *sync.WaitGroup
|
||||
stopper *stop.Group
|
||||
pathChan chan string
|
||||
countChan chan int
|
||||
sdCount int
|
||||
blobCount int
|
||||
errCount int
|
||||
}
|
||||
var uploadDeleteBlobsAfterUpload bool
|
||||
|
||||
func init() {
|
||||
var cmd = &cobra.Command{
|
||||
@@ -49,220 +26,30 @@ func init() {
|
||||
}
|
||||
cmd.PersistentFlags().IntVar(&uploadWorkers, "workers", 1, "How many worker threads to run at once")
|
||||
cmd.PersistentFlags().BoolVar(&uploadSkipExistsCheck, "skipExistsCheck", false, "Dont check if blobs exist before uploading")
|
||||
cmd.PersistentFlags().BoolVar(&uploadDeleteBlobsAfterUpload, "deleteBlobsAfterUpload", false, "Delete blobs after uploading them")
|
||||
rootCmd.AddCommand(cmd)
|
||||
}
|
||||
|
||||
func uploadCmd(cmd *cobra.Command, args []string) {
|
||||
startTime := time.Now()
|
||||
db := new(db.SQL)
|
||||
db := &db.SQL{
|
||||
LogQueries: log.GetLevel() == log.DebugLevel,
|
||||
}
|
||||
err := db.Connect(globalConfig.DBConn)
|
||||
checkErr(err)
|
||||
|
||||
params := uploaderParams{
|
||||
workerWG: &sync.WaitGroup{},
|
||||
counterWG: &sync.WaitGroup{},
|
||||
pathChan: make(chan string),
|
||||
countChan: make(chan int),
|
||||
stopper: stop.New()}
|
||||
st := store.NewDBBackedStore(
|
||||
store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName),
|
||||
db, false)
|
||||
|
||||
setInterrupt(params.stopper)
|
||||
uploader := reflector.NewUploader(db, st, uploadWorkers, uploadSkipExistsCheck, uploadDeleteBlobsAfterUpload)
|
||||
|
||||
paths, err := getPaths(args[0])
|
||||
checkErr(err)
|
||||
|
||||
totalCount := len(paths)
|
||||
|
||||
hashes := make([]string, len(paths))
|
||||
for i, p := range paths {
|
||||
hashes[i] = path.Base(p)
|
||||
}
|
||||
|
||||
log.Println("checking for existing blobs")
|
||||
|
||||
exists := make(map[string]bool)
|
||||
if !uploadSkipExistsCheck {
|
||||
exists, err = db.HasBlobs(hashes)
|
||||
checkErr(err)
|
||||
}
|
||||
existsCount := len(exists)
|
||||
|
||||
log.Printf("%d new blobs to upload", totalCount-existsCount)
|
||||
|
||||
startUploadWorkers(¶ms)
|
||||
params.counterWG.Add(1)
|
||||
go func() {
|
||||
defer params.counterWG.Done()
|
||||
runCountReceiver(¶ms, startTime, totalCount, existsCount)
|
||||
}()
|
||||
|
||||
Upload:
|
||||
for _, f := range paths {
|
||||
if exists[path.Base(f)] {
|
||||
continue
|
||||
}
|
||||
|
||||
select {
|
||||
case params.pathChan <- f:
|
||||
case <-params.stopper.Ch():
|
||||
log.Warnln("Caught interrupt, quitting at first opportunity...")
|
||||
break Upload
|
||||
}
|
||||
}
|
||||
|
||||
close(params.pathChan)
|
||||
params.workerWG.Wait()
|
||||
close(params.countChan)
|
||||
params.counterWG.Wait()
|
||||
params.stopper.Stop()
|
||||
|
||||
log.Println("SUMMARY")
|
||||
log.Printf("%d blobs total", totalCount)
|
||||
log.Printf("%d SD blobs uploaded", params.sdCount)
|
||||
log.Printf("%d content blobs uploaded", params.blobCount)
|
||||
log.Printf("%d blobs already stored", existsCount)
|
||||
log.Printf("%d errors encountered", params.errCount)
|
||||
}
|
||||
|
||||
func isJSON(data []byte) bool {
|
||||
var js json.RawMessage
|
||||
return json.Unmarshal(data, &js) == nil
|
||||
}
|
||||
|
||||
func newBlobStore() *store.DBBackedS3Store {
|
||||
db := new(db.SQL)
|
||||
err := db.Connect(globalConfig.DBConn)
|
||||
checkErr(err)
|
||||
|
||||
s3 := store.NewS3BlobStore(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
|
||||
return store.NewDBBackedS3Store(s3, db)
|
||||
}
|
||||
|
||||
func setInterrupt(stopper *stop.Group) {
|
||||
interruptChan := make(chan os.Signal, 1)
|
||||
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
|
||||
go func() {
|
||||
<-interruptChan
|
||||
stopper.Stop()
|
||||
uploader.Stop()
|
||||
}()
|
||||
}
|
||||
|
||||
func startUploadWorkers(params *uploaderParams) {
|
||||
for i := 0; i < uploadWorkers; i++ {
|
||||
params.workerWG.Add(1)
|
||||
go func(i int) {
|
||||
defer params.workerWG.Done()
|
||||
defer func(i int) {
|
||||
log.Printf("worker %d quitting", i)
|
||||
}(i)
|
||||
|
||||
blobStore := newBlobStore()
|
||||
launchFileUploader(params, blobStore, i)
|
||||
}(i)
|
||||
}
|
||||
}
|
||||
|
||||
func launchFileUploader(params *uploaderParams, blobStore *store.DBBackedS3Store, worker int) {
|
||||
for {
|
||||
select {
|
||||
case <-params.stopper.Ch():
|
||||
return
|
||||
case filepath, ok := <-params.pathChan:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
blob, err := ioutil.ReadFile(filepath)
|
||||
checkErr(err)
|
||||
|
||||
hash := peer.GetBlobHash(blob)
|
||||
if hash != path.Base(filepath) {
|
||||
log.Errorf("worker %d: file name does not match hash (%s != %s), skipping", worker, filepath, hash)
|
||||
select {
|
||||
case params.countChan <- errInc:
|
||||
case <-params.stopper.Ch():
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if isJSON(blob) {
|
||||
log.Printf("worker %d: PUTTING SD BLOB %s", worker, hash)
|
||||
err := blobStore.PutSD(hash, blob)
|
||||
if err != nil {
|
||||
log.Error("PutSD Error: ", err)
|
||||
}
|
||||
select {
|
||||
case params.countChan <- sdInc:
|
||||
case <-params.stopper.Ch():
|
||||
}
|
||||
} else {
|
||||
log.Printf("worker %d: putting %s", worker, hash)
|
||||
err = blobStore.Put(hash, blob)
|
||||
if err != nil {
|
||||
log.Error("put Blob Error: ", err)
|
||||
}
|
||||
select {
|
||||
case params.countChan <- blobInc:
|
||||
case <-params.stopper.Ch():
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func runCountReceiver(params *uploaderParams, startTime time.Time, totalCount int, existsCount int) {
|
||||
for {
|
||||
select {
|
||||
case <-params.stopper.Ch():
|
||||
return
|
||||
case countType, ok := <-params.countChan:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
switch countType {
|
||||
case sdInc:
|
||||
params.sdCount++
|
||||
case blobInc:
|
||||
params.blobCount++
|
||||
case errInc:
|
||||
params.errCount++
|
||||
}
|
||||
}
|
||||
if (params.sdCount+params.blobCount)%50 == 0 {
|
||||
log.Printf("%d of %d done (%s elapsed, %.3fs per blob)", params.sdCount+params.blobCount, totalCount-existsCount, time.Since(startTime).String(), time.Since(startTime).Seconds()/float64(params.sdCount+params.blobCount))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func getPaths(path string) ([]string, error) {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if info.Mode().IsRegular() {
|
||||
return []string{path}, nil
|
||||
}
|
||||
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
files, err := f.Readdir(-1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = f.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var filenames []string
|
||||
for _, file := range files {
|
||||
if !file.IsDir() {
|
||||
filenames = append(filenames, path+"/"+file.Name())
|
||||
}
|
||||
}
|
||||
|
||||
return filenames, nil
|
||||
err = uploader.Upload(args[0])
|
||||
checkErr(err)
|
||||
}
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"log"
|
||||
"fmt"
|
||||
|
||||
"github.com/lbryio/reflector.go/meta"
|
||||
|
||||
@@ -18,5 +18,5 @@ func init() {
|
||||
}
|
||||
|
||||
func versionCmd(cmd *cobra.Command, args []string) {
|
||||
log.Printf("version %s\n", meta.Version)
|
||||
fmt.Println(meta.VersionString())
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"aws_id": "YOUR-AWS-ID",
|
||||
"aws_secret": "YOUR-AWS-SECRET",
|
||||
"bucket_region": "YOUR-BUCKET-REGION",
|
||||
"bucket_name": "YOUR-BUCKET-NAME",
|
||||
"db_conn": "USER:PASSWORD@tcp(localhost:3306)/DBNAME",
|
||||
"slack_hook_url": ""
|
||||
}
|
||||
@@ -3,13 +3,22 @@ package db
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/lbry.go/querytools"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
"github.com/lbryio/lbry.go/v2/dht/bits"
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
qt "github.com/lbryio/lbry.go/v2/extras/query"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
_ "github.com/go-sql-driver/mysql" // blank import for db driver
|
||||
"github.com/go-sql-driver/mysql"
|
||||
_ "github.com/go-sql-driver/mysql" // blank import for db driver ensures its imported even if its not used
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/volatiletech/null"
|
||||
"go.uber.org/atomic"
|
||||
)
|
||||
|
||||
// SdBlob is a special blob that contains information on the rest of the blobs in the stream
|
||||
@@ -19,7 +28,7 @@ type SdBlob struct {
|
||||
Length int `json:"length"`
|
||||
BlobNum int `json:"blob_num"`
|
||||
BlobHash string `json:"blob_hash,omitempty"`
|
||||
Iv string `json:"iv"`
|
||||
IV string `json:"iv"`
|
||||
} `json:"blobs"`
|
||||
StreamType string `json:"stream_type"`
|
||||
Key string `json:"key"`
|
||||
@@ -27,17 +36,38 @@ type SdBlob struct {
|
||||
StreamHash string `json:"stream_hash"`
|
||||
}
|
||||
|
||||
type trackAccess int
|
||||
|
||||
const (
|
||||
TrackAccessNone trackAccess = iota // Don't track accesses
|
||||
TrackAccessStreams // Track accesses at the stream level
|
||||
TrackAccessBlobs // Track accesses at the blob level
|
||||
)
|
||||
|
||||
// SQL implements the DB interface
|
||||
type SQL struct {
|
||||
conn *sql.DB
|
||||
|
||||
// Track the approx last time a blob or stream was accessed
|
||||
TrackAccess trackAccess
|
||||
|
||||
// Instead of deleting a blob, marked it as not stored in the db
|
||||
SoftDelete bool
|
||||
|
||||
// Log executed queries. qt.InterpolateParams is cpu-heavy. This avoids that call if not needed.
|
||||
LogQueries bool
|
||||
}
|
||||
|
||||
func logQuery(query string, args ...interface{}) {
|
||||
s, err := querytools.InterpolateParams(query, args...)
|
||||
func (s SQL) logQuery(query string, args ...interface{}) {
|
||||
if !s.LogQueries {
|
||||
return
|
||||
}
|
||||
|
||||
qStr, err := qt.InterpolateParams(query, args...)
|
||||
if err != nil {
|
||||
log.Errorln(err)
|
||||
} else {
|
||||
log.Debugln(s)
|
||||
log.Debugln(qStr)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -53,6 +83,8 @@ func (s *SQL) Connect(dsn string) error {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
s.conn.SetMaxIdleConns(12)
|
||||
|
||||
return errors.Err(s.conn.Ping())
|
||||
}
|
||||
|
||||
@@ -62,55 +94,240 @@ func (s *SQL) AddBlob(hash string, length int, isStored bool) error {
|
||||
return errors.Err("not connected")
|
||||
}
|
||||
|
||||
return withTx(s.conn, func(tx *sql.Tx) error {
|
||||
return addBlob(tx, hash, length, isStored)
|
||||
})
|
||||
_, err := s.insertBlob(hash, length, isStored)
|
||||
return err
|
||||
}
|
||||
|
||||
func addBlob(tx *sql.Tx, hash string, length int, isStored bool) error {
|
||||
if length <= 0 {
|
||||
return errors.Err("length must be positive")
|
||||
//AddBlobs adds blobs to the database.
|
||||
func (s *SQL) AddBlobs(hash []string) error {
|
||||
if s.conn == nil {
|
||||
return errors.Err("not connected")
|
||||
}
|
||||
|
||||
err := execTx(tx,
|
||||
"INSERT INTO blob_ (hash, is_stored, length) VALUES (?,?,?) ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored))",
|
||||
[]interface{}{hash, isStored, length},
|
||||
batch := 10000
|
||||
totalBlobs := int64(len(hash))
|
||||
work := make(chan []string, 1000)
|
||||
stopper := stop.New()
|
||||
var totalInserted atomic.Int64
|
||||
start := time.Now()
|
||||
|
||||
go func() {
|
||||
for i := 0; i < len(hash); i += batch {
|
||||
j := i + batch
|
||||
if j > len(hash) {
|
||||
j = len(hash)
|
||||
}
|
||||
work <- hash[i:j]
|
||||
}
|
||||
log.Infof("done loading %d hashes in the work queue", len(hash))
|
||||
close(work)
|
||||
}()
|
||||
|
||||
for i := 0; i < runtime.NumCPU(); i++ {
|
||||
stopper.Add(1)
|
||||
go func(worker int) {
|
||||
log.Infof("starting worker %d", worker)
|
||||
defer stopper.Done()
|
||||
for hashes := range work {
|
||||
inserted := totalInserted.Load()
|
||||
remaining := totalBlobs - inserted
|
||||
if inserted > 0 {
|
||||
timePerBlob := time.Since(start).Microseconds() / inserted
|
||||
remainingTime := time.Duration(remaining*timePerBlob) * time.Microsecond
|
||||
log.Infof("[T%d] processing batch of %d items. ETA: %s", worker, len(hashes), remainingTime.String())
|
||||
}
|
||||
err := s.insertBlobs(hashes) // Process the batch.
|
||||
if err != nil {
|
||||
log.Errorf("error while inserting batch: %s", errors.FullTrace(err))
|
||||
}
|
||||
totalInserted.Add(int64(len(hashes)))
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
|
||||
stopper.Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *SQL) insertBlobs(hashes []string) error {
|
||||
var (
|
||||
q string
|
||||
//args []interface{}
|
||||
)
|
||||
dayAgo := time.Now().AddDate(0, 0, -1).Format("2006-01-02 15:04:05")
|
||||
q = "insert into blob_ (hash, is_stored, length, last_accessed_at) values "
|
||||
for _, hash := range hashes {
|
||||
// prepared statements slow everything down by a lot due to reflection
|
||||
// for this specific instance we'll go ahead and hardcode the query to make it go faster
|
||||
q += fmt.Sprintf("('%s',1,%d,'%s'),", hash, stream.MaxBlobSize, dayAgo)
|
||||
//args = append(args, hash, true, stream.MaxBlobSize, dayAgo)
|
||||
}
|
||||
q = strings.TrimSuffix(q, ",")
|
||||
_, err := s.exec(q)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// HasBlob checks if the database contains the blob information.
|
||||
func (s *SQL) HasBlob(hash string) (bool, error) {
|
||||
if s.conn == nil {
|
||||
return false, errors.Err("not connected")
|
||||
func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error) {
|
||||
if length <= 0 {
|
||||
return 0, errors.Err("length must be positive")
|
||||
}
|
||||
|
||||
query := "SELECT EXISTS(SELECT 1 FROM blob_ WHERE hash = ? AND is_stored = ?)"
|
||||
args := []interface{}{hash, true}
|
||||
var (
|
||||
q string
|
||||
args []interface{}
|
||||
)
|
||||
if s.TrackAccess == TrackAccessBlobs {
|
||||
args = []interface{}{hash, isStored, length, time.Now()}
|
||||
q = "INSERT INTO blob_ (hash, is_stored, length, last_accessed_at) VALUES (" + qt.Qs(len(args)) + ") ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored)), last_accessed_at = VALUES(last_accessed_at)"
|
||||
} else {
|
||||
args = []interface{}{hash, isStored, length}
|
||||
q = "INSERT INTO blob_ (hash, is_stored, length) VALUES (" + qt.Qs(len(args)) + ") ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored))"
|
||||
}
|
||||
|
||||
logQuery(query, args...)
|
||||
blobID, err := s.exec(q, args...)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
row := s.conn.QueryRow(query, args...)
|
||||
if blobID == 0 {
|
||||
err = s.conn.QueryRow("SELECT id FROM blob_ WHERE hash = ?", hash).Scan(&blobID)
|
||||
if err != nil {
|
||||
return 0, errors.Err(err)
|
||||
}
|
||||
if blobID == 0 {
|
||||
return 0, errors.Err("blob ID is 0 even after INSERTing and SELECTing")
|
||||
}
|
||||
|
||||
exists := false
|
||||
err := row.Scan(&exists)
|
||||
if s.TrackAccess == TrackAccessBlobs {
|
||||
err := s.touchBlobs([]uint64{uint64(blobID)})
|
||||
if err != nil {
|
||||
return 0, errors.Err(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return exists, errors.Err(err)
|
||||
return blobID, nil
|
||||
}
|
||||
|
||||
func (s *SQL) insertStream(hash string, sdBlobID int64) (int64, error) {
|
||||
var (
|
||||
q string
|
||||
args []interface{}
|
||||
)
|
||||
|
||||
if s.TrackAccess == TrackAccessStreams {
|
||||
args = []interface{}{hash, sdBlobID, time.Now()}
|
||||
q = "INSERT IGNORE INTO stream (hash, sd_blob_id, last_accessed_at) VALUES (" + qt.Qs(len(args)) + ")"
|
||||
} else {
|
||||
args = []interface{}{hash, sdBlobID}
|
||||
q = "INSERT IGNORE INTO stream (hash, sd_blob_id) VALUES (" + qt.Qs(len(args)) + ")"
|
||||
}
|
||||
|
||||
streamID, err := s.exec(q, args...)
|
||||
if err != nil {
|
||||
return 0, errors.Err(err)
|
||||
}
|
||||
|
||||
if streamID == 0 {
|
||||
err = s.conn.QueryRow("SELECT id FROM stream WHERE sd_blob_id = ?", sdBlobID).Scan(&streamID)
|
||||
if err != nil {
|
||||
return 0, errors.Err(err)
|
||||
}
|
||||
if streamID == 0 {
|
||||
return 0, errors.Err("stream ID is 0 even after INSERTing and SELECTing")
|
||||
}
|
||||
|
||||
if s.TrackAccess == TrackAccessStreams {
|
||||
err := s.touchStreams([]uint64{uint64(streamID)})
|
||||
if err != nil {
|
||||
return 0, errors.Err(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return streamID, nil
|
||||
}
|
||||
|
||||
// HasBlob checks if the database contains the blob information.
|
||||
func (s *SQL) HasBlob(hash string, touch bool) (bool, error) {
|
||||
exists, err := s.HasBlobs([]string{hash}, touch)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return exists[hash], nil
|
||||
}
|
||||
|
||||
// HasBlobs checks if the database contains the set of blobs and returns a bool map.
|
||||
func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
|
||||
if s.conn == nil {
|
||||
return nil, errors.Err("not connected")
|
||||
func (s *SQL) HasBlobs(hashes []string, touch bool) (map[string]bool, error) {
|
||||
exists, idsNeedingTouch, err := s.hasBlobs(hashes)
|
||||
|
||||
if touch {
|
||||
if s.TrackAccess == TrackAccessBlobs {
|
||||
_ = s.touchBlobs(idsNeedingTouch)
|
||||
} else if s.TrackAccess == TrackAccessStreams {
|
||||
_ = s.touchStreams(idsNeedingTouch)
|
||||
}
|
||||
}
|
||||
|
||||
var hash string
|
||||
return exists, err
|
||||
}
|
||||
|
||||
func (s *SQL) touchBlobs(blobIDs []uint64) error {
|
||||
if len(blobIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
query := "UPDATE blob_ SET last_accessed_at = ? WHERE id IN (" + qt.Qs(len(blobIDs)) + ")"
|
||||
args := make([]interface{}, len(blobIDs)+1)
|
||||
args[0] = time.Now()
|
||||
for i := range blobIDs {
|
||||
args[i+1] = blobIDs[i]
|
||||
}
|
||||
|
||||
startTime := time.Now()
|
||||
_, err := s.exec(query, args...)
|
||||
log.Debugf("touched %d blobs and took %s", len(blobIDs), time.Since(startTime))
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
func (s *SQL) touchStreams(streamIDs []uint64) error {
|
||||
if len(streamIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
query := "UPDATE stream SET last_accessed_at = ? WHERE id IN (" + qt.Qs(len(streamIDs)) + ")"
|
||||
args := make([]interface{}, len(streamIDs)+1)
|
||||
args[0] = time.Now()
|
||||
for i := range streamIDs {
|
||||
args[i+1] = streamIDs[i]
|
||||
}
|
||||
|
||||
startTime := time.Now()
|
||||
_, err := s.exec(query, args...)
|
||||
log.Debugf("touched %d streams and took %s", len(streamIDs), time.Since(startTime))
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
func (s *SQL) hasBlobs(hashes []string) (map[string]bool, []uint64, error) {
|
||||
if s.conn == nil {
|
||||
return nil, nil, errors.Err("not connected")
|
||||
}
|
||||
|
||||
var (
|
||||
hash string
|
||||
blobID uint64
|
||||
streamID null.Uint64
|
||||
lastAccessedAt null.Time
|
||||
)
|
||||
|
||||
var needsTouch []uint64
|
||||
exists := make(map[string]bool)
|
||||
maxBatchSize := 100
|
||||
|
||||
touchDeadline := time.Now().Add(-6 * time.Hour) // touch blob if last accessed before this time
|
||||
maxBatchSize := 10000
|
||||
doneIndex := 0
|
||||
|
||||
for len(hashes) > doneIndex {
|
||||
@@ -121,41 +338,171 @@ func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
|
||||
log.Debugf("getting hashes[%d:%d] of %d", doneIndex, sliceEnd, len(hashes))
|
||||
batch := hashes[doneIndex:sliceEnd]
|
||||
|
||||
query := "SELECT hash FROM blob_ WHERE is_stored = ? && hash IN (" + querytools.Qs(len(batch)) + ")"
|
||||
args := make([]interface{}, len(batch)+1)
|
||||
args[0] = true
|
||||
var query string
|
||||
if s.TrackAccess == TrackAccessBlobs {
|
||||
query = `SELECT b.hash, b.id, NULL, b.last_accessed_at
|
||||
FROM blob_ b
|
||||
WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)`
|
||||
} else if s.TrackAccess == TrackAccessStreams {
|
||||
query = `SELECT b.hash, b.id, s.id, s.last_accessed_at
|
||||
FROM blob_ b
|
||||
LEFT JOIN stream_blob sb ON b.id = sb.blob_id
|
||||
INNER JOIN stream s on (sb.stream_id = s.id or s.sd_blob_id = b.id)
|
||||
WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)`
|
||||
} else {
|
||||
query = `SELECT b.hash, b.id, NULL, NULL
|
||||
FROM blob_ b
|
||||
WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)`
|
||||
}
|
||||
|
||||
args := make([]interface{}, len(batch))
|
||||
for i := range batch {
|
||||
args[i+1] = batch[i]
|
||||
args[i] = batch[i]
|
||||
}
|
||||
|
||||
logQuery(query, args...)
|
||||
s.logQuery(query, args...)
|
||||
|
||||
rows, err := s.conn.Query(query, args...)
|
||||
if err != nil {
|
||||
closeRows(rows)
|
||||
return exists, err
|
||||
}
|
||||
|
||||
for rows.Next() {
|
||||
err := rows.Scan(&hash)
|
||||
err := func() error {
|
||||
startTime := time.Now()
|
||||
rows, err := s.conn.Query(query, args...)
|
||||
log.Debugf("hashes query took %s", time.Since(startTime))
|
||||
if err != nil {
|
||||
closeRows(rows)
|
||||
return exists, err
|
||||
return errors.Err(err)
|
||||
}
|
||||
exists[hash] = true
|
||||
}
|
||||
defer closeRows(rows)
|
||||
|
||||
err = rows.Err()
|
||||
for rows.Next() {
|
||||
err := rows.Scan(&hash, &blobID, &streamID, &lastAccessedAt)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
exists[hash] = true
|
||||
if !lastAccessedAt.Valid || lastAccessedAt.Time.Before(touchDeadline) {
|
||||
if s.TrackAccess == TrackAccessBlobs {
|
||||
needsTouch = append(needsTouch, blobID)
|
||||
} else if s.TrackAccess == TrackAccessStreams && !streamID.IsZero() {
|
||||
needsTouch = append(needsTouch, streamID.Uint64)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
err = rows.Err()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
doneIndex += len(batch)
|
||||
return nil
|
||||
}()
|
||||
if err != nil {
|
||||
closeRows(rows)
|
||||
return exists, err
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
closeRows(rows)
|
||||
doneIndex += len(batch)
|
||||
}
|
||||
|
||||
return exists, nil
|
||||
return exists, needsTouch, nil
|
||||
}
|
||||
|
||||
// Delete will remove (or soft-delete) the blob from the db
|
||||
// NOTE: If SoftDelete is enabled, streams will never be deleted
|
||||
func (s *SQL) Delete(hash string) error {
|
||||
if s.SoftDelete {
|
||||
_, err := s.exec("UPDATE blob_ SET is_stored = 0 WHERE hash = ?", hash)
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
_, err := s.exec("DELETE FROM stream WHERE sd_blob_id = (SELECT id FROM blob_ WHERE hash = ?)", hash)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
_, err = s.exec("DELETE FROM blob_ WHERE hash = ?", hash)
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
//LeastRecentlyAccessedHashes gets the least recently accessed blobs
|
||||
func (s *SQL) LeastRecentlyAccessedHashes(maxBlobs int) ([]string, error) {
|
||||
if s.conn == nil {
|
||||
return nil, errors.Err("not connected")
|
||||
}
|
||||
|
||||
if s.TrackAccess != TrackAccessBlobs {
|
||||
return nil, errors.Err("blob access tracking is disabled")
|
||||
}
|
||||
|
||||
query := "SELECT hash from blob_ where is_stored = 1 order by last_accessed_at limit ?"
|
||||
s.logQuery(query, maxBlobs)
|
||||
|
||||
rows, err := s.conn.Query(query, maxBlobs)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
defer closeRows(rows)
|
||||
|
||||
blobs := make([]string, 0, maxBlobs)
|
||||
for rows.Next() {
|
||||
var hash string
|
||||
err := rows.Scan(&hash)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
blobs = append(blobs, hash)
|
||||
}
|
||||
|
||||
return blobs, nil
|
||||
}
|
||||
|
||||
func (s *SQL) Count() (int, error) {
|
||||
if s.conn == nil {
|
||||
return 0, errors.Err("not connected")
|
||||
}
|
||||
|
||||
query := "SELECT count(id) from blob_"
|
||||
if s.SoftDelete {
|
||||
query += " where is_stored = 1"
|
||||
}
|
||||
s.logQuery(query)
|
||||
|
||||
var count int
|
||||
err := s.conn.QueryRow(query).Scan(&count)
|
||||
return count, errors.Err(err)
|
||||
}
|
||||
|
||||
// Block will mark a blob as blocked
|
||||
func (s *SQL) Block(hash string) error {
|
||||
query := "INSERT IGNORE INTO blocked SET hash = ?"
|
||||
args := []interface{}{hash}
|
||||
s.logQuery(query, args...)
|
||||
_, err := s.conn.Exec(query, args...)
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
// GetBlocked will return a list of blocked hashes
|
||||
func (s *SQL) GetBlocked() (map[string]bool, error) {
|
||||
query := "SELECT hash FROM blocked"
|
||||
s.logQuery(query)
|
||||
rows, err := s.conn.Query(query)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
defer closeRows(rows)
|
||||
|
||||
blocked := make(map[string]bool)
|
||||
|
||||
var hash string
|
||||
for rows.Next() {
|
||||
err := rows.Scan(&hash)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
blocked[hash] = true
|
||||
}
|
||||
|
||||
err = rows.Err()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
return blocked, nil
|
||||
}
|
||||
|
||||
// MissingBlobsForKnownStream returns missing blobs for an existing stream
|
||||
@@ -168,13 +515,14 @@ func (s *SQL) MissingBlobsForKnownStream(sdHash string) ([]string, error) {
|
||||
|
||||
query := `
|
||||
SELECT b.hash FROM blob_ b
|
||||
INNER JOIN stream_blob sb ON b.hash = sb.blob_hash
|
||||
INNER JOIN stream s ON s.hash = sb.stream_hash AND s.sd_hash = ?
|
||||
INNER JOIN stream_blob sb ON b.id = sb.blob_id
|
||||
INNER JOIN stream s ON s.id = sb.stream_id
|
||||
INNER JOIN blob_ sdb ON sdb.id = s.sd_blob_id AND sdb.hash = ?
|
||||
WHERE b.is_stored = 0
|
||||
`
|
||||
args := []interface{}{sdHash}
|
||||
|
||||
logQuery(query, args...)
|
||||
s.logQuery(query, args...)
|
||||
|
||||
rows, err := s.conn.Query(query, args...)
|
||||
if err != nil {
|
||||
@@ -201,52 +549,45 @@ func (s *SQL) MissingBlobsForKnownStream(sdHash string) ([]string, error) {
|
||||
return missingBlobs, errors.Err(err)
|
||||
}
|
||||
|
||||
// AddSDBlob takes the SD Hash number of blobs and the set of blobs. In a single db tx it inserts the sdblob information
|
||||
// into a stream, and inserts the associated blobs' information in the database. If a blob fails the transaction is
|
||||
// rolled back and error(s) are returned.
|
||||
// AddSDBlob insert the SD blob and all the content blobs. The content blobs are marked as "not stored",
|
||||
// but they are tracked so reflector knows what it is missing.
|
||||
func (s *SQL) AddSDBlob(sdHash string, sdBlobLength int, sdBlob SdBlob) error {
|
||||
if s.conn == nil {
|
||||
return errors.Err("not connected")
|
||||
}
|
||||
|
||||
return withTx(s.conn, func(tx *sql.Tx) error {
|
||||
// insert sd blob
|
||||
err := addBlob(tx, sdHash, sdBlobLength, true)
|
||||
sdBlobID, err := s.insertBlob(sdHash, sdBlobLength, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
streamID, err := s.insertStream(sdBlob.StreamHash, sdBlobID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// insert content blobs and connect them to stream
|
||||
for _, contentBlob := range sdBlob.Blobs {
|
||||
if contentBlob.BlobHash == "" {
|
||||
// null terminator blob
|
||||
continue
|
||||
}
|
||||
|
||||
blobID, err := s.insertBlob(contentBlob.BlobHash, contentBlob.Length, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// insert stream
|
||||
err = execTx(tx,
|
||||
"INSERT IGNORE INTO stream (hash, sd_hash) VALUES (?,?)",
|
||||
[]interface{}{sdBlob.StreamHash, sdHash},
|
||||
args := []interface{}{streamID, blobID, contentBlob.BlobNum}
|
||||
_, err = s.exec(
|
||||
"INSERT IGNORE INTO stream_blob (stream_id, blob_id, num) VALUES ("+qt.Qs(len(args))+")",
|
||||
args...,
|
||||
)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
// insert content blobs and connect them to stream
|
||||
for _, contentBlob := range sdBlob.Blobs {
|
||||
if contentBlob.BlobHash == "" {
|
||||
// null terminator blob
|
||||
continue
|
||||
}
|
||||
|
||||
err := addBlob(tx, contentBlob.BlobHash, contentBlob.Length, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = execTx(tx,
|
||||
"INSERT IGNORE INTO stream_blob (stream_hash, blob_hash, num) VALUES (?,?,?)",
|
||||
[]interface{}{sdBlob.StreamHash, contentBlob.BlobHash, contentBlob.BlobNum},
|
||||
)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetHashRange gets the smallest and biggest hashes in the db
|
||||
@@ -260,7 +601,7 @@ func (s *SQL) GetHashRange() (string, string, error) {
|
||||
|
||||
query := "SELECT MIN(hash), MAX(hash) from blob_"
|
||||
|
||||
logQuery(query)
|
||||
s.logQuery(query)
|
||||
|
||||
err := s.conn.QueryRow(query).Scan(&min, &max)
|
||||
return min, max, err
|
||||
@@ -284,7 +625,7 @@ func (s *SQL) GetStoredHashesInRange(ctx context.Context, start, end bits.Bitmap
|
||||
query := "SELECT hash FROM blob_ WHERE hash >= ? AND hash <= ? AND is_stored = 1"
|
||||
args := []interface{}{start.Hex(), end.Hex()}
|
||||
|
||||
logQuery(query, args...)
|
||||
s.logQuery(query, args...)
|
||||
|
||||
rows, err := s.conn.Query(query, args...)
|
||||
defer closeRows(rows)
|
||||
@@ -364,38 +705,77 @@ func closeRows(rows *sql.Rows) {
|
||||
}
|
||||
}
|
||||
|
||||
func execTx(tx *sql.Tx, query string, args []interface{}) error {
|
||||
logQuery(query, args...)
|
||||
_, err := tx.Exec(query, args...)
|
||||
return errors.Err(err)
|
||||
func (s *SQL) exec(query string, args ...interface{}) (int64, error) {
|
||||
s.logQuery(query, args...)
|
||||
attempt, maxAttempts := 0, 3
|
||||
Retry:
|
||||
attempt++
|
||||
result, err := s.conn.Exec(query, args...)
|
||||
if isLockTimeoutError(err) {
|
||||
if attempt <= maxAttempts {
|
||||
//Error 1205: Lock wait timeout exceeded; try restarting transaction
|
||||
goto Retry
|
||||
}
|
||||
err = errors.Prefix("Lock timeout for query "+query, err)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return 0, errors.Err(err)
|
||||
}
|
||||
|
||||
lastID, err := result.LastInsertId()
|
||||
return lastID, errors.Err(err)
|
||||
}
|
||||
|
||||
func isLockTimeoutError(err error) bool {
|
||||
e, ok := err.(*mysql.MySQLError)
|
||||
return ok && e != nil && e.Number == 1205
|
||||
}
|
||||
|
||||
/* SQL schema
|
||||
|
||||
in prod, set tx_isolation to READ-COMMITTED to improve db performance
|
||||
make sure you use latin1 or utf8 charset, NOT utf8mb4. that's a waste of space.
|
||||
|
||||
todo: could add UNIQUE KEY (stream_hash, num) to stream_blob ...
|
||||
|
||||
CREATE TABLE blob_ (
|
||||
id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT UNIQUE,
|
||||
hash char(96) NOT NULL,
|
||||
is_stored TINYINT(1) NOT NULL DEFAULT 0,
|
||||
length bigint(20) unsigned DEFAULT NULL,
|
||||
PRIMARY KEY (hash)
|
||||
last_accessed_at TIMESTAMP NULL DEFAULT NULL,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY blob_hash_idx (hash),
|
||||
KEY `blob_last_accessed_idx` (`last_accessed_at`),
|
||||
KEY `is_stored_idx` (`is_stored`)
|
||||
);
|
||||
|
||||
CREATE TABLE stream (
|
||||
id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT UNIQUE,
|
||||
hash char(96) NOT NULL,
|
||||
sd_hash char(96) NOT NULL,
|
||||
PRIMARY KEY (hash),
|
||||
KEY sd_hash_idx (sd_hash),
|
||||
FOREIGN KEY (sd_hash) REFERENCES blob_ (hash) ON DELETE RESTRICT ON UPDATE CASCADE
|
||||
sd_blob_id BIGINT UNSIGNED NOT NULL,
|
||||
last_accessed_at TIMESTAMP NULL DEFAULT NULL,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY stream_hash_idx (hash),
|
||||
KEY stream_sd_blob_id_idx (sd_blob_id),
|
||||
KEY last_accessed_at_idx (last_accessed_at),
|
||||
FOREIGN KEY (sd_blob_id) REFERENCES blob_ (id) ON DELETE RESTRICT ON UPDATE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE stream_blob (
|
||||
stream_hash char(96) NOT NULL,
|
||||
blob_hash char(96) NOT NULL,
|
||||
stream_id BIGINT UNSIGNED NOT NULL,
|
||||
blob_id BIGINT UNSIGNED NOT NULL,
|
||||
num int NOT NULL,
|
||||
PRIMARY KEY (stream_hash, blob_hash),
|
||||
FOREIGN KEY (stream_hash) REFERENCES stream (hash) ON DELETE CASCADE ON UPDATE CASCADE,
|
||||
FOREIGN KEY (blob_hash) REFERENCES blob_ (hash) ON DELETE CASCADE ON UPDATE CASCADE
|
||||
PRIMARY KEY (stream_id, blob_id),
|
||||
KEY stream_blob_blob_id_idx (blob_id),
|
||||
FOREIGN KEY (stream_id) REFERENCES stream (id) ON DELETE CASCADE ON UPDATE CASCADE,
|
||||
FOREIGN KEY (blob_id) REFERENCES blob_ (id) ON DELETE CASCADE ON UPDATE CASCADE
|
||||
);
|
||||
|
||||
could add UNIQUE KEY (stream_hash, num) to stream_blob ...
|
||||
CREATE TABLE blocked (
|
||||
hash char(96) NOT NULL,
|
||||
PRIMARY KEY (hash)
|
||||
);
|
||||
|
||||
*/
|
||||
@@ -1,399 +0,0 @@
|
||||
package bits
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"math/big"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
|
||||
"github.com/lyoshenka/bencode"
|
||||
)
|
||||
|
||||
// TODO: http://roaringbitmap.org/
|
||||
|
||||
const (
|
||||
NumBytes = 48 // bytes
|
||||
NumBits = NumBytes * 8
|
||||
)
|
||||
|
||||
// Bitmap is a generalized representation of an identifier or data that can be sorted, compared fast. Used by the DHT
|
||||
// package as a way to handle the unique identifiers of a DHT node.
|
||||
type Bitmap [NumBytes]byte
|
||||
|
||||
func (b Bitmap) RawString() string {
|
||||
return string(b[:])
|
||||
}
|
||||
|
||||
func (b Bitmap) String() string {
|
||||
return b.Hex()
|
||||
}
|
||||
|
||||
// BString returns the bitmap as a string of 0s and 1s
|
||||
func (b Bitmap) BString() string {
|
||||
var s string
|
||||
for _, byte := range b {
|
||||
s += strconv.FormatInt(int64(byte), 2)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// Hex returns a hexadecimal representation of the bitmap.
|
||||
func (b Bitmap) Hex() string {
|
||||
return hex.EncodeToString(b[:])
|
||||
}
|
||||
|
||||
// HexShort returns a hexadecimal representation of the first 4 bytes.
|
||||
func (b Bitmap) HexShort() string {
|
||||
return hex.EncodeToString(b[:4])
|
||||
}
|
||||
|
||||
// HexSimplified returns the hexadecimal representation with all leading 0's removed
|
||||
func (b Bitmap) HexSimplified() string {
|
||||
simple := strings.TrimLeft(b.Hex(), "0")
|
||||
if simple == "" {
|
||||
simple = "0"
|
||||
}
|
||||
return simple
|
||||
}
|
||||
|
||||
func (b Bitmap) Big() *big.Int {
|
||||
i := new(big.Int)
|
||||
i.SetString(b.Hex(), 16)
|
||||
return i
|
||||
}
|
||||
|
||||
// Cmp compares b and other and returns:
|
||||
//
|
||||
// -1 if b < other
|
||||
// 0 if b == other
|
||||
// +1 if b > other
|
||||
//
|
||||
func (b Bitmap) Cmp(other Bitmap) int {
|
||||
for k := range b {
|
||||
if b[k] < other[k] {
|
||||
return -1
|
||||
} else if b[k] > other[k] {
|
||||
return 1
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// Closer returns true if dist(b,x) < dist(b,y)
|
||||
func (b Bitmap) Closer(x, y Bitmap) bool {
|
||||
return x.Xor(b).Cmp(y.Xor(b)) < 0
|
||||
}
|
||||
|
||||
// Equals returns true if every byte in bitmap are equal, false otherwise
|
||||
func (b Bitmap) Equals(other Bitmap) bool {
|
||||
return b.Cmp(other) == 0
|
||||
}
|
||||
|
||||
// Copy returns a duplicate value for the bitmap.
|
||||
func (b Bitmap) Copy() Bitmap {
|
||||
var ret Bitmap
|
||||
copy(ret[:], b[:])
|
||||
return ret
|
||||
}
|
||||
|
||||
// Xor returns a diff bitmap. If they are equal, the returned bitmap will be all 0's. If 100% unique the returned
|
||||
// bitmap will be all 1's.
|
||||
func (b Bitmap) Xor(other Bitmap) Bitmap {
|
||||
var ret Bitmap
|
||||
for k := range b {
|
||||
ret[k] = b[k] ^ other[k]
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
// And returns a comparison bitmap, that for each byte returns the AND true table result
|
||||
func (b Bitmap) And(other Bitmap) Bitmap {
|
||||
var ret Bitmap
|
||||
for k := range b {
|
||||
ret[k] = b[k] & other[k]
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
// Or returns a comparison bitmap, that for each byte returns the OR true table result
|
||||
func (b Bitmap) Or(other Bitmap) Bitmap {
|
||||
var ret Bitmap
|
||||
for k := range b {
|
||||
ret[k] = b[k] | other[k]
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
// Not returns a complimentary bitmap that is an inverse. So b.NOT.NOT = b
|
||||
func (b Bitmap) Not() Bitmap {
|
||||
var ret Bitmap
|
||||
for k := range b {
|
||||
ret[k] = ^b[k]
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
func (b Bitmap) add(other Bitmap) (Bitmap, bool) {
|
||||
var ret Bitmap
|
||||
carry := false
|
||||
for i := NumBits - 1; i >= 0; i-- {
|
||||
bBit := getBit(b[:], i)
|
||||
oBit := getBit(other[:], i)
|
||||
setBit(ret[:], i, bBit != oBit != carry)
|
||||
carry = (bBit && oBit) || (bBit && carry) || (oBit && carry)
|
||||
}
|
||||
return ret, carry
|
||||
}
|
||||
|
||||
// Add returns a bitmap that treats both bitmaps as numbers and adding them together. Since the size of a bitmap is
|
||||
// limited, an overflow is possible when adding bitmaps.
|
||||
func (b Bitmap) Add(other Bitmap) Bitmap {
|
||||
ret, carry := b.add(other)
|
||||
if carry {
|
||||
panic("overflow in bitmap addition. limited to " + strconv.Itoa(NumBits) + " bits.")
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
// Sub returns a bitmap that treats both bitmaps as numbers and subtracts then via the inverse of the other and adding
|
||||
// then together a + (-b). Negative bitmaps are not supported so other must be greater than this.
|
||||
func (b Bitmap) Sub(other Bitmap) Bitmap {
|
||||
if b.Cmp(other) < 0 {
|
||||
// ToDo: Why is this not supported? Should it say not implemented? BitMap might have a generic use case outside of dht.
|
||||
panic("negative bitmaps not supported")
|
||||
}
|
||||
complement, _ := other.Not().add(FromShortHexP("1"))
|
||||
ret, _ := b.add(complement)
|
||||
return ret
|
||||
}
|
||||
|
||||
// Get returns the binary bit at the position passed.
|
||||
func (b Bitmap) Get(n int) bool {
|
||||
return getBit(b[:], n)
|
||||
}
|
||||
|
||||
// Set sets the binary bit at the position passed.
|
||||
func (b Bitmap) Set(n int, one bool) Bitmap {
|
||||
ret := b.Copy()
|
||||
setBit(ret[:], n, one)
|
||||
return ret
|
||||
}
|
||||
|
||||
// PrefixLen returns the number of leading 0 bits
|
||||
func (b Bitmap) PrefixLen() int {
|
||||
for i := range b {
|
||||
for j := 0; j < 8; j++ {
|
||||
if (b[i]>>uint8(7-j))&0x1 != 0 {
|
||||
return i*8 + j
|
||||
}
|
||||
}
|
||||
}
|
||||
return NumBits
|
||||
}
|
||||
|
||||
// Prefix returns a copy of b with the first n bits set to 1 (if `one` is true) or 0 (if `one` is false)
|
||||
// https://stackoverflow.com/a/23192263/182709
|
||||
func (b Bitmap) Prefix(n int, one bool) Bitmap {
|
||||
ret := b.Copy()
|
||||
|
||||
Outer:
|
||||
for i := range ret {
|
||||
for j := 0; j < 8; j++ {
|
||||
if i*8+j < n {
|
||||
if one {
|
||||
ret[i] |= 1 << uint(7-j)
|
||||
} else {
|
||||
ret[i] &= ^(1 << uint(7-j))
|
||||
}
|
||||
} else {
|
||||
break Outer
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ret
|
||||
}
|
||||
|
||||
// Suffix returns a copy of b with the last n bits set to 1 (if `one` is true) or 0 (if `one` is false)
|
||||
// https://stackoverflow.com/a/23192263/182709
|
||||
func (b Bitmap) Suffix(n int, one bool) Bitmap {
|
||||
ret := b.Copy()
|
||||
|
||||
Outer:
|
||||
for i := len(ret) - 1; i >= 0; i-- {
|
||||
for j := 7; j >= 0; j-- {
|
||||
if i*8+j >= NumBits-n {
|
||||
if one {
|
||||
ret[i] |= 1 << uint(7-j)
|
||||
} else {
|
||||
ret[i] &= ^(1 << uint(7-j))
|
||||
}
|
||||
} else {
|
||||
break Outer
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ret
|
||||
}
|
||||
|
||||
// MarshalBencode implements the Marshaller(bencode)/Message interface.
|
||||
func (b Bitmap) MarshalBencode() ([]byte, error) {
|
||||
str := string(b[:])
|
||||
return bencode.EncodeBytes(str)
|
||||
}
|
||||
|
||||
// UnmarshalBencode implements the Marshaller(bencode)/Message interface.
|
||||
func (b *Bitmap) UnmarshalBencode(encoded []byte) error {
|
||||
var str string
|
||||
err := bencode.DecodeBytes(encoded, &str)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(str) != NumBytes {
|
||||
return errors.Err("invalid bitmap length")
|
||||
}
|
||||
copy(b[:], str)
|
||||
return nil
|
||||
}
|
||||
|
||||
// FromBytes returns a bitmap as long as the byte array is of a specific length specified in the parameters.
|
||||
func FromBytes(data []byte) (Bitmap, error) {
|
||||
var bmp Bitmap
|
||||
|
||||
if len(data) != len(bmp) {
|
||||
return bmp, errors.Err("invalid bitmap of length %d", len(data))
|
||||
}
|
||||
|
||||
copy(bmp[:], data)
|
||||
return bmp, nil
|
||||
}
|
||||
|
||||
// FromBytesP returns a bitmap as long as the byte array is of a specific length specified in the parameters
|
||||
// otherwise it wil panic.
|
||||
func FromBytesP(data []byte) Bitmap {
|
||||
bmp, err := FromBytes(data)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return bmp
|
||||
}
|
||||
|
||||
//FromString returns a bitmap by converting the string to bytes and creating from bytes as long as the byte array
|
||||
// is of a specific length specified in the parameters
|
||||
func FromString(data string) (Bitmap, error) {
|
||||
return FromBytes([]byte(data))
|
||||
}
|
||||
|
||||
//FromStringP returns a bitmap by converting the string to bytes and creating from bytes as long as the byte array
|
||||
// is of a specific length specified in the parameters otherwise it wil panic.
|
||||
func FromStringP(data string) Bitmap {
|
||||
bmp, err := FromString(data)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return bmp
|
||||
}
|
||||
|
||||
//FromHex returns a bitmap by converting the hex string to bytes and creating from bytes as long as the byte array
|
||||
// is of a specific length specified in the parameters
|
||||
func FromHex(hexStr string) (Bitmap, error) {
|
||||
decoded, err := hex.DecodeString(hexStr)
|
||||
if err != nil {
|
||||
return Bitmap{}, errors.Err(err)
|
||||
}
|
||||
return FromBytes(decoded)
|
||||
}
|
||||
|
||||
//FromHexP returns a bitmap by converting the hex string to bytes and creating from bytes as long as the byte array
|
||||
// is of a specific length specified in the parameters otherwise it wil panic.
|
||||
func FromHexP(hexStr string) Bitmap {
|
||||
bmp, err := FromHex(hexStr)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return bmp
|
||||
}
|
||||
|
||||
//FromShortHex returns a bitmap by converting the hex string to bytes, adding the leading zeros prefix to the
|
||||
// hex string and creating from bytes as long as the byte array is of a specific length specified in the parameters
|
||||
func FromShortHex(hexStr string) (Bitmap, error) {
|
||||
return FromHex(strings.Repeat("0", NumBytes*2-len(hexStr)) + hexStr)
|
||||
}
|
||||
|
||||
//FromShortHexP returns a bitmap by converting the hex string to bytes, adding the leading zeros prefix to the
|
||||
// hex string and creating from bytes as long as the byte array is of a specific length specified in the parameters
|
||||
// otherwise it wil panic.
|
||||
func FromShortHexP(hexStr string) Bitmap {
|
||||
bmp, err := FromShortHex(hexStr)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return bmp
|
||||
}
|
||||
|
||||
func FromBigP(b *big.Int) Bitmap {
|
||||
return FromShortHexP(b.Text(16))
|
||||
}
|
||||
|
||||
// Max returns a bitmap with all bits set to 1
|
||||
func MaxP() Bitmap {
|
||||
return FromHexP(strings.Repeat("f", NumBytes*2))
|
||||
}
|
||||
|
||||
// Rand generates a cryptographically random bitmap with the confines of the parameters specified.
|
||||
func Rand() Bitmap {
|
||||
var id Bitmap
|
||||
_, err := rand.Read(id[:])
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
// RandInRangeP generates a cryptographically random bitmap and while it is greater than the high threshold
|
||||
// bitmap will subtract the diff between high and low until it is no longer greater that the high.
|
||||
func RandInRangeP(low, high Bitmap) Bitmap {
|
||||
diff := high.Sub(low)
|
||||
r := Rand()
|
||||
for r.Cmp(diff) > 0 {
|
||||
r = r.Sub(diff)
|
||||
}
|
||||
//ToDo - Adding the low at this point doesn't gurantee it will be within the range. Consider bitmaps as numbers and
|
||||
// I have a range of 50-100. If get to say 60, and add 50, I would be at 110. Should protect against this?
|
||||
return r.Add(low)
|
||||
}
|
||||
|
||||
func getBit(b []byte, n int) bool {
|
||||
i := n / 8
|
||||
j := n % 8
|
||||
return b[i]&(1<<uint(7-j)) > 0
|
||||
}
|
||||
|
||||
func setBit(b []byte, n int, one bool) {
|
||||
i := n / 8
|
||||
j := n % 8
|
||||
if one {
|
||||
b[i] |= 1 << uint(7-j)
|
||||
} else {
|
||||
b[i] &= ^(1 << uint(7-j))
|
||||
}
|
||||
}
|
||||
|
||||
// CLosest returns the closest bitmap to target. if no bitmaps are provided, target itself is returned
|
||||
func Closest(target Bitmap, bitmaps ...Bitmap) Bitmap {
|
||||
if len(bitmaps) == 0 {
|
||||
return target
|
||||
}
|
||||
|
||||
var closest *Bitmap
|
||||
for _, b := range bitmaps {
|
||||
if closest == nil || target.Closer(b, *closest) {
|
||||
closest = &b
|
||||
}
|
||||
}
|
||||
return *closest
|
||||
}
|
||||
@@ -1,386 +0,0 @@
|
||||
package bits
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/lyoshenka/bencode"
|
||||
)
|
||||
|
||||
func TestBitmap(t *testing.T) {
|
||||
a := Bitmap{
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
|
||||
12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
|
||||
24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35,
|
||||
36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,
|
||||
}
|
||||
b := Bitmap{
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
|
||||
12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
|
||||
24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35,
|
||||
36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 47, 46,
|
||||
}
|
||||
c := Bitmap{
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1,
|
||||
}
|
||||
|
||||
if !a.Equals(a) {
|
||||
t.Error("bitmap does not equal itself")
|
||||
}
|
||||
if a.Equals(b) {
|
||||
t.Error("bitmap equals another bitmap with different id")
|
||||
}
|
||||
|
||||
if !a.Xor(b).Equals(c) {
|
||||
t.Error(a.Xor(b))
|
||||
}
|
||||
|
||||
if c.PrefixLen() != 375 {
|
||||
t.Error(c.PrefixLen())
|
||||
}
|
||||
|
||||
if b.Cmp(a) < 0 {
|
||||
t.Error("bitmap fails Cmp test")
|
||||
}
|
||||
|
||||
if a.Closer(c, b) || !a.Closer(b, c) || c.Closer(a, b) || c.Closer(b, c) {
|
||||
t.Error("bitmap fails Closer test")
|
||||
}
|
||||
|
||||
id := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
|
||||
if FromHexP(id).Hex() != id {
|
||||
t.Error(FromHexP(id).Hex())
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmap_GetBit(t *testing.T) {
|
||||
tt := []struct {
|
||||
bit int
|
||||
expected bool
|
||||
panic bool
|
||||
}{
|
||||
{bit: 383, expected: false, panic: false},
|
||||
{bit: 382, expected: true, panic: false},
|
||||
{bit: 381, expected: false, panic: false},
|
||||
{bit: 380, expected: true, panic: false},
|
||||
}
|
||||
|
||||
b := FromShortHexP("a")
|
||||
|
||||
for _, test := range tt {
|
||||
actual := getBit(b[:], test.bit)
|
||||
if test.expected != actual {
|
||||
t.Errorf("getting bit %d of %s: expected %t, got %t", test.bit, b.HexSimplified(), test.expected, actual)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmap_SetBit(t *testing.T) {
|
||||
tt := []struct {
|
||||
hex string
|
||||
bit int
|
||||
one bool
|
||||
expected string
|
||||
panic bool
|
||||
}{
|
||||
{hex: "0", bit: 383, one: true, expected: "1", panic: false},
|
||||
{hex: "0", bit: 382, one: true, expected: "2", panic: false},
|
||||
{hex: "0", bit: 381, one: true, expected: "4", panic: false},
|
||||
{hex: "0", bit: 385, one: true, expected: "1", panic: true},
|
||||
{hex: "0", bit: 384, one: true, expected: "1", panic: true},
|
||||
}
|
||||
|
||||
for _, test := range tt {
|
||||
expected := FromShortHexP(test.expected)
|
||||
actual := FromShortHexP(test.hex)
|
||||
if test.panic {
|
||||
assertPanic(t, fmt.Sprintf("setting bit %d to %t", test.bit, test.one), func() { setBit(actual[:], test.bit, test.one) })
|
||||
} else {
|
||||
setBit(actual[:], test.bit, test.one)
|
||||
if !expected.Equals(actual) {
|
||||
t.Errorf("setting bit %d to %t: expected %s, got %s", test.bit, test.one, test.expected, actual.HexSimplified())
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmap_FromHexShort(t *testing.T) {
|
||||
tt := []struct {
|
||||
short string
|
||||
long string
|
||||
}{
|
||||
{short: "", long: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{short: "0", long: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{short: "00000", long: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{short: "9473745bc", long: "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000009473745bc"},
|
||||
{short: "09473745bc", long: "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000009473745bc"},
|
||||
{short: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
||||
long: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"},
|
||||
}
|
||||
|
||||
for _, test := range tt {
|
||||
short := FromShortHexP(test.short)
|
||||
long := FromHexP(test.long)
|
||||
if !short.Equals(long) {
|
||||
t.Errorf("short hex %s: expected %s, got %s", test.short, long.Hex(), short.Hex())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmapMarshal(t *testing.T) {
|
||||
b := FromStringP("123456789012345678901234567890123456789012345678")
|
||||
encoded, err := bencode.EncodeBytes(b)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
if string(encoded) != "48:123456789012345678901234567890123456789012345678" {
|
||||
t.Error("encoding does not match expected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmapMarshalEmbedded(t *testing.T) {
|
||||
e := struct {
|
||||
A string
|
||||
B Bitmap
|
||||
C int
|
||||
}{
|
||||
A: "1",
|
||||
B: FromStringP("222222222222222222222222222222222222222222222222"),
|
||||
C: 3,
|
||||
}
|
||||
|
||||
encoded, err := bencode.EncodeBytes(e)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
if string(encoded) != "d1:A1:11:B48:2222222222222222222222222222222222222222222222221:Ci3ee" {
|
||||
t.Error("encoding does not match expected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmapMarshalEmbedded2(t *testing.T) {
|
||||
encoded, err := bencode.EncodeBytes([]interface{}{
|
||||
FromStringP("333333333333333333333333333333333333333333333333"),
|
||||
})
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
if string(encoded) != "l48:333333333333333333333333333333333333333333333333e" {
|
||||
t.Error("encoding does not match expected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmap_PrefixLen(t *testing.T) {
|
||||
tt := []struct {
|
||||
hex string
|
||||
len int
|
||||
}{
|
||||
{len: 0, hex: "F00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{len: 0, hex: "800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{len: 1, hex: "700000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{len: 1, hex: "400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{len: 384, hex: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{len: 383, hex: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"},
|
||||
{len: 382, hex: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002"},
|
||||
{len: 382, hex: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003"},
|
||||
}
|
||||
|
||||
for _, test := range tt {
|
||||
len := FromHexP(test.hex).PrefixLen()
|
||||
if len != test.len {
|
||||
t.Errorf("got prefix len %d; expected %d for %s", len, test.len, test.hex)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmap_Prefix(t *testing.T) {
|
||||
allOne := FromHexP("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")
|
||||
|
||||
zerosTT := []struct {
|
||||
zeros int
|
||||
expected string
|
||||
}{
|
||||
{zeros: -123, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
|
||||
{zeros: 0, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
|
||||
{zeros: 1, expected: "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
|
||||
{zeros: 69, expected: "000000000000000007ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
|
||||
{zeros: 383, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"},
|
||||
{zeros: 384, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{zeros: 400, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
}
|
||||
|
||||
for _, test := range zerosTT {
|
||||
expected := FromHexP(test.expected)
|
||||
actual := allOne.Prefix(test.zeros, false)
|
||||
if !actual.Equals(expected) {
|
||||
t.Errorf("%d zeros: got %s; expected %s", test.zeros, actual.Hex(), expected.Hex())
|
||||
}
|
||||
}
|
||||
|
||||
for i := 0; i < NumBits; i++ {
|
||||
b := allOne.Prefix(i, false)
|
||||
if b.PrefixLen() != i {
|
||||
t.Errorf("got prefix len %d; expected %d for %s", b.PrefixLen(), i, b.Hex())
|
||||
}
|
||||
}
|
||||
|
||||
allZero := FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
|
||||
|
||||
onesTT := []struct {
|
||||
ones int
|
||||
expected string
|
||||
}{
|
||||
{ones: -123, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{ones: 0, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{ones: 1, expected: "800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{ones: 69, expected: "fffffffffffffffff8000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{ones: 383, expected: "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe"},
|
||||
{ones: 384, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
|
||||
{ones: 400, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
|
||||
}
|
||||
|
||||
for _, test := range onesTT {
|
||||
expected := FromHexP(test.expected)
|
||||
actual := allZero.Prefix(test.ones, true)
|
||||
if !actual.Equals(expected) {
|
||||
t.Errorf("%d ones: got %s; expected %s", test.ones, actual.Hex(), expected.Hex())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmap_Suffix(t *testing.T) {
|
||||
allOne := FromHexP("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")
|
||||
|
||||
zerosTT := []struct {
|
||||
zeros int
|
||||
expected string
|
||||
}{
|
||||
{zeros: -123, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
|
||||
{zeros: 0, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
|
||||
{zeros: 1, expected: "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe"},
|
||||
{zeros: 69, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe00000000000000000"},
|
||||
{zeros: 383, expected: "800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{zeros: 384, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{zeros: 400, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
}
|
||||
|
||||
for _, test := range zerosTT {
|
||||
expected := FromHexP(test.expected)
|
||||
actual := allOne.Suffix(test.zeros, false)
|
||||
if !actual.Equals(expected) {
|
||||
t.Errorf("%d zeros: got %s; expected %s", test.zeros, actual.Hex(), expected.Hex())
|
||||
}
|
||||
}
|
||||
|
||||
for i := 0; i < NumBits; i++ {
|
||||
b := allOne.Prefix(i, false)
|
||||
if b.PrefixLen() != i {
|
||||
t.Errorf("got prefix len %d; expected %d for %s", b.PrefixLen(), i, b.Hex())
|
||||
}
|
||||
}
|
||||
|
||||
allZero := FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
|
||||
|
||||
onesTT := []struct {
|
||||
ones int
|
||||
expected string
|
||||
}{
|
||||
{ones: -123, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{ones: 0, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{ones: 1, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"},
|
||||
{ones: 69, expected: "0000000000000000000000000000000000000000000000000000000000000000000000000000001fffffffffffffffff"},
|
||||
{ones: 383, expected: "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
|
||||
{ones: 384, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
|
||||
{ones: 400, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
|
||||
}
|
||||
|
||||
for _, test := range onesTT {
|
||||
expected := FromHexP(test.expected)
|
||||
actual := allZero.Suffix(test.ones, true)
|
||||
if !actual.Equals(expected) {
|
||||
t.Errorf("%d ones: got %s; expected %s", test.ones, actual.Hex(), expected.Hex())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmap_Add(t *testing.T) {
|
||||
tt := []struct {
|
||||
a, b, sum string
|
||||
panic bool
|
||||
}{
|
||||
{"0", "0", "0", false},
|
||||
{"0", "1", "1", false},
|
||||
{"1", "0", "1", false},
|
||||
{"1", "1", "2", false},
|
||||
{"8", "4", "c", false},
|
||||
{"1000", "0010", "1010", false},
|
||||
{"1111", "1111", "2222", false},
|
||||
{"ffff", "1", "10000", false},
|
||||
{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", false},
|
||||
{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "1", "", true},
|
||||
}
|
||||
|
||||
for _, test := range tt {
|
||||
a := FromShortHexP(test.a)
|
||||
b := FromShortHexP(test.b)
|
||||
expected := FromShortHexP(test.sum)
|
||||
if test.panic {
|
||||
assertPanic(t, fmt.Sprintf("adding %s and %s", test.a, test.b), func() { a.Add(b) })
|
||||
} else {
|
||||
actual := a.Add(b)
|
||||
if !expected.Equals(actual) {
|
||||
t.Errorf("adding %s and %s; expected %s, got %s", test.a, test.b, test.sum, actual.HexSimplified())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmap_Sub(t *testing.T) {
|
||||
tt := []struct {
|
||||
a, b, sum string
|
||||
panic bool
|
||||
}{
|
||||
{"0", "0", "0", false},
|
||||
{"1", "0", "1", false},
|
||||
{"1", "1", "0", false},
|
||||
{"8", "4", "4", false},
|
||||
{"f", "9", "6", false},
|
||||
{"f", "e", "1", false},
|
||||
{"10", "f", "1", false},
|
||||
{"2222", "1111", "1111", false},
|
||||
{"ffff", "1", "fffe", false},
|
||||
{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", false},
|
||||
{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0", false},
|
||||
{"0", "1", "", true},
|
||||
}
|
||||
|
||||
for _, test := range tt {
|
||||
a := FromShortHexP(test.a)
|
||||
b := FromShortHexP(test.b)
|
||||
expected := FromShortHexP(test.sum)
|
||||
if test.panic {
|
||||
assertPanic(t, fmt.Sprintf("subtracting %s - %s", test.a, test.b), func() { a.Sub(b) })
|
||||
} else {
|
||||
actual := a.Sub(b)
|
||||
if !expected.Equals(actual) {
|
||||
t.Errorf("subtracting %s - %s; expected %s, got %s", test.a, test.b, test.sum, actual.HexSimplified())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertPanic(t *testing.T, text string, f func()) {
|
||||
defer func() {
|
||||
if r := recover(); r == nil {
|
||||
t.Errorf("%s: did not panic as expected", text)
|
||||
}
|
||||
}()
|
||||
f()
|
||||
}
|
||||
@@ -1,65 +0,0 @@
|
||||
package bits
|
||||
|
||||
import (
|
||||
"math/big"
|
||||
|
||||
"github.com/lbryio/errors.go"
|
||||
)
|
||||
|
||||
// Range has a start and end
|
||||
type Range struct {
|
||||
Start Bitmap
|
||||
End Bitmap
|
||||
}
|
||||
|
||||
func MaxRange() Range {
|
||||
return Range{
|
||||
Start: Bitmap{},
|
||||
End: MaxP(),
|
||||
}
|
||||
}
|
||||
|
||||
// IntervalP divides the range into `num` intervals and returns the `n`th one
|
||||
// intervals are approximately the same size, but may not be exact because of rounding issues
|
||||
// the first interval always starts at the beginning of the range, and the last interval always ends at the end
|
||||
func (r Range) IntervalP(n, num int) Range {
|
||||
if num < 1 || n < 1 || n > num {
|
||||
panic(errors.Err("invalid interval %d of %d", n, num))
|
||||
}
|
||||
|
||||
start := r.intervalStart(n, num)
|
||||
end := r.End.Big()
|
||||
if n < num {
|
||||
end = r.intervalStart(n+1, num)
|
||||
end.Sub(end, big.NewInt(1))
|
||||
}
|
||||
|
||||
return Range{FromBigP(start), FromBigP(end)}
|
||||
}
|
||||
|
||||
func (r Range) intervalStart(n, num int) *big.Int {
|
||||
// formula:
|
||||
// size = (end - start) / num
|
||||
// rem = (end - start) % num
|
||||
// intervalStart = rangeStart + (size * n-1) + ((rem * n-1) % num)
|
||||
|
||||
size := new(big.Int)
|
||||
rem := new(big.Int)
|
||||
size.Sub(r.End.Big(), r.Start.Big()).DivMod(size, big.NewInt(int64(num)), rem)
|
||||
|
||||
size.Mul(size, big.NewInt(int64(n-1)))
|
||||
rem.Mul(rem, big.NewInt(int64(n-1))).Mod(rem, big.NewInt(int64(num)))
|
||||
|
||||
start := r.Start.Big()
|
||||
start.Add(start, size).Add(start, rem)
|
||||
|
||||
return start
|
||||
}
|
||||
|
||||
func (r Range) IntervalSize() *big.Int {
|
||||
return (&big.Int{}).Sub(r.End.Big(), r.Start.Big())
|
||||
}
|
||||
|
||||
func (r Range) Contains(b Bitmap) bool {
|
||||
return r.Start.Cmp(b) <= 0 && r.End.Cmp(b) >= 0
|
||||
}
|
||||
@@ -1,48 +0,0 @@
|
||||
package bits
|
||||
|
||||
import (
|
||||
"math/big"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMaxRange(t *testing.T) {
|
||||
start := FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
|
||||
end := FromHexP("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")
|
||||
r := MaxRange()
|
||||
|
||||
if !r.Start.Equals(start) {
|
||||
t.Error("max range does not start at the beginning")
|
||||
}
|
||||
if !r.End.Equals(end) {
|
||||
t.Error("max range does not end at the end")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRange_IntervalP(t *testing.T) {
|
||||
max := MaxRange()
|
||||
|
||||
numIntervals := 97
|
||||
expectedAvg := (&big.Int{}).Div(max.IntervalSize(), big.NewInt(int64(numIntervals)))
|
||||
maxDiff := big.NewInt(int64(numIntervals))
|
||||
|
||||
var lastEnd Bitmap
|
||||
|
||||
for i := 1; i <= numIntervals; i++ {
|
||||
ival := max.IntervalP(i, numIntervals)
|
||||
if i == 1 && !ival.Start.Equals(max.Start) {
|
||||
t.Error("first interval does not start at 0")
|
||||
}
|
||||
if i == numIntervals && !ival.End.Equals(max.End) {
|
||||
t.Error("last interval does not end at max")
|
||||
}
|
||||
if i > 1 && !ival.Start.Equals(lastEnd.Add(FromShortHexP("1"))) {
|
||||
t.Errorf("interval %d of %d: last end was %s, this start is %s", i, numIntervals, lastEnd.Hex(), ival.Start.Hex())
|
||||
}
|
||||
|
||||
if ival.IntervalSize().Cmp((&big.Int{}).Add(expectedAvg, maxDiff)) > 0 || ival.IntervalSize().Cmp((&big.Int{}).Sub(expectedAvg, maxDiff)) < 0 {
|
||||
t.Errorf("interval %d of %d: interval size is outside the normal range", i, numIntervals)
|
||||
}
|
||||
|
||||
lastEnd = ival.End
|
||||
}
|
||||
}
|
||||
@@ -1,212 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"math/rand"
|
||||
"net"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
)
|
||||
|
||||
const (
|
||||
bootstrapDefaultRefreshDuration = 15 * time.Minute
|
||||
)
|
||||
|
||||
// BootstrapNode is a configured node setup for testing.
|
||||
type BootstrapNode struct {
|
||||
Node
|
||||
|
||||
initialPingInterval time.Duration
|
||||
checkInterval time.Duration
|
||||
|
||||
nlock *sync.RWMutex
|
||||
peers map[bits.Bitmap]*peer
|
||||
nodeIDs []bits.Bitmap // necessary for efficient random ID selection
|
||||
}
|
||||
|
||||
// NewBootstrapNode returns a BootstrapNode pointer.
|
||||
func NewBootstrapNode(id bits.Bitmap, initialPingInterval, rePingInterval time.Duration) *BootstrapNode {
|
||||
b := &BootstrapNode{
|
||||
Node: *NewNode(id),
|
||||
|
||||
initialPingInterval: initialPingInterval,
|
||||
checkInterval: rePingInterval,
|
||||
|
||||
nlock: &sync.RWMutex{},
|
||||
peers: make(map[bits.Bitmap]*peer),
|
||||
nodeIDs: make([]bits.Bitmap, 0),
|
||||
}
|
||||
|
||||
b.requestHandler = b.handleRequest
|
||||
|
||||
return b
|
||||
}
|
||||
|
||||
// Add manually adds a contact
|
||||
func (b *BootstrapNode) Add(c Contact) {
|
||||
b.upsert(c)
|
||||
}
|
||||
|
||||
// Connect connects to the given connection and starts any background threads necessary
|
||||
func (b *BootstrapNode) Connect(conn UDPConn) error {
|
||||
err := b.Node.Connect(conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Infof("[%s] bootstrap: node connected", b.id.HexShort())
|
||||
|
||||
go func() {
|
||||
t := time.NewTicker(b.checkInterval / 5)
|
||||
for {
|
||||
select {
|
||||
case <-t.C:
|
||||
b.check()
|
||||
case <-b.grp.Ch():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// upsert adds the contact to the list, or updates the lastPinged time
|
||||
func (b *BootstrapNode) upsert(c Contact) {
|
||||
b.nlock.Lock()
|
||||
defer b.nlock.Unlock()
|
||||
|
||||
if peer, exists := b.peers[c.ID]; exists {
|
||||
log.Debugf("[%s] bootstrap: touching contact %s", b.id.HexShort(), peer.Contact.ID.HexShort())
|
||||
peer.Touch()
|
||||
return
|
||||
}
|
||||
|
||||
log.Debugf("[%s] bootstrap: adding new contact %s", b.id.HexShort(), c.ID.HexShort())
|
||||
b.peers[c.ID] = &peer{c, b.id.Xor(c.ID), time.Now(), 0}
|
||||
b.nodeIDs = append(b.nodeIDs, c.ID)
|
||||
}
|
||||
|
||||
// remove removes the contact from the list
|
||||
func (b *BootstrapNode) remove(c Contact) {
|
||||
b.nlock.Lock()
|
||||
defer b.nlock.Unlock()
|
||||
|
||||
_, exists := b.peers[c.ID]
|
||||
if !exists {
|
||||
return
|
||||
}
|
||||
|
||||
log.Debugf("[%s] bootstrap: removing contact %s", b.id.HexShort(), c.ID.HexShort())
|
||||
delete(b.peers, c.ID)
|
||||
for i := range b.nodeIDs {
|
||||
if b.nodeIDs[i].Equals(c.ID) {
|
||||
b.nodeIDs = append(b.nodeIDs[:i], b.nodeIDs[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// get returns up to `limit` random contacts from the list
|
||||
func (b *BootstrapNode) get(limit int) []Contact {
|
||||
b.nlock.RLock()
|
||||
defer b.nlock.RUnlock()
|
||||
|
||||
if len(b.peers) < limit {
|
||||
limit = len(b.peers)
|
||||
}
|
||||
|
||||
ret := make([]Contact, limit)
|
||||
for i, k := range randKeys(len(b.nodeIDs))[:limit] {
|
||||
ret[i] = b.peers[b.nodeIDs[k]].Contact
|
||||
}
|
||||
|
||||
return ret
|
||||
}
|
||||
|
||||
// ping pings a node. if the node responds, it is added to the list. otherwise, it is removed
|
||||
func (b *BootstrapNode) ping(c Contact) {
|
||||
log.Debugf("[%s] bootstrap: pinging %s", b.id.HexShort(), c.ID.HexShort())
|
||||
b.grp.Add(1)
|
||||
defer b.grp.Done()
|
||||
|
||||
resCh := b.SendAsync(c, Request{Method: pingMethod})
|
||||
|
||||
var res *Response
|
||||
|
||||
select {
|
||||
case res = <-resCh:
|
||||
case <-b.grp.Ch():
|
||||
return
|
||||
}
|
||||
|
||||
if res != nil && res.Data == pingSuccessResponse {
|
||||
b.upsert(c)
|
||||
} else {
|
||||
b.remove(c)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *BootstrapNode) check() {
|
||||
b.nlock.RLock()
|
||||
defer b.nlock.RUnlock()
|
||||
|
||||
for i := range b.peers {
|
||||
if !b.peers[i].ActiveInLast(b.checkInterval) {
|
||||
go b.ping(b.peers[i].Contact)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// handleRequest handles the requests received from udp.
|
||||
func (b *BootstrapNode) handleRequest(addr *net.UDPAddr, request Request) {
|
||||
switch request.Method {
|
||||
case pingMethod:
|
||||
err := b.sendMessage(addr, Response{ID: request.ID, NodeID: b.id, Data: pingSuccessResponse})
|
||||
if err != nil {
|
||||
log.Error("error sending response message - ", err)
|
||||
}
|
||||
case findNodeMethod:
|
||||
if request.Arg == nil {
|
||||
log.Errorln("request is missing arg")
|
||||
return
|
||||
}
|
||||
|
||||
err := b.sendMessage(addr, Response{
|
||||
ID: request.ID,
|
||||
NodeID: b.id,
|
||||
Contacts: b.get(bucketSize),
|
||||
})
|
||||
if err != nil {
|
||||
log.Error("error sending 'findnodemethod' response message - ", err)
|
||||
}
|
||||
}
|
||||
|
||||
go func() {
|
||||
b.nlock.RLock()
|
||||
_, exists := b.peers[request.NodeID]
|
||||
b.nlock.RUnlock()
|
||||
if !exists {
|
||||
log.Debugf("[%s] bootstrap: queuing %s to ping", b.id.HexShort(), request.NodeID.HexShort())
|
||||
<-time.After(b.initialPingInterval)
|
||||
b.nlock.RLock()
|
||||
_, exists = b.peers[request.NodeID]
|
||||
b.nlock.RUnlock()
|
||||
if !exists {
|
||||
b.ping(Contact{ID: request.NodeID, IP: addr.IP, Port: addr.Port})
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func randKeys(max int) []int {
|
||||
keys := make([]int, max)
|
||||
for k := range keys {
|
||||
keys[k] = k
|
||||
}
|
||||
rand.Shuffle(max, func(i, j int) {
|
||||
keys[i], keys[j] = keys[j], keys[i]
|
||||
})
|
||||
return keys
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"net"
|
||||
"testing"
|
||||
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
)
|
||||
|
||||
func TestBootstrapPing(t *testing.T) {
|
||||
b := NewBootstrapNode(bits.Rand(), 10, bootstrapDefaultRefreshDuration)
|
||||
|
||||
listener, err := net.ListenPacket(Network, "127.0.0.1:54320")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
err = b.Connect(listener.(*net.UDPConn))
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
b.Shutdown()
|
||||
}
|
||||
@@ -1,76 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
peerproto "github.com/lbryio/reflector.go/peer"
|
||||
)
|
||||
|
||||
const (
|
||||
Network = "udp4"
|
||||
DefaultPort = 4444
|
||||
|
||||
DefaultAnnounceRate = 10 // send at most this many announces per second
|
||||
DefaultReannounceTime = 50 * time.Minute // should be a bit less than hash expiration time
|
||||
|
||||
// TODO: all these constants should be defaults, and should be used to set values in the standard Config. then the code should use values in the config
|
||||
// TODO: alternatively, have a global Config for constants. at least that way tests can modify the values
|
||||
alpha = 3 // this is the constant alpha in the spec
|
||||
bucketSize = 8 // this is the constant k in the spec
|
||||
nodeIDLength = bits.NumBytes // bytes. this is the constant B in the spec
|
||||
messageIDLength = 20 // bytes.
|
||||
|
||||
udpRetry = 1
|
||||
udpTimeout = 5 * time.Second
|
||||
udpMaxMessageLength = 4096 // bytes. I think our longest message is ~676 bytes, so I rounded up to 1024
|
||||
// scratch that. a findValue could return more than K results if a lot of nodes are storing that value, so we need more buffer
|
||||
|
||||
maxPeerFails = 3 // after this many failures, a peer is considered bad and will be removed from the routing table
|
||||
//tExpire = 60 * time.Minute // the time after which a key/value pair expires; this is a time-to-live (TTL) from the original publication date
|
||||
tRefresh = 1 * time.Hour // the time after which an otherwise unaccessed bucket must be refreshed
|
||||
//tReplicate = 1 * time.Hour // the interval between Kademlia replication events, when a node is required to publish its entire database
|
||||
//tNodeRefresh = 15 * time.Minute // the time after which a good node becomes questionable if it has not messaged us
|
||||
|
||||
compactNodeInfoLength = nodeIDLength + 6 // nodeID + 4 for IP + 2 for port
|
||||
|
||||
tokenSecretRotationInterval = 5 * time.Minute // how often the token-generating secret is rotated
|
||||
)
|
||||
|
||||
// Config represents the configure of dht.
|
||||
type Config struct {
|
||||
// this node's address. format is `ip:port`
|
||||
Address string
|
||||
// the seed nodes through which we can join in dht network
|
||||
SeedNodes []string
|
||||
// the hex-encoded node id for this node. if string is empty, a random id will be generated
|
||||
NodeID string
|
||||
// print the state of the dht every X time
|
||||
PrintState time.Duration
|
||||
// the port that clients can use to download blobs using the LBRY peer protocol
|
||||
PeerProtocolPort int
|
||||
// if nonzero, an RPC server will listen to requests on this port and respond to them
|
||||
RPCPort int
|
||||
// the time after which the original publisher must reannounce a key/value pair
|
||||
ReannounceTime time.Duration
|
||||
// send at most this many announces per second
|
||||
AnnounceRate int
|
||||
// channel that will receive notifications about announcements
|
||||
AnnounceNotificationCh chan announceNotification
|
||||
}
|
||||
|
||||
// NewStandardConfig returns a Config pointer with default values.
|
||||
func NewStandardConfig() *Config {
|
||||
return &Config{
|
||||
Address: "0.0.0.0:" + strconv.Itoa(DefaultPort),
|
||||
SeedNodes: []string{
|
||||
"lbrynet1.lbry.io:4444",
|
||||
"lbrynet2.lbry.io:4444",
|
||||
"lbrynet3.lbry.io:4444",
|
||||
},
|
||||
PeerProtocolPort: peerproto.DefaultPort,
|
||||
ReannounceTime: DefaultReannounceTime,
|
||||
AnnounceRate: DefaultAnnounceRate,
|
||||
}
|
||||
}
|
||||
-118
@@ -1,118 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net"
|
||||
"sort"
|
||||
"strconv"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
|
||||
"github.com/lyoshenka/bencode"
|
||||
)
|
||||
|
||||
// TODO: if routing table is ever empty (aka the node is isolated), it should re-bootstrap
|
||||
|
||||
// Contact contains information for contacting another node on the network
|
||||
type Contact struct {
|
||||
ID bits.Bitmap
|
||||
IP net.IP
|
||||
Port int // the udp port used for the dht
|
||||
PeerPort int // the tcp port a peer can be contacted on for blob requests
|
||||
}
|
||||
|
||||
// Equals returns true if two contacts are the same.
|
||||
func (c Contact) Equals(other Contact, checkID bool) bool {
|
||||
return c.IP.Equal(other.IP) && c.Port == other.Port && (!checkID || c.ID == other.ID)
|
||||
}
|
||||
|
||||
// Addr returns the address of the contact.
|
||||
func (c Contact) Addr() *net.UDPAddr {
|
||||
return &net.UDPAddr{IP: c.IP, Port: c.Port}
|
||||
}
|
||||
|
||||
// String returns a short string representation of the contact
|
||||
func (c Contact) String() string {
|
||||
str := c.ID.HexShort() + "@" + c.Addr().String()
|
||||
if c.PeerPort != 0 {
|
||||
str += "(" + strconv.Itoa(c.PeerPort) + ")"
|
||||
}
|
||||
return str
|
||||
}
|
||||
|
||||
// MarshalCompact returns a compact byteslice representation of the contact
|
||||
// NOTE: The compact representation always uses the tcp PeerPort, not the udp Port. This is dumb, but that's how the python daemon does it
|
||||
func (c Contact) MarshalCompact() ([]byte, error) {
|
||||
if c.IP.To4() == nil {
|
||||
return nil, errors.Err("ip not set")
|
||||
}
|
||||
if c.PeerPort < 0 || c.PeerPort > 65535 {
|
||||
return nil, errors.Err("invalid port")
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
buf.Write(c.IP.To4())
|
||||
buf.WriteByte(byte(c.PeerPort >> 8))
|
||||
buf.WriteByte(byte(c.PeerPort))
|
||||
buf.Write(c.ID[:])
|
||||
|
||||
if buf.Len() != compactNodeInfoLength {
|
||||
return nil, errors.Err("i dont know how this happened")
|
||||
}
|
||||
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
// UnmarshalCompact unmarshals the compact byteslice representation of a contact.
|
||||
// NOTE: The compact representation always uses the tcp PeerPort, not the udp Port. This is dumb, but that's how the python daemon does it
|
||||
func (c *Contact) UnmarshalCompact(b []byte) error {
|
||||
if len(b) != compactNodeInfoLength {
|
||||
return errors.Err("invalid compact length")
|
||||
}
|
||||
c.IP = net.IPv4(b[0], b[1], b[2], b[3]).To4()
|
||||
c.PeerPort = int(uint16(b[5]) | uint16(b[4])<<8)
|
||||
c.ID = bits.FromBytesP(b[6:])
|
||||
return nil
|
||||
}
|
||||
|
||||
// MarshalBencode returns the serialized byte slice representation of a contact.
|
||||
func (c Contact) MarshalBencode() ([]byte, error) {
|
||||
return bencode.EncodeBytes([]interface{}{c.ID, c.IP.String(), c.Port})
|
||||
}
|
||||
|
||||
// UnmarshalBencode unmarshals the serialized byte slice into the appropriate fields of the contact.
|
||||
func (c *Contact) UnmarshalBencode(b []byte) error {
|
||||
var raw []bencode.RawMessage
|
||||
err := bencode.DecodeBytes(b, &raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(raw) != 3 {
|
||||
return errors.Err("contact must have 3 elements; got %d", len(raw))
|
||||
}
|
||||
|
||||
err = bencode.DecodeBytes(raw[0], &c.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var ipStr string
|
||||
err = bencode.DecodeBytes(raw[1], &ipStr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.IP = net.ParseIP(ipStr).To4()
|
||||
if c.IP == nil {
|
||||
return errors.Err("invalid IP")
|
||||
}
|
||||
|
||||
return bencode.DecodeBytes(raw[2], &c.Port)
|
||||
}
|
||||
|
||||
func sortByDistance(contacts []Contact, target bits.Bitmap) {
|
||||
sort.Slice(contacts, func(i, j int) bool {
|
||||
return contacts[i].ID.Xor(target).Cmp(contacts[j].ID.Xor(target)) < 0
|
||||
})
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"net"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
)
|
||||
|
||||
func TestCompactEncoding(t *testing.T) {
|
||||
c := Contact{
|
||||
ID: bits.FromHexP("1c8aff71b99462464d9eeac639595ab99664be3482cb91a29d87467515c7d9158fe72aa1f1582dab07d8f8b5db277f41"),
|
||||
IP: net.ParseIP("1.2.3.4"),
|
||||
PeerPort: int(55<<8 + 66),
|
||||
}
|
||||
|
||||
var compact []byte
|
||||
compact, err := c.MarshalCompact()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(compact) != compactNodeInfoLength {
|
||||
t.Fatalf("got length of %d; expected %d", len(compact), compactNodeInfoLength)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(compact, append([]byte{1, 2, 3, 4, 55, 66}, c.ID[:]...)) {
|
||||
t.Errorf("compact bytes not encoded correctly")
|
||||
}
|
||||
}
|
||||
-232
@@ -1,232 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/lbry.go/stop"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cast"
|
||||
)
|
||||
|
||||
var log *logrus.Logger
|
||||
|
||||
func UseLogger(l *logrus.Logger) {
|
||||
log = l
|
||||
}
|
||||
|
||||
func init() {
|
||||
log = logrus.StandardLogger()
|
||||
//log.SetFormatter(&log.TextFormatter{ForceColors: true})
|
||||
//log.SetLevel(log.DebugLevel)
|
||||
}
|
||||
|
||||
// DHT represents a DHT node.
|
||||
type DHT struct {
|
||||
// config
|
||||
conf *Config
|
||||
// local contact
|
||||
contact Contact
|
||||
// node
|
||||
node *Node
|
||||
// stopGroup to shut down DHT
|
||||
grp *stop.Group
|
||||
// channel is closed when DHT joins network
|
||||
joined chan struct{}
|
||||
// cache for store tokens
|
||||
tokenCache *tokenCache
|
||||
// hashes that need to be put into the announce queue or removed from the queue
|
||||
announceAddRemove chan queueEdit
|
||||
}
|
||||
|
||||
// New returns a DHT pointer. If config is nil, then config will be set to the default config.
|
||||
func New(config *Config) *DHT {
|
||||
if config == nil {
|
||||
config = NewStandardConfig()
|
||||
}
|
||||
|
||||
d := &DHT{
|
||||
conf: config,
|
||||
grp: stop.New(),
|
||||
joined: make(chan struct{}),
|
||||
announceAddRemove: make(chan queueEdit),
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func (dht *DHT) connect(conn UDPConn) error {
|
||||
contact, err := getContact(dht.conf.NodeID, dht.conf.Address)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dht.contact = contact
|
||||
dht.node = NewNode(contact.ID)
|
||||
dht.tokenCache = newTokenCache(dht.node, tokenSecretRotationInterval)
|
||||
|
||||
return dht.node.Connect(conn)
|
||||
}
|
||||
|
||||
// Start starts the dht
|
||||
func (dht *DHT) Start() error {
|
||||
listener, err := net.ListenPacket(Network, dht.conf.Address)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
conn := listener.(*net.UDPConn)
|
||||
|
||||
err = dht.connect(conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dht.join()
|
||||
log.Infof("[%s] DHT ready on %s (%d nodes found during join)",
|
||||
dht.node.id.HexShort(), dht.contact.Addr().String(), dht.node.rt.Count())
|
||||
|
||||
dht.grp.Add(1)
|
||||
go func() {
|
||||
dht.runAnnouncer()
|
||||
dht.grp.Done()
|
||||
}()
|
||||
|
||||
if dht.conf.RPCPort > 0 {
|
||||
dht.grp.Add(1)
|
||||
go func() {
|
||||
dht.runRPCServer(dht.conf.RPCPort)
|
||||
dht.grp.Done()
|
||||
}()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// join makes current node join the dht network.
|
||||
func (dht *DHT) join() {
|
||||
defer close(dht.joined) // if anyone's waiting for join to finish, they'll know its done
|
||||
|
||||
log.Infof("[%s] joining DHT network", dht.node.id.HexShort())
|
||||
|
||||
// ping nodes, which gets their real node IDs and adds them to the routing table
|
||||
atLeastOneNodeResponded := false
|
||||
for _, addr := range dht.conf.SeedNodes {
|
||||
err := dht.Ping(addr)
|
||||
if err != nil {
|
||||
log.Error(errors.Prefix(fmt.Sprintf("[%s] join", dht.node.id.HexShort()), err))
|
||||
} else {
|
||||
atLeastOneNodeResponded = true
|
||||
}
|
||||
}
|
||||
|
||||
if !atLeastOneNodeResponded {
|
||||
log.Errorf("[%s] join: no nodes responded to initial ping", dht.node.id.HexShort())
|
||||
return
|
||||
}
|
||||
|
||||
// now call iterativeFind on yourself
|
||||
_, _, err := FindContacts(dht.node, dht.node.id, false, dht.grp.Child())
|
||||
if err != nil {
|
||||
log.Errorf("[%s] join: %s", dht.node.id.HexShort(), err.Error())
|
||||
}
|
||||
|
||||
// TODO: after joining, refresh all buckets further away than our closest neighbor
|
||||
// http://xlattice.sourceforge.net/components/protocol/kademlia/specs.html#join
|
||||
}
|
||||
|
||||
// WaitUntilJoined blocks until the node joins the network.
|
||||
func (dht *DHT) WaitUntilJoined() {
|
||||
if dht.joined == nil {
|
||||
panic("dht not initialized")
|
||||
}
|
||||
<-dht.joined
|
||||
}
|
||||
|
||||
// Shutdown shuts down the dht
|
||||
func (dht *DHT) Shutdown() {
|
||||
log.Debugf("[%s] DHT shutting down", dht.node.id.HexShort())
|
||||
dht.grp.StopAndWait()
|
||||
dht.node.Shutdown()
|
||||
log.Debugf("[%s] DHT stopped", dht.node.id.HexShort())
|
||||
}
|
||||
|
||||
// Ping pings a given address, creates a temporary contact for sending a message, and returns an error if communication
|
||||
// fails.
|
||||
func (dht *DHT) Ping(addr string) error {
|
||||
raddr, err := net.ResolveUDPAddr(Network, addr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tmpNode := Contact{ID: bits.Rand(), IP: raddr.IP, Port: raddr.Port}
|
||||
res := dht.node.Send(tmpNode, Request{Method: pingMethod}, SendOptions{skipIDCheck: true})
|
||||
if res == nil {
|
||||
return errors.Err("no response from node %s", addr)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get returns the list of nodes that have the blob for the given hash
|
||||
func (dht *DHT) Get(hash bits.Bitmap) ([]Contact, error) {
|
||||
contacts, found, err := FindContacts(dht.node, hash, true, dht.grp.Child())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if found {
|
||||
return contacts, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// PrintState prints the current state of the DHT including address, nr outstanding transactions, stored hashes as well
|
||||
// as current bucket information.
|
||||
func (dht *DHT) PrintState() {
|
||||
log.Printf("DHT node %s at %s", dht.contact.String(), time.Now().Format(time.RFC822Z))
|
||||
log.Printf("Outstanding transactions: %d", dht.node.CountActiveTransactions())
|
||||
log.Printf("Stored hashes: %d", dht.node.store.CountStoredHashes())
|
||||
log.Printf("Buckets:")
|
||||
for _, line := range strings.Split(dht.node.rt.BucketInfo(), "\n") {
|
||||
log.Println(line)
|
||||
}
|
||||
}
|
||||
|
||||
func (dht DHT) ID() bits.Bitmap {
|
||||
return dht.contact.ID
|
||||
}
|
||||
|
||||
func getContact(nodeID, addr string) (Contact, error) {
|
||||
var c Contact
|
||||
if nodeID == "" {
|
||||
c.ID = bits.Rand()
|
||||
} else {
|
||||
c.ID = bits.FromHexP(nodeID)
|
||||
}
|
||||
|
||||
ip, port, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return c, errors.Err(err)
|
||||
} else if ip == "" {
|
||||
return c, errors.Err("address does not contain an IP")
|
||||
} else if port == "" {
|
||||
return c, errors.Err("address does not contain a port")
|
||||
}
|
||||
|
||||
c.IP = net.ParseIP(ip)
|
||||
if c.IP == nil {
|
||||
return c, errors.Err("invalid ip")
|
||||
}
|
||||
|
||||
c.Port, err = cast.ToIntE(port)
|
||||
if err != nil {
|
||||
return c, errors.Err(err)
|
||||
}
|
||||
|
||||
return c, nil
|
||||
}
|
||||
@@ -1,214 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"container/ring"
|
||||
"context"
|
||||
"math"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
type queueEdit struct {
|
||||
hash bits.Bitmap
|
||||
add bool
|
||||
}
|
||||
|
||||
const (
|
||||
announceStarted = "started"
|
||||
announceFinishd = "finished"
|
||||
)
|
||||
|
||||
type announceNotification struct {
|
||||
hash bits.Bitmap
|
||||
action string
|
||||
err error
|
||||
}
|
||||
|
||||
// Add adds the hash to the list of hashes this node is announcing
|
||||
func (dht *DHT) Add(hash bits.Bitmap) {
|
||||
dht.announceAddRemove <- queueEdit{hash: hash, add: true}
|
||||
}
|
||||
|
||||
// Remove removes the hash from the list of hashes this node is announcing
|
||||
func (dht *DHT) Remove(hash bits.Bitmap) {
|
||||
dht.announceAddRemove <- queueEdit{hash: hash, add: false}
|
||||
}
|
||||
|
||||
func (dht *DHT) runAnnouncer() {
|
||||
type hashAndTime struct {
|
||||
hash bits.Bitmap
|
||||
lastAnnounce time.Time
|
||||
}
|
||||
|
||||
var queue *ring.Ring
|
||||
hashes := make(map[bits.Bitmap]*ring.Ring)
|
||||
|
||||
var announceNextHash <-chan time.Time
|
||||
timer := time.NewTimer(math.MaxInt64)
|
||||
timer.Stop()
|
||||
|
||||
limitCh := make(chan time.Time)
|
||||
dht.grp.Add(1)
|
||||
go func() {
|
||||
defer dht.grp.Done()
|
||||
limiter := rate.NewLimiter(rate.Limit(dht.conf.AnnounceRate), dht.conf.AnnounceRate)
|
||||
for {
|
||||
err := limiter.Wait(context.Background()) // TODO: should use grp.ctx somehow? so when grp is closed, wait returns
|
||||
if err != nil {
|
||||
log.Error(errors.Prefix("rate limiter", err))
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case limitCh <- time.Now():
|
||||
case <-dht.grp.Ch():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
maintenance := time.NewTicker(1 * time.Minute)
|
||||
|
||||
// TODO: work to space hash announces out so they aren't bunched up around the reannounce time. track time since last announce. if its been more than the ideal time (reannounce time / numhashes), start announcing hashes early
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-dht.grp.Ch():
|
||||
return
|
||||
|
||||
case <-maintenance.C:
|
||||
maxAnnounce := dht.conf.AnnounceRate * int(dht.conf.ReannounceTime.Seconds())
|
||||
if len(hashes) > maxAnnounce {
|
||||
// TODO: send this to slack
|
||||
log.Warnf("DHT has %d hashes, but can only announce %d hashes in the %s reannounce window. Raise the announce rate or spawn more nodes.",
|
||||
len(hashes), maxAnnounce, dht.conf.ReannounceTime.String())
|
||||
}
|
||||
|
||||
case change := <-dht.announceAddRemove:
|
||||
if change.add {
|
||||
if _, exists := hashes[change.hash]; exists {
|
||||
continue
|
||||
}
|
||||
|
||||
r := ring.New(1)
|
||||
r.Value = hashAndTime{hash: change.hash}
|
||||
if queue != nil {
|
||||
queue.Prev().Link(r)
|
||||
}
|
||||
queue = r
|
||||
hashes[change.hash] = r
|
||||
announceNextHash = limitCh // announce next hash ASAP
|
||||
} else {
|
||||
r, exists := hashes[change.hash]
|
||||
if !exists {
|
||||
continue
|
||||
}
|
||||
|
||||
delete(hashes, change.hash)
|
||||
|
||||
if len(hashes) == 0 {
|
||||
queue = ring.New(0)
|
||||
announceNextHash = nil // no hashes to announce, wait indefinitely
|
||||
} else {
|
||||
if r == queue {
|
||||
queue = queue.Next() // don't lose our pointer
|
||||
}
|
||||
r.Prev().Link(r.Next())
|
||||
}
|
||||
}
|
||||
|
||||
case <-announceNextHash:
|
||||
dht.grp.Add(1)
|
||||
ht := queue.Value.(hashAndTime)
|
||||
|
||||
if !ht.lastAnnounce.IsZero() {
|
||||
nextAnnounce := ht.lastAnnounce.Add(dht.conf.ReannounceTime)
|
||||
if nextAnnounce.After(time.Now()) {
|
||||
timer.Reset(time.Until(nextAnnounce))
|
||||
announceNextHash = timer.C // wait until next hash should be announced
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if dht.conf.AnnounceNotificationCh != nil {
|
||||
dht.conf.AnnounceNotificationCh <- announceNotification{
|
||||
hash: ht.hash,
|
||||
action: announceStarted,
|
||||
}
|
||||
}
|
||||
|
||||
go func(hash bits.Bitmap) {
|
||||
defer dht.grp.Done()
|
||||
err := dht.announce(hash)
|
||||
if err != nil {
|
||||
log.Error(errors.Prefix("announce", err))
|
||||
}
|
||||
|
||||
if dht.conf.AnnounceNotificationCh != nil {
|
||||
dht.conf.AnnounceNotificationCh <- announceNotification{
|
||||
hash: ht.hash,
|
||||
action: announceFinishd,
|
||||
err: err,
|
||||
}
|
||||
}
|
||||
}(ht.hash)
|
||||
|
||||
queue.Value = hashAndTime{hash: ht.hash, lastAnnounce: time.Now()}
|
||||
queue = queue.Next()
|
||||
announceNextHash = limitCh // announce next hash ASAP
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Announce announces to the DHT that this node has the blob for the given hash
|
||||
func (dht *DHT) announce(hash bits.Bitmap) error {
|
||||
contacts, _, err := FindContacts(dht.node, hash, false, dht.grp.Child())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// self-store if we found less than K contacts, or we're closer than the farthest contact
|
||||
if len(contacts) < bucketSize {
|
||||
contacts = append(contacts, dht.contact)
|
||||
} else if hash.Closer(dht.node.id, contacts[bucketSize-1].ID) {
|
||||
contacts[bucketSize-1] = dht.contact
|
||||
}
|
||||
|
||||
wg := &sync.WaitGroup{}
|
||||
for _, c := range contacts {
|
||||
wg.Add(1)
|
||||
go func(c Contact) {
|
||||
dht.store(hash, c)
|
||||
wg.Done()
|
||||
}(c)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (dht *DHT) store(hash bits.Bitmap, c Contact) {
|
||||
if dht.contact.ID == c.ID {
|
||||
// self-store
|
||||
c.PeerPort = dht.conf.PeerProtocolPort
|
||||
dht.node.Store(hash, c)
|
||||
return
|
||||
}
|
||||
|
||||
dht.node.SendAsync(c, Request{
|
||||
Method: storeMethod,
|
||||
StoreArgs: &storeArgs{
|
||||
BlobHash: hash,
|
||||
Value: storeArgsValue{
|
||||
Token: dht.tokenCache.Get(c, hash, dht.grp.Ch()),
|
||||
LbryID: dht.contact.ID,
|
||||
Port: dht.conf.PeerProtocolPort,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
-181
@@ -1,181 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"net"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
)
|
||||
|
||||
func TestNodeFinder_FindNodes(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping slow nodeFinder test")
|
||||
}
|
||||
|
||||
bs, dhts := TestingCreateNetwork(t, 3, true, false)
|
||||
defer func() {
|
||||
for i := range dhts {
|
||||
dhts[i].Shutdown()
|
||||
}
|
||||
bs.Shutdown()
|
||||
}()
|
||||
|
||||
contacts, found, err := FindContacts(dhts[2].node, bits.Rand(), false, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if found {
|
||||
t.Fatal("something was found, but it should not have been")
|
||||
}
|
||||
|
||||
if len(contacts) != 3 {
|
||||
t.Errorf("expected 3 node, found %d", len(contacts))
|
||||
}
|
||||
|
||||
foundBootstrap := false
|
||||
foundOne := false
|
||||
foundTwo := false
|
||||
|
||||
for _, n := range contacts {
|
||||
if n.ID.Equals(bs.id) {
|
||||
foundBootstrap = true
|
||||
}
|
||||
if n.ID.Equals(dhts[0].node.id) {
|
||||
foundOne = true
|
||||
}
|
||||
if n.ID.Equals(dhts[1].node.id) {
|
||||
foundTwo = true
|
||||
}
|
||||
}
|
||||
|
||||
if !foundBootstrap {
|
||||
t.Errorf("did not find bootstrap node %s", bs.id.Hex())
|
||||
}
|
||||
if !foundOne {
|
||||
t.Errorf("did not find first node %s", dhts[0].node.id.Hex())
|
||||
}
|
||||
if !foundTwo {
|
||||
t.Errorf("did not find second node %s", dhts[1].node.id.Hex())
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodeFinder_FindNodes_NoBootstrap(t *testing.T) {
|
||||
_, dhts := TestingCreateNetwork(t, 3, false, false)
|
||||
defer func() {
|
||||
for i := range dhts {
|
||||
dhts[i].Shutdown()
|
||||
}
|
||||
}()
|
||||
|
||||
_, _, err := FindContacts(dhts[2].node, bits.Rand(), false, nil)
|
||||
if err == nil {
|
||||
t.Fatal("contact finder should have errored saying that there are no contacts in the routing table")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodeFinder_FindValue(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping slow nodeFinder test")
|
||||
}
|
||||
|
||||
bs, dhts := TestingCreateNetwork(t, 3, true, false)
|
||||
defer func() {
|
||||
for i := range dhts {
|
||||
dhts[i].Shutdown()
|
||||
}
|
||||
bs.Shutdown()
|
||||
}()
|
||||
|
||||
blobHashToFind := bits.Rand()
|
||||
nodeToFind := Contact{ID: bits.Rand(), IP: net.IPv4(1, 2, 3, 4), Port: 5678}
|
||||
dhts[0].node.store.Upsert(blobHashToFind, nodeToFind)
|
||||
|
||||
contacts, found, err := FindContacts(dhts[2].node, blobHashToFind, true, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if !found {
|
||||
t.Fatal("node was not found")
|
||||
}
|
||||
|
||||
if len(contacts) != 1 {
|
||||
t.Fatalf("expected one node, found %d", len(contacts))
|
||||
}
|
||||
|
||||
if !contacts[0].ID.Equals(nodeToFind.ID) {
|
||||
t.Fatalf("found node id %s, expected %s", contacts[0].ID.Hex(), nodeToFind.ID.Hex())
|
||||
}
|
||||
}
|
||||
|
||||
func TestDHT_LargeDHT(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping large DHT test")
|
||||
}
|
||||
|
||||
nodes := 100
|
||||
bs, dhts := TestingCreateNetwork(t, nodes, true, true)
|
||||
defer func() {
|
||||
for _, d := range dhts {
|
||||
go d.Shutdown()
|
||||
}
|
||||
bs.Shutdown()
|
||||
time.Sleep(1 * time.Second)
|
||||
}()
|
||||
|
||||
wg := &sync.WaitGroup{}
|
||||
ids := make([]bits.Bitmap, nodes)
|
||||
for i := range ids {
|
||||
ids[i] = bits.Rand()
|
||||
wg.Add(1)
|
||||
go func(index int) {
|
||||
defer wg.Done()
|
||||
err := dhts[index].announce(ids[index])
|
||||
if err != nil {
|
||||
t.Error("error announcing random bitmap - ", err)
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
// check that each node is in at learst 1 other routing table
|
||||
rtCounts := make(map[bits.Bitmap]int)
|
||||
for _, d := range dhts {
|
||||
for _, d2 := range dhts {
|
||||
if d.node.id.Equals(d2.node.id) {
|
||||
continue
|
||||
}
|
||||
c := d2.node.rt.GetClosest(d.node.id, 1)
|
||||
if len(c) > 1 {
|
||||
t.Error("rt returned more than one node when only one requested")
|
||||
} else if len(c) == 1 && c[0].ID.Equals(d.node.id) {
|
||||
rtCounts[d.node.id]++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for k, v := range rtCounts {
|
||||
if v == 0 {
|
||||
t.Errorf("%s was not in any routing tables", k.HexShort())
|
||||
}
|
||||
}
|
||||
|
||||
// check that each ID is stored by at least 3 nodes
|
||||
storeCounts := make(map[bits.Bitmap]int)
|
||||
for _, d := range dhts {
|
||||
for _, id := range ids {
|
||||
if len(d.node.store.Get(id)) > 0 {
|
||||
storeCounts[id]++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for k, v := range storeCounts {
|
||||
if v == 0 {
|
||||
t.Errorf("%s was not stored by any nodes", k.HexShort())
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
-463
@@ -1,463 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
|
||||
"github.com/lyoshenka/bencode"
|
||||
"github.com/spf13/cast"
|
||||
)
|
||||
|
||||
const (
|
||||
pingMethod = "ping"
|
||||
storeMethod = "store"
|
||||
findNodeMethod = "findNode"
|
||||
findValueMethod = "findValue"
|
||||
)
|
||||
|
||||
const (
|
||||
pingSuccessResponse = "pong"
|
||||
storeSuccessResponse = "OK"
|
||||
)
|
||||
|
||||
const (
|
||||
requestType = 0
|
||||
responseType = 1
|
||||
errorType = 2
|
||||
)
|
||||
|
||||
const (
|
||||
// these are strings because bencode requires bytestring keys
|
||||
headerTypeField = "0"
|
||||
headerMessageIDField = "1" // message id is 20 bytes long
|
||||
headerNodeIDField = "2" // node id is 48 bytes long
|
||||
headerPayloadField = "3"
|
||||
headerArgsField = "4"
|
||||
contactsField = "contacts"
|
||||
tokenField = "token"
|
||||
protocolVersionField = "protocolVersion"
|
||||
)
|
||||
|
||||
// Message is a DHT message
|
||||
type Message interface {
|
||||
bencode.Marshaler
|
||||
}
|
||||
|
||||
type messageID [messageIDLength]byte
|
||||
|
||||
// HexShort returns the first 8 hex characters of the hex encoded message id.
|
||||
func (m messageID) HexShort() string {
|
||||
return hex.EncodeToString(m[:])[:8]
|
||||
}
|
||||
|
||||
// UnmarshalBencode takes a byte slice and unmarshals the message id.
|
||||
func (m *messageID) UnmarshalBencode(encoded []byte) error {
|
||||
var str string
|
||||
err := bencode.DecodeBytes(encoded, &str)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
copy(m[:], str)
|
||||
return nil
|
||||
}
|
||||
|
||||
// MarshallBencode returns the encoded byte slice of the message id.
|
||||
func (m messageID) MarshalBencode() ([]byte, error) {
|
||||
str := string(m[:])
|
||||
return bencode.EncodeBytes(str)
|
||||
}
|
||||
|
||||
func newMessageID() messageID {
|
||||
var m messageID
|
||||
_, err := rand.Read(m[:])
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// Request represents a DHT request message
|
||||
type Request struct {
|
||||
ID messageID
|
||||
NodeID bits.Bitmap
|
||||
Method string
|
||||
Arg *bits.Bitmap
|
||||
StoreArgs *storeArgs
|
||||
ProtocolVersion int
|
||||
}
|
||||
|
||||
// MarshalBencode returns the serialized byte slice representation of the request
|
||||
func (r Request) MarshalBencode() ([]byte, error) {
|
||||
var args interface{}
|
||||
if r.StoreArgs != nil {
|
||||
args = r.StoreArgs
|
||||
} else if r.Arg != nil {
|
||||
args = []bits.Bitmap{*r.Arg}
|
||||
} else {
|
||||
args = []string{} // request must always have keys 0-4, so we use an empty list for PING
|
||||
}
|
||||
return bencode.EncodeBytes(map[string]interface{}{
|
||||
headerTypeField: requestType,
|
||||
headerMessageIDField: r.ID,
|
||||
headerNodeIDField: r.NodeID,
|
||||
headerPayloadField: r.Method,
|
||||
headerArgsField: args,
|
||||
})
|
||||
}
|
||||
|
||||
// UnmarshalBencode unmarshals the serialized byte slice into the appropriate fields of the request.
|
||||
func (r *Request) UnmarshalBencode(b []byte) error {
|
||||
var raw struct {
|
||||
ID messageID `bencode:"1"`
|
||||
NodeID bits.Bitmap `bencode:"2"`
|
||||
Method string `bencode:"3"`
|
||||
Args bencode.RawMessage `bencode:"4"`
|
||||
}
|
||||
err := bencode.DecodeBytes(b, &raw)
|
||||
if err != nil {
|
||||
return errors.Prefix("request unmarshal", err)
|
||||
}
|
||||
|
||||
r.ID = raw.ID
|
||||
r.NodeID = raw.NodeID
|
||||
r.Method = raw.Method
|
||||
|
||||
if r.Method == storeMethod {
|
||||
r.StoreArgs = &storeArgs{} // bencode wont find the unmarshaler on a null pointer. need to fix it.
|
||||
err = bencode.DecodeBytes(raw.Args, &r.StoreArgs)
|
||||
if err != nil {
|
||||
return errors.Prefix("request unmarshal", err)
|
||||
}
|
||||
} else if len(raw.Args) > 2 { // 2 because an empty list is `le`
|
||||
r.Arg, r.ProtocolVersion, err = processArgsAndProtoVersion(raw.Args)
|
||||
if err != nil {
|
||||
return errors.Prefix("request unmarshal", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func processArgsAndProtoVersion(raw bencode.RawMessage) (arg *bits.Bitmap, version int, err error) {
|
||||
var args []bencode.RawMessage
|
||||
err = bencode.DecodeBytes(raw, &args)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
if len(args) == 0 {
|
||||
return nil, 0, nil
|
||||
}
|
||||
|
||||
var extras map[string]int
|
||||
err = bencode.DecodeBytes(args[len(args)-1], &extras)
|
||||
if err == nil {
|
||||
if v, exists := extras[protocolVersionField]; exists {
|
||||
version = v
|
||||
args = args[:len(args)-1]
|
||||
}
|
||||
}
|
||||
|
||||
if len(args) > 0 {
|
||||
var b bits.Bitmap
|
||||
err = bencode.DecodeBytes(args[0], &b)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
arg = &b
|
||||
}
|
||||
|
||||
return arg, version, nil
|
||||
}
|
||||
|
||||
func (r Request) argsDebug() string {
|
||||
if r.StoreArgs != nil {
|
||||
return r.StoreArgs.BlobHash.HexShort() + ", " + r.StoreArgs.Value.LbryID.HexShort() + ":" + strconv.Itoa(r.StoreArgs.Value.Port)
|
||||
} else if r.Arg != nil {
|
||||
return r.Arg.HexShort()
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type storeArgsValue struct {
|
||||
Token string `bencode:"token"`
|
||||
LbryID bits.Bitmap `bencode:"lbryid"`
|
||||
Port int `bencode:"port"`
|
||||
}
|
||||
|
||||
type storeArgs struct {
|
||||
BlobHash bits.Bitmap
|
||||
Value storeArgsValue
|
||||
NodeID bits.Bitmap // original publisher id? I think this is getting fixed in the new dht stuff
|
||||
SelfStore bool // this is an int on the wire
|
||||
}
|
||||
|
||||
// MarshalBencode returns the serialized byte slice representation of the storage arguments.
|
||||
func (s storeArgs) MarshalBencode() ([]byte, error) {
|
||||
encodedValue, err := bencode.EncodeString(s.Value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
selfStoreStr := 0
|
||||
if s.SelfStore {
|
||||
selfStoreStr = 1
|
||||
}
|
||||
|
||||
return bencode.EncodeBytes([]interface{}{
|
||||
s.BlobHash,
|
||||
bencode.RawMessage(encodedValue),
|
||||
s.NodeID,
|
||||
selfStoreStr,
|
||||
})
|
||||
}
|
||||
|
||||
// UnmarshalBencode unmarshals the serialized byte slice into the appropriate fields of the store arguments.
|
||||
func (s *storeArgs) UnmarshalBencode(b []byte) error {
|
||||
var argsInt []bencode.RawMessage
|
||||
err := bencode.DecodeBytes(b, &argsInt)
|
||||
if err != nil {
|
||||
return errors.Prefix("storeArgs unmarshal", err)
|
||||
}
|
||||
|
||||
if len(argsInt) != 4 {
|
||||
return errors.Err("unexpected number of fields for store args. got " + cast.ToString(len(argsInt)))
|
||||
}
|
||||
|
||||
err = bencode.DecodeBytes(argsInt[0], &s.BlobHash)
|
||||
if err != nil {
|
||||
return errors.Prefix("storeArgs unmarshal", err)
|
||||
}
|
||||
|
||||
err = bencode.DecodeBytes(argsInt[1], &s.Value)
|
||||
if err != nil {
|
||||
return errors.Prefix("storeArgs unmarshal", err)
|
||||
}
|
||||
|
||||
err = bencode.DecodeBytes(argsInt[2], &s.NodeID)
|
||||
if err != nil {
|
||||
return errors.Prefix("storeArgs unmarshal", err)
|
||||
}
|
||||
|
||||
var selfStore int
|
||||
err = bencode.DecodeBytes(argsInt[3], &selfStore)
|
||||
if err != nil {
|
||||
return errors.Prefix("storeArgs unmarshal", err)
|
||||
}
|
||||
if selfStore == 0 {
|
||||
s.SelfStore = false
|
||||
} else if selfStore == 1 {
|
||||
s.SelfStore = true
|
||||
} else {
|
||||
return errors.Err("selfstore must be 1 or 0")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Response represents a DHT response message
|
||||
type Response struct {
|
||||
ID messageID
|
||||
NodeID bits.Bitmap
|
||||
Data string
|
||||
Contacts []Contact
|
||||
FindValueKey string
|
||||
Token string
|
||||
ProtocolVersion int
|
||||
}
|
||||
|
||||
func (r Response) argsDebug() string {
|
||||
if r.Data != "" {
|
||||
return r.Data
|
||||
}
|
||||
|
||||
str := "contacts "
|
||||
if r.FindValueKey != "" {
|
||||
str = "value for " + hex.EncodeToString([]byte(r.FindValueKey))[:8] + " "
|
||||
}
|
||||
|
||||
str += "|"
|
||||
for _, c := range r.Contacts {
|
||||
str += c.String() + ","
|
||||
}
|
||||
str = strings.TrimRight(str, ",") + "|"
|
||||
|
||||
if r.Token != "" {
|
||||
str += " token: " + hex.EncodeToString([]byte(r.Token))[:8]
|
||||
}
|
||||
|
||||
return str
|
||||
}
|
||||
|
||||
// MarshalBencode returns the serialized byte slice representation of the response.
|
||||
func (r Response) MarshalBencode() ([]byte, error) {
|
||||
data := map[string]interface{}{
|
||||
headerTypeField: responseType,
|
||||
headerMessageIDField: r.ID,
|
||||
headerNodeIDField: r.NodeID,
|
||||
}
|
||||
|
||||
if r.Data != "" {
|
||||
// ping or store
|
||||
data[headerPayloadField] = r.Data
|
||||
} else if r.FindValueKey != "" {
|
||||
// findValue success
|
||||
if r.Token == "" {
|
||||
return nil, errors.Err("response to findValue must have a token")
|
||||
}
|
||||
|
||||
var contacts [][]byte
|
||||
for _, c := range r.Contacts {
|
||||
compact, err := c.MarshalCompact()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
contacts = append(contacts, compact)
|
||||
}
|
||||
data[headerPayloadField] = map[string]interface{}{
|
||||
r.FindValueKey: contacts,
|
||||
tokenField: r.Token,
|
||||
}
|
||||
} else if r.Token != "" {
|
||||
// findValue failure falling back to findNode
|
||||
data[headerPayloadField] = map[string]interface{}{
|
||||
contactsField: r.Contacts,
|
||||
tokenField: r.Token,
|
||||
}
|
||||
} else {
|
||||
// straight up findNode
|
||||
data[headerPayloadField] = r.Contacts
|
||||
}
|
||||
|
||||
return bencode.EncodeBytes(data)
|
||||
}
|
||||
|
||||
// UnmarshalBencode unmarshals the serialized byte slice into the appropriate fields of the store arguments.
|
||||
func (r *Response) UnmarshalBencode(b []byte) error {
|
||||
var raw struct {
|
||||
ID messageID `bencode:"1"`
|
||||
NodeID bits.Bitmap `bencode:"2"`
|
||||
Data bencode.RawMessage `bencode:"3"`
|
||||
}
|
||||
err := bencode.DecodeBytes(b, &raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
r.ID = raw.ID
|
||||
r.NodeID = raw.NodeID
|
||||
|
||||
// maybe data is a string (response to ping or store)?
|
||||
err = bencode.DecodeBytes(raw.Data, &r.Data)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// maybe data is a list of contacts (response to findNode)?
|
||||
err = bencode.DecodeBytes(raw.Data, &r.Contacts)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// it must be a response to findValue
|
||||
var rawData map[string]bencode.RawMessage
|
||||
err = bencode.DecodeBytes(raw.Data, &rawData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if token, ok := rawData[tokenField]; ok {
|
||||
err = bencode.DecodeBytes(token, &r.Token)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
delete(rawData, tokenField) // so it doesnt mess up findValue key finding below
|
||||
}
|
||||
|
||||
if protocolVersion, ok := rawData[protocolVersionField]; ok {
|
||||
err = bencode.DecodeBytes(protocolVersion, &r.ProtocolVersion)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
delete(rawData, protocolVersionField) // so it doesnt mess up findValue key finding below
|
||||
}
|
||||
|
||||
if contacts, ok := rawData[contactsField]; ok {
|
||||
err = bencode.DecodeBytes(contacts, &r.Contacts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
for k, v := range rawData {
|
||||
r.FindValueKey = k
|
||||
var compactContacts [][]byte
|
||||
err = bencode.DecodeBytes(v, &compactContacts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, compact := range compactContacts {
|
||||
var c Contact
|
||||
err = c.UnmarshalCompact(compact)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r.Contacts = append(r.Contacts, c)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Error represents a DHT error response
|
||||
type Error struct {
|
||||
ID messageID
|
||||
NodeID bits.Bitmap
|
||||
ExceptionType string
|
||||
Response []string
|
||||
}
|
||||
|
||||
// MarshalBencode returns the serialized byte slice representation of an error message.
|
||||
func (e Error) MarshalBencode() ([]byte, error) {
|
||||
return bencode.EncodeBytes(map[string]interface{}{
|
||||
headerTypeField: errorType,
|
||||
headerMessageIDField: e.ID,
|
||||
headerNodeIDField: e.NodeID,
|
||||
headerPayloadField: e.ExceptionType,
|
||||
headerArgsField: e.Response,
|
||||
})
|
||||
}
|
||||
|
||||
// UnmarshalBencode unmarshals the serialized byte slice into the appropriate fields of the error message.
|
||||
func (e *Error) UnmarshalBencode(b []byte) error {
|
||||
var raw struct {
|
||||
ID messageID `bencode:"1"`
|
||||
NodeID bits.Bitmap `bencode:"2"`
|
||||
ExceptionType string `bencode:"3"`
|
||||
Args interface{} `bencode:"4"`
|
||||
}
|
||||
err := bencode.DecodeBytes(b, &raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
e.ID = raw.ID
|
||||
e.NodeID = raw.NodeID
|
||||
e.ExceptionType = raw.ExceptionType
|
||||
|
||||
if reflect.TypeOf(raw.Args).Kind() == reflect.Slice {
|
||||
v := reflect.ValueOf(raw.Args)
|
||||
for i := 0; i < v.Len(); i++ {
|
||||
e.Response = append(e.Response, cast.ToString(v.Index(i).Interface()))
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,223 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"net"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
"github.com/lyoshenka/bencode"
|
||||
)
|
||||
|
||||
func TestBencodeDecodeStoreArgs(t *testing.T) {
|
||||
blobHash := "3214D6C2F77FCB5E8D5FC07EDAFBA614F031CE8B2EAB49F924F8143F6DFBADE048D918710072FB98AB1B52B58F4E1468"
|
||||
lbryID := "7CE1B831DEC8689E44F80F547D2DEA171F6A625E1A4FF6C6165E645F953103DABEB068A622203F859C6C64658FD3AA3B"
|
||||
port := hex.EncodeToString([]byte("3333"))
|
||||
token := "17C2D8E1E48EF21567FE4AD5C8ED944B798D3B65AB58D0C9122AD6587D1B5FED472EA2CB12284CEFA1C21EFF302322BD"
|
||||
nodeID := "7CE1B831DEC8689E44F80F547D2DEA171F6A625E1A4FF6C6165E645F953103DABEB068A622203F859C6C64658FD3AA3B"
|
||||
selfStore := hex.EncodeToString([]byte("1"))
|
||||
|
||||
raw := "6C" + // start args list
|
||||
"3438 3A " + blobHash + // blob hash
|
||||
"64" + // start value dict
|
||||
"363A6C6272796964 3438 3A " + lbryID + // lbry id
|
||||
"343A706F7274 69 " + port + " 65" + // port
|
||||
"353A746F6B656E 3438 3A " + token + // token
|
||||
"65" + // end value dict
|
||||
"3438 3A " + nodeID + // node id
|
||||
"69 " + selfStore + " 65" + // self store (integer)
|
||||
"65" // end args list
|
||||
|
||||
raw = strings.ToLower(strings.Replace(raw, " ", "", -1))
|
||||
|
||||
data, err := hex.DecodeString(raw)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
storeArgs := &storeArgs{}
|
||||
err = bencode.DecodeBytes(data, storeArgs)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
if storeArgs.BlobHash.Hex() != strings.ToLower(blobHash) {
|
||||
t.Error("blob hash mismatch")
|
||||
}
|
||||
if storeArgs.Value.LbryID.Hex() != strings.ToLower(lbryID) {
|
||||
t.Error("lbryid mismatch")
|
||||
}
|
||||
if hex.EncodeToString([]byte(strconv.Itoa(storeArgs.Value.Port))) != port {
|
||||
t.Error("port mismatch")
|
||||
}
|
||||
if hex.EncodeToString([]byte(storeArgs.Value.Token)) != strings.ToLower(token) {
|
||||
t.Error("token mismatch")
|
||||
}
|
||||
if storeArgs.NodeID.Hex() != strings.ToLower(nodeID) {
|
||||
t.Error("node id mismatch")
|
||||
}
|
||||
if !storeArgs.SelfStore {
|
||||
t.Error("selfStore mismatch")
|
||||
}
|
||||
|
||||
reencoded, err := bencode.EncodeBytes(storeArgs)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
} else if !reflect.DeepEqual(reencoded, data) {
|
||||
t.Error("reencoded data does not match original")
|
||||
spew.Dump(reencoded, data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBencodeFindNodesResponse(t *testing.T) {
|
||||
res := Response{
|
||||
ID: newMessageID(),
|
||||
NodeID: bits.Rand(),
|
||||
Contacts: []Contact{
|
||||
{ID: bits.Rand(), IP: net.IPv4(1, 2, 3, 4).To4(), Port: 5678},
|
||||
{ID: bits.Rand(), IP: net.IPv4(4, 3, 2, 1).To4(), Port: 8765},
|
||||
},
|
||||
}
|
||||
|
||||
encoded, err := bencode.EncodeBytes(res)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var res2 Response
|
||||
err = bencode.DecodeBytes(encoded, &res2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
compareResponses(t, res, res2)
|
||||
}
|
||||
|
||||
func TestBencodeFindValueResponse(t *testing.T) {
|
||||
res := Response{
|
||||
ID: newMessageID(),
|
||||
NodeID: bits.Rand(),
|
||||
FindValueKey: bits.Rand().RawString(),
|
||||
Token: "arstarstarst",
|
||||
Contacts: []Contact{
|
||||
{ID: bits.Rand(), IP: net.IPv4(1, 2, 3, 4).To4(), PeerPort: 8765},
|
||||
},
|
||||
}
|
||||
|
||||
encoded, err := bencode.EncodeBytes(res)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var res2 Response
|
||||
err = bencode.DecodeBytes(encoded, &res2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
compareResponses(t, res, res2)
|
||||
}
|
||||
|
||||
func TestDecodeSomeStrings(t *testing.T) {
|
||||
strs := []string{
|
||||
"6469306569306569316532303a359ec440b8236dee0fb2500ebda9c4704ae6741469326534383ade269943996b4bef3ff41176668a0577f86aba7f1ea2996edd18f9c42430802c8085331345c5f0c44a7f352e2ba8ae59693365383a66696e644e6f64656934656c34383ade269943996b4bef3ff41176668a0577f86aba7f1ea2996edd18f9c42430802c8085331345c5f0c44a7f352e2ba8ae596565",
|
||||
"64313a30693165313a3132303a31f27ec6174783c6f00e4b9ebccd49c6bdce300e313a3234383a7b3785cd962a56d4d4574262b1b63cc95d062c825b38f73602c7dc297c1e6b8259cdcdb5e7edef216d514e0c31ad8637313a3364353a746f6b656e34383a071fdf9835995889920e9beb876ebe8e36e5d44f5822684b549457405e22a5bf3fe40335743b459f22c347c0e736eba434383a7bcc7f8804db7d0d40fcb9c508547e53ee4f7d4000758105ecce099fc931d9cccfbbf9a30829777afcf41f2b4b0aeb716c35343a627db2b20d05b109195e7d06a855c392fda9a05faa878e69b18cc1212c35371ba5cad6231e049a460cdc263e7f4701afad45b288f48435343a424434ae0d05bfd6e1b98cc921b441fa8b560ac29038c37ec4907ae023617bf2bbc54abd3403226daf479798edcce69743ea6699e6ff35343a4462ba020d059ed3d31ceb3d1ffb2c025abf065a14516a8942607f9b3fad4b697ccc232766bdf471b6518a3ed716f2712171fd046c4335343a47413e2b0d05bf6244e5edacf43f65ef4cfd4b6eac380244894cc364ec23e8ededfb085999a211e20f8de146d9e081bbb42dc61d9ca035343a627db2b20d05b109195e7d06a855c392fda9a05faa878e69b18cc1212c35371ba5cad6231e049a460cdc263e7f4701afad45b288f48435343a4462ba020d059ed3d31ceb3d1ffb2c025abf065a14516a8942607f9b3fad4b697ccc232766bdf471b6518a3ed716f2712171fd046c4335343a47413e2b0d05bf6244e5edacf43f65ef4cfd4b6eac380244894cc364ec23e8ededfb085999a211e20f8de146d9e081bbb42dc61d9ca035343a627db2b20d05b109195e7d06a855c392fda9a05faa878e69b18cc1212c35371ba5cad6231e049a460cdc263e7f4701afad45b288f48435343a68fe5c3e0d05bfd6e1b98cc921b441fa8b560ac29038c37ec4907ae023617bf2bbc54abd3403226daf479798edcce69743ea6699e6ff35343a4462ba020d059ed3d31ceb3d1ffb2c025abf065a14516a8942607f9b3fad4b697ccc232766bdf471b6518a3ed716f2712171fd046c4335343a47413e2b0d05bf6244e5edacf43f65ef4cfd4b6eac380244894cc364ec23e8ededfb085999a211e20f8de146d9e081bbb42dc61d9ca035343a67d9a7ea0d05223c42ef5979c7c8f2444311f7a8266d64d3f98e9d0b21726bc2714a6fea5426c9a46a68c49ea13b54ccc324adf0783035343a68fe5c3e0d05bfd6e1b98cc921b441fa8b560ac29038c37ec4907ae023617bf2bbc54abd3403226daf479798edcce69743ea6699e6ff35343a47413e2b0d05bf6244e5edacf43f65ef4cfd4b6eac380244894cc364ec23e8ededfb085999a211e20f8de146d9e081bbb42dc61d9ca035343a4462ba020d059ed3d31ceb3d1ffb2c025abf065a14516a8942607f9b3fad4b697ccc232766bdf471b6518a3ed716f2712171fd046c4335343a67d9a7ea0d05223c42ef5979c7c8f2444311f7a8266d64d3f98e9d0b21726bc2714a6fea5426c9a46a68c49ea13b54ccc324adf0783035343a424434ae0d05bfd6e1b98cc921b441fa8b560ac29038c37ec4907ae023617bf2bbc54abd3403226daf479798edcce69743ea6699e6ff35343a4ace39c80d0541ae66325db4f1fce530476942d339f3219913f0a237726e18746a65ff7b3c5558c82c274a6f1ed10bbaf7c1305ca8e635343a47413e2b0d05bf6244e5edacf43f65ef4cfd4b6eac380244894cc364ec23e8ededfb085999a211e20f8de146d9e081bbb42dc61d9ca035343a4462ba020d059ed3d31ceb3d1ffb2c025abf065a14516a8942607f9b3fad4b697ccc232766bdf471b6518a3ed716f2712171fd046c4335343a627db2b20d05b109195e7d06a855c392fda9a05faa878e69b18cc1212c35371ba5cad6231e049a460cdc263e7f4701afad45b288f48435343a424434ae0d05bfd6e1b98cc921b441fa8b560ac29038c37ec4907ae023617bf2bbc54abd3403226daf479798edcce69743ea6699e6ff35343a4ace39c80d0541ae66325db4f1fce530476942d339f3219913f0a237726e18746a65ff7b3c5558c82c274a6f1ed10bbaf7c1305ca8e635343a47413e2b0d05bf6244e5edacf43f65ef4cfd4b6eac380244894cc364ec23e8ededfb085999a211e20f8de146d9e081bbb42dc61d9ca035343a4462ba020d059ed3d31ceb3d1ffb2c025abf065a14516a8942607f9b3fad4b697ccc232766bdf471b6518a3ed716f2712171fd046c4335343a67d9a7ea0d05223c42ef5979c7c8f2444311f7a8266d64d3f98e9d0b21726bc2714a6fea5426c9a46a68c49ea13b54ccc324adf0783035343a4ace39c80d0541ae66325db4f1fce530476942d339f3219913f0a237726e18746a65ff7b3c5558c82c274a6f1ed10bbaf7c1305ca8e635343a47413e2b0d05bf6244e5edacf43f65ef4cfd4b6eac380244894cc364ec23e8ededfb085999a211e20f8de146d9e081bbb42dc61d9ca035343a4462ba020d059ed3d31ceb3d1ffb2c025abf065a14516a8942607f9b3fad4b697ccc232766bdf471b6518a3ed716f2712171fd046c4335343a67d9a7ea0d05223c42ef5979c7c8f2444311f7a8266d64d3f98e9d0b21726bc2714a6fea5426c9a46a68c49ea13b54ccc324adf0783035343a68fe5c3e0d05bfd6e1b98cc921b441fa8b560ac29038c37ec4907ae023617bf2bbc54abd3403226daf479798edcce69743ea6699e6ff35343a4ace39c80d0541ae66325db4f1fce530476942d339f3219913f0a237726e18746a65ff7b3c5558c82c274a6f1ed10bbaf7c1305ca8e635343a47413e2b0d05bf6244e5edacf43f65ef4cfd4b6eac380244894cc364ec23e8ededfb085999a211e20f8de146d9e081bbb42dc61d9ca035343a68fe5c3e0d05bfd6e1b98cc921b441fa8b560ac29038c37ec4907ae023617bf2bbc54abd3403226daf479798edcce69743ea6699e6ff35343a4ace39c80d0541ae66325db4f1fce530476942d339f3219913f0a237726e18746a65ff7b3c5558c82c274a6f1ed10bbaf7c1305ca8e635343a47413e2b0d05bf6244e5edacf43f65ef4cfd4b6eac380244894cc364ec23e8ededfb085999a211e20f8de146d9e081bbb42dc61d9ca035343a4462ba020d059ed3d31ceb3d1ffb2c025abf065a14516a8942607f9b3fad4b697ccc232766bdf471b6518a3ed716f2712171fd046c4335343a67d9a7ea0d05223c42ef5979c7c8f2444311f7a8266d64d3f98e9d0b21726bc2714a6fea5426c9a46a68c49ea13b54ccc324adf0783035343a68fe5c3e0d05bfd6e1b98cc921b441fa8b560ac29038c37ec4907ae023617bf2bbc54abd3403226daf479798edcce69743ea6699e6ff35343a4ace39c80d0541ae66325db4f1fce530476942d339f3219913f0a237726e18746a65ff7b3c5558c82c274a6f1ed10bbaf7c1305ca8e635343a47413e2b0d05bf6244e5edacf43fLine truncated
|
||||
"64313a30693165313a3132303aa19e41f1d887db705fbf0a58f120773069d6ff11313a3234383a6911c21c3b5ea6536f4fb170c87fdc8bf4201124c5fe5eeb5f0054ff48e899a1d6e089a30a12ba8683ebf79691d71439313a3364383a636f6e74616374736c6c34383a6dd3dedeec334bae70a3c5c1b58fef2d4a501af320a332cdd5db4b71945c0a90d11aa2013e8d0216258a643f42d9562631323a34362e32382e3230342e3738693434343465656c34383a7ec0edf204d940d21c17e1c979df1c94610711d7773ceda72a4aa925c5f7799bbb741caa8d6d877aa2776399f77c08a031333a36392e3131382e34332e313234693434343465656c34383a6ec15cd8b95718d010bdf9739fda1af9ded7ac8be7da5cced6dc7a08470c8109accc5bfb973e5c1dd711349c62a8a3c331323a34362e32382e3230342e3738693434343465656c34383a6d2603a690712d5a3d5addb228d31a622f798c64ab2d3576d2c3ab3c5a64cb863c05545f37aa384ce83003416ca5cea431323a34362e32382e3230342e3738693434343465656c34383a6c1d92202a7e74ec85dd4e3e1699303174c4b6460b171637bba0a1c068d72f389035e1ba8bbe70f6621c36d70d4d045031323a34362e32382e3230342e3738693434343465656c34383a6dac30fbe94c006515cffe4898bc534a105fd0403bfc3c8692c2751b7946949dbc9e0c9570477cf8f817d1efc9a0e02531343a31382e3232312e3139342e313933693434343465656c34383a6f2ee379c662ea9f6db86beed6023be300cd23b9f9c4610a195dcb0f258392ec77188854ea3a8c0fffe67844849eb6ca31333a32342e3131362e39302e323238693434343465656c34383a6c55b2dbb89010de9db88e0ad3510a79fe60983b394a9e66b0efe139a455973cdc4415beff82cbdfb63d98caad648ab031333a38382e3132392e3234352e33396934343434656565353a746f6b656e34383a51844b0d1e8a613c4ad783c03b323063d3a5ff063a640368d7754bcae277c22b2b06b46c3e5466c30acb773f77b686476565",
|
||||
"64313a30693165313a3132303af693334af099a987a4b55af2727472b51c1990db313a3234383ab6928ff25778a7bbb5d258d3b3a06e26db1654f3d2efce8c26681d43f7237cdf2e359a4d309c4473d5d89ec99fb4f573313a3364353a746f6b656e34383a61a45a40080ea118361f919ebf09cdc9d2c63f476e7bc4d193463f52b91dc4b0db4ed1004e32d4fc273645421e58479034383abe463e24af015c0d1ba496b4b1911aa98c8b42119295bbcc5c36829a2e2ff8f149743cbbc0af2c39669cb0750fd778576c35343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e95656565",
|
||||
"6469306569316569316532303a3e0929c88abb8fe1d718025efc7f4a3cd85de16269326534383a21b2e2d2996b4bef3ff41176668a0577f86aba7f1ea2996edd18f9c42430802c8085331345c5f0c44a7f352e2ba8ae596933656c6565",
|
||||
}
|
||||
|
||||
for i, str := range strs {
|
||||
raw, err := hex.DecodeString(str)
|
||||
if err != nil {
|
||||
t.Errorf("error hex-decoding string %d: %s", i, err)
|
||||
continue
|
||||
}
|
||||
|
||||
var decoded interface{}
|
||||
err = bencode.DecodeBytes(raw, &decoded)
|
||||
if err != nil {
|
||||
t.Errorf("error bencode-decoding string %d: %s", i, err)
|
||||
continue
|
||||
}
|
||||
//
|
||||
//t.Error("TODO")
|
||||
//continue
|
||||
//
|
||||
//spew.Dump(decoded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeFindNodeResponseWithNoNodes(t *testing.T) {
|
||||
raw, err := hex.DecodeString("6469306569316569316532303a3e0929c88abb8fe1d718025efc7f4a3cd85de16269326534383a21b2e2d2996b4bef3ff41176668a0577f86aba7f1ea2996edd18f9c42430802c8085331345c5f0c44a7f352e2ba8ae596933656c6565")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
response := Response{}
|
||||
err = bencode.DecodeBytes(raw, &response)
|
||||
//spew.Dump(response)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeRequestWithProtocolVersion(t *testing.T) {
|
||||
raw, err := hex.DecodeString("6469306569306569316532303a65e1b9afce87c44abc40b4bb466890d2797f0dd269326534383a1baf3dbba8746c7a739f35465e268ace823622c5e8ec1dd7d5d27af5795cfbc54f22a32fbd05d420f241266b3dd16831693365343a70696e676934656c6431353a70726f746f636f6c56657273696f6e693165656565")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
req := Request{}
|
||||
err = bencode.DecodeBytes(raw, &req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if req.ProtocolVersion != 1 {
|
||||
t.Error("protocol version was not detected correctly")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeResponseWithProtocolVersion(t *testing.T) {
|
||||
raw, err := hex.DecodeString("6469306569316569316532303a2b96f5de8be1e86dc2332a50eb313c97848064db69326534383a8b8eb692658ea3d7e7828e80a3133d524c6f82aaff370efa759d5b87821035a32de06724cb099f01a819695f829dff7f6933656431353a70726f746f636f6c56657273696f6e693165353a746f6b656e34383af2368fe4fd06ede6631ad8b153f3b5d8db724f8d520f4291e992e206dd02a216fb7dfd9b81686f11b626e3840df65fb434383a89c5c3f9794b0b24a03406e3b74361edb9ae70828e4c133512fc75db0a2d312673cdd4e30eed37892a46692d2fe439f36c35343a12dd65af0d058cd7d10d122fbe2eb5ae31062b7480011be588f20cfe6807b1939c42eea639059fa6365bfccb56ef8c9e574f49ba035c656565")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
res := Response{}
|
||||
err = bencode.DecodeBytes(raw, &res)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if res.ProtocolVersion != 1 {
|
||||
t.Error("protocol version was not detected correctly")
|
||||
}
|
||||
}
|
||||
|
||||
func compareResponses(t *testing.T, res, res2 Response) {
|
||||
if res.ID != res2.ID {
|
||||
t.Errorf("expected ID %s, got %s", res.ID, res2.ID)
|
||||
}
|
||||
if !res.NodeID.Equals(res2.NodeID) {
|
||||
t.Errorf("expected NodeID %s, got %s", res.NodeID.Hex(), res2.NodeID.Hex())
|
||||
}
|
||||
if res.Data != res2.Data {
|
||||
t.Errorf("expected Data %s, got %s", res.Data, res2.Data)
|
||||
}
|
||||
if res.FindValueKey != res2.FindValueKey {
|
||||
t.Errorf("expected FindValueKey %s, got %s", res.FindValueKey, res2.FindValueKey)
|
||||
}
|
||||
if res.Token != res2.Token {
|
||||
t.Errorf("expected Token %s, got %s", res.Token, res2.Token)
|
||||
}
|
||||
if !reflect.DeepEqual(res.Contacts, res2.Contacts) {
|
||||
t.Errorf("expected FindNodeData %s, got %s", spew.Sdump(res.Contacts), spew.Sdump(res2.Contacts))
|
||||
}
|
||||
}
|
||||
-474
@@ -1,474 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/errors.go"
|
||||
"github.com/lbryio/lbry.go/stop"
|
||||
"github.com/lbryio/lbry.go/util"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/lyoshenka/bencode"
|
||||
)
|
||||
|
||||
// packet represents the information receive from udp.
|
||||
type packet struct {
|
||||
data []byte
|
||||
raddr *net.UDPAddr
|
||||
}
|
||||
|
||||
// UDPConn allows using a mocked connection to test sending/receiving data
|
||||
// TODO: stop mocking this and use the real thing
|
||||
type UDPConn interface {
|
||||
ReadFromUDP([]byte) (int, *net.UDPAddr, error)
|
||||
WriteToUDP([]byte, *net.UDPAddr) (int, error)
|
||||
SetReadDeadline(time.Time) error
|
||||
SetWriteDeadline(time.Time) error
|
||||
Close() error
|
||||
}
|
||||
|
||||
// RequestHandlerFunc is exported handler for requests.
|
||||
type RequestHandlerFunc func(addr *net.UDPAddr, request Request)
|
||||
|
||||
// Node is a type representation of a node on the network.
|
||||
type Node struct {
|
||||
// the node's id
|
||||
id bits.Bitmap
|
||||
// UDP connection for sending and receiving data
|
||||
conn UDPConn
|
||||
// true if we've closed the connection on purpose
|
||||
connClosed bool
|
||||
// token manager
|
||||
tokens *tokenManager
|
||||
|
||||
// map of outstanding transactions + mutex
|
||||
txLock *sync.RWMutex
|
||||
transactions map[messageID]*transaction
|
||||
|
||||
// routing table
|
||||
rt *routingTable
|
||||
// data store
|
||||
store *contactStore
|
||||
|
||||
// overrides for request handlers
|
||||
requestHandler RequestHandlerFunc
|
||||
|
||||
// stop the node neatly and clean up after itself
|
||||
grp *stop.Group
|
||||
}
|
||||
|
||||
// NewNode returns an initialized Node's pointer.
|
||||
func NewNode(id bits.Bitmap) *Node {
|
||||
return &Node{
|
||||
id: id,
|
||||
rt: newRoutingTable(id),
|
||||
store: newStore(),
|
||||
|
||||
txLock: &sync.RWMutex{},
|
||||
transactions: make(map[messageID]*transaction),
|
||||
|
||||
grp: stop.New(),
|
||||
tokens: &tokenManager{},
|
||||
}
|
||||
}
|
||||
|
||||
// Connect connects to the given connection and starts any background threads necessary
|
||||
func (n *Node) Connect(conn UDPConn) error {
|
||||
n.conn = conn
|
||||
|
||||
n.tokens.Start(tokenSecretRotationInterval)
|
||||
|
||||
go func() {
|
||||
// stop tokens and close the connection when we're shutting down
|
||||
<-n.grp.Ch()
|
||||
n.tokens.Stop()
|
||||
n.connClosed = true
|
||||
err := n.conn.Close()
|
||||
if err != nil {
|
||||
log.Error("error closing node connection on shutdown - ", err)
|
||||
}
|
||||
}()
|
||||
|
||||
packets := make(chan packet)
|
||||
|
||||
n.grp.Add(1)
|
||||
go func() {
|
||||
defer n.grp.Done()
|
||||
|
||||
buf := make([]byte, udpMaxMessageLength)
|
||||
|
||||
for {
|
||||
bytesRead, raddr, err := n.conn.ReadFromUDP(buf)
|
||||
if err != nil {
|
||||
if n.connClosed {
|
||||
return
|
||||
}
|
||||
log.Errorf("udp read error: %v", err)
|
||||
continue
|
||||
} else if raddr == nil {
|
||||
log.Errorf("udp read with no raddr")
|
||||
continue
|
||||
}
|
||||
|
||||
data := make([]byte, bytesRead)
|
||||
copy(data, buf[:bytesRead]) // slices use the same underlying array, so we need a new one for each packet
|
||||
|
||||
select { // needs select here because packet consumer can quit and the packets channel gets filled up and blocks
|
||||
case packets <- packet{data: data, raddr: raddr}:
|
||||
case <-n.grp.Ch():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
n.grp.Add(1)
|
||||
go func() {
|
||||
defer n.grp.Done()
|
||||
|
||||
var pkt packet
|
||||
|
||||
for {
|
||||
select {
|
||||
case pkt = <-packets:
|
||||
n.handlePacket(pkt)
|
||||
case <-n.grp.Ch():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// TODO: turn this back on when you're sure it works right
|
||||
n.grp.Add(1)
|
||||
go func() {
|
||||
defer n.grp.Done()
|
||||
n.startRoutingTableGrooming()
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Shutdown shuts down the node
|
||||
func (n *Node) Shutdown() {
|
||||
log.Debugf("[%s] node shutting down", n.id.HexShort())
|
||||
n.grp.StopAndWait()
|
||||
log.Debugf("[%s] node stopped", n.id.HexShort())
|
||||
}
|
||||
|
||||
// handlePacket handles packets received from udp.
|
||||
func (n *Node) handlePacket(pkt packet) {
|
||||
//log.Debugf("[%s] Received message from %s (%d bytes) %s", n.id.HexShort(), pkt.raddr.String(), len(pkt.data), hex.EncodeToString(pkt.data))
|
||||
|
||||
if !util.InSlice(string(pkt.data[0:5]), []string{"d1:0i", "di0ei"}) {
|
||||
log.Errorf("[%s] data is not a well-formatted dict: (%d bytes) %s", n.id.HexShort(), len(pkt.data), hex.EncodeToString(pkt.data))
|
||||
return
|
||||
}
|
||||
|
||||
// the following is a bit of a hack, but it lets us avoid decoding every message twice
|
||||
// it depends on the data being a dict with 0 as the first key (so it starts with "d1:0i") and the message type as the first value
|
||||
// TODO: test this more thoroughly
|
||||
|
||||
switch pkt.data[5] {
|
||||
case '0' + requestType:
|
||||
request := Request{}
|
||||
err := bencode.DecodeBytes(pkt.data, &request)
|
||||
if err != nil {
|
||||
log.Errorf("[%s] error decoding request from %s: %s: (%d bytes) %s", n.id.HexShort(), pkt.raddr.String(), err.Error(), len(pkt.data), hex.EncodeToString(pkt.data))
|
||||
return
|
||||
}
|
||||
log.Debugf("[%s] query %s: received request from %s: %s(%s)", n.id.HexShort(), request.ID.HexShort(), request.NodeID.HexShort(), request.Method, request.argsDebug())
|
||||
n.handleRequest(pkt.raddr, request)
|
||||
|
||||
case '0' + responseType:
|
||||
response := Response{}
|
||||
err := bencode.DecodeBytes(pkt.data, &response)
|
||||
if err != nil {
|
||||
log.Errorf("[%s] error decoding response from %s: %s: (%d bytes) %s", n.id.HexShort(), pkt.raddr.String(), err.Error(), len(pkt.data), hex.EncodeToString(pkt.data))
|
||||
return
|
||||
}
|
||||
log.Debugf("[%s] query %s: received response from %s: %s", n.id.HexShort(), response.ID.HexShort(), response.NodeID.HexShort(), response.argsDebug())
|
||||
n.handleResponse(pkt.raddr, response)
|
||||
|
||||
case '0' + errorType:
|
||||
e := Error{}
|
||||
err := bencode.DecodeBytes(pkt.data, &e)
|
||||
if err != nil {
|
||||
log.Errorf("[%s] error decoding error from %s: %s: (%d bytes) %s", n.id.HexShort(), pkt.raddr.String(), err.Error(), len(pkt.data), hex.EncodeToString(pkt.data))
|
||||
return
|
||||
}
|
||||
log.Debugf("[%s] query %s: received error from %s: %s", n.id.HexShort(), e.ID.HexShort(), e.NodeID.HexShort(), e.ExceptionType)
|
||||
n.handleError(pkt.raddr, e)
|
||||
|
||||
default:
|
||||
log.Errorf("[%s] invalid message type: %s", n.id.HexShort(), string(pkt.data[5]))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// handleRequest handles the requests received from udp.
|
||||
func (n *Node) handleRequest(addr *net.UDPAddr, request Request) {
|
||||
if request.NodeID.Equals(n.id) {
|
||||
log.Warn("ignoring self-request")
|
||||
return
|
||||
}
|
||||
|
||||
// if a handler is overridden, call it instead
|
||||
if n.requestHandler != nil {
|
||||
n.requestHandler(addr, request)
|
||||
return
|
||||
}
|
||||
|
||||
switch request.Method {
|
||||
default:
|
||||
//n.sendMessage(addr, Error{ID: request.ID, NodeID: n.id, ExceptionType: "invalid-request-method"})
|
||||
log.Errorln("invalid request method")
|
||||
return
|
||||
case pingMethod:
|
||||
err := n.sendMessage(addr, Response{ID: request.ID, NodeID: n.id, Data: pingSuccessResponse})
|
||||
if err != nil {
|
||||
log.Error("error sending 'pingmethod' response message - ", err)
|
||||
}
|
||||
case storeMethod:
|
||||
// TODO: we should be sending the IP in the request, not just using the sender's IP
|
||||
// TODO: should we be using StoreArgs.NodeID or StoreArgs.Value.LbryID ???
|
||||
if n.tokens.Verify(request.StoreArgs.Value.Token, request.NodeID, addr) {
|
||||
n.Store(request.StoreArgs.BlobHash, Contact{ID: request.StoreArgs.NodeID, IP: addr.IP, Port: addr.Port, PeerPort: request.StoreArgs.Value.Port})
|
||||
|
||||
err := n.sendMessage(addr, Response{ID: request.ID, NodeID: n.id, Data: storeSuccessResponse})
|
||||
if err != nil {
|
||||
log.Error("error sending 'storemethod' response message - ", err)
|
||||
}
|
||||
} else {
|
||||
err := n.sendMessage(addr, Error{ID: request.ID, NodeID: n.id, ExceptionType: "invalid-token"})
|
||||
if err != nil {
|
||||
log.Error("error sending 'storemethod'response message for invalid-token - ", err)
|
||||
}
|
||||
}
|
||||
case findNodeMethod:
|
||||
if request.Arg == nil {
|
||||
log.Errorln("request is missing arg")
|
||||
return
|
||||
}
|
||||
err := n.sendMessage(addr, Response{
|
||||
ID: request.ID,
|
||||
NodeID: n.id,
|
||||
Contacts: n.rt.GetClosest(*request.Arg, bucketSize),
|
||||
})
|
||||
if err != nil {
|
||||
log.Error("error sending 'findnodemethod' response message - ", err)
|
||||
}
|
||||
|
||||
case findValueMethod:
|
||||
if request.Arg == nil {
|
||||
log.Errorln("request is missing arg")
|
||||
return
|
||||
}
|
||||
|
||||
res := Response{
|
||||
ID: request.ID,
|
||||
NodeID: n.id,
|
||||
Token: n.tokens.Get(request.NodeID, addr),
|
||||
}
|
||||
|
||||
if contacts := n.store.Get(*request.Arg); len(contacts) > 0 {
|
||||
res.FindValueKey = request.Arg.RawString()
|
||||
res.Contacts = contacts
|
||||
} else {
|
||||
res.Contacts = n.rt.GetClosest(*request.Arg, bucketSize)
|
||||
}
|
||||
|
||||
err := n.sendMessage(addr, res)
|
||||
if err != nil {
|
||||
log.Error("error sending 'findvaluemethod' response message - ", err)
|
||||
}
|
||||
}
|
||||
|
||||
// nodes that send us requests should not be inserted, only refreshed.
|
||||
// the routing table must only contain "good" nodes, which are nodes that reply to our requests
|
||||
// if a node is already good (aka in the table), its fine to refresh it
|
||||
// http://www.bittorrent.org/beps/bep_0005.html#routing-table
|
||||
n.rt.Fresh(Contact{ID: request.NodeID, IP: addr.IP, Port: addr.Port})
|
||||
}
|
||||
|
||||
// handleResponse handles responses received from udp.
|
||||
func (n *Node) handleResponse(addr *net.UDPAddr, response Response) {
|
||||
tx := n.txFind(response.ID, Contact{ID: response.NodeID, IP: addr.IP, Port: addr.Port})
|
||||
if tx != nil {
|
||||
select {
|
||||
case tx.res <- response:
|
||||
default:
|
||||
//log.Errorf("[%s] query %s: response received, but tx has no listener or multiple responses to the same tx", n.id.HexShort(), response.ID.HexShort())
|
||||
}
|
||||
}
|
||||
|
||||
n.rt.Update(Contact{ID: response.NodeID, IP: addr.IP, Port: addr.Port})
|
||||
}
|
||||
|
||||
// handleError handles errors received from udp.
|
||||
func (n *Node) handleError(addr *net.UDPAddr, e Error) {
|
||||
spew.Dump(e)
|
||||
n.rt.Fresh(Contact{ID: e.NodeID, IP: addr.IP, Port: addr.Port})
|
||||
}
|
||||
|
||||
// send sends data to a udp address
|
||||
func (n *Node) sendMessage(addr *net.UDPAddr, data Message) error {
|
||||
encoded, err := bencode.EncodeBytes(data)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
if req, ok := data.(Request); ok {
|
||||
log.Debugf("[%s] query %s: sending request to %s (%d bytes) %s(%s)",
|
||||
n.id.HexShort(), req.ID.HexShort(), addr.String(), len(encoded), req.Method, req.argsDebug())
|
||||
} else if res, ok := data.(Response); ok {
|
||||
log.Debugf("[%s] query %s: sending response to %s (%d bytes) %s",
|
||||
n.id.HexShort(), res.ID.HexShort(), addr.String(), len(encoded), res.argsDebug())
|
||||
} else {
|
||||
log.Debugf("[%s] (%d bytes) %s", n.id.HexShort(), len(encoded), spew.Sdump(data))
|
||||
}
|
||||
|
||||
err = n.conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
|
||||
if err != nil {
|
||||
if n.connClosed {
|
||||
return nil
|
||||
}
|
||||
log.Error("error setting write deadline - ", err)
|
||||
}
|
||||
|
||||
_, err = n.conn.WriteToUDP(encoded, addr)
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
// transaction represents a single query to the dht. it stores the queried contact, the request, and the response channel
|
||||
type transaction struct {
|
||||
contact Contact
|
||||
req Request
|
||||
res chan Response
|
||||
skipIDCheck bool
|
||||
}
|
||||
|
||||
// insert adds a transaction to the manager.
|
||||
func (n *Node) txInsert(tx *transaction) {
|
||||
n.txLock.Lock()
|
||||
defer n.txLock.Unlock()
|
||||
n.transactions[tx.req.ID] = tx
|
||||
}
|
||||
|
||||
// delete removes a transaction from the manager.
|
||||
func (n *Node) txDelete(id messageID) {
|
||||
n.txLock.Lock()
|
||||
defer n.txLock.Unlock()
|
||||
delete(n.transactions, id)
|
||||
}
|
||||
|
||||
// Find finds a transaction for the given id and contact
|
||||
func (n *Node) txFind(id messageID, c Contact) *transaction {
|
||||
n.txLock.RLock()
|
||||
defer n.txLock.RUnlock()
|
||||
|
||||
t, ok := n.transactions[id]
|
||||
if !ok || !t.contact.Equals(c, !t.skipIDCheck) {
|
||||
return nil
|
||||
}
|
||||
|
||||
return t
|
||||
}
|
||||
|
||||
// SendOptions controls the behavior of send calls
|
||||
type SendOptions struct {
|
||||
skipIDCheck bool
|
||||
}
|
||||
|
||||
// SendAsync sends a transaction and returns a channel that will eventually contain the transaction response
|
||||
// The response channel is closed when the transaction is completed or times out.
|
||||
func (n *Node) SendAsync(contact Contact, req Request, options ...SendOptions) <-chan *Response {
|
||||
ch := make(chan *Response, 1)
|
||||
|
||||
if contact.ID.Equals(n.id) {
|
||||
log.Error("sending query to self")
|
||||
close(ch)
|
||||
return ch
|
||||
}
|
||||
|
||||
go func() {
|
||||
defer close(ch)
|
||||
|
||||
req.ID = newMessageID()
|
||||
req.NodeID = n.id
|
||||
tx := &transaction{
|
||||
contact: contact,
|
||||
req: req,
|
||||
res: make(chan Response),
|
||||
}
|
||||
|
||||
if len(options) > 0 && options[0].skipIDCheck {
|
||||
tx.skipIDCheck = true
|
||||
}
|
||||
|
||||
n.txInsert(tx)
|
||||
defer n.txDelete(tx.req.ID)
|
||||
|
||||
for i := 0; i < udpRetry; i++ {
|
||||
err := n.sendMessage(contact.Addr(), tx.req)
|
||||
if err != nil {
|
||||
if !strings.Contains(err.Error(), "use of closed network connection") { // this only happens on localhost. real UDP has no connections
|
||||
log.Error("send error: ", err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
select {
|
||||
case res := <-tx.res:
|
||||
ch <- &res
|
||||
return
|
||||
case <-n.grp.Ch():
|
||||
return
|
||||
case <-time.After(udpTimeout):
|
||||
}
|
||||
}
|
||||
|
||||
// notify routing table about a failure to respond
|
||||
n.rt.Fail(tx.contact)
|
||||
}()
|
||||
|
||||
return ch
|
||||
}
|
||||
|
||||
// Send sends a transaction and blocks until the response is available. It returns a response, or nil
|
||||
// if the transaction timed out.
|
||||
func (n *Node) Send(contact Contact, req Request, options ...SendOptions) *Response {
|
||||
return <-n.SendAsync(contact, req, options...)
|
||||
}
|
||||
|
||||
// CountActiveTransactions returns the number of transactions in the manager
|
||||
func (n *Node) CountActiveTransactions() int {
|
||||
n.txLock.Lock()
|
||||
defer n.txLock.Unlock()
|
||||
return len(n.transactions)
|
||||
}
|
||||
|
||||
func (n *Node) startRoutingTableGrooming() {
|
||||
refreshTicker := time.NewTicker(tRefresh / 5) // how often to check for buckets that need to be refreshed
|
||||
for {
|
||||
select {
|
||||
case <-refreshTicker.C:
|
||||
RoutingTableRefresh(n, tRefresh, n.grp.Child())
|
||||
case <-n.grp.Ch():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Store stores a node contact in the node's contact store.
|
||||
func (n *Node) Store(hash bits.Bitmap, c Contact) {
|
||||
n.store.Upsert(hash, c)
|
||||
}
|
||||
|
||||
//AddKnownNode adds a known-good node to the routing table
|
||||
func (n *Node) AddKnownNode(c Contact) {
|
||||
n.rt.Update(c)
|
||||
}
|
||||
@@ -1,338 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/crypto"
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/lbry.go/stop"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/uber-go/atomic"
|
||||
)
|
||||
|
||||
// TODO: iterativeFindValue may be stopping early. if it gets a response with one peer, it should keep going because other nodes may know about more peers that have that blob
|
||||
// TODO: or, it should try a tcp handshake with peers as it finds them, to make sure they are still online and have the blob
|
||||
|
||||
var cfLog *logrus.Logger
|
||||
|
||||
func init() {
|
||||
cfLog = logrus.StandardLogger()
|
||||
}
|
||||
|
||||
func NodeFinderUseLogger(l *logrus.Logger) {
|
||||
cfLog = l
|
||||
}
|
||||
|
||||
type contactFinder struct {
|
||||
findValue bool // true if we're using findValue
|
||||
target bits.Bitmap
|
||||
node *Node
|
||||
|
||||
grp *stop.Group
|
||||
|
||||
findValueMutex *sync.Mutex
|
||||
findValueResult []Contact
|
||||
|
||||
activeContactsMutex *sync.Mutex
|
||||
activeContacts []Contact
|
||||
|
||||
shortlistMutex *sync.Mutex
|
||||
shortlist []Contact
|
||||
shortlistAdded map[bits.Bitmap]bool
|
||||
|
||||
closestContactMutex *sync.RWMutex
|
||||
closestContact *Contact
|
||||
notGettingCloser *atomic.Bool
|
||||
}
|
||||
|
||||
func FindContacts(node *Node, target bits.Bitmap, findValue bool, parentGrp *stop.Group) ([]Contact, bool, error) {
|
||||
cf := &contactFinder{
|
||||
node: node,
|
||||
target: target,
|
||||
findValue: findValue,
|
||||
findValueMutex: &sync.Mutex{},
|
||||
activeContactsMutex: &sync.Mutex{},
|
||||
shortlistMutex: &sync.Mutex{},
|
||||
shortlistAdded: make(map[bits.Bitmap]bool),
|
||||
grp: stop.New(parentGrp),
|
||||
closestContactMutex: &sync.RWMutex{},
|
||||
notGettingCloser: atomic.NewBool(false),
|
||||
}
|
||||
|
||||
return cf.Find()
|
||||
}
|
||||
|
||||
func (cf *contactFinder) Stop() {
|
||||
cf.grp.StopAndWait()
|
||||
}
|
||||
|
||||
func (cf *contactFinder) Find() ([]Contact, bool, error) {
|
||||
if cf.findValue {
|
||||
cf.debug("starting iterativeFindValue")
|
||||
} else {
|
||||
cf.debug("starting iterativeFindNode")
|
||||
}
|
||||
|
||||
cf.appendNewToShortlist(cf.node.rt.GetClosest(cf.target, alpha))
|
||||
if len(cf.shortlist) == 0 {
|
||||
return nil, false, errors.Err("[%s] find %s: no contacts in routing table", cf.node.id.HexShort(), cf.target.HexShort())
|
||||
}
|
||||
|
||||
go cf.cycle(false)
|
||||
timeout := 5 * time.Second
|
||||
CycleLoop:
|
||||
for {
|
||||
select {
|
||||
case <-time.After(timeout):
|
||||
go cf.cycle(false)
|
||||
case <-cf.grp.Ch():
|
||||
break CycleLoop
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: what to do if we have less than K active contacts, shortlist is empty, but we have other contacts in our routing table whom we have not contacted. prolly contact them
|
||||
|
||||
var contacts []Contact
|
||||
var found bool
|
||||
if cf.findValue && len(cf.findValueResult) > 0 {
|
||||
contacts = cf.findValueResult
|
||||
found = true
|
||||
} else {
|
||||
contacts = cf.activeContacts
|
||||
if len(contacts) > bucketSize {
|
||||
contacts = contacts[:bucketSize]
|
||||
}
|
||||
}
|
||||
|
||||
cf.Stop()
|
||||
return contacts, found, nil
|
||||
}
|
||||
|
||||
// cycle does a single cycle of sending alpha probes and checking results against closestNode
|
||||
func (cf *contactFinder) cycle(bigCycle bool) {
|
||||
cycleID := crypto.RandString(6)
|
||||
if bigCycle {
|
||||
cf.debug("LAUNCHING CYCLE %s, AND ITS A BIG CYCLE", cycleID)
|
||||
} else {
|
||||
cf.debug("LAUNCHING CYCLE %s", cycleID)
|
||||
}
|
||||
defer cf.debug("CYCLE %s DONE", cycleID)
|
||||
|
||||
cf.closestContactMutex.RLock()
|
||||
closestContact := cf.closestContact
|
||||
cf.closestContactMutex.RUnlock()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
ch := make(chan *Contact)
|
||||
|
||||
limit := alpha
|
||||
if bigCycle {
|
||||
limit = bucketSize
|
||||
}
|
||||
|
||||
for i := 0; i < limit; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
ch <- cf.probe(cycleID)
|
||||
}()
|
||||
}
|
||||
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(ch)
|
||||
}()
|
||||
|
||||
foundCloser := false
|
||||
for {
|
||||
c, more := <-ch
|
||||
if !more {
|
||||
break
|
||||
}
|
||||
if c != nil && (closestContact == nil || cf.target.Closer(c.ID, closestContact.ID)) {
|
||||
if closestContact != nil {
|
||||
cf.debug("|%s| best contact improved: %s -> %s", cycleID, closestContact.ID.HexShort(), c.ID.HexShort())
|
||||
} else {
|
||||
cf.debug("|%s| best contact starting at %s", cycleID, c.ID.HexShort())
|
||||
}
|
||||
foundCloser = true
|
||||
closestContact = c
|
||||
}
|
||||
}
|
||||
|
||||
if cf.isSearchFinished() {
|
||||
cf.grp.Stop()
|
||||
return
|
||||
}
|
||||
|
||||
if foundCloser {
|
||||
cf.closestContactMutex.Lock()
|
||||
// have to check again after locking in case other probes found a closer one in the meantime
|
||||
if cf.closestContact == nil || cf.target.Closer(closestContact.ID, cf.closestContact.ID) {
|
||||
cf.closestContact = closestContact
|
||||
}
|
||||
cf.closestContactMutex.Unlock()
|
||||
go cf.cycle(false)
|
||||
} else if !bigCycle {
|
||||
cf.debug("|%s| no improvement, running big cycle", cycleID)
|
||||
go cf.cycle(true)
|
||||
} else {
|
||||
// big cycle ran and there was no improvement, so we're done
|
||||
cf.debug("|%s| big cycle ran, still no improvement", cycleID)
|
||||
cf.notGettingCloser.Store(true)
|
||||
}
|
||||
}
|
||||
|
||||
// probe sends a single probe, updates the lists, and returns the closest contact it found
|
||||
func (cf *contactFinder) probe(cycleID string) *Contact {
|
||||
maybeContact := cf.popFromShortlist()
|
||||
if maybeContact == nil {
|
||||
cf.debug("|%s| no contacts in shortlist, returning", cycleID)
|
||||
return nil
|
||||
}
|
||||
|
||||
c := *maybeContact
|
||||
|
||||
if c.ID.Equals(cf.node.id) {
|
||||
return nil
|
||||
}
|
||||
|
||||
cf.debug("|%s| probe %s: launching", cycleID, c.ID.HexShort())
|
||||
|
||||
req := Request{Arg: &cf.target}
|
||||
if cf.findValue {
|
||||
req.Method = findValueMethod
|
||||
} else {
|
||||
req.Method = findNodeMethod
|
||||
}
|
||||
|
||||
var res *Response
|
||||
resCh := cf.node.SendAsync(c, req)
|
||||
select {
|
||||
case res = <-resCh:
|
||||
case <-cf.grp.Ch():
|
||||
cf.debug("|%s| probe %s: canceled", cycleID, c.ID.HexShort())
|
||||
return nil
|
||||
}
|
||||
|
||||
if res == nil {
|
||||
cf.debug("|%s| probe %s: req canceled or timed out", cycleID, c.ID.HexShort())
|
||||
return nil
|
||||
}
|
||||
|
||||
if cf.findValue && res.FindValueKey != "" {
|
||||
cf.debug("|%s| probe %s: got value", cycleID, c.ID.HexShort())
|
||||
cf.findValueMutex.Lock()
|
||||
cf.findValueResult = res.Contacts
|
||||
cf.findValueMutex.Unlock()
|
||||
cf.grp.Stop()
|
||||
return nil
|
||||
}
|
||||
|
||||
cf.debug("|%s| probe %s: got %s", cycleID, c.ID.HexShort(), res.argsDebug())
|
||||
cf.insertIntoActiveList(c)
|
||||
cf.appendNewToShortlist(res.Contacts)
|
||||
|
||||
cf.activeContactsMutex.Lock()
|
||||
contacts := cf.activeContacts
|
||||
if len(contacts) > bucketSize {
|
||||
contacts = contacts[:bucketSize]
|
||||
}
|
||||
contactsStr := ""
|
||||
for _, c := range contacts {
|
||||
contactsStr += c.ID.HexShort() + ", "
|
||||
}
|
||||
cf.activeContactsMutex.Unlock()
|
||||
|
||||
return cf.closest(res.Contacts...)
|
||||
}
|
||||
|
||||
// appendNewToShortlist appends any new contacts to the shortlist and sorts it by distance
|
||||
// contacts that have already been added to the shortlist in the past are ignored
|
||||
func (cf *contactFinder) appendNewToShortlist(contacts []Contact) {
|
||||
cf.shortlistMutex.Lock()
|
||||
defer cf.shortlistMutex.Unlock()
|
||||
|
||||
for _, c := range contacts {
|
||||
if _, ok := cf.shortlistAdded[c.ID]; !ok {
|
||||
cf.shortlist = append(cf.shortlist, c)
|
||||
cf.shortlistAdded[c.ID] = true
|
||||
}
|
||||
}
|
||||
|
||||
sortByDistance(cf.shortlist, cf.target)
|
||||
}
|
||||
|
||||
// popFromShortlist pops the first contact off the shortlist and returns it
|
||||
func (cf *contactFinder) popFromShortlist() *Contact {
|
||||
cf.shortlistMutex.Lock()
|
||||
defer cf.shortlistMutex.Unlock()
|
||||
|
||||
if len(cf.shortlist) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
first := cf.shortlist[0]
|
||||
cf.shortlist = cf.shortlist[1:]
|
||||
return &first
|
||||
}
|
||||
|
||||
// insertIntoActiveList inserts the contact into appropriate place in the list of active contacts (sorted by distance)
|
||||
func (cf *contactFinder) insertIntoActiveList(contact Contact) {
|
||||
cf.activeContactsMutex.Lock()
|
||||
defer cf.activeContactsMutex.Unlock()
|
||||
|
||||
inserted := false
|
||||
for i, n := range cf.activeContacts {
|
||||
if cf.target.Closer(contact.ID, n.ID) {
|
||||
cf.activeContacts = append(cf.activeContacts[:i], append([]Contact{contact}, cf.activeContacts[i:]...)...)
|
||||
inserted = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !inserted {
|
||||
cf.activeContacts = append(cf.activeContacts, contact)
|
||||
}
|
||||
}
|
||||
|
||||
// isSearchFinished returns true if the search is done and should be stopped
|
||||
func (cf *contactFinder) isSearchFinished() bool {
|
||||
if cf.findValue && len(cf.findValueResult) > 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
select {
|
||||
case <-cf.grp.Ch():
|
||||
return true
|
||||
default:
|
||||
}
|
||||
|
||||
if cf.notGettingCloser.Load() {
|
||||
return true
|
||||
}
|
||||
|
||||
cf.activeContactsMutex.Lock()
|
||||
defer cf.activeContactsMutex.Unlock()
|
||||
return len(cf.activeContacts) >= bucketSize
|
||||
}
|
||||
|
||||
func (cf *contactFinder) debug(format string, args ...interface{}) {
|
||||
args = append([]interface{}{cf.node.id.HexShort()}, append([]interface{}{cf.target.HexShort()}, args...)...)
|
||||
cfLog.Debugf("[%s] find %s: "+format, args...)
|
||||
}
|
||||
|
||||
func (cf *contactFinder) closest(contacts ...Contact) *Contact {
|
||||
if len(contacts) == 0 {
|
||||
return nil
|
||||
}
|
||||
closest := contacts[0]
|
||||
for _, c := range contacts {
|
||||
if cf.target.Closer(c.ID, closest.ID) {
|
||||
closest = c
|
||||
}
|
||||
}
|
||||
return &closest
|
||||
}
|
||||
@@ -1,422 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"net"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
"github.com/lyoshenka/bencode"
|
||||
)
|
||||
|
||||
func TestPing(t *testing.T) {
|
||||
dhtNodeID := bits.Rand()
|
||||
testNodeID := bits.Rand()
|
||||
|
||||
conn := newTestUDPConn("127.0.0.1:21217")
|
||||
|
||||
dht := New(&Config{Address: "127.0.0.1:21216", NodeID: dhtNodeID.Hex()})
|
||||
|
||||
err := dht.connect(conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer dht.Shutdown()
|
||||
|
||||
messageID := newMessageID()
|
||||
|
||||
data, err := bencode.EncodeBytes(map[string]interface{}{
|
||||
headerTypeField: requestType,
|
||||
headerMessageIDField: messageID,
|
||||
headerNodeIDField: testNodeID.RawString(),
|
||||
headerPayloadField: "ping",
|
||||
headerArgsField: []string{},
|
||||
})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
conn.toRead <- testUDPPacket{addr: conn.addr, data: data}
|
||||
timer := time.NewTimer(3 * time.Second)
|
||||
|
||||
select {
|
||||
case <-timer.C:
|
||||
t.Error("timeout")
|
||||
case resp := <-conn.writes:
|
||||
var response map[string]interface{}
|
||||
err := bencode.DecodeBytes(resp.data, &response)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(response) != 4 {
|
||||
t.Errorf("expected 4 response fields, got %d", len(response))
|
||||
}
|
||||
|
||||
_, ok := response[headerTypeField]
|
||||
if !ok {
|
||||
t.Error("missing type field")
|
||||
} else {
|
||||
rType, ok := response[headerTypeField].(int64)
|
||||
if !ok {
|
||||
t.Error("type is not an integer")
|
||||
} else if rType != responseType {
|
||||
t.Error("unexpected response type")
|
||||
}
|
||||
}
|
||||
|
||||
_, ok = response[headerMessageIDField]
|
||||
if !ok {
|
||||
t.Error("missing message id field")
|
||||
} else {
|
||||
rMessageID, ok := response[headerMessageIDField].(string)
|
||||
if !ok {
|
||||
t.Error("message ID is not a string")
|
||||
} else if rMessageID != string(messageID[:]) {
|
||||
t.Error("unexpected message ID")
|
||||
}
|
||||
}
|
||||
|
||||
_, ok = response[headerNodeIDField]
|
||||
if !ok {
|
||||
t.Error("missing node id field")
|
||||
} else {
|
||||
rNodeID, ok := response[headerNodeIDField].(string)
|
||||
if !ok {
|
||||
t.Error("node ID is not a string")
|
||||
} else if rNodeID != dhtNodeID.RawString() {
|
||||
t.Error("unexpected node ID")
|
||||
}
|
||||
}
|
||||
|
||||
_, ok = response[headerPayloadField]
|
||||
if !ok {
|
||||
t.Error("missing payload field")
|
||||
} else {
|
||||
rNodeID, ok := response[headerPayloadField].(string)
|
||||
if !ok {
|
||||
t.Error("payload is not a string")
|
||||
} else if rNodeID != pingSuccessResponse {
|
||||
t.Error("did not pong")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStore(t *testing.T) {
|
||||
dhtNodeID := bits.Rand()
|
||||
testNodeID := bits.Rand()
|
||||
|
||||
conn := newTestUDPConn("127.0.0.1:21217")
|
||||
|
||||
dht := New(&Config{Address: "127.0.0.1:21216", NodeID: dhtNodeID.Hex()})
|
||||
|
||||
err := dht.connect(conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer dht.Shutdown()
|
||||
|
||||
messageID := newMessageID()
|
||||
blobHashToStore := bits.Rand()
|
||||
|
||||
storeRequest := Request{
|
||||
ID: messageID,
|
||||
NodeID: testNodeID,
|
||||
Method: storeMethod,
|
||||
StoreArgs: &storeArgs{
|
||||
BlobHash: blobHashToStore,
|
||||
Value: storeArgsValue{
|
||||
Token: dht.node.tokens.Get(testNodeID, conn.addr),
|
||||
LbryID: testNodeID,
|
||||
Port: 9999,
|
||||
},
|
||||
NodeID: testNodeID,
|
||||
},
|
||||
}
|
||||
|
||||
_ = "64 " + // start message
|
||||
"313A30 693065" + // type: 0
|
||||
"313A31 3230 3A 6EB490B5788B63F0F7E6D92352024D0CBDEC2D3A" + // message id
|
||||
"313A32 3438 3A 7CE1B831DEC8689E44F80F547D2DEA171F6A625E1A4FF6C6165E645F953103DABEB068A622203F859C6C64658FD3AA3B" + // node id
|
||||
"313A33 35 3A 73746F7265" + // method
|
||||
"313A34 6C" + // start args list
|
||||
"3438 3A 3214D6C2F77FCB5E8D5FC07EDAFBA614F031CE8B2EAB49F924F8143F6DFBADE048D918710072FB98AB1B52B58F4E1468" + // block hash
|
||||
"64" + // start value dict
|
||||
"363A6C6272796964 3438 3A 7CE1B831DEC8689E44F80F547D2DEA171F6A625E1A4FF6C6165E645F953103DABEB068A622203F859C6C64658FD3AA3B" + // lbry id
|
||||
"343A706F7274 69 33333333 65" + // port
|
||||
"353A746F6B656E 3438 3A 17C2D8E1E48EF21567FE4AD5C8ED944B798D3B65AB58D0C9122AD6587D1B5FED472EA2CB12284CEFA1C21EFF302322BD" + // token
|
||||
"65" + // end value dict
|
||||
"3438 3A 7CE1B831DEC8689E44F80F547D2DEA171F6A625E1A4FF6C6165E645F953103DABEB068A622203F859C6C64658FD3AA3B" + // node id
|
||||
"693065" + // self store (integer)
|
||||
"65" + // end args list
|
||||
"65" // end message
|
||||
|
||||
data, err := bencode.EncodeBytes(storeRequest)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
conn.toRead <- testUDPPacket{addr: conn.addr, data: data}
|
||||
timer := time.NewTimer(3 * time.Second)
|
||||
|
||||
var response map[string]interface{}
|
||||
select {
|
||||
case <-timer.C:
|
||||
t.Fatal("timeout")
|
||||
case resp := <-conn.writes:
|
||||
err := bencode.DecodeBytes(resp.data, &response)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
verifyResponse(t, response, messageID, dhtNodeID.RawString())
|
||||
|
||||
_, ok := response[headerPayloadField]
|
||||
if !ok {
|
||||
t.Error("missing payload field")
|
||||
} else {
|
||||
rNodeID, ok := response[headerPayloadField].(string)
|
||||
if !ok {
|
||||
t.Error("payload is not a string")
|
||||
} else if rNodeID != storeSuccessResponse {
|
||||
t.Error("did not return OK")
|
||||
}
|
||||
}
|
||||
|
||||
if dht.node.store.CountStoredHashes() != 1 {
|
||||
t.Error("dht store has wrong number of items")
|
||||
}
|
||||
|
||||
items := dht.node.store.Get(blobHashToStore)
|
||||
if len(items) != 1 {
|
||||
t.Error("list created in store, but nothing in list")
|
||||
}
|
||||
if !items[0].ID.Equals(testNodeID) {
|
||||
t.Error("wrong value stored")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindNode(t *testing.T) {
|
||||
dhtNodeID := bits.Rand()
|
||||
testNodeID := bits.Rand()
|
||||
|
||||
conn := newTestUDPConn("127.0.0.1:21217")
|
||||
|
||||
dht := New(&Config{Address: "127.0.0.1:21216", NodeID: dhtNodeID.Hex()})
|
||||
|
||||
err := dht.connect(conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer dht.Shutdown()
|
||||
|
||||
nodesToInsert := 3
|
||||
var nodes []Contact
|
||||
for i := 0; i < nodesToInsert; i++ {
|
||||
n := Contact{ID: bits.Rand(), IP: net.ParseIP("127.0.0.1"), Port: 10000 + i}
|
||||
nodes = append(nodes, n)
|
||||
dht.node.rt.Update(n)
|
||||
}
|
||||
|
||||
messageID := newMessageID()
|
||||
blobHashToFind := bits.Rand()
|
||||
|
||||
request := Request{
|
||||
ID: messageID,
|
||||
NodeID: testNodeID,
|
||||
Method: findNodeMethod,
|
||||
Arg: &blobHashToFind,
|
||||
}
|
||||
|
||||
data, err := bencode.EncodeBytes(request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
conn.toRead <- testUDPPacket{addr: conn.addr, data: data}
|
||||
timer := time.NewTimer(3 * time.Second)
|
||||
|
||||
var response map[string]interface{}
|
||||
select {
|
||||
case <-timer.C:
|
||||
t.Fatal("timeout")
|
||||
case resp := <-conn.writes:
|
||||
err := bencode.DecodeBytes(resp.data, &response)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
verifyResponse(t, response, messageID, dhtNodeID.RawString())
|
||||
|
||||
_, ok := response[headerPayloadField]
|
||||
if !ok {
|
||||
t.Fatal("missing payload field")
|
||||
}
|
||||
|
||||
contacts, ok := response[headerPayloadField].([]interface{})
|
||||
if !ok {
|
||||
t.Fatal("payload is not a list")
|
||||
}
|
||||
|
||||
verifyContacts(t, contacts, nodes)
|
||||
}
|
||||
|
||||
func TestFindValueExisting(t *testing.T) {
|
||||
dhtNodeID := bits.Rand()
|
||||
testNodeID := bits.Rand()
|
||||
|
||||
conn := newTestUDPConn("127.0.0.1:21217")
|
||||
|
||||
dht := New(&Config{Address: "127.0.0.1:21216", NodeID: dhtNodeID.Hex()})
|
||||
|
||||
err := dht.connect(conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer dht.Shutdown()
|
||||
|
||||
nodesToInsert := 3
|
||||
for i := 0; i < nodesToInsert; i++ {
|
||||
n := Contact{ID: bits.Rand(), IP: net.ParseIP("127.0.0.1"), Port: 10000 + i}
|
||||
dht.node.rt.Update(n)
|
||||
}
|
||||
|
||||
//data, _ := hex.DecodeString("64313a30693065313a3132303a7de8e57d34e316abbb5a8a8da50dcd1ad4c80e0f313a3234383a7ce1b831dec8689e44f80f547d2dea171f6a625e1a4ff6c6165e645f953103dabeb068a622203f859c6c64658fd3aa3b313a33393a66696e6456616c7565313a346c34383aa47624b8e7ee1e54df0c45e2eb858feb0b705bd2a78d8b739be31ba188f4bd6f56b371c51fecc5280d5fd26ba4168e966565")
|
||||
|
||||
messageID := newMessageID()
|
||||
valueToFind := bits.Rand()
|
||||
|
||||
nodeToFind := Contact{ID: bits.Rand(), IP: net.ParseIP("1.2.3.4"), PeerPort: 1286}
|
||||
dht.node.store.Upsert(valueToFind, nodeToFind)
|
||||
dht.node.store.Upsert(valueToFind, nodeToFind)
|
||||
dht.node.store.Upsert(valueToFind, nodeToFind)
|
||||
|
||||
request := Request{
|
||||
ID: messageID,
|
||||
NodeID: testNodeID,
|
||||
Method: findValueMethod,
|
||||
Arg: &valueToFind,
|
||||
}
|
||||
|
||||
data, err := bencode.EncodeBytes(request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
conn.toRead <- testUDPPacket{addr: conn.addr, data: data}
|
||||
timer := time.NewTimer(3 * time.Second)
|
||||
|
||||
var response map[string]interface{}
|
||||
select {
|
||||
case <-timer.C:
|
||||
t.Fatal("timeout")
|
||||
case resp := <-conn.writes:
|
||||
err := bencode.DecodeBytes(resp.data, &response)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
verifyResponse(t, response, messageID, dhtNodeID.RawString())
|
||||
|
||||
_, ok := response[headerPayloadField]
|
||||
if !ok {
|
||||
t.Fatal("missing payload field")
|
||||
}
|
||||
|
||||
payload, ok := response[headerPayloadField].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatal("payload is not a dictionary")
|
||||
}
|
||||
|
||||
compactContacts, ok := payload[valueToFind.RawString()]
|
||||
if !ok {
|
||||
t.Fatal("payload is missing key for search value")
|
||||
}
|
||||
|
||||
contacts, ok := compactContacts.([]interface{})
|
||||
if !ok {
|
||||
t.Fatal("search results are not a list")
|
||||
}
|
||||
|
||||
verifyCompactContacts(t, contacts, []Contact{nodeToFind})
|
||||
}
|
||||
|
||||
func TestFindValueFallbackToFindNode(t *testing.T) {
|
||||
dhtNodeID := bits.Rand()
|
||||
testNodeID := bits.Rand()
|
||||
|
||||
conn := newTestUDPConn("127.0.0.1:21217")
|
||||
|
||||
dht := New(&Config{Address: "127.0.0.1:21216", NodeID: dhtNodeID.Hex()})
|
||||
|
||||
err := dht.connect(conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer dht.Shutdown()
|
||||
|
||||
nodesToInsert := 3
|
||||
var nodes []Contact
|
||||
for i := 0; i < nodesToInsert; i++ {
|
||||
n := Contact{ID: bits.Rand(), IP: net.ParseIP("127.0.0.1"), Port: 10000 + i}
|
||||
nodes = append(nodes, n)
|
||||
dht.node.rt.Update(n)
|
||||
}
|
||||
|
||||
messageID := newMessageID()
|
||||
valueToFind := bits.Rand()
|
||||
|
||||
request := Request{
|
||||
ID: messageID,
|
||||
NodeID: testNodeID,
|
||||
Method: findValueMethod,
|
||||
Arg: &valueToFind,
|
||||
}
|
||||
|
||||
data, err := bencode.EncodeBytes(request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
conn.toRead <- testUDPPacket{addr: conn.addr, data: data}
|
||||
timer := time.NewTimer(3 * time.Second)
|
||||
|
||||
var response map[string]interface{}
|
||||
select {
|
||||
case <-timer.C:
|
||||
t.Fatal("timeout")
|
||||
case resp := <-conn.writes:
|
||||
err := bencode.DecodeBytes(resp.data, &response)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
verifyResponse(t, response, messageID, dhtNodeID.RawString())
|
||||
|
||||
_, ok := response[headerPayloadField]
|
||||
if !ok {
|
||||
t.Fatal("missing payload field")
|
||||
}
|
||||
|
||||
payload, ok := response[headerPayloadField].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatal("payload is not a dictionary")
|
||||
}
|
||||
|
||||
contactsList, ok := payload[contactsField]
|
||||
if !ok {
|
||||
t.Fatal("payload is missing 'contacts' key")
|
||||
}
|
||||
|
||||
contacts, ok := contactsList.([]interface{})
|
||||
if !ok {
|
||||
t.Fatal("'contacts' is not a list")
|
||||
}
|
||||
|
||||
verifyContacts(t, contacts, nodes)
|
||||
}
|
||||
@@ -1,463 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/lbry.go/stop"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
)
|
||||
|
||||
// TODO: if routing table is ever empty (aka the node is isolated), it should re-bootstrap
|
||||
|
||||
// TODO: use a tree with bucket splitting instead of a fixed bucket list. include jack's optimization (see link in commit mesg)
|
||||
// https://github.com/lbryio/lbry/pull/1211/commits/341b27b6d21ac027671d42458826d02735aaae41
|
||||
|
||||
// peer is a contact with extra information
|
||||
type peer struct {
|
||||
Contact Contact
|
||||
Distance bits.Bitmap
|
||||
LastActivity time.Time
|
||||
// LastReplied time.Time
|
||||
// LastRequested time.Time
|
||||
// LastFailure time.Time
|
||||
// SecondLastFailure time.Time
|
||||
NumFailures int
|
||||
|
||||
//<lastPublished>,
|
||||
//<originallyPublished>
|
||||
// <originalPublisherID>
|
||||
}
|
||||
|
||||
func (p *peer) Touch() {
|
||||
p.LastActivity = time.Now()
|
||||
p.NumFailures = 0
|
||||
}
|
||||
|
||||
// ActiveSince returns whether a peer has responded in the last `d` duration
|
||||
// this is used to check if the peer is "good", meaning that we believe the peer will respond to our requests
|
||||
func (p *peer) ActiveInLast(d time.Duration) bool {
|
||||
return time.Since(p.LastActivity) < d
|
||||
}
|
||||
|
||||
// IsBad returns whether a peer is "bad", meaning that it has failed to respond to multiple pings in a row
|
||||
func (p *peer) IsBad(maxFalures int) bool {
|
||||
return p.NumFailures >= maxFalures
|
||||
}
|
||||
|
||||
// Fail marks a peer as having failed to respond. It returns whether or not the peer should be removed from the routing table
|
||||
func (p *peer) Fail() {
|
||||
p.NumFailures++
|
||||
}
|
||||
|
||||
type bucket struct {
|
||||
lock *sync.RWMutex
|
||||
peers []peer
|
||||
lastUpdate time.Time
|
||||
Range bits.Range // capitalized because `range` is a keyword
|
||||
}
|
||||
|
||||
func newBucket(r bits.Range) *bucket {
|
||||
return &bucket{
|
||||
peers: make([]peer, 0, bucketSize),
|
||||
lock: &sync.RWMutex{},
|
||||
Range: r,
|
||||
}
|
||||
}
|
||||
|
||||
// Len returns the number of peers in the bucket
|
||||
func (b bucket) Len() int {
|
||||
b.lock.RLock()
|
||||
defer b.lock.RUnlock()
|
||||
return len(b.peers)
|
||||
}
|
||||
|
||||
func (b bucket) Has(c Contact) bool {
|
||||
b.lock.RLock()
|
||||
defer b.lock.RUnlock()
|
||||
for _, p := range b.peers {
|
||||
if p.Contact.Equals(c, true) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Contacts returns a slice of the bucket's contacts
|
||||
func (b bucket) Contacts() []Contact {
|
||||
b.lock.RLock()
|
||||
defer b.lock.RUnlock()
|
||||
contacts := make([]Contact, len(b.peers))
|
||||
for i := range b.peers {
|
||||
contacts[i] = b.peers[i].Contact
|
||||
}
|
||||
return contacts
|
||||
}
|
||||
|
||||
// UpdatePeer marks a contact as having been successfully contacted. if insertIfNew and the contact is does not exist yet, it is inserted
|
||||
func (b *bucket) UpdatePeer(p peer, insertIfNew bool) error {
|
||||
b.lock.Lock()
|
||||
defer b.lock.Unlock()
|
||||
|
||||
if !b.Range.Contains(p.Distance) {
|
||||
return errors.Err("this bucket range does not cover this peer")
|
||||
}
|
||||
|
||||
peerIndex := find(p.Contact.ID, b.peers)
|
||||
if peerIndex >= 0 {
|
||||
b.lastUpdate = time.Now()
|
||||
b.peers[peerIndex].Touch()
|
||||
moveToBack(b.peers, peerIndex)
|
||||
} else if insertIfNew {
|
||||
hasRoom := true
|
||||
|
||||
if len(b.peers) >= bucketSize {
|
||||
hasRoom = false
|
||||
for i := range b.peers {
|
||||
if b.peers[i].IsBad(maxPeerFails) {
|
||||
// TODO: Ping contact first. Only remove if it does not respond
|
||||
b.peers = append(b.peers[:i], b.peers[i+1:]...)
|
||||
hasRoom = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if hasRoom {
|
||||
b.lastUpdate = time.Now()
|
||||
p.Touch()
|
||||
b.peers = append(b.peers, p)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// FailContact marks a contact as having failed, and removes it if it failed too many times
|
||||
func (b *bucket) FailContact(id bits.Bitmap) {
|
||||
b.lock.Lock()
|
||||
defer b.lock.Unlock()
|
||||
i := find(id, b.peers)
|
||||
if i >= 0 {
|
||||
// BEP5 says not to remove the contact until the bucket is full and you try to insert
|
||||
b.peers[i].Fail()
|
||||
}
|
||||
}
|
||||
|
||||
// find returns the contact in the bucket, or nil if the bucket does not contain the contact
|
||||
func find(id bits.Bitmap, peers []peer) int {
|
||||
for i := range peers {
|
||||
if peers[i].Contact.ID.Equals(id) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// NeedsRefresh returns true if bucket has not been updated in the last `refreshInterval`, false otherwise
|
||||
func (b *bucket) NeedsRefresh(refreshInterval time.Duration) bool {
|
||||
b.lock.RLock()
|
||||
defer b.lock.RUnlock()
|
||||
return time.Since(b.lastUpdate) > refreshInterval
|
||||
}
|
||||
|
||||
func (b *bucket) Split() (*bucket, *bucket) {
|
||||
b.lock.Lock()
|
||||
defer b.lock.Unlock()
|
||||
|
||||
left := newBucket(b.Range.IntervalP(1, 2))
|
||||
right := newBucket(b.Range.IntervalP(2, 2))
|
||||
left.lastUpdate = b.lastUpdate
|
||||
right.lastUpdate = b.lastUpdate
|
||||
|
||||
for _, p := range b.peers {
|
||||
if left.Range.Contains(p.Distance) {
|
||||
left.peers = append(left.peers, p)
|
||||
} else {
|
||||
right.peers = append(right.peers, p)
|
||||
}
|
||||
}
|
||||
|
||||
if len(b.peers) > 1 {
|
||||
if len(left.peers) == 0 {
|
||||
left, right = right.Split()
|
||||
left.Range.Start = b.Range.Start
|
||||
} else if len(right.peers) == 0 {
|
||||
left, right = left.Split()
|
||||
right.Range.End = b.Range.End
|
||||
}
|
||||
}
|
||||
|
||||
return left, right
|
||||
}
|
||||
|
||||
type routingTable struct {
|
||||
id bits.Bitmap
|
||||
buckets []*bucket
|
||||
mu *sync.RWMutex // this mutex is write-locked only when CHANGING THE NUMBER OF BUCKETS in the table
|
||||
}
|
||||
|
||||
func newRoutingTable(id bits.Bitmap) *routingTable {
|
||||
rt := routingTable{
|
||||
id: id,
|
||||
mu: &sync.RWMutex{},
|
||||
}
|
||||
rt.reset()
|
||||
return &rt
|
||||
}
|
||||
|
||||
func (rt *routingTable) reset() {
|
||||
rt.mu.Lock()
|
||||
defer rt.mu.Unlock()
|
||||
rt.buckets = []*bucket{newBucket(bits.MaxRange())}
|
||||
}
|
||||
|
||||
func (rt *routingTable) BucketInfo() string {
|
||||
rt.mu.RLock()
|
||||
defer rt.mu.RUnlock()
|
||||
|
||||
var bucketInfo []string
|
||||
for i, b := range rt.buckets {
|
||||
if b.Len() > 0 {
|
||||
contacts := b.Contacts()
|
||||
s := make([]string, len(contacts))
|
||||
for j, c := range contacts {
|
||||
s[j] = c.ID.HexShort()
|
||||
}
|
||||
bucketInfo = append(bucketInfo, fmt.Sprintf("bucket %d: (%d) %s", i, len(contacts), strings.Join(s, ", ")))
|
||||
}
|
||||
}
|
||||
if len(bucketInfo) == 0 {
|
||||
return "buckets are empty"
|
||||
}
|
||||
return strings.Join(bucketInfo, "\n")
|
||||
}
|
||||
|
||||
// Update inserts or refreshes a contact
|
||||
func (rt *routingTable) Update(c Contact) {
|
||||
rt.mu.Lock() // write lock, because updates may cause bucket splits
|
||||
defer rt.mu.Unlock()
|
||||
|
||||
b := rt.bucketFor(c.ID)
|
||||
|
||||
if rt.shouldSplit(b, c) {
|
||||
left, right := b.Split()
|
||||
|
||||
for i := range rt.buckets {
|
||||
if rt.buckets[i].Range.Start.Equals(left.Range.Start) {
|
||||
rt.buckets = append(rt.buckets[:i], append([]*bucket{left, right}, rt.buckets[i+1:]...)...)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if left.Range.Contains(c.ID) {
|
||||
b = left
|
||||
} else {
|
||||
b = right
|
||||
}
|
||||
}
|
||||
|
||||
err := b.UpdatePeer(peer{Contact: c, Distance: rt.id.Xor(c.ID)}, true)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Fresh refreshes a contact if its already in the routing table
|
||||
func (rt *routingTable) Fresh(c Contact) {
|
||||
rt.mu.RLock()
|
||||
defer rt.mu.RUnlock()
|
||||
err := rt.bucketFor(c.ID).UpdatePeer(peer{Contact: c, Distance: rt.id.Xor(c.ID)}, false)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
// FailContact marks a contact as having failed, and removes it if it failed too many times
|
||||
func (rt *routingTable) Fail(c Contact) {
|
||||
rt.mu.RLock()
|
||||
defer rt.mu.RUnlock()
|
||||
rt.bucketFor(c.ID).FailContact(c.ID)
|
||||
}
|
||||
|
||||
// GetClosest returns the closest `limit` contacts from the routing table.
|
||||
// This is a locking wrapper around getClosest()
|
||||
func (rt *routingTable) GetClosest(target bits.Bitmap, limit int) []Contact {
|
||||
rt.mu.RLock()
|
||||
defer rt.mu.RUnlock()
|
||||
return rt.getClosest(target, limit)
|
||||
}
|
||||
|
||||
// getClosest returns the closest `limit` contacts from the routing table
|
||||
func (rt *routingTable) getClosest(target bits.Bitmap, limit int) []Contact {
|
||||
var contacts []Contact
|
||||
for _, b := range rt.buckets {
|
||||
contacts = append(contacts, b.Contacts()...)
|
||||
}
|
||||
|
||||
sortByDistance(contacts, target)
|
||||
if len(contacts) > limit {
|
||||
contacts = contacts[:limit]
|
||||
}
|
||||
|
||||
return contacts
|
||||
}
|
||||
|
||||
// Count returns the number of contacts in the routing table
|
||||
func (rt *routingTable) Count() int {
|
||||
rt.mu.RLock()
|
||||
defer rt.mu.RUnlock()
|
||||
count := 0
|
||||
for _, bucket := range rt.buckets {
|
||||
count += bucket.Len()
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
// Len returns the number of buckets in the routing table
|
||||
func (rt *routingTable) Len() int {
|
||||
rt.mu.RLock()
|
||||
defer rt.mu.RUnlock()
|
||||
return len(rt.buckets)
|
||||
}
|
||||
|
||||
func (rt *routingTable) bucketFor(target bits.Bitmap) *bucket {
|
||||
if rt.id.Equals(target) {
|
||||
panic("routing table does not have a bucket for its own id")
|
||||
}
|
||||
distance := target.Xor(rt.id)
|
||||
for _, b := range rt.buckets {
|
||||
if b.Range.Contains(distance) {
|
||||
return b
|
||||
}
|
||||
}
|
||||
panic("target is not contained in any buckets")
|
||||
}
|
||||
|
||||
func (rt *routingTable) shouldSplit(b *bucket, c Contact) bool {
|
||||
if b.Has(c) {
|
||||
return false
|
||||
}
|
||||
if b.Len() >= bucketSize {
|
||||
if b.Range.Start.Equals(bits.Bitmap{}) { // this is the bucket covering our node id
|
||||
return true
|
||||
}
|
||||
kClosest := rt.getClosest(rt.id, bucketSize)
|
||||
kthClosest := kClosest[len(kClosest)-1]
|
||||
if rt.id.Closer(c.ID, kthClosest.ID) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
//func (rt *routingTable) printBucketInfo() {
|
||||
// fmt.Printf("there are %d contacts in %d buckets\n", rt.Count(), rt.Len())
|
||||
// for i, b := range rt.buckets {
|
||||
// fmt.Printf("bucket %d, %d contacts\n", i+1, len(b.peers))
|
||||
// fmt.Printf(" start : %s\n", b.Range.Start.String())
|
||||
// fmt.Printf(" stop : %s\n", b.Range.End.String())
|
||||
// fmt.Println("")
|
||||
// }
|
||||
//}
|
||||
|
||||
func (rt *routingTable) GetIDsForRefresh(refreshInterval time.Duration) []bits.Bitmap {
|
||||
var bitmaps []bits.Bitmap
|
||||
for i, bucket := range rt.buckets {
|
||||
if bucket.NeedsRefresh(refreshInterval) {
|
||||
bitmaps = append(bitmaps, bits.Rand().Prefix(i, false))
|
||||
}
|
||||
}
|
||||
return bitmaps
|
||||
}
|
||||
|
||||
const rtContactSep = "-"
|
||||
|
||||
type rtSave struct {
|
||||
ID string `json:"id"`
|
||||
Contacts []string `json:"contacts"`
|
||||
}
|
||||
|
||||
func (rt *routingTable) MarshalJSON() ([]byte, error) {
|
||||
var data rtSave
|
||||
data.ID = rt.id.Hex()
|
||||
for _, b := range rt.buckets {
|
||||
for _, c := range b.Contacts() {
|
||||
data.Contacts = append(data.Contacts, strings.Join([]string{c.ID.Hex(), c.IP.String(), strconv.Itoa(c.Port)}, rtContactSep))
|
||||
}
|
||||
}
|
||||
return json.Marshal(data)
|
||||
}
|
||||
|
||||
func (rt *routingTable) UnmarshalJSON(b []byte) error {
|
||||
var data rtSave
|
||||
err := json.Unmarshal(b, &data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rt.id, err = bits.FromHex(data.ID)
|
||||
if err != nil {
|
||||
return errors.Prefix("decoding ID", err)
|
||||
}
|
||||
rt.reset()
|
||||
|
||||
for _, s := range data.Contacts {
|
||||
parts := strings.Split(s, rtContactSep)
|
||||
if len(parts) != 3 {
|
||||
return errors.Err("decoding contact %s: wrong number of parts", s)
|
||||
}
|
||||
var c Contact
|
||||
c.ID, err = bits.FromHex(parts[0])
|
||||
if err != nil {
|
||||
return errors.Err("decoding contact %s: invalid ID: %s", s, err)
|
||||
}
|
||||
c.IP = net.ParseIP(parts[1])
|
||||
if c.IP == nil {
|
||||
return errors.Err("decoding contact %s: invalid IP", s)
|
||||
}
|
||||
c.Port, err = strconv.Atoi(parts[2])
|
||||
if err != nil {
|
||||
return errors.Err("decoding contact %s: invalid port: %s", s, err)
|
||||
}
|
||||
rt.Update(c)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// RoutingTableRefresh refreshes any buckets that need to be refreshed
|
||||
func RoutingTableRefresh(n *Node, refreshInterval time.Duration, parentGrp *stop.Group) {
|
||||
done := stop.New()
|
||||
|
||||
for _, id := range n.rt.GetIDsForRefresh(refreshInterval) {
|
||||
done.Add(1)
|
||||
go func(id bits.Bitmap) {
|
||||
defer done.Done()
|
||||
_, _, err := FindContacts(n, id, false, parentGrp)
|
||||
if err != nil {
|
||||
log.Error("error finding contact during routing table refresh - ", err)
|
||||
}
|
||||
}(id)
|
||||
}
|
||||
|
||||
done.Wait()
|
||||
done.Stop()
|
||||
}
|
||||
|
||||
func moveToBack(peers []peer, index int) {
|
||||
if index < 0 || len(peers) <= index+1 {
|
||||
return
|
||||
}
|
||||
p := peers[index]
|
||||
for i := index; i < len(peers)-1; i++ {
|
||||
peers[i] = peers[i+1]
|
||||
}
|
||||
peers[len(peers)-1] = p
|
||||
}
|
||||
@@ -1,328 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"math/big"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
|
||||
"github.com/sebdah/goldie"
|
||||
)
|
||||
|
||||
func TestBucket_Split(t *testing.T) {
|
||||
rt := newRoutingTable(bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"))
|
||||
if len(rt.buckets) != 1 {
|
||||
t.Errorf("there should only be one bucket so far")
|
||||
}
|
||||
if len(rt.buckets[0].peers) != 0 {
|
||||
t.Errorf("there should be no contacts yet")
|
||||
}
|
||||
|
||||
var tests = []struct {
|
||||
name string
|
||||
id bits.Bitmap
|
||||
expectedBucketCount int
|
||||
expectedTotalContacts int
|
||||
}{
|
||||
//fill first bucket
|
||||
{"b1-one", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100"), 1, 1},
|
||||
{"b1-two", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000200"), 1, 2},
|
||||
{"b1-three", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000300"), 1, 3},
|
||||
{"b1-four", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000400"), 1, 4},
|
||||
{"b1-five", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000500"), 1, 5},
|
||||
{"b1-six", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000600"), 1, 6},
|
||||
{"b1-seven", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000700"), 1, 7},
|
||||
{"b1-eight", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800"), 1, 8},
|
||||
|
||||
// split off second bucket and fill it
|
||||
{"b2-one", bits.FromHexP("001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 9},
|
||||
{"b2-two", bits.FromHexP("002000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 10},
|
||||
{"b2-three", bits.FromHexP("003000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 11},
|
||||
{"b2-four", bits.FromHexP("004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 12},
|
||||
{"b2-five", bits.FromHexP("005000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 13},
|
||||
{"b2-six", bits.FromHexP("006000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 14},
|
||||
{"b2-seven", bits.FromHexP("007000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 15},
|
||||
|
||||
// at this point there are two buckets. the first has 7 contacts, the second has 8
|
||||
|
||||
// inserts into the second bucket should be skipped
|
||||
{"dont-split", bits.FromHexP("009000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 15},
|
||||
|
||||
// ... unless the ID is closer than the kth-closest contact
|
||||
{"split-kth-closest", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), 2, 16},
|
||||
|
||||
{"b3-two", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002"), 3, 17},
|
||||
{"b3-three", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003"), 3, 18},
|
||||
}
|
||||
|
||||
for i, testCase := range tests {
|
||||
rt.Update(Contact{testCase.id, net.ParseIP("127.0.0.1"), 8000 + i, 0})
|
||||
|
||||
if len(rt.buckets) != testCase.expectedBucketCount {
|
||||
t.Errorf("failed test case %s. there should be %d buckets, got %d", testCase.name, testCase.expectedBucketCount, len(rt.buckets))
|
||||
}
|
||||
if rt.Count() != testCase.expectedTotalContacts {
|
||||
t.Errorf("failed test case %s. there should be %d contacts, got %d", testCase.name, testCase.expectedTotalContacts, rt.Count())
|
||||
}
|
||||
}
|
||||
|
||||
var testRanges = []struct {
|
||||
id bits.Bitmap
|
||||
expected int
|
||||
}{
|
||||
{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), 0},
|
||||
{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000005"), 0},
|
||||
{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000410"), 1},
|
||||
{bits.FromHexP("0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007f0"), 1},
|
||||
{bits.FromHexP("F00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800"), 2},
|
||||
{bits.FromHexP("F00000000000000000000000000000000000000000000000000F00000000000000000000000000000000000000000000"), 2},
|
||||
{bits.FromHexP("F0000000000000000000000000000000F0000000000000000000000000F0000000000000000000000000000000000000"), 2},
|
||||
}
|
||||
|
||||
for _, tt := range testRanges {
|
||||
bucket := bucketNumFor(rt, tt.id)
|
||||
if bucket != tt.expected {
|
||||
t.Errorf("bucketFor(%s, %s) => got %d, expected %d", tt.id.Hex(), rt.id.Hex(), bucket, tt.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func bucketNumFor(rt *routingTable, target bits.Bitmap) int {
|
||||
if rt.id.Equals(target) {
|
||||
panic("routing table does not have a bucket for its own id")
|
||||
}
|
||||
distance := target.Xor(rt.id)
|
||||
for i := range rt.buckets {
|
||||
if rt.buckets[i].Range.Contains(distance) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
panic("target is not contained in any buckets")
|
||||
}
|
||||
|
||||
func TestBucket_Split_Continuous(t *testing.T) {
|
||||
b := newBucket(bits.MaxRange())
|
||||
|
||||
left, right := b.Split()
|
||||
|
||||
if !left.Range.Start.Equals(b.Range.Start) {
|
||||
t.Errorf("left bucket start does not align with original bucket start. got %s, expected %s", left.Range.Start, b.Range.Start)
|
||||
}
|
||||
|
||||
if !right.Range.End.Equals(b.Range.End) {
|
||||
t.Errorf("right bucket end does not align with original bucket end. got %s, expected %s", right.Range.End, b.Range.End)
|
||||
}
|
||||
|
||||
leftEndNext := (&big.Int{}).Add(left.Range.End.Big(), big.NewInt(1))
|
||||
if !bits.FromBigP(leftEndNext).Equals(right.Range.Start) {
|
||||
t.Errorf("there's a gap between left bucket end and right bucket start. end is %s, start is %s", left.Range.End, right.Range.Start)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBucket_Split_KthClosest_DoSplit(t *testing.T) {
|
||||
rt := newRoutingTable(bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"))
|
||||
|
||||
// add 4 low IDs
|
||||
rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), net.ParseIP("127.0.0.1"), 8001, 0})
|
||||
rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002"), net.ParseIP("127.0.0.1"), 8002, 0})
|
||||
rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003"), net.ParseIP("127.0.0.1"), 8003, 0})
|
||||
rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004"), net.ParseIP("127.0.0.1"), 8004, 0})
|
||||
|
||||
// add 4 high IDs
|
||||
rt.Update(Contact{bits.FromHexP("800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8001, 0})
|
||||
rt.Update(Contact{bits.FromHexP("900000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8002, 0})
|
||||
rt.Update(Contact{bits.FromHexP("a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8003, 0})
|
||||
rt.Update(Contact{bits.FromHexP("b00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8004, 0})
|
||||
|
||||
// split the bucket and fill the high bucket
|
||||
rt.Update(Contact{bits.FromHexP("c00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8005, 0})
|
||||
rt.Update(Contact{bits.FromHexP("d00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8006, 0})
|
||||
rt.Update(Contact{bits.FromHexP("e00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8007, 0})
|
||||
rt.Update(Contact{bits.FromHexP("f00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8008, 0})
|
||||
|
||||
// add a high ID. it should split because the high ID is closer than the Kth closest ID
|
||||
rt.Update(Contact{bits.FromHexP("910000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.1"), 8009, 0})
|
||||
|
||||
if len(rt.buckets) != 3 {
|
||||
t.Errorf("expected 3 buckets, got %d", len(rt.buckets))
|
||||
}
|
||||
if rt.Count() != 13 {
|
||||
t.Errorf("expected 13 contacts, got %d", rt.Count())
|
||||
}
|
||||
}
|
||||
|
||||
func TestBucket_Split_KthClosest_DontSplit(t *testing.T) {
|
||||
rt := newRoutingTable(bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"))
|
||||
|
||||
// add 4 low IDs
|
||||
rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), net.ParseIP("127.0.0.1"), 8001, 0})
|
||||
rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002"), net.ParseIP("127.0.0.1"), 8002, 0})
|
||||
rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003"), net.ParseIP("127.0.0.1"), 8003, 0})
|
||||
rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004"), net.ParseIP("127.0.0.1"), 8004, 0})
|
||||
|
||||
// add 4 high IDs
|
||||
rt.Update(Contact{bits.FromHexP("800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8001, 0})
|
||||
rt.Update(Contact{bits.FromHexP("900000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8002, 0})
|
||||
rt.Update(Contact{bits.FromHexP("a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8003, 0})
|
||||
rt.Update(Contact{bits.FromHexP("b00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8004, 0})
|
||||
|
||||
// split the bucket and fill the high bucket
|
||||
rt.Update(Contact{bits.FromHexP("c00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8005, 0})
|
||||
rt.Update(Contact{bits.FromHexP("d00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8006, 0})
|
||||
rt.Update(Contact{bits.FromHexP("e00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8007, 0})
|
||||
rt.Update(Contact{bits.FromHexP("f00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8008, 0})
|
||||
|
||||
// add a really high ID. this should not split because its not closer than the Kth closest ID
|
||||
rt.Update(Contact{bits.FromHexP("ffff00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.1"), 8009, 0})
|
||||
|
||||
if len(rt.buckets) != 2 {
|
||||
t.Errorf("expected 2 buckets, got %d", len(rt.buckets))
|
||||
}
|
||||
if rt.Count() != 12 {
|
||||
t.Errorf("expected 12 contacts, got %d", rt.Count())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoutingTable_GetClosest(t *testing.T) {
|
||||
n1 := bits.FromHexP("FFFFFFFF0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
|
||||
n2 := bits.FromHexP("FFFFFFF00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
|
||||
n3 := bits.FromHexP("111111110000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
|
||||
rt := newRoutingTable(n1)
|
||||
rt.Update(Contact{n2, net.ParseIP("127.0.0.1"), 8001, 0})
|
||||
rt.Update(Contact{n3, net.ParseIP("127.0.0.1"), 8002, 0})
|
||||
|
||||
contacts := rt.GetClosest(bits.FromHexP("222222220000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 1)
|
||||
if len(contacts) != 1 {
|
||||
t.Fail()
|
||||
return
|
||||
}
|
||||
if !contacts[0].ID.Equals(n3) {
|
||||
t.Error(contacts[0])
|
||||
}
|
||||
contacts = rt.GetClosest(n2, 10)
|
||||
if len(contacts) != 2 {
|
||||
t.Error(len(contacts))
|
||||
return
|
||||
}
|
||||
if !contacts[0].ID.Equals(n2) {
|
||||
t.Error(contacts[0])
|
||||
}
|
||||
if !contacts[1].ID.Equals(n3) {
|
||||
t.Error(contacts[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoutingTable_GetClosest_Empty(t *testing.T) {
|
||||
n1 := bits.FromShortHexP("1")
|
||||
rt := newRoutingTable(n1)
|
||||
|
||||
contacts := rt.GetClosest(bits.FromShortHexP("a"), 3)
|
||||
if len(contacts) != 0 {
|
||||
t.Error("there shouldn't be any contacts")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoutingTable_Refresh(t *testing.T) {
|
||||
t.Skip("TODO: test routing table refreshing")
|
||||
}
|
||||
|
||||
func TestRoutingTable_MoveToBack(t *testing.T) {
|
||||
tt := map[string]struct {
|
||||
data []peer
|
||||
index int
|
||||
expected []peer
|
||||
}{
|
||||
"simpleMove": {
|
||||
data: []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
|
||||
index: 1,
|
||||
expected: []peer{{NumFailures: 0}, {NumFailures: 2}, {NumFailures: 3}, {NumFailures: 1}},
|
||||
},
|
||||
"moveFirst": {
|
||||
data: []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
|
||||
index: 0,
|
||||
expected: []peer{{NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}, {NumFailures: 0}},
|
||||
},
|
||||
"moveLast": {
|
||||
data: []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
|
||||
index: 3,
|
||||
expected: []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
|
||||
},
|
||||
"largeIndex": {
|
||||
data: []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
|
||||
index: 27,
|
||||
expected: []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
|
||||
},
|
||||
"negativeIndex": {
|
||||
data: []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
|
||||
index: -12,
|
||||
expected: []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
|
||||
},
|
||||
}
|
||||
|
||||
for name, test := range tt {
|
||||
moveToBack(test.data, test.index)
|
||||
expected := make([]string, len(test.expected))
|
||||
actual := make([]string, len(test.data))
|
||||
for i := range actual {
|
||||
actual[i] = strconv.Itoa(test.data[i].NumFailures)
|
||||
expected[i] = strconv.Itoa(test.expected[i].NumFailures)
|
||||
}
|
||||
|
||||
expJoin := strings.Join(expected, ",")
|
||||
actJoin := strings.Join(actual, ",")
|
||||
|
||||
if actJoin != expJoin {
|
||||
t.Errorf("%s failed: got %s; expected %s", name, actJoin, expJoin)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoutingTable_Save(t *testing.T) {
|
||||
t.Skip("fix me")
|
||||
id := bits.FromHexP("1c8aff71b99462464d9eeac639595ab99664be3482cb91a29d87467515c7d9158fe72aa1f1582dab07d8f8b5db277f41")
|
||||
rt := newRoutingTable(id)
|
||||
|
||||
for i, b := range rt.buckets {
|
||||
for j := 0; j < bucketSize; j++ {
|
||||
toAdd := b.Range.Start.Add(bits.FromShortHexP(strconv.Itoa(j)))
|
||||
if toAdd.Cmp(b.Range.End) <= 0 {
|
||||
rt.Update(Contact{
|
||||
ID: b.Range.Start.Add(bits.FromShortHexP(strconv.Itoa(j))),
|
||||
IP: net.ParseIP("1.2.3." + strconv.Itoa(j)),
|
||||
Port: 1 + i*bucketSize + j,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
data, err := json.MarshalIndent(rt, "", " ")
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
goldie.Assert(t, t.Name(), data)
|
||||
}
|
||||
|
||||
func TestRoutingTable_Load_ID(t *testing.T) {
|
||||
t.Skip("fix me")
|
||||
id := "1c8aff71b99462464d9eeac639595ab99664be3482cb91a29d87467515c7d9158fe72aa1f1582dab07d8f8b5db277f41"
|
||||
data := []byte(`{"id": "` + id + `","contacts": []}`)
|
||||
|
||||
rt := routingTable{}
|
||||
err := json.Unmarshal(data, &rt)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if rt.id.Hex() != id {
|
||||
t.Error("id mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoutingTable_Load_Contacts(t *testing.T) {
|
||||
t.Skip("TODO")
|
||||
}
|
||||
-187
@@ -1,187 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"sync"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
rpc2 "github.com/gorilla/rpc/v2"
|
||||
"github.com/gorilla/rpc/v2/json"
|
||||
)
|
||||
|
||||
type rpcReceiver struct {
|
||||
dht *DHT
|
||||
}
|
||||
|
||||
type RpcPingArgs struct {
|
||||
Address string
|
||||
}
|
||||
|
||||
func (rpc *rpcReceiver) Ping(r *http.Request, args *RpcPingArgs, result *string) error {
|
||||
if args.Address == "" {
|
||||
return errors.Err("no address given")
|
||||
}
|
||||
|
||||
err := rpc.dht.Ping(args.Address)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
*result = pingSuccessResponse
|
||||
return nil
|
||||
}
|
||||
|
||||
type RpcFindArgs struct {
|
||||
Key string
|
||||
NodeID string
|
||||
IP string
|
||||
Port int
|
||||
}
|
||||
|
||||
func (rpc *rpcReceiver) FindNode(r *http.Request, args *RpcFindArgs, result *[]Contact) error {
|
||||
key, err := bits.FromHex(args.Key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
toQuery, err := bits.FromHex(args.NodeID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
c := Contact{ID: toQuery, IP: net.ParseIP(args.IP), Port: args.Port}
|
||||
req := Request{Method: findNodeMethod, Arg: &key}
|
||||
|
||||
nodeResponse := rpc.dht.node.Send(c, req)
|
||||
if nodeResponse != nil && nodeResponse.Contacts != nil {
|
||||
*result = nodeResponse.Contacts
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type RpcFindValueResult struct {
|
||||
Contacts []Contact
|
||||
Value string
|
||||
}
|
||||
|
||||
func (rpc *rpcReceiver) FindValue(r *http.Request, args *RpcFindArgs, result *RpcFindValueResult) error {
|
||||
key, err := bits.FromHex(args.Key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
toQuery, err := bits.FromHex(args.NodeID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c := Contact{ID: toQuery, IP: net.ParseIP(args.IP), Port: args.Port}
|
||||
req := Request{Arg: &key, Method: findValueMethod}
|
||||
|
||||
nodeResponse := rpc.dht.node.Send(c, req)
|
||||
if nodeResponse != nil && nodeResponse.FindValueKey != "" {
|
||||
*result = RpcFindValueResult{Value: nodeResponse.FindValueKey}
|
||||
return nil
|
||||
}
|
||||
if nodeResponse != nil && nodeResponse.Contacts != nil {
|
||||
*result = RpcFindValueResult{Contacts: nodeResponse.Contacts}
|
||||
return nil
|
||||
}
|
||||
|
||||
return errors.Err("not sure what happened")
|
||||
}
|
||||
|
||||
type RpcIterativeFindValueArgs struct {
|
||||
Key string
|
||||
}
|
||||
|
||||
type RpcIterativeFindValueResult struct {
|
||||
Contacts []Contact
|
||||
FoundValue bool
|
||||
}
|
||||
|
||||
func (rpc *rpcReceiver) IterativeFindValue(r *http.Request, args *RpcIterativeFindValueArgs, result *RpcIterativeFindValueResult) error {
|
||||
key, err := bits.FromHex(args.Key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
foundContacts, found, err := FindContacts(rpc.dht.node, key, false, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result.Contacts = foundContacts
|
||||
result.FoundValue = found
|
||||
return nil
|
||||
}
|
||||
|
||||
type RpcBucketResponse struct {
|
||||
Start string
|
||||
End string
|
||||
NumContacts int
|
||||
Contacts []Contact
|
||||
}
|
||||
|
||||
type RpcRoutingTableResponse struct {
|
||||
NodeID string
|
||||
NumBuckets int
|
||||
Buckets []RpcBucketResponse
|
||||
}
|
||||
|
||||
func (rpc *rpcReceiver) GetRoutingTable(r *http.Request, args *struct{}, result *RpcRoutingTableResponse) error {
|
||||
result.NodeID = rpc.dht.node.id.String()
|
||||
result.NumBuckets = len(rpc.dht.node.rt.buckets)
|
||||
for _, b := range rpc.dht.node.rt.buckets {
|
||||
result.Buckets = append(result.Buckets, RpcBucketResponse{
|
||||
Start: b.Range.Start.String(),
|
||||
End: b.Range.End.String(),
|
||||
NumContacts: b.Len(),
|
||||
Contacts: b.Contacts(),
|
||||
})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (rpc *rpcReceiver) AddKnownNode(r *http.Request, args *Contact, result *string) error {
|
||||
rpc.dht.node.AddKnownNode(*args)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (dht *DHT) runRPCServer(port int) {
|
||||
addr := "0.0.0.0:" + strconv.Itoa(port)
|
||||
|
||||
s := rpc2.NewServer()
|
||||
s.RegisterCodec(json.NewCodec(), "application/json")
|
||||
s.RegisterCodec(json.NewCodec(), "application/json;charset=UTF-8")
|
||||
err := s.RegisterService(&rpcReceiver{dht: dht}, "rpc")
|
||||
if err != nil {
|
||||
log.Error(errors.Prefix("registering rpc service", err))
|
||||
return
|
||||
}
|
||||
|
||||
handler := mux.NewRouter()
|
||||
handler.Handle("/", s)
|
||||
server := &http.Server{Addr: addr, Handler: handler}
|
||||
|
||||
wg := sync.WaitGroup{}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
log.Printf("RPC server listening on %s", addr)
|
||||
err := server.ListenAndServe()
|
||||
if err != nil && err != http.ErrServerClosed {
|
||||
log.Error(err)
|
||||
}
|
||||
}()
|
||||
|
||||
<-dht.grp.Ch()
|
||||
err = server.Shutdown(context.Background())
|
||||
if err != nil {
|
||||
log.Error(errors.Prefix("shutting down rpc service", err))
|
||||
return
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
)
|
||||
|
||||
// TODO: expire stored data after tExpire time
|
||||
|
||||
type contactStore struct {
|
||||
// map of blob hashes to (map of node IDs to bools)
|
||||
hashes map[bits.Bitmap]map[bits.Bitmap]bool
|
||||
// stores the peers themselves, so they can be updated in one place
|
||||
contacts map[bits.Bitmap]Contact
|
||||
lock sync.RWMutex
|
||||
}
|
||||
|
||||
func newStore() *contactStore {
|
||||
return &contactStore{
|
||||
hashes: make(map[bits.Bitmap]map[bits.Bitmap]bool),
|
||||
contacts: make(map[bits.Bitmap]Contact),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *contactStore) Upsert(blobHash bits.Bitmap, contact Contact) {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
|
||||
if _, ok := s.hashes[blobHash]; !ok {
|
||||
s.hashes[blobHash] = make(map[bits.Bitmap]bool)
|
||||
}
|
||||
s.hashes[blobHash][contact.ID] = true
|
||||
s.contacts[contact.ID] = contact
|
||||
}
|
||||
|
||||
func (s *contactStore) Get(blobHash bits.Bitmap) []Contact {
|
||||
s.lock.RLock()
|
||||
defer s.lock.RUnlock()
|
||||
|
||||
var contacts []Contact
|
||||
if ids, ok := s.hashes[blobHash]; ok {
|
||||
for id := range ids {
|
||||
contact, ok := s.contacts[id]
|
||||
if !ok {
|
||||
panic("node id in IDs list, but not in nodeInfo")
|
||||
}
|
||||
contacts = append(contacts, contact)
|
||||
}
|
||||
}
|
||||
return contacts
|
||||
}
|
||||
|
||||
func (s *contactStore) RemoveTODO(contact Contact) {
|
||||
// TODO: remove peer from everywhere
|
||||
}
|
||||
|
||||
func (s *contactStore) CountStoredHashes() int {
|
||||
s.lock.RLock()
|
||||
defer s.lock.RUnlock()
|
||||
return len(s.hashes)
|
||||
}
|
||||
-312
@@ -1,312 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
)
|
||||
|
||||
var testingDHTIP = "127.0.0.1"
|
||||
var testingDHTFirstPort = 21000
|
||||
|
||||
// TestingCreateNetwork initializes a testable DHT network with a specific number of nodes, with bootstrap and concurrent options.
|
||||
func TestingCreateNetwork(t *testing.T, numNodes int, bootstrap, concurrent bool) (*BootstrapNode, []*DHT) {
|
||||
var bootstrapNode *BootstrapNode
|
||||
var seeds []string
|
||||
|
||||
if bootstrap {
|
||||
bootstrapAddress := testingDHTIP + ":" + strconv.Itoa(testingDHTFirstPort)
|
||||
seeds = []string{bootstrapAddress}
|
||||
bootstrapNode = NewBootstrapNode(bits.Rand(), 0, bootstrapDefaultRefreshDuration)
|
||||
listener, err := net.ListenPacket(Network, bootstrapAddress)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
err = bootstrapNode.Connect(listener.(*net.UDPConn))
|
||||
if err != nil {
|
||||
t.Error("error connecting bootstrap node - ", err)
|
||||
}
|
||||
}
|
||||
|
||||
if numNodes < 1 {
|
||||
return bootstrapNode, nil
|
||||
}
|
||||
|
||||
firstPort := testingDHTFirstPort + 1
|
||||
dhts := make([]*DHT, numNodes)
|
||||
|
||||
for i := 0; i < numNodes; i++ {
|
||||
c := NewStandardConfig()
|
||||
c.NodeID = bits.Rand().Hex()
|
||||
c.Address = testingDHTIP + ":" + strconv.Itoa(firstPort+i)
|
||||
c.SeedNodes = seeds
|
||||
dht := New(c)
|
||||
|
||||
go func() {
|
||||
err := dht.Start()
|
||||
if err != nil {
|
||||
t.Error("error starting dht - ", err)
|
||||
}
|
||||
}()
|
||||
if !concurrent {
|
||||
dht.WaitUntilJoined()
|
||||
}
|
||||
dhts[i] = dht
|
||||
}
|
||||
|
||||
if concurrent {
|
||||
for _, d := range dhts {
|
||||
d.WaitUntilJoined()
|
||||
}
|
||||
}
|
||||
|
||||
return bootstrapNode, dhts
|
||||
}
|
||||
|
||||
type timeoutErr struct {
|
||||
error
|
||||
}
|
||||
|
||||
func (t timeoutErr) Timeout() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (t timeoutErr) Temporary() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// TODO: just use a normal net.Conn instead of this mock conn
|
||||
|
||||
type testUDPPacket struct {
|
||||
data []byte
|
||||
addr *net.UDPAddr
|
||||
}
|
||||
|
||||
type testUDPConn struct {
|
||||
addr *net.UDPAddr
|
||||
toRead chan testUDPPacket
|
||||
writes chan testUDPPacket
|
||||
|
||||
readDeadline time.Time
|
||||
}
|
||||
|
||||
func newTestUDPConn(addr string) *testUDPConn {
|
||||
parts := strings.Split(addr, ":")
|
||||
if len(parts) != 2 {
|
||||
panic("addr needs ip and port")
|
||||
}
|
||||
port, err := strconv.Atoi(parts[1])
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return &testUDPConn{
|
||||
addr: &net.UDPAddr{IP: net.IP(parts[0]), Port: port},
|
||||
toRead: make(chan testUDPPacket),
|
||||
writes: make(chan testUDPPacket),
|
||||
}
|
||||
}
|
||||
|
||||
func (t testUDPConn) ReadFromUDP(b []byte) (int, *net.UDPAddr, error) {
|
||||
var timeoutCh <-chan time.Time
|
||||
if !t.readDeadline.IsZero() {
|
||||
timeoutCh = time.After(time.Until(t.readDeadline))
|
||||
}
|
||||
|
||||
select {
|
||||
case packet, ok := <-t.toRead:
|
||||
if !ok {
|
||||
return 0, nil, errors.Err("conn closed")
|
||||
}
|
||||
n := copy(b, packet.data)
|
||||
return n, packet.addr, nil
|
||||
case <-timeoutCh:
|
||||
return 0, nil, timeoutErr{errors.Err("timeout")}
|
||||
}
|
||||
}
|
||||
|
||||
func (t testUDPConn) WriteToUDP(b []byte, addr *net.UDPAddr) (int, error) {
|
||||
t.writes <- testUDPPacket{data: b, addr: addr}
|
||||
return len(b), nil
|
||||
}
|
||||
|
||||
func (t *testUDPConn) SetReadDeadline(tm time.Time) error {
|
||||
t.readDeadline = tm
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *testUDPConn) SetWriteDeadline(tm time.Time) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *testUDPConn) Close() error {
|
||||
close(t.toRead)
|
||||
t.writes = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
func verifyResponse(t *testing.T, resp map[string]interface{}, id messageID, dhtNodeID string) {
|
||||
if len(resp) != 4 {
|
||||
t.Errorf("expected 4 response fields, got %d", len(resp))
|
||||
}
|
||||
|
||||
_, ok := resp[headerTypeField]
|
||||
if !ok {
|
||||
t.Error("missing type field")
|
||||
} else {
|
||||
rType, ok := resp[headerTypeField].(int64)
|
||||
if !ok {
|
||||
t.Error("type is not an integer")
|
||||
} else if rType != responseType {
|
||||
t.Error("unexpected response type")
|
||||
}
|
||||
}
|
||||
|
||||
_, ok = resp[headerMessageIDField]
|
||||
if !ok {
|
||||
t.Error("missing message id field")
|
||||
} else {
|
||||
rMessageID, ok := resp[headerMessageIDField].(string)
|
||||
if !ok {
|
||||
t.Error("message ID is not a string")
|
||||
} else if rMessageID != string(id[:]) {
|
||||
t.Error("unexpected message ID")
|
||||
}
|
||||
if len(rMessageID) != messageIDLength {
|
||||
t.Errorf("message ID should be %d chars long", messageIDLength)
|
||||
}
|
||||
}
|
||||
|
||||
_, ok = resp[headerNodeIDField]
|
||||
if !ok {
|
||||
t.Error("missing node id field")
|
||||
} else {
|
||||
rNodeID, ok := resp[headerNodeIDField].(string)
|
||||
if !ok {
|
||||
t.Error("node ID is not a string")
|
||||
} else if rNodeID != dhtNodeID {
|
||||
t.Error("unexpected node ID")
|
||||
}
|
||||
if len(rNodeID) != nodeIDLength {
|
||||
t.Errorf("node ID should be %d chars long", nodeIDLength)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func verifyContacts(t *testing.T, contacts []interface{}, nodes []Contact) {
|
||||
if len(contacts) != len(nodes) {
|
||||
t.Errorf("got %d contacts; expected %d", len(contacts), len(nodes))
|
||||
return
|
||||
}
|
||||
|
||||
foundNodes := make(map[string]bool)
|
||||
|
||||
for _, c := range contacts {
|
||||
contact, ok := c.([]interface{})
|
||||
if !ok {
|
||||
t.Error("contact is not a list")
|
||||
return
|
||||
}
|
||||
|
||||
if len(contact) != 3 {
|
||||
t.Error("contact must be 3 items")
|
||||
return
|
||||
}
|
||||
|
||||
var currNode Contact
|
||||
currNodeFound := false
|
||||
|
||||
id, ok := contact[0].(string)
|
||||
if !ok {
|
||||
t.Error("contact id is not a string")
|
||||
} else {
|
||||
if _, ok := foundNodes[id]; ok {
|
||||
t.Errorf("contact %s appears multiple times", id)
|
||||
continue
|
||||
}
|
||||
for _, n := range nodes {
|
||||
if n.ID.RawString() == id {
|
||||
currNode = n
|
||||
currNodeFound = true
|
||||
foundNodes[id] = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !currNodeFound {
|
||||
t.Errorf("unexpected contact %s", id)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
ip, ok := contact[1].(string)
|
||||
if !ok {
|
||||
t.Error("contact IP is not a string")
|
||||
} else if !currNode.IP.Equal(net.ParseIP(ip)) {
|
||||
t.Errorf("contact IP mismatch. got %s; expected %s", ip, currNode.IP.String())
|
||||
}
|
||||
|
||||
port, ok := contact[2].(int64)
|
||||
if !ok {
|
||||
t.Error("contact port is not an int")
|
||||
} else if int(port) != currNode.Port {
|
||||
t.Errorf("contact port mismatch. got %d; expected %d", port, currNode.Port)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func verifyCompactContacts(t *testing.T, contacts []interface{}, nodes []Contact) {
|
||||
if len(contacts) != len(nodes) {
|
||||
t.Errorf("got %d contacts; expected %d", len(contacts), len(nodes))
|
||||
return
|
||||
}
|
||||
|
||||
foundNodes := make(map[string]bool)
|
||||
|
||||
for _, c := range contacts {
|
||||
compact, ok := c.(string)
|
||||
if !ok {
|
||||
t.Error("contact is not a string")
|
||||
return
|
||||
}
|
||||
|
||||
contact := Contact{}
|
||||
err := contact.UnmarshalCompact([]byte(compact))
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
var currNode Contact
|
||||
currNodeFound := false
|
||||
|
||||
if _, ok := foundNodes[contact.ID.Hex()]; ok {
|
||||
t.Errorf("contact %s appears multiple times", contact.ID.Hex())
|
||||
continue
|
||||
}
|
||||
for _, n := range nodes {
|
||||
if n.ID.Equals(contact.ID) {
|
||||
currNode = n
|
||||
currNodeFound = true
|
||||
foundNodes[contact.ID.Hex()] = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !currNodeFound {
|
||||
t.Errorf("unexpected contact %s", contact.ID.Hex())
|
||||
continue
|
||||
}
|
||||
|
||||
if !currNode.IP.Equal(contact.IP) {
|
||||
t.Errorf("contact IP mismatch. got %s; expected %s", contact.IP.String(), currNode.IP.String())
|
||||
}
|
||||
|
||||
if contact.Port != currNode.Port {
|
||||
t.Errorf("contact port mismatch. got %d; expected %d", contact.Port, currNode.Port)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,71 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
|
||||
"github.com/lbryio/lbry.go/stop"
|
||||
)
|
||||
|
||||
// TODO: this should be moved out of dht and into node, and it should be completely hidden inside node. dht should not need to know about tokens
|
||||
|
||||
type tokenCacheEntry struct {
|
||||
token string
|
||||
receivedAt time.Time
|
||||
}
|
||||
|
||||
type tokenCache struct {
|
||||
node *Node
|
||||
tokens map[string]tokenCacheEntry
|
||||
expiration time.Duration
|
||||
lock *sync.RWMutex
|
||||
}
|
||||
|
||||
func newTokenCache(node *Node, expiration time.Duration) *tokenCache {
|
||||
tc := &tokenCache{}
|
||||
tc.node = node
|
||||
tc.tokens = make(map[string]tokenCacheEntry)
|
||||
tc.expiration = expiration
|
||||
tc.lock = &sync.RWMutex{}
|
||||
return tc
|
||||
}
|
||||
|
||||
// TODO: if store fails, get new token. can happen if a node restarts but we have the token cached
|
||||
|
||||
func (tc *tokenCache) Get(c Contact, hash bits.Bitmap, cancelCh stop.Chan) string {
|
||||
tc.lock.RLock()
|
||||
token, exists := tc.tokens[c.String()]
|
||||
tc.lock.RUnlock()
|
||||
|
||||
if exists && time.Since(token.receivedAt) < tc.expiration {
|
||||
return token.token
|
||||
}
|
||||
|
||||
resCh := tc.node.SendAsync(c, Request{
|
||||
Method: findValueMethod,
|
||||
Arg: &hash,
|
||||
})
|
||||
|
||||
var res *Response
|
||||
|
||||
select {
|
||||
case res = <-resCh:
|
||||
case <-cancelCh:
|
||||
return ""
|
||||
}
|
||||
|
||||
if res == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
tc.lock.Lock()
|
||||
tc.tokens[c.String()] = tokenCacheEntry{
|
||||
token: res.Token,
|
||||
receivedAt: time.Now(),
|
||||
}
|
||||
tc.lock.Unlock()
|
||||
|
||||
return res.Token
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"net"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/stop"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
)
|
||||
|
||||
type tokenManager struct {
|
||||
secret []byte
|
||||
prevSecret []byte
|
||||
lock *sync.RWMutex
|
||||
stop *stop.Group
|
||||
}
|
||||
|
||||
func (tm *tokenManager) Start(interval time.Duration) {
|
||||
tm.secret = make([]byte, 64)
|
||||
tm.prevSecret = make([]byte, 64)
|
||||
tm.lock = &sync.RWMutex{}
|
||||
tm.stop = stop.New()
|
||||
|
||||
tm.rotateSecret()
|
||||
|
||||
tm.stop.Add(1)
|
||||
go func() {
|
||||
defer tm.stop.Done()
|
||||
tick := time.NewTicker(interval)
|
||||
for {
|
||||
select {
|
||||
case <-tick.C:
|
||||
tm.rotateSecret()
|
||||
case <-tm.stop.Ch():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (tm *tokenManager) Stop() {
|
||||
tm.stop.StopAndWait()
|
||||
}
|
||||
|
||||
func (tm *tokenManager) Get(nodeID bits.Bitmap, addr *net.UDPAddr) string {
|
||||
return genToken(tm.secret, nodeID, addr)
|
||||
}
|
||||
|
||||
func (tm *tokenManager) Verify(token string, nodeID bits.Bitmap, addr *net.UDPAddr) bool {
|
||||
return token == genToken(tm.secret, nodeID, addr) || token == genToken(tm.prevSecret, nodeID, addr)
|
||||
}
|
||||
|
||||
func genToken(secret []byte, nodeID bits.Bitmap, addr *net.UDPAddr) string {
|
||||
buf := bytes.Buffer{}
|
||||
buf.Write(nodeID[:])
|
||||
buf.Write(addr.IP)
|
||||
buf.WriteString(strconv.Itoa(addr.Port))
|
||||
buf.Write(secret)
|
||||
t := sha256.Sum256(buf.Bytes())
|
||||
return string(t[:])
|
||||
}
|
||||
|
||||
func (tm *tokenManager) rotateSecret() {
|
||||
tm.lock.Lock()
|
||||
defer tm.lock.Unlock()
|
||||
|
||||
copy(tm.prevSecret, tm.secret)
|
||||
|
||||
_, err := rand.Read(tm.secret)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
module github.com/lbryio/reflector.go
|
||||
|
||||
replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19
|
||||
|
||||
require (
|
||||
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
|
||||
github.com/aws/aws-sdk-go v1.27.0
|
||||
github.com/bluele/gcache v0.0.2
|
||||
github.com/brk0v/directio v0.0.0-20190225130936-69406e757cf7
|
||||
github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3
|
||||
github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d
|
||||
github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2
|
||||
github.com/davecgh/go-spew v1.1.1
|
||||
github.com/ekyoung/gin-nice-recovery v0.0.0-20160510022553-1654dca486db
|
||||
github.com/gin-gonic/gin v1.7.1
|
||||
github.com/go-sql-driver/mysql v1.6.0
|
||||
github.com/gogo/protobuf v1.2.1
|
||||
github.com/golang/protobuf v1.5.2
|
||||
github.com/google/go-cmp v0.5.6 // indirect
|
||||
github.com/google/gops v0.3.18
|
||||
github.com/gorilla/mux v1.8.0
|
||||
github.com/hashicorp/go-msgpack v0.5.5 // indirect
|
||||
github.com/hashicorp/golang-lru v0.5.4 // indirect
|
||||
github.com/hashicorp/serf v0.9.5
|
||||
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf
|
||||
github.com/johntdyer/slackrus v0.0.0-20210521205746-42486fb4c48c
|
||||
github.com/karrick/godirwalk v1.16.1
|
||||
github.com/lbryio/chainquery v1.9.0
|
||||
github.com/lbryio/lbry.go/v2 v2.7.2-0.20210416195322-6516df1418e3
|
||||
github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386
|
||||
github.com/lucas-clemente/quic-go v0.20.1
|
||||
github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6
|
||||
github.com/prometheus/client_golang v1.10.0
|
||||
github.com/sergi/go-diff v1.2.0 // indirect
|
||||
github.com/sirupsen/logrus v1.8.1
|
||||
github.com/spf13/afero v1.4.1 // indirect
|
||||
github.com/spf13/cast v1.3.0
|
||||
github.com/spf13/cobra v1.1.3
|
||||
github.com/spf13/viper v1.7.1 // indirect
|
||||
github.com/stretchr/testify v1.7.0
|
||||
github.com/volatiletech/null v8.0.0+incompatible
|
||||
go.uber.org/atomic v1.7.0
|
||||
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 // indirect
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
|
||||
golang.org/x/text v0.3.6 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
|
||||
google.golang.org/protobuf v1.27.1 // indirect
|
||||
)
|
||||
|
||||
go 1.16
|
||||
@@ -0,0 +1,930 @@
|
||||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo=
|
||||
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
|
||||
cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
|
||||
cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
|
||||
cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
|
||||
cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
|
||||
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
|
||||
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
|
||||
cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
|
||||
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
|
||||
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
|
||||
dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU=
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU=
|
||||
dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4=
|
||||
dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU=
|
||||
git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
|
||||
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
|
||||
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
|
||||
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
|
||||
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
|
||||
github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
|
||||
github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
|
||||
github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
|
||||
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
|
||||
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
|
||||
github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
|
||||
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
|
||||
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
|
||||
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM=
|
||||
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg=
|
||||
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
|
||||
github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
|
||||
github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A=
|
||||
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
|
||||
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
|
||||
github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
|
||||
github.com/aws/aws-sdk-go v1.27.0 h1:0xphMHGMLBrPMfxR2AmVjZKcMEESEgWF8Kru94BNByk=
|
||||
github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
||||
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
|
||||
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
|
||||
github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw=
|
||||
github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0=
|
||||
github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
|
||||
github.com/brk0v/directio v0.0.0-20190225130936-69406e757cf7 h1:7gNKWnX6OF+ERiXVw4I9RsHhZ52aumXdFE07nEx5v20=
|
||||
github.com/brk0v/directio v0.0.0-20190225130936-69406e757cf7/go.mod h1:M/KA3XJG5PJaApPiv4gWNsgcSJquOQTqumZNLyYE0KM=
|
||||
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo=
|
||||
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
|
||||
github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
|
||||
github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d h1:yJzD/yFppdVCf6ApMkVy8cUxV0XrxdP9rVf6D87/Mng=
|
||||
github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
|
||||
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd h1:R/opQEbFEy9JGkIguV40SvRY1uliPX8ifOvi6ICsFCw=
|
||||
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg=
|
||||
github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY=
|
||||
github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
|
||||
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3SkEwmHoWBmX1DNXhXZqlTpq6s4tyJGc=
|
||||
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
|
||||
github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
|
||||
github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
|
||||
github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2 h1:t8KYCwSKsOEZBFELI4Pn/phbp38iJ1RRAkDFNin1aak=
|
||||
github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M=
|
||||
github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
|
||||
github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
|
||||
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
||||
github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
|
||||
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
|
||||
github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
|
||||
github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
|
||||
github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
|
||||
github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
|
||||
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
|
||||
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
|
||||
github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
|
||||
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
|
||||
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
|
||||
github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
|
||||
github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
|
||||
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
|
||||
github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
|
||||
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
|
||||
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
|
||||
github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
|
||||
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
|
||||
github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
|
||||
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
|
||||
github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
|
||||
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
|
||||
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
|
||||
github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
|
||||
github.com/ekyoung/gin-nice-recovery v0.0.0-20160510022553-1654dca486db h1:oZ4U9IqO8NS+61OmGTBi8vopzqTRxwQeogyBHdrhjbc=
|
||||
github.com/ekyoung/gin-nice-recovery v0.0.0-20160510022553-1654dca486db/go.mod h1:Pk7/9x6tyChFTkahDvLBQMlvdsWvfC+yU8HTT5VD314=
|
||||
github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g=
|
||||
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
|
||||
github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
|
||||
github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
|
||||
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
|
||||
github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
|
||||
github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
|
||||
github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
|
||||
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
|
||||
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
|
||||
github.com/gin-gonic/gin v1.7.1 h1:qC89GU3p8TvKWMAVhEpmpB2CIb1hnqt2UdKZaP93mS8=
|
||||
github.com/gin-gonic/gin v1.7.1/go.mod h1:jD2toBW3GZUr5UMcdrwQA10I7RuaFOl/SGeDjXkfUtY=
|
||||
github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
|
||||
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
|
||||
github.com/go-errors/errors v1.1.1 h1:ljK/pL5ltg3qoN+OtN6yCv9HWSfMwxSx90GJCZQxYNg=
|
||||
github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-ini/ini v1.38.2/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-ini/ini v1.48.0 h1:TvO60hO/2xgaaTWp2P0wUe4CFxwdMzfbkv3+343Xzqw=
|
||||
github.com/go-ini/ini v1.48.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o=
|
||||
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
|
||||
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
|
||||
github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
|
||||
github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
|
||||
github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
|
||||
github.com/go-ozzo/ozzo-validation v3.6.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
|
||||
github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
|
||||
github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
|
||||
github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
|
||||
github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
|
||||
github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
|
||||
github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
|
||||
github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE=
|
||||
github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
|
||||
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
||||
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
||||
github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
|
||||
github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE=
|
||||
github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
|
||||
github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
|
||||
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
|
||||
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
|
||||
github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
|
||||
github.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g=
|
||||
github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
|
||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
|
||||
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
|
||||
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
|
||||
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
|
||||
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
|
||||
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
|
||||
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||
github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
|
||||
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
||||
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
|
||||
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
|
||||
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
|
||||
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/gops v0.3.18 h1:my259V+172PVFmduS2RAsq4FKH+HjKqdh7pLr17Ot8c=
|
||||
github.com/google/gops v0.3.18/go.mod h1:Pfp8hWGIFdV/7rY9/O/U5WgdjYQXf/GiEK4NVuVd2ZE=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
|
||||
github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f h1:TyqzGm2z1h3AGhjOoRYyeLcW4WlW81MDQkWa+rx/000=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
|
||||
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
|
||||
github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
|
||||
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
|
||||
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
|
||||
github.com/gorilla/rpc v1.1.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ=
|
||||
github.com/gorilla/rpc v1.2.0 h1:WvvdC2lNeT1SP32zrIce5l0ECBfbAlmrmSBsuc57wfk=
|
||||
github.com/gorilla/rpc v1.2.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ=
|
||||
github.com/gorilla/schema v1.0.2/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU=
|
||||
github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
|
||||
github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
|
||||
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
|
||||
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
|
||||
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
|
||||
github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
|
||||
github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE=
|
||||
github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
|
||||
github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
|
||||
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
|
||||
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
|
||||
github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0=
|
||||
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
|
||||
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
|
||||
github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI=
|
||||
github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
|
||||
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
|
||||
github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI=
|
||||
github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
|
||||
github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
|
||||
github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
|
||||
github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs=
|
||||
github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
|
||||
github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
|
||||
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE=
|
||||
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
|
||||
github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
|
||||
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
|
||||
github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
|
||||
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
|
||||
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
|
||||
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
|
||||
github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
|
||||
github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY=
|
||||
github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
|
||||
github.com/hashicorp/memberlist v0.2.2 h1:5+RffWKwqJ71YPu9mWsF7ZOscZmwfasdA8kbdC7AO2g=
|
||||
github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE=
|
||||
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
|
||||
github.com/hashicorp/serf v0.9.5 h1:EBWvyu9tcRszt3Bxp3KNssBMP1KuHWyO51lz9+786iM=
|
||||
github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg=
|
||||
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf h1:WfD7VjIE6z8dIvMsI4/s+1qr5EL+zoIGev1BQj1eoJ8=
|
||||
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h1:hyb9oH7vZsitZCiBt0ZvifOrB+qc8PS5IiilCIb87rg=
|
||||
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
|
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
|
||||
github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
|
||||
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
|
||||
github.com/jmoiron/sqlx v0.0.0-20170430194603-d9bd385d68c0/go.mod h1:IiEW3SEiiErVyFdH8NTuWjSifiEQKUoyK3LNqr2kCHU=
|
||||
github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22 h1:jKUP9TQ0c7X3w6+IPyMit07RE42MtTWNd77sN2cHngQ=
|
||||
github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22/go.mod h1:u0Jo4f2dNlTJeeOywkM6bLwxq6gC3pZ9rEFHn3AhTdk=
|
||||
github.com/johntdyer/slackrus v0.0.0-20170926115001-3992f319fd0a/go.mod h1:j1kV/8f3jowErEq4XyeypkCdvg5EeHkf0YCKCcq5Ybo=
|
||||
github.com/johntdyer/slackrus v0.0.0-20210521205746-42486fb4c48c h1:4eD0DM7IEOQI5egyzKdPXncs61BxMtYwwzJPCRQFU0Y=
|
||||
github.com/johntdyer/slackrus v0.0.0-20210521205746-42486fb4c48c/go.mod h1:j1kV/8f3jowErEq4XyeypkCdvg5EeHkf0YCKCcq5Ybo=
|
||||
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
|
||||
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
|
||||
github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
|
||||
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
||||
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
|
||||
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
github.com/jteeuwen/go-bindata v3.0.7+incompatible/go.mod h1:JVvhzYOiGBnFSYRyV00iY8q7/0PThjIYav1p9h5dmKs=
|
||||
github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
|
||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
|
||||
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
|
||||
github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw=
|
||||
github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
|
||||
github.com/keybase/go-ps v0.0.0-20190827175125-91aafc93ba19/go.mod h1:hY+WOq6m2FpbvyrI93sMaypsttvaIL5nhVR92dTMUcQ=
|
||||
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/lbryio/chainquery v1.9.0 h1:NfBZ3eKYwD3PqXU/vt+2tF3ox3WUWoW4J5YdEQ0rxw0=
|
||||
github.com/lbryio/chainquery v1.9.0/go.mod h1:7G8l7jNtANS1I7fQOvtzbiHsv6qKVmN4codXHc3C4kk=
|
||||
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8=
|
||||
github.com/lbryio/lbry.go v1.1.1-0.20190825202001-8fa28d3d656f h1:ovd2wPXzkT80vdP/FX5xcQeXu0i9RAo80SQ6qIsrAjM=
|
||||
github.com/lbryio/lbry.go v1.1.1-0.20190825202001-8fa28d3d656f/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
|
||||
github.com/lbryio/lbry.go/v2 v2.7.2-0.20210416195322-6516df1418e3 h1:hkVViG8qbOKJMf/M6mN+Hrtns4j55IU2dRavpJZWbxw=
|
||||
github.com/lbryio/lbry.go/v2 v2.7.2-0.20210416195322-6516df1418e3/go.mod h1:I1q8W9fwU+t0IWNiprPgE1SorWQwcO6ser0nzP3L5Pk=
|
||||
github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19 h1:/zWD8dVIl7bV1TdJWqPqy9tpqixzX2Qxgit48h3hQcY=
|
||||
github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
|
||||
github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
|
||||
github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4=
|
||||
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
|
||||
github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386 h1:JOQkGpeCM9FWkEHRx+kRPqySPCXElNW1em1++7tVS4M=
|
||||
github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
|
||||
github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
|
||||
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
|
||||
github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
|
||||
github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
|
||||
github.com/lucas-clemente/quic-go v0.20.1 h1:hb5m76V8QS/8Nw/suHvXqo3BMHAozvIkcnzpJdpanSk=
|
||||
github.com/lucas-clemente/quic-go v0.20.1/go.mod h1:fZq/HUDIM+mW6X6wtzORjC0E/WDBMKe5Hf9bgjISwLk=
|
||||
github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
|
||||
github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0=
|
||||
github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg=
|
||||
github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
|
||||
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 h1:mG83tLXWSRdcXMWfkoumVwhcCbf3jHF9QKv/m37BkM0=
|
||||
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCWffGOaDcjkw1iB7W9DVLp6GXmfcJY/7YZCWPA4=
|
||||
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
|
||||
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
|
||||
github.com/marten-seemann/qpack v0.2.1 h1:jvTsT/HpCn2UZJdP+UUB53FfUUgeOyG5K1ns0OJOGVs=
|
||||
github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc=
|
||||
github.com/marten-seemann/qtls-go1-15 v0.1.4 h1:RehYMOyRW8hPVEja1KBVsFVNSm35Jj9Mvs5yNoZZ28A=
|
||||
github.com/marten-seemann/qtls-go1-15 v0.1.4/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I=
|
||||
github.com/marten-seemann/qtls-go1-16 v0.1.3 h1:XEZ1xGorVy9u+lJq+WXNE+hiqRYLNvJGYmwfwKQN2gU=
|
||||
github.com/marten-seemann/qtls-go1-16 v0.1.3/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk=
|
||||
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
|
||||
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
|
||||
github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
|
||||
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||
github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
|
||||
github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
|
||||
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
||||
github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
|
||||
github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
|
||||
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
|
||||
github.com/miekg/dns v1.1.26 h1:gPxPSwALAeHJSjarOs00QjVdV9QoBvc1D2ujQUr5BzU=
|
||||
github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
|
||||
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
|
||||
github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI=
|
||||
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
|
||||
github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
|
||||
github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
|
||||
github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/mitchellh/mapstructure v0.0.0-20180511142126-bb74f1db0675/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
|
||||
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
|
||||
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg=
|
||||
github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU=
|
||||
github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k=
|
||||
github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w=
|
||||
github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
|
||||
github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
|
||||
github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
|
||||
github.com/nlopes/slack v0.5.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM=
|
||||
github.com/nlopes/slack v0.6.0 h1:jt0jxVQGhssx1Ib7naAOZEZcGdtIhTzkP0nopK0AsRA=
|
||||
github.com/nlopes/slack v0.6.0/go.mod h1:JzQ9m3PMAqcpeCam7UaHSuBuupz7CmpjehYMayT6YOk=
|
||||
github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
|
||||
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
|
||||
github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs=
|
||||
github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
|
||||
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
|
||||
github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
|
||||
github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA=
|
||||
github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
|
||||
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
|
||||
github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
|
||||
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
|
||||
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
|
||||
github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis=
|
||||
github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74=
|
||||
github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
|
||||
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
|
||||
github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA=
|
||||
github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
|
||||
github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
|
||||
github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
|
||||
github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
|
||||
github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM=
|
||||
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
||||
github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
|
||||
github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
||||
github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
|
||||
github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
|
||||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||
github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac=
|
||||
github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6 h1:2bae6N0SZjgzk+Zg8mzTsfmpwHXY9VBNp9UdjhaElA0=
|
||||
github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
|
||||
github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
|
||||
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
|
||||
github.com/pkg/profile v1.3.0/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
|
||||
github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
|
||||
github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s=
|
||||
github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
|
||||
github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
|
||||
github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
|
||||
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
|
||||
github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og=
|
||||
github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
|
||||
github.com/prometheus/client_golang v1.10.0 h1:/o0BDeWzLWXNZ+4q5gXltUvaMpJqckTa+jTNoB+z4cg=
|
||||
github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU=
|
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
||||
github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
||||
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
|
||||
github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
|
||||
github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
|
||||
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
|
||||
github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
|
||||
github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
|
||||
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
|
||||
github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
|
||||
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
|
||||
github.com/prometheus/common v0.18.0 h1:WCVKW7aL6LEe1uryfI9dnEc2ZqNB1Fn0ok930v0iL1Y=
|
||||
github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s=
|
||||
github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
|
||||
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
|
||||
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
|
||||
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
|
||||
github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
|
||||
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
|
||||
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
||||
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rubenv/sql-migrate v0.0.0-20170330050058-38004e7a77f2/go.mod h1:WS0rl9eEliYI8DPnr3TOwz4439pay+qNgzJoVya/DmY=
|
||||
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
|
||||
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
|
||||
github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
|
||||
github.com/sebdah/goldie v0.0.0-20180424091453-8784dd1ab561/go.mod h1:lvjGftC8oe7XPtyrOidaMi0rp5B9+XY/ZRUynGnuaxQ=
|
||||
github.com/sebdah/goldie v0.0.0-20190531093107-d313ffb52c77 h1:Msb6XRY62jQOueNNlB5LGin1rljK7c49NLniGwJG2bg=
|
||||
github.com/sebdah/goldie v0.0.0-20190531093107-d313ffb52c77/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
|
||||
github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
|
||||
github.com/sfreiberg/gotwilio v0.0.0-20180612161623-8fb7259ba8bf/go.mod h1:60PiR0SAnAcYSiwrXB6BaxeqHdXMf172toCosHfV+Yk=
|
||||
github.com/shirou/gopsutil/v3 v3.21.2/go.mod h1:ghfMypLDrFSWN2c9cDYFLHyynQ+QUht0cv/18ZqVczw=
|
||||
github.com/shopspring/decimal v0.0.0-20180607144847-19e3cb6c2930/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
|
||||
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
|
||||
github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5 h1:Gojs/hac/DoYEM7WEICT45+hNWczIeuL5D21e5/HPAw=
|
||||
github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
|
||||
github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY=
|
||||
github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM=
|
||||
github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0=
|
||||
github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
|
||||
github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
|
||||
github.com/shurcooL/gofontwoff v0.0.0-20180329035133-29b52fc0a18d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw=
|
||||
github.com/shurcooL/gopherjslib v0.0.0-20160914041154-feb6d3990c2c/go.mod h1:8d3azKNyqcHP1GaQE/c6dDgjkgSx2BZ4IoEi4F1reUI=
|
||||
github.com/shurcooL/highlight_diff v0.0.0-20170515013008-09bb4053de1b/go.mod h1:ZpfEhSmds4ytuByIcDnOLkTHGUI6KNqRNPDLHDk+mUU=
|
||||
github.com/shurcooL/highlight_go v0.0.0-20181028180052-98c3abbbae20/go.mod h1:UDKB5a1T23gOMUJrI+uSuH0VRDStOiUVSjBTRDVBVag=
|
||||
github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg=
|
||||
github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw=
|
||||
github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y=
|
||||
github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg=
|
||||
github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q=
|
||||
github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ=
|
||||
github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I=
|
||||
github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b5uSkrEVM1jQUspwbixRBhaIjIzL2xazXp6kntxYle0=
|
||||
github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ=
|
||||
github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk=
|
||||
github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
|
||||
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
|
||||
github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4=
|
||||
github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
|
||||
github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
|
||||
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w=
|
||||
github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
|
||||
github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
|
||||
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
|
||||
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
|
||||
github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
|
||||
github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE=
|
||||
github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA=
|
||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/spf13/afero v1.1.1/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
|
||||
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
|
||||
github.com/spf13/afero v1.4.1 h1:asw9sl74539yqavKaglDM5hFpdJVK0Y5Dr/JOgQ89nQ=
|
||||
github.com/spf13/afero v1.4.1/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
|
||||
github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
|
||||
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
|
||||
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
|
||||
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
|
||||
github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M=
|
||||
github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
|
||||
github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
|
||||
github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
|
||||
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
|
||||
github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
|
||||
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
|
||||
github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
|
||||
github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk=
|
||||
github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
|
||||
github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
|
||||
github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
|
||||
github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
|
||||
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
|
||||
github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
|
||||
github.com/tklauser/go-sysconf v0.3.4/go.mod h1:Cl2c8ZRWfHD5IrfHo9VN+FX9kCFjIOyVklgXycLB6ek=
|
||||
github.com/tklauser/numcpus v0.2.1/go.mod h1:9aU+wOc6WjUIZEwWMP62PL/41d65P+iks1gBkr4QyP8=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
|
||||
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
|
||||
github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
|
||||
github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
|
||||
github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
|
||||
github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
|
||||
github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
|
||||
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
|
||||
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
|
||||
github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
|
||||
github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
|
||||
github.com/volatiletech/inflect v0.0.0-20170731032912-e7201282ae8d h1:gI4/tqP6lCY5k6Sg+4k9qSoBXmPwG+xXgMpK7jivD4M=
|
||||
github.com/volatiletech/inflect v0.0.0-20170731032912-e7201282ae8d/go.mod h1:jspfvgf53t5NLUT4o9L1IX0kIBNKamGq1tWc/MgWK9Q=
|
||||
github.com/volatiletech/null v8.0.0+incompatible h1:7wP8m5d/gZ6kW/9GnrLtMCRre2dlEnaQ9Km5OXlK4zg=
|
||||
github.com/volatiletech/null v8.0.0+incompatible/go.mod h1:0wD98JzdqB+rLyZ70fN05VDbXbafIb0KU0MdVhCzmOQ=
|
||||
github.com/volatiletech/sqlboiler v3.4.0+incompatible h1:saQ6WxZ9wEJp33q3w/DHs7an7SYi1H7Yzf4/moxCbJU=
|
||||
github.com/volatiletech/sqlboiler v3.4.0+incompatible/go.mod h1:jLfDkkHWPbS2cWRLkyC20vQWaIQsASEY7gM7zSo11Yw=
|
||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
|
||||
github.com/xlab/treeprint v1.0.0/go.mod h1:IoImgRak9i3zJyuxOKUP1v4UZd1tMoKkq/Cimt1uhCg=
|
||||
github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE=
|
||||
github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0=
|
||||
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
|
||||
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
|
||||
go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
|
||||
go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
|
||||
go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
|
||||
go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
|
||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
|
||||
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
|
||||
go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
|
||||
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
|
||||
go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
|
||||
go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
|
||||
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
|
||||
go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
|
||||
go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
|
||||
golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw=
|
||||
golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
|
||||
golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
|
||||
golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
|
||||
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
|
||||
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
|
||||
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
|
||||
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20191009170851-d66e71096ffb/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0=
|
||||
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190520201301-c432e742b0af/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201231184435-2d18734c6014/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
|
||||
google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
|
||||
google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y=
|
||||
google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
|
||||
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
|
||||
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
|
||||
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
|
||||
google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
|
||||
google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
|
||||
google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
|
||||
google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||
google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
|
||||
google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
|
||||
google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
|
||||
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
|
||||
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
|
||||
google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
|
||||
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
|
||||
google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
|
||||
google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
|
||||
google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
||||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
|
||||
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
|
||||
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
|
||||
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=
|
||||
gopkg.in/gorp.v1 v1.7.1/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw=
|
||||
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
|
||||
gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/ini.v1 v1.48.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
|
||||
gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 h1:FpCr9V8wuOei4BAen+93HtVJ+XSi+KPbaPKm0Vj5R64=
|
||||
gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79/go.mod h1:gWkaRU7CoXpezCBWfWjm3999QqS+1pYPXGbqQCTMzo8=
|
||||
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
|
||||
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
|
||||
gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
|
||||
grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o=
|
||||
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
|
||||
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
|
||||
rsc.io/goversion v1.2.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo=
|
||||
sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
|
||||
sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU=
|
||||
sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck=
|
||||
sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
|
||||
@@ -0,0 +1,324 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
ee "github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type Server struct {
|
||||
srv *http.Server
|
||||
stop *stop.Stopper
|
||||
}
|
||||
|
||||
func NewServer(address string, path string) *Server {
|
||||
h := http.NewServeMux()
|
||||
h.Handle(path, promhttp.Handler())
|
||||
return &Server{
|
||||
srv: &http.Server{
|
||||
Addr: address,
|
||||
Handler: h,
|
||||
//https://blog.cloudflare.com/the-complete-guide-to-golang-net-http-timeouts/
|
||||
//https://blog.cloudflare.com/exposing-go-on-the-internet/
|
||||
ReadTimeout: 5 * time.Second,
|
||||
WriteTimeout: 10 * time.Second,
|
||||
IdleTimeout: 120 * time.Second,
|
||||
},
|
||||
stop: stop.New(),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) Start() {
|
||||
s.stop.Add(1)
|
||||
go func() {
|
||||
defer s.stop.Done()
|
||||
err := s.srv.ListenAndServe()
|
||||
if err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
log.Error(err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (s *Server) Shutdown() {
|
||||
_ = s.srv.Shutdown(context.Background())
|
||||
s.stop.StopAndWait()
|
||||
}
|
||||
|
||||
const (
|
||||
ns = "reflector"
|
||||
subsystemCache = "cache"
|
||||
subsystemITTT = "ittt"
|
||||
|
||||
labelDirection = "direction"
|
||||
labelErrorType = "error_type"
|
||||
|
||||
DirectionUpload = "upload" // to reflector
|
||||
DirectionDownload = "download" // from reflector
|
||||
|
||||
LabelCacheType = "cache_type"
|
||||
LabelComponent = "component"
|
||||
LabelSource = "source"
|
||||
|
||||
errConnReset = "conn_reset"
|
||||
errReadConnReset = "read_conn_reset"
|
||||
errWriteConnReset = "write_conn_reset"
|
||||
errReadConnTimedOut = "read_conn_timed_out"
|
||||
errNoNetworkActivity = "no_network_activity"
|
||||
errWriteConnTimedOut = "write_conn_timed_out"
|
||||
errWriteBrokenPipe = "write_broken_pipe"
|
||||
errEPipe = "e_pipe"
|
||||
errETimedout = "e_timedout"
|
||||
errIOTimeout = "io_timeout"
|
||||
errUnexpectedEOF = "unexpected_eof"
|
||||
errUnexpectedEOFStr = "unexpected_eof_str"
|
||||
errJSONSyntax = "json_syntax"
|
||||
errBlobTooBig = "blob_too_big"
|
||||
errDeadlineExceeded = "deadline_exceeded"
|
||||
errHashMismatch = "hash_mismatch"
|
||||
errZeroByteBlob = "zero_byte_blob"
|
||||
errInvalidCharacter = "invalid_character"
|
||||
errBlobNotFound = "blob_not_found"
|
||||
errNoErr = "no_error"
|
||||
errQuicProto = "quic_protocol_violation"
|
||||
errOther = "other"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrorCount = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "error_total",
|
||||
Help: "Total number of errors",
|
||||
}, []string{labelDirection, labelErrorType})
|
||||
|
||||
BlobDownloadCount = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "blob_download_total",
|
||||
Help: "Total number of blobs downloaded from reflector",
|
||||
})
|
||||
PeerDownloadCount = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "peer_download_total",
|
||||
Help: "Total number of blobs downloaded from reflector through tcp protocol",
|
||||
})
|
||||
Http3DownloadCount = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "http3_blob_download_total",
|
||||
Help: "Total number of blobs downloaded from reflector through QUIC protocol",
|
||||
})
|
||||
HttpDownloadCount = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "http_blob_download_total",
|
||||
Help: "Total number of blobs downloaded from reflector through HTTP protocol",
|
||||
})
|
||||
|
||||
CacheHitCount = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Subsystem: subsystemCache,
|
||||
Name: "hit_total",
|
||||
Help: "Total number of blobs retrieved from the cache storage",
|
||||
}, []string{LabelCacheType, LabelComponent})
|
||||
ThisHitCount = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Subsystem: subsystemITTT,
|
||||
Name: "this_hit_total",
|
||||
Help: "Total number of blobs retrieved from the this storage",
|
||||
})
|
||||
ThatHitCount = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Subsystem: subsystemITTT,
|
||||
Name: "that_hit_total",
|
||||
Help: "Total number of blobs retrieved from the that storage",
|
||||
})
|
||||
CacheMissCount = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Subsystem: subsystemCache,
|
||||
Name: "miss_total",
|
||||
Help: "Total number of blobs retrieved from origin rather than cache storage",
|
||||
}, []string{LabelCacheType, LabelComponent})
|
||||
CacheOriginRequestsCount = promauto.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: ns,
|
||||
Subsystem: subsystemCache,
|
||||
Name: "origin_requests_total",
|
||||
Help: "How many Get requests are in flight from the cache to the origin",
|
||||
}, []string{LabelCacheType, LabelComponent})
|
||||
//during thundering-herd situations, the metric below should be a lot smaller than the metric above
|
||||
CacheWaitingRequestsCount = promauto.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: ns,
|
||||
Subsystem: subsystemCache,
|
||||
Name: "waiting_requests_total",
|
||||
Help: "How many cache requests are waiting for an in-flight origin request",
|
||||
}, []string{LabelCacheType, LabelComponent})
|
||||
CacheLRUEvictCount = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Subsystem: subsystemCache,
|
||||
Name: "evict_total",
|
||||
Help: "Count of blobs evicted from cache",
|
||||
}, []string{LabelCacheType, LabelComponent})
|
||||
CacheRetrievalSpeed = promauto.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: ns,
|
||||
Name: "speed_mbps",
|
||||
Help: "Speed of blob retrieval from cache or from origin",
|
||||
}, []string{LabelCacheType, LabelComponent, LabelSource})
|
||||
|
||||
BlobUploadCount = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "blob_upload_total",
|
||||
Help: "Total number of blobs uploaded to reflector",
|
||||
})
|
||||
SDBlobUploadCount = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "sdblob_upload_total",
|
||||
Help: "Total number of SD blobs (and therefore streams) uploaded to reflector",
|
||||
})
|
||||
|
||||
MtrInBytesTcp = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "tcp_in_bytes",
|
||||
Help: "Total number of bytes downloaded through TCP",
|
||||
})
|
||||
MtrOutBytesTcp = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "tcp_out_bytes",
|
||||
Help: "Total number of bytes streamed out through TCP",
|
||||
})
|
||||
MtrInBytesUdp = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "udp_in_bytes",
|
||||
Help: "Total number of bytes downloaded through UDP",
|
||||
})
|
||||
MtrInBytesHttp = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "http_in_bytes",
|
||||
Help: "Total number of bytes downloaded through HTTP",
|
||||
})
|
||||
MtrOutBytesUdp = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "udp_out_bytes",
|
||||
Help: "Total number of bytes streamed out through UDP",
|
||||
})
|
||||
MtrOutBytesHttp = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "http_out_bytes",
|
||||
Help: "Total number of bytes streamed out through UDP",
|
||||
})
|
||||
MtrInBytesReflector = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "reflector_in_bytes",
|
||||
Help: "Total number of incoming bytes (from users)",
|
||||
})
|
||||
MtrOutBytesReflector = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "s3_out_bytes",
|
||||
Help: "Total number of outgoing bytes (to S3)",
|
||||
})
|
||||
MtrInBytesS3 = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "s3_in_bytes",
|
||||
Help: "Total number of incoming bytes (from S3-CF)",
|
||||
})
|
||||
Http3BlobReqQueue = promauto.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: ns,
|
||||
Name: "http3_blob_request_queue_size",
|
||||
Help: "Blob requests of https queue size",
|
||||
})
|
||||
HttpBlobReqQueue = promauto.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: ns,
|
||||
Name: "http_blob_request_queue_size",
|
||||
Help: "Blob requests queue size of the HTTP protocol",
|
||||
})
|
||||
RoutinesQueue = promauto.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: ns,
|
||||
Name: "routines",
|
||||
Help: "routines running by type",
|
||||
}, []string{"package", "kind"})
|
||||
)
|
||||
|
||||
func CacheLabels(name, component string) prometheus.Labels {
|
||||
return prometheus.Labels{
|
||||
LabelCacheType: name,
|
||||
LabelComponent: component,
|
||||
}
|
||||
}
|
||||
|
||||
func TrackError(direction string, e error) (shouldLog bool) { // shouldLog is a hack, but whatever
|
||||
if e == nil {
|
||||
return
|
||||
}
|
||||
|
||||
err := ee.Wrap(e, 0)
|
||||
errType := errOther
|
||||
if strings.Contains(err.Error(), "i/o timeout") {
|
||||
errType = errIOTimeout
|
||||
} else if errors.Is(e, syscall.ECONNRESET) {
|
||||
// Looks like we're getting this when direction == "download", but read_conn_reset and
|
||||
// write_conn_reset when its "upload"
|
||||
errType = errConnReset
|
||||
} else if errors.Is(e, context.DeadlineExceeded) {
|
||||
errType = errDeadlineExceeded
|
||||
} else if strings.Contains(err.Error(), "read: connection reset by peer") { // the other side closed the connection using TCP reset
|
||||
errType = errReadConnReset
|
||||
} else if strings.Contains(err.Error(), "write: connection reset by peer") { // the other side closed the connection using TCP reset
|
||||
errType = errWriteConnReset
|
||||
} else if errors.Is(e, syscall.ETIMEDOUT) {
|
||||
errType = errETimedout
|
||||
} else if strings.Contains(err.Error(), "read: connection timed out") { // the other side closed the connection using TCP reset
|
||||
//log.Warnln("read conn timed out is not the same as ETIMEDOUT")
|
||||
errType = errReadConnTimedOut
|
||||
} else if strings.Contains(err.Error(), "NO_ERROR: No recent network activity") { // the other side closed the QUIC connection
|
||||
//log.Warnln("read conn timed out is not the same as ETIMEDOUT")
|
||||
errType = errNoNetworkActivity
|
||||
} else if strings.Contains(err.Error(), "write: connection timed out") {
|
||||
errType = errWriteConnTimedOut
|
||||
} else if errors.Is(e, io.ErrUnexpectedEOF) {
|
||||
errType = errUnexpectedEOF
|
||||
} else if strings.Contains(err.Error(), "unexpected EOF") { // tried to read from closed pipe or socket
|
||||
errType = errUnexpectedEOFStr
|
||||
} else if errors.Is(e, syscall.EPIPE) {
|
||||
errType = errEPipe
|
||||
} else if strings.Contains(err.Error(), "write: broken pipe") { // tried to write to a pipe or socket that was closed by the peer
|
||||
// I believe this is the same as EPipe when direction == "download", but not for upload
|
||||
errType = errWriteBrokenPipe
|
||||
//} else if errors.Is(e, reflector.ErrBlobTooBig) { # this creates a circular import
|
||||
// errType = errBlobTooBig
|
||||
} else if strings.Contains(err.Error(), "blob must be at most") {
|
||||
//log.Warnln("blob must be at most X bytes is not the same as ErrBlobTooBig")
|
||||
errType = errBlobTooBig
|
||||
} else if strings.Contains(err.Error(), "hash of received blob data does not match hash from send request") {
|
||||
errType = errHashMismatch
|
||||
} else if strings.Contains(err.Error(), "blob not found") {
|
||||
errType = errBlobNotFound
|
||||
} else if strings.Contains(err.Error(), "0-byte blob received") {
|
||||
errType = errZeroByteBlob
|
||||
} else if strings.Contains(err.Error(), "PROTOCOL_VIOLATION: tried to retire connection") {
|
||||
errType = errQuicProto
|
||||
} else if strings.Contains(err.Error(), "invalid character") {
|
||||
errType = errInvalidCharacter
|
||||
} else if _, ok := e.(*json.SyntaxError); ok {
|
||||
errType = errJSONSyntax
|
||||
} else if strings.Contains(err.Error(), "NO_ERROR") {
|
||||
errType = errNoErr
|
||||
} else {
|
||||
log.Warnf("error '%s' for direction '%s' is not being tracked", err.TypeName(), direction)
|
||||
shouldLog = true
|
||||
}
|
||||
|
||||
ErrorCount.With(map[string]string{
|
||||
labelDirection: direction,
|
||||
labelErrorType: errType,
|
||||
}).Inc()
|
||||
|
||||
return
|
||||
}
|
||||
+348
@@ -0,0 +1,348 @@
|
||||
package lite_db
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
qt "github.com/lbryio/lbry.go/v2/extras/query"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
_ "github.com/go-sql-driver/mysql" // blank import for db driver ensures its imported even if its not used
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/volatiletech/null"
|
||||
)
|
||||
|
||||
// SdBlob is a special blob that contains information on the rest of the blobs in the stream
|
||||
type SdBlob struct {
|
||||
StreamName string `json:"stream_name"`
|
||||
Blobs []struct {
|
||||
Length int `json:"length"`
|
||||
BlobNum int `json:"blob_num"`
|
||||
BlobHash string `json:"blob_hash,omitempty"`
|
||||
IV string `json:"iv"`
|
||||
} `json:"blobs"`
|
||||
StreamType string `json:"stream_type"`
|
||||
Key string `json:"key"`
|
||||
SuggestedFileName string `json:"suggested_file_name"`
|
||||
StreamHash string `json:"stream_hash"`
|
||||
}
|
||||
|
||||
// SQL implements the DB interface
|
||||
type SQL struct {
|
||||
conn *sql.DB
|
||||
|
||||
TrackAccessTime bool
|
||||
}
|
||||
|
||||
func logQuery(query string, args ...interface{}) {
|
||||
s, err := qt.InterpolateParams(query, args...)
|
||||
if err != nil {
|
||||
log.Errorln(err)
|
||||
} else {
|
||||
log.Debugln(s)
|
||||
}
|
||||
}
|
||||
|
||||
// Connect will create a connection to the database
|
||||
func (s *SQL) Connect(dsn string) error {
|
||||
var err error
|
||||
// interpolateParams is necessary. otherwise uploading a stream with thousands of blobs
|
||||
// will hit MySQL's max_prepared_stmt_count limit because the prepared statements are all
|
||||
// opened inside a transaction. closing them manually doesn't seem to help
|
||||
dsn += "?parseTime=1&collation=utf8mb4_unicode_ci&interpolateParams=1"
|
||||
s.conn, err = sql.Open("mysql", dsn)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
s.conn.SetMaxIdleConns(12)
|
||||
|
||||
return errors.Err(s.conn.Ping())
|
||||
}
|
||||
|
||||
// AddBlob adds a blob to the database.
|
||||
func (s *SQL) AddBlob(hash string, length int) error {
|
||||
if s.conn == nil {
|
||||
return errors.Err("not connected")
|
||||
}
|
||||
|
||||
_, err := s.insertBlob(hash, length)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *SQL) insertBlob(hash string, length int) (int64, error) {
|
||||
if length <= 0 {
|
||||
return 0, errors.Err("length must be positive")
|
||||
}
|
||||
const isStored = true
|
||||
now := time.Now()
|
||||
args := []interface{}{hash, isStored, length, now}
|
||||
blobID, err := s.exec(
|
||||
"INSERT INTO blob_ (hash, is_stored, length, last_accessed_at) VALUES ("+qt.Qs(len(args))+") ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored)), last_accessed_at=VALUES(last_accessed_at)",
|
||||
args...,
|
||||
)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if blobID == 0 {
|
||||
err = s.conn.QueryRow("SELECT id FROM blob_ WHERE hash = ?", hash).Scan(&blobID)
|
||||
if err != nil {
|
||||
return 0, errors.Err(err)
|
||||
}
|
||||
if blobID == 0 {
|
||||
return 0, errors.Err("blob ID is 0 even after INSERTing and SELECTing")
|
||||
}
|
||||
}
|
||||
|
||||
return blobID, nil
|
||||
}
|
||||
|
||||
// HasBlob checks if the database contains the blob information.
|
||||
func (s *SQL) HasBlob(hash string) (bool, error) {
|
||||
exists, err := s.HasBlobs([]string{hash})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return exists[hash], nil
|
||||
}
|
||||
|
||||
// HasBlobs checks if the database contains the set of blobs and returns a bool map.
|
||||
func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
|
||||
exists, streamsNeedingTouch, err := s.hasBlobs(hashes)
|
||||
s.touch(streamsNeedingTouch)
|
||||
return exists, err
|
||||
}
|
||||
|
||||
func (s *SQL) touch(blobIDs []uint64) error {
|
||||
if len(blobIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
query := "UPDATE blob_ SET last_accessed_at = ? WHERE id IN (" + qt.Qs(len(blobIDs)) + ")"
|
||||
args := make([]interface{}, len(blobIDs)+1)
|
||||
args[0] = time.Now()
|
||||
for i := range blobIDs {
|
||||
args[i+1] = blobIDs[i]
|
||||
}
|
||||
|
||||
startTime := time.Now()
|
||||
_, err := s.exec(query, args...)
|
||||
log.Debugf("blobs access query touched %d blobs and took %s", len(blobIDs), time.Since(startTime))
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
func (s *SQL) hasBlobs(hashes []string) (map[string]bool, []uint64, error) {
|
||||
if s.conn == nil {
|
||||
return nil, nil, errors.Err("not connected")
|
||||
}
|
||||
|
||||
var (
|
||||
hash string
|
||||
blobID uint64
|
||||
lastAccessedAt null.Time
|
||||
)
|
||||
|
||||
var needsTouch []uint64
|
||||
exists := make(map[string]bool)
|
||||
|
||||
touchDeadline := time.Now().AddDate(0, 0, -1) // touch blob if last accessed before this time
|
||||
maxBatchSize := 10000
|
||||
doneIndex := 0
|
||||
|
||||
for len(hashes) > doneIndex {
|
||||
sliceEnd := doneIndex + maxBatchSize
|
||||
if sliceEnd > len(hashes) {
|
||||
sliceEnd = len(hashes)
|
||||
}
|
||||
log.Debugf("getting hashes[%d:%d] of %d", doneIndex, sliceEnd, len(hashes))
|
||||
batch := hashes[doneIndex:sliceEnd]
|
||||
|
||||
// TODO: this query doesn't work for SD blobs, which are not in the stream_blob table
|
||||
|
||||
query := `SELECT hash, id, last_accessed_at
|
||||
FROM blob_
|
||||
WHERE is_stored = ? and hash IN (` + qt.Qs(len(batch)) + `)`
|
||||
args := make([]interface{}, len(batch)+1)
|
||||
args[0] = true
|
||||
for i := range batch {
|
||||
args[i+1] = batch[i]
|
||||
}
|
||||
|
||||
logQuery(query, args...)
|
||||
|
||||
err := func() error {
|
||||
startTime := time.Now()
|
||||
rows, err := s.conn.Query(query, args...)
|
||||
log.Debugf("hashes query took %s", time.Since(startTime))
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer closeRows(rows)
|
||||
|
||||
for rows.Next() {
|
||||
err := rows.Scan(&hash, &blobID, &lastAccessedAt)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
exists[hash] = true
|
||||
if s.TrackAccessTime && (!lastAccessedAt.Valid || lastAccessedAt.Time.Before(touchDeadline)) {
|
||||
needsTouch = append(needsTouch, blobID)
|
||||
}
|
||||
}
|
||||
|
||||
err = rows.Err()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
doneIndex += len(batch)
|
||||
return nil
|
||||
}()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return exists, needsTouch, nil
|
||||
}
|
||||
|
||||
// Delete will remove the blob from the db
|
||||
func (s *SQL) Delete(hash string) error {
|
||||
_, err := s.exec("UPDATE blob_ set is_stored = ? WHERE hash = ?", 0, hash)
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
// AddSDBlob insert the SD blob and all the content blobs. The content blobs are marked as "not stored",
|
||||
// but they are tracked so reflector knows what it is missing.
|
||||
func (s *SQL) AddSDBlob(sdHash string, sdBlobLength int) error {
|
||||
if s.conn == nil {
|
||||
return errors.Err("not connected")
|
||||
}
|
||||
|
||||
_, err := s.insertBlob(sdHash, sdBlobLength)
|
||||
return err
|
||||
}
|
||||
|
||||
// GetHashRange gets the smallest and biggest hashes in the db
|
||||
func (s *SQL) GetLRUBlobs(maxBlobs int) ([]string, error) {
|
||||
if s.conn == nil {
|
||||
return nil, errors.Err("not connected")
|
||||
}
|
||||
|
||||
query := "SELECT hash from blob_ where is_stored = ? order by last_accessed_at limit ?"
|
||||
const isStored = true
|
||||
logQuery(query, isStored, maxBlobs)
|
||||
rows, err := s.conn.Query(query, isStored, maxBlobs)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
defer closeRows(rows)
|
||||
blobs := make([]string, 0, maxBlobs)
|
||||
for rows.Next() {
|
||||
var hash string
|
||||
err := rows.Scan(&hash)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
blobs = append(blobs, hash)
|
||||
}
|
||||
return blobs, nil
|
||||
}
|
||||
|
||||
func (s *SQL) AllBlobs() ([]string, error) {
|
||||
if s.conn == nil {
|
||||
return nil, errors.Err("not connected")
|
||||
}
|
||||
|
||||
query := "SELECT hash from blob_ where is_stored = ?" //TODO: maybe sorting them makes more sense?
|
||||
const isStored = true
|
||||
logQuery(query, isStored)
|
||||
rows, err := s.conn.Query(query, isStored)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
defer closeRows(rows)
|
||||
totalBlobs, err := s.BlobsCount()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
blobs := make([]string, 0, totalBlobs)
|
||||
for rows.Next() {
|
||||
var hash string
|
||||
err := rows.Scan(&hash)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
blobs = append(blobs, hash)
|
||||
}
|
||||
return blobs, nil
|
||||
}
|
||||
|
||||
func (s *SQL) BlobsCount() (int, error) {
|
||||
if s.conn == nil {
|
||||
return 0, errors.Err("not connected")
|
||||
}
|
||||
|
||||
query := "SELECT count(id) from blob_ where is_stored = ?" //TODO: maybe sorting them makes more sense?
|
||||
const isStored = true
|
||||
logQuery(query, isStored)
|
||||
var count int
|
||||
err := s.conn.QueryRow(query, isStored).Scan(&count)
|
||||
return count, errors.Err(err)
|
||||
}
|
||||
|
||||
func closeRows(rows *sql.Rows) {
|
||||
if rows != nil {
|
||||
err := rows.Close()
|
||||
if err != nil {
|
||||
log.Error("error closing rows: ", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *SQL) exec(query string, args ...interface{}) (int64, error) {
|
||||
logQuery(query, args...)
|
||||
attempt, maxAttempts := 0, 3
|
||||
Retry:
|
||||
attempt++
|
||||
result, err := s.conn.Exec(query, args...)
|
||||
if isLockTimeoutError(err) {
|
||||
if attempt <= maxAttempts {
|
||||
//Error 1205: Lock wait timeout exceeded; try restarting transaction
|
||||
goto Retry
|
||||
}
|
||||
err = errors.Prefix("Lock timeout for query "+query, err)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return 0, errors.Err(err)
|
||||
}
|
||||
|
||||
lastID, err := result.LastInsertId()
|
||||
return lastID, errors.Err(err)
|
||||
}
|
||||
|
||||
func isLockTimeoutError(err error) bool {
|
||||
e, ok := err.(*mysql.MySQLError)
|
||||
return ok && e != nil && e.Number == 1205
|
||||
}
|
||||
|
||||
/* SQL schema
|
||||
|
||||
in prod make sure you use latin1 or utf8 charset, NOT utf8mb4. that's a waste of space.
|
||||
|
||||
CREATE TABLE `blob_` (
|
||||
`id` bigint unsigned NOT NULL AUTO_INCREMENT,
|
||||
`hash` char(96) NOT NULL,
|
||||
`is_stored` tinyint(1) NOT NULL DEFAULT '0',
|
||||
`length` bigint unsigned DEFAULT NULL,
|
||||
`last_accessed_at` datetime DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `id` (`id`),
|
||||
UNIQUE KEY `blob_hash_idx` (`hash`),
|
||||
KEY `blob_last_accessed_idx` (`last_accessed_at`)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=latin1
|
||||
|
||||
*/
|
||||
@@ -1,7 +1,19 @@
|
||||
package main
|
||||
|
||||
import "github.com/lbryio/reflector.go/cmd"
|
||||
import (
|
||||
"math/rand"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/cmd"
|
||||
|
||||
"github.com/google/gops/agent"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := agent.Listen(agent.Options{}); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
cmd.Execute()
|
||||
}
|
||||
@@ -1,3 +1,36 @@
|
||||
package meta
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
var Version = ""
|
||||
var Time = ""
|
||||
var BuildTime time.Time
|
||||
|
||||
func init() {
|
||||
if Time != "" {
|
||||
t, err := strconv.Atoi(Time)
|
||||
if err == nil {
|
||||
BuildTime = time.Unix(int64(t), 0).UTC()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func VersionString() string {
|
||||
version := Version
|
||||
if version == "" {
|
||||
version = "<unset>"
|
||||
}
|
||||
|
||||
var buildTime string
|
||||
if BuildTime.IsZero() {
|
||||
buildTime = "<now>"
|
||||
} else {
|
||||
buildTime = BuildTime.Format(time.RFC3339)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("version %s, built %s", version, buildTime)
|
||||
}
|
||||
-316
@@ -1,316 +0,0 @@
|
||||
package peer
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"crypto/sha512"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/lbry.go/stop"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const (
|
||||
// DefaultPort is the port the peer server listens on if not passed in.
|
||||
DefaultPort = 3333
|
||||
// LbrycrdAddress to be used when paying for data. Not implemented yet.
|
||||
LbrycrdAddress = "bJxKvpD96kaJLriqVajZ7SaQTsWWyrGQct"
|
||||
)
|
||||
|
||||
// Server is an instance of a peer server that houses the listener and store.
|
||||
type Server struct {
|
||||
store store.BlobStore
|
||||
closed bool
|
||||
|
||||
grp *stop.Group
|
||||
}
|
||||
|
||||
// NewServer returns an initialized Server pointer.
|
||||
func NewServer(store store.BlobStore) *Server {
|
||||
return &Server{
|
||||
store: store,
|
||||
grp: stop.New(),
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown gracefully shuts down the peer server.
|
||||
func (s *Server) Shutdown() {
|
||||
log.Debug("shutting down peer server...")
|
||||
s.grp.StopAndWait()
|
||||
log.Debug("peer server stopped")
|
||||
}
|
||||
|
||||
// Start starts the server listener to handle connections.
|
||||
func (s *Server) Start(address string) error {
|
||||
log.Println("peer listening on " + address)
|
||||
l, err := net.Listen("tcp4", address)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
go s.listenForShutdown(l)
|
||||
s.grp.Add(1)
|
||||
go func() {
|
||||
s.listenAndServe(l)
|
||||
s.grp.Done()
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) listenForShutdown(listener net.Listener) {
|
||||
<-s.grp.Ch()
|
||||
s.closed = true
|
||||
err := listener.Close()
|
||||
if err != nil {
|
||||
log.Error("error closing listener for peer server - ", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) listenAndServe(listener net.Listener) {
|
||||
for {
|
||||
conn, err := listener.Accept()
|
||||
if err != nil {
|
||||
if s.closed {
|
||||
return
|
||||
}
|
||||
log.Error(err)
|
||||
} else {
|
||||
s.grp.Add(1)
|
||||
go func() {
|
||||
s.handleConnection(conn)
|
||||
s.grp.Done()
|
||||
}()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) handleConnection(conn net.Conn) {
|
||||
defer func() {
|
||||
if err := conn.Close(); err != nil {
|
||||
log.Error(errors.Prefix("closing peer conn", err))
|
||||
}
|
||||
}()
|
||||
|
||||
timeoutDuration := 5 * time.Second
|
||||
|
||||
for {
|
||||
var request []byte
|
||||
var response []byte
|
||||
|
||||
err := conn.SetReadDeadline(time.Now().Add(timeoutDuration))
|
||||
if err != nil {
|
||||
log.Error(errors.FullTrace(err))
|
||||
}
|
||||
|
||||
request, err = readNextRequest(conn)
|
||||
if err != nil {
|
||||
if err != io.EOF {
|
||||
log.Errorln(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
err = conn.SetReadDeadline(time.Time{})
|
||||
if err != nil {
|
||||
log.Error(errors.FullTrace(err))
|
||||
}
|
||||
|
||||
if strings.Contains(string(request), `"requested_blobs"`) {
|
||||
log.Debugln("received availability request")
|
||||
response, err = s.handleAvailabilityRequest(request)
|
||||
} else if strings.Contains(string(request), `"blob_data_payment_rate"`) {
|
||||
log.Debugln("received rate negotiation request")
|
||||
response, err = s.handlePaymentRateNegotiation(request)
|
||||
} else if strings.Contains(string(request), `"requested_blob"`) {
|
||||
log.Debugln("received blob request")
|
||||
response, err = s.handleBlobRequest(request)
|
||||
} else {
|
||||
log.Errorln("invalid request")
|
||||
spew.Dump(request)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
n, err := conn.Write(response)
|
||||
if err != nil {
|
||||
log.Errorln(err)
|
||||
return
|
||||
} else if n != len(response) {
|
||||
log.Errorln(io.ErrShortWrite)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) handleAvailabilityRequest(data []byte) ([]byte, error) {
|
||||
var request availabilityRequest
|
||||
err := json.Unmarshal(data, &request)
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
|
||||
availableBlobs := []string{}
|
||||
for _, blobHash := range request.RequestedBlobs {
|
||||
exists, err := s.store.Has(blobHash)
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
if exists {
|
||||
availableBlobs = append(availableBlobs, blobHash)
|
||||
}
|
||||
}
|
||||
|
||||
return json.Marshal(availabilityResponse{LbrycrdAddress: LbrycrdAddress, AvailableBlobs: availableBlobs})
|
||||
}
|
||||
|
||||
func (s *Server) handlePaymentRateNegotiation(data []byte) ([]byte, error) {
|
||||
var request paymentRateRequest
|
||||
err := json.Unmarshal(data, &request)
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
|
||||
offerReply := paymentRateAccepted
|
||||
if request.BlobDataPaymentRate < 0 {
|
||||
offerReply = paymentRateTooLow
|
||||
}
|
||||
|
||||
return json.Marshal(paymentRateResponse{BlobDataPaymentRate: offerReply})
|
||||
}
|
||||
|
||||
func (s *Server) handleBlobRequest(data []byte) ([]byte, error) {
|
||||
var request blobRequest
|
||||
err := json.Unmarshal(data, &request)
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
|
||||
log.Println("Sending blob " + request.RequestedBlob[:8])
|
||||
|
||||
blob, err := s.store.Get(request.RequestedBlob)
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
|
||||
response, err := json.Marshal(blobResponse{IncomingBlob: incomingBlob{
|
||||
BlobHash: GetBlobHash(blob),
|
||||
Length: len(blob),
|
||||
}})
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
|
||||
return append(response, blob...), nil
|
||||
}
|
||||
|
||||
func readNextRequest(conn net.Conn) ([]byte, error) {
|
||||
request := make([]byte, 0)
|
||||
eof := false
|
||||
buf := bufio.NewReader(conn)
|
||||
|
||||
for {
|
||||
chunk, err := buf.ReadBytes('}')
|
||||
if err != nil {
|
||||
if err != io.EOF {
|
||||
log.Errorln("read error:", err)
|
||||
return request, err
|
||||
}
|
||||
eof = true
|
||||
}
|
||||
|
||||
//log.Debugln("got", len(chunk), "bytes.")
|
||||
//spew.Dump(chunk)
|
||||
|
||||
if len(chunk) > 0 {
|
||||
request = append(request, chunk...)
|
||||
|
||||
if len(request) > maxRequestSize {
|
||||
return request, errRequestTooLarge
|
||||
}
|
||||
|
||||
// yes, this is how the peer protocol knows when the request finishes
|
||||
if isValidJSON(request) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if eof {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
//log.Debugln("total size:", len(request))
|
||||
//if len(request) > 0 {
|
||||
// spew.Dump(request)
|
||||
//}
|
||||
|
||||
if len(request) == 0 && eof {
|
||||
return []byte{}, io.EOF
|
||||
}
|
||||
|
||||
return request, nil
|
||||
}
|
||||
|
||||
func isValidJSON(b []byte) bool {
|
||||
var r json.RawMessage
|
||||
return json.Unmarshal(b, &r) == nil
|
||||
}
|
||||
|
||||
// GetBlobHash returns the sha512 hash hex encoded string of the blob byte slice.
|
||||
func GetBlobHash(blob []byte) string {
|
||||
hashBytes := sha512.Sum384(blob)
|
||||
return hex.EncodeToString(hashBytes[:])
|
||||
}
|
||||
|
||||
const (
|
||||
maxRequestSize = 64 * (2 ^ 10) // 64kb
|
||||
paymentRateAccepted = "RATE_ACCEPTED"
|
||||
paymentRateTooLow = "RATE_TOO_LOW"
|
||||
//ToDo: paymentRateUnset is not used but exists in the protocol.
|
||||
//paymentRateUnset = "RATE_UNSET"
|
||||
)
|
||||
|
||||
var errRequestTooLarge = errors.Base("request is too large")
|
||||
|
||||
type availabilityRequest struct {
|
||||
LbrycrdAddress bool `json:"lbrycrd_address"`
|
||||
RequestedBlobs []string `json:"requested_blobs"`
|
||||
}
|
||||
|
||||
type availabilityResponse struct {
|
||||
LbrycrdAddress string `json:"lbrycrd_address"`
|
||||
AvailableBlobs []string `json:"available_blobs"`
|
||||
}
|
||||
|
||||
type paymentRateRequest struct {
|
||||
BlobDataPaymentRate float64 `json:"blob_data_payment_rate"`
|
||||
}
|
||||
|
||||
type paymentRateResponse struct {
|
||||
BlobDataPaymentRate string `json:"blob_data_payment_rate"`
|
||||
}
|
||||
|
||||
type blobRequest struct {
|
||||
RequestedBlob string `json:"requested_blob"`
|
||||
}
|
||||
|
||||
type incomingBlob struct {
|
||||
Error string `json:"error,omitempty"`
|
||||
BlobHash string `json:"blob_hash"`
|
||||
Length int `json:"length"`
|
||||
}
|
||||
type blobResponse struct {
|
||||
IncomingBlob incomingBlob `json:"incoming_blob"`
|
||||
}
|
||||
+6
-6
@@ -7,14 +7,14 @@ import (
|
||||
|
||||
"github.com/lbryio/reflector.go/cluster"
|
||||
"github.com/lbryio/reflector.go/db"
|
||||
"github.com/lbryio/reflector.go/dht"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
"github.com/lbryio/reflector.go/peer"
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
"github.com/lbryio/reflector.go/server/peer"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/lbry.go/stop"
|
||||
"github.com/lbryio/lbry.go/v2/dht"
|
||||
"github.com/lbryio/lbry.go/v2/dht/bits"
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
@@ -79,7 +79,7 @@ func New(conf *Config) *Prism {
|
||||
dht: d,
|
||||
cluster: c,
|
||||
peer: peer.NewServer(conf.Blobs),
|
||||
reflector: reflector.NewServer(conf.Blobs),
|
||||
reflector: reflector.NewServer(conf.Blobs, conf.Blobs),
|
||||
|
||||
grp: stop.New(),
|
||||
}
|
||||
|
||||
+2
-1
@@ -4,8 +4,9 @@ import (
|
||||
"math/big"
|
||||
"testing"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/dht/bits"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
)
|
||||
|
||||
func TestAnnounceRange(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
package publish
|
||||
|
||||
import "strings"
|
||||
|
||||
func guessMimeType(ext string) (string, string) {
|
||||
if ext == "" {
|
||||
return "application/octet-stream", "binary"
|
||||
}
|
||||
|
||||
ext = strings.ToLower(strings.TrimLeft(strings.TrimSpace(ext), "."))
|
||||
|
||||
types := map[string]struct{ mime, t string }{
|
||||
"a": {"application/octet-stream", "binary"},
|
||||
"ai": {"application/postscript", "image"},
|
||||
"aif": {"audio/x-aiff", "audio"},
|
||||
"aifc": {"audio/x-aiff", "audio"},
|
||||
"aiff": {"audio/x-aiff", "audio"},
|
||||
"au": {"audio/basic", "audio"},
|
||||
"avi": {"video/x-msvideo", "video"},
|
||||
"bat": {"text/plain", "document"},
|
||||
"bcpio": {"application/x-bcpio", "binary"},
|
||||
"bin": {"application/octet-stream", "binary"},
|
||||
"bmp": {"image/bmp", "image"},
|
||||
"c": {"text/plain", "document"},
|
||||
"cdf": {"application/x-netcdf", "binary"},
|
||||
"cpio": {"application/x-cpio", "binary"},
|
||||
"csh": {"application/x-csh", "binary"},
|
||||
"css": {"text/css", "document"},
|
||||
"csv": {"text/csv", "document"},
|
||||
"dll": {"application/octet-stream", "binary"},
|
||||
"doc": {"application/msword", "document"},
|
||||
"dot": {"application/msword", "document"},
|
||||
"dvi": {"application/x-dvi", "binary"},
|
||||
"eml": {"message/rfc822", "document"},
|
||||
"eps": {"application/postscript", "document"},
|
||||
"epub": {"application/epub+zip", "document"},
|
||||
"etx": {"text/x-setext", "document"},
|
||||
"exe": {"application/octet-stream", "binary"},
|
||||
"gif": {"image/gif", "image"},
|
||||
"gtar": {"application/x-gtar", "binary"},
|
||||
"h": {"text/plain", "document"},
|
||||
"hdf": {"application/x-hdf", "binary"},
|
||||
"htm": {"text/html", "document"},
|
||||
"html": {"text/html", "document"},
|
||||
"ico": {"image/vnd.microsoft.icon", "image"},
|
||||
"ief": {"image/ief", "image"},
|
||||
"iges": {"model/iges", "model"},
|
||||
"jpe": {"image/jpeg", "image"},
|
||||
"jpeg": {"image/jpeg", "image"},
|
||||
"jpg": {"image/jpeg", "image"},
|
||||
"js": {"application/javascript", "document"},
|
||||
"json": {"application/json", "document"},
|
||||
"ksh": {"text/plain", "document"},
|
||||
"latex": {"application/x-latex", "binary"},
|
||||
"m1v": {"video/mpeg", "video"},
|
||||
"m3u": {"application/vnd.apple.mpegurl", "audio"},
|
||||
"m3u8": {"application/vnd.apple.mpegurl", "audio"},
|
||||
"man": {"application/x-troff-man", "document"},
|
||||
"markdown": {"text/markdown", "document"},
|
||||
"md": {"text/markdown", "document"},
|
||||
"me": {"application/x-troff-me", "binary"},
|
||||
"mht": {"message/rfc822", "document"},
|
||||
"mhtml": {"message/rfc822", "document"},
|
||||
"mif": {"application/x-mif", "binary"},
|
||||
"mov": {"video/quicktime", "video"},
|
||||
"movie": {"video/x-sgi-movie", "video"},
|
||||
"mp2": {"audio/mpeg", "audio"},
|
||||
"mp3": {"audio/mpeg", "audio"},
|
||||
"mp4": {"video/mp4", "video"},
|
||||
"mpa": {"video/mpeg", "video"},
|
||||
"mpe": {"video/mpeg", "video"},
|
||||
"mpeg": {"video/mpeg", "video"},
|
||||
"mpg": {"video/mpeg", "video"},
|
||||
"ms": {"application/x-troff-ms", "binary"},
|
||||
"nc": {"application/x-netcdf", "binary"},
|
||||
"nws": {"message/rfc822", "document"},
|
||||
"o": {"application/octet-stream", "binary"},
|
||||
"obj": {"application/octet-stream", "model"},
|
||||
"oda": {"application/oda", "binary"},
|
||||
"p12": {"application/x-pkcs12", "binary"},
|
||||
"p7c": {"application/pkcs7-mime", "binary"},
|
||||
"pbm": {"image/x-portable-bitmap", "image"},
|
||||
"pdf": {"application/pdf", "document"},
|
||||
"pfx": {"application/x-pkcs12", "binary"},
|
||||
"pgm": {"image/x-portable-graymap", "image"},
|
||||
"pl": {"text/plain", "document"},
|
||||
"png": {"image/png", "image"},
|
||||
"pnm": {"image/x-portable-anymap", "image"},
|
||||
"pot": {"application/vnd.ms-powerpoint", "document"},
|
||||
"ppa": {"application/vnd.ms-powerpoint", "document"},
|
||||
"ppm": {"image/x-portable-pixmap", "image"},
|
||||
"pps": {"application/vnd.ms-powerpoint", "document"},
|
||||
"ppt": {"application/vnd.ms-powerpoint", "document"},
|
||||
"ps": {"application/postscript", "document"},
|
||||
"pwz": {"application/vnd.ms-powerpoint", "document"},
|
||||
"py": {"text/x-python", "document"},
|
||||
"pyc": {"application/x-python-code", "binary"},
|
||||
"pyo": {"application/x-python-code", "binary"},
|
||||
"qt": {"video/quicktime", "video"},
|
||||
"ra": {"audio/x-pn-realaudio", "audio"},
|
||||
"ram": {"application/x-pn-realaudio", "audio"},
|
||||
"ras": {"image/x-cmu-raster", "image"},
|
||||
"rdf": {"application/xml", "binary"},
|
||||
"rgb": {"image/x-rgb", "image"},
|
||||
"roff": {"application/x-troff", "binary"},
|
||||
"rtx": {"text/richtext", "document"},
|
||||
"sgm": {"text/x-sgml", "document"},
|
||||
"sgml": {"text/x-sgml", "document"},
|
||||
"sh": {"application/x-sh", "document"},
|
||||
"shar": {"application/x-shar", "binary"},
|
||||
"snd": {"audio/basic", "audio"},
|
||||
"so": {"application/octet-stream", "binary"},
|
||||
"src": {"application/x-wais-source", "binary"},
|
||||
"stl": {"model/stl", "model"},
|
||||
"sv4cpio": {"application/x-sv4cpio", "binary"},
|
||||
"sv4crc": {"application/x-sv4crc", "binary"},
|
||||
"svg": {"image/svg+xml", "image"},
|
||||
"swf": {"application/x-shockwave-flash", "binary"},
|
||||
"t": {"application/x-troff", "binary"},
|
||||
"tar": {"application/x-tar", "binary"},
|
||||
"tcl": {"application/x-tcl", "binary"},
|
||||
"tex": {"application/x-tex", "binary"},
|
||||
"texi": {"application/x-texinfo", "binary"},
|
||||
"texinfo": {"application/x-texinfo", "binary"},
|
||||
"tif": {"image/tiff", "image"},
|
||||
"tiff": {"image/tiff", "image"},
|
||||
"tr": {"application/x-troff", "binary"},
|
||||
"tsv": {"text/tab-separated-values", "document"},
|
||||
"txt": {"text/plain", "document"},
|
||||
"ustar": {"application/x-ustar", "binary"},
|
||||
"vcf": {"text/x-vcard", "document"},
|
||||
"wav": {"audio/x-wav", "audio"},
|
||||
"webm": {"video/webm", "video"},
|
||||
"wiz": {"application/msword", "document"},
|
||||
"wsdl": {"application/xml", "document"},
|
||||
"xbm": {"image/x-xbitmap", "image"},
|
||||
"xlb": {"application/vnd.ms-excel", "document"},
|
||||
"xls": {"application/vnd.ms-excel", "document"},
|
||||
"xml": {"text/xml", "document"},
|
||||
"xpdl": {"application/xml", "document"},
|
||||
"xpm": {"image/x-xpixmap", "image"},
|
||||
"xsl": {"application/xml", "document"},
|
||||
"xwd": {"image/x-xwindowdump", "image"},
|
||||
"zip": {"application/zip", "binary"},
|
||||
|
||||
// These are non-standard types, commonly found in the wild.
|
||||
"cbr": {"application/vnd.comicbook-rar", "document"},
|
||||
"cbz": {"application/vnd.comicbook+zip", "document"},
|
||||
"flac": {"audio/flac", "audio"},
|
||||
"lbry": {"application/x-ext-lbry", "document"},
|
||||
"m4v": {"video/m4v", "video"},
|
||||
"mid": {"audio/midi", "audio"},
|
||||
"midi": {"audio/midi", "audio"},
|
||||
"mkv": {"video/x-matroska", "video"},
|
||||
"mobi": {"application/x-mobipocket-ebook", "document"},
|
||||
"oga": {"audio/ogg", "audio"},
|
||||
"ogv": {"video/ogg", "video"},
|
||||
"pct": {"image/pict", "image"},
|
||||
"pic": {"image/pict", "image"},
|
||||
"pict": {"image/pict", "image"},
|
||||
"prc": {"application/x-mobipocket-ebook", "document"},
|
||||
"rtf": {"application/rtf", "document"},
|
||||
"xul": {"text/xul", "document"},
|
||||
|
||||
// microsoft is special and has its own "standard"
|
||||
// https://docs.microsoft.com/en-us/windows/desktop/wmp/file-name-extensions
|
||||
"wmv": {"video/x-ms-wmv", "video"},
|
||||
}
|
||||
|
||||
if data, ok := types[ext]; ok {
|
||||
return data.mime, data.t
|
||||
}
|
||||
return "application/x-ext-" + ext, "binary"
|
||||
}
|
||||
@@ -0,0 +1,292 @@
|
||||
package publish
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/lbrycrd"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
pb "github.com/lbryio/types/v2/go"
|
||||
|
||||
"github.com/btcsuite/btcd/btcjson"
|
||||
"github.com/btcsuite/btcd/chaincfg/chainhash"
|
||||
"github.com/btcsuite/btcd/txscript"
|
||||
"github.com/btcsuite/btcd/wire"
|
||||
"github.com/btcsuite/btcutil"
|
||||
"github.com/golang/protobuf/proto"
|
||||
)
|
||||
|
||||
/* TODO:
|
||||
import cert from wallet
|
||||
get all utxos from chainquery
|
||||
create transaction
|
||||
sign it with the channel
|
||||
track state of utxos across publishes from this channel so that we can just do one query to get utxos
|
||||
prioritize only confirmed utxos
|
||||
|
||||
Handling all the issues we handle currently with lbrynet:
|
||||
"Couldn't find private key for id",
|
||||
"You already have a stream claim published under the name",
|
||||
"Cannot publish using channel",
|
||||
"txn-mempool-conflict",
|
||||
"too-long-mempool-chain",
|
||||
"Missing inputs",
|
||||
"Not enough funds to cover this transaction",
|
||||
*/
|
||||
|
||||
type Details struct {
|
||||
Title string
|
||||
Description string
|
||||
Author string
|
||||
Tags []string
|
||||
ReleaseTime int64
|
||||
}
|
||||
|
||||
func Publish(client *lbrycrd.Client, path, name, address string, details Details, reflectorAddress string) (*wire.MsgTx, *chainhash.Hash, error) {
|
||||
if name == "" {
|
||||
return nil, nil, errors.Err("name required")
|
||||
}
|
||||
|
||||
//TODO: sign claim if publishing into channel
|
||||
|
||||
addr, err := btcutil.DecodeAddress(address, &lbrycrd.MainNetParams)
|
||||
if errors.Is(err, btcutil.ErrUnknownAddressType) {
|
||||
return nil, nil, errors.Err(`unknown address type. here's what you need to make this work:
|
||||
- deprecatedrpc=validateaddress" and "deprecatedrpc=signrawtransaction" in your lbrycrd.conf
|
||||
- github.com/btcsuite/btcd pinned to hash 306aecffea32
|
||||
- github.com/btcsuite/btcutil pinned to 4c204d697803
|
||||
- github.com/lbryio/lbry.go/v2 (make sure you have v2 at the end)`)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
amount := 0.01
|
||||
changeAddr := addr // TODO: fix this? or maybe its fine?
|
||||
tx, err := baseTx(client, amount, changeAddr)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
st, stPB, err := makeStream(path)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
stPB.Author = details.Author
|
||||
stPB.ReleaseTime = details.ReleaseTime
|
||||
|
||||
claim := &pb.Claim{
|
||||
Title: details.Title,
|
||||
Description: details.Description,
|
||||
Type: &pb.Claim_Stream{Stream: stPB},
|
||||
}
|
||||
|
||||
err = addClaimToTx(tx, claim, name, amount, addr)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// sign and send
|
||||
signedTx, allInputsSigned, err := client.SignRawTransaction(tx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if !allInputsSigned {
|
||||
return nil, nil, errors.Err("not all inputs for the tx could be signed")
|
||||
}
|
||||
|
||||
err = reflect(st, reflectorAddress)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
txid, err := client.SendRawTransaction(signedTx, false)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return signedTx, txid, nil
|
||||
}
|
||||
|
||||
//TODO: lots of assumptions. hardcoded values need to be passed in or calculated
|
||||
func baseTx(client *lbrycrd.Client, amount float64, changeAddress btcutil.Address) (*wire.MsgTx, error) {
|
||||
txFee := 0.0002 // TODO: estimate this better?
|
||||
|
||||
inputs, total, err := coinChooser(client, amount+txFee)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
change := total - amount - txFee
|
||||
|
||||
// create base raw tx
|
||||
addresses := make(map[btcutil.Address]btcutil.Amount)
|
||||
//changeAddr, err := client.GetNewAddress("")
|
||||
changeAmount, err := btcutil.NewAmount(change)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
addresses[changeAddress] = changeAmount
|
||||
lockTime := int64(0)
|
||||
return client.CreateRawTransaction(inputs, addresses, &lockTime)
|
||||
}
|
||||
|
||||
func coinChooser(client *lbrycrd.Client, amount float64) ([]btcjson.TransactionInput, float64, error) {
|
||||
utxos, err := client.ListUnspentMin(1)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
sort.Slice(utxos, func(i, j int) bool { return utxos[i].Amount < utxos[j].Amount })
|
||||
|
||||
var utxo btcjson.ListUnspentResult
|
||||
for _, u := range utxos {
|
||||
if u.Spendable && u.Amount >= amount {
|
||||
utxo = u
|
||||
break
|
||||
}
|
||||
}
|
||||
if utxo.TxID == "" {
|
||||
return nil, 0, errors.Err("not enough utxos to create tx")
|
||||
}
|
||||
|
||||
return []btcjson.TransactionInput{{Txid: utxo.TxID, Vout: utxo.Vout}}, utxo.Amount, nil
|
||||
}
|
||||
|
||||
func addClaimToTx(tx *wire.MsgTx, claim *pb.Claim, name string, amount float64, claimAddress btcutil.Address) error {
|
||||
claimBytes, err := proto.Marshal(claim)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
claimBytes = append([]byte{0}, claimBytes...) // version 0 = no channel sig
|
||||
|
||||
amt, err := btcutil.NewAmount(amount)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
script, err := getClaimPayoutScript(name, claimBytes, claimAddress)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tx.AddTxOut(wire.NewTxOut(int64(amt), script))
|
||||
return nil
|
||||
}
|
||||
|
||||
func Decode(client *lbrycrd.Client, tx *wire.MsgTx) (string, error) {
|
||||
buf := bytes.NewBuffer(make([]byte, 0, tx.SerializeSize()))
|
||||
if err := tx.Serialize(buf); err != nil {
|
||||
return "", errors.Err(err)
|
||||
}
|
||||
//txHex := hex.EncodeToString(buf.Bytes())
|
||||
//spew.Dump(txHex)
|
||||
decoded, err := client.DecodeRawTransaction(buf.Bytes())
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
data, err := json.MarshalIndent(decoded, "", " ")
|
||||
return string(data), err
|
||||
}
|
||||
|
||||
func reflect(st stream.Stream, reflectorAddress string) error {
|
||||
// upload blobs to reflector
|
||||
c := reflector.Client{}
|
||||
err := c.Connect(reflectorAddress)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
for i, b := range st {
|
||||
if i == 0 {
|
||||
err = c.SendSDBlob(b)
|
||||
} else {
|
||||
err = c.SendBlob(b)
|
||||
}
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func makeStream(path string) (stream.Stream, *pb.Stream, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, nil, errors.Err(err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
enc := stream.NewEncoder(file)
|
||||
|
||||
s, err := enc.Stream()
|
||||
if err != nil {
|
||||
return nil, nil, errors.Err(err)
|
||||
}
|
||||
|
||||
streamProto := &pb.Stream{
|
||||
Source: &pb.Source{
|
||||
SdHash: enc.SDBlob().Hash(),
|
||||
Name: filepath.Base(file.Name()),
|
||||
Size: uint64(enc.SourceLen()),
|
||||
Hash: enc.SourceHash(),
|
||||
},
|
||||
}
|
||||
|
||||
mimeType, category := guessMimeType(filepath.Ext(file.Name()))
|
||||
streamProto.Source.MediaType = mimeType
|
||||
|
||||
switch category {
|
||||
case "video":
|
||||
//t, err := streamVideoMetadata(path)
|
||||
//if err != nil {
|
||||
// return nil, nil, err
|
||||
//}
|
||||
streamProto.Type = &pb.Stream_Video{}
|
||||
case "audio":
|
||||
streamProto.Type = &pb.Stream_Audio{}
|
||||
case "image":
|
||||
streamProto.Type = &pb.Stream_Image{}
|
||||
}
|
||||
|
||||
return s, streamProto, nil
|
||||
}
|
||||
|
||||
func getClaimPayoutScript(name string, value []byte, address btcutil.Address) ([]byte, error) {
|
||||
//OP_CLAIM_NAME <name> <value> OP_2DROP OP_DROP OP_DUP OP_HASH160 <address> OP_EQUALVERIFY OP_CHECKSIG
|
||||
|
||||
pkscript, err := txscript.PayToAddrScript(address)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
return txscript.NewScriptBuilder().
|
||||
AddOp(txscript.OP_NOP6). //OP_CLAIM_NAME
|
||||
AddData([]byte(name)). //<name>
|
||||
AddData(value). //<value>
|
||||
AddOp(txscript.OP_2DROP). //OP_2DROP
|
||||
AddOp(txscript.OP_DROP). //OP_DROP
|
||||
AddOps(pkscript). //OP_DUP OP_HASH160 <address> OP_EQUALVERIFY OP_CHECKSIG
|
||||
Script()
|
||||
}
|
||||
|
||||
//func streamVideoMetadata(path string) (*pb.Stream_Video, error) {
|
||||
// mi, err := mediainfo.GetMediaInfo(path)
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
// return &pb.Stream_Video{
|
||||
// Video: &pb.Video{
|
||||
// Duration: uint32(mi.General.Duration / 1000),
|
||||
// Height: uint32(mi.Video.Height),
|
||||
// Width: uint32(mi.Video.Width),
|
||||
// },
|
||||
// }, nil
|
||||
//}
|
||||
@@ -0,0 +1,59 @@
|
||||
package publish
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
)
|
||||
|
||||
func LoadWallet(r io.Reader) (WalletFile, error) {
|
||||
var w WalletFile
|
||||
err := json.NewDecoder(r).Decode(&w)
|
||||
return w, err
|
||||
}
|
||||
|
||||
type WalletFile struct {
|
||||
Name string `json:"name"`
|
||||
Version int `json:"version"`
|
||||
Preferences WalletPrefs `json:"preferences"`
|
||||
Accounts []Account `json:"accounts"`
|
||||
}
|
||||
|
||||
type Account struct {
|
||||
AddressGenerator AddressGenerator `json:"address_generator"`
|
||||
Certificates map[string]string `json:"certificates"`
|
||||
Encrypted bool `json:"encrypted"`
|
||||
Ledger string `json:"ledger"`
|
||||
ModifiedOn float64 `json:"modified_on"`
|
||||
Name string `json:"name"`
|
||||
PrivateKey string `json:"private_key"`
|
||||
PublicKey string `json:"public_key"`
|
||||
Seed string `json:"seed"`
|
||||
}
|
||||
|
||||
type AddressGenerator struct {
|
||||
Name string `json:"name"`
|
||||
Change AddressGenParams `json:"change"` // should "change" and "receiving" be replaced with a map[string]AddressGenParams?
|
||||
Receiving AddressGenParams `json:"receiving"`
|
||||
}
|
||||
|
||||
type AddressGenParams struct {
|
||||
Gap int `json:"gap"`
|
||||
MaximumUsesPerAddress int `json:"maximum_uses_per_address"`
|
||||
}
|
||||
|
||||
type WalletPrefs struct {
|
||||
Shared struct {
|
||||
Ts float64 `json:"ts"`
|
||||
Value struct {
|
||||
Type string `json:"type"`
|
||||
Value struct {
|
||||
AppWelcomeVersion int `json:"app_welcome_version"`
|
||||
Blocked []interface{} `json:"blocked"`
|
||||
Sharing3P bool `json:"sharing_3P"`
|
||||
Subscriptions []string `json:"subscriptions"`
|
||||
Tags []string `json:"tags"`
|
||||
} `json:"value"`
|
||||
Version string `json:"version"`
|
||||
} `json:"value"`
|
||||
} `json:"shared"`
|
||||
}
|
||||
@@ -1,25 +1,110 @@
|
||||
# Reflector
|
||||
|
||||
A reflector cluster to accept LBRY content for hosting en masse, rehost the content, and make money on data fees (TODO).
|
||||
This code includes Go implementations of the LBRY peer protocol, reflector protocol, and DHT.
|
||||
Reflector is a central piece of software that providers LBRY with the following features:
|
||||
- Blobs reflection: when something is published, we capture the data and store it on our servers for quicker retrieval
|
||||
- Blobs distribution: when a piece of content is requested and the LBRY network doesn't have it, reflector will retrieve it from its storage and distribute it
|
||||
- Blobs caching: reflectors can be chained together in multiple regions or servers to form a chain of cached content. We call those "blobcaches". They are layered so that content distribution is favorable in all the regions we deploy it to
|
||||
|
||||
There are a few other features embedded in reflector.go including publishing streams from Go, downloading or upload blobs, resolving content and more unfinished tools.
|
||||
|
||||
This code includes a Go implementations of the LBRY peer protocol, reflector protocol, and DHT.
|
||||
|
||||
## Installation
|
||||
|
||||
coming soon
|
||||
- Install mysql 8 (5.7 might work too)
|
||||
- add a reflector user and database with password `reflector` with localhost access only
|
||||
- Create the tables as described [here](https://github.com/lbryio/reflector.go/blob/ittt/db/db.go#L735) (the link might not update as the code does so just look for the schema in that file)
|
||||
|
||||
#### We do not support running reflector.go as a blob receiver, however if you want to run it as a private blobcache you may compile it yourself and run it as following:
|
||||
```bash
|
||||
./prism-bin reflector \
|
||||
--conf="none" \
|
||||
--disable-uploads=true \
|
||||
--use-db=false \
|
||||
--upstream-reflector="reflector.lbry.com" \
|
||||
--upstream-protocol="http" \
|
||||
--request-queue-size=200 \
|
||||
--disk-cache="2GB:/path/to/your/storage/:localdb" \
|
||||
```
|
||||
|
||||
Create a systemd script if you want to run it automatically on startup or as a service.
|
||||
|
||||
## Usage
|
||||
|
||||
coming soon
|
||||
Usage as reflector/blobcache:
|
||||
```bash
|
||||
Run reflector server
|
||||
|
||||
Usage:
|
||||
prism reflector [flags]
|
||||
|
||||
Flags:
|
||||
--disable-blocklist Disable blocklist watching/updating
|
||||
--disable-uploads Disable uploads to this reflector server
|
||||
--disk-cache string Where to cache blobs on the file system. format is 'sizeGB:CACHE_PATH:cachemanager' (cachemanagers: localdb/lfuda/lru) (default "100GB:/tmp/downloaded_blobs:localdb")
|
||||
-h, --help help for reflector
|
||||
--http-peer-port int The port reflector will distribute content from over HTTP protocol (default 5569)
|
||||
--http3-peer-port int The port reflector will distribute content from over HTTP3 protocol (default 5568)
|
||||
--mem-cache int enable in-memory cache with a max size of this many blobs
|
||||
--metrics-port int The port reflector will use for prometheus metrics (default 2112)
|
||||
--optional-disk-cache string Optional secondary file system cache for blobs. format is 'sizeGB:CACHE_PATH:cachemanager' (cachemanagers: localdb/lfuda/lru) (this would get hit before the one specified in disk-cache)
|
||||
--origin-endpoint string HTTP edge endpoint for standard HTTP retrieval
|
||||
--origin-endpoint-fallback string HTTP edge endpoint for standard HTTP retrieval if first origin fails
|
||||
--receiver-port int The port reflector will receive content from (default 5566)
|
||||
--request-queue-size int How many concurrent requests from downstream should be handled at once (the rest will wait) (default 200)
|
||||
--tcp-peer-port int The port reflector will distribute content from for the TCP (LBRY) protocol (default 5567)
|
||||
--upstream-protocol string protocol used to fetch blobs from another upstream reflector server (tcp/http3/http) (default "http")
|
||||
--upstream-reflector string host:port of a reflector server where blobs are fetched from
|
||||
--use-db Whether to connect to the reflector db or not (default true)
|
||||
|
||||
Global Flags:
|
||||
--conf string Path to config. Use 'none' to disable (default "config.json")
|
||||
-v, --verbose strings Verbose logging for specific components
|
||||
```
|
||||
|
||||
Other uses:
|
||||
|
||||
```bash
|
||||
Prism is a single entry point application with multiple sub modules which can be leveraged individually or together
|
||||
|
||||
Usage:
|
||||
prism [command]
|
||||
|
||||
Available Commands:
|
||||
check-integrity check blobs integrity for a given path
|
||||
cluster Start(join) to or Start a new cluster
|
||||
decode Decode a claim value
|
||||
dht Run dht node
|
||||
getstream Get a stream from a reflector server
|
||||
help Help about any command
|
||||
peer Run peer server
|
||||
populate-db populate local database with blobs from a disk storage
|
||||
publish Publish a file
|
||||
reflector Run reflector server
|
||||
resolve Resolve a URL
|
||||
send Send a file to a reflector
|
||||
sendblob Send a random blob to a reflector server
|
||||
start Runs full prism application with cluster, dht, peer server, and reflector server.
|
||||
test Test things
|
||||
upload Upload blobs to S3
|
||||
version Print the version
|
||||
|
||||
Flags:
|
||||
--conf string Path to config. Use 'none' to disable (default "config.json")
|
||||
-h, --help help for prism
|
||||
-v, --verbose strings Verbose logging for specific components
|
||||
```
|
||||
## Running from Source
|
||||
|
||||
This project requires [Go v1.10](https://golang.org/doc/install) or higher.
|
||||
This project requires [Go v1.16](https://golang.org/doc/install).
|
||||
|
||||
On Ubuntu you can install it with `sudo snap install go --classic`
|
||||
|
||||
```
|
||||
go get -u github.com/lbryio/reflector.go
|
||||
cd "$(go env GOPATH)/src/github.com/lbryio/reflector.go"
|
||||
git clone git@github.com:lbryio/reflector.go.git
|
||||
cd reflector.go
|
||||
make
|
||||
./prism
|
||||
./bin/prism-bin
|
||||
```
|
||||
|
||||
## Contributing
|
||||
@@ -32,9 +117,8 @@ This project is MIT licensed.
|
||||
|
||||
## Security
|
||||
|
||||
We take security seriously. Please contact security@lbry.io regarding any security issues.
|
||||
We take security seriously. Please contact security@lbry.com regarding any security issues.
|
||||
Our PGP key is [here](https://keybase.io/lbry/key.asc) if you need it.
|
||||
|
||||
## Contact
|
||||
|
||||
The primary contact for this project is [@lyoshenka](https://github.com/lyoshenka) (grin@lbry.io)
|
||||
The primary contact for this project is [@Nikooo777](https://github.com/Nikooo777) (niko-at-lbry.com)
|
||||
@@ -0,0 +1,159 @@
|
||||
package reflector
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
"github.com/lbryio/reflector.go/wallet"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const blocklistURL = "https://api.lbry.com/file/list_blocked"
|
||||
|
||||
func (s *Server) enableBlocklist(b store.Blocklister) {
|
||||
walletServers := []string{
|
||||
"spv25.lbry.com:50001",
|
||||
"spv26.lbry.com:50001",
|
||||
"spv19.lbry.com:50001",
|
||||
"spv14.lbry.com:50001",
|
||||
}
|
||||
|
||||
updateBlocklist(b, walletServers, s.grp.Ch())
|
||||
t := time.NewTicker(12 * time.Hour)
|
||||
for {
|
||||
select {
|
||||
case <-s.grp.Ch():
|
||||
return
|
||||
case <-t.C:
|
||||
updateBlocklist(b, walletServers, s.grp.Ch())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func updateBlocklist(b store.Blocklister, walletServers []string, stopper stop.Chan) {
|
||||
log.Debugf("blocklist update starting")
|
||||
values, err := blockedSdHashes(walletServers, stopper)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
for name, v := range values {
|
||||
if v.Err != nil {
|
||||
log.Error(errors.FullTrace(errors.Err("blocklist: %s: %s", name, v.Err)))
|
||||
continue
|
||||
}
|
||||
|
||||
err = b.Block(v.Value)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
}
|
||||
log.Debugf("blocklist update done")
|
||||
}
|
||||
|
||||
func blockedSdHashes(walletServers []string, stopper stop.Chan) (map[string]valOrErr, error) {
|
||||
client := http.Client{Timeout: 1 * time.Second}
|
||||
resp, err := client.Get(blocklistURL)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
defer func() {
|
||||
err := resp.Body.Close()
|
||||
if err != nil {
|
||||
log.Errorln(errors.Err(err))
|
||||
}
|
||||
}()
|
||||
|
||||
var r struct {
|
||||
Success bool `json:"success"`
|
||||
Error string `json:"error"`
|
||||
Data struct {
|
||||
Outpoints []string `json:"outpoints"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
if err = json.NewDecoder(resp.Body).Decode(&r); err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
if !r.Success {
|
||||
return nil, errors.Prefix("list_blocked API call", r.Error)
|
||||
}
|
||||
|
||||
return sdHashesForOutpoints(walletServers, r.Data.Outpoints, stopper)
|
||||
}
|
||||
|
||||
type valOrErr struct {
|
||||
Value string
|
||||
Err error
|
||||
}
|
||||
|
||||
// sdHashesForOutpoints queries wallet server for the sd hashes in a given outpoints
|
||||
func sdHashesForOutpoints(walletServers, outpoints []string, stopper stop.Chan) (map[string]valOrErr, error) {
|
||||
values := make(map[string]valOrErr)
|
||||
|
||||
node := wallet.NewNode()
|
||||
err := node.Connect(walletServers, nil)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
done := make(chan bool)
|
||||
metrics.RoutinesQueue.WithLabelValues("reflector", "sdhashesforoutput").Inc()
|
||||
go func() {
|
||||
defer metrics.RoutinesQueue.WithLabelValues("reflector", "sdhashesforoutput").Dec()
|
||||
select {
|
||||
case <-done:
|
||||
case <-stopper:
|
||||
}
|
||||
node.Shutdown()
|
||||
}()
|
||||
|
||||
OutpointLoop:
|
||||
for _, outpoint := range outpoints {
|
||||
select {
|
||||
case <-stopper:
|
||||
break OutpointLoop
|
||||
default:
|
||||
}
|
||||
|
||||
parts := strings.Split(outpoint, ":")
|
||||
if len(parts) != 2 {
|
||||
values[outpoint] = valOrErr{Err: errors.Err("invalid outpoint format")}
|
||||
continue
|
||||
}
|
||||
|
||||
nout, err := strconv.Atoi(parts[1])
|
||||
if err != nil {
|
||||
values[outpoint] = valOrErr{Err: errors.Prefix("invalid nout", err)}
|
||||
continue
|
||||
}
|
||||
|
||||
claim, err := node.GetClaimInTx(parts[0], nout)
|
||||
if err != nil {
|
||||
values[outpoint] = valOrErr{Err: err}
|
||||
continue
|
||||
}
|
||||
|
||||
hash := hex.EncodeToString(claim.GetStream().GetSource().GetSdHash())
|
||||
values[outpoint] = valOrErr{Value: hash, Err: nil}
|
||||
}
|
||||
|
||||
select {
|
||||
case done <- true:
|
||||
default: // in case of race where stopper got stopped right after loop finished
|
||||
}
|
||||
|
||||
return values, nil
|
||||
}
|
||||
+69
-33
@@ -2,11 +2,11 @@ package reflector
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log"
|
||||
"net"
|
||||
"strconv"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
)
|
||||
|
||||
// ErrBlobExists is a default error for when a blob already exists on the reflector server.
|
||||
@@ -35,23 +35,36 @@ func (c *Client) Close() error {
|
||||
return c.conn.Close()
|
||||
}
|
||||
|
||||
// SendBlob sends a send blob request to the client.
|
||||
func (c *Client) SendBlob(blob []byte) error {
|
||||
// SendBlob sends a blob to the server.
|
||||
func (c *Client) SendBlob(blob stream.Blob) error {
|
||||
return c.sendBlob(blob, false)
|
||||
}
|
||||
|
||||
// SendSDBlob sends an SD blob request to the server.
|
||||
func (c *Client) SendSDBlob(blob stream.Blob) error {
|
||||
return c.sendBlob(blob, true)
|
||||
}
|
||||
|
||||
// sendBlob does the actual blob sending
|
||||
func (c *Client) sendBlob(blob stream.Blob, isSDBlob bool) error {
|
||||
if !c.connected {
|
||||
return errors.Err("not connected")
|
||||
}
|
||||
|
||||
if len(blob) > maxBlobSize {
|
||||
return errors.Err("blob must be at most " + strconv.Itoa(maxBlobSize) + " bytes")
|
||||
} else if len(blob) == 0 {
|
||||
return errors.Err("blob is empty")
|
||||
if err := blob.ValidForSend(); err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
blobHash := BlobHash(blob)
|
||||
sendRequest, err := json.Marshal(sendBlobRequest{
|
||||
BlobSize: len(blob),
|
||||
BlobHash: blobHash,
|
||||
})
|
||||
blobHash := blob.HashHex()
|
||||
var req sendBlobRequest
|
||||
if isSDBlob {
|
||||
req.SdBlobSize = blob.Size()
|
||||
req.SdBlobHash = blobHash
|
||||
} else {
|
||||
req.BlobSize = blob.Size()
|
||||
req.BlobHash = blobHash
|
||||
}
|
||||
sendRequest, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -63,30 +76,51 @@ func (c *Client) SendBlob(blob []byte) error {
|
||||
|
||||
dec := json.NewDecoder(c.conn)
|
||||
|
||||
var sendResp sendBlobResponse
|
||||
err = dec.Decode(&sendResp)
|
||||
if err != nil {
|
||||
return err
|
||||
if isSDBlob {
|
||||
var sendResp sendSdBlobResponse
|
||||
err = dec.Decode(&sendResp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !sendResp.SendSdBlob {
|
||||
return errors.Prefix(blobHash[:8], ErrBlobExists)
|
||||
}
|
||||
log.Println("Sending SD blob " + blobHash[:8])
|
||||
} else {
|
||||
var sendResp sendBlobResponse
|
||||
err = dec.Decode(&sendResp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !sendResp.SendBlob {
|
||||
return errors.Prefix(blobHash[:8], ErrBlobExists)
|
||||
}
|
||||
log.Println("Sending blob " + blobHash[:8])
|
||||
}
|
||||
|
||||
if !sendResp.SendBlob {
|
||||
return ErrBlobExists
|
||||
}
|
||||
|
||||
log.Println("Sending blob " + blobHash[:8])
|
||||
|
||||
_, err = c.conn.Write(blob)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var transferResp blobTransferResponse
|
||||
err = dec.Decode(&transferResp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !transferResp.ReceivedBlob {
|
||||
return errors.Err("server did not received blob")
|
||||
if isSDBlob {
|
||||
var transferResp sdBlobTransferResponse
|
||||
err = dec.Decode(&transferResp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !transferResp.ReceivedSdBlob {
|
||||
return errors.Err("server did not received SD blob")
|
||||
}
|
||||
} else {
|
||||
var transferResp blobTransferResponse
|
||||
err = dec.Decode(&transferResp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !transferResp.ReceivedBlob {
|
||||
return errors.Err("server did not received blob")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -97,7 +131,7 @@ func (c *Client) doHandshake(version int) error {
|
||||
return errors.Err("not connected")
|
||||
}
|
||||
|
||||
handshake, err := json.Marshal(handshakeRequestResponse{Version: version})
|
||||
handshake, err := json.Marshal(handshakeRequestResponse{Version: &version})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -111,7 +145,9 @@ func (c *Client) doHandshake(version int) error {
|
||||
err = json.NewDecoder(c.conn).Decode(&resp)
|
||||
if err != nil {
|
||||
return err
|
||||
} else if resp.Version != version {
|
||||
} else if resp.Version == nil {
|
||||
return errors.Err("invalid handshake")
|
||||
} else if *resp.Version != version {
|
||||
return errors.Err("handshake version mismatch")
|
||||
}
|
||||
|
||||
|
||||
+101
-37
@@ -6,14 +6,16 @@ import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/lbry.go/stop"
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
@@ -27,23 +29,29 @@ const (
|
||||
network = "tcp4"
|
||||
protocolVersion1 = 0
|
||||
protocolVersion2 = 1
|
||||
maxBlobSize = 2 * 1024 * 1024
|
||||
maxBlobSize = stream.MaxBlobSize
|
||||
)
|
||||
|
||||
var ErrBlobTooBig = errors.Base("blob must be at most %d bytes", maxBlobSize)
|
||||
|
||||
// Server is and instance of the reflector server. It houses the blob store and listener.
|
||||
type Server struct {
|
||||
Timeout time.Duration // timeout to read or write next message
|
||||
|
||||
store store.BlobStore
|
||||
grp *stop.Group
|
||||
EnableBlocklist bool // if true, blocklist checking and blob deletion will be enabled
|
||||
|
||||
underlyingStore store.BlobStore
|
||||
outerStore store.BlobStore
|
||||
grp *stop.Group
|
||||
}
|
||||
|
||||
// NewServer returns an initialized reflector server pointer.
|
||||
func NewServer(store store.BlobStore) *Server {
|
||||
func NewServer(underlying store.BlobStore, outer store.BlobStore) *Server {
|
||||
return &Server{
|
||||
Timeout: DefaultTimeout,
|
||||
store: store,
|
||||
grp: stop.New(),
|
||||
Timeout: DefaultTimeout,
|
||||
underlyingStore: underlying,
|
||||
outerStore: outer,
|
||||
grp: stop.New(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -61,9 +69,10 @@ func (s *Server) Start(address string) error {
|
||||
return errors.Err(err)
|
||||
}
|
||||
log.Println("reflector listening on " + address)
|
||||
|
||||
s.grp.Add(1)
|
||||
metrics.RoutinesQueue.WithLabelValues("reflector", "listener").Inc()
|
||||
go func() {
|
||||
defer metrics.RoutinesQueue.WithLabelValues("reflector", "listener").Dec()
|
||||
<-s.grp.Ch()
|
||||
err := l.Close()
|
||||
if err != nil {
|
||||
@@ -73,11 +82,28 @@ func (s *Server) Start(address string) error {
|
||||
}()
|
||||
|
||||
s.grp.Add(1)
|
||||
metrics.RoutinesQueue.WithLabelValues("reflector", "start").Inc()
|
||||
go func() {
|
||||
defer metrics.RoutinesQueue.WithLabelValues("reflector", "start").Dec()
|
||||
s.listenAndServe(l)
|
||||
s.grp.Done()
|
||||
}()
|
||||
|
||||
if s.EnableBlocklist {
|
||||
if b, ok := s.underlyingStore.(store.Blocklister); ok {
|
||||
s.grp.Add(1)
|
||||
metrics.RoutinesQueue.WithLabelValues("reflector", "enableblocklist").Inc()
|
||||
go func() {
|
||||
defer metrics.RoutinesQueue.WithLabelValues("reflector", "enableblocklist").Dec()
|
||||
s.enableBlocklist(b)
|
||||
s.grp.Done()
|
||||
}()
|
||||
} else {
|
||||
//s.Shutdown()
|
||||
return errors.Err("blocklist is enabled but blob store does not support blocklisting")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -91,7 +117,9 @@ func (s *Server) listenAndServe(listener net.Listener) {
|
||||
log.Error(err)
|
||||
} else {
|
||||
s.grp.Add(1)
|
||||
metrics.RoutinesQueue.WithLabelValues("reflector", "server-listenandserve").Inc()
|
||||
go func() {
|
||||
defer metrics.RoutinesQueue.WithLabelValues("reflector", "server-listenandserve").Dec()
|
||||
s.handleConn(conn)
|
||||
s.grp.Done()
|
||||
}()
|
||||
@@ -106,7 +134,9 @@ func (s *Server) handleConn(conn net.Conn) {
|
||||
close(connNeedsClosing)
|
||||
}()
|
||||
s.grp.Add(1)
|
||||
metrics.RoutinesQueue.WithLabelValues("reflector", "server-handleconn").Inc()
|
||||
go func() {
|
||||
defer metrics.RoutinesQueue.WithLabelValues("reflector", "server-handleconn").Dec()
|
||||
defer s.grp.Done()
|
||||
select {
|
||||
case <-connNeedsClosing:
|
||||
@@ -146,7 +176,13 @@ func (s *Server) handleConn(conn net.Conn) {
|
||||
}
|
||||
|
||||
func (s *Server) doError(conn net.Conn, err error) error {
|
||||
log.Errorln(errors.FullTrace(err))
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
shouldLog := metrics.TrackError(metrics.DirectionUpload, err)
|
||||
if shouldLog {
|
||||
log.Errorln(errors.FullTrace(err))
|
||||
}
|
||||
if e2, ok := err.(*json.SyntaxError); ok {
|
||||
log.Errorf("syntax error at byte offset %d", e2.Offset)
|
||||
}
|
||||
@@ -159,39 +195,46 @@ func (s *Server) doError(conn net.Conn, err error) error {
|
||||
}
|
||||
|
||||
func (s *Server) receiveBlob(conn net.Conn) error {
|
||||
var err error
|
||||
|
||||
blobSize, blobHash, isSdBlob, err := s.readBlobRequest(conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
blobExists, err := s.store.Has(blobHash)
|
||||
if err != nil {
|
||||
return err
|
||||
var wantsBlob bool
|
||||
if bl, ok := s.underlyingStore.(store.Blocklister); ok {
|
||||
wantsBlob, err = bl.Wants(blobHash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
blobExists, err := s.underlyingStore.Has(blobHash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
wantsBlob = !blobExists
|
||||
}
|
||||
|
||||
var neededBlobs []string
|
||||
|
||||
if isSdBlob && blobExists {
|
||||
if fsc, ok := s.store.(neededBlobChecker); ok {
|
||||
neededBlobs, err = fsc.MissingBlobsForKnownStream(blobHash)
|
||||
if isSdBlob && !wantsBlob {
|
||||
if nbc, ok := s.underlyingStore.(neededBlobChecker); ok {
|
||||
neededBlobs, err = nbc.MissingBlobsForKnownStream(blobHash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
// if we can't confirm that we have the full stream, we have to say that the sd blob is
|
||||
// missing. if we say we have it, they wont try to send any content blobs
|
||||
blobExists = false
|
||||
// if we can't check for blobs in a stream, we have to say that the sd blob is
|
||||
// missing. if we say we have the sd blob, they wont try to send any content blobs
|
||||
wantsBlob = true
|
||||
}
|
||||
}
|
||||
|
||||
err = s.sendBlobResponse(conn, blobExists, isSdBlob, neededBlobs)
|
||||
err = s.sendBlobResponse(conn, wantsBlob, isSdBlob, neededBlobs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if blobExists {
|
||||
if !wantsBlob {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -217,14 +260,18 @@ func (s *Server) receiveBlob(conn net.Conn) error {
|
||||
log.Debugln("Got blob " + blobHash[:8])
|
||||
|
||||
if isSdBlob {
|
||||
err = s.store.PutSD(blobHash, blob)
|
||||
err = s.outerStore.PutSD(blobHash, blob)
|
||||
} else {
|
||||
err = s.store.Put(blobHash, blob)
|
||||
err = s.outerStore.Put(blobHash, blob)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
metrics.MtrInBytesReflector.Add(float64(len(blob)))
|
||||
metrics.BlobUploadCount.Inc()
|
||||
if isSdBlob {
|
||||
metrics.SDBlobUploadCount.Inc()
|
||||
}
|
||||
return s.sendTransferResponse(conn, true, isSdBlob)
|
||||
}
|
||||
|
||||
@@ -233,7 +280,9 @@ func (s *Server) doHandshake(conn net.Conn) error {
|
||||
err := s.read(conn, &handshake)
|
||||
if err != nil {
|
||||
return err
|
||||
} else if handshake.Version != protocolVersion1 && handshake.Version != protocolVersion2 {
|
||||
} else if handshake.Version == nil {
|
||||
return errors.Err("handshake is missing protocol version")
|
||||
} else if *handshake.Version != protocolVersion1 && *handshake.Version != protocolVersion2 {
|
||||
return errors.Err("protocol version not supported")
|
||||
}
|
||||
|
||||
@@ -268,7 +317,7 @@ func (s *Server) readBlobRequest(conn net.Conn) (int, string, bool, error) {
|
||||
return blobSize, blobHash, isSdBlob, errors.Err("blob hash is empty")
|
||||
}
|
||||
if blobSize > maxBlobSize {
|
||||
return blobSize, blobHash, isSdBlob, errors.Err("blob must be at most " + strconv.Itoa(maxBlobSize) + " bytes")
|
||||
return blobSize, blobHash, isSdBlob, errors.Err(ErrBlobTooBig)
|
||||
}
|
||||
if blobSize == 0 {
|
||||
return blobSize, blobHash, isSdBlob, errors.Err("0-byte blob received")
|
||||
@@ -277,14 +326,14 @@ func (s *Server) readBlobRequest(conn net.Conn) (int, string, bool, error) {
|
||||
return blobSize, blobHash, isSdBlob, nil
|
||||
}
|
||||
|
||||
func (s *Server) sendBlobResponse(conn net.Conn, blobExists, isSdBlob bool, neededBlobs []string) error {
|
||||
func (s *Server) sendBlobResponse(conn net.Conn, shouldSendBlob, isSdBlob bool, neededBlobs []string) error {
|
||||
var response []byte
|
||||
var err error
|
||||
|
||||
if isSdBlob {
|
||||
response, err = json.Marshal(sendSdBlobResponse{SendSdBlob: !blobExists, NeededBlobs: neededBlobs})
|
||||
response, err = json.Marshal(sendSdBlobResponse{SendSdBlob: shouldSendBlob, NeededBlobs: neededBlobs})
|
||||
} else {
|
||||
response, err = json.Marshal(sendBlobResponse{SendBlob: !blobExists})
|
||||
response, err = json.Marshal(sendBlobResponse{SendBlob: shouldSendBlob})
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -315,7 +364,16 @@ func (s *Server) read(conn net.Conn, v interface{}) error {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
return errors.Err(json.NewDecoder(conn).Decode(v))
|
||||
dec := json.NewDecoder(conn)
|
||||
err = dec.Decode(v)
|
||||
if err != nil {
|
||||
data, _ := ioutil.ReadAll(dec.Buffered())
|
||||
if len(data) > 0 {
|
||||
return errors.Err("%s. Data: %s", err.Error(), hex.EncodeToString(data))
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) readRawBlob(conn net.Conn, blobSize int) ([]byte, error) {
|
||||
@@ -351,17 +409,23 @@ func (s *Server) quitting() bool {
|
||||
}
|
||||
}
|
||||
|
||||
// BlobHash returns the sha512 hash hex encoded string of the blob byte slice.
|
||||
func BlobHash(blob []byte) string {
|
||||
hashBytes := sha512.Sum384(blob)
|
||||
return hex.EncodeToString(hashBytes[:])
|
||||
}
|
||||
|
||||
type errorResponse struct {
|
||||
Error string `json:"error"`
|
||||
func IsValidJSON(b []byte) bool {
|
||||
var r json.RawMessage
|
||||
return json.Unmarshal(b, &r) == nil
|
||||
}
|
||||
|
||||
//type errorResponse struct {
|
||||
// Error string `json:"error"`
|
||||
//}
|
||||
|
||||
type handshakeRequestResponse struct {
|
||||
Version int `json:"version"`
|
||||
Version *int `json:"version"`
|
||||
}
|
||||
|
||||
type sendBlobRequest struct {
|
||||
|
||||
+25
-22
@@ -2,19 +2,18 @@ package reflector
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"sort"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"encoding/json"
|
||||
"github.com/lbryio/lbry.go/v2/dht/bits"
|
||||
|
||||
"sort"
|
||||
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/phayes/freeport"
|
||||
)
|
||||
|
||||
@@ -24,7 +23,7 @@ func startServerOnRandomPort(t *testing.T) (*Server, int) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
srv := NewServer(&store.MemoryBlobStore{})
|
||||
srv := NewServer(store.NewMemStore(), store.NewMemStore())
|
||||
err = srv.Start("127.0.0.1:" + strconv.Itoa(port))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -121,7 +120,7 @@ func TestServer_Timeout(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
srv := NewServer(&store.MemoryBlobStore{})
|
||||
srv := NewServer(store.NewMemStore(), store.NewMemStore())
|
||||
srv.Timeout = testTimeout
|
||||
err = srv.Start("127.0.0.1:" + strconv.Itoa(port))
|
||||
if err != nil {
|
||||
@@ -146,8 +145,24 @@ func TestServer_Timeout(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
//func TestServer_InvalidJSONHandshake(t *testing.T) {
|
||||
// srv, port := startServerOnRandomPort(t)
|
||||
// defer srv.Shutdown()
|
||||
//
|
||||
// c := Client{}
|
||||
// err := c.Connect(":" + strconv.Itoa(port))
|
||||
// if err != nil {
|
||||
// t.Fatal("error connecting client to server", err)
|
||||
// }
|
||||
//
|
||||
// _, err = c.conn.Write([]byte(`{"stuff":4,tf}"`))
|
||||
// if err == nil {
|
||||
// t.Error("expected an error")
|
||||
// }
|
||||
//}
|
||||
|
||||
type mockPartialStore struct {
|
||||
store.MemoryBlobStore
|
||||
*store.MemStore
|
||||
missing []string
|
||||
}
|
||||
|
||||
@@ -167,7 +182,7 @@ func TestServer_PartialUpload(t *testing.T) {
|
||||
missing[i] = bits.Rand().String()
|
||||
}
|
||||
|
||||
st := store.BlobStore(&mockPartialStore{missing: missing})
|
||||
st := store.BlobStore(&mockPartialStore{MemStore: store.NewMemStore(), missing: missing})
|
||||
if _, ok := st.(neededBlobChecker); !ok {
|
||||
t.Fatal("mock does not implement the relevant interface")
|
||||
}
|
||||
@@ -176,7 +191,7 @@ func TestServer_PartialUpload(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
srv := NewServer(st)
|
||||
srv := NewServer(st, st)
|
||||
err = srv.Start("127.0.0.1:" + strconv.Itoa(port))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -226,18 +241,6 @@ func TestServer_PartialUpload(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
//func MakeRandStream(size int) ([]byte, [][]byte) {
|
||||
// blobs := make([][]byte, int(math.Ceil(float64(size)/maxBlobSize)))
|
||||
// for i := 0; i < len(blobs); i++ {
|
||||
// blobs[i] = randBlob(int(math.Min(maxBlobSize, float64(size))))
|
||||
// size -= maxBlobSize
|
||||
// }
|
||||
//
|
||||
// //TODO: create SD blob for the stream
|
||||
//
|
||||
// return nil, blobs
|
||||
//}
|
||||
|
||||
func randBlob(size int) []byte {
|
||||
//if size > maxBlobSize {
|
||||
// panic("blob size too big")
|
||||
|
||||
@@ -0,0 +1,271 @@
|
||||
package reflector
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/db"
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type increment int
|
||||
|
||||
const (
|
||||
sdInc increment = iota + 1
|
||||
blobInc
|
||||
errInc
|
||||
)
|
||||
|
||||
type Summary struct {
|
||||
Total, AlreadyStored, Sd, Blob, Err int
|
||||
}
|
||||
|
||||
type Uploader struct {
|
||||
db *db.SQL
|
||||
store *store.DBBackedStore // could just be store.BlobStore interface
|
||||
workers int
|
||||
skipExistsCheck bool
|
||||
deleteBlobsAfterUpload bool
|
||||
stopper *stop.Group
|
||||
countChan chan increment
|
||||
|
||||
count Summary
|
||||
}
|
||||
|
||||
func NewUploader(db *db.SQL, store *store.DBBackedStore, workers int, skipExistsCheck, deleteBlobsAfterUpload bool) *Uploader {
|
||||
return &Uploader{
|
||||
db: db,
|
||||
store: store,
|
||||
workers: workers,
|
||||
skipExistsCheck: skipExistsCheck,
|
||||
deleteBlobsAfterUpload: deleteBlobsAfterUpload,
|
||||
stopper: stop.New(),
|
||||
countChan: make(chan increment),
|
||||
}
|
||||
}
|
||||
|
||||
func (u *Uploader) Stop() {
|
||||
log.Infoln("stopping uploader")
|
||||
u.stopper.StopAndWait()
|
||||
}
|
||||
|
||||
func (u *Uploader) Upload(dirOrFilePath string) error {
|
||||
paths, err := getPaths(dirOrFilePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
u.count.Total = len(paths)
|
||||
|
||||
hashes := make([]string, len(paths))
|
||||
for i, p := range paths {
|
||||
hashes[i] = path.Base(p)
|
||||
}
|
||||
|
||||
log.Debug("checking for existing blobs")
|
||||
|
||||
var exists map[string]bool
|
||||
if !u.skipExistsCheck {
|
||||
exists, err = u.db.HasBlobs(hashes, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
u.count.AlreadyStored = len(exists)
|
||||
}
|
||||
|
||||
log.Debugf("%d new blobs to upload", u.count.Total-u.count.AlreadyStored)
|
||||
|
||||
workerWG := sync.WaitGroup{}
|
||||
pathChan := make(chan string)
|
||||
|
||||
for i := 0; i < u.workers; i++ {
|
||||
workerWG.Add(1)
|
||||
metrics.RoutinesQueue.WithLabelValues("reflector", "upload").Inc()
|
||||
go func(i int) {
|
||||
defer metrics.RoutinesQueue.WithLabelValues("reflector", "upload").Dec()
|
||||
defer workerWG.Done()
|
||||
defer func(i int) { log.Debugf("worker %d quitting", i) }(i)
|
||||
u.worker(pathChan)
|
||||
}(i)
|
||||
}
|
||||
|
||||
countWG := sync.WaitGroup{}
|
||||
countWG.Add(1)
|
||||
metrics.RoutinesQueue.WithLabelValues("reflector", "uploader").Inc()
|
||||
go func() {
|
||||
defer metrics.RoutinesQueue.WithLabelValues("reflector", "uploader").Dec()
|
||||
defer countWG.Done()
|
||||
u.counter()
|
||||
}()
|
||||
|
||||
Upload:
|
||||
for _, f := range paths {
|
||||
if exists != nil && exists[path.Base(f)] {
|
||||
continue
|
||||
}
|
||||
|
||||
select {
|
||||
case pathChan <- f:
|
||||
case <-u.stopper.Ch():
|
||||
break Upload
|
||||
}
|
||||
}
|
||||
|
||||
close(pathChan)
|
||||
workerWG.Wait()
|
||||
close(u.countChan)
|
||||
countWG.Wait()
|
||||
u.stopper.Stop()
|
||||
|
||||
log.Debugf(
|
||||
"upload stats: %d blobs total, %d already stored, %d SD blobs uploaded, %d content blobs uploaded, %d errors",
|
||||
u.count.Total, u.count.AlreadyStored, u.count.Sd, u.count.Blob, u.count.Err,
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
// worker reads paths from a channel, uploads them, and optionally deletes them
|
||||
func (u *Uploader) worker(pathChan chan string) {
|
||||
for {
|
||||
select {
|
||||
case <-u.stopper.Ch():
|
||||
return
|
||||
case filepath, ok := <-pathChan:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
err := u.uploadBlob(filepath)
|
||||
if err != nil {
|
||||
log.Errorln(err)
|
||||
} else if u.deleteBlobsAfterUpload {
|
||||
err = os.Remove(filepath)
|
||||
if err != nil {
|
||||
log.Errorln(errors.Prefix("deleting blob", err))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// uploadBlob uploads a blob
|
||||
func (u *Uploader) uploadBlob(filepath string) (err error) {
|
||||
defer func() {
|
||||
if err != nil {
|
||||
u.inc(errInc)
|
||||
}
|
||||
}()
|
||||
|
||||
blob, err := ioutil.ReadFile(filepath)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
hash := BlobHash(blob)
|
||||
if hash != path.Base(filepath) {
|
||||
return errors.Err("file name does not match hash (%s != %s), skipping", filepath, hash)
|
||||
}
|
||||
|
||||
if IsValidJSON(blob) {
|
||||
log.Debugf("uploading SD blob %s", hash)
|
||||
err := u.store.PutSD(hash, blob)
|
||||
if err != nil {
|
||||
return errors.Prefix("uploading SD blob "+hash, err)
|
||||
}
|
||||
u.inc(sdInc)
|
||||
} else {
|
||||
log.Debugf("uploading blob %s", hash)
|
||||
err = u.store.Put(hash, blob)
|
||||
if err != nil {
|
||||
return errors.Prefix("uploading blob "+hash, err)
|
||||
}
|
||||
u.inc(blobInc)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// counter updates the counts of how many sd blobs and content blobs were uploaded, and how many
|
||||
// errors were encountered. It occasionally prints the upload progress to debug.
|
||||
func (u *Uploader) counter() {
|
||||
start := time.Now()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-u.stopper.Ch():
|
||||
return
|
||||
case incrementType, ok := <-u.countChan:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
switch incrementType {
|
||||
case sdInc:
|
||||
u.count.Sd++
|
||||
case blobInc:
|
||||
u.count.Blob++
|
||||
case errInc:
|
||||
u.count.Err++
|
||||
}
|
||||
}
|
||||
if (u.count.Sd+u.count.Blob)%50 == 0 {
|
||||
log.Debugf("%d of %d done (%s elapsed, %.3fs per blob)", u.count.Sd+u.count.Blob, u.count.Total-u.count.AlreadyStored, time.Since(start).String(), time.Since(start).Seconds()/float64(u.count.Sd+u.count.Blob))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (u *Uploader) GetSummary() Summary {
|
||||
return u.count
|
||||
}
|
||||
|
||||
func (u *Uploader) inc(t increment) {
|
||||
select {
|
||||
case u.countChan <- t:
|
||||
case <-u.stopper.Ch():
|
||||
}
|
||||
}
|
||||
|
||||
// getPaths returns the paths for files to upload. it takes a path to a file or a dir. for a file,
|
||||
// it returns the full path to that file. for a dir, it returns the paths for all the files in the
|
||||
// dir
|
||||
func getPaths(dirOrFilePath string) ([]string, error) {
|
||||
info, err := os.Stat(dirOrFilePath)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
if info.Mode().IsRegular() {
|
||||
return []string{dirOrFilePath}, nil
|
||||
}
|
||||
|
||||
f, err := os.Open(dirOrFilePath)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
files, err := f.Readdir(-1)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
err = f.Close()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
var filenames []string
|
||||
for _, file := range files {
|
||||
if !file.IsDir() {
|
||||
filenames = append(filenames, dirOrFilePath+"/"+file.Name())
|
||||
}
|
||||
}
|
||||
|
||||
return filenames, nil
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
func (s *Server) getBlob(c *gin.Context) {
|
||||
waiter := &sync.WaitGroup{}
|
||||
waiter.Add(1)
|
||||
enqueue(&blobRequest{c: c, finished: waiter})
|
||||
waiter.Wait()
|
||||
}
|
||||
|
||||
func (s *Server) HandleGetBlob(c *gin.Context) {
|
||||
start := time.Now()
|
||||
hash := c.Query("hash")
|
||||
|
||||
if s.missesCache.Has(hash) {
|
||||
serialized, err := shared.NewBlobTrace(time.Since(start), "http").Serialize()
|
||||
c.Header("Via", serialized)
|
||||
if err != nil {
|
||||
_ = c.Error(errors.Err(err))
|
||||
c.String(http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
c.AbortWithStatus(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
blob, trace, err := s.store.Get(hash)
|
||||
if err != nil {
|
||||
serialized, serializeErr := trace.Serialize()
|
||||
if serializeErr != nil {
|
||||
_ = c.Error(errors.Prefix(serializeErr.Error(), err))
|
||||
c.String(http.StatusInternalServerError, errors.Prefix(serializeErr.Error(), err).Error())
|
||||
return
|
||||
}
|
||||
c.Header("Via", serialized)
|
||||
|
||||
if errors.Is(err, store.ErrBlobNotFound) {
|
||||
_ = s.missesCache.Set(hash, true)
|
||||
c.AbortWithStatus(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
_ = c.Error(err)
|
||||
c.String(http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
serialized, err := trace.Serialize()
|
||||
if err != nil {
|
||||
_ = c.Error(err)
|
||||
c.String(http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
metrics.MtrOutBytesHttp.Add(float64(len(blob)))
|
||||
metrics.BlobDownloadCount.Inc()
|
||||
metrics.HttpDownloadCount.Inc()
|
||||
c.Header("Via", serialized)
|
||||
c.Header("Content-Disposition", "filename="+hash)
|
||||
c.Data(http.StatusOK, "application/octet-stream", blob)
|
||||
}
|
||||
|
||||
func (s *Server) hasBlob(c *gin.Context) {
|
||||
hash := c.Query("hash")
|
||||
has, err := s.store.Has(hash)
|
||||
if err != nil {
|
||||
_ = c.Error(err)
|
||||
c.String(http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
if has {
|
||||
c.Status(http.StatusNoContent)
|
||||
return
|
||||
}
|
||||
c.Status(http.StatusNotFound)
|
||||
}
|
||||
|
||||
func (s *Server) recoveryHandler(c *gin.Context, err interface{}) {
|
||||
c.JSON(500, gin.H{
|
||||
"title": "Error",
|
||||
"err": err,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
|
||||
"github.com/bluele/gcache"
|
||||
nice "github.com/ekyoung/gin-nice-recovery"
|
||||
"github.com/gin-gonic/gin"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// Server is an instance of a peer server that houses the listener and store.
|
||||
type Server struct {
|
||||
store store.BlobStore
|
||||
grp *stop.Group
|
||||
concurrentRequests int
|
||||
missesCache gcache.Cache
|
||||
}
|
||||
|
||||
// NewServer returns an initialized Server pointer.
|
||||
func NewServer(store store.BlobStore, requestQueueSize int) *Server {
|
||||
return &Server{
|
||||
store: store,
|
||||
grp: stop.New(),
|
||||
concurrentRequests: requestQueueSize,
|
||||
missesCache: gcache.New(2000).Expiration(5 * time.Minute).ARC().Build(),
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown gracefully shuts down the peer server.
|
||||
func (s *Server) Shutdown() {
|
||||
log.Debug("shutting down HTTP server")
|
||||
s.grp.StopAndWait()
|
||||
log.Debug("HTTP server stopped")
|
||||
}
|
||||
|
||||
// Start starts the server listener to handle connections.
|
||||
func (s *Server) Start(address string) error {
|
||||
gin.SetMode(gin.ReleaseMode)
|
||||
router := gin.Default()
|
||||
router.GET("/blob", s.getBlob)
|
||||
router.HEAD("/blob", s.hasBlob)
|
||||
// Install nice.Recovery, passing the handler to call after recovery
|
||||
router.Use(nice.Recovery(s.recoveryHandler))
|
||||
srv := &http.Server{
|
||||
Addr: address,
|
||||
Handler: router,
|
||||
}
|
||||
go s.listenForShutdown(srv)
|
||||
go InitWorkers(s, s.concurrentRequests)
|
||||
// Initializing the server in a goroutine so that
|
||||
// it won't block the graceful shutdown handling below
|
||||
s.grp.Add(1)
|
||||
go func() {
|
||||
defer s.grp.Done()
|
||||
log.Println("HTTP server listening on " + address)
|
||||
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||
log.Fatalf("listen: %s\n", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) listenForShutdown(listener *http.Server) {
|
||||
<-s.grp.Ch()
|
||||
// The context is used to inform the server it has 5 seconds to finish
|
||||
// the request it is currently handling
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if err := listener.Shutdown(ctx); err != nil {
|
||||
log.Fatal("Server forced to shutdown:", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
type blobRequest struct {
|
||||
c *gin.Context
|
||||
finished *sync.WaitGroup
|
||||
}
|
||||
|
||||
var getReqCh = make(chan *blobRequest, 20000)
|
||||
|
||||
func InitWorkers(server *Server, workers int) {
|
||||
stopper := stop.New(server.grp)
|
||||
for i := 0; i < workers; i++ {
|
||||
metrics.RoutinesQueue.WithLabelValues("http", "worker").Inc()
|
||||
go func(worker int) {
|
||||
defer metrics.RoutinesQueue.WithLabelValues("http", "worker").Dec()
|
||||
for {
|
||||
select {
|
||||
case <-stopper.Ch():
|
||||
case r := <-getReqCh:
|
||||
process(server, r)
|
||||
metrics.HttpBlobReqQueue.Dec()
|
||||
}
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
}
|
||||
|
||||
func enqueue(b *blobRequest) {
|
||||
metrics.HttpBlobReqQueue.Inc()
|
||||
getReqCh <- b
|
||||
}
|
||||
|
||||
func process(server *Server, r *blobRequest) {
|
||||
server.HandleGetBlob(r.c)
|
||||
r.finished.Done()
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
package http3
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
"github.com/lucas-clemente/quic-go/http3"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// Client is an instance of a client connected to a server.
|
||||
type Client struct {
|
||||
Timeout time.Duration
|
||||
conn *http.Client
|
||||
roundTripper *http3.RoundTripper
|
||||
ServerAddr string
|
||||
}
|
||||
|
||||
// Close closes the connection with the client.
|
||||
func (c *Client) Close() error {
|
||||
c.conn.CloseIdleConnections()
|
||||
return c.roundTripper.Close()
|
||||
}
|
||||
|
||||
// GetStream gets a stream
|
||||
func (c *Client) GetStream(sdHash string, blobCache store.BlobStore) (stream.Stream, error) {
|
||||
var sd stream.SDBlob
|
||||
|
||||
b, _, err := c.GetBlob(sdHash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = sd.FromBlob(b)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s := make(stream.Stream, len(sd.BlobInfos)+1-1) // +1 for sd blob, -1 for last null blob
|
||||
s[0] = b
|
||||
|
||||
for i := 0; i < len(sd.BlobInfos)-1; i++ {
|
||||
var trace shared.BlobTrace
|
||||
s[i+1], trace, err = c.GetBlob(hex.EncodeToString(sd.BlobInfos[i].BlobHash))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
log.Debug(trace.String())
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// HasBlob checks if the blob is available
|
||||
func (c *Client) HasBlob(hash string) (bool, error) {
|
||||
resp, err := c.conn.Get(fmt.Sprintf("https://%s/has/%s", c.ServerAddr, hash))
|
||||
if err != nil {
|
||||
return false, errors.Err(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusOK {
|
||||
return true, nil
|
||||
}
|
||||
if resp.StatusCode == http.StatusNotFound {
|
||||
return false, nil
|
||||
}
|
||||
return false, errors.Err("non 200 status code returned: %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
// GetBlob gets a blob
|
||||
func (c *Client) GetBlob(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
start := time.Now()
|
||||
resp, err := c.conn.Get(fmt.Sprintf("https://%s/get/%s?trace=true", c.ServerAddr, hash))
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode == http.StatusNotFound {
|
||||
fmt.Printf("%s blob not found %d\n", hash, resp.StatusCode)
|
||||
return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err(store.ErrBlobNotFound)
|
||||
} else if resp.StatusCode != http.StatusOK {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err("non 200 status code returned: %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
tmp := getBuffer()
|
||||
defer putBuffer(tmp)
|
||||
serialized := resp.Header.Get("Via")
|
||||
trace := shared.NewBlobTrace(time.Since(start), "http3")
|
||||
if serialized != "" {
|
||||
parsedTrace, err := shared.Deserialize(serialized)
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), "http3"), err
|
||||
}
|
||||
trace = *parsedTrace
|
||||
}
|
||||
written, err := io.Copy(tmp, resp.Body)
|
||||
if err != nil {
|
||||
return nil, trace.Stack(time.Since(start), "http3"), errors.Err(err)
|
||||
}
|
||||
|
||||
blob := make([]byte, written)
|
||||
copy(blob, tmp.Bytes())
|
||||
|
||||
metrics.MtrInBytesUdp.Add(float64(len(blob)))
|
||||
|
||||
return blob, trace.Stack(time.Since(start), "http3"), nil
|
||||
}
|
||||
|
||||
// buffer pool to reduce GC
|
||||
// https://www.captaincodeman.com/2017/06/02/golang-buffer-pool-gotcha
|
||||
var buffers = sync.Pool{
|
||||
// New is called when a new instance is needed
|
||||
New: func() interface{} {
|
||||
buf := make([]byte, 0, stream.MaxBlobSize)
|
||||
return bytes.NewBuffer(buf)
|
||||
},
|
||||
}
|
||||
|
||||
// getBuffer fetches a buffer from the pool
|
||||
func getBuffer() *bytes.Buffer {
|
||||
return buffers.Get().(*bytes.Buffer)
|
||||
}
|
||||
|
||||
// putBuffer returns a buffer to the pool
|
||||
func putBuffer(buf *bytes.Buffer) {
|
||||
buf.Reset()
|
||||
buffers.Put(buf)
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
package http3
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/lucas-clemente/quic-go"
|
||||
"github.com/lucas-clemente/quic-go/http3"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// Server is an instance of a peer server that houses the listener and store.
|
||||
type Server struct {
|
||||
store store.BlobStore
|
||||
grp *stop.Group
|
||||
concurrentRequests int
|
||||
}
|
||||
|
||||
// NewServer returns an initialized Server pointer.
|
||||
func NewServer(store store.BlobStore, requestQueueSize int) *Server {
|
||||
return &Server{
|
||||
store: store,
|
||||
grp: stop.New(),
|
||||
concurrentRequests: requestQueueSize,
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown gracefully shuts down the peer server.
|
||||
func (s *Server) Shutdown() {
|
||||
log.Debug("shutting down http3 peer server")
|
||||
s.grp.StopAndWait()
|
||||
log.Debug("http3 peer server stopped")
|
||||
}
|
||||
|
||||
func (s *Server) logError(e error) {
|
||||
if e == nil {
|
||||
return
|
||||
}
|
||||
shouldLog := metrics.TrackError(metrics.DirectionDownload, e)
|
||||
if shouldLog {
|
||||
log.Errorln(errors.FullTrace(e))
|
||||
}
|
||||
}
|
||||
|
||||
type availabilityResponse struct {
|
||||
LbrycrdAddress string `json:"lbrycrd_address"`
|
||||
IsAvailable bool `json:"is_available"`
|
||||
}
|
||||
|
||||
// Start starts the server listener to handle connections.
|
||||
func (s *Server) Start(address string) error {
|
||||
log.Println("HTTP3 peer listening on " + address)
|
||||
window500M := 500 * 1 << 20
|
||||
|
||||
quicConf := &quic.Config{
|
||||
MaxStreamReceiveWindow: uint64(window500M),
|
||||
MaxConnectionReceiveWindow: uint64(window500M),
|
||||
EnableDatagrams: true,
|
||||
HandshakeIdleTimeout: 4 * time.Second,
|
||||
MaxIdleTimeout: 20 * time.Second,
|
||||
}
|
||||
r := mux.NewRouter()
|
||||
r.HandleFunc("/get/{hash}", func(w http.ResponseWriter, r *http.Request) {
|
||||
waiter := &sync.WaitGroup{}
|
||||
waiter.Add(1)
|
||||
enqueue(&blobRequest{request: r, reply: w, finished: waiter})
|
||||
waiter.Wait()
|
||||
})
|
||||
r.HandleFunc("/has/{hash}", func(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
requestedBlob := vars["hash"]
|
||||
blobExists, err := s.store.Has(requestedBlob)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
s.logError(err)
|
||||
return
|
||||
}
|
||||
if !blobExists {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
// LbrycrdAddress to be used when paying for data. Not implemented yet.
|
||||
const LbrycrdAddress = "bJxKvpD96kaJLriqVajZ7SaQTsWWyrGQct"
|
||||
resp, err := json.Marshal(availabilityResponse{
|
||||
LbrycrdAddress: LbrycrdAddress,
|
||||
IsAvailable: blobExists,
|
||||
})
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
s.logError(err)
|
||||
return
|
||||
}
|
||||
_, err = w.Write(resp)
|
||||
if err != nil {
|
||||
s.logError(err)
|
||||
}
|
||||
})
|
||||
server := http3.Server{
|
||||
Server: &http.Server{
|
||||
Handler: r,
|
||||
Addr: address,
|
||||
TLSConfig: generateTLSConfig(),
|
||||
},
|
||||
QuicConfig: quicConf,
|
||||
}
|
||||
go InitWorkers(s, s.concurrentRequests)
|
||||
go s.listenForShutdown(&server)
|
||||
s.grp.Add(1)
|
||||
go func() {
|
||||
s.listenAndServe(&server)
|
||||
s.grp.Done()
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Setup a bare-bones TLS config for the server
|
||||
func generateTLSConfig() *tls.Config {
|
||||
key, err := rsa.GenerateKey(rand.Reader, 1024)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
template := x509.Certificate{SerialNumber: big.NewInt(1)}
|
||||
certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})
|
||||
certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
|
||||
|
||||
tlsCert, err := tls.X509KeyPair(certPEM, keyPEM)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return &tls.Config{
|
||||
Certificates: []tls.Certificate{tlsCert},
|
||||
NextProtos: []string{"http3-reflector-server"},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) listenAndServe(server *http3.Server) {
|
||||
err := server.ListenAndServe()
|
||||
if err != nil && err.Error() != "server closed" {
|
||||
log.Errorln(errors.FullTrace(err))
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) listenForShutdown(listener *http3.Server) {
|
||||
<-s.grp.Ch()
|
||||
err := listener.Close()
|
||||
if err != nil {
|
||||
log.Error("error closing listener for peer server - ", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) HandleGetBlob(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
requestedBlob := vars["hash"]
|
||||
traceParam := r.URL.Query().Get("trace")
|
||||
var err error
|
||||
wantsTrace := false
|
||||
if traceParam != "" {
|
||||
wantsTrace, err = strconv.ParseBool(traceParam)
|
||||
if err != nil {
|
||||
wantsTrace = false
|
||||
}
|
||||
}
|
||||
|
||||
blob, trace, err := s.store.Get(requestedBlob)
|
||||
|
||||
if wantsTrace {
|
||||
serialized, err := trace.Serialize()
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
w.Header().Add("Via", serialized)
|
||||
log.Debug(trace.String())
|
||||
}
|
||||
if err != nil {
|
||||
if errors.Is(err, store.ErrBlobNotFound) {
|
||||
http.Error(w, err.Error(), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
fmt.Printf("%s: %s", requestedBlob, errors.FullTrace(err))
|
||||
s.logError(err)
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
_, err = w.Write(blob)
|
||||
if err != nil {
|
||||
s.logError(err)
|
||||
}
|
||||
metrics.MtrOutBytesUdp.Add(float64(len(blob)))
|
||||
metrics.BlobDownloadCount.Inc()
|
||||
metrics.Http3DownloadCount.Inc()
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
package http3
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
"github.com/lucas-clemente/quic-go"
|
||||
"github.com/lucas-clemente/quic-go/http3"
|
||||
)
|
||||
|
||||
// Store is a blob store that gets blobs from a peer.
|
||||
// It satisfies the store.BlobStore interface but cannot put or delete blobs.
|
||||
type Store struct {
|
||||
opts StoreOpts
|
||||
NotFoundCache *sync.Map
|
||||
}
|
||||
|
||||
// StoreOpts allows to set options for a new Store.
|
||||
type StoreOpts struct {
|
||||
Address string
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
// NewStore makes a new peer store.
|
||||
func NewStore(opts StoreOpts) *Store {
|
||||
return &Store{opts: opts, NotFoundCache: &sync.Map{}}
|
||||
}
|
||||
|
||||
func (p *Store) getClient() (*Client, error) {
|
||||
var qconf quic.Config
|
||||
window500M := 500 * 1 << 20
|
||||
qconf.MaxStreamReceiveWindow = uint64(window500M)
|
||||
qconf.MaxConnectionReceiveWindow = uint64(window500M)
|
||||
qconf.EnableDatagrams = true
|
||||
qconf.HandshakeIdleTimeout = 4 * time.Second
|
||||
qconf.MaxIdleTimeout = 20 * time.Second
|
||||
pool, err := x509.SystemCertPool()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
roundTripper := &http3.RoundTripper{
|
||||
TLSClientConfig: &tls.Config{
|
||||
RootCAs: pool,
|
||||
InsecureSkipVerify: true,
|
||||
},
|
||||
QuicConfig: &qconf,
|
||||
}
|
||||
connection := &http.Client{
|
||||
Transport: roundTripper,
|
||||
}
|
||||
c := &Client{
|
||||
conn: connection,
|
||||
roundTripper: roundTripper,
|
||||
ServerAddr: p.opts.Address,
|
||||
}
|
||||
return c, errors.Prefix("connection error", err)
|
||||
}
|
||||
|
||||
func (p *Store) Name() string { return "http3" }
|
||||
|
||||
// Has asks the peer if they have a hash
|
||||
func (p *Store) Has(hash string) (bool, error) {
|
||||
c, err := p.getClient()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer c.Close()
|
||||
return c.HasBlob(hash)
|
||||
}
|
||||
|
||||
// Get downloads the blob from the peer
|
||||
func (p *Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
start := time.Now()
|
||||
if lastChecked, ok := p.NotFoundCache.Load(hash); ok {
|
||||
if lastChecked.(time.Time).After(time.Now().Add(-5 * time.Minute)) {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), p.Name()+"-notfoundcache"), store.ErrBlobNotFound
|
||||
}
|
||||
}
|
||||
c, err := p.getClient()
|
||||
if err != nil && strings.Contains(err.Error(), "blob not found") {
|
||||
p.NotFoundCache.Store(hash, time.Now())
|
||||
}
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), p.Name()), err
|
||||
}
|
||||
defer c.Close()
|
||||
return c.GetBlob(hash)
|
||||
}
|
||||
|
||||
// Put is not supported
|
||||
func (p *Store) Put(hash string, blob stream.Blob) error {
|
||||
return errors.Err(shared.ErrNotImplemented)
|
||||
}
|
||||
|
||||
// PutSD is not supported
|
||||
func (p *Store) PutSD(hash string, blob stream.Blob) error {
|
||||
return errors.Err(shared.ErrNotImplemented)
|
||||
}
|
||||
|
||||
// Delete is not supported
|
||||
func (p *Store) Delete(hash string) error {
|
||||
return errors.Err(shared.ErrNotImplemented)
|
||||
}
|
||||
|
||||
// Shutdown is not supported
|
||||
func (p *Store) Shutdown() {
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package http3
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"sync"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
)
|
||||
|
||||
type blobRequest struct {
|
||||
request *http.Request
|
||||
reply http.ResponseWriter
|
||||
finished *sync.WaitGroup
|
||||
}
|
||||
|
||||
var getReqCh = make(chan *blobRequest, 20000)
|
||||
|
||||
func InitWorkers(server *Server, workers int) {
|
||||
stopper := stop.New(server.grp)
|
||||
for i := 0; i < workers; i++ {
|
||||
metrics.RoutinesQueue.WithLabelValues("http3", "worker").Inc()
|
||||
go func(worker int) {
|
||||
defer metrics.RoutinesQueue.WithLabelValues("http3", "worker").Dec()
|
||||
for {
|
||||
select {
|
||||
case <-stopper.Ch():
|
||||
case r := <-getReqCh:
|
||||
metrics.Http3BlobReqQueue.Dec()
|
||||
process(server, r)
|
||||
}
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
}
|
||||
|
||||
func enqueue(b *blobRequest) {
|
||||
metrics.Http3BlobReqQueue.Inc()
|
||||
getReqCh <- b
|
||||
}
|
||||
|
||||
func process(server *Server, r *blobRequest) {
|
||||
server.HandleGetBlob(r.reply, r.request)
|
||||
r.finished.Done()
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
package peer
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// Client is an instance of a client connected to a server.
|
||||
type Client struct {
|
||||
Timeout time.Duration
|
||||
conn net.Conn
|
||||
buf *bufio.Reader
|
||||
connected bool
|
||||
}
|
||||
|
||||
// Connect connects to a specific clients and errors if it cannot be contacted.
|
||||
func (c *Client) Connect(address string) error {
|
||||
var err error
|
||||
if c.Timeout == 0 {
|
||||
c.Timeout = 5 * time.Second
|
||||
}
|
||||
c.conn, err = net.Dial("tcp4", address)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.connected = true
|
||||
c.buf = bufio.NewReader(c.conn)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close closes the connection with the client.
|
||||
func (c *Client) Close() error {
|
||||
c.connected = false
|
||||
return c.conn.Close()
|
||||
}
|
||||
|
||||
// GetStream gets a stream
|
||||
func (c *Client) GetStream(sdHash string, blobCache store.BlobStore) (stream.Stream, error) {
|
||||
if !c.connected {
|
||||
return nil, errors.Err("not connected")
|
||||
}
|
||||
|
||||
var sd stream.SDBlob
|
||||
|
||||
b, trace, err := c.GetBlob(sdHash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
log.Debug(trace.String())
|
||||
|
||||
err = sd.FromBlob(b)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s := make(stream.Stream, len(sd.BlobInfos)+1-1) // +1 for sd blob, -1 for last null blob
|
||||
s[0] = b
|
||||
|
||||
for i := 0; i < len(sd.BlobInfos)-1; i++ {
|
||||
s[i+1], trace, err = c.GetBlob(hex.EncodeToString(sd.BlobInfos[i].BlobHash))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
log.Debug(trace.String())
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// HasBlob checks if the blob is available
|
||||
func (c *Client) HasBlob(hash string) (bool, error) {
|
||||
if !c.connected {
|
||||
return false, errors.Err("not connected")
|
||||
}
|
||||
|
||||
sendRequest, err := json.Marshal(availabilityRequest{
|
||||
RequestedBlobs: []string{hash},
|
||||
})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
err = c.write(sendRequest)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
var resp availabilityResponse
|
||||
err = c.read(&resp)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
for _, h := range resp.AvailableBlobs {
|
||||
if h == hash {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// GetBlob gets a blob
|
||||
func (c *Client) GetBlob(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
start := time.Now()
|
||||
if !c.connected {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), "tcp"), errors.Err("not connected")
|
||||
}
|
||||
|
||||
sendRequest, err := json.Marshal(blobRequest{
|
||||
RequestedBlob: hash,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), "tcp"), err
|
||||
}
|
||||
|
||||
err = c.write(sendRequest)
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), "tcp"), err
|
||||
}
|
||||
|
||||
var resp blobResponse
|
||||
err = c.read(&resp)
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), "tcp"), err
|
||||
}
|
||||
|
||||
trace := shared.NewBlobTrace(time.Since(start), "tcp")
|
||||
if resp.RequestTrace != nil {
|
||||
trace = *resp.RequestTrace
|
||||
}
|
||||
if resp.IncomingBlob.Error != "" {
|
||||
return nil, trace, errors.Prefix(hash[:8], resp.IncomingBlob.Error)
|
||||
}
|
||||
if resp.IncomingBlob.BlobHash != hash {
|
||||
return nil, trace.Stack(time.Since(start), "tcp"), errors.Prefix(hash[:8], "blob hash in response does not match requested hash")
|
||||
}
|
||||
if resp.IncomingBlob.Length <= 0 {
|
||||
return nil, trace, errors.Prefix(hash[:8], "length reported as <= 0")
|
||||
}
|
||||
|
||||
log.Debugf("receiving blob %s from %s", hash[:8], c.conn.RemoteAddr())
|
||||
|
||||
blob, err := c.readRawBlob(resp.IncomingBlob.Length)
|
||||
if err != nil {
|
||||
return nil, (*resp.RequestTrace).Stack(time.Since(start), "tcp"), err
|
||||
}
|
||||
metrics.MtrInBytesTcp.Add(float64(len(blob)))
|
||||
return blob, trace.Stack(time.Since(start), "tcp"), nil
|
||||
}
|
||||
|
||||
func (c *Client) read(v interface{}) error {
|
||||
err := c.conn.SetReadDeadline(time.Now().Add(c.Timeout))
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
m, err := readNextMessage(c.buf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Debugf("read %d bytes from %s", len(m), c.conn.RemoteAddr())
|
||||
|
||||
err = json.Unmarshal(m, v)
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
func (c *Client) readRawBlob(blobSize int) ([]byte, error) {
|
||||
err := c.conn.SetReadDeadline(time.Now().Add(c.Timeout))
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
blob := make([]byte, blobSize)
|
||||
n, err := io.ReadFull(c.buf, blob)
|
||||
log.Debugf("read %d bytes from %s", n, c.conn.RemoteAddr())
|
||||
return blob, errors.Err(err)
|
||||
}
|
||||
|
||||
func (c *Client) write(b []byte) error {
|
||||
err := c.conn.SetWriteDeadline(time.Now().Add(c.Timeout))
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
log.Debugf("writing %d bytes to %s", len(b), c.conn.RemoteAddr())
|
||||
|
||||
n, err := c.conn.Write(b)
|
||||
if err == nil && n != len(b) {
|
||||
err = io.ErrShortWrite
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
@@ -0,0 +1,404 @@
|
||||
package peer
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
ee "errors"
|
||||
"io"
|
||||
"net"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const (
|
||||
// DefaultPort is the port the peer server listens on if not passed in.
|
||||
DefaultPort = 3333
|
||||
// LbrycrdAddress to be used when paying for data. Not implemented yet.
|
||||
LbrycrdAddress = "bJxKvpD96kaJLriqVajZ7SaQTsWWyrGQct"
|
||||
)
|
||||
|
||||
// Server is an instance of a peer server that houses the listener and store.
|
||||
type Server struct {
|
||||
store store.BlobStore
|
||||
disk *store.DiskStore
|
||||
closed bool
|
||||
|
||||
grp *stop.Group
|
||||
}
|
||||
|
||||
// NewServer returns an initialized Server pointer.
|
||||
func NewServer(blobstore store.BlobStore) *Server {
|
||||
return &Server{
|
||||
store: blobstore,
|
||||
disk: store.NewDiskStore("/external/blobs", 2),
|
||||
grp: stop.New(),
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown gracefully shuts down the peer server.
|
||||
func (s *Server) Shutdown() {
|
||||
log.Debug("shutting down peer server")
|
||||
s.grp.StopAndWait()
|
||||
log.Debug("peer server stopped")
|
||||
}
|
||||
|
||||
// Start starts the server listener to handle connections.
|
||||
func (s *Server) Start(address string) error {
|
||||
log.Println("peer listening on " + address)
|
||||
l, err := net.Listen("tcp4", address)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
go s.listenForShutdown(l)
|
||||
s.grp.Add(1)
|
||||
go func() {
|
||||
s.listenAndServe(l)
|
||||
s.grp.Done()
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) listenForShutdown(listener net.Listener) {
|
||||
<-s.grp.Ch()
|
||||
s.closed = true
|
||||
err := listener.Close()
|
||||
if err != nil {
|
||||
log.Error("error closing listener for peer server - ", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) listenAndServe(listener net.Listener) {
|
||||
for {
|
||||
conn, err := listener.Accept()
|
||||
if err != nil {
|
||||
if s.closed {
|
||||
return
|
||||
}
|
||||
log.Error(errors.Prefix("accepting conn", err))
|
||||
} else {
|
||||
s.grp.Add(1)
|
||||
metrics.RoutinesQueue.WithLabelValues("peer", "server-handleconn").Inc()
|
||||
go func() {
|
||||
defer metrics.RoutinesQueue.WithLabelValues("peer", "server-handleconn").Dec()
|
||||
s.handleConnection(conn)
|
||||
s.grp.Done()
|
||||
}()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) handleConnection(conn net.Conn) {
|
||||
defer func() {
|
||||
if err := conn.Close(); err != nil {
|
||||
log.Error(errors.Prefix("closing peer conn", err))
|
||||
}
|
||||
}()
|
||||
|
||||
timeoutDuration := 1 * time.Minute
|
||||
buf := bufio.NewReader(conn)
|
||||
|
||||
for {
|
||||
var request []byte
|
||||
var response []byte
|
||||
|
||||
err := conn.SetReadDeadline(time.Now().Add(timeoutDuration))
|
||||
if err != nil {
|
||||
log.Error(errors.FullTrace(err))
|
||||
}
|
||||
|
||||
request, err = readNextMessage(buf)
|
||||
if err != nil {
|
||||
if err != io.EOF {
|
||||
s.logError(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
err = conn.SetReadDeadline(time.Time{})
|
||||
if err != nil {
|
||||
log.Error(errors.FullTrace(err))
|
||||
}
|
||||
|
||||
response, blobHash, err := s.handleCompositeRequest(request)
|
||||
if err != nil {
|
||||
log.Error(errors.FullTrace(err))
|
||||
return
|
||||
}
|
||||
|
||||
err = conn.SetWriteDeadline(time.Now().Add(timeoutDuration))
|
||||
if err != nil {
|
||||
log.Error(errors.FullTrace(err))
|
||||
}
|
||||
|
||||
n, err := conn.Write(response)
|
||||
s.disk.Sendfile(blobHash, conn)
|
||||
if err != nil {
|
||||
if !strings.Contains(err.Error(), "connection reset by peer") { // means the other side closed the connection using TCP reset
|
||||
s.logError(err)
|
||||
}
|
||||
return
|
||||
} else if n != len(response) {
|
||||
log.Errorln(io.ErrShortWrite)
|
||||
return
|
||||
}
|
||||
|
||||
err = conn.SetWriteDeadline(time.Time{})
|
||||
if err != nil {
|
||||
log.Error(errors.FullTrace(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) handleAvailabilityRequest(data []byte) ([]byte, error) {
|
||||
var request availabilityRequest
|
||||
err := json.Unmarshal(data, &request)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
availableBlobs := []string{}
|
||||
for _, blobHash := range request.RequestedBlobs {
|
||||
exists, err := s.store.Has(blobHash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if exists {
|
||||
availableBlobs = append(availableBlobs, blobHash)
|
||||
}
|
||||
}
|
||||
|
||||
return json.Marshal(availabilityResponse{LbrycrdAddress: LbrycrdAddress, AvailableBlobs: availableBlobs})
|
||||
}
|
||||
|
||||
//func (s *Server) handlePaymentRateNegotiation(data []byte) ([]byte, error) {
|
||||
// var request paymentRateRequest
|
||||
// err := json.Unmarshal(data, &request)
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
//
|
||||
// offerReply := paymentRateAccepted
|
||||
// if request.BlobDataPaymentRate < 0 {
|
||||
// offerReply = paymentRateTooLow
|
||||
// }
|
||||
//
|
||||
// return json.Marshal(paymentRateResponse{BlobDataPaymentRate: offerReply})
|
||||
//}
|
||||
//
|
||||
//func (s *Server) handleBlobRequest(data []byte) ([]byte, error) {
|
||||
// var request blobRequest
|
||||
// err := json.Unmarshal(data, &request)
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
//
|
||||
// log.Debugln("Sending blob " + request.RequestedBlob[:8])
|
||||
//
|
||||
// blob, err := s.store.Get(request.RequestedBlob)
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
//
|
||||
// response, err := json.Marshal(blobResponse{IncomingBlob: incomingBlob{
|
||||
// BlobHash: reflector.BlobHash(blob),
|
||||
// Length: len(blob),
|
||||
// }})
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
//
|
||||
// return append(response, blob...), nil
|
||||
//}
|
||||
|
||||
func (s *Server) handleCompositeRequest(data []byte) ([]byte, string, error) {
|
||||
var request compositeRequest
|
||||
err := json.Unmarshal(data, &request)
|
||||
if err != nil {
|
||||
var je *json.SyntaxError
|
||||
if ee.As(err, &je) {
|
||||
return nil, "", errors.Err("invalid json at offset %d in data %s", je.Offset, hex.EncodeToString(data))
|
||||
}
|
||||
return nil, "", errors.Err(err)
|
||||
}
|
||||
|
||||
response := compositeResponse{
|
||||
LbrycrdAddress: LbrycrdAddress,
|
||||
}
|
||||
|
||||
if len(request.RequestedBlobs) > 0 {
|
||||
var availableBlobs []string
|
||||
for _, blobHash := range request.RequestedBlobs {
|
||||
exists, err := s.store.Has(blobHash)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if exists {
|
||||
availableBlobs = append(availableBlobs, blobHash)
|
||||
}
|
||||
}
|
||||
response.AvailableBlobs = availableBlobs
|
||||
}
|
||||
|
||||
response.BlobDataPaymentRate = paymentRateAccepted
|
||||
if request.BlobDataPaymentRate < 0 {
|
||||
response.BlobDataPaymentRate = paymentRateTooLow
|
||||
}
|
||||
|
||||
var blob []byte
|
||||
if request.RequestedBlob != "" {
|
||||
if len(request.RequestedBlob) != stream.BlobHashHexLength {
|
||||
return nil, "", errors.Err("Invalid blob hash length")
|
||||
}
|
||||
|
||||
log.Debugln("Sending blob " + request.RequestedBlob[:8])
|
||||
|
||||
size, err := s.disk.Size(request.RequestedBlob)
|
||||
if err != nil {
|
||||
response.IncomingBlob = incomingBlob{
|
||||
Error: err.Error(),
|
||||
}
|
||||
} else {
|
||||
response.IncomingBlob = incomingBlob{
|
||||
BlobHash: request.RequestedBlob,
|
||||
Length: int(size),
|
||||
}
|
||||
}
|
||||
metrics.MtrOutBytesTcp.Add(float64(len(blob)))
|
||||
metrics.BlobDownloadCount.Inc()
|
||||
metrics.PeerDownloadCount.Inc()
|
||||
}
|
||||
|
||||
respData, err := json.Marshal(response)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
return respData, request.RequestedBlob, nil
|
||||
}
|
||||
|
||||
func (s *Server) logError(e error) {
|
||||
if e == nil {
|
||||
return
|
||||
}
|
||||
shouldLog := metrics.TrackError(metrics.DirectionDownload, e)
|
||||
if shouldLog {
|
||||
log.Errorln(errors.FullTrace(e))
|
||||
}
|
||||
}
|
||||
|
||||
func readNextMessage(buf *bufio.Reader) ([]byte, error) {
|
||||
msg := make([]byte, 0)
|
||||
eof := false
|
||||
|
||||
for {
|
||||
chunk, err := buf.ReadBytes('}')
|
||||
if err != nil {
|
||||
if err != io.EOF {
|
||||
//log.Errorln("readBytes error:", err) // logged by caller
|
||||
return msg, err
|
||||
}
|
||||
eof = true
|
||||
}
|
||||
|
||||
//log.Debugln("got", len(chunk), "bytes.")
|
||||
//spew.Dump(chunk)
|
||||
|
||||
if len(chunk) > 0 {
|
||||
msg = append(msg, chunk...)
|
||||
|
||||
if len(msg) > maxRequestSize {
|
||||
return msg, errRequestTooLarge
|
||||
}
|
||||
|
||||
// yes, this is how the peer protocol knows when the request finishes
|
||||
if reflector.IsValidJSON(msg) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if eof {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
//log.Debugln("total size:", len(request))
|
||||
//if len(request) > 0 {
|
||||
// spew.Dump(request)
|
||||
//}
|
||||
|
||||
if len(msg) == 0 && eof {
|
||||
return nil, io.EOF
|
||||
}
|
||||
|
||||
return msg, nil
|
||||
}
|
||||
|
||||
const (
|
||||
maxRequestSize = 64 * (2 ^ 10) // 64kb
|
||||
paymentRateAccepted = "RATE_ACCEPTED"
|
||||
paymentRateTooLow = "RATE_TOO_LOW"
|
||||
//ToDo: paymentRateUnset is not used but exists in the protocol.
|
||||
//paymentRateUnset = "RATE_UNSET"
|
||||
)
|
||||
|
||||
var errRequestTooLarge = errors.Base("request is too large")
|
||||
|
||||
type availabilityRequest struct {
|
||||
LbrycrdAddress bool `json:"lbrycrd_address"`
|
||||
RequestedBlobs []string `json:"requested_blobs"`
|
||||
}
|
||||
|
||||
type availabilityResponse struct {
|
||||
LbrycrdAddress string `json:"lbrycrd_address"`
|
||||
AvailableBlobs []string `json:"available_blobs"`
|
||||
}
|
||||
|
||||
type paymentRateRequest struct {
|
||||
BlobDataPaymentRate float64 `json:"blob_data_payment_rate"`
|
||||
}
|
||||
|
||||
type paymentRateResponse struct {
|
||||
BlobDataPaymentRate string `json:"blob_data_payment_rate"`
|
||||
}
|
||||
|
||||
type blobRequest struct {
|
||||
RequestedBlob string `json:"requested_blob"`
|
||||
}
|
||||
|
||||
type incomingBlob struct {
|
||||
Error string `json:"error,omitempty"`
|
||||
BlobHash string `json:"blob_hash"`
|
||||
Length int `json:"length"`
|
||||
}
|
||||
type blobResponse struct {
|
||||
IncomingBlob incomingBlob `json:"incoming_blob"`
|
||||
RequestTrace *shared.BlobTrace
|
||||
}
|
||||
|
||||
type compositeRequest struct {
|
||||
LbrycrdAddress bool `json:"lbrycrd_address"`
|
||||
RequestedBlobs []string `json:"requested_blobs"`
|
||||
BlobDataPaymentRate float64 `json:"blob_data_payment_rate"`
|
||||
RequestedBlob string `json:"requested_blob"`
|
||||
}
|
||||
|
||||
type compositeResponse struct {
|
||||
LbrycrdAddress string `json:"lbrycrd_address,omitempty"`
|
||||
AvailableBlobs []string `json:"available_blobs,omitempty"`
|
||||
BlobDataPaymentRate string `json:"blob_data_payment_rate,omitempty"`
|
||||
IncomingBlob incomingBlob `json:"incoming_blob,omitempty"`
|
||||
}
|
||||
@@ -34,7 +34,7 @@ var availabilityRequests = []pair{
|
||||
}
|
||||
|
||||
func getServer(t *testing.T, withBlobs bool) *Server {
|
||||
st := store.MemoryBlobStore{}
|
||||
st := store.NewMemStore()
|
||||
if withBlobs {
|
||||
for k, v := range blobs {
|
||||
err := st.Put(k, v)
|
||||
@@ -43,7 +43,7 @@ func getServer(t *testing.T, withBlobs bool) *Server {
|
||||
}
|
||||
}
|
||||
}
|
||||
return NewServer(&st)
|
||||
return NewServer(st)
|
||||
}
|
||||
|
||||
func TestAvailabilityRequest_NoBlobs(t *testing.T) {
|
||||
@@ -0,0 +1,82 @@
|
||||
package peer
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
)
|
||||
|
||||
// Store is a blob store that gets blobs from a peer.
|
||||
// It satisfies the store.BlobStore interface but cannot put or delete blobs.
|
||||
type Store struct {
|
||||
opts StoreOpts
|
||||
}
|
||||
|
||||
// StoreOpts allows to set options for a new Store.
|
||||
type StoreOpts struct {
|
||||
Address string
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
// NewStore makes a new peer store.
|
||||
func NewStore(opts StoreOpts) *Store {
|
||||
return &Store{opts: opts}
|
||||
}
|
||||
|
||||
func (p *Store) getClient() (*Client, error) {
|
||||
c := &Client{Timeout: p.opts.Timeout}
|
||||
err := c.Connect(p.opts.Address)
|
||||
return c, errors.Prefix("connection error", err)
|
||||
}
|
||||
|
||||
func (p *Store) Name() string { return "peer" }
|
||||
|
||||
// Has asks the peer if they have a hash
|
||||
func (p *Store) Has(hash string) (bool, error) {
|
||||
c, err := p.getClient()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer c.Close()
|
||||
return c.HasBlob(hash)
|
||||
}
|
||||
|
||||
// Get downloads the blob from the peer
|
||||
func (p *Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
start := time.Now()
|
||||
c, err := p.getClient()
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), p.Name()), err
|
||||
}
|
||||
defer c.Close()
|
||||
blob, trace, err := c.GetBlob(hash)
|
||||
if err != nil && strings.Contains(err.Error(), "blob not found") {
|
||||
return nil, trace, store.ErrBlobNotFound
|
||||
}
|
||||
|
||||
return blob, trace, err
|
||||
}
|
||||
|
||||
// Put is not supported
|
||||
func (p *Store) Put(hash string, blob stream.Blob) error {
|
||||
return errors.Err(shared.ErrNotImplemented)
|
||||
}
|
||||
|
||||
// PutSD is not supported
|
||||
func (p *Store) PutSD(hash string, blob stream.Blob) error {
|
||||
return errors.Err(shared.ErrNotImplemented)
|
||||
}
|
||||
|
||||
// Delete is not supported
|
||||
func (p *Store) Delete(hash string) error {
|
||||
return errors.Err(shared.ErrNotImplemented)
|
||||
}
|
||||
|
||||
// Shutdown is not supported
|
||||
func (p *Store) Shutdown() {
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
package shared
|
||||
|
||||
import "github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
|
||||
//ErrNotImplemented is a standard error when a store that implements the store interface does not implement a method
|
||||
var ErrNotImplemented = errors.Base("this store does not implement this method")
|
||||
@@ -0,0 +1,82 @@
|
||||
package shared
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
)
|
||||
|
||||
type BlobStack struct {
|
||||
Timing time.Duration `json:"timing"`
|
||||
OriginName string `json:"origin_name"`
|
||||
HostName string `json:"host_name"`
|
||||
}
|
||||
type BlobTrace struct {
|
||||
Stacks []BlobStack `json:"stacks"`
|
||||
}
|
||||
|
||||
var hostName *string
|
||||
|
||||
func getHostName() string {
|
||||
if hostName == nil {
|
||||
hn, err := os.Hostname()
|
||||
if err != nil {
|
||||
hn = "unknown"
|
||||
}
|
||||
hostName = &hn
|
||||
}
|
||||
return *hostName
|
||||
}
|
||||
func (b *BlobTrace) Stack(timing time.Duration, originName string) BlobTrace {
|
||||
b.Stacks = append(b.Stacks, BlobStack{
|
||||
Timing: timing,
|
||||
OriginName: originName,
|
||||
HostName: getHostName(),
|
||||
})
|
||||
return *b
|
||||
}
|
||||
func (b *BlobTrace) Merge(otherTrance BlobTrace) BlobTrace {
|
||||
b.Stacks = append(b.Stacks, otherTrance.Stacks...)
|
||||
return *b
|
||||
}
|
||||
func NewBlobTrace(timing time.Duration, originName string) BlobTrace {
|
||||
b := BlobTrace{}
|
||||
b.Stacks = append(b.Stacks, BlobStack{
|
||||
Timing: timing,
|
||||
OriginName: originName,
|
||||
HostName: getHostName(),
|
||||
})
|
||||
return b
|
||||
}
|
||||
|
||||
func (b BlobTrace) String() string {
|
||||
var fullTrace string
|
||||
for i, stack := range b.Stacks {
|
||||
delta := time.Duration(0)
|
||||
if i > 0 {
|
||||
delta = stack.Timing - b.Stacks[i-1].Timing
|
||||
}
|
||||
fullTrace += fmt.Sprintf("[%d](%s) origin: %s - timing: %s - delta: %s\n", i, stack.HostName, stack.OriginName, stack.Timing.String(), delta.String())
|
||||
}
|
||||
return fullTrace
|
||||
}
|
||||
|
||||
func (b BlobTrace) Serialize() (string, error) {
|
||||
t, err := json.Marshal(b)
|
||||
if err != nil {
|
||||
return "", errors.Err(err)
|
||||
}
|
||||
return string(t), nil
|
||||
}
|
||||
|
||||
func Deserialize(serializedData string) (*BlobTrace, error) {
|
||||
var trace BlobTrace
|
||||
err := json.Unmarshal([]byte(serializedData), &trace)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
return &trace, nil
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package shared
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestBlobTrace_Serialize(t *testing.T) {
|
||||
hostName = util.PtrToString("test_machine")
|
||||
stack := NewBlobTrace(10*time.Second, "test")
|
||||
stack.Stack(20*time.Second, "test2")
|
||||
stack.Stack(30*time.Second, "test3")
|
||||
serialized, err := stack.Serialize()
|
||||
assert.NoError(t, err)
|
||||
t.Log(serialized)
|
||||
expected := "{\"stacks\":[{\"timing\":10000000000,\"origin_name\":\"test\",\"host_name\":\"test_machine\"},{\"timing\":20000000000,\"origin_name\":\"test2\",\"host_name\":\"test_machine\"},{\"timing\":30000000000,\"origin_name\":\"test3\",\"host_name\":\"test_machine\"}]}"
|
||||
assert.Equal(t, expected, serialized)
|
||||
}
|
||||
|
||||
func TestBlobTrace_Deserialize(t *testing.T) {
|
||||
hostName = util.PtrToString("test_machine")
|
||||
serialized := "{\"stacks\":[{\"timing\":10000000000,\"origin_name\":\"test\"},{\"timing\":20000000000,\"origin_name\":\"test2\"},{\"timing\":30000000000,\"origin_name\":\"test3\"}]}"
|
||||
stack, err := Deserialize(serialized)
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, stack.Stacks, 3)
|
||||
assert.Equal(t, stack.Stacks[0].Timing, 10*time.Second)
|
||||
assert.Equal(t, stack.Stacks[1].Timing, 20*time.Second)
|
||||
assert.Equal(t, stack.Stacks[2].Timing, 30*time.Second)
|
||||
assert.Equal(t, stack.Stacks[0].OriginName, "test")
|
||||
assert.Equal(t, stack.Stacks[1].OriginName, "test2")
|
||||
assert.Equal(t, stack.Stacks[2].OriginName, "test3")
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
// +build linux
|
||||
|
||||
package store
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
func timespecToTime(ts syscall.Timespec) time.Time {
|
||||
return time.Unix(int64(ts.Sec), int64(ts.Nsec))
|
||||
}
|
||||
|
||||
func atime(fi os.FileInfo) time.Time {
|
||||
return timespecToTime(fi.Sys().(*syscall.Stat_t).Atim)
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
// +build !linux
|
||||
|
||||
package store
|
||||
|
||||
import (
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
func atime(fi os.FileInfo) time.Time {
|
||||
return fi.ModTime()
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// CachingStore combines two stores, typically a local and a remote store, to improve performance.
|
||||
// Accessed blobs are stored in and retrieved from the cache. If they are not in the cache, they
|
||||
// are retrieved from the origin and cached. Puts are cached and also forwarded to the origin.
|
||||
type CachingStore struct {
|
||||
origin, cache BlobStore
|
||||
component string
|
||||
}
|
||||
|
||||
// NewCachingStore makes a new caching disk store and returns a pointer to it.
|
||||
func NewCachingStore(component string, origin, cache BlobStore) *CachingStore {
|
||||
return &CachingStore{
|
||||
component: component,
|
||||
origin: WithSingleFlight(component, origin),
|
||||
cache: WithSingleFlight(component, cache),
|
||||
}
|
||||
}
|
||||
|
||||
const nameCaching = "caching"
|
||||
|
||||
// Name is the cache type name
|
||||
func (c *CachingStore) Name() string { return nameCaching }
|
||||
|
||||
// Has checks the cache and then the origin for a hash. It returns true if either store has it.
|
||||
func (c *CachingStore) Has(hash string) (bool, error) {
|
||||
has, err := c.cache.Has(hash)
|
||||
if has || err != nil {
|
||||
return has, err
|
||||
}
|
||||
return c.origin.Has(hash)
|
||||
}
|
||||
|
||||
// Get tries to get the blob from the cache first, falling back to the origin. If the blob comes
|
||||
// from the origin, it is also stored in the cache.
|
||||
func (c *CachingStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
start := time.Now()
|
||||
blob, trace, err := c.cache.Get(hash)
|
||||
if err == nil || !errors.Is(err, ErrBlobNotFound) {
|
||||
metrics.CacheHitCount.With(metrics.CacheLabels(c.cache.Name(), c.component)).Inc()
|
||||
rate := float64(len(blob)) / 1024 / 1024 / time.Since(start).Seconds()
|
||||
metrics.CacheRetrievalSpeed.With(map[string]string{
|
||||
metrics.LabelCacheType: c.cache.Name(),
|
||||
metrics.LabelComponent: c.component,
|
||||
metrics.LabelSource: "cache",
|
||||
}).Set(rate)
|
||||
return blob, trace.Stack(time.Since(start), c.Name()), err
|
||||
}
|
||||
|
||||
metrics.CacheMissCount.With(metrics.CacheLabels(c.cache.Name(), c.component)).Inc()
|
||||
|
||||
blob, trace, err = c.origin.Get(hash)
|
||||
if err != nil {
|
||||
return nil, trace.Stack(time.Since(start), c.Name()), err
|
||||
}
|
||||
// do not do this async unless you're prepared to deal with mayhem
|
||||
err = c.cache.Put(hash, blob)
|
||||
if err != nil {
|
||||
log.Errorf("error saving blob to underlying cache: %s", errors.FullTrace(err))
|
||||
}
|
||||
return blob, trace.Stack(time.Since(start), c.Name()), nil
|
||||
}
|
||||
|
||||
// Put stores the blob in the origin and the cache
|
||||
func (c *CachingStore) Put(hash string, blob stream.Blob) error {
|
||||
err := c.origin.Put(hash, blob)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return c.cache.Put(hash, blob)
|
||||
}
|
||||
|
||||
// PutSD stores the sd blob in the origin and the cache
|
||||
func (c *CachingStore) PutSD(hash string, blob stream.Blob) error {
|
||||
err := c.origin.PutSD(hash, blob)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return c.cache.PutSD(hash, blob)
|
||||
}
|
||||
|
||||
// Delete deletes the blob from the origin and the cache
|
||||
func (c *CachingStore) Delete(hash string) error {
|
||||
err := c.origin.Delete(hash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return c.cache.Delete(hash)
|
||||
}
|
||||
|
||||
// Shutdown shuts down the store gracefully
|
||||
func (c *CachingStore) Shutdown() {
|
||||
c.origin.Shutdown()
|
||||
c.cache.Shutdown()
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
)
|
||||
|
||||
func TestCachingStore_Put(t *testing.T) {
|
||||
origin := NewMemStore()
|
||||
cache := NewMemStore()
|
||||
s := NewCachingStore("test", origin, cache)
|
||||
|
||||
b := []byte("this is a blob of stuff")
|
||||
hash := "hash"
|
||||
|
||||
err := s.Put(hash, b)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
has, err := origin.Has(hash)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !has {
|
||||
t.Errorf("failed to store blob in origin")
|
||||
}
|
||||
|
||||
has, err = cache.Has(hash)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !has {
|
||||
t.Errorf("failed to store blob in cache")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCachingStore_CacheMiss(t *testing.T) {
|
||||
origin := NewMemStore()
|
||||
cache := NewMemStore()
|
||||
s := NewCachingStore("test", origin, cache)
|
||||
|
||||
b := []byte("this is a blob of stuff")
|
||||
hash := "hash"
|
||||
err := origin.Put(hash, b)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
res, stack, err := s.Get(hash)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(b, res) {
|
||||
t.Errorf("expected Get() to return %s, got %s", string(b), string(res))
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond) //storing to cache is done async so let's give it some time
|
||||
|
||||
has, err := cache.Has(hash)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !has {
|
||||
t.Errorf("Get() did not copy blob to cache")
|
||||
}
|
||||
t.Logf("stack: %s", stack.String())
|
||||
|
||||
res, stack, err = cache.Get(hash)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(b, res) {
|
||||
t.Errorf("expected cached Get() to return %s, got %s", string(b), string(res))
|
||||
}
|
||||
t.Logf("stack: %s", stack.String())
|
||||
}
|
||||
|
||||
func TestCachingStore_ThunderingHerd(t *testing.T) {
|
||||
storeDelay := 100 * time.Millisecond
|
||||
origin := NewSlowBlobStore(storeDelay)
|
||||
cache := NewMemStore()
|
||||
s := NewCachingStore("test", origin, cache)
|
||||
|
||||
b := []byte("this is a blob of stuff")
|
||||
hash := "hash"
|
||||
err := origin.Put(hash, b)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
wg := &sync.WaitGroup{}
|
||||
|
||||
getNoErr := func() {
|
||||
res, _, err := s.Get(hash)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(b, res) {
|
||||
t.Errorf("expected Get() to return %s, got %s", string(b), string(res))
|
||||
}
|
||||
wg.Done()
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
|
||||
wg.Add(4)
|
||||
go func() {
|
||||
go getNoErr()
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
go getNoErr()
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
go getNoErr()
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
go getNoErr()
|
||||
}()
|
||||
|
||||
wg.Wait()
|
||||
duration := time.Since(start)
|
||||
|
||||
// only the first getNoErr() should hit the origin. the rest should wait for the first request to return
|
||||
// once the first returns, the others should return immediately
|
||||
// therefore, if the delay much longer than 100ms, it means subsequent requests also went to the origin
|
||||
expectedMaxDelay := storeDelay + 5*time.Millisecond // a bit of extra time to let requests finish
|
||||
if duration > expectedMaxDelay {
|
||||
t.Errorf("Expected delay of at most %s, got %s", expectedMaxDelay, duration)
|
||||
}
|
||||
}
|
||||
|
||||
// SlowBlobStore adds a delay to each request
|
||||
type SlowBlobStore struct {
|
||||
mem *MemStore
|
||||
delay time.Duration
|
||||
}
|
||||
|
||||
func NewSlowBlobStore(delay time.Duration) *SlowBlobStore {
|
||||
return &SlowBlobStore{
|
||||
mem: NewMemStore(),
|
||||
delay: delay,
|
||||
}
|
||||
}
|
||||
func (s *SlowBlobStore) Name() string {
|
||||
return "slow"
|
||||
}
|
||||
|
||||
func (s *SlowBlobStore) Has(hash string) (bool, error) {
|
||||
time.Sleep(s.delay)
|
||||
return s.mem.Has(hash)
|
||||
}
|
||||
|
||||
func (s *SlowBlobStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
time.Sleep(s.delay)
|
||||
return s.mem.Get(hash)
|
||||
}
|
||||
|
||||
func (s *SlowBlobStore) Put(hash string, blob stream.Blob) error {
|
||||
time.Sleep(s.delay)
|
||||
return s.mem.Put(hash, blob)
|
||||
}
|
||||
|
||||
func (s *SlowBlobStore) PutSD(hash string, blob stream.Blob) error {
|
||||
time.Sleep(s.delay)
|
||||
return s.mem.PutSD(hash, blob)
|
||||
}
|
||||
|
||||
func (s *SlowBlobStore) Delete(hash string) error {
|
||||
time.Sleep(s.delay)
|
||||
return s.mem.Delete(hash)
|
||||
}
|
||||
|
||||
func (s *SlowBlobStore) Shutdown() {
|
||||
return
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
"github.com/lbryio/reflector.go/meta"
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// CloudFrontROStore reads from cloudfront. All writes panic.
|
||||
type CloudFrontROStore struct {
|
||||
endpoint string // cloudflare endpoint
|
||||
}
|
||||
|
||||
// NewCloudFrontROStore returns an initialized CloudFrontROStore store pointer.
|
||||
func NewCloudFrontROStore(endpoint string) *CloudFrontROStore {
|
||||
return &CloudFrontROStore{endpoint: endpoint}
|
||||
}
|
||||
|
||||
const nameCloudFrontRO = "cloudfront_ro"
|
||||
|
||||
// Name is the cache type name
|
||||
func (c *CloudFrontROStore) Name() string { return nameCloudFrontRO }
|
||||
|
||||
// Has checks if the hash is in the store.
|
||||
func (c *CloudFrontROStore) Has(hash string) (bool, error) {
|
||||
status, body, err := c.cfRequest(http.MethodHead, hash)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer body.Close()
|
||||
|
||||
switch status {
|
||||
case http.StatusNotFound, http.StatusForbidden:
|
||||
return false, nil
|
||||
case http.StatusOK:
|
||||
return true, nil
|
||||
default:
|
||||
return false, errors.Err("unexpected status %d", status)
|
||||
}
|
||||
}
|
||||
|
||||
// Get gets the blob from Cloudfront.
|
||||
func (c *CloudFrontROStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
log.Debugf("Getting %s from S3", hash[:8])
|
||||
start := time.Now()
|
||||
defer func(t time.Time) {
|
||||
log.Debugf("Getting %s from S3 took %s", hash[:8], time.Since(t).String())
|
||||
}(start)
|
||||
|
||||
status, body, err := c.cfRequest(http.MethodGet, hash)
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), c.Name()), err
|
||||
}
|
||||
defer body.Close()
|
||||
switch status {
|
||||
case http.StatusNotFound, http.StatusForbidden:
|
||||
return nil, shared.NewBlobTrace(time.Since(start), c.Name()), errors.Err(ErrBlobNotFound)
|
||||
case http.StatusOK:
|
||||
b, err := ioutil.ReadAll(body)
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), c.Name()), errors.Err(err)
|
||||
}
|
||||
metrics.MtrInBytesS3.Add(float64(len(b)))
|
||||
return b, shared.NewBlobTrace(time.Since(start), c.Name()), nil
|
||||
default:
|
||||
return nil, shared.NewBlobTrace(time.Since(start), c.Name()), errors.Err("unexpected status %d", status)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *CloudFrontROStore) cfRequest(method, hash string) (int, io.ReadCloser, error) {
|
||||
url := c.endpoint + hash
|
||||
req, err := http.NewRequest(method, url, nil)
|
||||
if err != nil {
|
||||
return 0, nil, errors.Err(err)
|
||||
}
|
||||
req.Header.Add("User-Agent", "reflector.go/"+meta.Version)
|
||||
|
||||
res, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return 0, nil, errors.Err(err)
|
||||
}
|
||||
|
||||
return res.StatusCode, res.Body, nil
|
||||
}
|
||||
|
||||
func (c *CloudFrontROStore) Put(_ string, _ stream.Blob) error {
|
||||
return errors.Err(shared.ErrNotImplemented)
|
||||
}
|
||||
|
||||
func (c *CloudFrontROStore) PutSD(_ string, _ stream.Blob) error {
|
||||
return errors.Err(shared.ErrNotImplemented)
|
||||
}
|
||||
|
||||
func (c *CloudFrontROStore) Delete(_ string) error {
|
||||
return errors.Err(shared.ErrNotImplemented)
|
||||
}
|
||||
|
||||
// Shutdown shuts down the store gracefully
|
||||
func (c *CloudFrontROStore) Shutdown() {
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
)
|
||||
|
||||
// CloudFrontRWStore combines a Cloudfront and an S3 store. Reads go to Cloudfront/Wasabi, writes go to S3.
|
||||
type CloudFrontRWStore struct {
|
||||
cf *ITTTStore
|
||||
s3 *S3Store
|
||||
}
|
||||
|
||||
// NewCloudFrontRWStore returns an initialized CloudFrontRWStore store pointer.
|
||||
// NOTE: It panics if either argument is nil.
|
||||
func NewCloudFrontRWStore(cf *ITTTStore, s3 *S3Store) *CloudFrontRWStore {
|
||||
if cf == nil || s3 == nil {
|
||||
panic("both stores must be set")
|
||||
}
|
||||
return &CloudFrontRWStore{cf: cf, s3: s3}
|
||||
}
|
||||
|
||||
const nameCloudFrontRW = "cloudfront_rw"
|
||||
|
||||
// Name is the cache type name
|
||||
func (c *CloudFrontRWStore) Name() string { return nameCloudFrontRW }
|
||||
|
||||
// Has checks if the hash is in the store.
|
||||
func (c *CloudFrontRWStore) Has(hash string) (bool, error) {
|
||||
return c.cf.Has(hash)
|
||||
}
|
||||
|
||||
// Get gets the blob from Cloudfront.
|
||||
func (c *CloudFrontRWStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
start := time.Now()
|
||||
blob, trace, err := c.cf.Get(hash)
|
||||
return blob, trace.Stack(time.Since(start), c.Name()), err
|
||||
}
|
||||
|
||||
// Put stores the blob on S3
|
||||
func (c *CloudFrontRWStore) Put(hash string, blob stream.Blob) error {
|
||||
return c.s3.Put(hash, blob)
|
||||
}
|
||||
|
||||
// PutSD stores the sd blob on S3
|
||||
func (c *CloudFrontRWStore) PutSD(hash string, blob stream.Blob) error {
|
||||
return c.s3.PutSD(hash, blob)
|
||||
}
|
||||
|
||||
// Delete deletes the blob from S3
|
||||
func (c *CloudFrontRWStore) Delete(hash string) error {
|
||||
return c.s3.Delete(hash)
|
||||
}
|
||||
|
||||
// Shutdown shuts down the store gracefully
|
||||
func (c *CloudFrontRWStore) Shutdown() {
|
||||
c.s3.Shutdown()
|
||||
c.cf.Shutdown()
|
||||
}
|
||||
+155
-20
@@ -2,35 +2,67 @@ package store
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/reflector.go/db"
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// DBBackedS3Store is an instance of an S3 Store that is backed by a DB for what is stored.
|
||||
type DBBackedS3Store struct {
|
||||
s3 *S3BlobStore
|
||||
db *db.SQL
|
||||
// DBBackedStore is a store that's backed by a DB. The DB contains data about what's in the store.
|
||||
type DBBackedStore struct {
|
||||
blobs BlobStore
|
||||
db *db.SQL
|
||||
blockedMu sync.RWMutex
|
||||
blocked map[string]bool
|
||||
deleteOnMiss bool
|
||||
}
|
||||
|
||||
// NewDBBackedS3Store returns an initialized store pointer.
|
||||
func NewDBBackedS3Store(s3 *S3BlobStore, db *db.SQL) *DBBackedS3Store {
|
||||
return &DBBackedS3Store{s3: s3, db: db}
|
||||
// NewDBBackedStore returns an initialized store pointer.
|
||||
func NewDBBackedStore(blobs BlobStore, db *db.SQL, deleteOnMiss bool) *DBBackedStore {
|
||||
return &DBBackedStore{blobs: blobs, db: db, deleteOnMiss: deleteOnMiss}
|
||||
}
|
||||
|
||||
// Has returns T/F or Error ( if the DB errors ) if store contains the blob.
|
||||
func (d *DBBackedS3Store) Has(hash string) (bool, error) {
|
||||
return d.db.HasBlob(hash)
|
||||
const nameDBBacked = "db-backed"
|
||||
|
||||
// Name is the cache type name
|
||||
func (d *DBBackedStore) Name() string { return nameDBBacked }
|
||||
|
||||
// Has returns true if the blob is in the store
|
||||
func (d *DBBackedStore) Has(hash string) (bool, error) {
|
||||
return d.db.HasBlob(hash, false)
|
||||
}
|
||||
|
||||
// Get returns the byte slice of the blob or an error.
|
||||
func (d *DBBackedS3Store) Get(hash string) ([]byte, error) {
|
||||
return d.s3.Get(hash)
|
||||
// Get gets the blob
|
||||
func (d *DBBackedStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
start := time.Now()
|
||||
has, err := d.db.HasBlob(hash, true)
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), err
|
||||
}
|
||||
if !has {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), ErrBlobNotFound
|
||||
}
|
||||
|
||||
b, stack, err := d.blobs.Get(hash)
|
||||
if d.deleteOnMiss && errors.Is(err, ErrBlobNotFound) {
|
||||
e2 := d.Delete(hash)
|
||||
if e2 != nil {
|
||||
log.Errorf("error while deleting blob from db: %s", errors.FullTrace(err))
|
||||
}
|
||||
}
|
||||
|
||||
return b, stack.Stack(time.Since(start), d.Name()), err
|
||||
}
|
||||
|
||||
// Put stores the blob in the S3 store and stores the blob information in the DB.
|
||||
func (d *DBBackedS3Store) Put(hash string, blob []byte) error {
|
||||
err := d.s3.Put(hash, blob)
|
||||
func (d *DBBackedStore) Put(hash string, blob stream.Blob) error {
|
||||
err := d.blobs.Put(hash, blob)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -40,17 +72,17 @@ func (d *DBBackedS3Store) Put(hash string, blob []byte) error {
|
||||
|
||||
// PutSD stores the SDBlob in the S3 store. It will return an error if the sd blob is missing the stream hash or if
|
||||
// there is an error storing the blob information in the DB.
|
||||
func (d *DBBackedS3Store) PutSD(hash string, blob []byte) error {
|
||||
func (d *DBBackedStore) PutSD(hash string, blob stream.Blob) error {
|
||||
var blobContents db.SdBlob
|
||||
err := json.Unmarshal(blob, &blobContents)
|
||||
if err != nil {
|
||||
return err
|
||||
return errors.Err(err)
|
||||
}
|
||||
if blobContents.StreamHash == "" {
|
||||
return errors.Err("sd blob is missing stream hash")
|
||||
}
|
||||
|
||||
err = d.s3.PutSD(hash, blob)
|
||||
err = d.blobs.PutSD(hash, blob)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -58,9 +90,112 @@ func (d *DBBackedS3Store) PutSD(hash string, blob []byte) error {
|
||||
return d.db.AddSDBlob(hash, len(blob), blobContents)
|
||||
}
|
||||
|
||||
func (d *DBBackedStore) Delete(hash string) error {
|
||||
err := d.blobs.Delete(hash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return d.db.Delete(hash)
|
||||
}
|
||||
|
||||
// Block deletes the blob and prevents it from being uploaded in the future
|
||||
func (d *DBBackedStore) Block(hash string) error {
|
||||
if blocked, err := d.isBlocked(hash); blocked || err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Debugf("blocking %s", hash)
|
||||
|
||||
err := d.db.Block(hash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
has, err := d.db.HasBlob(hash, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if has {
|
||||
err = d.blobs.Delete(hash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = d.db.Delete(hash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return d.markBlocked(hash)
|
||||
}
|
||||
|
||||
// Wants returns false if the hash exists or is blocked, true otherwise
|
||||
func (d *DBBackedStore) Wants(hash string) (bool, error) {
|
||||
blocked, err := d.isBlocked(hash)
|
||||
if blocked || err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
has, err := d.Has(hash)
|
||||
return !has, err
|
||||
}
|
||||
|
||||
// MissingBlobsForKnownStream returns missing blobs for an existing stream
|
||||
// WARNING: if the stream does NOT exist, no blob hashes will be returned, which looks
|
||||
// like no blobs are missing
|
||||
func (d *DBBackedS3Store) MissingBlobsForKnownStream(sdHash string) ([]string, error) {
|
||||
func (d *DBBackedStore) MissingBlobsForKnownStream(sdHash string) ([]string, error) {
|
||||
return d.db.MissingBlobsForKnownStream(sdHash)
|
||||
}
|
||||
|
||||
func (d *DBBackedStore) markBlocked(hash string) error {
|
||||
err := d.initBlocked()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
d.blockedMu.Lock()
|
||||
defer d.blockedMu.Unlock()
|
||||
|
||||
d.blocked[hash] = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *DBBackedStore) isBlocked(hash string) (bool, error) {
|
||||
err := d.initBlocked()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
d.blockedMu.RLock()
|
||||
defer d.blockedMu.RUnlock()
|
||||
|
||||
return d.blocked[hash], nil
|
||||
}
|
||||
|
||||
func (d *DBBackedStore) initBlocked() error {
|
||||
// first check without blocking since this is the most likely scenario
|
||||
if d.blocked != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
d.blockedMu.Lock()
|
||||
defer d.blockedMu.Unlock()
|
||||
|
||||
// check again in case of race condition
|
||||
if d.blocked != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var err error
|
||||
d.blocked, err = d.db.GetBlocked()
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// Shutdown shuts down the store gracefully
|
||||
func (d *DBBackedStore) Shutdown() {
|
||||
d.blobs.Shutdown()
|
||||
}
|
||||
+234
@@ -0,0 +1,234 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha512"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"os"
|
||||
"path"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
"github.com/lbryio/reflector.go/store/speedwalk"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
"github.com/brk0v/directio"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"go.uber.org/atomic"
|
||||
)
|
||||
|
||||
// DiskStore stores blobs on a local disk
|
||||
type DiskStore struct {
|
||||
// the location of blobs on disk
|
||||
blobDir string
|
||||
// store files in subdirectories based on the first N chars in the filename. 0 = don't create subdirectories.
|
||||
prefixLength int
|
||||
|
||||
// true if initOnce ran, false otherwise
|
||||
initialized bool
|
||||
|
||||
concurrentChecks atomic.Int32
|
||||
}
|
||||
|
||||
const maxConcurrentChecks = 30
|
||||
|
||||
// NewDiskStore returns an initialized file disk store pointer.
|
||||
func NewDiskStore(dir string, prefixLength int) *DiskStore {
|
||||
return &DiskStore{
|
||||
blobDir: dir,
|
||||
prefixLength: prefixLength,
|
||||
}
|
||||
}
|
||||
|
||||
const nameDisk = "disk"
|
||||
|
||||
// Name is the cache type name
|
||||
func (d *DiskStore) Name() string { return nameDisk }
|
||||
|
||||
// Has returns T/F or Error if it the blob stored already. It will error with any IO disk error.
|
||||
func (d *DiskStore) Has(hash string) (bool, error) {
|
||||
err := d.initOnce()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
_, err = os.Stat(d.path(hash))
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return false, nil
|
||||
}
|
||||
return false, errors.Err(err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (d *DiskStore) Size(hash string) (int64, error) {
|
||||
stat, err := os.Stat(d.path(hash))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return stat.Size(), nil
|
||||
}
|
||||
|
||||
// Get returns the blob or an error if the blob doesn't exist.
|
||||
func (d *DiskStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
start := time.Now()
|
||||
err := d.initOnce()
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), err
|
||||
}
|
||||
|
||||
blob, err := ioutil.ReadFile(d.path(hash))
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(ErrBlobNotFound)
|
||||
}
|
||||
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(err)
|
||||
}
|
||||
|
||||
// this is a rather poor yet effective way of throttling how many blobs can be checked concurrently
|
||||
// poor because there is a possible race condition between the check and the actual +1
|
||||
if d.concurrentChecks.Load() < maxConcurrentChecks {
|
||||
d.concurrentChecks.Add(1)
|
||||
defer d.concurrentChecks.Sub(1)
|
||||
hashBytes := sha512.Sum384(blob)
|
||||
readHash := hex.EncodeToString(hashBytes[:])
|
||||
if hash != readHash {
|
||||
message := fmt.Sprintf("[%s] found a broken blob while reading from disk. Actual hash: %s", hash, readHash)
|
||||
log.Errorf("%s", message)
|
||||
err := d.Delete(hash)
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), err
|
||||
}
|
||||
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(message)
|
||||
}
|
||||
}
|
||||
|
||||
return blob, shared.NewBlobTrace(time.Since(start), d.Name()), nil
|
||||
}
|
||||
|
||||
// Put stores the blob on disk
|
||||
func (d *DiskStore) Put(hash string, blob stream.Blob) error {
|
||||
err := d.initOnce()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = d.ensureDirExists(d.dir(hash))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Open file with O_DIRECT
|
||||
f, err := os.OpenFile(d.tmpPath(hash), openFileFlags, 0644)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
// Use directio writer
|
||||
dio, err := directio.New(f)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer dio.Flush()
|
||||
// Write the body to file
|
||||
_, err = io.Copy(dio, bytes.NewReader(blob))
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
err = os.Rename(d.tmpPath(hash), d.path(hash))
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
// PutSD stores the sd blob on the disk
|
||||
func (d *DiskStore) PutSD(hash string, blob stream.Blob) error {
|
||||
return d.Put(hash, blob)
|
||||
}
|
||||
|
||||
// Delete deletes the blob from the store
|
||||
func (d *DiskStore) Delete(hash string) error {
|
||||
err := d.initOnce()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
has, err := d.Has(hash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !has {
|
||||
return nil
|
||||
}
|
||||
|
||||
err = os.Remove(d.path(hash))
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
// list returns the hashes of blobs that already exist in the blobDir
|
||||
func (d *DiskStore) list() ([]string, error) {
|
||||
err := d.initOnce()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return speedwalk.AllFiles(d.blobDir, true)
|
||||
}
|
||||
|
||||
func (d *DiskStore) dir(hash string) string {
|
||||
if d.prefixLength <= 0 || len(hash) < d.prefixLength {
|
||||
return d.blobDir
|
||||
}
|
||||
return path.Join(d.blobDir, hash[:d.prefixLength])
|
||||
}
|
||||
func (d *DiskStore) tmpDir(hash string) string {
|
||||
return path.Join(d.blobDir, "tmp")
|
||||
}
|
||||
func (d *DiskStore) path(hash string) string {
|
||||
return path.Join(d.dir(hash), hash)
|
||||
}
|
||||
func (d *DiskStore) tmpPath(hash string) string {
|
||||
return path.Join(d.tmpDir(hash), hash)
|
||||
}
|
||||
func (d *DiskStore) ensureDirExists(dir string) error {
|
||||
return errors.Err(os.MkdirAll(dir, 0755))
|
||||
}
|
||||
|
||||
func (d *DiskStore) initOnce() error {
|
||||
if d.initialized {
|
||||
return nil
|
||||
}
|
||||
|
||||
err := d.ensureDirExists(d.blobDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = d.ensureDirExists(path.Join(d.blobDir, "tmp"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
d.initialized = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// Shutdown shuts down the store gracefully
|
||||
func (d *DiskStore) Shutdown() {
|
||||
}
|
||||
|
||||
func (d *DiskStore) Sendfile(hash string, conn net.Conn) error {
|
||||
f, err := os.Open(d.path(hash))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = io.Copy(conn, f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestDiskStore_Get(t *testing.T) {
|
||||
tmpDir, err := ioutil.TempDir("", "reflector_test_*")
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(tmpDir)
|
||||
d := NewDiskStore(tmpDir, 2)
|
||||
|
||||
hash := "f428b8265d65dad7f8ffa52922bba836404cbd62f3ecfe10adba6b444f8f658938e54f5981ac4de39644d5b93d89a94b"
|
||||
data := []byte("oyuntyausntoyaunpdoyruoyduanrstjwfjyuwf")
|
||||
|
||||
expectedPath := path.Join(tmpDir, hash[:2], hash)
|
||||
err = os.MkdirAll(filepath.Dir(expectedPath), os.ModePerm)
|
||||
require.NoError(t, err)
|
||||
err = ioutil.WriteFile(expectedPath, data, os.ModePerm)
|
||||
require.NoError(t, err)
|
||||
|
||||
blob, _, err := d.Get(hash)
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, data, blob)
|
||||
}
|
||||
|
||||
func TestDiskStore_GetNonexistentBlob(t *testing.T) {
|
||||
tmpDir, err := ioutil.TempDir("", "reflector_test_*")
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(tmpDir)
|
||||
d := NewDiskStore(tmpDir, 2)
|
||||
|
||||
blob, _, err := d.Get("nonexistent")
|
||||
assert.Nil(t, blob)
|
||||
assert.True(t, errors.Is(err, ErrBlobNotFound))
|
||||
}
|
||||
@@ -1,97 +0,0 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
)
|
||||
|
||||
// FileBlobStore is a local disk store.
|
||||
type FileBlobStore struct {
|
||||
dir string
|
||||
|
||||
initialized bool
|
||||
}
|
||||
|
||||
// NewFileBlobStore returns an initialized file disk store pointer.
|
||||
func NewFileBlobStore(dir string) *FileBlobStore {
|
||||
return &FileBlobStore{dir: dir}
|
||||
}
|
||||
|
||||
func (f *FileBlobStore) path(hash string) string {
|
||||
return path.Join(f.dir, hash)
|
||||
}
|
||||
|
||||
func (f *FileBlobStore) initOnce() error {
|
||||
if f.initialized {
|
||||
return nil
|
||||
}
|
||||
|
||||
if stat, err := os.Stat(f.dir); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
err2 := os.Mkdir(f.dir, 0755)
|
||||
if err2 != nil {
|
||||
return err2
|
||||
}
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
} else if !stat.IsDir() {
|
||||
return errors.Err("blob dir exists but is not a dir")
|
||||
}
|
||||
|
||||
f.initialized = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// Has returns T/F or Error if it the blob stored already. It will error with any IO disk error.
|
||||
func (f *FileBlobStore) Has(hash string) (bool, error) {
|
||||
err := f.initOnce()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
_, err = os.Stat(f.path(hash))
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// Get returns the byte slice of the blob stored or will error if the blob doesn't exist.
|
||||
func (f *FileBlobStore) Get(hash string) ([]byte, error) {
|
||||
err := f.initOnce()
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
|
||||
file, err := os.Open(f.path(hash))
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return []byte{}, errors.Err(ErrBlobNotFound)
|
||||
}
|
||||
return []byte{}, err
|
||||
}
|
||||
|
||||
return ioutil.ReadAll(file)
|
||||
}
|
||||
|
||||
// Put stores the blob on disk
|
||||
func (f *FileBlobStore) Put(hash string, blob []byte) error {
|
||||
err := f.initOnce()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return ioutil.WriteFile(f.path(hash), blob, 0644)
|
||||
}
|
||||
|
||||
// PutSD stores the sd blob on the disk
|
||||
func (f *FileBlobStore) PutSD(hash string, blob []byte) error {
|
||||
return f.Put(hash, blob)
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
// +build darwin
|
||||
|
||||
package store
|
||||
|
||||
import (
|
||||
"os"
|
||||
)
|
||||
|
||||
var openFileFlags = os.O_WRONLY | os.O_CREATE
|
||||
@@ -0,0 +1,10 @@
|
||||
// +build linux
|
||||
|
||||
package store
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
var openFileFlags = os.O_WRONLY | os.O_CREATE | syscall.O_DIRECT
|
||||
+159
@@ -0,0 +1,159 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
"github.com/bluele/gcache"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// GcacheStore adds a max cache size and Greedy-Dual-Size-Frequency cache eviction strategy to a BlobStore
|
||||
type GcacheStore struct {
|
||||
// underlying store
|
||||
store BlobStore
|
||||
// cache implementation
|
||||
cache gcache.Cache
|
||||
}
|
||||
type EvictionStrategy int
|
||||
|
||||
const (
|
||||
//LFU Discards the least frequently used items first.
|
||||
LFU EvictionStrategy = iota
|
||||
//ARC Constantly balances between LRU and LFU, to improve the combined result.
|
||||
ARC
|
||||
//LRU Discards the least recently used items first.
|
||||
LRU
|
||||
//SIMPLE has no clear priority for evict cache. It depends on key-value map order.
|
||||
SIMPLE
|
||||
)
|
||||
|
||||
// NewGcacheStore initialize a new LRUStore
|
||||
func NewGcacheStore(component string, store BlobStore, maxSize int, strategy EvictionStrategy) *GcacheStore {
|
||||
cacheBuilder := gcache.New(maxSize)
|
||||
var cache gcache.Cache
|
||||
evictFunc := func(key interface{}, value interface{}) {
|
||||
logrus.Infof("evicting %s", key)
|
||||
metrics.CacheLRUEvictCount.With(metrics.CacheLabels(store.Name(), component)).Inc()
|
||||
_ = store.Delete(key.(string)) // TODO: log this error. may happen if underlying entry is gone but cache entry still there
|
||||
}
|
||||
switch strategy {
|
||||
case LFU:
|
||||
cache = cacheBuilder.LFU().EvictedFunc(evictFunc).Build()
|
||||
case ARC:
|
||||
cache = cacheBuilder.ARC().EvictedFunc(evictFunc).Build()
|
||||
case LRU:
|
||||
cache = cacheBuilder.LRU().EvictedFunc(evictFunc).Build()
|
||||
case SIMPLE:
|
||||
cache = cacheBuilder.Simple().EvictedFunc(evictFunc).Build()
|
||||
|
||||
}
|
||||
l := &GcacheStore{
|
||||
store: store,
|
||||
cache: cache,
|
||||
}
|
||||
go func() {
|
||||
if lstr, ok := store.(lister); ok {
|
||||
err := l.loadExisting(lstr, int(maxSize))
|
||||
if err != nil {
|
||||
panic(err) // TODO: what should happen here? panic? return nil? just keep going?
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return l
|
||||
}
|
||||
|
||||
const nameGcache = "gcache"
|
||||
|
||||
// Name is the cache type name
|
||||
func (l *GcacheStore) Name() string { return nameGcache }
|
||||
|
||||
// Has returns whether the blob is in the store, without updating the recent-ness.
|
||||
func (l *GcacheStore) Has(hash string) (bool, error) {
|
||||
return l.cache.Has(hash), nil
|
||||
}
|
||||
|
||||
// Get returns the blob or an error if the blob doesn't exist.
|
||||
func (l *GcacheStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
start := time.Now()
|
||||
_, err := l.cache.Get(hash)
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), l.Name()), errors.Err(ErrBlobNotFound)
|
||||
}
|
||||
blob, stack, err := l.store.Get(hash)
|
||||
if errors.Is(err, ErrBlobNotFound) {
|
||||
// Blob disappeared from underlying store
|
||||
l.cache.Remove(hash)
|
||||
}
|
||||
return blob, stack.Stack(time.Since(start), l.Name()), err
|
||||
}
|
||||
|
||||
// Put stores the blob. Following LFUDA rules it's not guaranteed that a SET will store the value!!!
|
||||
func (l *GcacheStore) Put(hash string, blob stream.Blob) error {
|
||||
_ = l.cache.Set(hash, true)
|
||||
has, _ := l.Has(hash)
|
||||
if has {
|
||||
err := l.store.Put(hash, blob)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// PutSD stores the sd blob. Following LFUDA rules it's not guaranteed that a SET will store the value!!!
|
||||
func (l *GcacheStore) PutSD(hash string, blob stream.Blob) error {
|
||||
_ = l.cache.Set(hash, true)
|
||||
has, _ := l.Has(hash)
|
||||
if has {
|
||||
err := l.store.PutSD(hash, blob)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete deletes the blob from the store
|
||||
func (l *GcacheStore) Delete(hash string) error {
|
||||
err := l.store.Delete(hash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// This must come after store.Delete()
|
||||
// Remove triggers onEvict function, which also tries to delete blob from store
|
||||
// We need to delete it manually first so any errors can be propagated up
|
||||
l.cache.Remove(hash)
|
||||
return nil
|
||||
}
|
||||
|
||||
// loadExisting imports existing blobs from the underlying store into the LRU cache
|
||||
func (l *GcacheStore) loadExisting(store lister, maxItems int) error {
|
||||
logrus.Infof("loading at most %d items", maxItems)
|
||||
existing, err := store.list()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
logrus.Infof("read %d files from underlying store", len(existing))
|
||||
|
||||
added := 0
|
||||
for _, h := range existing {
|
||||
_ = l.cache.Set(h, true)
|
||||
added++
|
||||
if maxItems > 0 && added >= maxItems { // underlying cache is bigger than the cache
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Shutdown shuts down the store gracefully
|
||||
func (l *GcacheStore) Shutdown() {
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const cacheMaxSize = 3
|
||||
|
||||
func getTestGcacheStore() (*GcacheStore, *MemStore) {
|
||||
m := NewMemStore()
|
||||
return NewGcacheStore("test", m, cacheMaxSize, LFU), m
|
||||
}
|
||||
|
||||
func TestGcacheStore_Eviction(t *testing.T) {
|
||||
lfu, mem := getTestGcacheStore()
|
||||
b := []byte("x")
|
||||
for i := 0; i < 3; i++ {
|
||||
err := lfu.Put(fmt.Sprintf("%d", i), b)
|
||||
require.NoError(t, err)
|
||||
for j := 0; j < 3-i; j++ {
|
||||
_, _, err = lfu.Get(fmt.Sprintf("%d", i))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
}
|
||||
for k, v := range map[string]bool{
|
||||
"0": true,
|
||||
"1": true,
|
||||
"2": true,
|
||||
} {
|
||||
has, err := lfu.Has(k)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, v, has)
|
||||
}
|
||||
err := lfu.Put("3", b)
|
||||
require.NoError(t, err)
|
||||
for k, v := range map[string]bool{
|
||||
"0": true,
|
||||
"1": true,
|
||||
"2": false,
|
||||
"3": true,
|
||||
} {
|
||||
has, err := lfu.Has(k)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, v, has)
|
||||
}
|
||||
assert.Equal(t, cacheMaxSize, len(mem.Debug()))
|
||||
|
||||
err = lfu.Delete("0")
|
||||
assert.NoError(t, err)
|
||||
err = lfu.Delete("1")
|
||||
assert.NoError(t, err)
|
||||
err = lfu.Delete("3")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 0, len(mem.Debug()))
|
||||
}
|
||||
|
||||
func TestGcacheStore_UnderlyingBlobMissing(t *testing.T) {
|
||||
lfu, mem := getTestGcacheStore()
|
||||
hash := "hash"
|
||||
b := []byte("this is a blob of stuff")
|
||||
err := lfu.Put(hash, b)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = mem.Delete(hash)
|
||||
require.NoError(t, err)
|
||||
|
||||
// hash still exists in lru
|
||||
assert.True(t, lfu.cache.Has(hash))
|
||||
|
||||
blob, _, err := lfu.Get(hash)
|
||||
assert.Nil(t, blob)
|
||||
assert.True(t, errors.Is(err, ErrBlobNotFound), "expected (%s) %s, got (%s) %s",
|
||||
reflect.TypeOf(ErrBlobNotFound).String(), ErrBlobNotFound.Error(),
|
||||
reflect.TypeOf(err).String(), err.Error())
|
||||
|
||||
// lru.Get() removes hash if underlying store doesn't have it
|
||||
assert.False(t, lfu.cache.Has(hash))
|
||||
}
|
||||
|
||||
func TestGcacheStore_loadExisting(t *testing.T) {
|
||||
tmpDir, err := ioutil.TempDir("", "reflector_test_*")
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(tmpDir)
|
||||
d := NewDiskStore(tmpDir, 2)
|
||||
|
||||
hash := "hash"
|
||||
b := []byte("this is a blob of stuff")
|
||||
err = d.Put(hash, b)
|
||||
require.NoError(t, err)
|
||||
|
||||
existing, err := d.list()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, len(existing), "blob should exist in cache")
|
||||
assert.Equal(t, hash, existing[0])
|
||||
|
||||
lfu := NewGcacheStore("test", d, 3, LFU) // lru should load existing blobs when it's created
|
||||
time.Sleep(100 * time.Millisecond) // async load so let's wait...
|
||||
has, err := lfu.Has(hash)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, has, "hash should be loaded from disk store but it's not")
|
||||
}
|
||||
+160
@@ -0,0 +1,160 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
)
|
||||
|
||||
// HttpStore is a store that works on top of the HTTP protocol
|
||||
type HttpStore struct {
|
||||
upstream string
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
func NewHttpStore(upstream string) *HttpStore {
|
||||
return &HttpStore{
|
||||
upstream: "http://" + upstream,
|
||||
httpClient: getClient(),
|
||||
}
|
||||
}
|
||||
|
||||
const nameHttp = "http"
|
||||
|
||||
func (n *HttpStore) Name() string { return nameHttp }
|
||||
func (n *HttpStore) Has(hash string) (bool, error) {
|
||||
url := n.upstream + "/blob?hash=" + hash
|
||||
|
||||
req, err := http.NewRequest("HEAD", url, nil)
|
||||
if err != nil {
|
||||
return false, errors.Err(err)
|
||||
}
|
||||
|
||||
res, err := n.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return false, errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode == http.StatusNotFound {
|
||||
return false, nil
|
||||
}
|
||||
if res.StatusCode == http.StatusNoContent {
|
||||
return true, nil
|
||||
}
|
||||
var body []byte
|
||||
if res.Body != nil {
|
||||
body, _ = ioutil.ReadAll(res.Body)
|
||||
}
|
||||
return false, errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
|
||||
}
|
||||
|
||||
func (n *HttpStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
start := time.Now()
|
||||
url := n.upstream + "/blob?hash=" + hash
|
||||
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), n.Name()), errors.Err(err)
|
||||
}
|
||||
|
||||
res, err := n.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), n.Name()), errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
tmp := getBuffer()
|
||||
defer putBuffer(tmp)
|
||||
serialized := res.Header.Get("Via")
|
||||
trace := shared.NewBlobTrace(time.Since(start), n.Name())
|
||||
if serialized != "" {
|
||||
parsedTrace, err := shared.Deserialize(serialized)
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), n.Name()), err
|
||||
}
|
||||
trace = *parsedTrace
|
||||
}
|
||||
|
||||
if res.StatusCode == http.StatusNotFound {
|
||||
return nil, trace.Stack(time.Since(start), n.Name()), ErrBlobNotFound
|
||||
}
|
||||
if res.StatusCode == http.StatusOK {
|
||||
written, err := io.Copy(tmp, res.Body)
|
||||
if err != nil {
|
||||
return nil, trace.Stack(time.Since(start), n.Name()), errors.Err(err)
|
||||
}
|
||||
|
||||
blob := make([]byte, written)
|
||||
copy(blob, tmp.Bytes())
|
||||
metrics.MtrInBytesHttp.Add(float64(len(blob)))
|
||||
return blob, trace.Stack(time.Since(start), n.Name()), nil
|
||||
}
|
||||
var body []byte
|
||||
if res.Body != nil {
|
||||
body, _ = ioutil.ReadAll(res.Body)
|
||||
}
|
||||
|
||||
return nil, trace.Stack(time.Since(start), n.Name()), errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
|
||||
}
|
||||
|
||||
func (n *HttpStore) Put(string, stream.Blob) error {
|
||||
return shared.ErrNotImplemented
|
||||
}
|
||||
func (n *HttpStore) PutSD(string, stream.Blob) error {
|
||||
return shared.ErrNotImplemented
|
||||
}
|
||||
func (n *HttpStore) Delete(string) error {
|
||||
return shared.ErrNotImplemented
|
||||
}
|
||||
func (n *HttpStore) Shutdown() {}
|
||||
|
||||
// buffer pool to reduce GC
|
||||
// https://www.captaincodeman.com/2017/06/02/golang-buffer-pool-gotcha
|
||||
var buffers = sync.Pool{
|
||||
// New is called when a new instance is needed
|
||||
New: func() interface{} {
|
||||
buf := make([]byte, 0, stream.MaxBlobSize)
|
||||
return bytes.NewBuffer(buf)
|
||||
},
|
||||
}
|
||||
|
||||
// getBuffer fetches a buffer from the pool
|
||||
func getBuffer() *bytes.Buffer {
|
||||
return buffers.Get().(*bytes.Buffer)
|
||||
}
|
||||
|
||||
// putBuffer returns a buffer to the pool
|
||||
func putBuffer(buf *bytes.Buffer) {
|
||||
buf.Reset()
|
||||
buffers.Put(buf)
|
||||
}
|
||||
|
||||
// getClient gets an http client that's customized to be more performant when dealing with blobs of 2MB in size (most of our blobs)
|
||||
func getClient() *http.Client {
|
||||
// Customize the Transport to have larger connection pool
|
||||
defaultTransport := &http.Transport{
|
||||
DialContext: (&net.Dialer{
|
||||
Timeout: 30 * time.Second,
|
||||
KeepAlive: 30 * time.Second,
|
||||
}).DialContext,
|
||||
ForceAttemptHTTP2: true,
|
||||
MaxIdleConns: 100,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
TLSHandshakeTimeout: 10 * time.Second,
|
||||
ExpectContinueTimeout: 1 * time.Second,
|
||||
DisableCompression: true,
|
||||
MaxIdleConnsPerHost: 100,
|
||||
ReadBufferSize: stream.MaxBlobSize + 1024*10, //add an extra few KBs to make sure it fits the extra information
|
||||
}
|
||||
|
||||
return &http.Client{Transport: defaultTransport}
|
||||
}
|
||||
Loaded 100 of 112 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user