Compare commits
85
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c4ac60941a | ||
|
|
8b9ec771d0 | ||
|
|
eaf5f7478b | ||
|
|
47425c2c50 | ||
|
|
594a45a271 | ||
|
|
fd39b09e95 | ||
|
|
fce8beea4d | ||
|
|
96e0c2e2b2 | ||
|
|
bb5df9896a | ||
|
|
accd79c81f | ||
|
|
32dae2704f | ||
|
|
2ed1e23228 | ||
|
|
d61f7c892f | ||
|
|
027409fe40 | ||
|
|
38401bf8d6 | ||
|
|
825e699114 | ||
|
|
6509c7dc52 | ||
|
|
c088a65041 | ||
|
|
c7643fb5da | ||
|
|
c3ab5dd2a5 | ||
|
|
2047fe6c05 | ||
|
|
6166ff37cf | ||
|
|
970585c75d | ||
|
|
71549c0dea | ||
|
|
46ae5a502a | ||
|
|
fb2f807e6a | ||
|
|
ad1eea1e8e | ||
|
|
dfb77d3547 | ||
|
|
96427136ef | ||
|
|
c649636eeb | ||
|
|
2e81b1ab03 | ||
|
|
fa2ec38d3e | ||
|
|
2cacdaf22d | ||
|
|
3b269050af | ||
|
|
59ff828794 | ||
|
|
4569d04522 | ||
|
|
7a618f4228 | ||
|
|
0d996d7415 | ||
|
|
2744080c6b | ||
|
|
41b513439f | ||
|
|
31daec7054 | ||
|
|
00feaaf76c | ||
|
|
a98990f573 | ||
|
|
5788f5aa55 | ||
|
|
922a9402aa | ||
|
|
05e0a506b4 | ||
|
|
a8fe33ecf4 | ||
|
|
030ed4914d | ||
|
|
e03810903a | ||
|
|
b0b2d21a08 | ||
|
|
557988d2cb | ||
|
|
6acb99c74d | ||
|
|
d77f5d17f3 | ||
|
|
609ebdb74e | ||
|
|
5baa66f0aa | ||
|
|
75ad143d84 | ||
|
|
14d6d32a41 | ||
|
|
e3ae0ef5c9 | ||
|
|
db791e26ef | ||
|
|
dc104dd17b | ||
|
|
e67e3dbf85 | ||
|
|
e14a1f36cf | ||
|
|
11ebfb822b | ||
|
|
4293e772e2 | ||
|
|
686ac662cc | ||
|
|
747a6ec2b2 | ||
|
|
046868c4e0 | ||
|
|
53d3eea8fb | ||
|
|
61e83d86de | ||
|
|
9fb824790b | ||
|
|
6fd0526376 | ||
|
|
cd236708b0 | ||
|
|
83921b8c2d | ||
|
|
c042fe08c3 | ||
|
|
59e92ccf88 | ||
|
|
cb386de4c7 | ||
|
|
146adcf7fd | ||
|
|
5d844fc3ea | ||
|
|
4ed52e20d7 | ||
|
|
7ab2d355d5 | ||
|
|
064035f1d8 | ||
|
|
e158fa1bb2 | ||
|
|
25ec93a6b9 | ||
|
|
22f42d5559 | ||
|
|
473e7e3b07 |
No files matched your search
+1
-2
@@ -1,4 +1,3 @@
|
||||
/vendor
|
||||
/blobs
|
||||
/config.json*
|
||||
/prism-bin
|
||||
/bin
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"Disable": ["golint"],
|
||||
"Vendor": true
|
||||
}
|
||||
+31
-30
@@ -2,30 +2,21 @@ os: linux
|
||||
dist: trusty
|
||||
language: go
|
||||
|
||||
# Only the last two Go releases are supported by the Go team with security
|
||||
# updates. Any versions older than that should be considered deprecated.
|
||||
# Don't bother testing with them. tip builds your code with the latest
|
||||
# development version of Go. This can warn you that your code will break
|
||||
# in the next version of Go. Don't worry! Later we declare that test runs
|
||||
# are allowed to fail on Go tip.
|
||||
matrix:
|
||||
include:
|
||||
- go: 1.10.2
|
||||
env: DEPLOY=true # dont deploy tip build
|
||||
- go: master
|
||||
env: DEPLOY=false # dont deploy tip build
|
||||
allow_failures:
|
||||
- go: master
|
||||
# Don't wait for tip tests to finish. Mark the test run green if the
|
||||
# tests pass on the stable versions of Go.
|
||||
fast_finish: true
|
||||
env:
|
||||
- GO111MODULE=on
|
||||
|
||||
go:
|
||||
- 1.11.x
|
||||
|
||||
cache:
|
||||
directories:
|
||||
- $HOME/.cache/go-build
|
||||
- $HOME/gopath/pkg/mod
|
||||
|
||||
# Don't email me the results of the test runs.
|
||||
notifications:
|
||||
email: false
|
||||
|
||||
# Skip the install step. Don't `go get` dependencies. Only build with the
|
||||
# code in vendor/
|
||||
# Skip the install step. Don't `go get` dependencies. Only build with the code in vendor/
|
||||
install: true
|
||||
|
||||
# Anything in before_script that returns a nonzero exit code will
|
||||
@@ -35,8 +26,8 @@ before_script:
|
||||
# All the .go files, excluding vendor/ and model (auto generated)
|
||||
- GO_FILES=$(find . -iname '*.go' ! -iname '*_test.go' -type f | grep -v /vendor/ )
|
||||
- go get golang.org/x/tools/cmd/goimports # Used in build script for generated files
|
||||
- go get github.com/golang/lint/golint # Linter
|
||||
- go get honnef.co/go/tools/cmd/megacheck # Badass static analyzer/linter
|
||||
# - go get github.com/golang/lint/golint # Linter
|
||||
# - go get honnef.co/go/tools/cmd/megacheck # Badass static analyzer/linter
|
||||
- go get github.com/jgautheron/gocyclo # Check against high complexity
|
||||
- go get github.com/mdempsky/unconvert # Identifies unnecessary type conversions
|
||||
- go get github.com/kisielk/errcheck # Checks for unhandled errors
|
||||
@@ -48,12 +39,10 @@ before_script:
|
||||
# script always run to completion (set +e). All of these code checks are must haves
|
||||
# in a modern Go project.
|
||||
script:
|
||||
# Build Prism successfully
|
||||
- make
|
||||
# Fail if a .go file hasn't been formatted with gofmt
|
||||
- test -z $(gofmt -s -l $GO_FILES)
|
||||
# Run unit tests
|
||||
- go test ./...
|
||||
- make test
|
||||
# Checks for unused vars and fields on structs
|
||||
- varcheck ./...
|
||||
- structcheck ./...
|
||||
@@ -62,21 +51,33 @@ script:
|
||||
# forbid code with huge functions
|
||||
#- gocyclo -ignore "_test.go" -avg -over 19 $GO_FILES
|
||||
# checks for unhandled errors
|
||||
- errcheck ./...
|
||||
# - errcheck ./... # COMMENTED OUT UNTIL https://github.com/kisielk/errcheck/issues/155 is fixed
|
||||
# "go vet on steroids" + linter - ignore autogen code
|
||||
- megacheck -simple.exit-non-zero=true ./...
|
||||
# - megacheck -simple.exit-non-zero=true ./... # DISABLED UNTIL https://github.com/dominikh/go-tools/issues/328 is fixed
|
||||
# check for unnecessary conversions - ignore autogen code
|
||||
- unconvert ./...
|
||||
# - unconvert ./... # SEEMS TO BE BROKEN WITH GO MODULES
|
||||
# one last linter - ignore autogen code
|
||||
#- golint -set_exit_status $(go list ./... | grep -v /vendor/ )
|
||||
# Finally, build the binary
|
||||
- make
|
||||
|
||||
deploy:
|
||||
- provider: s3
|
||||
local_dir: ./bin
|
||||
skip_cleanup: true
|
||||
on:
|
||||
repo: lbryio/reflector.go
|
||||
access_key_id:
|
||||
secure: "ABmoSTxTee1GubJmmi+MlyBcnqRT0ywEOtTL4hrH7T+Vj5UPcNBhjqGRiMXzO3MdrVWom19OnqMvsvAdehrB4uQJVlX/+zB26gTpEiiU5dWJgerNAYmfxDGNz/p9pyYWLvVY/cymSlJ4HkggJVg+dMTBTOP+gTNgQUhHL7pAd/Z5HMPmPW/rlNlQgDdczoY44OdiniuHny4Lr7mo8dcT4dcyJysPpHLCPGWp3kXcffIEbgO02aQZ14BbX9oU9xm1BacYnr4t8gRCU+uK0mt7TREHUzZ8bQBRXUymmpeeEQshXdifdwtk6sdRGIZiTi0NmQ7kYter6oh6623TH6rpNy/lkxKi823FbSPC1XZGH7CwcbnSIEwY++7e9PT0EA14VuDUF7+iXC/gpRkVQwUoMcoC9rHBjkXW+SiwHWoUeDKXrfVO9QGEJTi5bDRNwklwu2zkYWeRIPzfDaVayUBYwjunjT1eST997ygmJwJTJ6DeOWLx+WFlmwBJfQpBwbrlMrceiNjreMwGB6ffXEBzuM6S5inXGIChllGDpNyJeFVcH9zPpU0JsIpWoRjwLNFOiEhFsnzK5rvA2SKmlOOnSlgOCBQnFi2SmLDM790KoY8qjan2s984chQz2qsKnhYuVpJhG6LKgrI1QwE0KHr80spVFfewq11AU4Q+lfA9aAQ="
|
||||
secret_access_key:
|
||||
secure: "AxYRTy/GnjeTJKQdeJ/AEeAd+yXs783bFDKdyKNswtsHlU8sWPQgNcvTLpVqnSQMpiwkGDGi/70rvR5C+AT3SIWNw13RYrgBRpduQU0J+B2JS+3dN2DIePu25uvs++Wo22OfS8I+UjZ1mWY1SSHI2spPXvDCq5tb+Ih8nlYflEyAtxU9Oq2R3Kus2tkIlRnL25sP/2fY7RvuJFYIV63z8ZIJRzB5WzOeERqnXq2zfwos+hycAqyo/VevJnWAYTEDsvBuSODOpZF+QfKtIQ2rYSoqy8Lq1M6UOZimnC3Ulea4euBVf2ssBCnI7csGNG5UzkTiwrPDi2xIP8nM01rHW1yHJ7tQsJaghnUsfw2t6ui4ZofvbbOFTN/YCloHITifEi8Tc1/17isi3y+kX5yQ/Nk5UNry0Wbt91CP+nkL/ZmA5grkBXDL7VJMmB60TnO3ap24CtwBQartN3LoWs7h+4ov+LqbCt6IqpJVWQWlwJeb2MFPFByALtBpsqAyL1SxXlGNpPa94CuXxfQ6Bv436PtefA5FlTzR8uMmqsjWciv06bVnSvVlFEVovN2Fkplrjt7AASJ/8KJs4THDg4k61nfd8roAHx6ewQzl4wCWKCikQ0MuFd2mVHwdrbnCH1mIHuPRyvWMMIAK0ooc1/rmKiJlpgumjxoFYNE10MXtt+I="
|
||||
bucket: "lbry-reflector-binary"
|
||||
|
||||
- provider: releases
|
||||
file: prism-bin
|
||||
file: ./bin/prism-bin
|
||||
skip_cleanup: true
|
||||
on:
|
||||
repo: lbryio/reflector.go
|
||||
tags: true
|
||||
condition: "$DEPLOY = true"
|
||||
api_key:
|
||||
secure: epAlhp3SUr8hhISarJ22n6tRw2TEa4s4oNFIvJUb5HGECVp1SYN7ao0ln5NoNLmfJS60pi911i/kMhhi21/uhZ0kCYlEhhIE2pc1zsiAxK9L9ENCssJ205HfVbe5grhwskLGzgjhU9OznO8WtmyOPWXr0it8M8RCTjx6rEC0A33Id3WMYyhP938Sj9CxEYeH4KS8wFvBXkgBVtrgaYwRTCIROFddHFXOb9jyNhqQ1RbfKtllsVtQhVk5WMlomheBNSS4vr6WMS4X4+2okFqnLtiSn1wrn5I/94UQbnrI1juVnQj0K+j32EyQbAOt4T2cLW3GtG0jhaYKyNMT9ycDCdVACPSDELlHWjeyoes9bnhUFftm6kDbQxwA1UsTF1yG8tMKXxBSmYyoT7qDloi6pBifZMrFXL61uTs6yhVB9LS/2oqg4sc0Ne87bRcn4OxsBeVCe3kbBHDTR/NTyF2gNPtRvgMAWULxTVcUm9VYdO0IWvAig5g4Row0DnFzEquD6CzezbRWD9WyZyV/AFyYHeeQ2PO7jTw0/3M7aDX33Fuhh34lehzmrC03cfgD/wZW+spxozIcQCYdiJqVw+u+/NvbNr0kkFzE9zW26JEmUFTyDvKxvnza1Kwtww3EgH6zaOL8r4yVbb54rePRvLw7pl93zlfJnEB2MCPqJOY5ZpU=
|
||||
Generated
-435
@@ -1,435 +0,0 @@
|
||||
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
|
||||
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:d64110a78451e373c5a952d2625323dbbe3bfe41c67f9652ea9668a6ceb4f645"
|
||||
name = "github.com/armon/go-metrics"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "3c58d8115a78a6879e5df75ae900846768d36895"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:c0b6dbbb56a745020d5939bdde2197241a1c6109f226cca57b16f46916be5e94"
|
||||
name = "github.com/aws/aws-sdk-go"
|
||||
packages = [
|
||||
"aws",
|
||||
"aws/awserr",
|
||||
"aws/awsutil",
|
||||
"aws/client",
|
||||
"aws/client/metadata",
|
||||
"aws/corehandlers",
|
||||
"aws/credentials",
|
||||
"aws/credentials/ec2rolecreds",
|
||||
"aws/credentials/endpointcreds",
|
||||
"aws/credentials/stscreds",
|
||||
"aws/csm",
|
||||
"aws/defaults",
|
||||
"aws/ec2metadata",
|
||||
"aws/endpoints",
|
||||
"aws/request",
|
||||
"aws/session",
|
||||
"aws/signer/v4",
|
||||
"internal/sdkio",
|
||||
"internal/sdkrand",
|
||||
"internal/sdkuri",
|
||||
"internal/shareddefaults",
|
||||
"private/protocol",
|
||||
"private/protocol/eventstream",
|
||||
"private/protocol/eventstream/eventstreamapi",
|
||||
"private/protocol/query",
|
||||
"private/protocol/query/queryutil",
|
||||
"private/protocol/rest",
|
||||
"private/protocol/restxml",
|
||||
"private/protocol/xml/xmlutil",
|
||||
"service/s3",
|
||||
"service/s3/s3iface",
|
||||
"service/s3/s3manager",
|
||||
"service/sts",
|
||||
]
|
||||
pruneopts = ""
|
||||
revision = "c0447dbaaf195bb477fd2d511b8e4665e04b9017"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:e250643be8120824556f39df6ef128fc2be490fc96e0cb64b1a8ecf96bbe3ce6"
|
||||
name = "github.com/btcsuite/btcutil"
|
||||
packages = ["base58"]
|
||||
pruneopts = ""
|
||||
revision = "ab6388e0c60ae4834a1f57511e20c17b5f78be4b"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:0a39ec8bf5629610a4bc7873a92039ee509246da3cef1a0ea60f1ed7e5f9cea5"
|
||||
name = "github.com/davecgh/go-spew"
|
||||
packages = ["spew"]
|
||||
pruneopts = ""
|
||||
revision = "346938d642f2ec3594ed81d874461961cd0faa76"
|
||||
version = "v1.1.0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:968d8903d598e3fae738325d3410f33f07ea6a2b9ee5591e9c262ee37df6845a"
|
||||
name = "github.com/go-errors/errors"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "a6af135bd4e28680facf08a3d206b454abc877a4"
|
||||
version = "v1.0.1"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:d19c78214e03e297e9e30d2eb11892f731358b2951f2a5c7374658a156373e4c"
|
||||
name = "github.com/go-ini/ini"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "358ee7663966325963d4e8b2e1fbd570c5195153"
|
||||
version = "v1.38.1"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:27b11ca1ad214ead955ff5480e8575e74c5df4e4dc02b04256a8d92131e1d3ad"
|
||||
name = "github.com/go-sql-driver/mysql"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "99ff426eb706cffe92ff3d058e168b278cabf7c7"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:dbbeb8ddb0be949954c8157ee8439c2adfd8dc1c9510eb44a6e58cb68c3dce28"
|
||||
name = "github.com/gorilla/context"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "08b5f424b9271eedf6f9f0ce86cb9396ed337a42"
|
||||
version = "v1.1.1"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:c2c8666b4836c81a1d247bdf21c6a6fc1ab586538ab56f74437c2e0df5c375e1"
|
||||
name = "github.com/gorilla/mux"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "e3702bed27f0d39777b0b37b664b6280e8ef8fbf"
|
||||
version = "v1.6.2"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:b4817bdb0da3054166de058111943ac58b315aead2fed4ee838625a4e304f74c"
|
||||
name = "github.com/gorilla/rpc"
|
||||
packages = [
|
||||
"v2",
|
||||
"v2/json",
|
||||
]
|
||||
pruneopts = ""
|
||||
revision = "22c016f3df3febe0c1f6727598b6389507e03a18"
|
||||
version = "v1.1.0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:fe1b4d4cbe48c0d55507c55f8663aa4185576cc58fa0c8be03bb8f19dfe17a9c"
|
||||
name = "github.com/gorilla/websocket"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "ea4d1f681babbce9545c9c5f3d5194a789c89f5b"
|
||||
version = "v1.2.0"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:4fe55793760295fbef367890352b720784243e0ad19b5ee242519a4682bb9ef8"
|
||||
name = "github.com/hashicorp/errwrap"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "d6c0cd88035724dd42e0f335ae30161c20575ecc"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:4423ee95d6ee30bb22f680445c58889bb5b91e1b955405bf34374a053784a8a2"
|
||||
name = "github.com/hashicorp/go-immutable-radix"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "7f3cd4390caab3250a57f30efdb2a65dd7649ecf"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:6a611e691e739173805cb54019b5c39bb9d46455526dff31e0e6fe3aaca52776"
|
||||
name = "github.com/hashicorp/go-msgpack"
|
||||
packages = ["codec"]
|
||||
pruneopts = ""
|
||||
revision = "fa3f63826f7c23912c15263591e65d54d080b458"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:deaebb4a98ca748bbad7eb653f3a675749500020823a086448ffcd7ba6b8b02d"
|
||||
name = "github.com/hashicorp/go-multierror"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "3d5d8f294aa03d8e98859feac328afbdf1ae0703"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:74f54e6ef2339f1de1e8c4b6674442118bd89e619b2fbd949ef2337330067994"
|
||||
name = "github.com/hashicorp/go-sockaddr"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "6d291a969b86c4b633730bfc6b8b9d64c3aafed9"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:9c776d7d9c54b7ed89f119e449983c3f24c0023e75001d6092442412ebca6b94"
|
||||
name = "github.com/hashicorp/golang-lru"
|
||||
packages = ["simplelru"]
|
||||
pruneopts = ""
|
||||
revision = "0fb14efe8c47ae851c0034ed7a448854d3d34cf3"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:d7ce65372f495908f80fc1f80f4dab5d763d9a1de544abd95aa719e4262d0dd5"
|
||||
name = "github.com/hashicorp/memberlist"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "ce8abaa0c60c2d6bee7219f5ddf500e0a1457b28"
|
||||
version = "v0.1.0"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:4fd01ac3766b886665cfd335cc63819ec4e4538dcc1180c05d6edc089619962c"
|
||||
name = "github.com/hashicorp/serf"
|
||||
packages = [
|
||||
"coordinate",
|
||||
"serf",
|
||||
]
|
||||
pruneopts = ""
|
||||
revision = "984a73625de3138f44deb38d00878fab39eb6447"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:870d441fe217b8e689d7949fef6e43efbc787e50f200cb1e70dbca9204a1d6be"
|
||||
name = "github.com/inconshreveable/mousetrap"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
|
||||
version = "v1.0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:4f767a115bc8e08576f6d38ab73c376fc1b1cd3bb5041171c9e8668cc7739b52"
|
||||
name = "github.com/jmespath/go-jmespath"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "0b12d6b5"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:1e4cb5c6d4e92e2fb34f1545e071fb61e81c71fc0c325445db19b9f3981cb343"
|
||||
name = "github.com/johntdyer/slack-go"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "95fac1160b220c5abcf8b0ef88e9c3cb213c09f4"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:132aae3fa5ad407b53f57cf6ebe274e414ec22d1700899023319931fa90b63e2"
|
||||
name = "github.com/johntdyer/slackrus"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "f7aae3243a0786c5a974bce71ed951c459876e64"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:65bb92382468ce7a1ee218b5f352cd0927d358569be7521dd7cb42e74589a308"
|
||||
name = "github.com/lbryio/errors.go"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "ad03d3cc6a5c27c94bbe3412cf7de0eae1a9bd7c"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:76fd7507e6014c598a01f1b3d558774d2a3114c438403bc98123870d2aecec62"
|
||||
name = "github.com/lbryio/lbry.go"
|
||||
packages = [
|
||||
"crypto",
|
||||
"errors",
|
||||
"null",
|
||||
"querytools",
|
||||
"stop",
|
||||
"util",
|
||||
]
|
||||
pruneopts = ""
|
||||
revision = "e2c96944fc485d3ab5e164da78f8439a94c5aa85"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:cabf2bf5e49edfe0c34cb9c6a256f2a99e6cc8c5e660855c8f3dafe1f81d5dcd"
|
||||
name = "github.com/lyoshenka/bencode"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "b7abd7672df533e627eddbf3a5a529786e8bda7f"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:f0bad0fece0fb73c6ea249c18d8e80ffbe86be0457715b04463068f04686cf39"
|
||||
name = "github.com/miekg/dns"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "5a2b9fab83ff0f8bfc99684bd5f43a37abe560f1"
|
||||
version = "v1.0.8"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:ba3b2eb0ae6fd3deac4386c02fe9d2279c9520738eb9db2f0667e74d5c7a0a61"
|
||||
name = "github.com/nlopes/slack"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "0db1d5eae1116bf7c8ed96c6749acfbf4daaec3e"
|
||||
version = "v0.3.0"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:af967afd3cbc6b0145937f4dcab78bcd93e7b2f2b618fb2bcaf7069ad5c638fa"
|
||||
name = "github.com/phayes/freeport"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "e27662a4a9d6b2083dfd7e7b5d0e30985daca925"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:6ee36f2cea425916d81fdaaf983469fc18f91b3cf090cfe90fa0a9d85b8bfab7"
|
||||
name = "github.com/sean-/seed"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "e2103e2c35297fb7e17febb81e49b312087a2372"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:65d0c541c450f6f8bb9654a3f1938407a835bf41faade00bb0fa1416de215d00"
|
||||
name = "github.com/sebdah/goldie"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "8784dd1ab561dcf43d141f6678e9e41f3d0dff55"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:1747c026a603e3c9f33f238e1d1390df2c8f48876b6bcb7a9c52c7b479e040f4"
|
||||
name = "github.com/sirupsen/logrus"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "d329d24db4313262a3b0a24d8aeb1dc4bd294fb0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:d0b38ba6da419a6d4380700218eeec8623841d44a856bb57369c172fbf692ab4"
|
||||
name = "github.com/spf13/cast"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "8965335b8c7107321228e3e3702cab9832751bac"
|
||||
version = "v1.2.0"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:b5212c335a490d958a6b1b5b48901b46e682b82fa9af3a238fa88df6eaa60873"
|
||||
name = "github.com/spf13/cobra"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "7c4570c3ebeb8129a1f7456d0908a8b676b6f9f1"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:8e243c568f36b09031ec18dff5f7d2769dcf5ca4d624ea511c8e3197dc3d352d"
|
||||
name = "github.com/spf13/pflag"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "583c0c0531f06d5278b7d917446061adc344b5cd"
|
||||
version = "v1.0.1"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:f05efcac20bea32e1fcefde9a3cbefb07e02053666c4a67681ad18c8efc682d3"
|
||||
name = "github.com/uber-go/atomic"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "ca680462431f8c7f96686d91b1d9e5203c4b075b"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:16b935c128f178647036048862a21e8bfd66d1e83fb19787a8b356bdcf0de899"
|
||||
name = "golang.org/x/crypto"
|
||||
packages = [
|
||||
"ed25519",
|
||||
"ed25519/internal/edwards25519",
|
||||
"ripemd160",
|
||||
"sha3",
|
||||
"ssh/terminal",
|
||||
]
|
||||
pruneopts = ""
|
||||
revision = "f027049dab0ad238e394a753dba2d14753473a04"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:15f2fc8cc79d90b0d4d712f04bd3eb3a3856ff3dd6b610a1425113ead3501610"
|
||||
name = "golang.org/x/net"
|
||||
packages = [
|
||||
"bpf",
|
||||
"context",
|
||||
"internal/iana",
|
||||
"internal/socket",
|
||||
"ipv4",
|
||||
"ipv6",
|
||||
]
|
||||
pruneopts = ""
|
||||
revision = "19491d39cadbd9cd33f26ca22cc89ba4ba38251c"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:e8f649ecfae7835a0a27ef39fd2180f6d3c12bc422e2ae55cd611f0b283b3e6e"
|
||||
name = "golang.org/x/sys"
|
||||
packages = [
|
||||
"unix",
|
||||
"windows",
|
||||
]
|
||||
pruneopts = ""
|
||||
revision = "0718ef2ef256118d53a01598f179001ec2af7626"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:55a681cb66f28755765fa5fa5104cbd8dc85c55c02d206f9f89566451e3fe1aa"
|
||||
name = "golang.org/x/time"
|
||||
packages = ["rate"]
|
||||
pruneopts = ""
|
||||
revision = "fbb02b2291d28baffd63558aa44b4b56f178d650"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:eede11c81b63c8f6fd06ef24ba0a640dc077196ec9b7a58ecde03c82eee2f151"
|
||||
name = "google.golang.org/appengine"
|
||||
packages = ["cloudsql"]
|
||||
pruneopts = ""
|
||||
revision = "b1f26356af11148e710935ed1ac8a7f5702c7612"
|
||||
version = "v1.1.0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:05eca53b271663de74078b5484b1995a8d56668a51434a698dc5d0863035d575"
|
||||
name = "gopkg.in/nullbio/null.v6"
|
||||
packages = ["convert"]
|
||||
pruneopts = ""
|
||||
revision = "40264a2e6b7972d183906cf17663983c23231c82"
|
||||
version = "v6.3"
|
||||
|
||||
[solve-meta]
|
||||
analyzer-name = "dep"
|
||||
analyzer-version = 1
|
||||
input-imports = [
|
||||
"github.com/aws/aws-sdk-go/aws",
|
||||
"github.com/aws/aws-sdk-go/aws/awserr",
|
||||
"github.com/aws/aws-sdk-go/aws/credentials",
|
||||
"github.com/aws/aws-sdk-go/aws/session",
|
||||
"github.com/aws/aws-sdk-go/service/s3",
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager",
|
||||
"github.com/davecgh/go-spew/spew",
|
||||
"github.com/go-sql-driver/mysql",
|
||||
"github.com/gorilla/mux",
|
||||
"github.com/gorilla/rpc/v2",
|
||||
"github.com/gorilla/rpc/v2/json",
|
||||
"github.com/hashicorp/serf/serf",
|
||||
"github.com/johntdyer/slackrus",
|
||||
"github.com/lbryio/errors.go",
|
||||
"github.com/lbryio/lbry.go/crypto",
|
||||
"github.com/lbryio/lbry.go/errors",
|
||||
"github.com/lbryio/lbry.go/querytools",
|
||||
"github.com/lbryio/lbry.go/stop",
|
||||
"github.com/lbryio/lbry.go/util",
|
||||
"github.com/lyoshenka/bencode",
|
||||
"github.com/phayes/freeport",
|
||||
"github.com/sebdah/goldie",
|
||||
"github.com/sirupsen/logrus",
|
||||
"github.com/spf13/cast",
|
||||
"github.com/spf13/cobra",
|
||||
"github.com/uber-go/atomic",
|
||||
"golang.org/x/time/rate",
|
||||
]
|
||||
solver-name = "gps-cdcl"
|
||||
solver-version = 1
|
||||
-39
@@ -1,39 +0,0 @@
|
||||
[[constraint]]
|
||||
name = "github.com/aws/aws-sdk-go"
|
||||
branch = "master"
|
||||
|
||||
[[constraint]]
|
||||
name = "github.com/go-sql-driver/mysql"
|
||||
branch = "master"
|
||||
|
||||
[[constraint]]
|
||||
name = "github.com/hashicorp/serf"
|
||||
branch = "master"
|
||||
|
||||
[[constraint]]
|
||||
name = "github.com/lbryio/lbry.go"
|
||||
branch = "master"
|
||||
|
||||
[[constraint]]
|
||||
name = "github.com/sirupsen/logrus"
|
||||
branch = "master"
|
||||
|
||||
[[constraint]]
|
||||
name = "github.com/spf13/cobra"
|
||||
branch = "master"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/lyoshenka/bencode"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/uber-go/atomic"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/phayes/freeport"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/johntdyer/slackrus"
|
||||
@@ -1,27 +1,25 @@
|
||||
BINARY=prism-bin
|
||||
|
||||
DIR = $(shell cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)
|
||||
VENDOR_DIR = vendor
|
||||
BIN_DIR = ${DIR}/bin
|
||||
IMPORT_PATH = github.com/lbryio/reflector.go
|
||||
|
||||
VERSION = $(shell git --git-dir=${DIR}/.git describe --dirty --always --long --abbrev=7)
|
||||
LDFLAGS = -ldflags "-X ${IMPORT_PATH}/meta.Version=${VERSION}"
|
||||
LDFLAGS = -ldflags "-X ${IMPORT_PATH}/meta.Version=${VERSION} -X ${IMPORT_PATH}/meta.Time=$(shell date +%s)"
|
||||
|
||||
|
||||
.PHONY: build dep clean test
|
||||
.PHONY: build clean test lint
|
||||
.DEFAULT_GOAL: build
|
||||
|
||||
|
||||
build: dep
|
||||
CGO_ENABLED=0 go build ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${DIR}/${BINARY} main.go
|
||||
|
||||
dep: | $(VENDOR_DIR)
|
||||
|
||||
$(VENDOR_DIR):
|
||||
go get github.com/golang/dep/cmd/dep && dep ensure
|
||||
build:
|
||||
mkdir -p ${BIN_DIR} && CGO_ENABLED=0 go build ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/${BINARY} main.go
|
||||
|
||||
clean:
|
||||
if [ -f ${DIR}/${BINARY} ]; then rm ${DIR}/${BINARY}; fi
|
||||
if [ -f ${BIN_DIR}/${BINARY} ]; then rm ${BIN_DIR}/${BINARY}; fi
|
||||
|
||||
test:
|
||||
go test ./... -v -cover
|
||||
|
||||
lint:
|
||||
go get github.com/alecthomas/gometalinter && gometalinter --install && gometalinter ./...
|
||||
+3
-3
@@ -6,9 +6,9 @@ import (
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/crypto"
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/lbry.go/stop"
|
||||
"github.com/lbryio/lbry.go/extras/crypto"
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
"github.com/lbryio/lbry.go/extras/stop"
|
||||
|
||||
"github.com/hashicorp/serf/serf"
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
+1
-1
@@ -6,7 +6,7 @@ import (
|
||||
"strconv"
|
||||
"syscall"
|
||||
|
||||
"github.com/lbryio/lbry.go/crypto"
|
||||
"github.com/lbryio/lbry.go/extras/crypto"
|
||||
"github.com/lbryio/reflector.go/cluster"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
+6
-6
@@ -1,7 +1,6 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"log"
|
||||
"net"
|
||||
"os"
|
||||
"os/signal"
|
||||
@@ -9,15 +8,16 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/dht"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
"github.com/lbryio/lbry.go/dht"
|
||||
"github.com/lbryio/lbry.go/dht/bits"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var dhtNodeID string
|
||||
var dhtPort int
|
||||
var dhtRpcPort int
|
||||
var dhtRPCPort int
|
||||
var dhtSeeds []string
|
||||
|
||||
func init() {
|
||||
@@ -30,7 +30,7 @@ func init() {
|
||||
}
|
||||
cmd.PersistentFlags().StringVar(&dhtNodeID, "nodeID", "", "nodeID in hex")
|
||||
cmd.PersistentFlags().IntVar(&dhtPort, "port", 4567, "Port to start DHT on")
|
||||
cmd.PersistentFlags().IntVar(&dhtRpcPort, "rpcPort", 0, "Port to listen for rpc commands on")
|
||||
cmd.PersistentFlags().IntVar(&dhtRPCPort, "rpcPort", 0, "Port to listen for rpc commands on")
|
||||
cmd.PersistentFlags().StringSliceVar(&dhtSeeds, "seeds", []string{}, "Addresses of seed nodes")
|
||||
rootCmd.AddCommand(cmd)
|
||||
}
|
||||
@@ -56,7 +56,7 @@ func dhtCmd(cmd *cobra.Command, args []string) {
|
||||
|
||||
dhtConf := dht.NewStandardConfig()
|
||||
dhtConf.Address = "0.0.0.0:" + strconv.Itoa(dhtPort)
|
||||
dhtConf.RPCPort = dhtRpcPort
|
||||
dhtConf.RPCPort = dhtRPCPort
|
||||
if len(dhtSeeds) > 0 {
|
||||
dhtConf.SeedNodes = dhtSeeds
|
||||
}
|
||||
|
||||
+44
-15
@@ -7,9 +7,11 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
|
||||
"github.com/lbryio/reflector.go/db"
|
||||
"github.com/lbryio/reflector.go/meta"
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
"github.com/lbryio/reflector.go/peer"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
@@ -26,24 +28,48 @@ func init() {
|
||||
}
|
||||
|
||||
func reflectorCmd(cmd *cobra.Command, args []string) {
|
||||
log.Printf("reflector version %s", meta.Version)
|
||||
db := new(db.SQL)
|
||||
err := db.Connect(globalConfig.DBConn)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
log.Printf("reflector version %s, built %s", meta.Version, meta.BuildTime.Format(time.RFC3339))
|
||||
|
||||
// flip this flag to false when doing db maintenance. uploads will not work (as reflector server wont be running)
|
||||
// but downloads will still work straight from s3
|
||||
useDB := false
|
||||
|
||||
s3 := store.NewS3BlobStore(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
|
||||
combo := store.NewDBBackedS3Store(s3, db)
|
||||
|
||||
reflectorServer := reflector.NewServer(combo)
|
||||
reflectorServer.Timeout = 30 * time.Second
|
||||
if globalConfig.SlackHookURL != "" {
|
||||
reflectorServer.StatLogger = log.StandardLogger()
|
||||
reflectorServer.StatReportFrequency = 10 * time.Minute
|
||||
var err error
|
||||
|
||||
var blobStore store.BlobStore = s3
|
||||
var reflectorServer *reflector.Server
|
||||
|
||||
if useDB {
|
||||
db := new(db.SQL)
|
||||
err = db.Connect(globalConfig.DBConn)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
blobStore = store.NewDBBackedS3Store(s3, db)
|
||||
|
||||
reflectorServer = reflector.NewServer(blobStore)
|
||||
reflectorServer.Timeout = 3 * time.Minute
|
||||
if globalConfig.SlackHookURL != "" {
|
||||
reflectorServer.StatLogger = log.StandardLogger()
|
||||
reflectorServer.StatReportFrequency = 1 * time.Hour
|
||||
}
|
||||
reflectorServer.EnableBlocklist = true
|
||||
|
||||
err = reflectorServer.Start(":" + strconv.Itoa(reflector.DefaultPort))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
err = reflectorServer.Start(":" + strconv.Itoa(reflector.DefaultPort))
|
||||
peerServer := peer.NewServer(blobStore)
|
||||
if globalConfig.SlackHookURL != "" {
|
||||
peerServer.StatLogger = log.StandardLogger()
|
||||
peerServer.StatReportFrequency = 1 * time.Hour
|
||||
}
|
||||
err = peerServer.Start(":5567")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
@@ -51,5 +77,8 @@ func reflectorCmd(cmd *cobra.Command, args []string) {
|
||||
interruptChan := make(chan os.Signal, 1)
|
||||
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
|
||||
<-interruptChan
|
||||
reflectorServer.Shutdown()
|
||||
peerServer.Shutdown()
|
||||
if reflectorServer != nil {
|
||||
reflectorServer.Shutdown()
|
||||
}
|
||||
}
|
||||
+14
-3
@@ -5,9 +5,10 @@ import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/lbry.go/util"
|
||||
"github.com/lbryio/reflector.go/dht"
|
||||
"github.com/lbryio/lbry.go/dht"
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
"github.com/lbryio/lbry.go/extras/util"
|
||||
"github.com/lbryio/reflector.go/updater"
|
||||
|
||||
"github.com/johntdyer/slackrus"
|
||||
"github.com/sirupsen/logrus"
|
||||
@@ -22,6 +23,8 @@ type Config struct {
|
||||
BucketName string `json:"bucket_name"`
|
||||
DBConn string `json:"db_conn"`
|
||||
SlackHookURL string `json:"slack_hook_url"`
|
||||
UpdateBinURL string `json:"update_bin_url"`
|
||||
UpdateCmd string `json:"update_cmd"`
|
||||
}
|
||||
|
||||
var verbose []string
|
||||
@@ -102,6 +105,14 @@ func preRun(cmd *cobra.Command, args []string) {
|
||||
logrus.AddHook(hook)
|
||||
debugLogger.AddHook(hook)
|
||||
}
|
||||
|
||||
if globalConfig.UpdateBinURL != "" {
|
||||
if globalConfig.UpdateCmd == "" {
|
||||
logrus.Warnln("update_cmd is empty in conf file")
|
||||
}
|
||||
logrus.Println("starting update checker")
|
||||
go updater.Run(globalConfig.UpdateBinURL, globalConfig.UpdateCmd)
|
||||
}
|
||||
}
|
||||
|
||||
func checkErr(err error) {
|
||||
|
||||
+5
-3
@@ -5,6 +5,8 @@ import (
|
||||
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
|
||||
"github.com/lbryio/lbry.go/stream"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
@@ -25,13 +27,13 @@ func sendBlobCmd(cmd *cobra.Command, args []string) {
|
||||
c := reflector.Client{}
|
||||
err := c.Connect(addr)
|
||||
if err != nil {
|
||||
log.Fatal("error connecting client to server", err)
|
||||
log.Fatal("error connecting client to server: ", err)
|
||||
}
|
||||
|
||||
blob := make([]byte, 1024)
|
||||
blob := make(stream.Blob, 1024)
|
||||
_, err = rand.Read(blob)
|
||||
if err != nil {
|
||||
log.Fatal("failed to make random blob", err)
|
||||
log.Fatal("failed to make random blob: ", err)
|
||||
}
|
||||
|
||||
err = c.SendBlob(blob)
|
||||
|
||||
+2
-2
@@ -7,10 +7,10 @@ import (
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"github.com/lbryio/lbry.go/dht"
|
||||
"github.com/lbryio/lbry.go/dht/bits"
|
||||
"github.com/lbryio/reflector.go/cluster"
|
||||
"github.com/lbryio/reflector.go/db"
|
||||
"github.com/lbryio/reflector.go/dht"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
"github.com/lbryio/reflector.go/peer"
|
||||
"github.com/lbryio/reflector.go/prism"
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/meta"
|
||||
"github.com/lbryio/reflector.go/peer"
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func init() {
|
||||
var cmd = &cobra.Command{
|
||||
Use: "test",
|
||||
Short: "Test things",
|
||||
Run: testCmd,
|
||||
}
|
||||
rootCmd.AddCommand(cmd)
|
||||
}
|
||||
|
||||
func testCmd(cmd *cobra.Command, args []string) {
|
||||
log.Printf("reflector version %s", meta.Version)
|
||||
|
||||
memStore := &store.MemoryBlobStore{}
|
||||
|
||||
reflectorServer := reflector.NewServer(memStore)
|
||||
reflectorServer.Timeout = 3 * time.Minute
|
||||
|
||||
err := reflectorServer.Start(":" + strconv.Itoa(reflector.DefaultPort))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
peerServer := peer.NewServer(memStore)
|
||||
err = peerServer.Start(":" + strconv.Itoa(reflector.DefaultPort+1))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
interruptChan := make(chan os.Signal, 1)
|
||||
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
|
||||
<-interruptChan
|
||||
peerServer.Shutdown()
|
||||
reflectorServer.Shutdown()
|
||||
|
||||
fmt.Println("Blobs in store")
|
||||
for hash, blob := range memStore.Debug() {
|
||||
fmt.Printf("%s: %d bytes\n", hash, len(blob))
|
||||
}
|
||||
}
|
||||
+8
-226
@@ -1,45 +1,20 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/db"
|
||||
"github.com/lbryio/reflector.go/peer"
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/stop"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var uploadWorkers int
|
||||
var uploadSkipExistsCheck bool
|
||||
|
||||
const (
|
||||
sdInc = 1
|
||||
blobInc = 2
|
||||
errInc = 3
|
||||
)
|
||||
|
||||
type uploaderParams struct {
|
||||
workerWG *sync.WaitGroup
|
||||
counterWG *sync.WaitGroup
|
||||
stopper *stop.Group
|
||||
pathChan chan string
|
||||
countChan chan int
|
||||
sdCount int
|
||||
blobCount int
|
||||
errCount int
|
||||
}
|
||||
|
||||
func init() {
|
||||
var cmd = &cobra.Command{
|
||||
Use: "upload PATH",
|
||||
@@ -53,216 +28,23 @@ func init() {
|
||||
}
|
||||
|
||||
func uploadCmd(cmd *cobra.Command, args []string) {
|
||||
startTime := time.Now()
|
||||
db := new(db.SQL)
|
||||
err := db.Connect(globalConfig.DBConn)
|
||||
checkErr(err)
|
||||
|
||||
params := uploaderParams{
|
||||
workerWG: &sync.WaitGroup{},
|
||||
counterWG: &sync.WaitGroup{},
|
||||
pathChan: make(chan string),
|
||||
countChan: make(chan int),
|
||||
stopper: stop.New()}
|
||||
st := store.NewDBBackedS3Store(
|
||||
store.NewS3BlobStore(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName),
|
||||
db)
|
||||
|
||||
setInterrupt(params.stopper)
|
||||
uploader := reflector.NewUploader(db, st, uploadWorkers, uploadSkipExistsCheck)
|
||||
|
||||
paths, err := getPaths(args[0])
|
||||
checkErr(err)
|
||||
|
||||
totalCount := len(paths)
|
||||
|
||||
hashes := make([]string, len(paths))
|
||||
for i, p := range paths {
|
||||
hashes[i] = path.Base(p)
|
||||
}
|
||||
|
||||
log.Println("checking for existing blobs")
|
||||
|
||||
exists := make(map[string]bool)
|
||||
if !uploadSkipExistsCheck {
|
||||
exists, err = db.HasBlobs(hashes)
|
||||
checkErr(err)
|
||||
}
|
||||
existsCount := len(exists)
|
||||
|
||||
log.Printf("%d new blobs to upload", totalCount-existsCount)
|
||||
|
||||
startUploadWorkers(¶ms)
|
||||
params.counterWG.Add(1)
|
||||
go func() {
|
||||
defer params.counterWG.Done()
|
||||
runCountReceiver(¶ms, startTime, totalCount, existsCount)
|
||||
}()
|
||||
|
||||
Upload:
|
||||
for _, f := range paths {
|
||||
if exists[path.Base(f)] {
|
||||
continue
|
||||
}
|
||||
|
||||
select {
|
||||
case params.pathChan <- f:
|
||||
case <-params.stopper.Ch():
|
||||
log.Warnln("Caught interrupt, quitting at first opportunity...")
|
||||
break Upload
|
||||
}
|
||||
}
|
||||
|
||||
close(params.pathChan)
|
||||
params.workerWG.Wait()
|
||||
close(params.countChan)
|
||||
params.counterWG.Wait()
|
||||
params.stopper.Stop()
|
||||
|
||||
log.Println("SUMMARY")
|
||||
log.Printf("%d blobs total", totalCount)
|
||||
log.Printf("%d SD blobs uploaded", params.sdCount)
|
||||
log.Printf("%d content blobs uploaded", params.blobCount)
|
||||
log.Printf("%d blobs already stored", existsCount)
|
||||
log.Printf("%d errors encountered", params.errCount)
|
||||
}
|
||||
|
||||
func isJSON(data []byte) bool {
|
||||
var js json.RawMessage
|
||||
return json.Unmarshal(data, &js) == nil
|
||||
}
|
||||
|
||||
func newBlobStore() *store.DBBackedS3Store {
|
||||
db := new(db.SQL)
|
||||
err := db.Connect(globalConfig.DBConn)
|
||||
checkErr(err)
|
||||
|
||||
s3 := store.NewS3BlobStore(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
|
||||
return store.NewDBBackedS3Store(s3, db)
|
||||
}
|
||||
|
||||
func setInterrupt(stopper *stop.Group) {
|
||||
interruptChan := make(chan os.Signal, 1)
|
||||
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
|
||||
go func() {
|
||||
<-interruptChan
|
||||
stopper.Stop()
|
||||
uploader.Stop()
|
||||
}()
|
||||
}
|
||||
|
||||
func startUploadWorkers(params *uploaderParams) {
|
||||
for i := 0; i < uploadWorkers; i++ {
|
||||
params.workerWG.Add(1)
|
||||
go func(i int) {
|
||||
defer params.workerWG.Done()
|
||||
defer func(i int) {
|
||||
log.Printf("worker %d quitting", i)
|
||||
}(i)
|
||||
|
||||
blobStore := newBlobStore()
|
||||
launchFileUploader(params, blobStore, i)
|
||||
}(i)
|
||||
}
|
||||
}
|
||||
|
||||
func launchFileUploader(params *uploaderParams, blobStore *store.DBBackedS3Store, worker int) {
|
||||
for {
|
||||
select {
|
||||
case <-params.stopper.Ch():
|
||||
return
|
||||
case filepath, ok := <-params.pathChan:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
blob, err := ioutil.ReadFile(filepath)
|
||||
checkErr(err)
|
||||
|
||||
hash := peer.GetBlobHash(blob)
|
||||
if hash != path.Base(filepath) {
|
||||
log.Errorf("worker %d: file name does not match hash (%s != %s), skipping", worker, filepath, hash)
|
||||
select {
|
||||
case params.countChan <- errInc:
|
||||
case <-params.stopper.Ch():
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if isJSON(blob) {
|
||||
log.Printf("worker %d: PUTTING SD BLOB %s", worker, hash)
|
||||
err := blobStore.PutSD(hash, blob)
|
||||
if err != nil {
|
||||
log.Error("PutSD Error: ", err)
|
||||
}
|
||||
select {
|
||||
case params.countChan <- sdInc:
|
||||
case <-params.stopper.Ch():
|
||||
}
|
||||
} else {
|
||||
log.Printf("worker %d: putting %s", worker, hash)
|
||||
err = blobStore.Put(hash, blob)
|
||||
if err != nil {
|
||||
log.Error("put Blob Error: ", err)
|
||||
}
|
||||
select {
|
||||
case params.countChan <- blobInc:
|
||||
case <-params.stopper.Ch():
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func runCountReceiver(params *uploaderParams, startTime time.Time, totalCount int, existsCount int) {
|
||||
for {
|
||||
select {
|
||||
case <-params.stopper.Ch():
|
||||
return
|
||||
case countType, ok := <-params.countChan:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
switch countType {
|
||||
case sdInc:
|
||||
params.sdCount++
|
||||
case blobInc:
|
||||
params.blobCount++
|
||||
case errInc:
|
||||
params.errCount++
|
||||
}
|
||||
}
|
||||
if (params.sdCount+params.blobCount)%50 == 0 {
|
||||
log.Printf("%d of %d done (%s elapsed, %.3fs per blob)", params.sdCount+params.blobCount, totalCount-existsCount, time.Since(startTime).String(), time.Since(startTime).Seconds()/float64(params.sdCount+params.blobCount))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func getPaths(path string) ([]string, error) {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if info.Mode().IsRegular() {
|
||||
return []string{path}, nil
|
||||
}
|
||||
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
files, err := f.Readdir(-1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = f.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var filenames []string
|
||||
for _, file := range files {
|
||||
if !file.IsDir() {
|
||||
filenames = append(filenames, path+"/"+file.Name())
|
||||
}
|
||||
}
|
||||
|
||||
return filenames, nil
|
||||
err = uploader.Upload(args[0])
|
||||
checkErr(err)
|
||||
}
|
||||
+3
-3
@@ -1,10 +1,10 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"log"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/meta"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
@@ -18,5 +18,5 @@ func init() {
|
||||
}
|
||||
|
||||
func versionCmd(cmd *cobra.Command, args []string) {
|
||||
log.Printf("version %s\n", meta.Version)
|
||||
fmt.Printf("version %s (built %s)\n", meta.Version, meta.BuildTime.Format(time.RFC3339))
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"aws_id": "YOUR-AWS-ID",
|
||||
"aws_secret": "YOUR-AWS-SECRET",
|
||||
"bucket_region": "YOUR-BUCKET-REGION",
|
||||
"bucket_name": "YOUR-BUCKET-NAME",
|
||||
"db_conn": "USER:PASSWORD@tcp(localhost:3306)/DBNAME",
|
||||
"slack_hook_url": ""
|
||||
}
|
||||
@@ -3,12 +3,14 @@ package db
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/lbry.go/querytools"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
"github.com/lbryio/lbry.go/dht/bits"
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
qt "github.com/lbryio/lbry.go/extras/query"
|
||||
|
||||
_ "github.com/go-sql-driver/mysql" // blank import for db driver
|
||||
"github.com/go-sql-driver/mysql"
|
||||
_ "github.com/go-sql-driver/mysql" // blank import for db driver ensures its imported even if its not used
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
@@ -19,7 +21,7 @@ type SdBlob struct {
|
||||
Length int `json:"length"`
|
||||
BlobNum int `json:"blob_num"`
|
||||
BlobHash string `json:"blob_hash,omitempty"`
|
||||
Iv string `json:"iv"`
|
||||
IV string `json:"iv"`
|
||||
} `json:"blobs"`
|
||||
StreamType string `json:"stream_type"`
|
||||
Key string `json:"key"`
|
||||
@@ -33,7 +35,7 @@ type SQL struct {
|
||||
}
|
||||
|
||||
func logQuery(query string, args ...interface{}) {
|
||||
s, err := querytools.InterpolateParams(query, args...)
|
||||
s, err := qt.InterpolateParams(query, args...)
|
||||
if err != nil {
|
||||
log.Errorln(err)
|
||||
} else {
|
||||
@@ -53,6 +55,8 @@ func (s *SQL) Connect(dsn string) error {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
s.conn.SetMaxIdleConns(12)
|
||||
|
||||
return errors.Err(s.conn.Ping())
|
||||
}
|
||||
|
||||
@@ -62,44 +66,64 @@ func (s *SQL) AddBlob(hash string, length int, isStored bool) error {
|
||||
return errors.Err("not connected")
|
||||
}
|
||||
|
||||
return withTx(s.conn, func(tx *sql.Tx) error {
|
||||
return addBlob(tx, hash, length, isStored)
|
||||
})
|
||||
_, err := s.insertBlob(hash, length, isStored)
|
||||
return err
|
||||
}
|
||||
|
||||
func addBlob(tx *sql.Tx, hash string, length int, isStored bool) error {
|
||||
func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error) {
|
||||
if length <= 0 {
|
||||
return errors.Err("length must be positive")
|
||||
return 0, errors.Err("length must be positive")
|
||||
}
|
||||
|
||||
err := execTx(tx,
|
||||
blobID, err := s.exec(
|
||||
"INSERT INTO blob_ (hash, is_stored, length) VALUES (?,?,?) ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored))",
|
||||
[]interface{}{hash, isStored, length},
|
||||
hash, isStored, length,
|
||||
)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return nil
|
||||
if blobID == 0 {
|
||||
err = s.conn.QueryRow("SELECT id FROM blob_ WHERE hash = ?", hash).Scan(&blobID)
|
||||
if err != nil {
|
||||
return 0, errors.Err(err)
|
||||
}
|
||||
if blobID == 0 {
|
||||
return 0, errors.Err("blob ID is 0 even after INSERTing and SELECTing")
|
||||
}
|
||||
}
|
||||
|
||||
return blobID, nil
|
||||
}
|
||||
|
||||
func (s *SQL) insertStream(hash, sdHash string, sdBlobID int64) (int64, error) {
|
||||
streamID, err := s.exec(
|
||||
"INSERT IGNORE INTO stream (hash, sd_hash, sd_blob_id) VALUES (?,?, ?)",
|
||||
hash, sdHash, sdBlobID,
|
||||
)
|
||||
if err != nil {
|
||||
return 0, errors.Err(err)
|
||||
}
|
||||
|
||||
if streamID == 0 {
|
||||
err = s.conn.QueryRow("SELECT id FROM stream WHERE sd_blob_id = ?", sdBlobID).Scan(&streamID)
|
||||
if err != nil {
|
||||
return 0, errors.Err(err)
|
||||
}
|
||||
if streamID == 0 {
|
||||
return 0, errors.Err("stream ID is 0 even after INSERTing and SELECTing")
|
||||
}
|
||||
}
|
||||
return streamID, nil
|
||||
}
|
||||
|
||||
// HasBlob checks if the database contains the blob information.
|
||||
func (s *SQL) HasBlob(hash string) (bool, error) {
|
||||
if s.conn == nil {
|
||||
return false, errors.Err("not connected")
|
||||
exists, err := s.HasBlobs([]string{hash})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
query := "SELECT EXISTS(SELECT 1 FROM blob_ WHERE hash = ? AND is_stored = ?)"
|
||||
args := []interface{}{hash, true}
|
||||
|
||||
logQuery(query, args...)
|
||||
|
||||
row := s.conn.QueryRow(query, args...)
|
||||
|
||||
exists := false
|
||||
err := row.Scan(&exists)
|
||||
|
||||
return exists, errors.Err(err)
|
||||
return exists[hash], nil
|
||||
}
|
||||
|
||||
// HasBlobs checks if the database contains the set of blobs and returns a bool map.
|
||||
@@ -110,7 +134,7 @@ func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
|
||||
|
||||
var hash string
|
||||
exists := make(map[string]bool)
|
||||
maxBatchSize := 100
|
||||
maxBatchSize := 10000
|
||||
doneIndex := 0
|
||||
|
||||
for len(hashes) > doneIndex {
|
||||
@@ -121,7 +145,7 @@ func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
|
||||
log.Debugf("getting hashes[%d:%d] of %d", doneIndex, sliceEnd, len(hashes))
|
||||
batch := hashes[doneIndex:sliceEnd]
|
||||
|
||||
query := "SELECT hash FROM blob_ WHERE is_stored = ? && hash IN (" + querytools.Qs(len(batch)) + ")"
|
||||
query := "SELECT hash FROM blob_ WHERE is_stored = ? && hash IN (" + qt.Qs(len(batch)) + ")"
|
||||
args := make([]interface{}, len(batch)+1)
|
||||
args[0] = true
|
||||
for i := range batch {
|
||||
@@ -130,34 +154,88 @@ func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
|
||||
|
||||
logQuery(query, args...)
|
||||
|
||||
rows, err := s.conn.Query(query, args...)
|
||||
if err != nil {
|
||||
closeRows(rows)
|
||||
return exists, err
|
||||
}
|
||||
|
||||
for rows.Next() {
|
||||
err := rows.Scan(&hash)
|
||||
err := func() error {
|
||||
startTime := time.Now()
|
||||
rows, err := s.conn.Query(query, args...)
|
||||
log.Debugf("hashes query took %s", time.Since(startTime))
|
||||
if err != nil {
|
||||
closeRows(rows)
|
||||
return exists, err
|
||||
return errors.Err(err)
|
||||
}
|
||||
exists[hash] = true
|
||||
}
|
||||
defer closeRows(rows)
|
||||
|
||||
err = rows.Err()
|
||||
for rows.Next() {
|
||||
err := rows.Scan(&hash)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
exists[hash] = true
|
||||
}
|
||||
|
||||
err = rows.Err()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
doneIndex += len(batch)
|
||||
return nil
|
||||
}()
|
||||
if err != nil {
|
||||
closeRows(rows)
|
||||
return exists, err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
closeRows(rows)
|
||||
doneIndex += len(batch)
|
||||
}
|
||||
|
||||
return exists, nil
|
||||
}
|
||||
|
||||
// Delete will remove the blob from the db
|
||||
func (s *SQL) Delete(hash string) error {
|
||||
_, err := s.exec("DELETE FROM stream WHERE sd_hash = ?", hash)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
_, err = s.exec("DELETE FROM blob_ WHERE hash = ?", hash)
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
// Block will mark a blob as blocked
|
||||
func (s *SQL) Block(hash string) error {
|
||||
query := "INSERT IGNORE INTO blocked SET hash = ?"
|
||||
args := []interface{}{hash}
|
||||
logQuery(query, args...)
|
||||
_, err := s.conn.Exec(query, args...)
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
// GetBlocked will return a list of blocked hashes
|
||||
func (s *SQL) GetBlocked() (map[string]bool, error) {
|
||||
query := "SELECT hash FROM blocked"
|
||||
logQuery(query)
|
||||
rows, err := s.conn.Query(query)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
defer closeRows(rows)
|
||||
|
||||
blocked := make(map[string]bool)
|
||||
|
||||
var hash string
|
||||
for rows.Next() {
|
||||
err := rows.Scan(&hash)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
blocked[hash] = true
|
||||
}
|
||||
|
||||
err = rows.Err()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
return blocked, nil
|
||||
}
|
||||
|
||||
// MissingBlobsForKnownStream returns missing blobs for an existing stream
|
||||
// WARNING: if the stream does NOT exist, no blob hashes will be returned, which looks
|
||||
// like no blobs are missing
|
||||
@@ -201,52 +279,44 @@ func (s *SQL) MissingBlobsForKnownStream(sdHash string) ([]string, error) {
|
||||
return missingBlobs, errors.Err(err)
|
||||
}
|
||||
|
||||
// AddSDBlob takes the SD Hash number of blobs and the set of blobs. In a single db tx it inserts the sdblob information
|
||||
// into a stream, and inserts the associated blobs' information in the database. If a blob fails the transaction is
|
||||
// rolled back and error(s) are returned.
|
||||
// AddSDBlob insert the SD blob and all the content blobs. The content blobs are marked as "not stored",
|
||||
// but they are tracked so reflector knows what it is missing.
|
||||
func (s *SQL) AddSDBlob(sdHash string, sdBlobLength int, sdBlob SdBlob) error {
|
||||
if s.conn == nil {
|
||||
return errors.Err("not connected")
|
||||
}
|
||||
|
||||
return withTx(s.conn, func(tx *sql.Tx) error {
|
||||
// insert sd blob
|
||||
err := addBlob(tx, sdHash, sdBlobLength, true)
|
||||
sdBlobID, err := s.insertBlob(sdHash, sdBlobLength, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
streamID, err := s.insertStream(sdBlob.StreamHash, sdHash, sdBlobID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// insert content blobs and connect them to stream
|
||||
for _, contentBlob := range sdBlob.Blobs {
|
||||
if contentBlob.BlobHash == "" {
|
||||
// null terminator blob
|
||||
continue
|
||||
}
|
||||
|
||||
blobID, err := s.insertBlob(contentBlob.BlobHash, contentBlob.Length, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// insert stream
|
||||
err = execTx(tx,
|
||||
"INSERT IGNORE INTO stream (hash, sd_hash) VALUES (?,?)",
|
||||
[]interface{}{sdBlob.StreamHash, sdHash},
|
||||
_, err = s.exec(
|
||||
"INSERT IGNORE INTO stream_blob (stream_id, stream_hash, blob_id, blob_hash, num) VALUES (?,?,?,?,?)",
|
||||
streamID, sdBlob.StreamHash, blobID, contentBlob.BlobHash, contentBlob.BlobNum,
|
||||
)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
// insert content blobs and connect them to stream
|
||||
for _, contentBlob := range sdBlob.Blobs {
|
||||
if contentBlob.BlobHash == "" {
|
||||
// null terminator blob
|
||||
continue
|
||||
}
|
||||
|
||||
err := addBlob(tx, contentBlob.BlobHash, contentBlob.Length, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = execTx(tx,
|
||||
"INSERT IGNORE INTO stream_blob (stream_hash, blob_hash, num) VALUES (?,?,?)",
|
||||
[]interface{}{sdBlob.StreamHash, contentBlob.BlobHash, contentBlob.BlobNum},
|
||||
)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetHashRange gets the smallest and biggest hashes in the db
|
||||
@@ -364,38 +434,161 @@ func closeRows(rows *sql.Rows) {
|
||||
}
|
||||
}
|
||||
|
||||
func execTx(tx *sql.Tx, query string, args []interface{}) error {
|
||||
func (s *SQL) exec(query string, args ...interface{}) (int64, error) {
|
||||
logQuery(query, args...)
|
||||
_, err := tx.Exec(query, args...)
|
||||
return errors.Err(err)
|
||||
attempt, maxAttempts := 0, 3
|
||||
Retry:
|
||||
attempt++
|
||||
result, err := s.conn.Exec(query, args...)
|
||||
if isLockTimeoutError(err) {
|
||||
if attempt <= maxAttempts {
|
||||
//Error 1205: Lock wait timeout exceeded; try restarting transaction
|
||||
goto Retry
|
||||
}
|
||||
err = errors.Prefix("Lock timeout for query "+query, err)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return 0, errors.Err(err)
|
||||
}
|
||||
|
||||
lastID, err := result.LastInsertId()
|
||||
return lastID, errors.Err(err)
|
||||
}
|
||||
|
||||
func isLockTimeoutError(err error) bool {
|
||||
e, ok := err.(*mysql.MySQLError)
|
||||
return ok && e != nil && e.Number == 1205
|
||||
}
|
||||
|
||||
/* SQL schema
|
||||
|
||||
in prod, set tx_isolation to READ-COMMITTED to improve db performance
|
||||
make sure you use latin1 or utf8 charset, NOT utf8mb4. that's a waste of space.
|
||||
|
||||
todo: could add UNIQUE KEY (stream_hash, num) to stream_blob ...
|
||||
|
||||
CREATE TABLE blob_ (
|
||||
id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT UNIQUE,
|
||||
hash char(96) NOT NULL,
|
||||
is_stored TINYINT(1) NOT NULL DEFAULT 0,
|
||||
length bigint(20) unsigned DEFAULT NULL,
|
||||
PRIMARY KEY (hash)
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY blob_hash_idx (hash)
|
||||
);
|
||||
|
||||
CREATE TABLE stream (
|
||||
id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT UNIQUE,
|
||||
hash char(96) NOT NULL,
|
||||
sd_hash char(96) NOT NULL,
|
||||
PRIMARY KEY (hash),
|
||||
KEY sd_hash_idx (sd_hash),
|
||||
FOREIGN KEY (sd_hash) REFERENCES blob_ (hash) ON DELETE RESTRICT ON UPDATE CASCADE
|
||||
sd_blob_id BIGINT UNSIGNED NOT NULL,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY stream_hash_idx (hash),
|
||||
KEY stream_sd_blob_id_idx (sd_blob_id),
|
||||
FOREIGN KEY (sd_blob_id) REFERENCES blob_ (id) ON DELETE RESTRICT ON UPDATE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE stream_blob (
|
||||
stream_hash char(96) NOT NULL,
|
||||
blob_hash char(96) NOT NULL,
|
||||
stream_id BIGINT UNSIGNED NOT NULL,
|
||||
blob_id BIGINT UNSIGNED NOT NULL,
|
||||
num int NOT NULL,
|
||||
PRIMARY KEY (stream_hash, blob_hash),
|
||||
FOREIGN KEY (stream_hash) REFERENCES stream (hash) ON DELETE CASCADE ON UPDATE CASCADE,
|
||||
FOREIGN KEY (blob_hash) REFERENCES blob_ (hash) ON DELETE CASCADE ON UPDATE CASCADE
|
||||
PRIMARY KEY (stream_id, blob_id),
|
||||
KEY stream_blob_blob_id_idx (blob_id),
|
||||
FOREIGN KEY (stream_id) REFERENCES stream (id) ON DELETE CASCADE ON UPDATE CASCADE,
|
||||
FOREIGN KEY (blob_id) REFERENCES blob_ (id) ON DELETE CASCADE ON UPDATE CASCADE
|
||||
);
|
||||
|
||||
could add UNIQUE KEY (stream_hash, num) to stream_blob ...
|
||||
CREATE TABLE blocked (
|
||||
hash char(96) NOT NULL,
|
||||
PRIMARY KEY (hash)
|
||||
);
|
||||
|
||||
|
||||
FOR THE MIGRATION, USE THESE TRIGGERS
|
||||
|
||||
CREATE TRIGGER tgr_stream_insert AFTER INSERT ON stream FOR EACH ROW INSERT INTO stream_new SET id = NEW.id, hash = NEW.hash, sd_blob_id = (SELECT id from blob_ where hash = NEW.sd_hash);
|
||||
CREATE TRIGGER tgr_stream_delete AFTER DELETE ON stream FOR EACH ROW DELETE FROM stream_new WHERE id = OLD.id;
|
||||
|
||||
CREATE TRIGGER tgr_blob_insert AFTER INSERT ON blob_ FOR EACH ROW INSERT INTO blob_new SET id = NEW.id, hash = NEW.hash, is_stored = NEW.is_stored, length = NEW.length;
|
||||
CREATE TRIGGER tgr_blob_update AFTER UPDATE ON blob_ FOR EACH ROW UPDATE blob_new SET hash = NEW.hash, is_stored = NEW.is_stored, length = NEW.length WHERE id = NEW.id;
|
||||
CREATE TRIGGER tgr_blob_delete AFTER DELETE ON blob_ FOR EACH ROW DELETE FROM blob_new WHERE id = OLD.id;
|
||||
|
||||
CREATE TRIGGER tgr_stream_blob_insert AFTER INSERT ON stream_blob FOR EACH ROW INSERT INTO stream_blob_new SET stream_id = NEW.stream_id, blob_id = NEW.blob_id, num = NEW.num;
|
||||
CREATE TRIGGER tgr_stream_blob_delete AFTER DELETE ON stream_blob FOR EACH ROW DELETE FROM stream_blob_new WHERE stream_id = OLD.stream_id AND blob_id = OLD.blob_id;
|
||||
|
||||
|
||||
|
||||
DROP PROCEDURE IF EXISTS copyblobs;
|
||||
DELIMITER $$
|
||||
CREATE PROCEDURE copyblobs()
|
||||
BEGIN
|
||||
DECLARE first_trigger_blob_id INT DEFAULT 124802284; # ID of first blob that was copied using the triggers. dont copy anything after that.
|
||||
DECLARE i INT DEFAULT 0;
|
||||
DECLARE minid BIGINT UNSIGNED DEFAULT 0;
|
||||
SELECT min(id) INTO minid FROM blob_ WHERE id < first_trigger_blob_id AND id > (SELECT coalesce(max(id),0) from blob_new where id < first_trigger_blob_id);
|
||||
wloop: WHILE minid is not null DO
|
||||
#IF (i >= 100) THEN
|
||||
# LEAVE wloop;
|
||||
#END IF;
|
||||
SET i = i + 1;
|
||||
IF (i % 5000 = 0) THEN
|
||||
SELECT concat('loop ', i, ', id ', minid) as progress;
|
||||
END IF;
|
||||
|
||||
|
||||
IF (i % 10 = 1) THEN # we start our loops on 1, like normal people
|
||||
START TRANSACTION;
|
||||
END IF;
|
||||
|
||||
INSERT INTO blob_new (id, hash, is_stored, length) SELECT id, hash, is_stored, length from blob_ where id = minid;
|
||||
|
||||
IF (i % 10 = 0) THEN
|
||||
COMMIT;
|
||||
END IF;
|
||||
|
||||
SELECT min(id) INTO minid FROM blob_ WHERE id < first_trigger_blob_id AND id > minid;
|
||||
END WHILE wloop;
|
||||
COMMIT;
|
||||
END$$
|
||||
DELIMITER ;
|
||||
|
||||
|
||||
|
||||
DROP PROCEDURE IF EXISTS copystreams;
|
||||
DELIMITER $$
|
||||
CREATE PROCEDURE copystreams()
|
||||
BEGIN
|
||||
DECLARE first_trigger_stream_id INT DEFAULT 1465749; # ID of first stream that was copied using the triggers. dont copy anything after that.
|
||||
DECLARE i INT DEFAULT 0;
|
||||
DECLARE minid BIGINT UNSIGNED DEFAULT 0;
|
||||
DECLARE streamhash char(96);
|
||||
SELECT min(id) INTO minid FROM stream WHERE id < first_trigger_stream_id AND id > (SELECT coalesce(max(id),0) from stream_new where id < first_trigger_stream_id);
|
||||
wloop: WHILE minid is not null DO
|
||||
#IF (i >= 10) THEN
|
||||
# LEAVE wloop;
|
||||
#END IF;
|
||||
SET i = i + 1;
|
||||
|
||||
IF (i % 5000 = 0) THEN
|
||||
SELECT concat('loop ', i, ', id ', minid) as progress;
|
||||
END IF;
|
||||
|
||||
IF (i % 10 = 1) THEN # we start our loops on 1, like normal people
|
||||
START TRANSACTION;
|
||||
END IF;
|
||||
|
||||
SELECT hash INTO streamhash FROM stream WHERE id = minid;
|
||||
INSERT INTO stream_new (id, hash, sd_blob_id) SELECT s.id, s.hash, b.id FROM stream s INNER JOIN blob_ b ON s.sd_hash = b.hash WHERE s.id = minid;
|
||||
INSERT INTO stream_blob_new SELECT minid, b.id, sb.num FROM stream_blob sb INNER JOIN blob_ b ON sb.blob_hash = b.hash WHERE sb.stream_hash = streamhash;
|
||||
|
||||
IF (i % 10 = 0) THEN
|
||||
COMMIT;
|
||||
END IF;
|
||||
|
||||
SELECT min(id) INTO minid FROM stream WHERE id < first_trigger_stream_id AND id > minid;
|
||||
END WHILE wloop;
|
||||
COMMIT;
|
||||
END$$
|
||||
DELIMITER ;
|
||||
|
||||
|
||||
*/
|
||||
@@ -1,399 +0,0 @@
|
||||
package bits
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"math/big"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
|
||||
"github.com/lyoshenka/bencode"
|
||||
)
|
||||
|
||||
// TODO: http://roaringbitmap.org/
|
||||
|
||||
const (
|
||||
NumBytes = 48 // bytes
|
||||
NumBits = NumBytes * 8
|
||||
)
|
||||
|
||||
// Bitmap is a generalized representation of an identifier or data that can be sorted, compared fast. Used by the DHT
|
||||
// package as a way to handle the unique identifiers of a DHT node.
|
||||
type Bitmap [NumBytes]byte
|
||||
|
||||
func (b Bitmap) RawString() string {
|
||||
return string(b[:])
|
||||
}
|
||||
|
||||
func (b Bitmap) String() string {
|
||||
return b.Hex()
|
||||
}
|
||||
|
||||
// BString returns the bitmap as a string of 0s and 1s
|
||||
func (b Bitmap) BString() string {
|
||||
var s string
|
||||
for _, byte := range b {
|
||||
s += strconv.FormatInt(int64(byte), 2)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// Hex returns a hexadecimal representation of the bitmap.
|
||||
func (b Bitmap) Hex() string {
|
||||
return hex.EncodeToString(b[:])
|
||||
}
|
||||
|
||||
// HexShort returns a hexadecimal representation of the first 4 bytes.
|
||||
func (b Bitmap) HexShort() string {
|
||||
return hex.EncodeToString(b[:4])
|
||||
}
|
||||
|
||||
// HexSimplified returns the hexadecimal representation with all leading 0's removed
|
||||
func (b Bitmap) HexSimplified() string {
|
||||
simple := strings.TrimLeft(b.Hex(), "0")
|
||||
if simple == "" {
|
||||
simple = "0"
|
||||
}
|
||||
return simple
|
||||
}
|
||||
|
||||
func (b Bitmap) Big() *big.Int {
|
||||
i := new(big.Int)
|
||||
i.SetString(b.Hex(), 16)
|
||||
return i
|
||||
}
|
||||
|
||||
// Cmp compares b and other and returns:
|
||||
//
|
||||
// -1 if b < other
|
||||
// 0 if b == other
|
||||
// +1 if b > other
|
||||
//
|
||||
func (b Bitmap) Cmp(other Bitmap) int {
|
||||
for k := range b {
|
||||
if b[k] < other[k] {
|
||||
return -1
|
||||
} else if b[k] > other[k] {
|
||||
return 1
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// Closer returns true if dist(b,x) < dist(b,y)
|
||||
func (b Bitmap) Closer(x, y Bitmap) bool {
|
||||
return x.Xor(b).Cmp(y.Xor(b)) < 0
|
||||
}
|
||||
|
||||
// Equals returns true if every byte in bitmap are equal, false otherwise
|
||||
func (b Bitmap) Equals(other Bitmap) bool {
|
||||
return b.Cmp(other) == 0
|
||||
}
|
||||
|
||||
// Copy returns a duplicate value for the bitmap.
|
||||
func (b Bitmap) Copy() Bitmap {
|
||||
var ret Bitmap
|
||||
copy(ret[:], b[:])
|
||||
return ret
|
||||
}
|
||||
|
||||
// Xor returns a diff bitmap. If they are equal, the returned bitmap will be all 0's. If 100% unique the returned
|
||||
// bitmap will be all 1's.
|
||||
func (b Bitmap) Xor(other Bitmap) Bitmap {
|
||||
var ret Bitmap
|
||||
for k := range b {
|
||||
ret[k] = b[k] ^ other[k]
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
// And returns a comparison bitmap, that for each byte returns the AND true table result
|
||||
func (b Bitmap) And(other Bitmap) Bitmap {
|
||||
var ret Bitmap
|
||||
for k := range b {
|
||||
ret[k] = b[k] & other[k]
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
// Or returns a comparison bitmap, that for each byte returns the OR true table result
|
||||
func (b Bitmap) Or(other Bitmap) Bitmap {
|
||||
var ret Bitmap
|
||||
for k := range b {
|
||||
ret[k] = b[k] | other[k]
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
// Not returns a complimentary bitmap that is an inverse. So b.NOT.NOT = b
|
||||
func (b Bitmap) Not() Bitmap {
|
||||
var ret Bitmap
|
||||
for k := range b {
|
||||
ret[k] = ^b[k]
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
func (b Bitmap) add(other Bitmap) (Bitmap, bool) {
|
||||
var ret Bitmap
|
||||
carry := false
|
||||
for i := NumBits - 1; i >= 0; i-- {
|
||||
bBit := getBit(b[:], i)
|
||||
oBit := getBit(other[:], i)
|
||||
setBit(ret[:], i, bBit != oBit != carry)
|
||||
carry = (bBit && oBit) || (bBit && carry) || (oBit && carry)
|
||||
}
|
||||
return ret, carry
|
||||
}
|
||||
|
||||
// Add returns a bitmap that treats both bitmaps as numbers and adding them together. Since the size of a bitmap is
|
||||
// limited, an overflow is possible when adding bitmaps.
|
||||
func (b Bitmap) Add(other Bitmap) Bitmap {
|
||||
ret, carry := b.add(other)
|
||||
if carry {
|
||||
panic("overflow in bitmap addition. limited to " + strconv.Itoa(NumBits) + " bits.")
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
// Sub returns a bitmap that treats both bitmaps as numbers and subtracts then via the inverse of the other and adding
|
||||
// then together a + (-b). Negative bitmaps are not supported so other must be greater than this.
|
||||
func (b Bitmap) Sub(other Bitmap) Bitmap {
|
||||
if b.Cmp(other) < 0 {
|
||||
// ToDo: Why is this not supported? Should it say not implemented? BitMap might have a generic use case outside of dht.
|
||||
panic("negative bitmaps not supported")
|
||||
}
|
||||
complement, _ := other.Not().add(FromShortHexP("1"))
|
||||
ret, _ := b.add(complement)
|
||||
return ret
|
||||
}
|
||||
|
||||
// Get returns the binary bit at the position passed.
|
||||
func (b Bitmap) Get(n int) bool {
|
||||
return getBit(b[:], n)
|
||||
}
|
||||
|
||||
// Set sets the binary bit at the position passed.
|
||||
func (b Bitmap) Set(n int, one bool) Bitmap {
|
||||
ret := b.Copy()
|
||||
setBit(ret[:], n, one)
|
||||
return ret
|
||||
}
|
||||
|
||||
// PrefixLen returns the number of leading 0 bits
|
||||
func (b Bitmap) PrefixLen() int {
|
||||
for i := range b {
|
||||
for j := 0; j < 8; j++ {
|
||||
if (b[i]>>uint8(7-j))&0x1 != 0 {
|
||||
return i*8 + j
|
||||
}
|
||||
}
|
||||
}
|
||||
return NumBits
|
||||
}
|
||||
|
||||
// Prefix returns a copy of b with the first n bits set to 1 (if `one` is true) or 0 (if `one` is false)
|
||||
// https://stackoverflow.com/a/23192263/182709
|
||||
func (b Bitmap) Prefix(n int, one bool) Bitmap {
|
||||
ret := b.Copy()
|
||||
|
||||
Outer:
|
||||
for i := range ret {
|
||||
for j := 0; j < 8; j++ {
|
||||
if i*8+j < n {
|
||||
if one {
|
||||
ret[i] |= 1 << uint(7-j)
|
||||
} else {
|
||||
ret[i] &= ^(1 << uint(7-j))
|
||||
}
|
||||
} else {
|
||||
break Outer
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ret
|
||||
}
|
||||
|
||||
// Suffix returns a copy of b with the last n bits set to 1 (if `one` is true) or 0 (if `one` is false)
|
||||
// https://stackoverflow.com/a/23192263/182709
|
||||
func (b Bitmap) Suffix(n int, one bool) Bitmap {
|
||||
ret := b.Copy()
|
||||
|
||||
Outer:
|
||||
for i := len(ret) - 1; i >= 0; i-- {
|
||||
for j := 7; j >= 0; j-- {
|
||||
if i*8+j >= NumBits-n {
|
||||
if one {
|
||||
ret[i] |= 1 << uint(7-j)
|
||||
} else {
|
||||
ret[i] &= ^(1 << uint(7-j))
|
||||
}
|
||||
} else {
|
||||
break Outer
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ret
|
||||
}
|
||||
|
||||
// MarshalBencode implements the Marshaller(bencode)/Message interface.
|
||||
func (b Bitmap) MarshalBencode() ([]byte, error) {
|
||||
str := string(b[:])
|
||||
return bencode.EncodeBytes(str)
|
||||
}
|
||||
|
||||
// UnmarshalBencode implements the Marshaller(bencode)/Message interface.
|
||||
func (b *Bitmap) UnmarshalBencode(encoded []byte) error {
|
||||
var str string
|
||||
err := bencode.DecodeBytes(encoded, &str)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(str) != NumBytes {
|
||||
return errors.Err("invalid bitmap length")
|
||||
}
|
||||
copy(b[:], str)
|
||||
return nil
|
||||
}
|
||||
|
||||
// FromBytes returns a bitmap as long as the byte array is of a specific length specified in the parameters.
|
||||
func FromBytes(data []byte) (Bitmap, error) {
|
||||
var bmp Bitmap
|
||||
|
||||
if len(data) != len(bmp) {
|
||||
return bmp, errors.Err("invalid bitmap of length %d", len(data))
|
||||
}
|
||||
|
||||
copy(bmp[:], data)
|
||||
return bmp, nil
|
||||
}
|
||||
|
||||
// FromBytesP returns a bitmap as long as the byte array is of a specific length specified in the parameters
|
||||
// otherwise it wil panic.
|
||||
func FromBytesP(data []byte) Bitmap {
|
||||
bmp, err := FromBytes(data)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return bmp
|
||||
}
|
||||
|
||||
//FromString returns a bitmap by converting the string to bytes and creating from bytes as long as the byte array
|
||||
// is of a specific length specified in the parameters
|
||||
func FromString(data string) (Bitmap, error) {
|
||||
return FromBytes([]byte(data))
|
||||
}
|
||||
|
||||
//FromStringP returns a bitmap by converting the string to bytes and creating from bytes as long as the byte array
|
||||
// is of a specific length specified in the parameters otherwise it wil panic.
|
||||
func FromStringP(data string) Bitmap {
|
||||
bmp, err := FromString(data)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return bmp
|
||||
}
|
||||
|
||||
//FromHex returns a bitmap by converting the hex string to bytes and creating from bytes as long as the byte array
|
||||
// is of a specific length specified in the parameters
|
||||
func FromHex(hexStr string) (Bitmap, error) {
|
||||
decoded, err := hex.DecodeString(hexStr)
|
||||
if err != nil {
|
||||
return Bitmap{}, errors.Err(err)
|
||||
}
|
||||
return FromBytes(decoded)
|
||||
}
|
||||
|
||||
//FromHexP returns a bitmap by converting the hex string to bytes and creating from bytes as long as the byte array
|
||||
// is of a specific length specified in the parameters otherwise it wil panic.
|
||||
func FromHexP(hexStr string) Bitmap {
|
||||
bmp, err := FromHex(hexStr)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return bmp
|
||||
}
|
||||
|
||||
//FromShortHex returns a bitmap by converting the hex string to bytes, adding the leading zeros prefix to the
|
||||
// hex string and creating from bytes as long as the byte array is of a specific length specified in the parameters
|
||||
func FromShortHex(hexStr string) (Bitmap, error) {
|
||||
return FromHex(strings.Repeat("0", NumBytes*2-len(hexStr)) + hexStr)
|
||||
}
|
||||
|
||||
//FromShortHexP returns a bitmap by converting the hex string to bytes, adding the leading zeros prefix to the
|
||||
// hex string and creating from bytes as long as the byte array is of a specific length specified in the parameters
|
||||
// otherwise it wil panic.
|
||||
func FromShortHexP(hexStr string) Bitmap {
|
||||
bmp, err := FromShortHex(hexStr)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return bmp
|
||||
}
|
||||
|
||||
func FromBigP(b *big.Int) Bitmap {
|
||||
return FromShortHexP(b.Text(16))
|
||||
}
|
||||
|
||||
// Max returns a bitmap with all bits set to 1
|
||||
func MaxP() Bitmap {
|
||||
return FromHexP(strings.Repeat("f", NumBytes*2))
|
||||
}
|
||||
|
||||
// Rand generates a cryptographically random bitmap with the confines of the parameters specified.
|
||||
func Rand() Bitmap {
|
||||
var id Bitmap
|
||||
_, err := rand.Read(id[:])
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
// RandInRangeP generates a cryptographically random bitmap and while it is greater than the high threshold
|
||||
// bitmap will subtract the diff between high and low until it is no longer greater that the high.
|
||||
func RandInRangeP(low, high Bitmap) Bitmap {
|
||||
diff := high.Sub(low)
|
||||
r := Rand()
|
||||
for r.Cmp(diff) > 0 {
|
||||
r = r.Sub(diff)
|
||||
}
|
||||
//ToDo - Adding the low at this point doesn't gurantee it will be within the range. Consider bitmaps as numbers and
|
||||
// I have a range of 50-100. If get to say 60, and add 50, I would be at 110. Should protect against this?
|
||||
return r.Add(low)
|
||||
}
|
||||
|
||||
func getBit(b []byte, n int) bool {
|
||||
i := n / 8
|
||||
j := n % 8
|
||||
return b[i]&(1<<uint(7-j)) > 0
|
||||
}
|
||||
|
||||
func setBit(b []byte, n int, one bool) {
|
||||
i := n / 8
|
||||
j := n % 8
|
||||
if one {
|
||||
b[i] |= 1 << uint(7-j)
|
||||
} else {
|
||||
b[i] &= ^(1 << uint(7-j))
|
||||
}
|
||||
}
|
||||
|
||||
// CLosest returns the closest bitmap to target. if no bitmaps are provided, target itself is returned
|
||||
func Closest(target Bitmap, bitmaps ...Bitmap) Bitmap {
|
||||
if len(bitmaps) == 0 {
|
||||
return target
|
||||
}
|
||||
|
||||
var closest *Bitmap
|
||||
for _, b := range bitmaps {
|
||||
if closest == nil || target.Closer(b, *closest) {
|
||||
closest = &b
|
||||
}
|
||||
}
|
||||
return *closest
|
||||
}
|
||||
@@ -1,386 +0,0 @@
|
||||
package bits
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/lyoshenka/bencode"
|
||||
)
|
||||
|
||||
func TestBitmap(t *testing.T) {
|
||||
a := Bitmap{
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
|
||||
12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
|
||||
24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35,
|
||||
36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,
|
||||
}
|
||||
b := Bitmap{
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
|
||||
12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
|
||||
24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35,
|
||||
36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 47, 46,
|
||||
}
|
||||
c := Bitmap{
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1,
|
||||
}
|
||||
|
||||
if !a.Equals(a) {
|
||||
t.Error("bitmap does not equal itself")
|
||||
}
|
||||
if a.Equals(b) {
|
||||
t.Error("bitmap equals another bitmap with different id")
|
||||
}
|
||||
|
||||
if !a.Xor(b).Equals(c) {
|
||||
t.Error(a.Xor(b))
|
||||
}
|
||||
|
||||
if c.PrefixLen() != 375 {
|
||||
t.Error(c.PrefixLen())
|
||||
}
|
||||
|
||||
if b.Cmp(a) < 0 {
|
||||
t.Error("bitmap fails Cmp test")
|
||||
}
|
||||
|
||||
if a.Closer(c, b) || !a.Closer(b, c) || c.Closer(a, b) || c.Closer(b, c) {
|
||||
t.Error("bitmap fails Closer test")
|
||||
}
|
||||
|
||||
id := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
|
||||
if FromHexP(id).Hex() != id {
|
||||
t.Error(FromHexP(id).Hex())
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmap_GetBit(t *testing.T) {
|
||||
tt := []struct {
|
||||
bit int
|
||||
expected bool
|
||||
panic bool
|
||||
}{
|
||||
{bit: 383, expected: false, panic: false},
|
||||
{bit: 382, expected: true, panic: false},
|
||||
{bit: 381, expected: false, panic: false},
|
||||
{bit: 380, expected: true, panic: false},
|
||||
}
|
||||
|
||||
b := FromShortHexP("a")
|
||||
|
||||
for _, test := range tt {
|
||||
actual := getBit(b[:], test.bit)
|
||||
if test.expected != actual {
|
||||
t.Errorf("getting bit %d of %s: expected %t, got %t", test.bit, b.HexSimplified(), test.expected, actual)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmap_SetBit(t *testing.T) {
|
||||
tt := []struct {
|
||||
hex string
|
||||
bit int
|
||||
one bool
|
||||
expected string
|
||||
panic bool
|
||||
}{
|
||||
{hex: "0", bit: 383, one: true, expected: "1", panic: false},
|
||||
{hex: "0", bit: 382, one: true, expected: "2", panic: false},
|
||||
{hex: "0", bit: 381, one: true, expected: "4", panic: false},
|
||||
{hex: "0", bit: 385, one: true, expected: "1", panic: true},
|
||||
{hex: "0", bit: 384, one: true, expected: "1", panic: true},
|
||||
}
|
||||
|
||||
for _, test := range tt {
|
||||
expected := FromShortHexP(test.expected)
|
||||
actual := FromShortHexP(test.hex)
|
||||
if test.panic {
|
||||
assertPanic(t, fmt.Sprintf("setting bit %d to %t", test.bit, test.one), func() { setBit(actual[:], test.bit, test.one) })
|
||||
} else {
|
||||
setBit(actual[:], test.bit, test.one)
|
||||
if !expected.Equals(actual) {
|
||||
t.Errorf("setting bit %d to %t: expected %s, got %s", test.bit, test.one, test.expected, actual.HexSimplified())
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmap_FromHexShort(t *testing.T) {
|
||||
tt := []struct {
|
||||
short string
|
||||
long string
|
||||
}{
|
||||
{short: "", long: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{short: "0", long: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{short: "00000", long: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{short: "9473745bc", long: "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000009473745bc"},
|
||||
{short: "09473745bc", long: "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000009473745bc"},
|
||||
{short: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
||||
long: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"},
|
||||
}
|
||||
|
||||
for _, test := range tt {
|
||||
short := FromShortHexP(test.short)
|
||||
long := FromHexP(test.long)
|
||||
if !short.Equals(long) {
|
||||
t.Errorf("short hex %s: expected %s, got %s", test.short, long.Hex(), short.Hex())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmapMarshal(t *testing.T) {
|
||||
b := FromStringP("123456789012345678901234567890123456789012345678")
|
||||
encoded, err := bencode.EncodeBytes(b)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
if string(encoded) != "48:123456789012345678901234567890123456789012345678" {
|
||||
t.Error("encoding does not match expected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmapMarshalEmbedded(t *testing.T) {
|
||||
e := struct {
|
||||
A string
|
||||
B Bitmap
|
||||
C int
|
||||
}{
|
||||
A: "1",
|
||||
B: FromStringP("222222222222222222222222222222222222222222222222"),
|
||||
C: 3,
|
||||
}
|
||||
|
||||
encoded, err := bencode.EncodeBytes(e)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
if string(encoded) != "d1:A1:11:B48:2222222222222222222222222222222222222222222222221:Ci3ee" {
|
||||
t.Error("encoding does not match expected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmapMarshalEmbedded2(t *testing.T) {
|
||||
encoded, err := bencode.EncodeBytes([]interface{}{
|
||||
FromStringP("333333333333333333333333333333333333333333333333"),
|
||||
})
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
if string(encoded) != "l48:333333333333333333333333333333333333333333333333e" {
|
||||
t.Error("encoding does not match expected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmap_PrefixLen(t *testing.T) {
|
||||
tt := []struct {
|
||||
hex string
|
||||
len int
|
||||
}{
|
||||
{len: 0, hex: "F00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{len: 0, hex: "800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{len: 1, hex: "700000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{len: 1, hex: "400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{len: 384, hex: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{len: 383, hex: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"},
|
||||
{len: 382, hex: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002"},
|
||||
{len: 382, hex: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003"},
|
||||
}
|
||||
|
||||
for _, test := range tt {
|
||||
len := FromHexP(test.hex).PrefixLen()
|
||||
if len != test.len {
|
||||
t.Errorf("got prefix len %d; expected %d for %s", len, test.len, test.hex)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmap_Prefix(t *testing.T) {
|
||||
allOne := FromHexP("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")
|
||||
|
||||
zerosTT := []struct {
|
||||
zeros int
|
||||
expected string
|
||||
}{
|
||||
{zeros: -123, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
|
||||
{zeros: 0, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
|
||||
{zeros: 1, expected: "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
|
||||
{zeros: 69, expected: "000000000000000007ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
|
||||
{zeros: 383, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"},
|
||||
{zeros: 384, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{zeros: 400, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
}
|
||||
|
||||
for _, test := range zerosTT {
|
||||
expected := FromHexP(test.expected)
|
||||
actual := allOne.Prefix(test.zeros, false)
|
||||
if !actual.Equals(expected) {
|
||||
t.Errorf("%d zeros: got %s; expected %s", test.zeros, actual.Hex(), expected.Hex())
|
||||
}
|
||||
}
|
||||
|
||||
for i := 0; i < NumBits; i++ {
|
||||
b := allOne.Prefix(i, false)
|
||||
if b.PrefixLen() != i {
|
||||
t.Errorf("got prefix len %d; expected %d for %s", b.PrefixLen(), i, b.Hex())
|
||||
}
|
||||
}
|
||||
|
||||
allZero := FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
|
||||
|
||||
onesTT := []struct {
|
||||
ones int
|
||||
expected string
|
||||
}{
|
||||
{ones: -123, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{ones: 0, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{ones: 1, expected: "800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{ones: 69, expected: "fffffffffffffffff8000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{ones: 383, expected: "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe"},
|
||||
{ones: 384, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
|
||||
{ones: 400, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
|
||||
}
|
||||
|
||||
for _, test := range onesTT {
|
||||
expected := FromHexP(test.expected)
|
||||
actual := allZero.Prefix(test.ones, true)
|
||||
if !actual.Equals(expected) {
|
||||
t.Errorf("%d ones: got %s; expected %s", test.ones, actual.Hex(), expected.Hex())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmap_Suffix(t *testing.T) {
|
||||
allOne := FromHexP("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")
|
||||
|
||||
zerosTT := []struct {
|
||||
zeros int
|
||||
expected string
|
||||
}{
|
||||
{zeros: -123, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
|
||||
{zeros: 0, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
|
||||
{zeros: 1, expected: "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe"},
|
||||
{zeros: 69, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe00000000000000000"},
|
||||
{zeros: 383, expected: "800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{zeros: 384, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{zeros: 400, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
}
|
||||
|
||||
for _, test := range zerosTT {
|
||||
expected := FromHexP(test.expected)
|
||||
actual := allOne.Suffix(test.zeros, false)
|
||||
if !actual.Equals(expected) {
|
||||
t.Errorf("%d zeros: got %s; expected %s", test.zeros, actual.Hex(), expected.Hex())
|
||||
}
|
||||
}
|
||||
|
||||
for i := 0; i < NumBits; i++ {
|
||||
b := allOne.Prefix(i, false)
|
||||
if b.PrefixLen() != i {
|
||||
t.Errorf("got prefix len %d; expected %d for %s", b.PrefixLen(), i, b.Hex())
|
||||
}
|
||||
}
|
||||
|
||||
allZero := FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
|
||||
|
||||
onesTT := []struct {
|
||||
ones int
|
||||
expected string
|
||||
}{
|
||||
{ones: -123, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{ones: 0, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{ones: 1, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"},
|
||||
{ones: 69, expected: "0000000000000000000000000000000000000000000000000000000000000000000000000000001fffffffffffffffff"},
|
||||
{ones: 383, expected: "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
|
||||
{ones: 384, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
|
||||
{ones: 400, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
|
||||
}
|
||||
|
||||
for _, test := range onesTT {
|
||||
expected := FromHexP(test.expected)
|
||||
actual := allZero.Suffix(test.ones, true)
|
||||
if !actual.Equals(expected) {
|
||||
t.Errorf("%d ones: got %s; expected %s", test.ones, actual.Hex(), expected.Hex())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmap_Add(t *testing.T) {
|
||||
tt := []struct {
|
||||
a, b, sum string
|
||||
panic bool
|
||||
}{
|
||||
{"0", "0", "0", false},
|
||||
{"0", "1", "1", false},
|
||||
{"1", "0", "1", false},
|
||||
{"1", "1", "2", false},
|
||||
{"8", "4", "c", false},
|
||||
{"1000", "0010", "1010", false},
|
||||
{"1111", "1111", "2222", false},
|
||||
{"ffff", "1", "10000", false},
|
||||
{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", false},
|
||||
{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "1", "", true},
|
||||
}
|
||||
|
||||
for _, test := range tt {
|
||||
a := FromShortHexP(test.a)
|
||||
b := FromShortHexP(test.b)
|
||||
expected := FromShortHexP(test.sum)
|
||||
if test.panic {
|
||||
assertPanic(t, fmt.Sprintf("adding %s and %s", test.a, test.b), func() { a.Add(b) })
|
||||
} else {
|
||||
actual := a.Add(b)
|
||||
if !expected.Equals(actual) {
|
||||
t.Errorf("adding %s and %s; expected %s, got %s", test.a, test.b, test.sum, actual.HexSimplified())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmap_Sub(t *testing.T) {
|
||||
tt := []struct {
|
||||
a, b, sum string
|
||||
panic bool
|
||||
}{
|
||||
{"0", "0", "0", false},
|
||||
{"1", "0", "1", false},
|
||||
{"1", "1", "0", false},
|
||||
{"8", "4", "4", false},
|
||||
{"f", "9", "6", false},
|
||||
{"f", "e", "1", false},
|
||||
{"10", "f", "1", false},
|
||||
{"2222", "1111", "1111", false},
|
||||
{"ffff", "1", "fffe", false},
|
||||
{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", false},
|
||||
{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0", false},
|
||||
{"0", "1", "", true},
|
||||
}
|
||||
|
||||
for _, test := range tt {
|
||||
a := FromShortHexP(test.a)
|
||||
b := FromShortHexP(test.b)
|
||||
expected := FromShortHexP(test.sum)
|
||||
if test.panic {
|
||||
assertPanic(t, fmt.Sprintf("subtracting %s - %s", test.a, test.b), func() { a.Sub(b) })
|
||||
} else {
|
||||
actual := a.Sub(b)
|
||||
if !expected.Equals(actual) {
|
||||
t.Errorf("subtracting %s - %s; expected %s, got %s", test.a, test.b, test.sum, actual.HexSimplified())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertPanic(t *testing.T, text string, f func()) {
|
||||
defer func() {
|
||||
if r := recover(); r == nil {
|
||||
t.Errorf("%s: did not panic as expected", text)
|
||||
}
|
||||
}()
|
||||
f()
|
||||
}
|
||||
@@ -1,65 +0,0 @@
|
||||
package bits
|
||||
|
||||
import (
|
||||
"math/big"
|
||||
|
||||
"github.com/lbryio/errors.go"
|
||||
)
|
||||
|
||||
// Range has a start and end
|
||||
type Range struct {
|
||||
Start Bitmap
|
||||
End Bitmap
|
||||
}
|
||||
|
||||
func MaxRange() Range {
|
||||
return Range{
|
||||
Start: Bitmap{},
|
||||
End: MaxP(),
|
||||
}
|
||||
}
|
||||
|
||||
// IntervalP divides the range into `num` intervals and returns the `n`th one
|
||||
// intervals are approximately the same size, but may not be exact because of rounding issues
|
||||
// the first interval always starts at the beginning of the range, and the last interval always ends at the end
|
||||
func (r Range) IntervalP(n, num int) Range {
|
||||
if num < 1 || n < 1 || n > num {
|
||||
panic(errors.Err("invalid interval %d of %d", n, num))
|
||||
}
|
||||
|
||||
start := r.intervalStart(n, num)
|
||||
end := r.End.Big()
|
||||
if n < num {
|
||||
end = r.intervalStart(n+1, num)
|
||||
end.Sub(end, big.NewInt(1))
|
||||
}
|
||||
|
||||
return Range{FromBigP(start), FromBigP(end)}
|
||||
}
|
||||
|
||||
func (r Range) intervalStart(n, num int) *big.Int {
|
||||
// formula:
|
||||
// size = (end - start) / num
|
||||
// rem = (end - start) % num
|
||||
// intervalStart = rangeStart + (size * n-1) + ((rem * n-1) % num)
|
||||
|
||||
size := new(big.Int)
|
||||
rem := new(big.Int)
|
||||
size.Sub(r.End.Big(), r.Start.Big()).DivMod(size, big.NewInt(int64(num)), rem)
|
||||
|
||||
size.Mul(size, big.NewInt(int64(n-1)))
|
||||
rem.Mul(rem, big.NewInt(int64(n-1))).Mod(rem, big.NewInt(int64(num)))
|
||||
|
||||
start := r.Start.Big()
|
||||
start.Add(start, size).Add(start, rem)
|
||||
|
||||
return start
|
||||
}
|
||||
|
||||
func (r Range) IntervalSize() *big.Int {
|
||||
return (&big.Int{}).Sub(r.End.Big(), r.Start.Big())
|
||||
}
|
||||
|
||||
func (r Range) Contains(b Bitmap) bool {
|
||||
return r.Start.Cmp(b) <= 0 && r.End.Cmp(b) >= 0
|
||||
}
|
||||
@@ -1,48 +0,0 @@
|
||||
package bits
|
||||
|
||||
import (
|
||||
"math/big"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMaxRange(t *testing.T) {
|
||||
start := FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
|
||||
end := FromHexP("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")
|
||||
r := MaxRange()
|
||||
|
||||
if !r.Start.Equals(start) {
|
||||
t.Error("max range does not start at the beginning")
|
||||
}
|
||||
if !r.End.Equals(end) {
|
||||
t.Error("max range does not end at the end")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRange_IntervalP(t *testing.T) {
|
||||
max := MaxRange()
|
||||
|
||||
numIntervals := 97
|
||||
expectedAvg := (&big.Int{}).Div(max.IntervalSize(), big.NewInt(int64(numIntervals)))
|
||||
maxDiff := big.NewInt(int64(numIntervals))
|
||||
|
||||
var lastEnd Bitmap
|
||||
|
||||
for i := 1; i <= numIntervals; i++ {
|
||||
ival := max.IntervalP(i, numIntervals)
|
||||
if i == 1 && !ival.Start.Equals(max.Start) {
|
||||
t.Error("first interval does not start at 0")
|
||||
}
|
||||
if i == numIntervals && !ival.End.Equals(max.End) {
|
||||
t.Error("last interval does not end at max")
|
||||
}
|
||||
if i > 1 && !ival.Start.Equals(lastEnd.Add(FromShortHexP("1"))) {
|
||||
t.Errorf("interval %d of %d: last end was %s, this start is %s", i, numIntervals, lastEnd.Hex(), ival.Start.Hex())
|
||||
}
|
||||
|
||||
if ival.IntervalSize().Cmp((&big.Int{}).Add(expectedAvg, maxDiff)) > 0 || ival.IntervalSize().Cmp((&big.Int{}).Sub(expectedAvg, maxDiff)) < 0 {
|
||||
t.Errorf("interval %d of %d: interval size is outside the normal range", i, numIntervals)
|
||||
}
|
||||
|
||||
lastEnd = ival.End
|
||||
}
|
||||
}
|
||||
@@ -1,212 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"math/rand"
|
||||
"net"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
)
|
||||
|
||||
const (
|
||||
bootstrapDefaultRefreshDuration = 15 * time.Minute
|
||||
)
|
||||
|
||||
// BootstrapNode is a configured node setup for testing.
|
||||
type BootstrapNode struct {
|
||||
Node
|
||||
|
||||
initialPingInterval time.Duration
|
||||
checkInterval time.Duration
|
||||
|
||||
nlock *sync.RWMutex
|
||||
peers map[bits.Bitmap]*peer
|
||||
nodeIDs []bits.Bitmap // necessary for efficient random ID selection
|
||||
}
|
||||
|
||||
// NewBootstrapNode returns a BootstrapNode pointer.
|
||||
func NewBootstrapNode(id bits.Bitmap, initialPingInterval, rePingInterval time.Duration) *BootstrapNode {
|
||||
b := &BootstrapNode{
|
||||
Node: *NewNode(id),
|
||||
|
||||
initialPingInterval: initialPingInterval,
|
||||
checkInterval: rePingInterval,
|
||||
|
||||
nlock: &sync.RWMutex{},
|
||||
peers: make(map[bits.Bitmap]*peer),
|
||||
nodeIDs: make([]bits.Bitmap, 0),
|
||||
}
|
||||
|
||||
b.requestHandler = b.handleRequest
|
||||
|
||||
return b
|
||||
}
|
||||
|
||||
// Add manually adds a contact
|
||||
func (b *BootstrapNode) Add(c Contact) {
|
||||
b.upsert(c)
|
||||
}
|
||||
|
||||
// Connect connects to the given connection and starts any background threads necessary
|
||||
func (b *BootstrapNode) Connect(conn UDPConn) error {
|
||||
err := b.Node.Connect(conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Infof("[%s] bootstrap: node connected", b.id.HexShort())
|
||||
|
||||
go func() {
|
||||
t := time.NewTicker(b.checkInterval / 5)
|
||||
for {
|
||||
select {
|
||||
case <-t.C:
|
||||
b.check()
|
||||
case <-b.grp.Ch():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// upsert adds the contact to the list, or updates the lastPinged time
|
||||
func (b *BootstrapNode) upsert(c Contact) {
|
||||
b.nlock.Lock()
|
||||
defer b.nlock.Unlock()
|
||||
|
||||
if peer, exists := b.peers[c.ID]; exists {
|
||||
log.Debugf("[%s] bootstrap: touching contact %s", b.id.HexShort(), peer.Contact.ID.HexShort())
|
||||
peer.Touch()
|
||||
return
|
||||
}
|
||||
|
||||
log.Debugf("[%s] bootstrap: adding new contact %s", b.id.HexShort(), c.ID.HexShort())
|
||||
b.peers[c.ID] = &peer{c, b.id.Xor(c.ID), time.Now(), 0}
|
||||
b.nodeIDs = append(b.nodeIDs, c.ID)
|
||||
}
|
||||
|
||||
// remove removes the contact from the list
|
||||
func (b *BootstrapNode) remove(c Contact) {
|
||||
b.nlock.Lock()
|
||||
defer b.nlock.Unlock()
|
||||
|
||||
_, exists := b.peers[c.ID]
|
||||
if !exists {
|
||||
return
|
||||
}
|
||||
|
||||
log.Debugf("[%s] bootstrap: removing contact %s", b.id.HexShort(), c.ID.HexShort())
|
||||
delete(b.peers, c.ID)
|
||||
for i := range b.nodeIDs {
|
||||
if b.nodeIDs[i].Equals(c.ID) {
|
||||
b.nodeIDs = append(b.nodeIDs[:i], b.nodeIDs[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// get returns up to `limit` random contacts from the list
|
||||
func (b *BootstrapNode) get(limit int) []Contact {
|
||||
b.nlock.RLock()
|
||||
defer b.nlock.RUnlock()
|
||||
|
||||
if len(b.peers) < limit {
|
||||
limit = len(b.peers)
|
||||
}
|
||||
|
||||
ret := make([]Contact, limit)
|
||||
for i, k := range randKeys(len(b.nodeIDs))[:limit] {
|
||||
ret[i] = b.peers[b.nodeIDs[k]].Contact
|
||||
}
|
||||
|
||||
return ret
|
||||
}
|
||||
|
||||
// ping pings a node. if the node responds, it is added to the list. otherwise, it is removed
|
||||
func (b *BootstrapNode) ping(c Contact) {
|
||||
log.Debugf("[%s] bootstrap: pinging %s", b.id.HexShort(), c.ID.HexShort())
|
||||
b.grp.Add(1)
|
||||
defer b.grp.Done()
|
||||
|
||||
resCh := b.SendAsync(c, Request{Method: pingMethod})
|
||||
|
||||
var res *Response
|
||||
|
||||
select {
|
||||
case res = <-resCh:
|
||||
case <-b.grp.Ch():
|
||||
return
|
||||
}
|
||||
|
||||
if res != nil && res.Data == pingSuccessResponse {
|
||||
b.upsert(c)
|
||||
} else {
|
||||
b.remove(c)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *BootstrapNode) check() {
|
||||
b.nlock.RLock()
|
||||
defer b.nlock.RUnlock()
|
||||
|
||||
for i := range b.peers {
|
||||
if !b.peers[i].ActiveInLast(b.checkInterval) {
|
||||
go b.ping(b.peers[i].Contact)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// handleRequest handles the requests received from udp.
|
||||
func (b *BootstrapNode) handleRequest(addr *net.UDPAddr, request Request) {
|
||||
switch request.Method {
|
||||
case pingMethod:
|
||||
err := b.sendMessage(addr, Response{ID: request.ID, NodeID: b.id, Data: pingSuccessResponse})
|
||||
if err != nil {
|
||||
log.Error("error sending response message - ", err)
|
||||
}
|
||||
case findNodeMethod:
|
||||
if request.Arg == nil {
|
||||
log.Errorln("request is missing arg")
|
||||
return
|
||||
}
|
||||
|
||||
err := b.sendMessage(addr, Response{
|
||||
ID: request.ID,
|
||||
NodeID: b.id,
|
||||
Contacts: b.get(bucketSize),
|
||||
})
|
||||
if err != nil {
|
||||
log.Error("error sending 'findnodemethod' response message - ", err)
|
||||
}
|
||||
}
|
||||
|
||||
go func() {
|
||||
b.nlock.RLock()
|
||||
_, exists := b.peers[request.NodeID]
|
||||
b.nlock.RUnlock()
|
||||
if !exists {
|
||||
log.Debugf("[%s] bootstrap: queuing %s to ping", b.id.HexShort(), request.NodeID.HexShort())
|
||||
<-time.After(b.initialPingInterval)
|
||||
b.nlock.RLock()
|
||||
_, exists = b.peers[request.NodeID]
|
||||
b.nlock.RUnlock()
|
||||
if !exists {
|
||||
b.ping(Contact{ID: request.NodeID, IP: addr.IP, Port: addr.Port})
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func randKeys(max int) []int {
|
||||
keys := make([]int, max)
|
||||
for k := range keys {
|
||||
keys[k] = k
|
||||
}
|
||||
rand.Shuffle(max, func(i, j int) {
|
||||
keys[i], keys[j] = keys[j], keys[i]
|
||||
})
|
||||
return keys
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"net"
|
||||
"testing"
|
||||
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
)
|
||||
|
||||
func TestBootstrapPing(t *testing.T) {
|
||||
b := NewBootstrapNode(bits.Rand(), 10, bootstrapDefaultRefreshDuration)
|
||||
|
||||
listener, err := net.ListenPacket(Network, "127.0.0.1:54320")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
err = b.Connect(listener.(*net.UDPConn))
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
b.Shutdown()
|
||||
}
|
||||
@@ -1,76 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
peerproto "github.com/lbryio/reflector.go/peer"
|
||||
)
|
||||
|
||||
const (
|
||||
Network = "udp4"
|
||||
DefaultPort = 4444
|
||||
|
||||
DefaultAnnounceRate = 10 // send at most this many announces per second
|
||||
DefaultReannounceTime = 50 * time.Minute // should be a bit less than hash expiration time
|
||||
|
||||
// TODO: all these constants should be defaults, and should be used to set values in the standard Config. then the code should use values in the config
|
||||
// TODO: alternatively, have a global Config for constants. at least that way tests can modify the values
|
||||
alpha = 3 // this is the constant alpha in the spec
|
||||
bucketSize = 8 // this is the constant k in the spec
|
||||
nodeIDLength = bits.NumBytes // bytes. this is the constant B in the spec
|
||||
messageIDLength = 20 // bytes.
|
||||
|
||||
udpRetry = 1
|
||||
udpTimeout = 5 * time.Second
|
||||
udpMaxMessageLength = 4096 // bytes. I think our longest message is ~676 bytes, so I rounded up to 1024
|
||||
// scratch that. a findValue could return more than K results if a lot of nodes are storing that value, so we need more buffer
|
||||
|
||||
maxPeerFails = 3 // after this many failures, a peer is considered bad and will be removed from the routing table
|
||||
//tExpire = 60 * time.Minute // the time after which a key/value pair expires; this is a time-to-live (TTL) from the original publication date
|
||||
tRefresh = 1 * time.Hour // the time after which an otherwise unaccessed bucket must be refreshed
|
||||
//tReplicate = 1 * time.Hour // the interval between Kademlia replication events, when a node is required to publish its entire database
|
||||
//tNodeRefresh = 15 * time.Minute // the time after which a good node becomes questionable if it has not messaged us
|
||||
|
||||
compactNodeInfoLength = nodeIDLength + 6 // nodeID + 4 for IP + 2 for port
|
||||
|
||||
tokenSecretRotationInterval = 5 * time.Minute // how often the token-generating secret is rotated
|
||||
)
|
||||
|
||||
// Config represents the configure of dht.
|
||||
type Config struct {
|
||||
// this node's address. format is `ip:port`
|
||||
Address string
|
||||
// the seed nodes through which we can join in dht network
|
||||
SeedNodes []string
|
||||
// the hex-encoded node id for this node. if string is empty, a random id will be generated
|
||||
NodeID string
|
||||
// print the state of the dht every X time
|
||||
PrintState time.Duration
|
||||
// the port that clients can use to download blobs using the LBRY peer protocol
|
||||
PeerProtocolPort int
|
||||
// if nonzero, an RPC server will listen to requests on this port and respond to them
|
||||
RPCPort int
|
||||
// the time after which the original publisher must reannounce a key/value pair
|
||||
ReannounceTime time.Duration
|
||||
// send at most this many announces per second
|
||||
AnnounceRate int
|
||||
// channel that will receive notifications about announcements
|
||||
AnnounceNotificationCh chan announceNotification
|
||||
}
|
||||
|
||||
// NewStandardConfig returns a Config pointer with default values.
|
||||
func NewStandardConfig() *Config {
|
||||
return &Config{
|
||||
Address: "0.0.0.0:" + strconv.Itoa(DefaultPort),
|
||||
SeedNodes: []string{
|
||||
"lbrynet1.lbry.io:4444",
|
||||
"lbrynet2.lbry.io:4444",
|
||||
"lbrynet3.lbry.io:4444",
|
||||
},
|
||||
PeerProtocolPort: peerproto.DefaultPort,
|
||||
ReannounceTime: DefaultReannounceTime,
|
||||
AnnounceRate: DefaultAnnounceRate,
|
||||
}
|
||||
}
|
||||
-118
@@ -1,118 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net"
|
||||
"sort"
|
||||
"strconv"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
|
||||
"github.com/lyoshenka/bencode"
|
||||
)
|
||||
|
||||
// TODO: if routing table is ever empty (aka the node is isolated), it should re-bootstrap
|
||||
|
||||
// Contact contains information for contacting another node on the network
|
||||
type Contact struct {
|
||||
ID bits.Bitmap
|
||||
IP net.IP
|
||||
Port int // the udp port used for the dht
|
||||
PeerPort int // the tcp port a peer can be contacted on for blob requests
|
||||
}
|
||||
|
||||
// Equals returns true if two contacts are the same.
|
||||
func (c Contact) Equals(other Contact, checkID bool) bool {
|
||||
return c.IP.Equal(other.IP) && c.Port == other.Port && (!checkID || c.ID == other.ID)
|
||||
}
|
||||
|
||||
// Addr returns the address of the contact.
|
||||
func (c Contact) Addr() *net.UDPAddr {
|
||||
return &net.UDPAddr{IP: c.IP, Port: c.Port}
|
||||
}
|
||||
|
||||
// String returns a short string representation of the contact
|
||||
func (c Contact) String() string {
|
||||
str := c.ID.HexShort() + "@" + c.Addr().String()
|
||||
if c.PeerPort != 0 {
|
||||
str += "(" + strconv.Itoa(c.PeerPort) + ")"
|
||||
}
|
||||
return str
|
||||
}
|
||||
|
||||
// MarshalCompact returns a compact byteslice representation of the contact
|
||||
// NOTE: The compact representation always uses the tcp PeerPort, not the udp Port. This is dumb, but that's how the python daemon does it
|
||||
func (c Contact) MarshalCompact() ([]byte, error) {
|
||||
if c.IP.To4() == nil {
|
||||
return nil, errors.Err("ip not set")
|
||||
}
|
||||
if c.PeerPort < 0 || c.PeerPort > 65535 {
|
||||
return nil, errors.Err("invalid port")
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
buf.Write(c.IP.To4())
|
||||
buf.WriteByte(byte(c.PeerPort >> 8))
|
||||
buf.WriteByte(byte(c.PeerPort))
|
||||
buf.Write(c.ID[:])
|
||||
|
||||
if buf.Len() != compactNodeInfoLength {
|
||||
return nil, errors.Err("i dont know how this happened")
|
||||
}
|
||||
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
// UnmarshalCompact unmarshals the compact byteslice representation of a contact.
|
||||
// NOTE: The compact representation always uses the tcp PeerPort, not the udp Port. This is dumb, but that's how the python daemon does it
|
||||
func (c *Contact) UnmarshalCompact(b []byte) error {
|
||||
if len(b) != compactNodeInfoLength {
|
||||
return errors.Err("invalid compact length")
|
||||
}
|
||||
c.IP = net.IPv4(b[0], b[1], b[2], b[3]).To4()
|
||||
c.PeerPort = int(uint16(b[5]) | uint16(b[4])<<8)
|
||||
c.ID = bits.FromBytesP(b[6:])
|
||||
return nil
|
||||
}
|
||||
|
||||
// MarshalBencode returns the serialized byte slice representation of a contact.
|
||||
func (c Contact) MarshalBencode() ([]byte, error) {
|
||||
return bencode.EncodeBytes([]interface{}{c.ID, c.IP.String(), c.Port})
|
||||
}
|
||||
|
||||
// UnmarshalBencode unmarshals the serialized byte slice into the appropriate fields of the contact.
|
||||
func (c *Contact) UnmarshalBencode(b []byte) error {
|
||||
var raw []bencode.RawMessage
|
||||
err := bencode.DecodeBytes(b, &raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(raw) != 3 {
|
||||
return errors.Err("contact must have 3 elements; got %d", len(raw))
|
||||
}
|
||||
|
||||
err = bencode.DecodeBytes(raw[0], &c.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var ipStr string
|
||||
err = bencode.DecodeBytes(raw[1], &ipStr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.IP = net.ParseIP(ipStr).To4()
|
||||
if c.IP == nil {
|
||||
return errors.Err("invalid IP")
|
||||
}
|
||||
|
||||
return bencode.DecodeBytes(raw[2], &c.Port)
|
||||
}
|
||||
|
||||
func sortByDistance(contacts []Contact, target bits.Bitmap) {
|
||||
sort.Slice(contacts, func(i, j int) bool {
|
||||
return contacts[i].ID.Xor(target).Cmp(contacts[j].ID.Xor(target)) < 0
|
||||
})
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"net"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
)
|
||||
|
||||
func TestCompactEncoding(t *testing.T) {
|
||||
c := Contact{
|
||||
ID: bits.FromHexP("1c8aff71b99462464d9eeac639595ab99664be3482cb91a29d87467515c7d9158fe72aa1f1582dab07d8f8b5db277f41"),
|
||||
IP: net.ParseIP("1.2.3.4"),
|
||||
PeerPort: int(55<<8 + 66),
|
||||
}
|
||||
|
||||
var compact []byte
|
||||
compact, err := c.MarshalCompact()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(compact) != compactNodeInfoLength {
|
||||
t.Fatalf("got length of %d; expected %d", len(compact), compactNodeInfoLength)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(compact, append([]byte{1, 2, 3, 4, 55, 66}, c.ID[:]...)) {
|
||||
t.Errorf("compact bytes not encoded correctly")
|
||||
}
|
||||
}
|
||||
-232
@@ -1,232 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/lbry.go/stop"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cast"
|
||||
)
|
||||
|
||||
var log *logrus.Logger
|
||||
|
||||
func UseLogger(l *logrus.Logger) {
|
||||
log = l
|
||||
}
|
||||
|
||||
func init() {
|
||||
log = logrus.StandardLogger()
|
||||
//log.SetFormatter(&log.TextFormatter{ForceColors: true})
|
||||
//log.SetLevel(log.DebugLevel)
|
||||
}
|
||||
|
||||
// DHT represents a DHT node.
|
||||
type DHT struct {
|
||||
// config
|
||||
conf *Config
|
||||
// local contact
|
||||
contact Contact
|
||||
// node
|
||||
node *Node
|
||||
// stopGroup to shut down DHT
|
||||
grp *stop.Group
|
||||
// channel is closed when DHT joins network
|
||||
joined chan struct{}
|
||||
// cache for store tokens
|
||||
tokenCache *tokenCache
|
||||
// hashes that need to be put into the announce queue or removed from the queue
|
||||
announceAddRemove chan queueEdit
|
||||
}
|
||||
|
||||
// New returns a DHT pointer. If config is nil, then config will be set to the default config.
|
||||
func New(config *Config) *DHT {
|
||||
if config == nil {
|
||||
config = NewStandardConfig()
|
||||
}
|
||||
|
||||
d := &DHT{
|
||||
conf: config,
|
||||
grp: stop.New(),
|
||||
joined: make(chan struct{}),
|
||||
announceAddRemove: make(chan queueEdit),
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func (dht *DHT) connect(conn UDPConn) error {
|
||||
contact, err := getContact(dht.conf.NodeID, dht.conf.Address)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dht.contact = contact
|
||||
dht.node = NewNode(contact.ID)
|
||||
dht.tokenCache = newTokenCache(dht.node, tokenSecretRotationInterval)
|
||||
|
||||
return dht.node.Connect(conn)
|
||||
}
|
||||
|
||||
// Start starts the dht
|
||||
func (dht *DHT) Start() error {
|
||||
listener, err := net.ListenPacket(Network, dht.conf.Address)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
conn := listener.(*net.UDPConn)
|
||||
|
||||
err = dht.connect(conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dht.join()
|
||||
log.Infof("[%s] DHT ready on %s (%d nodes found during join)",
|
||||
dht.node.id.HexShort(), dht.contact.Addr().String(), dht.node.rt.Count())
|
||||
|
||||
dht.grp.Add(1)
|
||||
go func() {
|
||||
dht.runAnnouncer()
|
||||
dht.grp.Done()
|
||||
}()
|
||||
|
||||
if dht.conf.RPCPort > 0 {
|
||||
dht.grp.Add(1)
|
||||
go func() {
|
||||
dht.runRPCServer(dht.conf.RPCPort)
|
||||
dht.grp.Done()
|
||||
}()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// join makes current node join the dht network.
|
||||
func (dht *DHT) join() {
|
||||
defer close(dht.joined) // if anyone's waiting for join to finish, they'll know its done
|
||||
|
||||
log.Infof("[%s] joining DHT network", dht.node.id.HexShort())
|
||||
|
||||
// ping nodes, which gets their real node IDs and adds them to the routing table
|
||||
atLeastOneNodeResponded := false
|
||||
for _, addr := range dht.conf.SeedNodes {
|
||||
err := dht.Ping(addr)
|
||||
if err != nil {
|
||||
log.Error(errors.Prefix(fmt.Sprintf("[%s] join", dht.node.id.HexShort()), err))
|
||||
} else {
|
||||
atLeastOneNodeResponded = true
|
||||
}
|
||||
}
|
||||
|
||||
if !atLeastOneNodeResponded {
|
||||
log.Errorf("[%s] join: no nodes responded to initial ping", dht.node.id.HexShort())
|
||||
return
|
||||
}
|
||||
|
||||
// now call iterativeFind on yourself
|
||||
_, _, err := FindContacts(dht.node, dht.node.id, false, dht.grp.Child())
|
||||
if err != nil {
|
||||
log.Errorf("[%s] join: %s", dht.node.id.HexShort(), err.Error())
|
||||
}
|
||||
|
||||
// TODO: after joining, refresh all buckets further away than our closest neighbor
|
||||
// http://xlattice.sourceforge.net/components/protocol/kademlia/specs.html#join
|
||||
}
|
||||
|
||||
// WaitUntilJoined blocks until the node joins the network.
|
||||
func (dht *DHT) WaitUntilJoined() {
|
||||
if dht.joined == nil {
|
||||
panic("dht not initialized")
|
||||
}
|
||||
<-dht.joined
|
||||
}
|
||||
|
||||
// Shutdown shuts down the dht
|
||||
func (dht *DHT) Shutdown() {
|
||||
log.Debugf("[%s] DHT shutting down", dht.node.id.HexShort())
|
||||
dht.grp.StopAndWait()
|
||||
dht.node.Shutdown()
|
||||
log.Debugf("[%s] DHT stopped", dht.node.id.HexShort())
|
||||
}
|
||||
|
||||
// Ping pings a given address, creates a temporary contact for sending a message, and returns an error if communication
|
||||
// fails.
|
||||
func (dht *DHT) Ping(addr string) error {
|
||||
raddr, err := net.ResolveUDPAddr(Network, addr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tmpNode := Contact{ID: bits.Rand(), IP: raddr.IP, Port: raddr.Port}
|
||||
res := dht.node.Send(tmpNode, Request{Method: pingMethod}, SendOptions{skipIDCheck: true})
|
||||
if res == nil {
|
||||
return errors.Err("no response from node %s", addr)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get returns the list of nodes that have the blob for the given hash
|
||||
func (dht *DHT) Get(hash bits.Bitmap) ([]Contact, error) {
|
||||
contacts, found, err := FindContacts(dht.node, hash, true, dht.grp.Child())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if found {
|
||||
return contacts, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// PrintState prints the current state of the DHT including address, nr outstanding transactions, stored hashes as well
|
||||
// as current bucket information.
|
||||
func (dht *DHT) PrintState() {
|
||||
log.Printf("DHT node %s at %s", dht.contact.String(), time.Now().Format(time.RFC822Z))
|
||||
log.Printf("Outstanding transactions: %d", dht.node.CountActiveTransactions())
|
||||
log.Printf("Stored hashes: %d", dht.node.store.CountStoredHashes())
|
||||
log.Printf("Buckets:")
|
||||
for _, line := range strings.Split(dht.node.rt.BucketInfo(), "\n") {
|
||||
log.Println(line)
|
||||
}
|
||||
}
|
||||
|
||||
func (dht DHT) ID() bits.Bitmap {
|
||||
return dht.contact.ID
|
||||
}
|
||||
|
||||
func getContact(nodeID, addr string) (Contact, error) {
|
||||
var c Contact
|
||||
if nodeID == "" {
|
||||
c.ID = bits.Rand()
|
||||
} else {
|
||||
c.ID = bits.FromHexP(nodeID)
|
||||
}
|
||||
|
||||
ip, port, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return c, errors.Err(err)
|
||||
} else if ip == "" {
|
||||
return c, errors.Err("address does not contain an IP")
|
||||
} else if port == "" {
|
||||
return c, errors.Err("address does not contain a port")
|
||||
}
|
||||
|
||||
c.IP = net.ParseIP(ip)
|
||||
if c.IP == nil {
|
||||
return c, errors.Err("invalid ip")
|
||||
}
|
||||
|
||||
c.Port, err = cast.ToIntE(port)
|
||||
if err != nil {
|
||||
return c, errors.Err(err)
|
||||
}
|
||||
|
||||
return c, nil
|
||||
}
|
||||
@@ -1,214 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"container/ring"
|
||||
"context"
|
||||
"math"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
type queueEdit struct {
|
||||
hash bits.Bitmap
|
||||
add bool
|
||||
}
|
||||
|
||||
const (
|
||||
announceStarted = "started"
|
||||
announceFinishd = "finished"
|
||||
)
|
||||
|
||||
type announceNotification struct {
|
||||
hash bits.Bitmap
|
||||
action string
|
||||
err error
|
||||
}
|
||||
|
||||
// Add adds the hash to the list of hashes this node is announcing
|
||||
func (dht *DHT) Add(hash bits.Bitmap) {
|
||||
dht.announceAddRemove <- queueEdit{hash: hash, add: true}
|
||||
}
|
||||
|
||||
// Remove removes the hash from the list of hashes this node is announcing
|
||||
func (dht *DHT) Remove(hash bits.Bitmap) {
|
||||
dht.announceAddRemove <- queueEdit{hash: hash, add: false}
|
||||
}
|
||||
|
||||
func (dht *DHT) runAnnouncer() {
|
||||
type hashAndTime struct {
|
||||
hash bits.Bitmap
|
||||
lastAnnounce time.Time
|
||||
}
|
||||
|
||||
var queue *ring.Ring
|
||||
hashes := make(map[bits.Bitmap]*ring.Ring)
|
||||
|
||||
var announceNextHash <-chan time.Time
|
||||
timer := time.NewTimer(math.MaxInt64)
|
||||
timer.Stop()
|
||||
|
||||
limitCh := make(chan time.Time)
|
||||
dht.grp.Add(1)
|
||||
go func() {
|
||||
defer dht.grp.Done()
|
||||
limiter := rate.NewLimiter(rate.Limit(dht.conf.AnnounceRate), dht.conf.AnnounceRate)
|
||||
for {
|
||||
err := limiter.Wait(context.Background()) // TODO: should use grp.ctx somehow? so when grp is closed, wait returns
|
||||
if err != nil {
|
||||
log.Error(errors.Prefix("rate limiter", err))
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case limitCh <- time.Now():
|
||||
case <-dht.grp.Ch():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
maintenance := time.NewTicker(1 * time.Minute)
|
||||
|
||||
// TODO: work to space hash announces out so they aren't bunched up around the reannounce time. track time since last announce. if its been more than the ideal time (reannounce time / numhashes), start announcing hashes early
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-dht.grp.Ch():
|
||||
return
|
||||
|
||||
case <-maintenance.C:
|
||||
maxAnnounce := dht.conf.AnnounceRate * int(dht.conf.ReannounceTime.Seconds())
|
||||
if len(hashes) > maxAnnounce {
|
||||
// TODO: send this to slack
|
||||
log.Warnf("DHT has %d hashes, but can only announce %d hashes in the %s reannounce window. Raise the announce rate or spawn more nodes.",
|
||||
len(hashes), maxAnnounce, dht.conf.ReannounceTime.String())
|
||||
}
|
||||
|
||||
case change := <-dht.announceAddRemove:
|
||||
if change.add {
|
||||
if _, exists := hashes[change.hash]; exists {
|
||||
continue
|
||||
}
|
||||
|
||||
r := ring.New(1)
|
||||
r.Value = hashAndTime{hash: change.hash}
|
||||
if queue != nil {
|
||||
queue.Prev().Link(r)
|
||||
}
|
||||
queue = r
|
||||
hashes[change.hash] = r
|
||||
announceNextHash = limitCh // announce next hash ASAP
|
||||
} else {
|
||||
r, exists := hashes[change.hash]
|
||||
if !exists {
|
||||
continue
|
||||
}
|
||||
|
||||
delete(hashes, change.hash)
|
||||
|
||||
if len(hashes) == 0 {
|
||||
queue = ring.New(0)
|
||||
announceNextHash = nil // no hashes to announce, wait indefinitely
|
||||
} else {
|
||||
if r == queue {
|
||||
queue = queue.Next() // don't lose our pointer
|
||||
}
|
||||
r.Prev().Link(r.Next())
|
||||
}
|
||||
}
|
||||
|
||||
case <-announceNextHash:
|
||||
dht.grp.Add(1)
|
||||
ht := queue.Value.(hashAndTime)
|
||||
|
||||
if !ht.lastAnnounce.IsZero() {
|
||||
nextAnnounce := ht.lastAnnounce.Add(dht.conf.ReannounceTime)
|
||||
if nextAnnounce.After(time.Now()) {
|
||||
timer.Reset(time.Until(nextAnnounce))
|
||||
announceNextHash = timer.C // wait until next hash should be announced
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if dht.conf.AnnounceNotificationCh != nil {
|
||||
dht.conf.AnnounceNotificationCh <- announceNotification{
|
||||
hash: ht.hash,
|
||||
action: announceStarted,
|
||||
}
|
||||
}
|
||||
|
||||
go func(hash bits.Bitmap) {
|
||||
defer dht.grp.Done()
|
||||
err := dht.announce(hash)
|
||||
if err != nil {
|
||||
log.Error(errors.Prefix("announce", err))
|
||||
}
|
||||
|
||||
if dht.conf.AnnounceNotificationCh != nil {
|
||||
dht.conf.AnnounceNotificationCh <- announceNotification{
|
||||
hash: ht.hash,
|
||||
action: announceFinishd,
|
||||
err: err,
|
||||
}
|
||||
}
|
||||
}(ht.hash)
|
||||
|
||||
queue.Value = hashAndTime{hash: ht.hash, lastAnnounce: time.Now()}
|
||||
queue = queue.Next()
|
||||
announceNextHash = limitCh // announce next hash ASAP
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Announce announces to the DHT that this node has the blob for the given hash
|
||||
func (dht *DHT) announce(hash bits.Bitmap) error {
|
||||
contacts, _, err := FindContacts(dht.node, hash, false, dht.grp.Child())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// self-store if we found less than K contacts, or we're closer than the farthest contact
|
||||
if len(contacts) < bucketSize {
|
||||
contacts = append(contacts, dht.contact)
|
||||
} else if hash.Closer(dht.node.id, contacts[bucketSize-1].ID) {
|
||||
contacts[bucketSize-1] = dht.contact
|
||||
}
|
||||
|
||||
wg := &sync.WaitGroup{}
|
||||
for _, c := range contacts {
|
||||
wg.Add(1)
|
||||
go func(c Contact) {
|
||||
dht.store(hash, c)
|
||||
wg.Done()
|
||||
}(c)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (dht *DHT) store(hash bits.Bitmap, c Contact) {
|
||||
if dht.contact.ID == c.ID {
|
||||
// self-store
|
||||
c.PeerPort = dht.conf.PeerProtocolPort
|
||||
dht.node.Store(hash, c)
|
||||
return
|
||||
}
|
||||
|
||||
dht.node.SendAsync(c, Request{
|
||||
Method: storeMethod,
|
||||
StoreArgs: &storeArgs{
|
||||
BlobHash: hash,
|
||||
Value: storeArgsValue{
|
||||
Token: dht.tokenCache.Get(c, hash, dht.grp.Ch()),
|
||||
LbryID: dht.contact.ID,
|
||||
Port: dht.conf.PeerProtocolPort,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
-181
@@ -1,181 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"net"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
)
|
||||
|
||||
func TestNodeFinder_FindNodes(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping slow nodeFinder test")
|
||||
}
|
||||
|
||||
bs, dhts := TestingCreateNetwork(t, 3, true, false)
|
||||
defer func() {
|
||||
for i := range dhts {
|
||||
dhts[i].Shutdown()
|
||||
}
|
||||
bs.Shutdown()
|
||||
}()
|
||||
|
||||
contacts, found, err := FindContacts(dhts[2].node, bits.Rand(), false, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if found {
|
||||
t.Fatal("something was found, but it should not have been")
|
||||
}
|
||||
|
||||
if len(contacts) != 3 {
|
||||
t.Errorf("expected 3 node, found %d", len(contacts))
|
||||
}
|
||||
|
||||
foundBootstrap := false
|
||||
foundOne := false
|
||||
foundTwo := false
|
||||
|
||||
for _, n := range contacts {
|
||||
if n.ID.Equals(bs.id) {
|
||||
foundBootstrap = true
|
||||
}
|
||||
if n.ID.Equals(dhts[0].node.id) {
|
||||
foundOne = true
|
||||
}
|
||||
if n.ID.Equals(dhts[1].node.id) {
|
||||
foundTwo = true
|
||||
}
|
||||
}
|
||||
|
||||
if !foundBootstrap {
|
||||
t.Errorf("did not find bootstrap node %s", bs.id.Hex())
|
||||
}
|
||||
if !foundOne {
|
||||
t.Errorf("did not find first node %s", dhts[0].node.id.Hex())
|
||||
}
|
||||
if !foundTwo {
|
||||
t.Errorf("did not find second node %s", dhts[1].node.id.Hex())
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodeFinder_FindNodes_NoBootstrap(t *testing.T) {
|
||||
_, dhts := TestingCreateNetwork(t, 3, false, false)
|
||||
defer func() {
|
||||
for i := range dhts {
|
||||
dhts[i].Shutdown()
|
||||
}
|
||||
}()
|
||||
|
||||
_, _, err := FindContacts(dhts[2].node, bits.Rand(), false, nil)
|
||||
if err == nil {
|
||||
t.Fatal("contact finder should have errored saying that there are no contacts in the routing table")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodeFinder_FindValue(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping slow nodeFinder test")
|
||||
}
|
||||
|
||||
bs, dhts := TestingCreateNetwork(t, 3, true, false)
|
||||
defer func() {
|
||||
for i := range dhts {
|
||||
dhts[i].Shutdown()
|
||||
}
|
||||
bs.Shutdown()
|
||||
}()
|
||||
|
||||
blobHashToFind := bits.Rand()
|
||||
nodeToFind := Contact{ID: bits.Rand(), IP: net.IPv4(1, 2, 3, 4), Port: 5678}
|
||||
dhts[0].node.store.Upsert(blobHashToFind, nodeToFind)
|
||||
|
||||
contacts, found, err := FindContacts(dhts[2].node, blobHashToFind, true, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if !found {
|
||||
t.Fatal("node was not found")
|
||||
}
|
||||
|
||||
if len(contacts) != 1 {
|
||||
t.Fatalf("expected one node, found %d", len(contacts))
|
||||
}
|
||||
|
||||
if !contacts[0].ID.Equals(nodeToFind.ID) {
|
||||
t.Fatalf("found node id %s, expected %s", contacts[0].ID.Hex(), nodeToFind.ID.Hex())
|
||||
}
|
||||
}
|
||||
|
||||
func TestDHT_LargeDHT(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping large DHT test")
|
||||
}
|
||||
|
||||
nodes := 100
|
||||
bs, dhts := TestingCreateNetwork(t, nodes, true, true)
|
||||
defer func() {
|
||||
for _, d := range dhts {
|
||||
go d.Shutdown()
|
||||
}
|
||||
bs.Shutdown()
|
||||
time.Sleep(1 * time.Second)
|
||||
}()
|
||||
|
||||
wg := &sync.WaitGroup{}
|
||||
ids := make([]bits.Bitmap, nodes)
|
||||
for i := range ids {
|
||||
ids[i] = bits.Rand()
|
||||
wg.Add(1)
|
||||
go func(index int) {
|
||||
defer wg.Done()
|
||||
err := dhts[index].announce(ids[index])
|
||||
if err != nil {
|
||||
t.Error("error announcing random bitmap - ", err)
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
// check that each node is in at learst 1 other routing table
|
||||
rtCounts := make(map[bits.Bitmap]int)
|
||||
for _, d := range dhts {
|
||||
for _, d2 := range dhts {
|
||||
if d.node.id.Equals(d2.node.id) {
|
||||
continue
|
||||
}
|
||||
c := d2.node.rt.GetClosest(d.node.id, 1)
|
||||
if len(c) > 1 {
|
||||
t.Error("rt returned more than one node when only one requested")
|
||||
} else if len(c) == 1 && c[0].ID.Equals(d.node.id) {
|
||||
rtCounts[d.node.id]++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for k, v := range rtCounts {
|
||||
if v == 0 {
|
||||
t.Errorf("%s was not in any routing tables", k.HexShort())
|
||||
}
|
||||
}
|
||||
|
||||
// check that each ID is stored by at least 3 nodes
|
||||
storeCounts := make(map[bits.Bitmap]int)
|
||||
for _, d := range dhts {
|
||||
for _, id := range ids {
|
||||
if len(d.node.store.Get(id)) > 0 {
|
||||
storeCounts[id]++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for k, v := range storeCounts {
|
||||
if v == 0 {
|
||||
t.Errorf("%s was not stored by any nodes", k.HexShort())
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
-463
@@ -1,463 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
|
||||
"github.com/lyoshenka/bencode"
|
||||
"github.com/spf13/cast"
|
||||
)
|
||||
|
||||
const (
|
||||
pingMethod = "ping"
|
||||
storeMethod = "store"
|
||||
findNodeMethod = "findNode"
|
||||
findValueMethod = "findValue"
|
||||
)
|
||||
|
||||
const (
|
||||
pingSuccessResponse = "pong"
|
||||
storeSuccessResponse = "OK"
|
||||
)
|
||||
|
||||
const (
|
||||
requestType = 0
|
||||
responseType = 1
|
||||
errorType = 2
|
||||
)
|
||||
|
||||
const (
|
||||
// these are strings because bencode requires bytestring keys
|
||||
headerTypeField = "0"
|
||||
headerMessageIDField = "1" // message id is 20 bytes long
|
||||
headerNodeIDField = "2" // node id is 48 bytes long
|
||||
headerPayloadField = "3"
|
||||
headerArgsField = "4"
|
||||
contactsField = "contacts"
|
||||
tokenField = "token"
|
||||
protocolVersionField = "protocolVersion"
|
||||
)
|
||||
|
||||
// Message is a DHT message
|
||||
type Message interface {
|
||||
bencode.Marshaler
|
||||
}
|
||||
|
||||
type messageID [messageIDLength]byte
|
||||
|
||||
// HexShort returns the first 8 hex characters of the hex encoded message id.
|
||||
func (m messageID) HexShort() string {
|
||||
return hex.EncodeToString(m[:])[:8]
|
||||
}
|
||||
|
||||
// UnmarshalBencode takes a byte slice and unmarshals the message id.
|
||||
func (m *messageID) UnmarshalBencode(encoded []byte) error {
|
||||
var str string
|
||||
err := bencode.DecodeBytes(encoded, &str)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
copy(m[:], str)
|
||||
return nil
|
||||
}
|
||||
|
||||
// MarshallBencode returns the encoded byte slice of the message id.
|
||||
func (m messageID) MarshalBencode() ([]byte, error) {
|
||||
str := string(m[:])
|
||||
return bencode.EncodeBytes(str)
|
||||
}
|
||||
|
||||
func newMessageID() messageID {
|
||||
var m messageID
|
||||
_, err := rand.Read(m[:])
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// Request represents a DHT request message
|
||||
type Request struct {
|
||||
ID messageID
|
||||
NodeID bits.Bitmap
|
||||
Method string
|
||||
Arg *bits.Bitmap
|
||||
StoreArgs *storeArgs
|
||||
ProtocolVersion int
|
||||
}
|
||||
|
||||
// MarshalBencode returns the serialized byte slice representation of the request
|
||||
func (r Request) MarshalBencode() ([]byte, error) {
|
||||
var args interface{}
|
||||
if r.StoreArgs != nil {
|
||||
args = r.StoreArgs
|
||||
} else if r.Arg != nil {
|
||||
args = []bits.Bitmap{*r.Arg}
|
||||
} else {
|
||||
args = []string{} // request must always have keys 0-4, so we use an empty list for PING
|
||||
}
|
||||
return bencode.EncodeBytes(map[string]interface{}{
|
||||
headerTypeField: requestType,
|
||||
headerMessageIDField: r.ID,
|
||||
headerNodeIDField: r.NodeID,
|
||||
headerPayloadField: r.Method,
|
||||
headerArgsField: args,
|
||||
})
|
||||
}
|
||||
|
||||
// UnmarshalBencode unmarshals the serialized byte slice into the appropriate fields of the request.
|
||||
func (r *Request) UnmarshalBencode(b []byte) error {
|
||||
var raw struct {
|
||||
ID messageID `bencode:"1"`
|
||||
NodeID bits.Bitmap `bencode:"2"`
|
||||
Method string `bencode:"3"`
|
||||
Args bencode.RawMessage `bencode:"4"`
|
||||
}
|
||||
err := bencode.DecodeBytes(b, &raw)
|
||||
if err != nil {
|
||||
return errors.Prefix("request unmarshal", err)
|
||||
}
|
||||
|
||||
r.ID = raw.ID
|
||||
r.NodeID = raw.NodeID
|
||||
r.Method = raw.Method
|
||||
|
||||
if r.Method == storeMethod {
|
||||
r.StoreArgs = &storeArgs{} // bencode wont find the unmarshaler on a null pointer. need to fix it.
|
||||
err = bencode.DecodeBytes(raw.Args, &r.StoreArgs)
|
||||
if err != nil {
|
||||
return errors.Prefix("request unmarshal", err)
|
||||
}
|
||||
} else if len(raw.Args) > 2 { // 2 because an empty list is `le`
|
||||
r.Arg, r.ProtocolVersion, err = processArgsAndProtoVersion(raw.Args)
|
||||
if err != nil {
|
||||
return errors.Prefix("request unmarshal", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func processArgsAndProtoVersion(raw bencode.RawMessage) (arg *bits.Bitmap, version int, err error) {
|
||||
var args []bencode.RawMessage
|
||||
err = bencode.DecodeBytes(raw, &args)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
if len(args) == 0 {
|
||||
return nil, 0, nil
|
||||
}
|
||||
|
||||
var extras map[string]int
|
||||
err = bencode.DecodeBytes(args[len(args)-1], &extras)
|
||||
if err == nil {
|
||||
if v, exists := extras[protocolVersionField]; exists {
|
||||
version = v
|
||||
args = args[:len(args)-1]
|
||||
}
|
||||
}
|
||||
|
||||
if len(args) > 0 {
|
||||
var b bits.Bitmap
|
||||
err = bencode.DecodeBytes(args[0], &b)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
arg = &b
|
||||
}
|
||||
|
||||
return arg, version, nil
|
||||
}
|
||||
|
||||
func (r Request) argsDebug() string {
|
||||
if r.StoreArgs != nil {
|
||||
return r.StoreArgs.BlobHash.HexShort() + ", " + r.StoreArgs.Value.LbryID.HexShort() + ":" + strconv.Itoa(r.StoreArgs.Value.Port)
|
||||
} else if r.Arg != nil {
|
||||
return r.Arg.HexShort()
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type storeArgsValue struct {
|
||||
Token string `bencode:"token"`
|
||||
LbryID bits.Bitmap `bencode:"lbryid"`
|
||||
Port int `bencode:"port"`
|
||||
}
|
||||
|
||||
type storeArgs struct {
|
||||
BlobHash bits.Bitmap
|
||||
Value storeArgsValue
|
||||
NodeID bits.Bitmap // original publisher id? I think this is getting fixed in the new dht stuff
|
||||
SelfStore bool // this is an int on the wire
|
||||
}
|
||||
|
||||
// MarshalBencode returns the serialized byte slice representation of the storage arguments.
|
||||
func (s storeArgs) MarshalBencode() ([]byte, error) {
|
||||
encodedValue, err := bencode.EncodeString(s.Value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
selfStoreStr := 0
|
||||
if s.SelfStore {
|
||||
selfStoreStr = 1
|
||||
}
|
||||
|
||||
return bencode.EncodeBytes([]interface{}{
|
||||
s.BlobHash,
|
||||
bencode.RawMessage(encodedValue),
|
||||
s.NodeID,
|
||||
selfStoreStr,
|
||||
})
|
||||
}
|
||||
|
||||
// UnmarshalBencode unmarshals the serialized byte slice into the appropriate fields of the store arguments.
|
||||
func (s *storeArgs) UnmarshalBencode(b []byte) error {
|
||||
var argsInt []bencode.RawMessage
|
||||
err := bencode.DecodeBytes(b, &argsInt)
|
||||
if err != nil {
|
||||
return errors.Prefix("storeArgs unmarshal", err)
|
||||
}
|
||||
|
||||
if len(argsInt) != 4 {
|
||||
return errors.Err("unexpected number of fields for store args. got " + cast.ToString(len(argsInt)))
|
||||
}
|
||||
|
||||
err = bencode.DecodeBytes(argsInt[0], &s.BlobHash)
|
||||
if err != nil {
|
||||
return errors.Prefix("storeArgs unmarshal", err)
|
||||
}
|
||||
|
||||
err = bencode.DecodeBytes(argsInt[1], &s.Value)
|
||||
if err != nil {
|
||||
return errors.Prefix("storeArgs unmarshal", err)
|
||||
}
|
||||
|
||||
err = bencode.DecodeBytes(argsInt[2], &s.NodeID)
|
||||
if err != nil {
|
||||
return errors.Prefix("storeArgs unmarshal", err)
|
||||
}
|
||||
|
||||
var selfStore int
|
||||
err = bencode.DecodeBytes(argsInt[3], &selfStore)
|
||||
if err != nil {
|
||||
return errors.Prefix("storeArgs unmarshal", err)
|
||||
}
|
||||
if selfStore == 0 {
|
||||
s.SelfStore = false
|
||||
} else if selfStore == 1 {
|
||||
s.SelfStore = true
|
||||
} else {
|
||||
return errors.Err("selfstore must be 1 or 0")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Response represents a DHT response message
|
||||
type Response struct {
|
||||
ID messageID
|
||||
NodeID bits.Bitmap
|
||||
Data string
|
||||
Contacts []Contact
|
||||
FindValueKey string
|
||||
Token string
|
||||
ProtocolVersion int
|
||||
}
|
||||
|
||||
func (r Response) argsDebug() string {
|
||||
if r.Data != "" {
|
||||
return r.Data
|
||||
}
|
||||
|
||||
str := "contacts "
|
||||
if r.FindValueKey != "" {
|
||||
str = "value for " + hex.EncodeToString([]byte(r.FindValueKey))[:8] + " "
|
||||
}
|
||||
|
||||
str += "|"
|
||||
for _, c := range r.Contacts {
|
||||
str += c.String() + ","
|
||||
}
|
||||
str = strings.TrimRight(str, ",") + "|"
|
||||
|
||||
if r.Token != "" {
|
||||
str += " token: " + hex.EncodeToString([]byte(r.Token))[:8]
|
||||
}
|
||||
|
||||
return str
|
||||
}
|
||||
|
||||
// MarshalBencode returns the serialized byte slice representation of the response.
|
||||
func (r Response) MarshalBencode() ([]byte, error) {
|
||||
data := map[string]interface{}{
|
||||
headerTypeField: responseType,
|
||||
headerMessageIDField: r.ID,
|
||||
headerNodeIDField: r.NodeID,
|
||||
}
|
||||
|
||||
if r.Data != "" {
|
||||
// ping or store
|
||||
data[headerPayloadField] = r.Data
|
||||
} else if r.FindValueKey != "" {
|
||||
// findValue success
|
||||
if r.Token == "" {
|
||||
return nil, errors.Err("response to findValue must have a token")
|
||||
}
|
||||
|
||||
var contacts [][]byte
|
||||
for _, c := range r.Contacts {
|
||||
compact, err := c.MarshalCompact()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
contacts = append(contacts, compact)
|
||||
}
|
||||
data[headerPayloadField] = map[string]interface{}{
|
||||
r.FindValueKey: contacts,
|
||||
tokenField: r.Token,
|
||||
}
|
||||
} else if r.Token != "" {
|
||||
// findValue failure falling back to findNode
|
||||
data[headerPayloadField] = map[string]interface{}{
|
||||
contactsField: r.Contacts,
|
||||
tokenField: r.Token,
|
||||
}
|
||||
} else {
|
||||
// straight up findNode
|
||||
data[headerPayloadField] = r.Contacts
|
||||
}
|
||||
|
||||
return bencode.EncodeBytes(data)
|
||||
}
|
||||
|
||||
// UnmarshalBencode unmarshals the serialized byte slice into the appropriate fields of the store arguments.
|
||||
func (r *Response) UnmarshalBencode(b []byte) error {
|
||||
var raw struct {
|
||||
ID messageID `bencode:"1"`
|
||||
NodeID bits.Bitmap `bencode:"2"`
|
||||
Data bencode.RawMessage `bencode:"3"`
|
||||
}
|
||||
err := bencode.DecodeBytes(b, &raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
r.ID = raw.ID
|
||||
r.NodeID = raw.NodeID
|
||||
|
||||
// maybe data is a string (response to ping or store)?
|
||||
err = bencode.DecodeBytes(raw.Data, &r.Data)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// maybe data is a list of contacts (response to findNode)?
|
||||
err = bencode.DecodeBytes(raw.Data, &r.Contacts)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// it must be a response to findValue
|
||||
var rawData map[string]bencode.RawMessage
|
||||
err = bencode.DecodeBytes(raw.Data, &rawData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if token, ok := rawData[tokenField]; ok {
|
||||
err = bencode.DecodeBytes(token, &r.Token)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
delete(rawData, tokenField) // so it doesnt mess up findValue key finding below
|
||||
}
|
||||
|
||||
if protocolVersion, ok := rawData[protocolVersionField]; ok {
|
||||
err = bencode.DecodeBytes(protocolVersion, &r.ProtocolVersion)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
delete(rawData, protocolVersionField) // so it doesnt mess up findValue key finding below
|
||||
}
|
||||
|
||||
if contacts, ok := rawData[contactsField]; ok {
|
||||
err = bencode.DecodeBytes(contacts, &r.Contacts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
for k, v := range rawData {
|
||||
r.FindValueKey = k
|
||||
var compactContacts [][]byte
|
||||
err = bencode.DecodeBytes(v, &compactContacts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, compact := range compactContacts {
|
||||
var c Contact
|
||||
err = c.UnmarshalCompact(compact)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r.Contacts = append(r.Contacts, c)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Error represents a DHT error response
|
||||
type Error struct {
|
||||
ID messageID
|
||||
NodeID bits.Bitmap
|
||||
ExceptionType string
|
||||
Response []string
|
||||
}
|
||||
|
||||
// MarshalBencode returns the serialized byte slice representation of an error message.
|
||||
func (e Error) MarshalBencode() ([]byte, error) {
|
||||
return bencode.EncodeBytes(map[string]interface{}{
|
||||
headerTypeField: errorType,
|
||||
headerMessageIDField: e.ID,
|
||||
headerNodeIDField: e.NodeID,
|
||||
headerPayloadField: e.ExceptionType,
|
||||
headerArgsField: e.Response,
|
||||
})
|
||||
}
|
||||
|
||||
// UnmarshalBencode unmarshals the serialized byte slice into the appropriate fields of the error message.
|
||||
func (e *Error) UnmarshalBencode(b []byte) error {
|
||||
var raw struct {
|
||||
ID messageID `bencode:"1"`
|
||||
NodeID bits.Bitmap `bencode:"2"`
|
||||
ExceptionType string `bencode:"3"`
|
||||
Args interface{} `bencode:"4"`
|
||||
}
|
||||
err := bencode.DecodeBytes(b, &raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
e.ID = raw.ID
|
||||
e.NodeID = raw.NodeID
|
||||
e.ExceptionType = raw.ExceptionType
|
||||
|
||||
if reflect.TypeOf(raw.Args).Kind() == reflect.Slice {
|
||||
v := reflect.ValueOf(raw.Args)
|
||||
for i := 0; i < v.Len(); i++ {
|
||||
e.Response = append(e.Response, cast.ToString(v.Index(i).Interface()))
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,223 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"net"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
"github.com/lyoshenka/bencode"
|
||||
)
|
||||
|
||||
func TestBencodeDecodeStoreArgs(t *testing.T) {
|
||||
blobHash := "3214D6C2F77FCB5E8D5FC07EDAFBA614F031CE8B2EAB49F924F8143F6DFBADE048D918710072FB98AB1B52B58F4E1468"
|
||||
lbryID := "7CE1B831DEC8689E44F80F547D2DEA171F6A625E1A4FF6C6165E645F953103DABEB068A622203F859C6C64658FD3AA3B"
|
||||
port := hex.EncodeToString([]byte("3333"))
|
||||
token := "17C2D8E1E48EF21567FE4AD5C8ED944B798D3B65AB58D0C9122AD6587D1B5FED472EA2CB12284CEFA1C21EFF302322BD"
|
||||
nodeID := "7CE1B831DEC8689E44F80F547D2DEA171F6A625E1A4FF6C6165E645F953103DABEB068A622203F859C6C64658FD3AA3B"
|
||||
selfStore := hex.EncodeToString([]byte("1"))
|
||||
|
||||
raw := "6C" + // start args list
|
||||
"3438 3A " + blobHash + // blob hash
|
||||
"64" + // start value dict
|
||||
"363A6C6272796964 3438 3A " + lbryID + // lbry id
|
||||
"343A706F7274 69 " + port + " 65" + // port
|
||||
"353A746F6B656E 3438 3A " + token + // token
|
||||
"65" + // end value dict
|
||||
"3438 3A " + nodeID + // node id
|
||||
"69 " + selfStore + " 65" + // self store (integer)
|
||||
"65" // end args list
|
||||
|
||||
raw = strings.ToLower(strings.Replace(raw, " ", "", -1))
|
||||
|
||||
data, err := hex.DecodeString(raw)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
storeArgs := &storeArgs{}
|
||||
err = bencode.DecodeBytes(data, storeArgs)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
if storeArgs.BlobHash.Hex() != strings.ToLower(blobHash) {
|
||||
t.Error("blob hash mismatch")
|
||||
}
|
||||
if storeArgs.Value.LbryID.Hex() != strings.ToLower(lbryID) {
|
||||
t.Error("lbryid mismatch")
|
||||
}
|
||||
if hex.EncodeToString([]byte(strconv.Itoa(storeArgs.Value.Port))) != port {
|
||||
t.Error("port mismatch")
|
||||
}
|
||||
if hex.EncodeToString([]byte(storeArgs.Value.Token)) != strings.ToLower(token) {
|
||||
t.Error("token mismatch")
|
||||
}
|
||||
if storeArgs.NodeID.Hex() != strings.ToLower(nodeID) {
|
||||
t.Error("node id mismatch")
|
||||
}
|
||||
if !storeArgs.SelfStore {
|
||||
t.Error("selfStore mismatch")
|
||||
}
|
||||
|
||||
reencoded, err := bencode.EncodeBytes(storeArgs)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
} else if !reflect.DeepEqual(reencoded, data) {
|
||||
t.Error("reencoded data does not match original")
|
||||
spew.Dump(reencoded, data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBencodeFindNodesResponse(t *testing.T) {
|
||||
res := Response{
|
||||
ID: newMessageID(),
|
||||
NodeID: bits.Rand(),
|
||||
Contacts: []Contact{
|
||||
{ID: bits.Rand(), IP: net.IPv4(1, 2, 3, 4).To4(), Port: 5678},
|
||||
{ID: bits.Rand(), IP: net.IPv4(4, 3, 2, 1).To4(), Port: 8765},
|
||||
},
|
||||
}
|
||||
|
||||
encoded, err := bencode.EncodeBytes(res)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var res2 Response
|
||||
err = bencode.DecodeBytes(encoded, &res2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
compareResponses(t, res, res2)
|
||||
}
|
||||
|
||||
func TestBencodeFindValueResponse(t *testing.T) {
|
||||
res := Response{
|
||||
ID: newMessageID(),
|
||||
NodeID: bits.Rand(),
|
||||
FindValueKey: bits.Rand().RawString(),
|
||||
Token: "arstarstarst",
|
||||
Contacts: []Contact{
|
||||
{ID: bits.Rand(), IP: net.IPv4(1, 2, 3, 4).To4(), PeerPort: 8765},
|
||||
},
|
||||
}
|
||||
|
||||
encoded, err := bencode.EncodeBytes(res)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var res2 Response
|
||||
err = bencode.DecodeBytes(encoded, &res2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
compareResponses(t, res, res2)
|
||||
}
|
||||
|
||||
func TestDecodeSomeStrings(t *testing.T) {
|
||||
strs := []string{
|
||||
"6469306569306569316532303a359ec440b8236dee0fb2500ebda9c4704ae6741469326534383ade269943996b4bef3ff41176668a0577f86aba7f1ea2996edd18f9c42430802c8085331345c5f0c44a7f352e2ba8ae59693365383a66696e644e6f64656934656c34383ade269943996b4bef3ff41176668a0577f86aba7f1ea2996edd18f9c42430802c8085331345c5f0c44a7f352e2ba8ae596565",
|
||||
"64313a30693165313a3132303a31f27ec6174783c6f00e4b9ebccd49c6bdce300e313a3234383a7b3785cd962a56d4d4574262b1b63cc95d062c825b38f73602c7dc297c1e6b8259cdcdb5e7edef216d514e0c31ad8637313a3364353a746f6b656e34383a071fdf9835995889920e9beb876ebe8e36e5d44f5822684b549457405e22a5bf3fe40335743b459f22c347c0e736eba434383a7bcc7f8804db7d0d40fcb9c508547e53ee4f7d4000758105ecce099fc931d9cccfbbf9a30829777afcf41f2b4b0aeb716c35343a627db2b20d05b109195e7d06a855c392fda9a05faa878e69b18cc1212c35371ba5cad6231e049a460cdc263e7f4701afad45b288f48435343a424434ae0d05bfd6e1b98cc921b441fa8b560ac29038c37ec4907ae023617bf2bbc54abd3403226daf479798edcce69743ea6699e6ff35343a4462ba020d059ed3d31ceb3d1ffb2c025abf065a14516a8942607f9b3fad4b697ccc232766bdf471b6518a3ed716f2712171fd046c4335343a47413e2b0d05bf6244e5edacf43f65ef4cfd4b6eac380244894cc364ec23e8ededfb085999a211e20f8de146d9e081bbb42dc61d9ca035343a627db2b20d05b109195e7d06a855c392fda9a05faa878e69b18cc1212c35371ba5cad6231e049a460cdc263e7f4701afad45b288f48435343a4462ba020d059ed3d31ceb3d1ffb2c025abf065a14516a8942607f9b3fad4b697ccc232766bdf471b6518a3ed716f2712171fd046c4335343a47413e2b0d05bf6244e5edacf43f65ef4cfd4b6eac380244894cc364ec23e8ededfb085999a211e20f8de146d9e081bbb42dc61d9ca035343a627db2b20d05b109195e7d06a855c392fda9a05faa878e69b18cc1212c35371ba5cad6231e049a460cdc263e7f4701afad45b288f48435343a68fe5c3e0d05bfd6e1b98cc921b441fa8b560ac29038c37ec4907ae023617bf2bbc54abd3403226daf479798edcce69743ea6699e6ff35343a4462ba020d059ed3d31ceb3d1ffb2c025abf065a14516a8942607f9b3fad4b697ccc232766bdf471b6518a3ed716f2712171fd046c4335343a47413e2b0d05bf6244e5edacf43f65ef4cfd4b6eac380244894cc364ec23e8ededfb085999a211e20f8de146d9e081bbb42dc61d9ca035343a67d9a7ea0d05223c42ef5979c7c8f2444311f7a8266d64d3f98e9d0b21726bc2714a6fea5426c9a46a68c49ea13b54ccc324adf0783035343a68fe5c3e0d05bfd6e1b98cc921b441fa8b560ac29038c37ec4907ae023617bf2bbc54abd3403226daf479798edcce69743ea6699e6ff35343a47413e2b0d05bf6244e5edacf43f65ef4cfd4b6eac380244894cc364ec23e8ededfb085999a211e20f8de146d9e081bbb42dc61d9ca035343a4462ba020d059ed3d31ceb3d1ffb2c025abf065a14516a8942607f9b3fad4b697ccc232766bdf471b6518a3ed716f2712171fd046c4335343a67d9a7ea0d05223c42ef5979c7c8f2444311f7a8266d64d3f98e9d0b21726bc2714a6fea5426c9a46a68c49ea13b54ccc324adf0783035343a424434ae0d05bfd6e1b98cc921b441fa8b560ac29038c37ec4907ae023617bf2bbc54abd3403226daf479798edcce69743ea6699e6ff35343a4ace39c80d0541ae66325db4f1fce530476942d339f3219913f0a237726e18746a65ff7b3c5558c82c274a6f1ed10bbaf7c1305ca8e635343a47413e2b0d05bf6244e5edacf43f65ef4cfd4b6eac380244894cc364ec23e8ededfb085999a211e20f8de146d9e081bbb42dc61d9ca035343a4462ba020d059ed3d31ceb3d1ffb2c025abf065a14516a8942607f9b3fad4b697ccc232766bdf471b6518a3ed716f2712171fd046c4335343a627db2b20d05b109195e7d06a855c392fda9a05faa878e69b18cc1212c35371ba5cad6231e049a460cdc263e7f4701afad45b288f48435343a424434ae0d05bfd6e1b98cc921b441fa8b560ac29038c37ec4907ae023617bf2bbc54abd3403226daf479798edcce69743ea6699e6ff35343a4ace39c80d0541ae66325db4f1fce530476942d339f3219913f0a237726e18746a65ff7b3c5558c82c274a6f1ed10bbaf7c1305ca8e635343a47413e2b0d05bf6244e5edacf43f65ef4cfd4b6eac380244894cc364ec23e8ededfb085999a211e20f8de146d9e081bbb42dc61d9ca035343a4462ba020d059ed3d31ceb3d1ffb2c025abf065a14516a8942607f9b3fad4b697ccc232766bdf471b6518a3ed716f2712171fd046c4335343a67d9a7ea0d05223c42ef5979c7c8f2444311f7a8266d64d3f98e9d0b21726bc2714a6fea5426c9a46a68c49ea13b54ccc324adf0783035343a4ace39c80d0541ae66325db4f1fce530476942d339f3219913f0a237726e18746a65ff7b3c5558c82c274a6f1ed10bbaf7c1305ca8e635343a47413e2b0d05bf6244e5edacf43f65ef4cfd4b6eac380244894cc364ec23e8ededfb085999a211e20f8de146d9e081bbb42dc61d9ca035343a4462ba020d059ed3d31ceb3d1ffb2c025abf065a14516a8942607f9b3fad4b697ccc232766bdf471b6518a3ed716f2712171fd046c4335343a67d9a7ea0d05223c42ef5979c7c8f2444311f7a8266d64d3f98e9d0b21726bc2714a6fea5426c9a46a68c49ea13b54ccc324adf0783035343a68fe5c3e0d05bfd6e1b98cc921b441fa8b560ac29038c37ec4907ae023617bf2bbc54abd3403226daf479798edcce69743ea6699e6ff35343a4ace39c80d0541ae66325db4f1fce530476942d339f3219913f0a237726e18746a65ff7b3c5558c82c274a6f1ed10bbaf7c1305ca8e635343a47413e2b0d05bf6244e5edacf43f65ef4cfd4b6eac380244894cc364ec23e8ededfb085999a211e20f8de146d9e081bbb42dc61d9ca035343a68fe5c3e0d05bfd6e1b98cc921b441fa8b560ac29038c37ec4907ae023617bf2bbc54abd3403226daf479798edcce69743ea6699e6ff35343a4ace39c80d0541ae66325db4f1fce530476942d339f3219913f0a237726e18746a65ff7b3c5558c82c274a6f1ed10bbaf7c1305ca8e635343a47413e2b0d05bf6244e5edacf43f65ef4cfd4b6eac380244894cc364ec23e8ededfb085999a211e20f8de146d9e081bbb42dc61d9ca035343a4462ba020d059ed3d31ceb3d1ffb2c025abf065a14516a8942607f9b3fad4b697ccc232766bdf471b6518a3ed716f2712171fd046c4335343a67d9a7ea0d05223c42ef5979c7c8f2444311f7a8266d64d3f98e9d0b21726bc2714a6fea5426c9a46a68c49ea13b54ccc324adf0783035343a68fe5c3e0d05bfd6e1b98cc921b441fa8b560ac29038c37ec4907ae023617bf2bbc54abd3403226daf479798edcce69743ea6699e6ff35343a4ace39c80d0541ae66325db4f1fce530476942d339f3219913f0a237726e18746a65ff7b3c5558c82c274a6f1ed10bbaf7c1305ca8e635343a47413e2b0d05bf6244e5edacf43fLine truncated
|
||||
"64313a30693165313a3132303aa19e41f1d887db705fbf0a58f120773069d6ff11313a3234383a6911c21c3b5ea6536f4fb170c87fdc8bf4201124c5fe5eeb5f0054ff48e899a1d6e089a30a12ba8683ebf79691d71439313a3364383a636f6e74616374736c6c34383a6dd3dedeec334bae70a3c5c1b58fef2d4a501af320a332cdd5db4b71945c0a90d11aa2013e8d0216258a643f42d9562631323a34362e32382e3230342e3738693434343465656c34383a7ec0edf204d940d21c17e1c979df1c94610711d7773ceda72a4aa925c5f7799bbb741caa8d6d877aa2776399f77c08a031333a36392e3131382e34332e313234693434343465656c34383a6ec15cd8b95718d010bdf9739fda1af9ded7ac8be7da5cced6dc7a08470c8109accc5bfb973e5c1dd711349c62a8a3c331323a34362e32382e3230342e3738693434343465656c34383a6d2603a690712d5a3d5addb228d31a622f798c64ab2d3576d2c3ab3c5a64cb863c05545f37aa384ce83003416ca5cea431323a34362e32382e3230342e3738693434343465656c34383a6c1d92202a7e74ec85dd4e3e1699303174c4b6460b171637bba0a1c068d72f389035e1ba8bbe70f6621c36d70d4d045031323a34362e32382e3230342e3738693434343465656c34383a6dac30fbe94c006515cffe4898bc534a105fd0403bfc3c8692c2751b7946949dbc9e0c9570477cf8f817d1efc9a0e02531343a31382e3232312e3139342e313933693434343465656c34383a6f2ee379c662ea9f6db86beed6023be300cd23b9f9c4610a195dcb0f258392ec77188854ea3a8c0fffe67844849eb6ca31333a32342e3131362e39302e323238693434343465656c34383a6c55b2dbb89010de9db88e0ad3510a79fe60983b394a9e66b0efe139a455973cdc4415beff82cbdfb63d98caad648ab031333a38382e3132392e3234352e33396934343434656565353a746f6b656e34383a51844b0d1e8a613c4ad783c03b323063d3a5ff063a640368d7754bcae277c22b2b06b46c3e5466c30acb773f77b686476565",
|
||||
"64313a30693165313a3132303af693334af099a987a4b55af2727472b51c1990db313a3234383ab6928ff25778a7bbb5d258d3b3a06e26db1654f3d2efce8c26681d43f7237cdf2e359a4d309c4473d5d89ec99fb4f573313a3364353a746f6b656e34383a61a45a40080ea118361f919ebf09cdc9d2c63f476e7bc4d193463f52b91dc4b0db4ed1004e32d4fc273645421e58479034383abe463e24af015c0d1ba496b4b1911aa98c8b42119295bbcc5c36829a2e2ff8f149743cbbc0af2c39669cb0750fd778576c35343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e95656565",
|
||||
"6469306569316569316532303a3e0929c88abb8fe1d718025efc7f4a3cd85de16269326534383a21b2e2d2996b4bef3ff41176668a0577f86aba7f1ea2996edd18f9c42430802c8085331345c5f0c44a7f352e2ba8ae596933656c6565",
|
||||
}
|
||||
|
||||
for i, str := range strs {
|
||||
raw, err := hex.DecodeString(str)
|
||||
if err != nil {
|
||||
t.Errorf("error hex-decoding string %d: %s", i, err)
|
||||
continue
|
||||
}
|
||||
|
||||
var decoded interface{}
|
||||
err = bencode.DecodeBytes(raw, &decoded)
|
||||
if err != nil {
|
||||
t.Errorf("error bencode-decoding string %d: %s", i, err)
|
||||
continue
|
||||
}
|
||||
//
|
||||
//t.Error("TODO")
|
||||
//continue
|
||||
//
|
||||
//spew.Dump(decoded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeFindNodeResponseWithNoNodes(t *testing.T) {
|
||||
raw, err := hex.DecodeString("6469306569316569316532303a3e0929c88abb8fe1d718025efc7f4a3cd85de16269326534383a21b2e2d2996b4bef3ff41176668a0577f86aba7f1ea2996edd18f9c42430802c8085331345c5f0c44a7f352e2ba8ae596933656c6565")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
response := Response{}
|
||||
err = bencode.DecodeBytes(raw, &response)
|
||||
//spew.Dump(response)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeRequestWithProtocolVersion(t *testing.T) {
|
||||
raw, err := hex.DecodeString("6469306569306569316532303a65e1b9afce87c44abc40b4bb466890d2797f0dd269326534383a1baf3dbba8746c7a739f35465e268ace823622c5e8ec1dd7d5d27af5795cfbc54f22a32fbd05d420f241266b3dd16831693365343a70696e676934656c6431353a70726f746f636f6c56657273696f6e693165656565")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
req := Request{}
|
||||
err = bencode.DecodeBytes(raw, &req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if req.ProtocolVersion != 1 {
|
||||
t.Error("protocol version was not detected correctly")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeResponseWithProtocolVersion(t *testing.T) {
|
||||
raw, err := hex.DecodeString("6469306569316569316532303a2b96f5de8be1e86dc2332a50eb313c97848064db69326534383a8b8eb692658ea3d7e7828e80a3133d524c6f82aaff370efa759d5b87821035a32de06724cb099f01a819695f829dff7f6933656431353a70726f746f636f6c56657273696f6e693165353a746f6b656e34383af2368fe4fd06ede6631ad8b153f3b5d8db724f8d520f4291e992e206dd02a216fb7dfd9b81686f11b626e3840df65fb434383a89c5c3f9794b0b24a03406e3b74361edb9ae70828e4c133512fc75db0a2d312673cdd4e30eed37892a46692d2fe439f36c35343a12dd65af0d058cd7d10d122fbe2eb5ae31062b7480011be588f20cfe6807b1939c42eea639059fa6365bfccb56ef8c9e574f49ba035c656565")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
res := Response{}
|
||||
err = bencode.DecodeBytes(raw, &res)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if res.ProtocolVersion != 1 {
|
||||
t.Error("protocol version was not detected correctly")
|
||||
}
|
||||
}
|
||||
|
||||
func compareResponses(t *testing.T, res, res2 Response) {
|
||||
if res.ID != res2.ID {
|
||||
t.Errorf("expected ID %s, got %s", res.ID, res2.ID)
|
||||
}
|
||||
if !res.NodeID.Equals(res2.NodeID) {
|
||||
t.Errorf("expected NodeID %s, got %s", res.NodeID.Hex(), res2.NodeID.Hex())
|
||||
}
|
||||
if res.Data != res2.Data {
|
||||
t.Errorf("expected Data %s, got %s", res.Data, res2.Data)
|
||||
}
|
||||
if res.FindValueKey != res2.FindValueKey {
|
||||
t.Errorf("expected FindValueKey %s, got %s", res.FindValueKey, res2.FindValueKey)
|
||||
}
|
||||
if res.Token != res2.Token {
|
||||
t.Errorf("expected Token %s, got %s", res.Token, res2.Token)
|
||||
}
|
||||
if !reflect.DeepEqual(res.Contacts, res2.Contacts) {
|
||||
t.Errorf("expected FindNodeData %s, got %s", spew.Sdump(res.Contacts), spew.Sdump(res2.Contacts))
|
||||
}
|
||||
}
|
||||
-474
@@ -1,474 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/errors.go"
|
||||
"github.com/lbryio/lbry.go/stop"
|
||||
"github.com/lbryio/lbry.go/util"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/lyoshenka/bencode"
|
||||
)
|
||||
|
||||
// packet represents the information receive from udp.
|
||||
type packet struct {
|
||||
data []byte
|
||||
raddr *net.UDPAddr
|
||||
}
|
||||
|
||||
// UDPConn allows using a mocked connection to test sending/receiving data
|
||||
// TODO: stop mocking this and use the real thing
|
||||
type UDPConn interface {
|
||||
ReadFromUDP([]byte) (int, *net.UDPAddr, error)
|
||||
WriteToUDP([]byte, *net.UDPAddr) (int, error)
|
||||
SetReadDeadline(time.Time) error
|
||||
SetWriteDeadline(time.Time) error
|
||||
Close() error
|
||||
}
|
||||
|
||||
// RequestHandlerFunc is exported handler for requests.
|
||||
type RequestHandlerFunc func(addr *net.UDPAddr, request Request)
|
||||
|
||||
// Node is a type representation of a node on the network.
|
||||
type Node struct {
|
||||
// the node's id
|
||||
id bits.Bitmap
|
||||
// UDP connection for sending and receiving data
|
||||
conn UDPConn
|
||||
// true if we've closed the connection on purpose
|
||||
connClosed bool
|
||||
// token manager
|
||||
tokens *tokenManager
|
||||
|
||||
// map of outstanding transactions + mutex
|
||||
txLock *sync.RWMutex
|
||||
transactions map[messageID]*transaction
|
||||
|
||||
// routing table
|
||||
rt *routingTable
|
||||
// data store
|
||||
store *contactStore
|
||||
|
||||
// overrides for request handlers
|
||||
requestHandler RequestHandlerFunc
|
||||
|
||||
// stop the node neatly and clean up after itself
|
||||
grp *stop.Group
|
||||
}
|
||||
|
||||
// NewNode returns an initialized Node's pointer.
|
||||
func NewNode(id bits.Bitmap) *Node {
|
||||
return &Node{
|
||||
id: id,
|
||||
rt: newRoutingTable(id),
|
||||
store: newStore(),
|
||||
|
||||
txLock: &sync.RWMutex{},
|
||||
transactions: make(map[messageID]*transaction),
|
||||
|
||||
grp: stop.New(),
|
||||
tokens: &tokenManager{},
|
||||
}
|
||||
}
|
||||
|
||||
// Connect connects to the given connection and starts any background threads necessary
|
||||
func (n *Node) Connect(conn UDPConn) error {
|
||||
n.conn = conn
|
||||
|
||||
n.tokens.Start(tokenSecretRotationInterval)
|
||||
|
||||
go func() {
|
||||
// stop tokens and close the connection when we're shutting down
|
||||
<-n.grp.Ch()
|
||||
n.tokens.Stop()
|
||||
n.connClosed = true
|
||||
err := n.conn.Close()
|
||||
if err != nil {
|
||||
log.Error("error closing node connection on shutdown - ", err)
|
||||
}
|
||||
}()
|
||||
|
||||
packets := make(chan packet)
|
||||
|
||||
n.grp.Add(1)
|
||||
go func() {
|
||||
defer n.grp.Done()
|
||||
|
||||
buf := make([]byte, udpMaxMessageLength)
|
||||
|
||||
for {
|
||||
bytesRead, raddr, err := n.conn.ReadFromUDP(buf)
|
||||
if err != nil {
|
||||
if n.connClosed {
|
||||
return
|
||||
}
|
||||
log.Errorf("udp read error: %v", err)
|
||||
continue
|
||||
} else if raddr == nil {
|
||||
log.Errorf("udp read with no raddr")
|
||||
continue
|
||||
}
|
||||
|
||||
data := make([]byte, bytesRead)
|
||||
copy(data, buf[:bytesRead]) // slices use the same underlying array, so we need a new one for each packet
|
||||
|
||||
select { // needs select here because packet consumer can quit and the packets channel gets filled up and blocks
|
||||
case packets <- packet{data: data, raddr: raddr}:
|
||||
case <-n.grp.Ch():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
n.grp.Add(1)
|
||||
go func() {
|
||||
defer n.grp.Done()
|
||||
|
||||
var pkt packet
|
||||
|
||||
for {
|
||||
select {
|
||||
case pkt = <-packets:
|
||||
n.handlePacket(pkt)
|
||||
case <-n.grp.Ch():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// TODO: turn this back on when you're sure it works right
|
||||
n.grp.Add(1)
|
||||
go func() {
|
||||
defer n.grp.Done()
|
||||
n.startRoutingTableGrooming()
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Shutdown shuts down the node
|
||||
func (n *Node) Shutdown() {
|
||||
log.Debugf("[%s] node shutting down", n.id.HexShort())
|
||||
n.grp.StopAndWait()
|
||||
log.Debugf("[%s] node stopped", n.id.HexShort())
|
||||
}
|
||||
|
||||
// handlePacket handles packets received from udp.
|
||||
func (n *Node) handlePacket(pkt packet) {
|
||||
//log.Debugf("[%s] Received message from %s (%d bytes) %s", n.id.HexShort(), pkt.raddr.String(), len(pkt.data), hex.EncodeToString(pkt.data))
|
||||
|
||||
if !util.InSlice(string(pkt.data[0:5]), []string{"d1:0i", "di0ei"}) {
|
||||
log.Errorf("[%s] data is not a well-formatted dict: (%d bytes) %s", n.id.HexShort(), len(pkt.data), hex.EncodeToString(pkt.data))
|
||||
return
|
||||
}
|
||||
|
||||
// the following is a bit of a hack, but it lets us avoid decoding every message twice
|
||||
// it depends on the data being a dict with 0 as the first key (so it starts with "d1:0i") and the message type as the first value
|
||||
// TODO: test this more thoroughly
|
||||
|
||||
switch pkt.data[5] {
|
||||
case '0' + requestType:
|
||||
request := Request{}
|
||||
err := bencode.DecodeBytes(pkt.data, &request)
|
||||
if err != nil {
|
||||
log.Errorf("[%s] error decoding request from %s: %s: (%d bytes) %s", n.id.HexShort(), pkt.raddr.String(), err.Error(), len(pkt.data), hex.EncodeToString(pkt.data))
|
||||
return
|
||||
}
|
||||
log.Debugf("[%s] query %s: received request from %s: %s(%s)", n.id.HexShort(), request.ID.HexShort(), request.NodeID.HexShort(), request.Method, request.argsDebug())
|
||||
n.handleRequest(pkt.raddr, request)
|
||||
|
||||
case '0' + responseType:
|
||||
response := Response{}
|
||||
err := bencode.DecodeBytes(pkt.data, &response)
|
||||
if err != nil {
|
||||
log.Errorf("[%s] error decoding response from %s: %s: (%d bytes) %s", n.id.HexShort(), pkt.raddr.String(), err.Error(), len(pkt.data), hex.EncodeToString(pkt.data))
|
||||
return
|
||||
}
|
||||
log.Debugf("[%s] query %s: received response from %s: %s", n.id.HexShort(), response.ID.HexShort(), response.NodeID.HexShort(), response.argsDebug())
|
||||
n.handleResponse(pkt.raddr, response)
|
||||
|
||||
case '0' + errorType:
|
||||
e := Error{}
|
||||
err := bencode.DecodeBytes(pkt.data, &e)
|
||||
if err != nil {
|
||||
log.Errorf("[%s] error decoding error from %s: %s: (%d bytes) %s", n.id.HexShort(), pkt.raddr.String(), err.Error(), len(pkt.data), hex.EncodeToString(pkt.data))
|
||||
return
|
||||
}
|
||||
log.Debugf("[%s] query %s: received error from %s: %s", n.id.HexShort(), e.ID.HexShort(), e.NodeID.HexShort(), e.ExceptionType)
|
||||
n.handleError(pkt.raddr, e)
|
||||
|
||||
default:
|
||||
log.Errorf("[%s] invalid message type: %s", n.id.HexShort(), string(pkt.data[5]))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// handleRequest handles the requests received from udp.
|
||||
func (n *Node) handleRequest(addr *net.UDPAddr, request Request) {
|
||||
if request.NodeID.Equals(n.id) {
|
||||
log.Warn("ignoring self-request")
|
||||
return
|
||||
}
|
||||
|
||||
// if a handler is overridden, call it instead
|
||||
if n.requestHandler != nil {
|
||||
n.requestHandler(addr, request)
|
||||
return
|
||||
}
|
||||
|
||||
switch request.Method {
|
||||
default:
|
||||
//n.sendMessage(addr, Error{ID: request.ID, NodeID: n.id, ExceptionType: "invalid-request-method"})
|
||||
log.Errorln("invalid request method")
|
||||
return
|
||||
case pingMethod:
|
||||
err := n.sendMessage(addr, Response{ID: request.ID, NodeID: n.id, Data: pingSuccessResponse})
|
||||
if err != nil {
|
||||
log.Error("error sending 'pingmethod' response message - ", err)
|
||||
}
|
||||
case storeMethod:
|
||||
// TODO: we should be sending the IP in the request, not just using the sender's IP
|
||||
// TODO: should we be using StoreArgs.NodeID or StoreArgs.Value.LbryID ???
|
||||
if n.tokens.Verify(request.StoreArgs.Value.Token, request.NodeID, addr) {
|
||||
n.Store(request.StoreArgs.BlobHash, Contact{ID: request.StoreArgs.NodeID, IP: addr.IP, Port: addr.Port, PeerPort: request.StoreArgs.Value.Port})
|
||||
|
||||
err := n.sendMessage(addr, Response{ID: request.ID, NodeID: n.id, Data: storeSuccessResponse})
|
||||
if err != nil {
|
||||
log.Error("error sending 'storemethod' response message - ", err)
|
||||
}
|
||||
} else {
|
||||
err := n.sendMessage(addr, Error{ID: request.ID, NodeID: n.id, ExceptionType: "invalid-token"})
|
||||
if err != nil {
|
||||
log.Error("error sending 'storemethod'response message for invalid-token - ", err)
|
||||
}
|
||||
}
|
||||
case findNodeMethod:
|
||||
if request.Arg == nil {
|
||||
log.Errorln("request is missing arg")
|
||||
return
|
||||
}
|
||||
err := n.sendMessage(addr, Response{
|
||||
ID: request.ID,
|
||||
NodeID: n.id,
|
||||
Contacts: n.rt.GetClosest(*request.Arg, bucketSize),
|
||||
})
|
||||
if err != nil {
|
||||
log.Error("error sending 'findnodemethod' response message - ", err)
|
||||
}
|
||||
|
||||
case findValueMethod:
|
||||
if request.Arg == nil {
|
||||
log.Errorln("request is missing arg")
|
||||
return
|
||||
}
|
||||
|
||||
res := Response{
|
||||
ID: request.ID,
|
||||
NodeID: n.id,
|
||||
Token: n.tokens.Get(request.NodeID, addr),
|
||||
}
|
||||
|
||||
if contacts := n.store.Get(*request.Arg); len(contacts) > 0 {
|
||||
res.FindValueKey = request.Arg.RawString()
|
||||
res.Contacts = contacts
|
||||
} else {
|
||||
res.Contacts = n.rt.GetClosest(*request.Arg, bucketSize)
|
||||
}
|
||||
|
||||
err := n.sendMessage(addr, res)
|
||||
if err != nil {
|
||||
log.Error("error sending 'findvaluemethod' response message - ", err)
|
||||
}
|
||||
}
|
||||
|
||||
// nodes that send us requests should not be inserted, only refreshed.
|
||||
// the routing table must only contain "good" nodes, which are nodes that reply to our requests
|
||||
// if a node is already good (aka in the table), its fine to refresh it
|
||||
// http://www.bittorrent.org/beps/bep_0005.html#routing-table
|
||||
n.rt.Fresh(Contact{ID: request.NodeID, IP: addr.IP, Port: addr.Port})
|
||||
}
|
||||
|
||||
// handleResponse handles responses received from udp.
|
||||
func (n *Node) handleResponse(addr *net.UDPAddr, response Response) {
|
||||
tx := n.txFind(response.ID, Contact{ID: response.NodeID, IP: addr.IP, Port: addr.Port})
|
||||
if tx != nil {
|
||||
select {
|
||||
case tx.res <- response:
|
||||
default:
|
||||
//log.Errorf("[%s] query %s: response received, but tx has no listener or multiple responses to the same tx", n.id.HexShort(), response.ID.HexShort())
|
||||
}
|
||||
}
|
||||
|
||||
n.rt.Update(Contact{ID: response.NodeID, IP: addr.IP, Port: addr.Port})
|
||||
}
|
||||
|
||||
// handleError handles errors received from udp.
|
||||
func (n *Node) handleError(addr *net.UDPAddr, e Error) {
|
||||
spew.Dump(e)
|
||||
n.rt.Fresh(Contact{ID: e.NodeID, IP: addr.IP, Port: addr.Port})
|
||||
}
|
||||
|
||||
// send sends data to a udp address
|
||||
func (n *Node) sendMessage(addr *net.UDPAddr, data Message) error {
|
||||
encoded, err := bencode.EncodeBytes(data)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
if req, ok := data.(Request); ok {
|
||||
log.Debugf("[%s] query %s: sending request to %s (%d bytes) %s(%s)",
|
||||
n.id.HexShort(), req.ID.HexShort(), addr.String(), len(encoded), req.Method, req.argsDebug())
|
||||
} else if res, ok := data.(Response); ok {
|
||||
log.Debugf("[%s] query %s: sending response to %s (%d bytes) %s",
|
||||
n.id.HexShort(), res.ID.HexShort(), addr.String(), len(encoded), res.argsDebug())
|
||||
} else {
|
||||
log.Debugf("[%s] (%d bytes) %s", n.id.HexShort(), len(encoded), spew.Sdump(data))
|
||||
}
|
||||
|
||||
err = n.conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
|
||||
if err != nil {
|
||||
if n.connClosed {
|
||||
return nil
|
||||
}
|
||||
log.Error("error setting write deadline - ", err)
|
||||
}
|
||||
|
||||
_, err = n.conn.WriteToUDP(encoded, addr)
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
// transaction represents a single query to the dht. it stores the queried contact, the request, and the response channel
|
||||
type transaction struct {
|
||||
contact Contact
|
||||
req Request
|
||||
res chan Response
|
||||
skipIDCheck bool
|
||||
}
|
||||
|
||||
// insert adds a transaction to the manager.
|
||||
func (n *Node) txInsert(tx *transaction) {
|
||||
n.txLock.Lock()
|
||||
defer n.txLock.Unlock()
|
||||
n.transactions[tx.req.ID] = tx
|
||||
}
|
||||
|
||||
// delete removes a transaction from the manager.
|
||||
func (n *Node) txDelete(id messageID) {
|
||||
n.txLock.Lock()
|
||||
defer n.txLock.Unlock()
|
||||
delete(n.transactions, id)
|
||||
}
|
||||
|
||||
// Find finds a transaction for the given id and contact
|
||||
func (n *Node) txFind(id messageID, c Contact) *transaction {
|
||||
n.txLock.RLock()
|
||||
defer n.txLock.RUnlock()
|
||||
|
||||
t, ok := n.transactions[id]
|
||||
if !ok || !t.contact.Equals(c, !t.skipIDCheck) {
|
||||
return nil
|
||||
}
|
||||
|
||||
return t
|
||||
}
|
||||
|
||||
// SendOptions controls the behavior of send calls
|
||||
type SendOptions struct {
|
||||
skipIDCheck bool
|
||||
}
|
||||
|
||||
// SendAsync sends a transaction and returns a channel that will eventually contain the transaction response
|
||||
// The response channel is closed when the transaction is completed or times out.
|
||||
func (n *Node) SendAsync(contact Contact, req Request, options ...SendOptions) <-chan *Response {
|
||||
ch := make(chan *Response, 1)
|
||||
|
||||
if contact.ID.Equals(n.id) {
|
||||
log.Error("sending query to self")
|
||||
close(ch)
|
||||
return ch
|
||||
}
|
||||
|
||||
go func() {
|
||||
defer close(ch)
|
||||
|
||||
req.ID = newMessageID()
|
||||
req.NodeID = n.id
|
||||
tx := &transaction{
|
||||
contact: contact,
|
||||
req: req,
|
||||
res: make(chan Response),
|
||||
}
|
||||
|
||||
if len(options) > 0 && options[0].skipIDCheck {
|
||||
tx.skipIDCheck = true
|
||||
}
|
||||
|
||||
n.txInsert(tx)
|
||||
defer n.txDelete(tx.req.ID)
|
||||
|
||||
for i := 0; i < udpRetry; i++ {
|
||||
err := n.sendMessage(contact.Addr(), tx.req)
|
||||
if err != nil {
|
||||
if !strings.Contains(err.Error(), "use of closed network connection") { // this only happens on localhost. real UDP has no connections
|
||||
log.Error("send error: ", err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
select {
|
||||
case res := <-tx.res:
|
||||
ch <- &res
|
||||
return
|
||||
case <-n.grp.Ch():
|
||||
return
|
||||
case <-time.After(udpTimeout):
|
||||
}
|
||||
}
|
||||
|
||||
// notify routing table about a failure to respond
|
||||
n.rt.Fail(tx.contact)
|
||||
}()
|
||||
|
||||
return ch
|
||||
}
|
||||
|
||||
// Send sends a transaction and blocks until the response is available. It returns a response, or nil
|
||||
// if the transaction timed out.
|
||||
func (n *Node) Send(contact Contact, req Request, options ...SendOptions) *Response {
|
||||
return <-n.SendAsync(contact, req, options...)
|
||||
}
|
||||
|
||||
// CountActiveTransactions returns the number of transactions in the manager
|
||||
func (n *Node) CountActiveTransactions() int {
|
||||
n.txLock.Lock()
|
||||
defer n.txLock.Unlock()
|
||||
return len(n.transactions)
|
||||
}
|
||||
|
||||
func (n *Node) startRoutingTableGrooming() {
|
||||
refreshTicker := time.NewTicker(tRefresh / 5) // how often to check for buckets that need to be refreshed
|
||||
for {
|
||||
select {
|
||||
case <-refreshTicker.C:
|
||||
RoutingTableRefresh(n, tRefresh, n.grp.Child())
|
||||
case <-n.grp.Ch():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Store stores a node contact in the node's contact store.
|
||||
func (n *Node) Store(hash bits.Bitmap, c Contact) {
|
||||
n.store.Upsert(hash, c)
|
||||
}
|
||||
|
||||
//AddKnownNode adds a known-good node to the routing table
|
||||
func (n *Node) AddKnownNode(c Contact) {
|
||||
n.rt.Update(c)
|
||||
}
|
||||
@@ -1,338 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/crypto"
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/lbry.go/stop"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/uber-go/atomic"
|
||||
)
|
||||
|
||||
// TODO: iterativeFindValue may be stopping early. if it gets a response with one peer, it should keep going because other nodes may know about more peers that have that blob
|
||||
// TODO: or, it should try a tcp handshake with peers as it finds them, to make sure they are still online and have the blob
|
||||
|
||||
var cfLog *logrus.Logger
|
||||
|
||||
func init() {
|
||||
cfLog = logrus.StandardLogger()
|
||||
}
|
||||
|
||||
func NodeFinderUseLogger(l *logrus.Logger) {
|
||||
cfLog = l
|
||||
}
|
||||
|
||||
type contactFinder struct {
|
||||
findValue bool // true if we're using findValue
|
||||
target bits.Bitmap
|
||||
node *Node
|
||||
|
||||
grp *stop.Group
|
||||
|
||||
findValueMutex *sync.Mutex
|
||||
findValueResult []Contact
|
||||
|
||||
activeContactsMutex *sync.Mutex
|
||||
activeContacts []Contact
|
||||
|
||||
shortlistMutex *sync.Mutex
|
||||
shortlist []Contact
|
||||
shortlistAdded map[bits.Bitmap]bool
|
||||
|
||||
closestContactMutex *sync.RWMutex
|
||||
closestContact *Contact
|
||||
notGettingCloser *atomic.Bool
|
||||
}
|
||||
|
||||
func FindContacts(node *Node, target bits.Bitmap, findValue bool, parentGrp *stop.Group) ([]Contact, bool, error) {
|
||||
cf := &contactFinder{
|
||||
node: node,
|
||||
target: target,
|
||||
findValue: findValue,
|
||||
findValueMutex: &sync.Mutex{},
|
||||
activeContactsMutex: &sync.Mutex{},
|
||||
shortlistMutex: &sync.Mutex{},
|
||||
shortlistAdded: make(map[bits.Bitmap]bool),
|
||||
grp: stop.New(parentGrp),
|
||||
closestContactMutex: &sync.RWMutex{},
|
||||
notGettingCloser: atomic.NewBool(false),
|
||||
}
|
||||
|
||||
return cf.Find()
|
||||
}
|
||||
|
||||
func (cf *contactFinder) Stop() {
|
||||
cf.grp.StopAndWait()
|
||||
}
|
||||
|
||||
func (cf *contactFinder) Find() ([]Contact, bool, error) {
|
||||
if cf.findValue {
|
||||
cf.debug("starting iterativeFindValue")
|
||||
} else {
|
||||
cf.debug("starting iterativeFindNode")
|
||||
}
|
||||
|
||||
cf.appendNewToShortlist(cf.node.rt.GetClosest(cf.target, alpha))
|
||||
if len(cf.shortlist) == 0 {
|
||||
return nil, false, errors.Err("[%s] find %s: no contacts in routing table", cf.node.id.HexShort(), cf.target.HexShort())
|
||||
}
|
||||
|
||||
go cf.cycle(false)
|
||||
timeout := 5 * time.Second
|
||||
CycleLoop:
|
||||
for {
|
||||
select {
|
||||
case <-time.After(timeout):
|
||||
go cf.cycle(false)
|
||||
case <-cf.grp.Ch():
|
||||
break CycleLoop
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: what to do if we have less than K active contacts, shortlist is empty, but we have other contacts in our routing table whom we have not contacted. prolly contact them
|
||||
|
||||
var contacts []Contact
|
||||
var found bool
|
||||
if cf.findValue && len(cf.findValueResult) > 0 {
|
||||
contacts = cf.findValueResult
|
||||
found = true
|
||||
} else {
|
||||
contacts = cf.activeContacts
|
||||
if len(contacts) > bucketSize {
|
||||
contacts = contacts[:bucketSize]
|
||||
}
|
||||
}
|
||||
|
||||
cf.Stop()
|
||||
return contacts, found, nil
|
||||
}
|
||||
|
||||
// cycle does a single cycle of sending alpha probes and checking results against closestNode
|
||||
func (cf *contactFinder) cycle(bigCycle bool) {
|
||||
cycleID := crypto.RandString(6)
|
||||
if bigCycle {
|
||||
cf.debug("LAUNCHING CYCLE %s, AND ITS A BIG CYCLE", cycleID)
|
||||
} else {
|
||||
cf.debug("LAUNCHING CYCLE %s", cycleID)
|
||||
}
|
||||
defer cf.debug("CYCLE %s DONE", cycleID)
|
||||
|
||||
cf.closestContactMutex.RLock()
|
||||
closestContact := cf.closestContact
|
||||
cf.closestContactMutex.RUnlock()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
ch := make(chan *Contact)
|
||||
|
||||
limit := alpha
|
||||
if bigCycle {
|
||||
limit = bucketSize
|
||||
}
|
||||
|
||||
for i := 0; i < limit; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
ch <- cf.probe(cycleID)
|
||||
}()
|
||||
}
|
||||
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(ch)
|
||||
}()
|
||||
|
||||
foundCloser := false
|
||||
for {
|
||||
c, more := <-ch
|
||||
if !more {
|
||||
break
|
||||
}
|
||||
if c != nil && (closestContact == nil || cf.target.Closer(c.ID, closestContact.ID)) {
|
||||
if closestContact != nil {
|
||||
cf.debug("|%s| best contact improved: %s -> %s", cycleID, closestContact.ID.HexShort(), c.ID.HexShort())
|
||||
} else {
|
||||
cf.debug("|%s| best contact starting at %s", cycleID, c.ID.HexShort())
|
||||
}
|
||||
foundCloser = true
|
||||
closestContact = c
|
||||
}
|
||||
}
|
||||
|
||||
if cf.isSearchFinished() {
|
||||
cf.grp.Stop()
|
||||
return
|
||||
}
|
||||
|
||||
if foundCloser {
|
||||
cf.closestContactMutex.Lock()
|
||||
// have to check again after locking in case other probes found a closer one in the meantime
|
||||
if cf.closestContact == nil || cf.target.Closer(closestContact.ID, cf.closestContact.ID) {
|
||||
cf.closestContact = closestContact
|
||||
}
|
||||
cf.closestContactMutex.Unlock()
|
||||
go cf.cycle(false)
|
||||
} else if !bigCycle {
|
||||
cf.debug("|%s| no improvement, running big cycle", cycleID)
|
||||
go cf.cycle(true)
|
||||
} else {
|
||||
// big cycle ran and there was no improvement, so we're done
|
||||
cf.debug("|%s| big cycle ran, still no improvement", cycleID)
|
||||
cf.notGettingCloser.Store(true)
|
||||
}
|
||||
}
|
||||
|
||||
// probe sends a single probe, updates the lists, and returns the closest contact it found
|
||||
func (cf *contactFinder) probe(cycleID string) *Contact {
|
||||
maybeContact := cf.popFromShortlist()
|
||||
if maybeContact == nil {
|
||||
cf.debug("|%s| no contacts in shortlist, returning", cycleID)
|
||||
return nil
|
||||
}
|
||||
|
||||
c := *maybeContact
|
||||
|
||||
if c.ID.Equals(cf.node.id) {
|
||||
return nil
|
||||
}
|
||||
|
||||
cf.debug("|%s| probe %s: launching", cycleID, c.ID.HexShort())
|
||||
|
||||
req := Request{Arg: &cf.target}
|
||||
if cf.findValue {
|
||||
req.Method = findValueMethod
|
||||
} else {
|
||||
req.Method = findNodeMethod
|
||||
}
|
||||
|
||||
var res *Response
|
||||
resCh := cf.node.SendAsync(c, req)
|
||||
select {
|
||||
case res = <-resCh:
|
||||
case <-cf.grp.Ch():
|
||||
cf.debug("|%s| probe %s: canceled", cycleID, c.ID.HexShort())
|
||||
return nil
|
||||
}
|
||||
|
||||
if res == nil {
|
||||
cf.debug("|%s| probe %s: req canceled or timed out", cycleID, c.ID.HexShort())
|
||||
return nil
|
||||
}
|
||||
|
||||
if cf.findValue && res.FindValueKey != "" {
|
||||
cf.debug("|%s| probe %s: got value", cycleID, c.ID.HexShort())
|
||||
cf.findValueMutex.Lock()
|
||||
cf.findValueResult = res.Contacts
|
||||
cf.findValueMutex.Unlock()
|
||||
cf.grp.Stop()
|
||||
return nil
|
||||
}
|
||||
|
||||
cf.debug("|%s| probe %s: got %s", cycleID, c.ID.HexShort(), res.argsDebug())
|
||||
cf.insertIntoActiveList(c)
|
||||
cf.appendNewToShortlist(res.Contacts)
|
||||
|
||||
cf.activeContactsMutex.Lock()
|
||||
contacts := cf.activeContacts
|
||||
if len(contacts) > bucketSize {
|
||||
contacts = contacts[:bucketSize]
|
||||
}
|
||||
contactsStr := ""
|
||||
for _, c := range contacts {
|
||||
contactsStr += c.ID.HexShort() + ", "
|
||||
}
|
||||
cf.activeContactsMutex.Unlock()
|
||||
|
||||
return cf.closest(res.Contacts...)
|
||||
}
|
||||
|
||||
// appendNewToShortlist appends any new contacts to the shortlist and sorts it by distance
|
||||
// contacts that have already been added to the shortlist in the past are ignored
|
||||
func (cf *contactFinder) appendNewToShortlist(contacts []Contact) {
|
||||
cf.shortlistMutex.Lock()
|
||||
defer cf.shortlistMutex.Unlock()
|
||||
|
||||
for _, c := range contacts {
|
||||
if _, ok := cf.shortlistAdded[c.ID]; !ok {
|
||||
cf.shortlist = append(cf.shortlist, c)
|
||||
cf.shortlistAdded[c.ID] = true
|
||||
}
|
||||
}
|
||||
|
||||
sortByDistance(cf.shortlist, cf.target)
|
||||
}
|
||||
|
||||
// popFromShortlist pops the first contact off the shortlist and returns it
|
||||
func (cf *contactFinder) popFromShortlist() *Contact {
|
||||
cf.shortlistMutex.Lock()
|
||||
defer cf.shortlistMutex.Unlock()
|
||||
|
||||
if len(cf.shortlist) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
first := cf.shortlist[0]
|
||||
cf.shortlist = cf.shortlist[1:]
|
||||
return &first
|
||||
}
|
||||
|
||||
// insertIntoActiveList inserts the contact into appropriate place in the list of active contacts (sorted by distance)
|
||||
func (cf *contactFinder) insertIntoActiveList(contact Contact) {
|
||||
cf.activeContactsMutex.Lock()
|
||||
defer cf.activeContactsMutex.Unlock()
|
||||
|
||||
inserted := false
|
||||
for i, n := range cf.activeContacts {
|
||||
if cf.target.Closer(contact.ID, n.ID) {
|
||||
cf.activeContacts = append(cf.activeContacts[:i], append([]Contact{contact}, cf.activeContacts[i:]...)...)
|
||||
inserted = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !inserted {
|
||||
cf.activeContacts = append(cf.activeContacts, contact)
|
||||
}
|
||||
}
|
||||
|
||||
// isSearchFinished returns true if the search is done and should be stopped
|
||||
func (cf *contactFinder) isSearchFinished() bool {
|
||||
if cf.findValue && len(cf.findValueResult) > 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
select {
|
||||
case <-cf.grp.Ch():
|
||||
return true
|
||||
default:
|
||||
}
|
||||
|
||||
if cf.notGettingCloser.Load() {
|
||||
return true
|
||||
}
|
||||
|
||||
cf.activeContactsMutex.Lock()
|
||||
defer cf.activeContactsMutex.Unlock()
|
||||
return len(cf.activeContacts) >= bucketSize
|
||||
}
|
||||
|
||||
func (cf *contactFinder) debug(format string, args ...interface{}) {
|
||||
args = append([]interface{}{cf.node.id.HexShort()}, append([]interface{}{cf.target.HexShort()}, args...)...)
|
||||
cfLog.Debugf("[%s] find %s: "+format, args...)
|
||||
}
|
||||
|
||||
func (cf *contactFinder) closest(contacts ...Contact) *Contact {
|
||||
if len(contacts) == 0 {
|
||||
return nil
|
||||
}
|
||||
closest := contacts[0]
|
||||
for _, c := range contacts {
|
||||
if cf.target.Closer(c.ID, closest.ID) {
|
||||
closest = c
|
||||
}
|
||||
}
|
||||
return &closest
|
||||
}
|
||||
@@ -1,422 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"net"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
"github.com/lyoshenka/bencode"
|
||||
)
|
||||
|
||||
func TestPing(t *testing.T) {
|
||||
dhtNodeID := bits.Rand()
|
||||
testNodeID := bits.Rand()
|
||||
|
||||
conn := newTestUDPConn("127.0.0.1:21217")
|
||||
|
||||
dht := New(&Config{Address: "127.0.0.1:21216", NodeID: dhtNodeID.Hex()})
|
||||
|
||||
err := dht.connect(conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer dht.Shutdown()
|
||||
|
||||
messageID := newMessageID()
|
||||
|
||||
data, err := bencode.EncodeBytes(map[string]interface{}{
|
||||
headerTypeField: requestType,
|
||||
headerMessageIDField: messageID,
|
||||
headerNodeIDField: testNodeID.RawString(),
|
||||
headerPayloadField: "ping",
|
||||
headerArgsField: []string{},
|
||||
})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
conn.toRead <- testUDPPacket{addr: conn.addr, data: data}
|
||||
timer := time.NewTimer(3 * time.Second)
|
||||
|
||||
select {
|
||||
case <-timer.C:
|
||||
t.Error("timeout")
|
||||
case resp := <-conn.writes:
|
||||
var response map[string]interface{}
|
||||
err := bencode.DecodeBytes(resp.data, &response)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(response) != 4 {
|
||||
t.Errorf("expected 4 response fields, got %d", len(response))
|
||||
}
|
||||
|
||||
_, ok := response[headerTypeField]
|
||||
if !ok {
|
||||
t.Error("missing type field")
|
||||
} else {
|
||||
rType, ok := response[headerTypeField].(int64)
|
||||
if !ok {
|
||||
t.Error("type is not an integer")
|
||||
} else if rType != responseType {
|
||||
t.Error("unexpected response type")
|
||||
}
|
||||
}
|
||||
|
||||
_, ok = response[headerMessageIDField]
|
||||
if !ok {
|
||||
t.Error("missing message id field")
|
||||
} else {
|
||||
rMessageID, ok := response[headerMessageIDField].(string)
|
||||
if !ok {
|
||||
t.Error("message ID is not a string")
|
||||
} else if rMessageID != string(messageID[:]) {
|
||||
t.Error("unexpected message ID")
|
||||
}
|
||||
}
|
||||
|
||||
_, ok = response[headerNodeIDField]
|
||||
if !ok {
|
||||
t.Error("missing node id field")
|
||||
} else {
|
||||
rNodeID, ok := response[headerNodeIDField].(string)
|
||||
if !ok {
|
||||
t.Error("node ID is not a string")
|
||||
} else if rNodeID != dhtNodeID.RawString() {
|
||||
t.Error("unexpected node ID")
|
||||
}
|
||||
}
|
||||
|
||||
_, ok = response[headerPayloadField]
|
||||
if !ok {
|
||||
t.Error("missing payload field")
|
||||
} else {
|
||||
rNodeID, ok := response[headerPayloadField].(string)
|
||||
if !ok {
|
||||
t.Error("payload is not a string")
|
||||
} else if rNodeID != pingSuccessResponse {
|
||||
t.Error("did not pong")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStore(t *testing.T) {
|
||||
dhtNodeID := bits.Rand()
|
||||
testNodeID := bits.Rand()
|
||||
|
||||
conn := newTestUDPConn("127.0.0.1:21217")
|
||||
|
||||
dht := New(&Config{Address: "127.0.0.1:21216", NodeID: dhtNodeID.Hex()})
|
||||
|
||||
err := dht.connect(conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer dht.Shutdown()
|
||||
|
||||
messageID := newMessageID()
|
||||
blobHashToStore := bits.Rand()
|
||||
|
||||
storeRequest := Request{
|
||||
ID: messageID,
|
||||
NodeID: testNodeID,
|
||||
Method: storeMethod,
|
||||
StoreArgs: &storeArgs{
|
||||
BlobHash: blobHashToStore,
|
||||
Value: storeArgsValue{
|
||||
Token: dht.node.tokens.Get(testNodeID, conn.addr),
|
||||
LbryID: testNodeID,
|
||||
Port: 9999,
|
||||
},
|
||||
NodeID: testNodeID,
|
||||
},
|
||||
}
|
||||
|
||||
_ = "64 " + // start message
|
||||
"313A30 693065" + // type: 0
|
||||
"313A31 3230 3A 6EB490B5788B63F0F7E6D92352024D0CBDEC2D3A" + // message id
|
||||
"313A32 3438 3A 7CE1B831DEC8689E44F80F547D2DEA171F6A625E1A4FF6C6165E645F953103DABEB068A622203F859C6C64658FD3AA3B" + // node id
|
||||
"313A33 35 3A 73746F7265" + // method
|
||||
"313A34 6C" + // start args list
|
||||
"3438 3A 3214D6C2F77FCB5E8D5FC07EDAFBA614F031CE8B2EAB49F924F8143F6DFBADE048D918710072FB98AB1B52B58F4E1468" + // block hash
|
||||
"64" + // start value dict
|
||||
"363A6C6272796964 3438 3A 7CE1B831DEC8689E44F80F547D2DEA171F6A625E1A4FF6C6165E645F953103DABEB068A622203F859C6C64658FD3AA3B" + // lbry id
|
||||
"343A706F7274 69 33333333 65" + // port
|
||||
"353A746F6B656E 3438 3A 17C2D8E1E48EF21567FE4AD5C8ED944B798D3B65AB58D0C9122AD6587D1B5FED472EA2CB12284CEFA1C21EFF302322BD" + // token
|
||||
"65" + // end value dict
|
||||
"3438 3A 7CE1B831DEC8689E44F80F547D2DEA171F6A625E1A4FF6C6165E645F953103DABEB068A622203F859C6C64658FD3AA3B" + // node id
|
||||
"693065" + // self store (integer)
|
||||
"65" + // end args list
|
||||
"65" // end message
|
||||
|
||||
data, err := bencode.EncodeBytes(storeRequest)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
conn.toRead <- testUDPPacket{addr: conn.addr, data: data}
|
||||
timer := time.NewTimer(3 * time.Second)
|
||||
|
||||
var response map[string]interface{}
|
||||
select {
|
||||
case <-timer.C:
|
||||
t.Fatal("timeout")
|
||||
case resp := <-conn.writes:
|
||||
err := bencode.DecodeBytes(resp.data, &response)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
verifyResponse(t, response, messageID, dhtNodeID.RawString())
|
||||
|
||||
_, ok := response[headerPayloadField]
|
||||
if !ok {
|
||||
t.Error("missing payload field")
|
||||
} else {
|
||||
rNodeID, ok := response[headerPayloadField].(string)
|
||||
if !ok {
|
||||
t.Error("payload is not a string")
|
||||
} else if rNodeID != storeSuccessResponse {
|
||||
t.Error("did not return OK")
|
||||
}
|
||||
}
|
||||
|
||||
if dht.node.store.CountStoredHashes() != 1 {
|
||||
t.Error("dht store has wrong number of items")
|
||||
}
|
||||
|
||||
items := dht.node.store.Get(blobHashToStore)
|
||||
if len(items) != 1 {
|
||||
t.Error("list created in store, but nothing in list")
|
||||
}
|
||||
if !items[0].ID.Equals(testNodeID) {
|
||||
t.Error("wrong value stored")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindNode(t *testing.T) {
|
||||
dhtNodeID := bits.Rand()
|
||||
testNodeID := bits.Rand()
|
||||
|
||||
conn := newTestUDPConn("127.0.0.1:21217")
|
||||
|
||||
dht := New(&Config{Address: "127.0.0.1:21216", NodeID: dhtNodeID.Hex()})
|
||||
|
||||
err := dht.connect(conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer dht.Shutdown()
|
||||
|
||||
nodesToInsert := 3
|
||||
var nodes []Contact
|
||||
for i := 0; i < nodesToInsert; i++ {
|
||||
n := Contact{ID: bits.Rand(), IP: net.ParseIP("127.0.0.1"), Port: 10000 + i}
|
||||
nodes = append(nodes, n)
|
||||
dht.node.rt.Update(n)
|
||||
}
|
||||
|
||||
messageID := newMessageID()
|
||||
blobHashToFind := bits.Rand()
|
||||
|
||||
request := Request{
|
||||
ID: messageID,
|
||||
NodeID: testNodeID,
|
||||
Method: findNodeMethod,
|
||||
Arg: &blobHashToFind,
|
||||
}
|
||||
|
||||
data, err := bencode.EncodeBytes(request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
conn.toRead <- testUDPPacket{addr: conn.addr, data: data}
|
||||
timer := time.NewTimer(3 * time.Second)
|
||||
|
||||
var response map[string]interface{}
|
||||
select {
|
||||
case <-timer.C:
|
||||
t.Fatal("timeout")
|
||||
case resp := <-conn.writes:
|
||||
err := bencode.DecodeBytes(resp.data, &response)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
verifyResponse(t, response, messageID, dhtNodeID.RawString())
|
||||
|
||||
_, ok := response[headerPayloadField]
|
||||
if !ok {
|
||||
t.Fatal("missing payload field")
|
||||
}
|
||||
|
||||
contacts, ok := response[headerPayloadField].([]interface{})
|
||||
if !ok {
|
||||
t.Fatal("payload is not a list")
|
||||
}
|
||||
|
||||
verifyContacts(t, contacts, nodes)
|
||||
}
|
||||
|
||||
func TestFindValueExisting(t *testing.T) {
|
||||
dhtNodeID := bits.Rand()
|
||||
testNodeID := bits.Rand()
|
||||
|
||||
conn := newTestUDPConn("127.0.0.1:21217")
|
||||
|
||||
dht := New(&Config{Address: "127.0.0.1:21216", NodeID: dhtNodeID.Hex()})
|
||||
|
||||
err := dht.connect(conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer dht.Shutdown()
|
||||
|
||||
nodesToInsert := 3
|
||||
for i := 0; i < nodesToInsert; i++ {
|
||||
n := Contact{ID: bits.Rand(), IP: net.ParseIP("127.0.0.1"), Port: 10000 + i}
|
||||
dht.node.rt.Update(n)
|
||||
}
|
||||
|
||||
//data, _ := hex.DecodeString("64313a30693065313a3132303a7de8e57d34e316abbb5a8a8da50dcd1ad4c80e0f313a3234383a7ce1b831dec8689e44f80f547d2dea171f6a625e1a4ff6c6165e645f953103dabeb068a622203f859c6c64658fd3aa3b313a33393a66696e6456616c7565313a346c34383aa47624b8e7ee1e54df0c45e2eb858feb0b705bd2a78d8b739be31ba188f4bd6f56b371c51fecc5280d5fd26ba4168e966565")
|
||||
|
||||
messageID := newMessageID()
|
||||
valueToFind := bits.Rand()
|
||||
|
||||
nodeToFind := Contact{ID: bits.Rand(), IP: net.ParseIP("1.2.3.4"), PeerPort: 1286}
|
||||
dht.node.store.Upsert(valueToFind, nodeToFind)
|
||||
dht.node.store.Upsert(valueToFind, nodeToFind)
|
||||
dht.node.store.Upsert(valueToFind, nodeToFind)
|
||||
|
||||
request := Request{
|
||||
ID: messageID,
|
||||
NodeID: testNodeID,
|
||||
Method: findValueMethod,
|
||||
Arg: &valueToFind,
|
||||
}
|
||||
|
||||
data, err := bencode.EncodeBytes(request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
conn.toRead <- testUDPPacket{addr: conn.addr, data: data}
|
||||
timer := time.NewTimer(3 * time.Second)
|
||||
|
||||
var response map[string]interface{}
|
||||
select {
|
||||
case <-timer.C:
|
||||
t.Fatal("timeout")
|
||||
case resp := <-conn.writes:
|
||||
err := bencode.DecodeBytes(resp.data, &response)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
verifyResponse(t, response, messageID, dhtNodeID.RawString())
|
||||
|
||||
_, ok := response[headerPayloadField]
|
||||
if !ok {
|
||||
t.Fatal("missing payload field")
|
||||
}
|
||||
|
||||
payload, ok := response[headerPayloadField].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatal("payload is not a dictionary")
|
||||
}
|
||||
|
||||
compactContacts, ok := payload[valueToFind.RawString()]
|
||||
if !ok {
|
||||
t.Fatal("payload is missing key for search value")
|
||||
}
|
||||
|
||||
contacts, ok := compactContacts.([]interface{})
|
||||
if !ok {
|
||||
t.Fatal("search results are not a list")
|
||||
}
|
||||
|
||||
verifyCompactContacts(t, contacts, []Contact{nodeToFind})
|
||||
}
|
||||
|
||||
func TestFindValueFallbackToFindNode(t *testing.T) {
|
||||
dhtNodeID := bits.Rand()
|
||||
testNodeID := bits.Rand()
|
||||
|
||||
conn := newTestUDPConn("127.0.0.1:21217")
|
||||
|
||||
dht := New(&Config{Address: "127.0.0.1:21216", NodeID: dhtNodeID.Hex()})
|
||||
|
||||
err := dht.connect(conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer dht.Shutdown()
|
||||
|
||||
nodesToInsert := 3
|
||||
var nodes []Contact
|
||||
for i := 0; i < nodesToInsert; i++ {
|
||||
n := Contact{ID: bits.Rand(), IP: net.ParseIP("127.0.0.1"), Port: 10000 + i}
|
||||
nodes = append(nodes, n)
|
||||
dht.node.rt.Update(n)
|
||||
}
|
||||
|
||||
messageID := newMessageID()
|
||||
valueToFind := bits.Rand()
|
||||
|
||||
request := Request{
|
||||
ID: messageID,
|
||||
NodeID: testNodeID,
|
||||
Method: findValueMethod,
|
||||
Arg: &valueToFind,
|
||||
}
|
||||
|
||||
data, err := bencode.EncodeBytes(request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
conn.toRead <- testUDPPacket{addr: conn.addr, data: data}
|
||||
timer := time.NewTimer(3 * time.Second)
|
||||
|
||||
var response map[string]interface{}
|
||||
select {
|
||||
case <-timer.C:
|
||||
t.Fatal("timeout")
|
||||
case resp := <-conn.writes:
|
||||
err := bencode.DecodeBytes(resp.data, &response)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
verifyResponse(t, response, messageID, dhtNodeID.RawString())
|
||||
|
||||
_, ok := response[headerPayloadField]
|
||||
if !ok {
|
||||
t.Fatal("missing payload field")
|
||||
}
|
||||
|
||||
payload, ok := response[headerPayloadField].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatal("payload is not a dictionary")
|
||||
}
|
||||
|
||||
contactsList, ok := payload[contactsField]
|
||||
if !ok {
|
||||
t.Fatal("payload is missing 'contacts' key")
|
||||
}
|
||||
|
||||
contacts, ok := contactsList.([]interface{})
|
||||
if !ok {
|
||||
t.Fatal("'contacts' is not a list")
|
||||
}
|
||||
|
||||
verifyContacts(t, contacts, nodes)
|
||||
}
|
||||
@@ -1,463 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/lbry.go/stop"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
)
|
||||
|
||||
// TODO: if routing table is ever empty (aka the node is isolated), it should re-bootstrap
|
||||
|
||||
// TODO: use a tree with bucket splitting instead of a fixed bucket list. include jack's optimization (see link in commit mesg)
|
||||
// https://github.com/lbryio/lbry/pull/1211/commits/341b27b6d21ac027671d42458826d02735aaae41
|
||||
|
||||
// peer is a contact with extra information
|
||||
type peer struct {
|
||||
Contact Contact
|
||||
Distance bits.Bitmap
|
||||
LastActivity time.Time
|
||||
// LastReplied time.Time
|
||||
// LastRequested time.Time
|
||||
// LastFailure time.Time
|
||||
// SecondLastFailure time.Time
|
||||
NumFailures int
|
||||
|
||||
//<lastPublished>,
|
||||
//<originallyPublished>
|
||||
// <originalPublisherID>
|
||||
}
|
||||
|
||||
func (p *peer) Touch() {
|
||||
p.LastActivity = time.Now()
|
||||
p.NumFailures = 0
|
||||
}
|
||||
|
||||
// ActiveSince returns whether a peer has responded in the last `d` duration
|
||||
// this is used to check if the peer is "good", meaning that we believe the peer will respond to our requests
|
||||
func (p *peer) ActiveInLast(d time.Duration) bool {
|
||||
return time.Since(p.LastActivity) < d
|
||||
}
|
||||
|
||||
// IsBad returns whether a peer is "bad", meaning that it has failed to respond to multiple pings in a row
|
||||
func (p *peer) IsBad(maxFalures int) bool {
|
||||
return p.NumFailures >= maxFalures
|
||||
}
|
||||
|
||||
// Fail marks a peer as having failed to respond. It returns whether or not the peer should be removed from the routing table
|
||||
func (p *peer) Fail() {
|
||||
p.NumFailures++
|
||||
}
|
||||
|
||||
type bucket struct {
|
||||
lock *sync.RWMutex
|
||||
peers []peer
|
||||
lastUpdate time.Time
|
||||
Range bits.Range // capitalized because `range` is a keyword
|
||||
}
|
||||
|
||||
func newBucket(r bits.Range) *bucket {
|
||||
return &bucket{
|
||||
peers: make([]peer, 0, bucketSize),
|
||||
lock: &sync.RWMutex{},
|
||||
Range: r,
|
||||
}
|
||||
}
|
||||
|
||||
// Len returns the number of peers in the bucket
|
||||
func (b bucket) Len() int {
|
||||
b.lock.RLock()
|
||||
defer b.lock.RUnlock()
|
||||
return len(b.peers)
|
||||
}
|
||||
|
||||
func (b bucket) Has(c Contact) bool {
|
||||
b.lock.RLock()
|
||||
defer b.lock.RUnlock()
|
||||
for _, p := range b.peers {
|
||||
if p.Contact.Equals(c, true) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Contacts returns a slice of the bucket's contacts
|
||||
func (b bucket) Contacts() []Contact {
|
||||
b.lock.RLock()
|
||||
defer b.lock.RUnlock()
|
||||
contacts := make([]Contact, len(b.peers))
|
||||
for i := range b.peers {
|
||||
contacts[i] = b.peers[i].Contact
|
||||
}
|
||||
return contacts
|
||||
}
|
||||
|
||||
// UpdatePeer marks a contact as having been successfully contacted. if insertIfNew and the contact is does not exist yet, it is inserted
|
||||
func (b *bucket) UpdatePeer(p peer, insertIfNew bool) error {
|
||||
b.lock.Lock()
|
||||
defer b.lock.Unlock()
|
||||
|
||||
if !b.Range.Contains(p.Distance) {
|
||||
return errors.Err("this bucket range does not cover this peer")
|
||||
}
|
||||
|
||||
peerIndex := find(p.Contact.ID, b.peers)
|
||||
if peerIndex >= 0 {
|
||||
b.lastUpdate = time.Now()
|
||||
b.peers[peerIndex].Touch()
|
||||
moveToBack(b.peers, peerIndex)
|
||||
} else if insertIfNew {
|
||||
hasRoom := true
|
||||
|
||||
if len(b.peers) >= bucketSize {
|
||||
hasRoom = false
|
||||
for i := range b.peers {
|
||||
if b.peers[i].IsBad(maxPeerFails) {
|
||||
// TODO: Ping contact first. Only remove if it does not respond
|
||||
b.peers = append(b.peers[:i], b.peers[i+1:]...)
|
||||
hasRoom = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if hasRoom {
|
||||
b.lastUpdate = time.Now()
|
||||
p.Touch()
|
||||
b.peers = append(b.peers, p)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// FailContact marks a contact as having failed, and removes it if it failed too many times
|
||||
func (b *bucket) FailContact(id bits.Bitmap) {
|
||||
b.lock.Lock()
|
||||
defer b.lock.Unlock()
|
||||
i := find(id, b.peers)
|
||||
if i >= 0 {
|
||||
// BEP5 says not to remove the contact until the bucket is full and you try to insert
|
||||
b.peers[i].Fail()
|
||||
}
|
||||
}
|
||||
|
||||
// find returns the contact in the bucket, or nil if the bucket does not contain the contact
|
||||
func find(id bits.Bitmap, peers []peer) int {
|
||||
for i := range peers {
|
||||
if peers[i].Contact.ID.Equals(id) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// NeedsRefresh returns true if bucket has not been updated in the last `refreshInterval`, false otherwise
|
||||
func (b *bucket) NeedsRefresh(refreshInterval time.Duration) bool {
|
||||
b.lock.RLock()
|
||||
defer b.lock.RUnlock()
|
||||
return time.Since(b.lastUpdate) > refreshInterval
|
||||
}
|
||||
|
||||
func (b *bucket) Split() (*bucket, *bucket) {
|
||||
b.lock.Lock()
|
||||
defer b.lock.Unlock()
|
||||
|
||||
left := newBucket(b.Range.IntervalP(1, 2))
|
||||
right := newBucket(b.Range.IntervalP(2, 2))
|
||||
left.lastUpdate = b.lastUpdate
|
||||
right.lastUpdate = b.lastUpdate
|
||||
|
||||
for _, p := range b.peers {
|
||||
if left.Range.Contains(p.Distance) {
|
||||
left.peers = append(left.peers, p)
|
||||
} else {
|
||||
right.peers = append(right.peers, p)
|
||||
}
|
||||
}
|
||||
|
||||
if len(b.peers) > 1 {
|
||||
if len(left.peers) == 0 {
|
||||
left, right = right.Split()
|
||||
left.Range.Start = b.Range.Start
|
||||
} else if len(right.peers) == 0 {
|
||||
left, right = left.Split()
|
||||
right.Range.End = b.Range.End
|
||||
}
|
||||
}
|
||||
|
||||
return left, right
|
||||
}
|
||||
|
||||
type routingTable struct {
|
||||
id bits.Bitmap
|
||||
buckets []*bucket
|
||||
mu *sync.RWMutex // this mutex is write-locked only when CHANGING THE NUMBER OF BUCKETS in the table
|
||||
}
|
||||
|
||||
func newRoutingTable(id bits.Bitmap) *routingTable {
|
||||
rt := routingTable{
|
||||
id: id,
|
||||
mu: &sync.RWMutex{},
|
||||
}
|
||||
rt.reset()
|
||||
return &rt
|
||||
}
|
||||
|
||||
func (rt *routingTable) reset() {
|
||||
rt.mu.Lock()
|
||||
defer rt.mu.Unlock()
|
||||
rt.buckets = []*bucket{newBucket(bits.MaxRange())}
|
||||
}
|
||||
|
||||
func (rt *routingTable) BucketInfo() string {
|
||||
rt.mu.RLock()
|
||||
defer rt.mu.RUnlock()
|
||||
|
||||
var bucketInfo []string
|
||||
for i, b := range rt.buckets {
|
||||
if b.Len() > 0 {
|
||||
contacts := b.Contacts()
|
||||
s := make([]string, len(contacts))
|
||||
for j, c := range contacts {
|
||||
s[j] = c.ID.HexShort()
|
||||
}
|
||||
bucketInfo = append(bucketInfo, fmt.Sprintf("bucket %d: (%d) %s", i, len(contacts), strings.Join(s, ", ")))
|
||||
}
|
||||
}
|
||||
if len(bucketInfo) == 0 {
|
||||
return "buckets are empty"
|
||||
}
|
||||
return strings.Join(bucketInfo, "\n")
|
||||
}
|
||||
|
||||
// Update inserts or refreshes a contact
|
||||
func (rt *routingTable) Update(c Contact) {
|
||||
rt.mu.Lock() // write lock, because updates may cause bucket splits
|
||||
defer rt.mu.Unlock()
|
||||
|
||||
b := rt.bucketFor(c.ID)
|
||||
|
||||
if rt.shouldSplit(b, c) {
|
||||
left, right := b.Split()
|
||||
|
||||
for i := range rt.buckets {
|
||||
if rt.buckets[i].Range.Start.Equals(left.Range.Start) {
|
||||
rt.buckets = append(rt.buckets[:i], append([]*bucket{left, right}, rt.buckets[i+1:]...)...)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if left.Range.Contains(c.ID) {
|
||||
b = left
|
||||
} else {
|
||||
b = right
|
||||
}
|
||||
}
|
||||
|
||||
err := b.UpdatePeer(peer{Contact: c, Distance: rt.id.Xor(c.ID)}, true)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Fresh refreshes a contact if its already in the routing table
|
||||
func (rt *routingTable) Fresh(c Contact) {
|
||||
rt.mu.RLock()
|
||||
defer rt.mu.RUnlock()
|
||||
err := rt.bucketFor(c.ID).UpdatePeer(peer{Contact: c, Distance: rt.id.Xor(c.ID)}, false)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
// FailContact marks a contact as having failed, and removes it if it failed too many times
|
||||
func (rt *routingTable) Fail(c Contact) {
|
||||
rt.mu.RLock()
|
||||
defer rt.mu.RUnlock()
|
||||
rt.bucketFor(c.ID).FailContact(c.ID)
|
||||
}
|
||||
|
||||
// GetClosest returns the closest `limit` contacts from the routing table.
|
||||
// This is a locking wrapper around getClosest()
|
||||
func (rt *routingTable) GetClosest(target bits.Bitmap, limit int) []Contact {
|
||||
rt.mu.RLock()
|
||||
defer rt.mu.RUnlock()
|
||||
return rt.getClosest(target, limit)
|
||||
}
|
||||
|
||||
// getClosest returns the closest `limit` contacts from the routing table
|
||||
func (rt *routingTable) getClosest(target bits.Bitmap, limit int) []Contact {
|
||||
var contacts []Contact
|
||||
for _, b := range rt.buckets {
|
||||
contacts = append(contacts, b.Contacts()...)
|
||||
}
|
||||
|
||||
sortByDistance(contacts, target)
|
||||
if len(contacts) > limit {
|
||||
contacts = contacts[:limit]
|
||||
}
|
||||
|
||||
return contacts
|
||||
}
|
||||
|
||||
// Count returns the number of contacts in the routing table
|
||||
func (rt *routingTable) Count() int {
|
||||
rt.mu.RLock()
|
||||
defer rt.mu.RUnlock()
|
||||
count := 0
|
||||
for _, bucket := range rt.buckets {
|
||||
count += bucket.Len()
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
// Len returns the number of buckets in the routing table
|
||||
func (rt *routingTable) Len() int {
|
||||
rt.mu.RLock()
|
||||
defer rt.mu.RUnlock()
|
||||
return len(rt.buckets)
|
||||
}
|
||||
|
||||
func (rt *routingTable) bucketFor(target bits.Bitmap) *bucket {
|
||||
if rt.id.Equals(target) {
|
||||
panic("routing table does not have a bucket for its own id")
|
||||
}
|
||||
distance := target.Xor(rt.id)
|
||||
for _, b := range rt.buckets {
|
||||
if b.Range.Contains(distance) {
|
||||
return b
|
||||
}
|
||||
}
|
||||
panic("target is not contained in any buckets")
|
||||
}
|
||||
|
||||
func (rt *routingTable) shouldSplit(b *bucket, c Contact) bool {
|
||||
if b.Has(c) {
|
||||
return false
|
||||
}
|
||||
if b.Len() >= bucketSize {
|
||||
if b.Range.Start.Equals(bits.Bitmap{}) { // this is the bucket covering our node id
|
||||
return true
|
||||
}
|
||||
kClosest := rt.getClosest(rt.id, bucketSize)
|
||||
kthClosest := kClosest[len(kClosest)-1]
|
||||
if rt.id.Closer(c.ID, kthClosest.ID) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
//func (rt *routingTable) printBucketInfo() {
|
||||
// fmt.Printf("there are %d contacts in %d buckets\n", rt.Count(), rt.Len())
|
||||
// for i, b := range rt.buckets {
|
||||
// fmt.Printf("bucket %d, %d contacts\n", i+1, len(b.peers))
|
||||
// fmt.Printf(" start : %s\n", b.Range.Start.String())
|
||||
// fmt.Printf(" stop : %s\n", b.Range.End.String())
|
||||
// fmt.Println("")
|
||||
// }
|
||||
//}
|
||||
|
||||
func (rt *routingTable) GetIDsForRefresh(refreshInterval time.Duration) []bits.Bitmap {
|
||||
var bitmaps []bits.Bitmap
|
||||
for i, bucket := range rt.buckets {
|
||||
if bucket.NeedsRefresh(refreshInterval) {
|
||||
bitmaps = append(bitmaps, bits.Rand().Prefix(i, false))
|
||||
}
|
||||
}
|
||||
return bitmaps
|
||||
}
|
||||
|
||||
const rtContactSep = "-"
|
||||
|
||||
type rtSave struct {
|
||||
ID string `json:"id"`
|
||||
Contacts []string `json:"contacts"`
|
||||
}
|
||||
|
||||
func (rt *routingTable) MarshalJSON() ([]byte, error) {
|
||||
var data rtSave
|
||||
data.ID = rt.id.Hex()
|
||||
for _, b := range rt.buckets {
|
||||
for _, c := range b.Contacts() {
|
||||
data.Contacts = append(data.Contacts, strings.Join([]string{c.ID.Hex(), c.IP.String(), strconv.Itoa(c.Port)}, rtContactSep))
|
||||
}
|
||||
}
|
||||
return json.Marshal(data)
|
||||
}
|
||||
|
||||
func (rt *routingTable) UnmarshalJSON(b []byte) error {
|
||||
var data rtSave
|
||||
err := json.Unmarshal(b, &data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rt.id, err = bits.FromHex(data.ID)
|
||||
if err != nil {
|
||||
return errors.Prefix("decoding ID", err)
|
||||
}
|
||||
rt.reset()
|
||||
|
||||
for _, s := range data.Contacts {
|
||||
parts := strings.Split(s, rtContactSep)
|
||||
if len(parts) != 3 {
|
||||
return errors.Err("decoding contact %s: wrong number of parts", s)
|
||||
}
|
||||
var c Contact
|
||||
c.ID, err = bits.FromHex(parts[0])
|
||||
if err != nil {
|
||||
return errors.Err("decoding contact %s: invalid ID: %s", s, err)
|
||||
}
|
||||
c.IP = net.ParseIP(parts[1])
|
||||
if c.IP == nil {
|
||||
return errors.Err("decoding contact %s: invalid IP", s)
|
||||
}
|
||||
c.Port, err = strconv.Atoi(parts[2])
|
||||
if err != nil {
|
||||
return errors.Err("decoding contact %s: invalid port: %s", s, err)
|
||||
}
|
||||
rt.Update(c)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// RoutingTableRefresh refreshes any buckets that need to be refreshed
|
||||
func RoutingTableRefresh(n *Node, refreshInterval time.Duration, parentGrp *stop.Group) {
|
||||
done := stop.New()
|
||||
|
||||
for _, id := range n.rt.GetIDsForRefresh(refreshInterval) {
|
||||
done.Add(1)
|
||||
go func(id bits.Bitmap) {
|
||||
defer done.Done()
|
||||
_, _, err := FindContacts(n, id, false, parentGrp)
|
||||
if err != nil {
|
||||
log.Error("error finding contact during routing table refresh - ", err)
|
||||
}
|
||||
}(id)
|
||||
}
|
||||
|
||||
done.Wait()
|
||||
done.Stop()
|
||||
}
|
||||
|
||||
func moveToBack(peers []peer, index int) {
|
||||
if index < 0 || len(peers) <= index+1 {
|
||||
return
|
||||
}
|
||||
p := peers[index]
|
||||
for i := index; i < len(peers)-1; i++ {
|
||||
peers[i] = peers[i+1]
|
||||
}
|
||||
peers[len(peers)-1] = p
|
||||
}
|
||||
@@ -1,328 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"math/big"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
|
||||
"github.com/sebdah/goldie"
|
||||
)
|
||||
|
||||
func TestBucket_Split(t *testing.T) {
|
||||
rt := newRoutingTable(bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"))
|
||||
if len(rt.buckets) != 1 {
|
||||
t.Errorf("there should only be one bucket so far")
|
||||
}
|
||||
if len(rt.buckets[0].peers) != 0 {
|
||||
t.Errorf("there should be no contacts yet")
|
||||
}
|
||||
|
||||
var tests = []struct {
|
||||
name string
|
||||
id bits.Bitmap
|
||||
expectedBucketCount int
|
||||
expectedTotalContacts int
|
||||
}{
|
||||
//fill first bucket
|
||||
{"b1-one", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100"), 1, 1},
|
||||
{"b1-two", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000200"), 1, 2},
|
||||
{"b1-three", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000300"), 1, 3},
|
||||
{"b1-four", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000400"), 1, 4},
|
||||
{"b1-five", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000500"), 1, 5},
|
||||
{"b1-six", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000600"), 1, 6},
|
||||
{"b1-seven", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000700"), 1, 7},
|
||||
{"b1-eight", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800"), 1, 8},
|
||||
|
||||
// split off second bucket and fill it
|
||||
{"b2-one", bits.FromHexP("001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 9},
|
||||
{"b2-two", bits.FromHexP("002000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 10},
|
||||
{"b2-three", bits.FromHexP("003000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 11},
|
||||
{"b2-four", bits.FromHexP("004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 12},
|
||||
{"b2-five", bits.FromHexP("005000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 13},
|
||||
{"b2-six", bits.FromHexP("006000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 14},
|
||||
{"b2-seven", bits.FromHexP("007000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 15},
|
||||
|
||||
// at this point there are two buckets. the first has 7 contacts, the second has 8
|
||||
|
||||
// inserts into the second bucket should be skipped
|
||||
{"dont-split", bits.FromHexP("009000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 15},
|
||||
|
||||
// ... unless the ID is closer than the kth-closest contact
|
||||
{"split-kth-closest", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), 2, 16},
|
||||
|
||||
{"b3-two", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002"), 3, 17},
|
||||
{"b3-three", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003"), 3, 18},
|
||||
}
|
||||
|
||||
for i, testCase := range tests {
|
||||
rt.Update(Contact{testCase.id, net.ParseIP("127.0.0.1"), 8000 + i, 0})
|
||||
|
||||
if len(rt.buckets) != testCase.expectedBucketCount {
|
||||
t.Errorf("failed test case %s. there should be %d buckets, got %d", testCase.name, testCase.expectedBucketCount, len(rt.buckets))
|
||||
}
|
||||
if rt.Count() != testCase.expectedTotalContacts {
|
||||
t.Errorf("failed test case %s. there should be %d contacts, got %d", testCase.name, testCase.expectedTotalContacts, rt.Count())
|
||||
}
|
||||
}
|
||||
|
||||
var testRanges = []struct {
|
||||
id bits.Bitmap
|
||||
expected int
|
||||
}{
|
||||
{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), 0},
|
||||
{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000005"), 0},
|
||||
{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000410"), 1},
|
||||
{bits.FromHexP("0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007f0"), 1},
|
||||
{bits.FromHexP("F00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800"), 2},
|
||||
{bits.FromHexP("F00000000000000000000000000000000000000000000000000F00000000000000000000000000000000000000000000"), 2},
|
||||
{bits.FromHexP("F0000000000000000000000000000000F0000000000000000000000000F0000000000000000000000000000000000000"), 2},
|
||||
}
|
||||
|
||||
for _, tt := range testRanges {
|
||||
bucket := bucketNumFor(rt, tt.id)
|
||||
if bucket != tt.expected {
|
||||
t.Errorf("bucketFor(%s, %s) => got %d, expected %d", tt.id.Hex(), rt.id.Hex(), bucket, tt.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func bucketNumFor(rt *routingTable, target bits.Bitmap) int {
|
||||
if rt.id.Equals(target) {
|
||||
panic("routing table does not have a bucket for its own id")
|
||||
}
|
||||
distance := target.Xor(rt.id)
|
||||
for i := range rt.buckets {
|
||||
if rt.buckets[i].Range.Contains(distance) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
panic("target is not contained in any buckets")
|
||||
}
|
||||
|
||||
func TestBucket_Split_Continuous(t *testing.T) {
|
||||
b := newBucket(bits.MaxRange())
|
||||
|
||||
left, right := b.Split()
|
||||
|
||||
if !left.Range.Start.Equals(b.Range.Start) {
|
||||
t.Errorf("left bucket start does not align with original bucket start. got %s, expected %s", left.Range.Start, b.Range.Start)
|
||||
}
|
||||
|
||||
if !right.Range.End.Equals(b.Range.End) {
|
||||
t.Errorf("right bucket end does not align with original bucket end. got %s, expected %s", right.Range.End, b.Range.End)
|
||||
}
|
||||
|
||||
leftEndNext := (&big.Int{}).Add(left.Range.End.Big(), big.NewInt(1))
|
||||
if !bits.FromBigP(leftEndNext).Equals(right.Range.Start) {
|
||||
t.Errorf("there's a gap between left bucket end and right bucket start. end is %s, start is %s", left.Range.End, right.Range.Start)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBucket_Split_KthClosest_DoSplit(t *testing.T) {
|
||||
rt := newRoutingTable(bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"))
|
||||
|
||||
// add 4 low IDs
|
||||
rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), net.ParseIP("127.0.0.1"), 8001, 0})
|
||||
rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002"), net.ParseIP("127.0.0.1"), 8002, 0})
|
||||
rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003"), net.ParseIP("127.0.0.1"), 8003, 0})
|
||||
rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004"), net.ParseIP("127.0.0.1"), 8004, 0})
|
||||
|
||||
// add 4 high IDs
|
||||
rt.Update(Contact{bits.FromHexP("800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8001, 0})
|
||||
rt.Update(Contact{bits.FromHexP("900000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8002, 0})
|
||||
rt.Update(Contact{bits.FromHexP("a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8003, 0})
|
||||
rt.Update(Contact{bits.FromHexP("b00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8004, 0})
|
||||
|
||||
// split the bucket and fill the high bucket
|
||||
rt.Update(Contact{bits.FromHexP("c00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8005, 0})
|
||||
rt.Update(Contact{bits.FromHexP("d00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8006, 0})
|
||||
rt.Update(Contact{bits.FromHexP("e00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8007, 0})
|
||||
rt.Update(Contact{bits.FromHexP("f00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8008, 0})
|
||||
|
||||
// add a high ID. it should split because the high ID is closer than the Kth closest ID
|
||||
rt.Update(Contact{bits.FromHexP("910000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.1"), 8009, 0})
|
||||
|
||||
if len(rt.buckets) != 3 {
|
||||
t.Errorf("expected 3 buckets, got %d", len(rt.buckets))
|
||||
}
|
||||
if rt.Count() != 13 {
|
||||
t.Errorf("expected 13 contacts, got %d", rt.Count())
|
||||
}
|
||||
}
|
||||
|
||||
func TestBucket_Split_KthClosest_DontSplit(t *testing.T) {
|
||||
rt := newRoutingTable(bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"))
|
||||
|
||||
// add 4 low IDs
|
||||
rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), net.ParseIP("127.0.0.1"), 8001, 0})
|
||||
rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002"), net.ParseIP("127.0.0.1"), 8002, 0})
|
||||
rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003"), net.ParseIP("127.0.0.1"), 8003, 0})
|
||||
rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004"), net.ParseIP("127.0.0.1"), 8004, 0})
|
||||
|
||||
// add 4 high IDs
|
||||
rt.Update(Contact{bits.FromHexP("800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8001, 0})
|
||||
rt.Update(Contact{bits.FromHexP("900000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8002, 0})
|
||||
rt.Update(Contact{bits.FromHexP("a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8003, 0})
|
||||
rt.Update(Contact{bits.FromHexP("b00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8004, 0})
|
||||
|
||||
// split the bucket and fill the high bucket
|
||||
rt.Update(Contact{bits.FromHexP("c00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8005, 0})
|
||||
rt.Update(Contact{bits.FromHexP("d00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8006, 0})
|
||||
rt.Update(Contact{bits.FromHexP("e00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8007, 0})
|
||||
rt.Update(Contact{bits.FromHexP("f00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8008, 0})
|
||||
|
||||
// add a really high ID. this should not split because its not closer than the Kth closest ID
|
||||
rt.Update(Contact{bits.FromHexP("ffff00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.1"), 8009, 0})
|
||||
|
||||
if len(rt.buckets) != 2 {
|
||||
t.Errorf("expected 2 buckets, got %d", len(rt.buckets))
|
||||
}
|
||||
if rt.Count() != 12 {
|
||||
t.Errorf("expected 12 contacts, got %d", rt.Count())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoutingTable_GetClosest(t *testing.T) {
|
||||
n1 := bits.FromHexP("FFFFFFFF0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
|
||||
n2 := bits.FromHexP("FFFFFFF00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
|
||||
n3 := bits.FromHexP("111111110000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
|
||||
rt := newRoutingTable(n1)
|
||||
rt.Update(Contact{n2, net.ParseIP("127.0.0.1"), 8001, 0})
|
||||
rt.Update(Contact{n3, net.ParseIP("127.0.0.1"), 8002, 0})
|
||||
|
||||
contacts := rt.GetClosest(bits.FromHexP("222222220000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 1)
|
||||
if len(contacts) != 1 {
|
||||
t.Fail()
|
||||
return
|
||||
}
|
||||
if !contacts[0].ID.Equals(n3) {
|
||||
t.Error(contacts[0])
|
||||
}
|
||||
contacts = rt.GetClosest(n2, 10)
|
||||
if len(contacts) != 2 {
|
||||
t.Error(len(contacts))
|
||||
return
|
||||
}
|
||||
if !contacts[0].ID.Equals(n2) {
|
||||
t.Error(contacts[0])
|
||||
}
|
||||
if !contacts[1].ID.Equals(n3) {
|
||||
t.Error(contacts[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoutingTable_GetClosest_Empty(t *testing.T) {
|
||||
n1 := bits.FromShortHexP("1")
|
||||
rt := newRoutingTable(n1)
|
||||
|
||||
contacts := rt.GetClosest(bits.FromShortHexP("a"), 3)
|
||||
if len(contacts) != 0 {
|
||||
t.Error("there shouldn't be any contacts")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoutingTable_Refresh(t *testing.T) {
|
||||
t.Skip("TODO: test routing table refreshing")
|
||||
}
|
||||
|
||||
func TestRoutingTable_MoveToBack(t *testing.T) {
|
||||
tt := map[string]struct {
|
||||
data []peer
|
||||
index int
|
||||
expected []peer
|
||||
}{
|
||||
"simpleMove": {
|
||||
data: []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
|
||||
index: 1,
|
||||
expected: []peer{{NumFailures: 0}, {NumFailures: 2}, {NumFailures: 3}, {NumFailures: 1}},
|
||||
},
|
||||
"moveFirst": {
|
||||
data: []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
|
||||
index: 0,
|
||||
expected: []peer{{NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}, {NumFailures: 0}},
|
||||
},
|
||||
"moveLast": {
|
||||
data: []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
|
||||
index: 3,
|
||||
expected: []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
|
||||
},
|
||||
"largeIndex": {
|
||||
data: []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
|
||||
index: 27,
|
||||
expected: []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
|
||||
},
|
||||
"negativeIndex": {
|
||||
data: []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
|
||||
index: -12,
|
||||
expected: []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
|
||||
},
|
||||
}
|
||||
|
||||
for name, test := range tt {
|
||||
moveToBack(test.data, test.index)
|
||||
expected := make([]string, len(test.expected))
|
||||
actual := make([]string, len(test.data))
|
||||
for i := range actual {
|
||||
actual[i] = strconv.Itoa(test.data[i].NumFailures)
|
||||
expected[i] = strconv.Itoa(test.expected[i].NumFailures)
|
||||
}
|
||||
|
||||
expJoin := strings.Join(expected, ",")
|
||||
actJoin := strings.Join(actual, ",")
|
||||
|
||||
if actJoin != expJoin {
|
||||
t.Errorf("%s failed: got %s; expected %s", name, actJoin, expJoin)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoutingTable_Save(t *testing.T) {
|
||||
t.Skip("fix me")
|
||||
id := bits.FromHexP("1c8aff71b99462464d9eeac639595ab99664be3482cb91a29d87467515c7d9158fe72aa1f1582dab07d8f8b5db277f41")
|
||||
rt := newRoutingTable(id)
|
||||
|
||||
for i, b := range rt.buckets {
|
||||
for j := 0; j < bucketSize; j++ {
|
||||
toAdd := b.Range.Start.Add(bits.FromShortHexP(strconv.Itoa(j)))
|
||||
if toAdd.Cmp(b.Range.End) <= 0 {
|
||||
rt.Update(Contact{
|
||||
ID: b.Range.Start.Add(bits.FromShortHexP(strconv.Itoa(j))),
|
||||
IP: net.ParseIP("1.2.3." + strconv.Itoa(j)),
|
||||
Port: 1 + i*bucketSize + j,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
data, err := json.MarshalIndent(rt, "", " ")
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
goldie.Assert(t, t.Name(), data)
|
||||
}
|
||||
|
||||
func TestRoutingTable_Load_ID(t *testing.T) {
|
||||
t.Skip("fix me")
|
||||
id := "1c8aff71b99462464d9eeac639595ab99664be3482cb91a29d87467515c7d9158fe72aa1f1582dab07d8f8b5db277f41"
|
||||
data := []byte(`{"id": "` + id + `","contacts": []}`)
|
||||
|
||||
rt := routingTable{}
|
||||
err := json.Unmarshal(data, &rt)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if rt.id.Hex() != id {
|
||||
t.Error("id mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoutingTable_Load_Contacts(t *testing.T) {
|
||||
t.Skip("TODO")
|
||||
}
|
||||
-187
@@ -1,187 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"sync"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
rpc2 "github.com/gorilla/rpc/v2"
|
||||
"github.com/gorilla/rpc/v2/json"
|
||||
)
|
||||
|
||||
type rpcReceiver struct {
|
||||
dht *DHT
|
||||
}
|
||||
|
||||
type RpcPingArgs struct {
|
||||
Address string
|
||||
}
|
||||
|
||||
func (rpc *rpcReceiver) Ping(r *http.Request, args *RpcPingArgs, result *string) error {
|
||||
if args.Address == "" {
|
||||
return errors.Err("no address given")
|
||||
}
|
||||
|
||||
err := rpc.dht.Ping(args.Address)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
*result = pingSuccessResponse
|
||||
return nil
|
||||
}
|
||||
|
||||
type RpcFindArgs struct {
|
||||
Key string
|
||||
NodeID string
|
||||
IP string
|
||||
Port int
|
||||
}
|
||||
|
||||
func (rpc *rpcReceiver) FindNode(r *http.Request, args *RpcFindArgs, result *[]Contact) error {
|
||||
key, err := bits.FromHex(args.Key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
toQuery, err := bits.FromHex(args.NodeID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
c := Contact{ID: toQuery, IP: net.ParseIP(args.IP), Port: args.Port}
|
||||
req := Request{Method: findNodeMethod, Arg: &key}
|
||||
|
||||
nodeResponse := rpc.dht.node.Send(c, req)
|
||||
if nodeResponse != nil && nodeResponse.Contacts != nil {
|
||||
*result = nodeResponse.Contacts
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type RpcFindValueResult struct {
|
||||
Contacts []Contact
|
||||
Value string
|
||||
}
|
||||
|
||||
func (rpc *rpcReceiver) FindValue(r *http.Request, args *RpcFindArgs, result *RpcFindValueResult) error {
|
||||
key, err := bits.FromHex(args.Key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
toQuery, err := bits.FromHex(args.NodeID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c := Contact{ID: toQuery, IP: net.ParseIP(args.IP), Port: args.Port}
|
||||
req := Request{Arg: &key, Method: findValueMethod}
|
||||
|
||||
nodeResponse := rpc.dht.node.Send(c, req)
|
||||
if nodeResponse != nil && nodeResponse.FindValueKey != "" {
|
||||
*result = RpcFindValueResult{Value: nodeResponse.FindValueKey}
|
||||
return nil
|
||||
}
|
||||
if nodeResponse != nil && nodeResponse.Contacts != nil {
|
||||
*result = RpcFindValueResult{Contacts: nodeResponse.Contacts}
|
||||
return nil
|
||||
}
|
||||
|
||||
return errors.Err("not sure what happened")
|
||||
}
|
||||
|
||||
type RpcIterativeFindValueArgs struct {
|
||||
Key string
|
||||
}
|
||||
|
||||
type RpcIterativeFindValueResult struct {
|
||||
Contacts []Contact
|
||||
FoundValue bool
|
||||
}
|
||||
|
||||
func (rpc *rpcReceiver) IterativeFindValue(r *http.Request, args *RpcIterativeFindValueArgs, result *RpcIterativeFindValueResult) error {
|
||||
key, err := bits.FromHex(args.Key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
foundContacts, found, err := FindContacts(rpc.dht.node, key, false, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result.Contacts = foundContacts
|
||||
result.FoundValue = found
|
||||
return nil
|
||||
}
|
||||
|
||||
type RpcBucketResponse struct {
|
||||
Start string
|
||||
End string
|
||||
NumContacts int
|
||||
Contacts []Contact
|
||||
}
|
||||
|
||||
type RpcRoutingTableResponse struct {
|
||||
NodeID string
|
||||
NumBuckets int
|
||||
Buckets []RpcBucketResponse
|
||||
}
|
||||
|
||||
func (rpc *rpcReceiver) GetRoutingTable(r *http.Request, args *struct{}, result *RpcRoutingTableResponse) error {
|
||||
result.NodeID = rpc.dht.node.id.String()
|
||||
result.NumBuckets = len(rpc.dht.node.rt.buckets)
|
||||
for _, b := range rpc.dht.node.rt.buckets {
|
||||
result.Buckets = append(result.Buckets, RpcBucketResponse{
|
||||
Start: b.Range.Start.String(),
|
||||
End: b.Range.End.String(),
|
||||
NumContacts: b.Len(),
|
||||
Contacts: b.Contacts(),
|
||||
})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (rpc *rpcReceiver) AddKnownNode(r *http.Request, args *Contact, result *string) error {
|
||||
rpc.dht.node.AddKnownNode(*args)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (dht *DHT) runRPCServer(port int) {
|
||||
addr := "0.0.0.0:" + strconv.Itoa(port)
|
||||
|
||||
s := rpc2.NewServer()
|
||||
s.RegisterCodec(json.NewCodec(), "application/json")
|
||||
s.RegisterCodec(json.NewCodec(), "application/json;charset=UTF-8")
|
||||
err := s.RegisterService(&rpcReceiver{dht: dht}, "rpc")
|
||||
if err != nil {
|
||||
log.Error(errors.Prefix("registering rpc service", err))
|
||||
return
|
||||
}
|
||||
|
||||
handler := mux.NewRouter()
|
||||
handler.Handle("/", s)
|
||||
server := &http.Server{Addr: addr, Handler: handler}
|
||||
|
||||
wg := sync.WaitGroup{}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
log.Printf("RPC server listening on %s", addr)
|
||||
err := server.ListenAndServe()
|
||||
if err != nil && err != http.ErrServerClosed {
|
||||
log.Error(err)
|
||||
}
|
||||
}()
|
||||
|
||||
<-dht.grp.Ch()
|
||||
err = server.Shutdown(context.Background())
|
||||
if err != nil {
|
||||
log.Error(errors.Prefix("shutting down rpc service", err))
|
||||
return
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
)
|
||||
|
||||
// TODO: expire stored data after tExpire time
|
||||
|
||||
type contactStore struct {
|
||||
// map of blob hashes to (map of node IDs to bools)
|
||||
hashes map[bits.Bitmap]map[bits.Bitmap]bool
|
||||
// stores the peers themselves, so they can be updated in one place
|
||||
contacts map[bits.Bitmap]Contact
|
||||
lock sync.RWMutex
|
||||
}
|
||||
|
||||
func newStore() *contactStore {
|
||||
return &contactStore{
|
||||
hashes: make(map[bits.Bitmap]map[bits.Bitmap]bool),
|
||||
contacts: make(map[bits.Bitmap]Contact),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *contactStore) Upsert(blobHash bits.Bitmap, contact Contact) {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
|
||||
if _, ok := s.hashes[blobHash]; !ok {
|
||||
s.hashes[blobHash] = make(map[bits.Bitmap]bool)
|
||||
}
|
||||
s.hashes[blobHash][contact.ID] = true
|
||||
s.contacts[contact.ID] = contact
|
||||
}
|
||||
|
||||
func (s *contactStore) Get(blobHash bits.Bitmap) []Contact {
|
||||
s.lock.RLock()
|
||||
defer s.lock.RUnlock()
|
||||
|
||||
var contacts []Contact
|
||||
if ids, ok := s.hashes[blobHash]; ok {
|
||||
for id := range ids {
|
||||
contact, ok := s.contacts[id]
|
||||
if !ok {
|
||||
panic("node id in IDs list, but not in nodeInfo")
|
||||
}
|
||||
contacts = append(contacts, contact)
|
||||
}
|
||||
}
|
||||
return contacts
|
||||
}
|
||||
|
||||
func (s *contactStore) RemoveTODO(contact Contact) {
|
||||
// TODO: remove peer from everywhere
|
||||
}
|
||||
|
||||
func (s *contactStore) CountStoredHashes() int {
|
||||
s.lock.RLock()
|
||||
defer s.lock.RUnlock()
|
||||
return len(s.hashes)
|
||||
}
|
||||
-312
@@ -1,312 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
)
|
||||
|
||||
var testingDHTIP = "127.0.0.1"
|
||||
var testingDHTFirstPort = 21000
|
||||
|
||||
// TestingCreateNetwork initializes a testable DHT network with a specific number of nodes, with bootstrap and concurrent options.
|
||||
func TestingCreateNetwork(t *testing.T, numNodes int, bootstrap, concurrent bool) (*BootstrapNode, []*DHT) {
|
||||
var bootstrapNode *BootstrapNode
|
||||
var seeds []string
|
||||
|
||||
if bootstrap {
|
||||
bootstrapAddress := testingDHTIP + ":" + strconv.Itoa(testingDHTFirstPort)
|
||||
seeds = []string{bootstrapAddress}
|
||||
bootstrapNode = NewBootstrapNode(bits.Rand(), 0, bootstrapDefaultRefreshDuration)
|
||||
listener, err := net.ListenPacket(Network, bootstrapAddress)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
err = bootstrapNode.Connect(listener.(*net.UDPConn))
|
||||
if err != nil {
|
||||
t.Error("error connecting bootstrap node - ", err)
|
||||
}
|
||||
}
|
||||
|
||||
if numNodes < 1 {
|
||||
return bootstrapNode, nil
|
||||
}
|
||||
|
||||
firstPort := testingDHTFirstPort + 1
|
||||
dhts := make([]*DHT, numNodes)
|
||||
|
||||
for i := 0; i < numNodes; i++ {
|
||||
c := NewStandardConfig()
|
||||
c.NodeID = bits.Rand().Hex()
|
||||
c.Address = testingDHTIP + ":" + strconv.Itoa(firstPort+i)
|
||||
c.SeedNodes = seeds
|
||||
dht := New(c)
|
||||
|
||||
go func() {
|
||||
err := dht.Start()
|
||||
if err != nil {
|
||||
t.Error("error starting dht - ", err)
|
||||
}
|
||||
}()
|
||||
if !concurrent {
|
||||
dht.WaitUntilJoined()
|
||||
}
|
||||
dhts[i] = dht
|
||||
}
|
||||
|
||||
if concurrent {
|
||||
for _, d := range dhts {
|
||||
d.WaitUntilJoined()
|
||||
}
|
||||
}
|
||||
|
||||
return bootstrapNode, dhts
|
||||
}
|
||||
|
||||
type timeoutErr struct {
|
||||
error
|
||||
}
|
||||
|
||||
func (t timeoutErr) Timeout() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (t timeoutErr) Temporary() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// TODO: just use a normal net.Conn instead of this mock conn
|
||||
|
||||
type testUDPPacket struct {
|
||||
data []byte
|
||||
addr *net.UDPAddr
|
||||
}
|
||||
|
||||
type testUDPConn struct {
|
||||
addr *net.UDPAddr
|
||||
toRead chan testUDPPacket
|
||||
writes chan testUDPPacket
|
||||
|
||||
readDeadline time.Time
|
||||
}
|
||||
|
||||
func newTestUDPConn(addr string) *testUDPConn {
|
||||
parts := strings.Split(addr, ":")
|
||||
if len(parts) != 2 {
|
||||
panic("addr needs ip and port")
|
||||
}
|
||||
port, err := strconv.Atoi(parts[1])
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return &testUDPConn{
|
||||
addr: &net.UDPAddr{IP: net.IP(parts[0]), Port: port},
|
||||
toRead: make(chan testUDPPacket),
|
||||
writes: make(chan testUDPPacket),
|
||||
}
|
||||
}
|
||||
|
||||
func (t testUDPConn) ReadFromUDP(b []byte) (int, *net.UDPAddr, error) {
|
||||
var timeoutCh <-chan time.Time
|
||||
if !t.readDeadline.IsZero() {
|
||||
timeoutCh = time.After(time.Until(t.readDeadline))
|
||||
}
|
||||
|
||||
select {
|
||||
case packet, ok := <-t.toRead:
|
||||
if !ok {
|
||||
return 0, nil, errors.Err("conn closed")
|
||||
}
|
||||
n := copy(b, packet.data)
|
||||
return n, packet.addr, nil
|
||||
case <-timeoutCh:
|
||||
return 0, nil, timeoutErr{errors.Err("timeout")}
|
||||
}
|
||||
}
|
||||
|
||||
func (t testUDPConn) WriteToUDP(b []byte, addr *net.UDPAddr) (int, error) {
|
||||
t.writes <- testUDPPacket{data: b, addr: addr}
|
||||
return len(b), nil
|
||||
}
|
||||
|
||||
func (t *testUDPConn) SetReadDeadline(tm time.Time) error {
|
||||
t.readDeadline = tm
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *testUDPConn) SetWriteDeadline(tm time.Time) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *testUDPConn) Close() error {
|
||||
close(t.toRead)
|
||||
t.writes = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
func verifyResponse(t *testing.T, resp map[string]interface{}, id messageID, dhtNodeID string) {
|
||||
if len(resp) != 4 {
|
||||
t.Errorf("expected 4 response fields, got %d", len(resp))
|
||||
}
|
||||
|
||||
_, ok := resp[headerTypeField]
|
||||
if !ok {
|
||||
t.Error("missing type field")
|
||||
} else {
|
||||
rType, ok := resp[headerTypeField].(int64)
|
||||
if !ok {
|
||||
t.Error("type is not an integer")
|
||||
} else if rType != responseType {
|
||||
t.Error("unexpected response type")
|
||||
}
|
||||
}
|
||||
|
||||
_, ok = resp[headerMessageIDField]
|
||||
if !ok {
|
||||
t.Error("missing message id field")
|
||||
} else {
|
||||
rMessageID, ok := resp[headerMessageIDField].(string)
|
||||
if !ok {
|
||||
t.Error("message ID is not a string")
|
||||
} else if rMessageID != string(id[:]) {
|
||||
t.Error("unexpected message ID")
|
||||
}
|
||||
if len(rMessageID) != messageIDLength {
|
||||
t.Errorf("message ID should be %d chars long", messageIDLength)
|
||||
}
|
||||
}
|
||||
|
||||
_, ok = resp[headerNodeIDField]
|
||||
if !ok {
|
||||
t.Error("missing node id field")
|
||||
} else {
|
||||
rNodeID, ok := resp[headerNodeIDField].(string)
|
||||
if !ok {
|
||||
t.Error("node ID is not a string")
|
||||
} else if rNodeID != dhtNodeID {
|
||||
t.Error("unexpected node ID")
|
||||
}
|
||||
if len(rNodeID) != nodeIDLength {
|
||||
t.Errorf("node ID should be %d chars long", nodeIDLength)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func verifyContacts(t *testing.T, contacts []interface{}, nodes []Contact) {
|
||||
if len(contacts) != len(nodes) {
|
||||
t.Errorf("got %d contacts; expected %d", len(contacts), len(nodes))
|
||||
return
|
||||
}
|
||||
|
||||
foundNodes := make(map[string]bool)
|
||||
|
||||
for _, c := range contacts {
|
||||
contact, ok := c.([]interface{})
|
||||
if !ok {
|
||||
t.Error("contact is not a list")
|
||||
return
|
||||
}
|
||||
|
||||
if len(contact) != 3 {
|
||||
t.Error("contact must be 3 items")
|
||||
return
|
||||
}
|
||||
|
||||
var currNode Contact
|
||||
currNodeFound := false
|
||||
|
||||
id, ok := contact[0].(string)
|
||||
if !ok {
|
||||
t.Error("contact id is not a string")
|
||||
} else {
|
||||
if _, ok := foundNodes[id]; ok {
|
||||
t.Errorf("contact %s appears multiple times", id)
|
||||
continue
|
||||
}
|
||||
for _, n := range nodes {
|
||||
if n.ID.RawString() == id {
|
||||
currNode = n
|
||||
currNodeFound = true
|
||||
foundNodes[id] = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !currNodeFound {
|
||||
t.Errorf("unexpected contact %s", id)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
ip, ok := contact[1].(string)
|
||||
if !ok {
|
||||
t.Error("contact IP is not a string")
|
||||
} else if !currNode.IP.Equal(net.ParseIP(ip)) {
|
||||
t.Errorf("contact IP mismatch. got %s; expected %s", ip, currNode.IP.String())
|
||||
}
|
||||
|
||||
port, ok := contact[2].(int64)
|
||||
if !ok {
|
||||
t.Error("contact port is not an int")
|
||||
} else if int(port) != currNode.Port {
|
||||
t.Errorf("contact port mismatch. got %d; expected %d", port, currNode.Port)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func verifyCompactContacts(t *testing.T, contacts []interface{}, nodes []Contact) {
|
||||
if len(contacts) != len(nodes) {
|
||||
t.Errorf("got %d contacts; expected %d", len(contacts), len(nodes))
|
||||
return
|
||||
}
|
||||
|
||||
foundNodes := make(map[string]bool)
|
||||
|
||||
for _, c := range contacts {
|
||||
compact, ok := c.(string)
|
||||
if !ok {
|
||||
t.Error("contact is not a string")
|
||||
return
|
||||
}
|
||||
|
||||
contact := Contact{}
|
||||
err := contact.UnmarshalCompact([]byte(compact))
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
var currNode Contact
|
||||
currNodeFound := false
|
||||
|
||||
if _, ok := foundNodes[contact.ID.Hex()]; ok {
|
||||
t.Errorf("contact %s appears multiple times", contact.ID.Hex())
|
||||
continue
|
||||
}
|
||||
for _, n := range nodes {
|
||||
if n.ID.Equals(contact.ID) {
|
||||
currNode = n
|
||||
currNodeFound = true
|
||||
foundNodes[contact.ID.Hex()] = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !currNodeFound {
|
||||
t.Errorf("unexpected contact %s", contact.ID.Hex())
|
||||
continue
|
||||
}
|
||||
|
||||
if !currNode.IP.Equal(contact.IP) {
|
||||
t.Errorf("contact IP mismatch. got %s; expected %s", contact.IP.String(), currNode.IP.String())
|
||||
}
|
||||
|
||||
if contact.Port != currNode.Port {
|
||||
t.Errorf("contact port mismatch. got %d; expected %d", contact.Port, currNode.Port)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,71 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
|
||||
"github.com/lbryio/lbry.go/stop"
|
||||
)
|
||||
|
||||
// TODO: this should be moved out of dht and into node, and it should be completely hidden inside node. dht should not need to know about tokens
|
||||
|
||||
type tokenCacheEntry struct {
|
||||
token string
|
||||
receivedAt time.Time
|
||||
}
|
||||
|
||||
type tokenCache struct {
|
||||
node *Node
|
||||
tokens map[string]tokenCacheEntry
|
||||
expiration time.Duration
|
||||
lock *sync.RWMutex
|
||||
}
|
||||
|
||||
func newTokenCache(node *Node, expiration time.Duration) *tokenCache {
|
||||
tc := &tokenCache{}
|
||||
tc.node = node
|
||||
tc.tokens = make(map[string]tokenCacheEntry)
|
||||
tc.expiration = expiration
|
||||
tc.lock = &sync.RWMutex{}
|
||||
return tc
|
||||
}
|
||||
|
||||
// TODO: if store fails, get new token. can happen if a node restarts but we have the token cached
|
||||
|
||||
func (tc *tokenCache) Get(c Contact, hash bits.Bitmap, cancelCh stop.Chan) string {
|
||||
tc.lock.RLock()
|
||||
token, exists := tc.tokens[c.String()]
|
||||
tc.lock.RUnlock()
|
||||
|
||||
if exists && time.Since(token.receivedAt) < tc.expiration {
|
||||
return token.token
|
||||
}
|
||||
|
||||
resCh := tc.node.SendAsync(c, Request{
|
||||
Method: findValueMethod,
|
||||
Arg: &hash,
|
||||
})
|
||||
|
||||
var res *Response
|
||||
|
||||
select {
|
||||
case res = <-resCh:
|
||||
case <-cancelCh:
|
||||
return ""
|
||||
}
|
||||
|
||||
if res == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
tc.lock.Lock()
|
||||
tc.tokens[c.String()] = tokenCacheEntry{
|
||||
token: res.Token,
|
||||
receivedAt: time.Now(),
|
||||
}
|
||||
tc.lock.Unlock()
|
||||
|
||||
return res.Token
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"net"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/stop"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
)
|
||||
|
||||
type tokenManager struct {
|
||||
secret []byte
|
||||
prevSecret []byte
|
||||
lock *sync.RWMutex
|
||||
stop *stop.Group
|
||||
}
|
||||
|
||||
func (tm *tokenManager) Start(interval time.Duration) {
|
||||
tm.secret = make([]byte, 64)
|
||||
tm.prevSecret = make([]byte, 64)
|
||||
tm.lock = &sync.RWMutex{}
|
||||
tm.stop = stop.New()
|
||||
|
||||
tm.rotateSecret()
|
||||
|
||||
tm.stop.Add(1)
|
||||
go func() {
|
||||
defer tm.stop.Done()
|
||||
tick := time.NewTicker(interval)
|
||||
for {
|
||||
select {
|
||||
case <-tick.C:
|
||||
tm.rotateSecret()
|
||||
case <-tm.stop.Ch():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (tm *tokenManager) Stop() {
|
||||
tm.stop.StopAndWait()
|
||||
}
|
||||
|
||||
func (tm *tokenManager) Get(nodeID bits.Bitmap, addr *net.UDPAddr) string {
|
||||
return genToken(tm.secret, nodeID, addr)
|
||||
}
|
||||
|
||||
func (tm *tokenManager) Verify(token string, nodeID bits.Bitmap, addr *net.UDPAddr) bool {
|
||||
return token == genToken(tm.secret, nodeID, addr) || token == genToken(tm.prevSecret, nodeID, addr)
|
||||
}
|
||||
|
||||
func genToken(secret []byte, nodeID bits.Bitmap, addr *net.UDPAddr) string {
|
||||
buf := bytes.Buffer{}
|
||||
buf.Write(nodeID[:])
|
||||
buf.Write(addr.IP)
|
||||
buf.WriteString(strconv.Itoa(addr.Port))
|
||||
buf.Write(secret)
|
||||
t := sha256.Sum256(buf.Bytes())
|
||||
return string(t[:])
|
||||
}
|
||||
|
||||
func (tm *tokenManager) rotateSecret() {
|
||||
tm.lock.Lock()
|
||||
defer tm.lock.Unlock()
|
||||
|
||||
copy(tm.prevSecret, tm.secret)
|
||||
|
||||
_, err := rand.Read(tm.secret)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
module github.com/lbryio/reflector.go
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.34.0 // indirect
|
||||
github.com/armon/go-metrics v0.0.0-20180713145231-3c58d8115a78
|
||||
github.com/aws/aws-sdk-go v1.16.11
|
||||
github.com/btcsuite/btcd v0.0.0-20190109040709-5bda5314ca95 // indirect
|
||||
github.com/btcsuite/btcutil v0.0.0-20180706230648-ab6388e0c60a
|
||||
github.com/btcsuite/goleveldb v1.0.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1
|
||||
github.com/go-errors/errors v1.0.1
|
||||
github.com/go-ini/ini v1.41.0
|
||||
github.com/go-sql-driver/mysql v0.0.0-20180719071942-99ff426eb706
|
||||
github.com/golang/mock v1.2.0 // indirect
|
||||
github.com/golang/protobuf v1.3.1
|
||||
github.com/gorilla/context v1.1.1
|
||||
github.com/gorilla/mux v1.6.2
|
||||
github.com/gorilla/rpc v1.1.0
|
||||
github.com/gorilla/websocket v1.4.0
|
||||
github.com/hashicorp/errwrap v0.0.0-20180715044906-d6c0cd880357
|
||||
github.com/hashicorp/go-immutable-radix v0.0.0-20180129170900-7f3cd4390caa
|
||||
github.com/hashicorp/go-msgpack v0.0.0-20150518234257-fa3f63826f7c
|
||||
github.com/hashicorp/go-multierror v0.0.0-20180717150148-3d5d8f294aa0
|
||||
github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86
|
||||
github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47
|
||||
github.com/hashicorp/memberlist v0.1.0
|
||||
github.com/hashicorp/serf v0.0.0-20180530155958-984a73625de3
|
||||
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf
|
||||
github.com/inconshreveable/mousetrap v1.0.0
|
||||
github.com/jessevdk/go-flags v1.4.0 // indirect
|
||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af
|
||||
github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22
|
||||
github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07
|
||||
github.com/kkdai/bstream v0.0.0-20181106074824-b3251f7901ec // indirect
|
||||
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c
|
||||
github.com/lbryio/lbry.go v0.0.0-20190109223729-30c312501602
|
||||
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c
|
||||
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5
|
||||
github.com/miekg/dns v1.0.8
|
||||
github.com/mitchellh/mapstructure v1.1.2 // indirect
|
||||
github.com/nlopes/slack v0.4.0
|
||||
github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6
|
||||
github.com/pkg/errors v0.8.1 // indirect
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529
|
||||
github.com/sebdah/goldie v0.0.0-20180424091453-8784dd1ab561
|
||||
github.com/sergi/go-diff v1.0.0
|
||||
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 // indirect
|
||||
github.com/sirupsen/logrus v1.3.0
|
||||
github.com/spf13/cast v1.3.0
|
||||
github.com/spf13/cobra v0.0.0-20180722215644-7c4570c3ebeb
|
||||
github.com/spf13/pflag v1.0.1
|
||||
github.com/uber-go/atomic v1.3.2
|
||||
golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc
|
||||
golang.org/x/lint v0.0.0-20181217174547-8f45f776aaf1 // indirect
|
||||
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e
|
||||
golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890 // indirect
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 // indirect
|
||||
golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c
|
||||
golang.org/x/tools v0.0.0-20190109165630-d30e00c24034 // indirect
|
||||
google.golang.org/appengine v1.4.0
|
||||
google.golang.org/genproto v0.0.0-20190108161440-ae2f86662275 // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
|
||||
gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79
|
||||
gopkg.in/yaml.v2 v2.2.2 // indirect
|
||||
honnef.co/go/tools v0.0.0-20190109154334-5bcec433c8ea // indirect
|
||||
)
|
||||
@@ -0,0 +1,220 @@
|
||||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
|
||||
github.com/armon/go-metrics v0.0.0-20180713145231-3c58d8115a78 h1:mdRSArcFLfW0VoL34LZAKSz6LkkK4jFxVx2xYavACMg=
|
||||
github.com/armon/go-metrics v0.0.0-20180713145231-3c58d8115a78/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
|
||||
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
|
||||
github.com/aws/aws-sdk-go v0.0.0-20180806205706-c0447dbaaf19 h1:2di6C9H5QJ+D5LPazymJz8s2kRd8YkbN7knV17yH1Yw=
|
||||
github.com/aws/aws-sdk-go v0.0.0-20180806205706-c0447dbaaf19/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
|
||||
github.com/aws/aws-sdk-go v1.16.11 h1:g/c7gJeVyHoXCxM2fddS85bPGVkBF8s2q8t3fyElegc=
|
||||
github.com/aws/aws-sdk-go v1.16.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
||||
github.com/btcsuite/btcd v0.0.0-20180531025944-86fed781132a/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ=
|
||||
github.com/btcsuite/btcd v0.0.0-20190109040709-5bda5314ca95/go.mod h1:d3C0AkH6BRcvO8T0UEPu53cnw4IbV63x1bEjildYhO0=
|
||||
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
|
||||
github.com/btcsuite/btcutil v0.0.0-20180524032703-d4cc87b86016/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
|
||||
github.com/btcsuite/btcutil v0.0.0-20180706230648-ab6388e0c60a h1:RQMUrEILyYJEoAT34XS/kLu40vC0+po/UfxrBBA4qZE=
|
||||
github.com/btcsuite/btcutil v0.0.0-20180706230648-ab6388e0c60a/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
|
||||
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg=
|
||||
github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY=
|
||||
github.com/btcsuite/goleveldb v1.0.0/go.mod h1:QiK9vBlgftBg6rWQIj6wFzbPfRjiykIEhBH4obrXJ/I=
|
||||
github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
|
||||
github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
|
||||
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
|
||||
github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
|
||||
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
|
||||
github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-ini/ini v1.38.1 h1:hbtfM8emWUVo9GnXSloXYyFbXxZ+tG6sbepSStoe1FY=
|
||||
github.com/go-ini/ini v1.38.1/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-ini/ini v1.38.2/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-ini/ini v1.41.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
|
||||
github.com/go-sql-driver/mysql v0.0.0-20180719071942-99ff426eb706 h1:P3NPKb7qq581SeMCB+dU1SuCX1kQh8VoQ/4HmT2ftQY=
|
||||
github.com/go-sql-driver/mysql v0.0.0-20180719071942-99ff426eb706/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
|
||||
github.com/golang/lint v0.0.0-20181217174547-8f45f776aaf1/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
|
||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
|
||||
github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk=
|
||||
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
|
||||
github.com/gorilla/rpc v1.1.0 h1:marKfvVP0Gpd/jHlVBKCQ8RAoUPdX7K1Nuh6l1BNh7A=
|
||||
github.com/gorilla/rpc v1.1.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ=
|
||||
github.com/gorilla/websocket v1.2.0 h1:VJtLvh6VQym50czpZzx07z/kw9EgAxI3x1ZB8taTMQQ=
|
||||
github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
|
||||
github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
|
||||
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
|
||||
github.com/hashicorp/errwrap v0.0.0-20180715044906-d6c0cd880357 h1:Rem2+U35z1QtPQc6r+WolF7yXiefXqDKyk+lN2pE164=
|
||||
github.com/hashicorp/errwrap v0.0.0-20180715044906-d6c0cd880357/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/go-immutable-radix v0.0.0-20180129170900-7f3cd4390caa h1:0nA8i+6Rwqaq9xlpmVxxTwk6rxiEhX+E6Wh4vPNHiS8=
|
||||
github.com/hashicorp/go-immutable-radix v0.0.0-20180129170900-7f3cd4390caa/go.mod h1:6ij3Z20p+OhOkCSrA0gImAWoHYQRGbnlcuk6XYTiaRw=
|
||||
github.com/hashicorp/go-msgpack v0.0.0-20150518234257-fa3f63826f7c h1:BTAbnbegUIMB6xmQCwWE8yRzbA4XSpnZY5hvRJC188I=
|
||||
github.com/hashicorp/go-msgpack v0.0.0-20150518234257-fa3f63826f7c/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
|
||||
github.com/hashicorp/go-multierror v0.0.0-20180717150148-3d5d8f294aa0 h1:j30noezaCfvNLcdMYSvHLv81DxYRSt1grlpseG67vhU=
|
||||
github.com/hashicorp/go-multierror v0.0.0-20180717150148-3d5d8f294aa0/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=
|
||||
github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86 h1:7YOlAIO2YWnJZkQp7B5eFykaIY7C9JndqAFQyVV5BhM=
|
||||
github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
|
||||
github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47 h1:UnszMmmmm5vLwWzDjTFVIkfhvWF1NdrmChl8L2NUDCw=
|
||||
github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/memberlist v0.1.0 h1:qSsCiC0WYD39lbSitKNt40e30uorm2Ss/d4JGU1hzH8=
|
||||
github.com/hashicorp/memberlist v0.1.0/go.mod h1:ncdBp14cuox2iFOq3kDiquKU6fqsTBc3W6JvZwjxxsE=
|
||||
github.com/hashicorp/serf v0.0.0-20180530155958-984a73625de3 h1:NUr1hG6WO9sI1x8ofSimmpqfJ+rEHiHP/PLEA33rcfQ=
|
||||
github.com/hashicorp/serf v0.0.0-20180530155958-984a73625de3/go.mod h1:h/Ru6tmZazX7WO/GDmwdpS975F019L4t5ng5IgwbNrE=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf h1:WfD7VjIE6z8dIvMsI4/s+1qr5EL+zoIGev1BQj1eoJ8=
|
||||
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h1:hyb9oH7vZsitZCiBt0ZvifOrB+qc8PS5IiilCIb87rg=
|
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Aowv3l/EQUlocDHW2Cp4G9WJVH7uyH8QFJE=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
|
||||
github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22 h1:jKUP9TQ0c7X3w6+IPyMit07RE42MtTWNd77sN2cHngQ=
|
||||
github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22/go.mod h1:u0Jo4f2dNlTJeeOywkM6bLwxq6gC3pZ9rEFHn3AhTdk=
|
||||
github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07 h1:+kBG/8rjCa6vxJZbUjAiE4MQmBEBYc8nLEb51frnvBY=
|
||||
github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07/go.mod h1:j1kV/8f3jowErEq4XyeypkCdvg5EeHkf0YCKCcq5Ybo=
|
||||
github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
|
||||
github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
|
||||
github.com/kkdai/bstream v0.0.0-20181106074824-b3251f7901ec/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M=
|
||||
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8=
|
||||
github.com/lbryio/lbry.go v0.0.0-20180803110248-e2c96944fc48 h1:ojUrZuL8vqHxyhivNdpjDaIBkjzWktS9FOTTFkVG8yU=
|
||||
github.com/lbryio/lbry.go v0.0.0-20180803110248-e2c96944fc48/go.mod h1:jyMyYNmA1t7GTkgYs6z2kMnDFTFzlGBr+IkG9LXHk9M=
|
||||
github.com/lbryio/lbry.go v0.0.0-20190109223729-30c312501602 h1:HSv40ELfMPW2GIpyWwX2JaSAaYz5XW1GNweYc89F2kc=
|
||||
github.com/lbryio/lbry.go v0.0.0-20190109223729-30c312501602/go.mod h1:YEuFJD/oHNra6BFy+NfuvS84Wg6RMWJFGtiCCCc6MmQ=
|
||||
github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4=
|
||||
github.com/lbryio/types v0.0.0-20171215152337-0a913ba650dd h1:5wQgwcaLqLMaFIPju2QHcCseSHlArxpQZ1szEGzrp6Y=
|
||||
github.com/lbryio/types v0.0.0-20171215152337-0a913ba650dd/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
|
||||
github.com/lbryio/types v0.0.0-20181001180206-594241d24e00 h1:1qRpd8lcyVigX+kYkwQL13gpOURyytgvxZtuIQfPPX8=
|
||||
github.com/lbryio/types v0.0.0-20181001180206-594241d24e00/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
|
||||
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c h1:m3O7561xBQ00lfUVayW4c6SnpVbUDQtPUwGcGYSUYQA=
|
||||
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
|
||||
github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0=
|
||||
github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg=
|
||||
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 h1:mG83tLXWSRdcXMWfkoumVwhcCbf3jHF9QKv/m37BkM0=
|
||||
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCWffGOaDcjkw1iB7W9DVLp6GXmfcJY/7YZCWPA4=
|
||||
github.com/miekg/dns v1.0.8 h1:Zi8HNpze3NeRWH1PQV6O71YcvJRQ6j0lORO6DAEmAAI=
|
||||
github.com/miekg/dns v1.0.8/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
|
||||
github.com/mitchellh/mapstructure v0.0.0-20180511142126-bb74f1db0675/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/nlopes/slack v0.2.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM=
|
||||
github.com/nlopes/slack v0.3.0 h1:jCxvaS8wC4Bb1jnbqZMjCDkOOgy4spvQWcrw/TF0L0E=
|
||||
github.com/nlopes/slack v0.3.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM=
|
||||
github.com/nlopes/slack v0.4.0 h1:OVnHm7lv5gGT5gkcHsZAyw++oHVFihbjWbL3UceUpiA=
|
||||
github.com/nlopes/slack v0.4.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
|
||||
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6 h1:2bae6N0SZjgzk+Zg8mzTsfmpwHXY9VBNp9UdjhaElA0=
|
||||
github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
|
||||
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
|
||||
github.com/sebdah/goldie v0.0.0-20180424091453-8784dd1ab561 h1:IY+sDBJR/wRtsxq+626xJnt4Tw7/ROA9cDIR8MMhWyg=
|
||||
github.com/sebdah/goldie v0.0.0-20180424091453-8784dd1ab561/go.mod h1:lvjGftC8oe7XPtyrOidaMi0rp5B9+XY/ZRUynGnuaxQ=
|
||||
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/shopspring/decimal v0.0.0-20180607144847-19e3cb6c2930/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
|
||||
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
|
||||
github.com/sirupsen/logrus v0.0.0-20180523074243-ea8897e79973/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
|
||||
github.com/sirupsen/logrus v0.0.0-20180731161355-d329d24db431 h1:tJ2phHULXperiIq6Cl3t4MLypicinjmlM3Y+lNEipuo=
|
||||
github.com/sirupsen/logrus v0.0.0-20180731161355-d329d24db431/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
|
||||
github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME=
|
||||
github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
|
||||
github.com/spf13/cast v1.2.0 h1:HHl1DSRbEQN2i8tJmtS6ViPyHx35+p51amrdsiTCrkg=
|
||||
github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
|
||||
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
|
||||
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
|
||||
github.com/spf13/cobra v0.0.0-20180722215644-7c4570c3ebeb h1:9EsYJzSlhhaP+nYmMOcptMF2VEUH52jxPzt/TX14KWM=
|
||||
github.com/spf13/cobra v0.0.0-20180722215644-7c4570c3ebeb/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
|
||||
github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4=
|
||||
github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/uber-go/atomic v0.0.0-20180806045314-ca680462431f h1:vKpCeeburRE8ZXnuj9ptRqjm7WLD0O7ug28tRJuWM54=
|
||||
github.com/uber-go/atomic v0.0.0-20180806045314-ca680462431f/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
|
||||
github.com/uber-go/atomic v1.3.2 h1:Azu9lPBWRNKzYXSIwRfgRuDuS0YKsK4NFhiQv98gkxo=
|
||||
github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
|
||||
github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE=
|
||||
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20180807104621-f027049dab0a h1:PulT0Y50PcfTWomfsD39bSQyVrjjWdIuJKfyR4nOCJw=
|
||||
golang.org/x/crypto v0.0.0-20180807104621-f027049dab0a/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc h1:F5tKCVGp+MUAHhKp5MZtGqAlGX3+oCsiL1Q629FL90M=
|
||||
golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20181217174547-8f45f776aaf1/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180807145015-19491d39cadb h1:H5qWxL6CeEdaEGPqB5nnxJja1ULIcpuu91vIqAuGoJ0=
|
||||
golang.org/x/net v0.0.0-20180807145015-19491d39cadb/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e h1:bRhVy7zSSasaqNksaRZiA5EEI+Ei4I1nO5Jh72wfHlg=
|
||||
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180807141123-0718ef2ef256 h1:tKa4dTsBBeG8RnHO9sDPtJYNJNOc4ilC49ePrvii4To=
|
||||
golang.org/x/sys v0.0.0-20180807141123-0718ef2ef256/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb h1:1w588/yEchbPNpa9sEvOcMZYbWHedwJjg4VOAdDHWHk=
|
||||
golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2IVY3KZs6p9mix0ziNYJM=
|
||||
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190109165630-d30e00c24034/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190108161440-ae2f86662275/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
|
||||
google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
|
||||
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
|
||||
gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
|
||||
gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 h1:FpCr9V8wuOei4BAen+93HtVJ+XSi+KPbaPKm0Vj5R64=
|
||||
gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79/go.mod h1:gWkaRU7CoXpezCBWfWjm3999QqS+1pYPXGbqQCTMzo8=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190109154334-5bcec433c8ea/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
@@ -1,7 +1,13 @@
|
||||
package main
|
||||
|
||||
import "github.com/lbryio/reflector.go/cmd"
|
||||
import (
|
||||
"math/rand"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/cmd"
|
||||
)
|
||||
|
||||
func main() {
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
cmd.Execute()
|
||||
}
|
||||
@@ -1,3 +1,20 @@
|
||||
package meta
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
var Version = ""
|
||||
var Time = ""
|
||||
var BuildTime time.Time
|
||||
|
||||
func init() {
|
||||
if Time != "" {
|
||||
t, err := strconv.Atoi(Time)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
BuildTime = time.Unix(int64(t), 0).UTC()
|
||||
}
|
||||
}
|
||||
+135
-39
@@ -2,19 +2,18 @@ package peer
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"crypto/sha512"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/lbry.go/stop"
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
"github.com/lbryio/lbry.go/extras/stop"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
@@ -27,10 +26,14 @@ const (
|
||||
|
||||
// Server is an instance of a peer server that houses the listener and store.
|
||||
type Server struct {
|
||||
StatLogger *log.Logger // logger to log stats
|
||||
StatReportFrequency time.Duration // how often to log stats
|
||||
|
||||
store store.BlobStore
|
||||
closed bool
|
||||
|
||||
grp *stop.Group
|
||||
grp *stop.Group
|
||||
stats *reflector.Stats
|
||||
}
|
||||
|
||||
// NewServer returns an initialized Server pointer.
|
||||
@@ -44,6 +47,7 @@ func NewServer(store store.BlobStore) *Server {
|
||||
// Shutdown gracefully shuts down the peer server.
|
||||
func (s *Server) Shutdown() {
|
||||
log.Debug("shutting down peer server...")
|
||||
s.stats.Shutdown()
|
||||
s.grp.StopAndWait()
|
||||
log.Debug("peer server stopped")
|
||||
}
|
||||
@@ -63,6 +67,11 @@ func (s *Server) Start(address string) error {
|
||||
s.grp.Done()
|
||||
}()
|
||||
|
||||
s.stats = reflector.NewStatLogger("DOWNLOAD", s.StatLogger, s.StatReportFrequency, s.grp.Child())
|
||||
if s.StatLogger != nil && s.StatReportFrequency > 0 {
|
||||
s.stats.Start()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -82,7 +91,7 @@ func (s *Server) listenAndServe(listener net.Listener) {
|
||||
if s.closed {
|
||||
return
|
||||
}
|
||||
log.Error(err)
|
||||
log.Error(errors.Prefix("accepting conn", err))
|
||||
} else {
|
||||
s.grp.Add(1)
|
||||
go func() {
|
||||
@@ -100,7 +109,7 @@ func (s *Server) handleConnection(conn net.Conn) {
|
||||
}
|
||||
}()
|
||||
|
||||
timeoutDuration := 5 * time.Second
|
||||
timeoutDuration := 1 * time.Minute
|
||||
|
||||
for {
|
||||
var request []byte
|
||||
@@ -114,7 +123,7 @@ func (s *Server) handleConnection(conn net.Conn) {
|
||||
request, err = readNextRequest(conn)
|
||||
if err != nil {
|
||||
if err != io.EOF {
|
||||
log.Errorln(err)
|
||||
s.logError(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -124,33 +133,32 @@ func (s *Server) handleConnection(conn net.Conn) {
|
||||
log.Error(errors.FullTrace(err))
|
||||
}
|
||||
|
||||
if strings.Contains(string(request), `"requested_blobs"`) {
|
||||
log.Debugln("received availability request")
|
||||
response, err = s.handleAvailabilityRequest(request)
|
||||
} else if strings.Contains(string(request), `"blob_data_payment_rate"`) {
|
||||
log.Debugln("received rate negotiation request")
|
||||
response, err = s.handlePaymentRateNegotiation(request)
|
||||
} else if strings.Contains(string(request), `"requested_blob"`) {
|
||||
log.Debugln("received blob request")
|
||||
response, err = s.handleBlobRequest(request)
|
||||
} else {
|
||||
log.Errorln("invalid request")
|
||||
spew.Dump(request)
|
||||
return
|
||||
}
|
||||
response, err = s.handleCompositeRequest(request)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
err = conn.SetWriteDeadline(time.Now().Add(timeoutDuration))
|
||||
if err != nil {
|
||||
log.Error(errors.FullTrace(err))
|
||||
}
|
||||
|
||||
n, err := conn.Write(response)
|
||||
if err != nil {
|
||||
log.Errorln(err)
|
||||
if !strings.Contains(err.Error(), "connection reset by peer") { // means the other side closed the connection using TCP reset
|
||||
s.logError(err)
|
||||
}
|
||||
return
|
||||
} else if n != len(response) {
|
||||
log.Errorln(io.ErrShortWrite)
|
||||
return
|
||||
}
|
||||
|
||||
err = conn.SetWriteDeadline(time.Time{})
|
||||
if err != nil {
|
||||
log.Error(errors.FullTrace(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -197,7 +205,7 @@ func (s *Server) handleBlobRequest(data []byte) ([]byte, error) {
|
||||
return []byte{}, err
|
||||
}
|
||||
|
||||
log.Println("Sending blob " + request.RequestedBlob[:8])
|
||||
log.Debugln("Sending blob " + request.RequestedBlob[:8])
|
||||
|
||||
blob, err := s.store.Get(request.RequestedBlob)
|
||||
if err != nil {
|
||||
@@ -205,7 +213,7 @@ func (s *Server) handleBlobRequest(data []byte) ([]byte, error) {
|
||||
}
|
||||
|
||||
response, err := json.Marshal(blobResponse{IncomingBlob: incomingBlob{
|
||||
BlobHash: GetBlobHash(blob),
|
||||
BlobHash: reflector.BlobHash(blob),
|
||||
Length: len(blob),
|
||||
}})
|
||||
if err != nil {
|
||||
@@ -215,6 +223,91 @@ func (s *Server) handleBlobRequest(data []byte) ([]byte, error) {
|
||||
return append(response, blob...), nil
|
||||
}
|
||||
|
||||
func (s *Server) handleCompositeRequest(data []byte) ([]byte, error) {
|
||||
var request compositeRequest
|
||||
err := json.Unmarshal(data, &request)
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
|
||||
response := compositeResponse{
|
||||
LbrycrdAddress: LbrycrdAddress,
|
||||
}
|
||||
|
||||
if len(request.RequestedBlobs) > 0 {
|
||||
var availableBlobs []string
|
||||
for _, blobHash := range request.RequestedBlobs {
|
||||
exists, err := s.store.Has(blobHash)
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
if exists {
|
||||
availableBlobs = append(availableBlobs, blobHash)
|
||||
}
|
||||
}
|
||||
response.AvailableBlobs = availableBlobs
|
||||
}
|
||||
|
||||
response.BlobDataPaymentRate = paymentRateAccepted
|
||||
if request.BlobDataPaymentRate < 0 {
|
||||
response.BlobDataPaymentRate = paymentRateTooLow
|
||||
}
|
||||
|
||||
var blob []byte
|
||||
if request.RequestedBlob != "" {
|
||||
log.Debugln("Sending blob " + request.RequestedBlob[:8])
|
||||
|
||||
blob, err = s.store.Get(request.RequestedBlob)
|
||||
if errors.Is(err, store.ErrBlobNotFound) {
|
||||
response.IncomingBlob = incomingBlob{
|
||||
Error: err.Error(),
|
||||
}
|
||||
} else if err != nil {
|
||||
return []byte{}, err
|
||||
} else {
|
||||
response.IncomingBlob = incomingBlob{
|
||||
BlobHash: reflector.BlobHash(blob),
|
||||
Length: len(blob),
|
||||
}
|
||||
s.stats.AddBlob()
|
||||
}
|
||||
}
|
||||
|
||||
respData, err := json.Marshal(response)
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
|
||||
return append(respData, blob...), nil
|
||||
}
|
||||
|
||||
func (s *Server) logError(e error) {
|
||||
if e == nil {
|
||||
return
|
||||
}
|
||||
shouldLog := s.stats.AddError(e)
|
||||
if shouldLog {
|
||||
log.Errorln(errors.FullTrace(e))
|
||||
}
|
||||
|
||||
return
|
||||
|
||||
// old stuff below. its here for posterity, because we're gonna have to deal with these errors someday for real
|
||||
|
||||
//err := errors.Wrap(e, 0)
|
||||
|
||||
// these happen because the peer protocol does not have a way to cancel blob downloads
|
||||
// so the client will just close the connection if its in the middle of downloading a blob
|
||||
// but receives the blob from a different peer first or simply goes offline (timeout)
|
||||
//if strings.Contains(err.Error(), "connection reset by peer") ||
|
||||
// strings.Contains(err.Error(), "i/o timeout") ||
|
||||
// strings.Contains(err.Error(), "broken pipe") {
|
||||
// return
|
||||
//}
|
||||
//
|
||||
//log.Error(errors.FullTrace(e))
|
||||
}
|
||||
|
||||
func readNextRequest(conn net.Conn) ([]byte, error) {
|
||||
request := make([]byte, 0)
|
||||
eof := false
|
||||
@@ -224,7 +317,7 @@ func readNextRequest(conn net.Conn) ([]byte, error) {
|
||||
chunk, err := buf.ReadBytes('}')
|
||||
if err != nil {
|
||||
if err != io.EOF {
|
||||
log.Errorln("read error:", err)
|
||||
//log.Errorln("readBytes error:", err) // logged by caller
|
||||
return request, err
|
||||
}
|
||||
eof = true
|
||||
@@ -241,7 +334,7 @@ func readNextRequest(conn net.Conn) ([]byte, error) {
|
||||
}
|
||||
|
||||
// yes, this is how the peer protocol knows when the request finishes
|
||||
if isValidJSON(request) {
|
||||
if reflector.IsValidJSON(request) {
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -263,17 +356,6 @@ func readNextRequest(conn net.Conn) ([]byte, error) {
|
||||
return request, nil
|
||||
}
|
||||
|
||||
func isValidJSON(b []byte) bool {
|
||||
var r json.RawMessage
|
||||
return json.Unmarshal(b, &r) == nil
|
||||
}
|
||||
|
||||
// GetBlobHash returns the sha512 hash hex encoded string of the blob byte slice.
|
||||
func GetBlobHash(blob []byte) string {
|
||||
hashBytes := sha512.Sum384(blob)
|
||||
return hex.EncodeToString(hashBytes[:])
|
||||
}
|
||||
|
||||
const (
|
||||
maxRequestSize = 64 * (2 ^ 10) // 64kb
|
||||
paymentRateAccepted = "RATE_ACCEPTED"
|
||||
@@ -314,3 +396,17 @@ type incomingBlob struct {
|
||||
type blobResponse struct {
|
||||
IncomingBlob incomingBlob `json:"incoming_blob"`
|
||||
}
|
||||
|
||||
type compositeRequest struct {
|
||||
LbrycrdAddress bool `json:"lbrycrd_address"`
|
||||
RequestedBlobs []string `json:"requested_blobs"`
|
||||
BlobDataPaymentRate float64 `json:"blob_data_payment_rate"`
|
||||
RequestedBlob string `json:"requested_blob"`
|
||||
}
|
||||
|
||||
type compositeResponse struct {
|
||||
LbrycrdAddress string `json:"lbrycrd_address,omitempty"`
|
||||
AvailableBlobs []string `json:"available_blobs,omitempty"`
|
||||
BlobDataPaymentRate string `json:"blob_data_payment_rate,omitempty"`
|
||||
IncomingBlob incomingBlob `json:"incoming_blob,omitempty"`
|
||||
}
|
||||
+4
-4
@@ -5,16 +5,16 @@ import (
|
||||
"strconv"
|
||||
"sync"
|
||||
|
||||
"github.com/lbryio/lbry.go/dht"
|
||||
"github.com/lbryio/lbry.go/dht/bits"
|
||||
"github.com/lbryio/reflector.go/cluster"
|
||||
"github.com/lbryio/reflector.go/db"
|
||||
"github.com/lbryio/reflector.go/dht"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
"github.com/lbryio/reflector.go/peer"
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/lbry.go/stop"
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
"github.com/lbryio/lbry.go/extras/stop"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
+1
-1
@@ -5,7 +5,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
"github.com/lbryio/lbry.go/dht/bits"
|
||||
)
|
||||
|
||||
func TestAnnounceRange(t *testing.T) {
|
||||
|
||||
@@ -19,7 +19,7 @@ This project requires [Go v1.10](https://golang.org/doc/install) or higher.
|
||||
go get -u github.com/lbryio/reflector.go
|
||||
cd "$(go env GOPATH)/src/github.com/lbryio/reflector.go"
|
||||
make
|
||||
./prism
|
||||
./bin/prism-bin
|
||||
```
|
||||
|
||||
## Contributing
|
||||
@@ -32,9 +32,9 @@ This project is MIT licensed.
|
||||
|
||||
## Security
|
||||
|
||||
We take security seriously. Please contact security@lbry.io regarding any security issues.
|
||||
We take security seriously. Please contact security@lbry.com regarding any security issues.
|
||||
Our PGP key is [here](https://keybase.io/lbry/key.asc) if you need it.
|
||||
|
||||
## Contact
|
||||
|
||||
The primary contact for this project is [@lyoshenka](https://github.com/lyoshenka) (grin@lbry.io)
|
||||
The primary contact for this project is [@lyoshenka](https://github.com/lyoshenka) (grin@lbry.com)
|
||||
@@ -0,0 +1,173 @@
|
||||
package reflector
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
"github.com/lbryio/reflector.go/wallet"
|
||||
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
types1 "github.com/lbryio/types/v1/go"
|
||||
types2 "github.com/lbryio/types/v2/go"
|
||||
|
||||
"github.com/golang/protobuf/proto"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const blocklistURL = "https://api.lbry.io/file/list_blocked"
|
||||
|
||||
func (s *Server) enableBlocklist(b store.Blocklister) {
|
||||
// TODO: updateBlocklist should be killed when server is shutting down
|
||||
updateBlocklist(b)
|
||||
t := time.NewTicker(12 * time.Hour)
|
||||
for {
|
||||
select {
|
||||
case <-s.grp.Ch():
|
||||
return
|
||||
case <-t.C:
|
||||
updateBlocklist(b)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func updateBlocklist(b store.Blocklister) {
|
||||
values, err := blockedSdHashes()
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, v := range values {
|
||||
if v.Err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
err = b.Block(v.Value)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func blockedSdHashes() (map[string]valOrErr, error) {
|
||||
resp, err := http.Get(blocklistURL)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
defer func() {
|
||||
err := resp.Body.Close()
|
||||
if err != nil {
|
||||
log.Errorln(err)
|
||||
}
|
||||
}()
|
||||
|
||||
var r struct {
|
||||
Success bool `json:"success"`
|
||||
Error string `json:"error"`
|
||||
Data struct {
|
||||
Outpoints []string `json:"outpoints"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
if err = json.NewDecoder(resp.Body).Decode(&r); err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
if !r.Success {
|
||||
return nil, errors.Prefix("list_blocked API call", r.Error)
|
||||
}
|
||||
|
||||
return sdHashesForOutpoints(r.Data.Outpoints)
|
||||
}
|
||||
|
||||
type valOrErr struct {
|
||||
Value string
|
||||
Err error
|
||||
}
|
||||
|
||||
// sdHashesForOutpoints queries wallet server for the sd hashes in a given outpoints
|
||||
func sdHashesForOutpoints(outpoints []string) (map[string]valOrErr, error) {
|
||||
values := make(map[string]valOrErr)
|
||||
|
||||
node := wallet.NewNode()
|
||||
defer node.Shutdown()
|
||||
err := node.Connect([]string{
|
||||
"lbryumx1.lbry.io:50001",
|
||||
"lbryumx2.lbry.io:50001",
|
||||
}, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, outpoint := range outpoints {
|
||||
parts := strings.Split(outpoint, ":")
|
||||
if len(parts) != 2 {
|
||||
values[outpoint] = valOrErr{Err: errors.Err("invalid outpoint format")}
|
||||
continue
|
||||
}
|
||||
|
||||
nout, err := strconv.Atoi(parts[1])
|
||||
if err != nil {
|
||||
values[outpoint] = valOrErr{Err: errors.Prefix("invalid nout", err)}
|
||||
continue
|
||||
}
|
||||
|
||||
resp, err := node.GetClaimsInTx(parts[0])
|
||||
if err != nil {
|
||||
values[outpoint] = valOrErr{Err: err}
|
||||
continue
|
||||
}
|
||||
|
||||
var value []byte
|
||||
for _, tx := range resp.Result {
|
||||
if tx.Nout != nout {
|
||||
continue
|
||||
}
|
||||
|
||||
value, err = hex.DecodeString(tx.Value)
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
values[outpoint] = valOrErr{Err: err}
|
||||
continue
|
||||
} else if value == nil {
|
||||
values[outpoint] = valOrErr{Err: errors.Err("outpoint not found")}
|
||||
continue
|
||||
}
|
||||
|
||||
hash, err := hashFromClaim(value)
|
||||
values[outpoint] = valOrErr{Value: hash, Err: err}
|
||||
}
|
||||
|
||||
return values, nil
|
||||
}
|
||||
|
||||
func hashFromClaim(value []byte) (string, error) {
|
||||
claim := &types1.Claim{}
|
||||
err := proto.Unmarshal(value, claim)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if claim.GetStream().GetSource().GetSourceType() == types1.Source_lbry_sd_hash && claim.GetStream().GetSource().GetSource() != nil {
|
||||
return hex.EncodeToString(claim.GetStream().GetSource().GetSource()), nil
|
||||
}
|
||||
|
||||
claim2 := &types2.Claim{}
|
||||
err = proto.Unmarshal(value, claim2)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
stream, ok := claim2.GetType().(*types2.Claim_Stream)
|
||||
if !ok || stream == nil {
|
||||
return "", errors.Err("not a stream claim")
|
||||
}
|
||||
|
||||
return hex.EncodeToString(claim2.GetStream().GetSource().GetSdHash()), nil
|
||||
}
|
||||
+13
-12
@@ -3,9 +3,10 @@ package reflector
|
||||
import (
|
||||
"encoding/json"
|
||||
"net"
|
||||
"strconv"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
"github.com/lbryio/lbry.go/stream"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
@@ -36,20 +37,18 @@ func (c *Client) Close() error {
|
||||
}
|
||||
|
||||
// SendBlob sends a send blob request to the client.
|
||||
func (c *Client) SendBlob(blob []byte) error {
|
||||
func (c *Client) SendBlob(blob stream.Blob) error {
|
||||
if !c.connected {
|
||||
return errors.Err("not connected")
|
||||
}
|
||||
|
||||
if len(blob) > maxBlobSize {
|
||||
return errors.Err("blob must be at most " + strconv.Itoa(maxBlobSize) + " bytes")
|
||||
} else if len(blob) == 0 {
|
||||
return errors.Err("blob is empty")
|
||||
if err := blob.ValidForSend(); err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
blobHash := BlobHash(blob)
|
||||
blobHash := blob.HashHex()
|
||||
sendRequest, err := json.Marshal(sendBlobRequest{
|
||||
BlobSize: len(blob),
|
||||
BlobSize: blob.Size(),
|
||||
BlobHash: blobHash,
|
||||
})
|
||||
if err != nil {
|
||||
@@ -70,7 +69,7 @@ func (c *Client) SendBlob(blob []byte) error {
|
||||
}
|
||||
|
||||
if !sendResp.SendBlob {
|
||||
return ErrBlobExists
|
||||
return errors.Prefix(blobHash[:8], ErrBlobExists)
|
||||
}
|
||||
|
||||
log.Println("Sending blob " + blobHash[:8])
|
||||
@@ -97,7 +96,7 @@ func (c *Client) doHandshake(version int) error {
|
||||
return errors.Err("not connected")
|
||||
}
|
||||
|
||||
handshake, err := json.Marshal(handshakeRequestResponse{Version: version})
|
||||
handshake, err := json.Marshal(handshakeRequestResponse{Version: &version})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -111,7 +110,9 @@ func (c *Client) doHandshake(version int) error {
|
||||
err = json.NewDecoder(c.conn).Decode(&resp)
|
||||
if err != nil {
|
||||
return err
|
||||
} else if resp.Version != version {
|
||||
} else if resp.Version == nil {
|
||||
return errors.Err("invalid handshake")
|
||||
} else if *resp.Version != version {
|
||||
return errors.Err("handshake version mismatch")
|
||||
}
|
||||
|
||||
|
||||
+70
-26
@@ -6,14 +6,16 @@ import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/lbry.go/stop"
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
"github.com/lbryio/lbry.go/extras/stop"
|
||||
"github.com/lbryio/lbry.go/stream"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
@@ -27,7 +29,7 @@ const (
|
||||
network = "tcp4"
|
||||
protocolVersion1 = 0
|
||||
protocolVersion2 = 1
|
||||
maxBlobSize = 2 * 1024 * 1024
|
||||
maxBlobSize = stream.MaxBlobSize
|
||||
)
|
||||
|
||||
// Server is and instance of the reflector server. It houses the blob store and listener.
|
||||
@@ -37,9 +39,11 @@ type Server struct {
|
||||
StatLogger *log.Logger // logger to log stats
|
||||
StatReportFrequency time.Duration // how often to log stats
|
||||
|
||||
EnableBlocklist bool // if true, blocklist checking and blob deletion will be enabled
|
||||
|
||||
store store.BlobStore
|
||||
grp *stop.Group
|
||||
stats *stats
|
||||
stats *Stats
|
||||
}
|
||||
|
||||
// NewServer returns an initialized reflector server pointer.
|
||||
@@ -54,6 +58,7 @@ func NewServer(store store.BlobStore) *Server {
|
||||
// Shutdown shuts down the reflector server gracefully.
|
||||
func (s *Server) Shutdown() {
|
||||
log.Println("shutting down reflector server...")
|
||||
s.stats.Shutdown()
|
||||
s.grp.StopAndWait()
|
||||
log.Println("reflector server stopped")
|
||||
}
|
||||
@@ -82,11 +87,24 @@ func (s *Server) Start(address string) error {
|
||||
s.grp.Done()
|
||||
}()
|
||||
|
||||
s.stats = newStatLogger(s.StatLogger, s.StatReportFrequency, s.grp.Child())
|
||||
s.stats = NewStatLogger("UPLOAD", s.StatLogger, s.StatReportFrequency, s.grp.Child())
|
||||
if s.StatLogger != nil && s.StatReportFrequency > 0 {
|
||||
s.stats.Start()
|
||||
}
|
||||
|
||||
if s.EnableBlocklist {
|
||||
if b, ok := s.store.(store.Blocklister); ok {
|
||||
s.grp.Add(1)
|
||||
go func() {
|
||||
s.enableBlocklist(b)
|
||||
s.grp.Done()
|
||||
}()
|
||||
} else {
|
||||
//s.Shutdown()
|
||||
return errors.Err("blocklist is enabled but blob store does not support blocklisting")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -155,8 +173,10 @@ func (s *Server) handleConn(conn net.Conn) {
|
||||
}
|
||||
|
||||
func (s *Server) doError(conn net.Conn, err error) error {
|
||||
log.Errorln(errors.FullTrace(err))
|
||||
s.stats.AddError(err)
|
||||
shouldLog := s.stats.AddError(err)
|
||||
if shouldLog {
|
||||
log.Errorln(errors.FullTrace(err))
|
||||
}
|
||||
if e2, ok := err.(*json.SyntaxError); ok {
|
||||
log.Errorf("syntax error at byte offset %d", e2.Offset)
|
||||
}
|
||||
@@ -169,39 +189,46 @@ func (s *Server) doError(conn net.Conn, err error) error {
|
||||
}
|
||||
|
||||
func (s *Server) receiveBlob(conn net.Conn) error {
|
||||
var err error
|
||||
|
||||
blobSize, blobHash, isSdBlob, err := s.readBlobRequest(conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
blobExists, err := s.store.Has(blobHash)
|
||||
if err != nil {
|
||||
return err
|
||||
var wantsBlob bool
|
||||
if bl, ok := s.store.(store.Blocklister); ok {
|
||||
wantsBlob, err = bl.Wants(blobHash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
blobExists, err := s.store.Has(blobHash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
wantsBlob = !blobExists
|
||||
}
|
||||
|
||||
var neededBlobs []string
|
||||
|
||||
if isSdBlob && blobExists {
|
||||
if fsc, ok := s.store.(neededBlobChecker); ok {
|
||||
neededBlobs, err = fsc.MissingBlobsForKnownStream(blobHash)
|
||||
if isSdBlob && !wantsBlob {
|
||||
if nbc, ok := s.store.(neededBlobChecker); ok {
|
||||
neededBlobs, err = nbc.MissingBlobsForKnownStream(blobHash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
// if we can't confirm that we have the full stream, we have to say that the sd blob is
|
||||
// missing. if we say we have it, they wont try to send any content blobs
|
||||
blobExists = false
|
||||
// if we can't check for blobs in a stream, we have to say that the sd blob is
|
||||
// missing. if we say we have the sd blob, they wont try to send any content blobs
|
||||
wantsBlob = true
|
||||
}
|
||||
}
|
||||
|
||||
err = s.sendBlobResponse(conn, blobExists, isSdBlob, neededBlobs)
|
||||
err = s.sendBlobResponse(conn, wantsBlob, isSdBlob, neededBlobs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if blobExists {
|
||||
if !wantsBlob {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -247,7 +274,9 @@ func (s *Server) doHandshake(conn net.Conn) error {
|
||||
err := s.read(conn, &handshake)
|
||||
if err != nil {
|
||||
return err
|
||||
} else if handshake.Version != protocolVersion1 && handshake.Version != protocolVersion2 {
|
||||
} else if handshake.Version == nil {
|
||||
return errors.Err("handshake is missing protocol version")
|
||||
} else if *handshake.Version != protocolVersion1 && *handshake.Version != protocolVersion2 {
|
||||
return errors.Err("protocol version not supported")
|
||||
}
|
||||
|
||||
@@ -291,14 +320,14 @@ func (s *Server) readBlobRequest(conn net.Conn) (int, string, bool, error) {
|
||||
return blobSize, blobHash, isSdBlob, nil
|
||||
}
|
||||
|
||||
func (s *Server) sendBlobResponse(conn net.Conn, blobExists, isSdBlob bool, neededBlobs []string) error {
|
||||
func (s *Server) sendBlobResponse(conn net.Conn, shouldSendBlob, isSdBlob bool, neededBlobs []string) error {
|
||||
var response []byte
|
||||
var err error
|
||||
|
||||
if isSdBlob {
|
||||
response, err = json.Marshal(sendSdBlobResponse{SendSdBlob: !blobExists, NeededBlobs: neededBlobs})
|
||||
response, err = json.Marshal(sendSdBlobResponse{SendSdBlob: shouldSendBlob, NeededBlobs: neededBlobs})
|
||||
} else {
|
||||
response, err = json.Marshal(sendBlobResponse{SendBlob: !blobExists})
|
||||
response, err = json.Marshal(sendBlobResponse{SendBlob: shouldSendBlob})
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -329,7 +358,16 @@ func (s *Server) read(conn net.Conn, v interface{}) error {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
return errors.Err(json.NewDecoder(conn).Decode(v))
|
||||
dec := json.NewDecoder(conn)
|
||||
err = dec.Decode(v)
|
||||
if err != nil {
|
||||
data, _ := ioutil.ReadAll(dec.Buffered())
|
||||
if len(data) > 0 {
|
||||
return errors.Err("%s. Data: %s", err.Error(), hex.EncodeToString(data))
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) readRawBlob(conn net.Conn, blobSize int) ([]byte, error) {
|
||||
@@ -365,17 +403,23 @@ func (s *Server) quitting() bool {
|
||||
}
|
||||
}
|
||||
|
||||
// BlobHash returns the sha512 hash hex encoded string of the blob byte slice.
|
||||
func BlobHash(blob []byte) string {
|
||||
hashBytes := sha512.Sum384(blob)
|
||||
return hex.EncodeToString(hashBytes[:])
|
||||
}
|
||||
|
||||
func IsValidJSON(b []byte) bool {
|
||||
var r json.RawMessage
|
||||
return json.Unmarshal(b, &r) == nil
|
||||
}
|
||||
|
||||
//type errorResponse struct {
|
||||
// Error string `json:"error"`
|
||||
//}
|
||||
|
||||
type handshakeRequestResponse struct {
|
||||
Version int `json:"version"`
|
||||
Version *int `json:"version"`
|
||||
}
|
||||
|
||||
type sendBlobRequest struct {
|
||||
|
||||
+20
-18
@@ -2,19 +2,17 @@ package reflector
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"sort"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/lbryio/lbry.go/dht/bits"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"encoding/json"
|
||||
|
||||
"sort"
|
||||
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/phayes/freeport"
|
||||
)
|
||||
|
||||
@@ -146,6 +144,22 @@ func TestServer_Timeout(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
//func TestServer_InvalidJSONHandshake(t *testing.T) {
|
||||
// srv, port := startServerOnRandomPort(t)
|
||||
// defer srv.Shutdown()
|
||||
//
|
||||
// c := Client{}
|
||||
// err := c.Connect(":" + strconv.Itoa(port))
|
||||
// if err != nil {
|
||||
// t.Fatal("error connecting client to server", err)
|
||||
// }
|
||||
//
|
||||
// _, err = c.conn.Write([]byte(`{"stuff":4,tf}"`))
|
||||
// if err == nil {
|
||||
// t.Error("expected an error")
|
||||
// }
|
||||
//}
|
||||
|
||||
type mockPartialStore struct {
|
||||
store.MemoryBlobStore
|
||||
missing []string
|
||||
@@ -226,18 +240,6 @@ func TestServer_PartialUpload(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
//func MakeRandStream(size int) ([]byte, [][]byte) {
|
||||
// blobs := make([][]byte, int(math.Ceil(float64(size)/maxBlobSize)))
|
||||
// for i := 0; i < len(blobs); i++ {
|
||||
// blobs[i] = randBlob(int(math.Min(maxBlobSize, float64(size))))
|
||||
// size -= maxBlobSize
|
||||
// }
|
||||
//
|
||||
// //TODO: create SD blob for the stream
|
||||
//
|
||||
// return nil, blobs
|
||||
//}
|
||||
|
||||
func randBlob(size int) []byte {
|
||||
//if size > maxBlobSize {
|
||||
// panic("blob size too big")
|
||||
|
||||
+48
-19
@@ -2,37 +2,44 @@ package reflector
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/lbry.go/stop"
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
"github.com/lbryio/lbry.go/extras/stop"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type stats struct {
|
||||
// TODO: store daily stats too. and maybe other intervals
|
||||
|
||||
type Stats struct {
|
||||
mu *sync.Mutex
|
||||
blobs int
|
||||
streams int
|
||||
errors map[string]int
|
||||
started bool
|
||||
|
||||
name string
|
||||
logger *log.Logger
|
||||
logFreq time.Duration
|
||||
grp *stop.Group
|
||||
}
|
||||
|
||||
func newStatLogger(logger *log.Logger, logFreq time.Duration, parentGrp *stop.Group) *stats {
|
||||
return &stats{
|
||||
func NewStatLogger(name string, logger *log.Logger, logFreq time.Duration, parentGrp *stop.Group) *Stats {
|
||||
return &Stats{
|
||||
mu: &sync.Mutex{},
|
||||
grp: stop.New(parentGrp),
|
||||
logger: logger,
|
||||
logFreq: logFreq,
|
||||
errors: make(map[string]int),
|
||||
name: name,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *stats) Start() {
|
||||
func (s *Stats) Start() {
|
||||
s.started = true
|
||||
s.grp.Add(1)
|
||||
go func() {
|
||||
defer s.grp.Done()
|
||||
@@ -40,31 +47,51 @@ func (s *stats) Start() {
|
||||
}()
|
||||
}
|
||||
|
||||
func (s *stats) Shutdown() {
|
||||
func (s *Stats) Shutdown() {
|
||||
if !s.started {
|
||||
return
|
||||
}
|
||||
s.log()
|
||||
s.grp.StopAndWait()
|
||||
s.started = false
|
||||
}
|
||||
|
||||
func (s *stats) AddBlob() {
|
||||
func (s *Stats) AddBlob() {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.blobs += 1
|
||||
s.blobs++
|
||||
}
|
||||
func (s *stats) AddStream() {
|
||||
func (s *Stats) AddStream() {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.streams += 1
|
||||
s.streams++
|
||||
}
|
||||
func (s *stats) AddError(e error) {
|
||||
|
||||
func (s *Stats) AddError(e error) (shouldLog bool) { // shouldLog is a hack, but whatever
|
||||
if e == nil {
|
||||
return
|
||||
}
|
||||
err := errors.Wrap(e, 0)
|
||||
name := err.TypeName()
|
||||
if strings.Contains(err.Error(), "i/o timeout") { // hit a read or write deadline
|
||||
name = "i/o timeout"
|
||||
} else if strings.Contains(err.Error(), "read: connection reset by peer") { // the other side closed the connection using TCP reset
|
||||
name = "read conn reset"
|
||||
} else if strings.Contains(err.Error(), "unexpected EOF") { // tried to read from closed pipe or socket
|
||||
name = "unexpected EOF"
|
||||
} else if strings.Contains(err.Error(), "write: broken pipe") { // tried to write to a pipe or socket that was closed by the peer
|
||||
name = "write broken pipe"
|
||||
} else {
|
||||
shouldLog = true
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.errors[err.TypeName()] += 1
|
||||
s.errors[name]++
|
||||
return
|
||||
}
|
||||
|
||||
func (s *stats) runSlackLogger() {
|
||||
func (s *Stats) runSlackLogger() {
|
||||
t := time.NewTicker(s.logFreq)
|
||||
for {
|
||||
select {
|
||||
@@ -76,7 +103,7 @@ func (s *stats) runSlackLogger() {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *stats) log() {
|
||||
func (s *Stats) log() {
|
||||
s.mu.Lock()
|
||||
blobs, streams := s.blobs, s.streams
|
||||
s.blobs, s.streams = 0, 0
|
||||
@@ -86,8 +113,10 @@ func (s *stats) log() {
|
||||
delete(s.errors, name)
|
||||
}
|
||||
s.mu.Unlock()
|
||||
s.logger.Printf(
|
||||
"Stats: %d blobs, %d streams, errors: %s",
|
||||
blobs, streams, errStr[:len(errStr)-2], // trim last comma and space
|
||||
)
|
||||
|
||||
if len(errStr) > 2 {
|
||||
errStr = errStr[:len(errStr)-2] // trim last comma and space
|
||||
}
|
||||
|
||||
s.logger.Printf("%s stats: %d blobs, %d streams, errors: %s", s.name, blobs, streams, errStr)
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
package reflector
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/db"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
"github.com/lbryio/lbry.go/extras/stop"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type increment int
|
||||
|
||||
const (
|
||||
sdInc increment = iota + 1
|
||||
blobInc
|
||||
errInc
|
||||
)
|
||||
|
||||
type Uploader struct {
|
||||
db *db.SQL
|
||||
store *store.DBBackedS3Store // could just be store.BlobStore interface
|
||||
workers int
|
||||
skipExistsCheck bool
|
||||
stopper *stop.Group
|
||||
countChan chan increment
|
||||
|
||||
count struct {
|
||||
total, alreadyStored, sd, blob, err int
|
||||
}
|
||||
}
|
||||
|
||||
func NewUploader(db *db.SQL, store *store.DBBackedS3Store, workers int, skipExistsCheck bool) *Uploader {
|
||||
return &Uploader{
|
||||
db: db,
|
||||
store: store,
|
||||
workers: workers,
|
||||
skipExistsCheck: skipExistsCheck,
|
||||
stopper: stop.New(),
|
||||
countChan: make(chan increment),
|
||||
}
|
||||
}
|
||||
|
||||
func (u *Uploader) Stop() {
|
||||
log.Infoln("stopping uploader")
|
||||
u.stopper.StopAndWait()
|
||||
}
|
||||
|
||||
func (u *Uploader) Upload(dirOrFilePath string) error {
|
||||
paths, err := getPaths(dirOrFilePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
u.count.total = len(paths)
|
||||
|
||||
hashes := make([]string, len(paths))
|
||||
for i, p := range paths {
|
||||
hashes[i] = path.Base(p)
|
||||
}
|
||||
|
||||
log.Infoln("checking for existing blobs")
|
||||
|
||||
var exists map[string]bool
|
||||
if !u.skipExistsCheck {
|
||||
exists, err = u.db.HasBlobs(hashes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
u.count.alreadyStored = len(exists)
|
||||
}
|
||||
|
||||
log.Infof("%d new blobs to upload", u.count.total-u.count.alreadyStored)
|
||||
|
||||
workerWG := sync.WaitGroup{}
|
||||
pathChan := make(chan string)
|
||||
|
||||
for i := 0; i < u.workers; i++ {
|
||||
workerWG.Add(1)
|
||||
go func(i int) {
|
||||
defer workerWG.Done()
|
||||
defer func(i int) { log.Debugf("worker %d quitting", i) }(i)
|
||||
u.worker(pathChan)
|
||||
}(i)
|
||||
}
|
||||
|
||||
countWG := sync.WaitGroup{}
|
||||
countWG.Add(1)
|
||||
go func() {
|
||||
defer countWG.Done()
|
||||
u.counter()
|
||||
}()
|
||||
|
||||
Upload:
|
||||
for _, f := range paths {
|
||||
if exists != nil && exists[path.Base(f)] {
|
||||
continue
|
||||
}
|
||||
|
||||
select {
|
||||
case pathChan <- f:
|
||||
case <-u.stopper.Ch():
|
||||
break Upload
|
||||
}
|
||||
}
|
||||
|
||||
close(pathChan)
|
||||
workerWG.Wait()
|
||||
close(u.countChan)
|
||||
countWG.Wait()
|
||||
u.stopper.Stop()
|
||||
|
||||
log.Infoln("SUMMARY")
|
||||
log.Infof("%d blobs total", u.count.total)
|
||||
log.Infof("%d blobs already stored", u.count.alreadyStored)
|
||||
log.Infof("%d SD blobs uploaded", u.count.sd)
|
||||
log.Infof("%d content blobs uploaded", u.count.blob)
|
||||
log.Infof("%d errors encountered", u.count.err)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// worker reads paths from a channel and uploads them
|
||||
func (u *Uploader) worker(pathChan chan string) {
|
||||
for {
|
||||
select {
|
||||
case <-u.stopper.Ch():
|
||||
return
|
||||
case filepath, ok := <-pathChan:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
err := u.uploadBlob(filepath)
|
||||
if err != nil {
|
||||
log.Errorln(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// uploadBlob uploads a blob
|
||||
func (u *Uploader) uploadBlob(filepath string) (err error) {
|
||||
defer func() {
|
||||
if err != nil {
|
||||
u.inc(errInc)
|
||||
}
|
||||
}()
|
||||
|
||||
blob, err := ioutil.ReadFile(filepath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
hash := BlobHash(blob)
|
||||
if hash != path.Base(filepath) {
|
||||
return errors.Err("file name does not match hash (%s != %s), skipping", filepath, hash)
|
||||
}
|
||||
|
||||
if IsValidJSON(blob) {
|
||||
log.Debugf("Uploading SD blob %s", hash)
|
||||
err := u.store.PutSD(hash, blob)
|
||||
if err != nil {
|
||||
return errors.Prefix("Uploading SD blob "+hash, err)
|
||||
}
|
||||
u.inc(sdInc)
|
||||
} else {
|
||||
log.Debugf("Uploading blob %s", hash)
|
||||
err = u.store.Put(hash, blob)
|
||||
if err != nil {
|
||||
return errors.Prefix("Uploading blob "+hash, err)
|
||||
}
|
||||
u.inc(blobInc)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// counter updates the counts of how many sd blobs and content blobs were uploaded, and how many
|
||||
// errors were encountered. It occasionally prints the upload progress to debug.
|
||||
func (u *Uploader) counter() {
|
||||
start := time.Now()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-u.stopper.Ch():
|
||||
return
|
||||
case incrementType, ok := <-u.countChan:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
switch incrementType {
|
||||
case sdInc:
|
||||
u.count.sd++
|
||||
case blobInc:
|
||||
u.count.blob++
|
||||
case errInc:
|
||||
u.count.err++
|
||||
}
|
||||
}
|
||||
if (u.count.sd+u.count.blob)%50 == 0 {
|
||||
log.Infof("%d of %d done (%s elapsed, %.3fs per blob)", u.count.sd+u.count.blob, u.count.total-u.count.alreadyStored, time.Since(start).String(), time.Since(start).Seconds()/float64(u.count.sd+u.count.blob))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (u *Uploader) inc(t increment) {
|
||||
select {
|
||||
case u.countChan <- t:
|
||||
case <-u.stopper.Ch():
|
||||
}
|
||||
}
|
||||
|
||||
// getPaths returns the paths for files to upload. it takes a path to a file or a dir. for a file,
|
||||
// it returns the full path to that file. for a dir, it returns the paths for all the files in the
|
||||
// dir
|
||||
func getPaths(dirOrFilePath string) ([]string, error) {
|
||||
info, err := os.Stat(dirOrFilePath)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
if info.Mode().IsRegular() {
|
||||
return []string{dirOrFilePath}, nil
|
||||
}
|
||||
|
||||
f, err := os.Open(dirOrFilePath)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
files, err := f.Readdir(-1)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
err = f.Close()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
var filenames []string
|
||||
for _, file := range files {
|
||||
if !file.IsDir() {
|
||||
filenames = append(filenames, dirOrFilePath+"/"+file.Name())
|
||||
}
|
||||
}
|
||||
|
||||
return filenames, nil
|
||||
}
|
||||
+108
-5
@@ -2,15 +2,20 @@ package store
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"sync"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/reflector.go/db"
|
||||
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// DBBackedS3Store is an instance of an S3 Store that is backed by a DB for what is stored.
|
||||
type DBBackedS3Store struct {
|
||||
s3 *S3BlobStore
|
||||
db *db.SQL
|
||||
s3 *S3BlobStore
|
||||
db *db.SQL
|
||||
blockedMu sync.RWMutex
|
||||
blocked map[string]bool
|
||||
}
|
||||
|
||||
// NewDBBackedS3Store returns an initialized store pointer.
|
||||
@@ -18,12 +23,12 @@ func NewDBBackedS3Store(s3 *S3BlobStore, db *db.SQL) *DBBackedS3Store {
|
||||
return &DBBackedS3Store{s3: s3, db: db}
|
||||
}
|
||||
|
||||
// Has returns T/F or Error ( if the DB errors ) if store contains the blob.
|
||||
// Has returns true if the blob is in the store
|
||||
func (d *DBBackedS3Store) Has(hash string) (bool, error) {
|
||||
return d.db.HasBlob(hash)
|
||||
}
|
||||
|
||||
// Get returns the byte slice of the blob or an error.
|
||||
// Get gets the blob
|
||||
func (d *DBBackedS3Store) Get(hash string) ([]byte, error) {
|
||||
return d.s3.Get(hash)
|
||||
}
|
||||
@@ -58,9 +63,107 @@ func (d *DBBackedS3Store) PutSD(hash string, blob []byte) error {
|
||||
return d.db.AddSDBlob(hash, len(blob), blobContents)
|
||||
}
|
||||
|
||||
func (d *DBBackedS3Store) Delete(hash string) error {
|
||||
err := d.s3.Delete(hash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return d.db.Delete(hash)
|
||||
}
|
||||
|
||||
// Block deletes the blob and prevents it from being uploaded in the future
|
||||
func (d *DBBackedS3Store) Block(hash string) error {
|
||||
if blocked, err := d.isBlocked(hash); blocked || err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Debugf("blocking %s", hash)
|
||||
|
||||
err := d.db.Block(hash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
has, err := d.db.HasBlob(hash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if has {
|
||||
err = d.s3.Delete(hash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = d.db.Delete(hash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return d.markBlocked(hash)
|
||||
}
|
||||
|
||||
// Wants returns false if the hash exists or is blocked, true otherwise
|
||||
func (d *DBBackedS3Store) Wants(hash string) (bool, error) {
|
||||
blocked, err := d.isBlocked(hash)
|
||||
if blocked || err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
has, err := d.Has(hash)
|
||||
return !has, err
|
||||
}
|
||||
|
||||
// MissingBlobsForKnownStream returns missing blobs for an existing stream
|
||||
// WARNING: if the stream does NOT exist, no blob hashes will be returned, which looks
|
||||
// like no blobs are missing
|
||||
func (d *DBBackedS3Store) MissingBlobsForKnownStream(sdHash string) ([]string, error) {
|
||||
return d.db.MissingBlobsForKnownStream(sdHash)
|
||||
}
|
||||
|
||||
func (d *DBBackedS3Store) markBlocked(hash string) error {
|
||||
err := d.initBlocked()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
d.blockedMu.Lock()
|
||||
defer d.blockedMu.Unlock()
|
||||
|
||||
d.blocked[hash] = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *DBBackedS3Store) isBlocked(hash string) (bool, error) {
|
||||
err := d.initBlocked()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
d.blockedMu.RLock()
|
||||
defer d.blockedMu.RUnlock()
|
||||
|
||||
return d.blocked[hash], nil
|
||||
}
|
||||
|
||||
func (d *DBBackedS3Store) initBlocked() error {
|
||||
// first check without blocking since this is the most likely scenario
|
||||
if d.blocked != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
d.blockedMu.Lock()
|
||||
defer d.blockedMu.Unlock()
|
||||
|
||||
// check again in case of race condition
|
||||
if d.blocked != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var err error
|
||||
d.blocked, err = d.db.GetBlocked()
|
||||
|
||||
return err
|
||||
}
|
||||
+19
-1
@@ -5,7 +5,7 @@ import (
|
||||
"os"
|
||||
"path"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
)
|
||||
|
||||
// FileBlobStore is a local disk store.
|
||||
@@ -95,3 +95,21 @@ func (f *FileBlobStore) Put(hash string, blob []byte) error {
|
||||
func (f *FileBlobStore) PutSD(hash string, blob []byte) error {
|
||||
return f.Put(hash, blob)
|
||||
}
|
||||
|
||||
// Delete deletes the blob from the store
|
||||
func (f *FileBlobStore) Delete(hash string) error {
|
||||
err := f.initOnce()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
has, err := f.Has(hash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !has {
|
||||
return nil
|
||||
}
|
||||
|
||||
return os.Remove(f.path(hash))
|
||||
}
|
||||
+12
-1
@@ -1,6 +1,6 @@
|
||||
package store
|
||||
|
||||
import "github.com/lbryio/lbry.go/errors"
|
||||
import "github.com/lbryio/lbry.go/extras/errors"
|
||||
|
||||
// MemoryBlobStore is an in memory only blob store with no persistence.
|
||||
type MemoryBlobStore struct {
|
||||
@@ -41,3 +41,14 @@ func (m *MemoryBlobStore) Put(hash string, blob []byte) error {
|
||||
func (m *MemoryBlobStore) PutSD(hash string, blob []byte) error {
|
||||
return m.Put(hash, blob)
|
||||
}
|
||||
|
||||
// Delete deletes the blob from the store
|
||||
func (m *MemoryBlobStore) Delete(hash string) error {
|
||||
delete(m.blobs, hash)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Debug returns the blobs in memory. It's useful for testing and debugging.
|
||||
func (m *MemoryBlobStore) Debug() map[string][]byte {
|
||||
return m.blobs
|
||||
}
|
||||
@@ -4,7 +4,7 @@ import (
|
||||
"bytes"
|
||||
"testing"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
)
|
||||
|
||||
func TestMemoryBlobStore_Put(t *testing.T) {
|
||||
|
||||
+21
-4
@@ -5,7 +5,7 @@ import (
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
@@ -120,9 +120,10 @@ func (s *S3BlobStore) Put(hash string, blob []byte) error {
|
||||
}(time.Now())
|
||||
|
||||
_, err = s3manager.NewUploader(s.session).Upload(&s3manager.UploadInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(hash),
|
||||
Body: bytes.NewBuffer(blob),
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(hash),
|
||||
Body: bytes.NewBuffer(blob),
|
||||
StorageClass: aws.String(s3.StorageClassIntelligentTiering),
|
||||
})
|
||||
|
||||
return err
|
||||
@@ -133,3 +134,19 @@ func (s *S3BlobStore) PutSD(hash string, blob []byte) error {
|
||||
//Todo - handle missing stream for consistency
|
||||
return s.Put(hash, blob)
|
||||
}
|
||||
|
||||
func (s *S3BlobStore) Delete(hash string) error {
|
||||
err := s.initOnce()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Debugf("Deleting %s from S3", hash[:8])
|
||||
|
||||
_, err = s3.New(s.session).DeleteObject(&s3.DeleteObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(hash),
|
||||
})
|
||||
|
||||
return err
|
||||
}
|
||||
+18
-5
@@ -1,13 +1,26 @@
|
||||
package store
|
||||
|
||||
import "github.com/lbryio/lbry.go/errors"
|
||||
import "github.com/lbryio/lbry.go/extras/errors"
|
||||
|
||||
// BlobStore is an interface with methods for consistently handling blob storage.
|
||||
type BlobStore interface {
|
||||
Has(string) (bool, error)
|
||||
Get(string) ([]byte, error)
|
||||
Put(string, []byte) error
|
||||
PutSD(string, []byte) error
|
||||
// Does blob exist in the store
|
||||
Has(hash string) (bool, error)
|
||||
// Get the blob from the store
|
||||
Get(hash string) ([]byte, error)
|
||||
// Put the blob into the store
|
||||
Put(hash string, blob []byte) error
|
||||
// Put an SD blob into the store
|
||||
PutSD(hash string, blob []byte) error
|
||||
// Delete the blob from the store
|
||||
Delete(hash string) error
|
||||
}
|
||||
|
||||
type Blocklister interface {
|
||||
// Block deletes the blob and prevents it from being uploaded in the future
|
||||
Block(hash string) error
|
||||
// Wants returns false if the hash exists or is blocked, true otherwise
|
||||
Wants(hash string) (bool, error)
|
||||
}
|
||||
|
||||
//ErrBlobNotFound is a standard error when a blob is not found in the store.
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
package updater
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/meta"
|
||||
|
||||
"github.com/inconshreveable/go-update"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// TODO: this needs to be shut down cleanly
|
||||
|
||||
func Run(url, updateCmd string) {
|
||||
t := time.NewTicker(1 * time.Minute)
|
||||
for {
|
||||
<-t.C
|
||||
|
||||
r, err := http.Head(url)
|
||||
if err != nil {
|
||||
logrus.Errorln(err)
|
||||
continue
|
||||
} else if r.StatusCode >= 300 {
|
||||
logrus.Errorf("HEAD request returned status code %d", r.StatusCode)
|
||||
continue
|
||||
}
|
||||
|
||||
lm, err := time.Parse(time.RFC1123, r.Header.Get("Last-Modified"))
|
||||
if err != nil {
|
||||
logrus.Errorln(err)
|
||||
continue
|
||||
}
|
||||
|
||||
if !lm.After(meta.BuildTime.Add(5 * time.Minute)) { // buffer for compile and upload time
|
||||
continue
|
||||
}
|
||||
|
||||
logrus.Println("updating binary...")
|
||||
|
||||
err = updateBin(url)
|
||||
if err != nil {
|
||||
logrus.Errorln(err)
|
||||
// todo: may need to recover here, if stuck without binary
|
||||
continue
|
||||
}
|
||||
|
||||
parts := strings.Fields(updateCmd)
|
||||
err = exec.Command(parts[0], parts[1:]...).Start()
|
||||
if err != nil {
|
||||
logrus.Errorln(err)
|
||||
continue
|
||||
}
|
||||
|
||||
return // stop checking for updates in case restart takes a while
|
||||
}
|
||||
}
|
||||
|
||||
func updateBin(url string) error {
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
err := resp.Body.Close()
|
||||
if err != nil {
|
||||
logrus.Errorln(err)
|
||||
}
|
||||
}()
|
||||
|
||||
return update.Apply(resp.Body, update.Options{})
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package wallet
|
||||
|
||||
// ServerVersion returns the server's version.
|
||||
// https://electrumx.readthedocs.io/en/latest/protocol-methods.html#server-version
|
||||
func (n *Node) ServerVersion() (string, error) {
|
||||
resp := &struct {
|
||||
Result []string `json:"result"`
|
||||
}{}
|
||||
err := n.request("server.version", []string{"reflector.go", ProtocolVersion}, resp)
|
||||
|
||||
var v string
|
||||
if len(resp.Result) >= 2 {
|
||||
v = resp.Result[1]
|
||||
}
|
||||
|
||||
return v, err
|
||||
}
|
||||
|
||||
type GetClaimsInTxResp struct {
|
||||
Jsonrpc string `json:"jsonrpc"`
|
||||
ID int `json:"id"`
|
||||
Result []struct {
|
||||
Name string `json:"name"`
|
||||
ClaimID string `json:"claim_id"`
|
||||
Txid string `json:"txid"`
|
||||
Nout int `json:"nout"`
|
||||
Amount int `json:"amount"`
|
||||
Depth int `json:"depth"`
|
||||
Height int `json:"height"`
|
||||
Value string `json:"value"`
|
||||
ClaimSequence int `json:"claim_sequence"`
|
||||
Address string `json:"address"`
|
||||
Supports []interface{} `json:"supports"` // TODO: finish me
|
||||
EffectiveAmount int `json:"effective_amount"`
|
||||
ValidAtHeight int `json:"valid_at_height"`
|
||||
} `json:"result"`
|
||||
}
|
||||
|
||||
func (n *Node) GetClaimsInTx(txid string) (*GetClaimsInTxResp, error) {
|
||||
var resp GetClaimsInTxResp
|
||||
err := n.request("blockchain.claimtrie.getclaimsintx", []string{txid}, &resp)
|
||||
return &resp, err
|
||||
}
|
||||
@@ -0,0 +1,240 @@
|
||||
package wallet
|
||||
|
||||
// copied from https://github.com/d4l3k/go-electrum
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"math/rand"
|
||||
"net"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/extras/errors"
|
||||
"github.com/lbryio/lbry.go/extras/stop"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/uber-go/atomic"
|
||||
)
|
||||
|
||||
const (
|
||||
ClientVersion = "0.0.1"
|
||||
ProtocolVersion = "1.0"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrNotImplemented = errors.Base("not implemented")
|
||||
ErrNodeConnected = errors.Base("node already connected")
|
||||
ErrConnectFailed = errors.Base("failed to connect")
|
||||
ErrTimeout = errors.Base("timeout")
|
||||
)
|
||||
|
||||
type response struct {
|
||||
data []byte
|
||||
err error
|
||||
}
|
||||
|
||||
type Node struct {
|
||||
transport *TCPTransport
|
||||
nextId atomic.Uint32
|
||||
grp *stop.Group
|
||||
|
||||
handlersMu *sync.RWMutex
|
||||
handlers map[uint32]chan response
|
||||
|
||||
pushHandlersMu *sync.RWMutex
|
||||
pushHandlers map[string][]chan response
|
||||
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
// NewNode creates a new node.
|
||||
func NewNode() *Node {
|
||||
return &Node{
|
||||
handlers: make(map[uint32]chan response),
|
||||
pushHandlers: make(map[string][]chan response),
|
||||
handlersMu: &sync.RWMutex{},
|
||||
pushHandlersMu: &sync.RWMutex{},
|
||||
grp: stop.New(),
|
||||
timeout: 1 * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
// Connect creates a new connection to the specified address.
|
||||
func (n *Node) Connect(addrs []string, config *tls.Config) error {
|
||||
if n.transport != nil {
|
||||
return ErrNodeConnected
|
||||
}
|
||||
|
||||
// shuffle addresses for load balancing
|
||||
rand.Shuffle(len(addrs), func(i, j int) { addrs[i], addrs[j] = addrs[j], addrs[i] })
|
||||
|
||||
var err error
|
||||
|
||||
for _, addr := range addrs {
|
||||
n.transport, err = NewTransport(addr, config)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
if e, ok := err.(*net.OpError); ok && e.Err.Error() == "no such host" {
|
||||
// net.errNoSuchHost is not exported, so we have to string-match
|
||||
continue
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
if n.transport == nil {
|
||||
return ErrConnectFailed
|
||||
}
|
||||
|
||||
log.Debugf("wallet connected to %s", n.transport.conn.RemoteAddr())
|
||||
|
||||
n.grp.Add(1)
|
||||
go func() {
|
||||
defer n.grp.Done()
|
||||
<-n.grp.Ch()
|
||||
n.transport.Shutdown()
|
||||
}()
|
||||
|
||||
n.grp.Add(1)
|
||||
go func() {
|
||||
defer n.grp.Done()
|
||||
n.handleErrors()
|
||||
}()
|
||||
|
||||
n.grp.Add(1)
|
||||
go func() {
|
||||
defer n.grp.Done()
|
||||
n.listen()
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (n *Node) Shutdown() {
|
||||
n.grp.StopAndWait()
|
||||
}
|
||||
|
||||
func (n *Node) handleErrors() {
|
||||
for {
|
||||
select {
|
||||
case <-n.grp.Ch():
|
||||
return
|
||||
case err := <-n.transport.Errors():
|
||||
n.err(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// err handles errors produced by the foreign node.
|
||||
func (n *Node) err(err error) {
|
||||
// TODO: Better error handling.
|
||||
log.Error(err)
|
||||
}
|
||||
|
||||
// listen processes messages from the server.
|
||||
func (n *Node) listen() {
|
||||
for {
|
||||
select {
|
||||
case <-n.grp.Ch():
|
||||
return
|
||||
case bytes := <-n.transport.Responses():
|
||||
msg := &struct {
|
||||
Id uint32 `json:"id"`
|
||||
Method string `json:"method"`
|
||||
Error struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
} `json:"error"`
|
||||
}{}
|
||||
if err := json.Unmarshal(bytes, msg); err != nil {
|
||||
n.err(err)
|
||||
continue
|
||||
}
|
||||
|
||||
r := response{}
|
||||
if len(msg.Error.Message) > 0 {
|
||||
r.err = errors.Base("%d: %s", msg.Error.Code, msg.Error.Message)
|
||||
} else {
|
||||
r.data = bytes
|
||||
}
|
||||
|
||||
if len(msg.Method) > 0 {
|
||||
n.pushHandlersMu.RLock()
|
||||
handlers := n.pushHandlers[msg.Method]
|
||||
n.pushHandlersMu.RUnlock()
|
||||
|
||||
for _, handler := range handlers {
|
||||
select {
|
||||
case handler <- r:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
n.handlersMu.RLock()
|
||||
c, ok := n.handlers[msg.Id]
|
||||
n.handlersMu.RUnlock()
|
||||
if ok {
|
||||
c <- r
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// listenPush returns a channel of messages matching the method.
|
||||
//func (n *Node) listenPush(method string) <-chan []byte {
|
||||
// c := make(chan []byte, 1)
|
||||
// n.pushHandlersMu.Lock()
|
||||
// defer n.pushHandlersMu.Unlock()
|
||||
// n.pushHandlers[method] = append(n.pushHandlers[method], c)
|
||||
// return c
|
||||
//}
|
||||
|
||||
// request makes a request to the server and unmarshals the response into v.
|
||||
func (n *Node) request(method string, params []string, v interface{}) error {
|
||||
msg := struct {
|
||||
Id uint32 `json:"id"`
|
||||
Method string `json:"method"`
|
||||
Params []string `json:"params"`
|
||||
}{
|
||||
Id: n.nextId.Load(),
|
||||
Method: method,
|
||||
Params: params,
|
||||
}
|
||||
n.nextId.Inc()
|
||||
|
||||
bytes, err := json.Marshal(msg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bytes = append(bytes, delimiter)
|
||||
|
||||
c := make(chan response, 1)
|
||||
|
||||
n.handlersMu.Lock()
|
||||
n.handlers[msg.Id] = c
|
||||
n.handlersMu.Unlock()
|
||||
|
||||
err = n.transport.Send(bytes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var r response
|
||||
select {
|
||||
case r = <-c:
|
||||
case <-time.After(n.timeout):
|
||||
r = response{err: errors.Err(ErrTimeout)}
|
||||
}
|
||||
|
||||
n.handlersMu.Lock()
|
||||
delete(n.handlers, msg.Id)
|
||||
n.handlersMu.Unlock()
|
||||
|
||||
if r.err != nil {
|
||||
return r.err
|
||||
}
|
||||
|
||||
return json.Unmarshal(r.data, v)
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
package wallet
|
||||
|
||||
// copied from https://github.com/d4l3k/go-electrum
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"crypto/tls"
|
||||
"net"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/extras/stop"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type TCPTransport struct {
|
||||
conn net.Conn
|
||||
responses chan []byte
|
||||
errors chan error
|
||||
grp *stop.Group
|
||||
}
|
||||
|
||||
func NewTransport(addr string, config *tls.Config) (*TCPTransport, error) {
|
||||
var conn net.Conn
|
||||
var err error
|
||||
|
||||
timeout := 5 * time.Second
|
||||
if config != nil {
|
||||
conn, err = tls.DialWithDialer(&net.Dialer{Timeout: timeout}, "tcp", addr, config)
|
||||
} else {
|
||||
conn, err = net.DialTimeout("tcp", addr, timeout)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
t := &TCPTransport{
|
||||
conn: conn,
|
||||
responses: make(chan []byte),
|
||||
errors: make(chan error),
|
||||
grp: stop.New(),
|
||||
}
|
||||
|
||||
t.grp.Add(1)
|
||||
go func() {
|
||||
defer t.grp.Done()
|
||||
<-t.grp.Ch()
|
||||
t.close()
|
||||
}()
|
||||
|
||||
t.grp.Add(1)
|
||||
go func() {
|
||||
t.grp.Done()
|
||||
t.listen()
|
||||
}()
|
||||
|
||||
return t, nil
|
||||
}
|
||||
|
||||
const delimiter = byte('\n')
|
||||
|
||||
func (t *TCPTransport) listen() {
|
||||
reader := bufio.NewReader(t.conn)
|
||||
for {
|
||||
line, err := reader.ReadBytes(delimiter)
|
||||
if err != nil {
|
||||
t.error(err)
|
||||
return
|
||||
}
|
||||
|
||||
log.Debugf("%s -> %s", t.conn.RemoteAddr(), line)
|
||||
|
||||
t.responses <- line
|
||||
}
|
||||
}
|
||||
|
||||
func (t *TCPTransport) Send(body []byte) error {
|
||||
log.Debugf("%s <- %s", t.conn.RemoteAddr(), body)
|
||||
_, err := t.conn.Write(body)
|
||||
return err
|
||||
}
|
||||
|
||||
func (t *TCPTransport) error(err error) {
|
||||
select {
|
||||
case t.errors <- err:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (t *TCPTransport) Responses() <-chan []byte { return t.responses }
|
||||
func (t *TCPTransport) Errors() <-chan error { return t.errors }
|
||||
|
||||
func (t *TCPTransport) Shutdown() {
|
||||
t.grp.StopAndWait()
|
||||
}
|
||||
|
||||
func (t *TCPTransport) close() {
|
||||
err := t.conn.Close()
|
||||
if err != nil {
|
||||
t.error(err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user