96 Commits
Author SHA1 Message Date
Alex Grintsvayg c4ac60941a copystreams 2019-07-30 18:01:06 -04:00
Alex Grintsvayg 8b9ec771d0 commit every 10 inserts 2019-07-12 21:49:07 -04:00
Alex Grintsvayg eaf5f7478b migration triggers and procedures 2019-07-11 13:46:19 -04:00
Alex Grintsvayg 47425c2c50 Merge branch 'db_restructure'
* db_restructure:
  make new changes backwards-compatible while we migrate
  use IDs instead of hashes for join table
2019-07-10 12:48:15 -04:00
Alex Grintsvayg 594a45a271 make new changes backwards-compatible while we migrate 2019-07-10 11:28:56 -04:00
Alex Grintsvayg fd39b09e95 use IDs instead of hashes for join table 2019-07-10 11:27:19 -04:00
Alex Grintsvayg fce8beea4d dont try to connect to db when useDB is off 2019-07-10 09:39:21 -04:00
Niko Storni 96e0c2e2b2 temporarily turn off reflector 2019-07-10 15:22:00 +02:00
Niko bb5df9896a one more try 2019-07-06 02:31:13 -04:00
Niko accd79c81f turn off db again 2019-07-05 18:55:25 -04:00
Niko 32dae2704f turn reflector back on 2019-07-05 17:21:04 -04:00
Alex Grintsvayg 2ed1e23228 fix error counter 2019-07-03 16:17:31 -04:00
Alex Grintsvayg d61f7c892f off 2019-07-03 15:41:12 -04:00
Alex Grintsvayg 027409fe40 Merge branch 'uploader'
* uploader:
  separate uploader into reusable component
2019-07-03 13:41:56 -04:00
Alex Grintsvayg 38401bf8d6 separate uploader into reusable component 2019-07-03 13:39:37 -04:00
Alex Grintsvayg 825e699114 turn db back on 2019-07-03 13:02:39 -04:00
Alex Grintsvayg 6509c7dc52 turn off db 2019-07-02 09:33:15 -04:00
Alex Grintsvayg c088a65041 make it easy to turn the db on and off 2019-07-02 09:32:53 -04:00
Alex Grintsvayg c7643fb5da check blocklist first, its cheaper 2019-07-02 09:21:26 -04:00
Alex Grintsvayg c3ab5dd2a5 log timed-out query 2019-06-27 15:34:12 -04:00
Alex Grintsvayg 2047fe6c05 maybe removing this transaction will improve the situation? 2019-06-27 15:30:38 -04:00
Alex Grintsvayg 6166ff37cf retry timed-out queries 2019-06-26 14:36:05 -04:00
Alex Grintsvayg 970585c75d slight edit 2019-06-26 11:47:53 -04:00
Alex Grintsvayg 71549c0dea limit idle connections 2019-06-26 11:02:37 -04:00
Alex Grintsvayg 46ae5a502a Revert "temporarily disable reflector and db"
This reverts commit fb2f807e6a.
2019-06-25 13:46:04 -04:00
Alex Grintsvayg fb2f807e6a temporarily disable reflector and db 2019-06-25 13:08:35 -04:00
Alex Grintsvayg ad1eea1e8e a bit of debugging 2019-06-25 09:13:25 -04:00
Alex Grintsvayg dfb77d3547 bump up batch size to speed up querying 2019-06-25 08:44:37 -04:00
Niko Storni 96427136ef disable megacheck until it's fixed 2019-06-20 19:13:57 +02:00
Alex Grintsvayg c649636eeb better error when connection fails 2019-06-05 11:03:55 -04:00
Alex Grintsvayg 2e81b1ab03 add stat tracking for blob download. stream download will always say 0 2019-05-01 15:42:23 -04:00
Alex Grintsvayg fa2ec38d3e start logging received json on decode error 2019-05-01 15:39:14 -04:00
Alex Grintsvayg 2cacdaf22d support new protobufs, stop using victor's wallet server 2019-04-29 17:19:59 -04:00
Alex Grintsvayg 3b269050af move aws deploy up so you see it first 2019-03-22 09:12:10 -04:00
Mark Beamer Jr 59ff828794 dont go get go lint 2019-03-21 23:37:01 -04:00
Thomas Zarebczan 4569d04522 5 second timeout? wtf 2019-03-21 23:22:31 -04:00
Alex Grintsvayg 7a618f4228 ignore write error for closed connection 2019-03-18 16:29:28 -04:00
Alex Grintsvayg 0d996d7415 lbry.com switch 2019-03-18 12:28:29 -04:00
Alex Grintsvayg 2744080c6b remove extra log statement 2019-02-08 20:19:58 -05:00
Alex Grintsvayg 41b513439f enable travis build cache 2019-02-08 17:33:43 -05:00
Alex Grintsvayg 31daec7054 comment out unreachable code 2019-02-08 17:29:31 -05:00
Alex Grintsvayg 00feaaf76c more unique error messages 2019-02-08 17:28:09 -05:00
Alex Grintsvayg a98990f573 track peer errors as well 2019-02-08 14:56:41 -05:00
Alex Grin 5788f5aa55 Merge pull request #24 from lbryio/lower_verbosity_on_slack_blob_server
supress timeouts and connection reset from logs as they are common for blob serving
2019-02-06 15:52:18 -05:00
Victor Shyba 922a9402aa supress timeouts and connection reset from logs as they are common for blob server 2019-02-06 17:04:57 -03:00
Alex Grintsvayg 05e0a506b4 better debug output 2019-01-29 14:42:45 -05:00
Alex Grintsvayg a8fe33ecf4 this is how the daemon actually finds the reflector peer port 2019-01-29 14:33:55 -05:00
Alex Grintsvayg 030ed4914d test reflector server 2019-01-29 14:28:40 -05:00
Alex Grintsvayg e03810903a ignore some errors in peer proto, add write deadline to peer proto 2019-01-25 16:49:45 -05:00
Alex Grintsvayg b0b2d21a08 reduce logging on sending blobs 2019-01-17 16:49:19 -05:00
Alex Grintsvayg 557988d2cb minor renames 2019-01-16 11:45:28 -05:00
Alex Grintsvayg 6acb99c74d fixed a bunch of errors in wallet node 2019-01-16 11:41:58 -05:00
Alex Grintsvayg d77f5d17f3 Merge branch 'composite-request'
* composite-request:
  also run peer server on reflector
  handle new composite requests
2019-01-16 09:57:45 -05:00
Alex Grintsvayg 609ebdb74e also run peer server on reflector 2019-01-16 09:54:47 -05:00
Alex Grintsvayg 5baa66f0aa handle new composite requests 2019-01-15 14:23:08 -05:00
Alex Grintsvayg 75ad143d84 switch to lbry.go/stream 2019-01-10 08:02:43 -05:00
Alex Grintsvayg 14d6d32a41 move dht to lbry.go repo 2019-01-09 17:52:30 -05:00
Alex Grintsvayg e3ae0ef5c9 minor 2018-12-26 18:36:04 -05:00
Alex Grintsvayg db791e26ef store blobs in s3 intelligent-tiering storage class 2018-12-26 15:05:32 -05:00
Alex Grintsvayg dc104dd17b switch to go modules, drop testing against go master 2018-12-26 14:12:04 -05:00
Alex Grintsvayg e67e3dbf85 typo 2018-10-08 13:29:05 -04:00
Alex Grintsvayg e14a1f36cf Merge branch 'stream'
* stream:
  started work on go blob primitives. successfully matched python's blob crypto (excluding canonical JSON)
2018-10-04 15:05:37 -04:00
Alex Grintsvayg 11ebfb822b started work on go blob primitives. successfully matched python's blob crypto (excluding canonical JSON) 2018-10-04 15:05:19 -04:00
Alex Grintsvayg 4293e772e2 readme 2018-09-27 16:46:49 -04:00
Alex Grintsvayg 686ac662cc more common errors 2018-09-26 16:00:19 -04:00
Alex Grintsvayg 747a6ec2b2 make linter happy 2018-09-24 10:48:01 -04:00
Alex Grintsvayg 046868c4e0 Merge branch 'blocklist'
* blocklist:
  fixes from nikos review
  actually done now
  done, but NEEDS MIGRATION AND TESTING
  beginning of blocklist support
  lbryumx client
2018-09-24 10:03:54 -04:00
Alex Grintsvayg 53d3eea8fb fixes from nikos review 2018-09-21 09:18:39 -04:00
Alex Grintsvayg 61e83d86de actually done now 2018-09-20 11:29:35 -04:00
Alex Grintsvayg 9fb824790b done, but NEEDS MIGRATION AND TESTING 2018-09-20 11:29:35 -04:00
Alex Grintsvayg 6fd0526376 beginning of blocklist support 2018-09-20 11:29:35 -04:00
Alex Grintsvayg cd236708b0 lbryumx client 2018-09-20 11:29:35 -04:00
Alex Grintsvayg 83921b8c2d bump timeout until lbryio/lbry#1377 is resolved 2018-09-17 12:14:17 -04:00
Alex Grintsvayg c042fe08c3 test log-on-shutdown 2018-09-04 11:23:17 -04:00
Alex Grintsvayg 59e92ccf88 reports stats on shutdown 2018-09-04 11:08:34 -04:00
Alex Grintsvayg cb386de4c7 bump stat freq to 1 hour 2018-09-04 09:49:54 -04:00
Alex Grintsvayg 146adcf7fd fix panic when there were no recent errors 2018-08-31 06:38:38 -04:00
Alex Grintsvayg 5d844fc3ea fix shouldLog 2018-08-30 21:10:39 -04:00
Alex Grintsvayg 4ed52e20d7 todos 2018-08-30 21:08:59 -04:00
Alex Grintsvayg 7ab2d355d5 some linting 2018-08-30 20:20:15 -04:00
Alex Grintsvayg 064035f1d8 s3 finally working 2018-08-30 19:52:29 -04:00
Alex Grintsvayg e158fa1bb2 make dep in travis 2018-08-30 19:31:21 -04:00
Alex Grintsvayg 25ec93a6b9 auto-self-update 2018-08-30 17:18:42 -04:00
Alex Grintsvayg 22f42d5559 include build time in binary 2018-08-30 14:41:50 -04:00
Alex Grintsvayg 473e7e3b07 consolidate errors that happen a lot and we know about 2018-08-29 09:04:09 -04:00
Alex Grintsvayg d7a4b34d8c unused type 2018-08-28 15:27:32 -04:00
Alex Grintsvayg 605afd1d6c add stats reporting to reflector 2018-08-28 11:18:06 -04:00
Alex Grintsvayg 204198bf9d bump timeout to 30 seconds 2018-08-20 12:05:22 -04:00
Alex Grintsvayg fe9cf091fc send the expected response instead of the error message 2018-08-20 11:52:29 -04:00
Alex Grintsvayg 65df7bc627 make CI happy 2018-08-15 21:19:38 -04:00
Alex Grintsvayg 75886211b1 fix partial stream upload 2018-08-15 20:18:54 -04:00
Alex Grintsvayg 391f983630 bump timeout again 2018-08-15 16:12:55 -04:00
Alex Grintsvayg 44cf4d085c dont log every blob you get 2018-08-15 15:52:41 -04:00
Alex Grintsvayg bd8a35e366 expose reflector timeout, bump it up 2018-08-15 15:50:09 -04:00
Alex Grintsvayg 4284c3b1f9 sendblob command 2018-08-15 14:25:15 -04:00
Alex Grintsvayg 4c8eda783f bugfix 2018-08-15 11:26:36 -04:00
67 changed files with 2462 additions and 9500 deletions

No files matched your search

+1 -2
View File
@@ -1,4 +1,3 @@
/vendor
/blobs
/config.json*
/prism-bin
/bin
+4
View File
@@ -0,0 +1,4 @@
{
"Disable": ["golint"],
"Vendor": true
}
+31 -30
View File
@@ -2,30 +2,21 @@ os: linux
dist: trusty
language: go
# Only the last two Go releases are supported by the Go team with security
# updates. Any versions older than that should be considered deprecated.
# Don't bother testing with them. tip builds your code with the latest
# development version of Go. This can warn you that your code will break
# in the next version of Go. Don't worry! Later we declare that test runs
# are allowed to fail on Go tip.
matrix:
include:
- go: 1.10.2
env: DEPLOY=true # dont deploy tip build
- go: master
env: DEPLOY=false # dont deploy tip build
allow_failures:
- go: master
# Don't wait for tip tests to finish. Mark the test run green if the
# tests pass on the stable versions of Go.
fast_finish: true
env:
- GO111MODULE=on
go:
- 1.11.x
cache:
directories:
- $HOME/.cache/go-build
- $HOME/gopath/pkg/mod
# Don't email me the results of the test runs.
notifications:
email: false
# Skip the install step. Don't `go get` dependencies. Only build with the
# code in vendor/
# Skip the install step. Don't `go get` dependencies. Only build with the code in vendor/
install: true
# Anything in before_script that returns a nonzero exit code will
@@ -35,8 +26,8 @@ before_script:
# All the .go files, excluding vendor/ and model (auto generated)
- GO_FILES=$(find . -iname '*.go' ! -iname '*_test.go' -type f | grep -v /vendor/ )
- go get golang.org/x/tools/cmd/goimports # Used in build script for generated files
- go get github.com/golang/lint/golint # Linter
- go get honnef.co/go/tools/cmd/megacheck # Badass static analyzer/linter
# - go get github.com/golang/lint/golint # Linter
# - go get honnef.co/go/tools/cmd/megacheck # Badass static analyzer/linter
- go get github.com/jgautheron/gocyclo # Check against high complexity
- go get github.com/mdempsky/unconvert # Identifies unnecessary type conversions
- go get github.com/kisielk/errcheck # Checks for unhandled errors
@@ -48,12 +39,10 @@ before_script:
# script always run to completion (set +e). All of these code checks are must haves
# in a modern Go project.
script:
# Build Prism successfully
- make
# Fail if a .go file hasn't been formatted with gofmt
- test -z $(gofmt -s -l $GO_FILES)
# Run unit tests
- go test ./...
- make test
# Checks for unused vars and fields on structs
- varcheck ./...
- structcheck ./...
@@ -62,21 +51,33 @@ script:
# forbid code with huge functions
#- gocyclo -ignore "_test.go" -avg -over 19 $GO_FILES
# checks for unhandled errors
- errcheck ./...
# - errcheck ./... # COMMENTED OUT UNTIL https://github.com/kisielk/errcheck/issues/155 is fixed
# "go vet on steroids" + linter - ignore autogen code
- megacheck -simple.exit-non-zero=true ./...
# - megacheck -simple.exit-non-zero=true ./... # DISABLED UNTIL https://github.com/dominikh/go-tools/issues/328 is fixed
# check for unnecessary conversions - ignore autogen code
- unconvert ./...
# - unconvert ./... # SEEMS TO BE BROKEN WITH GO MODULES
# one last linter - ignore autogen code
#- golint -set_exit_status $(go list ./... | grep -v /vendor/ )
# Finally, build the binary
- make
deploy:
- provider: s3
local_dir: ./bin
skip_cleanup: true
on:
repo: lbryio/reflector.go
access_key_id:
secure: "ABmoSTxTee1GubJmmi+MlyBcnqRT0ywEOtTL4hrH7T+Vj5UPcNBhjqGRiMXzO3MdrVWom19OnqMvsvAdehrB4uQJVlX/+zB26gTpEiiU5dWJgerNAYmfxDGNz/p9pyYWLvVY/cymSlJ4HkggJVg+dMTBTOP+gTNgQUhHL7pAd/Z5HMPmPW/rlNlQgDdczoY44OdiniuHny4Lr7mo8dcT4dcyJysPpHLCPGWp3kXcffIEbgO02aQZ14BbX9oU9xm1BacYnr4t8gRCU+uK0mt7TREHUzZ8bQBRXUymmpeeEQshXdifdwtk6sdRGIZiTi0NmQ7kYter6oh6623TH6rpNy/lkxKi823FbSPC1XZGH7CwcbnSIEwY++7e9PT0EA14VuDUF7+iXC/gpRkVQwUoMcoC9rHBjkXW+SiwHWoUeDKXrfVO9QGEJTi5bDRNwklwu2zkYWeRIPzfDaVayUBYwjunjT1eST997ygmJwJTJ6DeOWLx+WFlmwBJfQpBwbrlMrceiNjreMwGB6ffXEBzuM6S5inXGIChllGDpNyJeFVcH9zPpU0JsIpWoRjwLNFOiEhFsnzK5rvA2SKmlOOnSlgOCBQnFi2SmLDM790KoY8qjan2s984chQz2qsKnhYuVpJhG6LKgrI1QwE0KHr80spVFfewq11AU4Q+lfA9aAQ="
secret_access_key:
secure: "AxYRTy/GnjeTJKQdeJ/AEeAd+yXs783bFDKdyKNswtsHlU8sWPQgNcvTLpVqnSQMpiwkGDGi/70rvR5C+AT3SIWNw13RYrgBRpduQU0J+B2JS+3dN2DIePu25uvs++Wo22OfS8I+UjZ1mWY1SSHI2spPXvDCq5tb+Ih8nlYflEyAtxU9Oq2R3Kus2tkIlRnL25sP/2fY7RvuJFYIV63z8ZIJRzB5WzOeERqnXq2zfwos+hycAqyo/VevJnWAYTEDsvBuSODOpZF+QfKtIQ2rYSoqy8Lq1M6UOZimnC3Ulea4euBVf2ssBCnI7csGNG5UzkTiwrPDi2xIP8nM01rHW1yHJ7tQsJaghnUsfw2t6ui4ZofvbbOFTN/YCloHITifEi8Tc1/17isi3y+kX5yQ/Nk5UNry0Wbt91CP+nkL/ZmA5grkBXDL7VJMmB60TnO3ap24CtwBQartN3LoWs7h+4ov+LqbCt6IqpJVWQWlwJeb2MFPFByALtBpsqAyL1SxXlGNpPa94CuXxfQ6Bv436PtefA5FlTzR8uMmqsjWciv06bVnSvVlFEVovN2Fkplrjt7AASJ/8KJs4THDg4k61nfd8roAHx6ewQzl4wCWKCikQ0MuFd2mVHwdrbnCH1mIHuPRyvWMMIAK0ooc1/rmKiJlpgumjxoFYNE10MXtt+I="
bucket: "lbry-reflector-binary"
- provider: releases
file: prism-bin
file: ./bin/prism-bin
skip_cleanup: true
on:
repo: lbryio/reflector.go
tags: true
condition: "$DEPLOY = true"
api_key:
secure: epAlhp3SUr8hhISarJ22n6tRw2TEa4s4oNFIvJUb5HGECVp1SYN7ao0ln5NoNLmfJS60pi911i/kMhhi21/uhZ0kCYlEhhIE2pc1zsiAxK9L9ENCssJ205HfVbe5grhwskLGzgjhU9OznO8WtmyOPWXr0it8M8RCTjx6rEC0A33Id3WMYyhP938Sj9CxEYeH4KS8wFvBXkgBVtrgaYwRTCIROFddHFXOb9jyNhqQ1RbfKtllsVtQhVk5WMlomheBNSS4vr6WMS4X4+2okFqnLtiSn1wrn5I/94UQbnrI1juVnQj0K+j32EyQbAOt4T2cLW3GtG0jhaYKyNMT9ycDCdVACPSDELlHWjeyoes9bnhUFftm6kDbQxwA1UsTF1yG8tMKXxBSmYyoT7qDloi6pBifZMrFXL61uTs6yhVB9LS/2oqg4sc0Ne87bRcn4OxsBeVCe3kbBHDTR/NTyF2gNPtRvgMAWULxTVcUm9VYdO0IWvAig5g4Row0DnFzEquD6CzezbRWD9WyZyV/AFyYHeeQ2PO7jTw0/3M7aDX33Fuhh34lehzmrC03cfgD/wZW+spxozIcQCYdiJqVw+u+/NvbNr0kkFzE9zW26JEmUFTyDvKxvnza1Kwtww3EgH6zaOL8r4yVbb54rePRvLw7pl93zlfJnEB2MCPqJOY5ZpU=
Generated
-435
View File
@@ -1,435 +0,0 @@
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
[[projects]]
branch = "master"
digest = "1:d64110a78451e373c5a952d2625323dbbe3bfe41c67f9652ea9668a6ceb4f645"
name = "github.com/armon/go-metrics"
packages = ["."]
pruneopts = ""
revision = "3c58d8115a78a6879e5df75ae900846768d36895"
[[projects]]
branch = "master"
digest = "1:c0b6dbbb56a745020d5939bdde2197241a1c6109f226cca57b16f46916be5e94"
name = "github.com/aws/aws-sdk-go"
packages = [
"aws",
"aws/awserr",
"aws/awsutil",
"aws/client",
"aws/client/metadata",
"aws/corehandlers",
"aws/credentials",
"aws/credentials/ec2rolecreds",
"aws/credentials/endpointcreds",
"aws/credentials/stscreds",
"aws/csm",
"aws/defaults",
"aws/ec2metadata",
"aws/endpoints",
"aws/request",
"aws/session",
"aws/signer/v4",
"internal/sdkio",
"internal/sdkrand",
"internal/sdkuri",
"internal/shareddefaults",
"private/protocol",
"private/protocol/eventstream",
"private/protocol/eventstream/eventstreamapi",
"private/protocol/query",
"private/protocol/query/queryutil",
"private/protocol/rest",
"private/protocol/restxml",
"private/protocol/xml/xmlutil",
"service/s3",
"service/s3/s3iface",
"service/s3/s3manager",
"service/sts",
]
pruneopts = ""
revision = "c0447dbaaf195bb477fd2d511b8e4665e04b9017"
[[projects]]
branch = "master"
digest = "1:e250643be8120824556f39df6ef128fc2be490fc96e0cb64b1a8ecf96bbe3ce6"
name = "github.com/btcsuite/btcutil"
packages = ["base58"]
pruneopts = ""
revision = "ab6388e0c60ae4834a1f57511e20c17b5f78be4b"
[[projects]]
digest = "1:0a39ec8bf5629610a4bc7873a92039ee509246da3cef1a0ea60f1ed7e5f9cea5"
name = "github.com/davecgh/go-spew"
packages = ["spew"]
pruneopts = ""
revision = "346938d642f2ec3594ed81d874461961cd0faa76"
version = "v1.1.0"
[[projects]]
digest = "1:968d8903d598e3fae738325d3410f33f07ea6a2b9ee5591e9c262ee37df6845a"
name = "github.com/go-errors/errors"
packages = ["."]
pruneopts = ""
revision = "a6af135bd4e28680facf08a3d206b454abc877a4"
version = "v1.0.1"
[[projects]]
digest = "1:d19c78214e03e297e9e30d2eb11892f731358b2951f2a5c7374658a156373e4c"
name = "github.com/go-ini/ini"
packages = ["."]
pruneopts = ""
revision = "358ee7663966325963d4e8b2e1fbd570c5195153"
version = "v1.38.1"
[[projects]]
branch = "master"
digest = "1:27b11ca1ad214ead955ff5480e8575e74c5df4e4dc02b04256a8d92131e1d3ad"
name = "github.com/go-sql-driver/mysql"
packages = ["."]
pruneopts = ""
revision = "99ff426eb706cffe92ff3d058e168b278cabf7c7"
[[projects]]
digest = "1:dbbeb8ddb0be949954c8157ee8439c2adfd8dc1c9510eb44a6e58cb68c3dce28"
name = "github.com/gorilla/context"
packages = ["."]
pruneopts = ""
revision = "08b5f424b9271eedf6f9f0ce86cb9396ed337a42"
version = "v1.1.1"
[[projects]]
digest = "1:c2c8666b4836c81a1d247bdf21c6a6fc1ab586538ab56f74437c2e0df5c375e1"
name = "github.com/gorilla/mux"
packages = ["."]
pruneopts = ""
revision = "e3702bed27f0d39777b0b37b664b6280e8ef8fbf"
version = "v1.6.2"
[[projects]]
digest = "1:b4817bdb0da3054166de058111943ac58b315aead2fed4ee838625a4e304f74c"
name = "github.com/gorilla/rpc"
packages = [
"v2",
"v2/json",
]
pruneopts = ""
revision = "22c016f3df3febe0c1f6727598b6389507e03a18"
version = "v1.1.0"
[[projects]]
digest = "1:fe1b4d4cbe48c0d55507c55f8663aa4185576cc58fa0c8be03bb8f19dfe17a9c"
name = "github.com/gorilla/websocket"
packages = ["."]
pruneopts = ""
revision = "ea4d1f681babbce9545c9c5f3d5194a789c89f5b"
version = "v1.2.0"
[[projects]]
branch = "master"
digest = "1:4fe55793760295fbef367890352b720784243e0ad19b5ee242519a4682bb9ef8"
name = "github.com/hashicorp/errwrap"
packages = ["."]
pruneopts = ""
revision = "d6c0cd88035724dd42e0f335ae30161c20575ecc"
[[projects]]
branch = "master"
digest = "1:4423ee95d6ee30bb22f680445c58889bb5b91e1b955405bf34374a053784a8a2"
name = "github.com/hashicorp/go-immutable-radix"
packages = ["."]
pruneopts = ""
revision = "7f3cd4390caab3250a57f30efdb2a65dd7649ecf"
[[projects]]
branch = "master"
digest = "1:6a611e691e739173805cb54019b5c39bb9d46455526dff31e0e6fe3aaca52776"
name = "github.com/hashicorp/go-msgpack"
packages = ["codec"]
pruneopts = ""
revision = "fa3f63826f7c23912c15263591e65d54d080b458"
[[projects]]
branch = "master"
digest = "1:deaebb4a98ca748bbad7eb653f3a675749500020823a086448ffcd7ba6b8b02d"
name = "github.com/hashicorp/go-multierror"
packages = ["."]
pruneopts = ""
revision = "3d5d8f294aa03d8e98859feac328afbdf1ae0703"
[[projects]]
branch = "master"
digest = "1:74f54e6ef2339f1de1e8c4b6674442118bd89e619b2fbd949ef2337330067994"
name = "github.com/hashicorp/go-sockaddr"
packages = ["."]
pruneopts = ""
revision = "6d291a969b86c4b633730bfc6b8b9d64c3aafed9"
[[projects]]
branch = "master"
digest = "1:9c776d7d9c54b7ed89f119e449983c3f24c0023e75001d6092442412ebca6b94"
name = "github.com/hashicorp/golang-lru"
packages = ["simplelru"]
pruneopts = ""
revision = "0fb14efe8c47ae851c0034ed7a448854d3d34cf3"
[[projects]]
digest = "1:d7ce65372f495908f80fc1f80f4dab5d763d9a1de544abd95aa719e4262d0dd5"
name = "github.com/hashicorp/memberlist"
packages = ["."]
pruneopts = ""
revision = "ce8abaa0c60c2d6bee7219f5ddf500e0a1457b28"
version = "v0.1.0"
[[projects]]
branch = "master"
digest = "1:4fd01ac3766b886665cfd335cc63819ec4e4538dcc1180c05d6edc089619962c"
name = "github.com/hashicorp/serf"
packages = [
"coordinate",
"serf",
]
pruneopts = ""
revision = "984a73625de3138f44deb38d00878fab39eb6447"
[[projects]]
digest = "1:870d441fe217b8e689d7949fef6e43efbc787e50f200cb1e70dbca9204a1d6be"
name = "github.com/inconshreveable/mousetrap"
packages = ["."]
pruneopts = ""
revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
version = "v1.0"
[[projects]]
digest = "1:4f767a115bc8e08576f6d38ab73c376fc1b1cd3bb5041171c9e8668cc7739b52"
name = "github.com/jmespath/go-jmespath"
packages = ["."]
pruneopts = ""
revision = "0b12d6b5"
[[projects]]
branch = "master"
digest = "1:1e4cb5c6d4e92e2fb34f1545e071fb61e81c71fc0c325445db19b9f3981cb343"
name = "github.com/johntdyer/slack-go"
packages = ["."]
pruneopts = ""
revision = "95fac1160b220c5abcf8b0ef88e9c3cb213c09f4"
[[projects]]
branch = "master"
digest = "1:132aae3fa5ad407b53f57cf6ebe274e414ec22d1700899023319931fa90b63e2"
name = "github.com/johntdyer/slackrus"
packages = ["."]
pruneopts = ""
revision = "f7aae3243a0786c5a974bce71ed951c459876e64"
[[projects]]
branch = "master"
digest = "1:65bb92382468ce7a1ee218b5f352cd0927d358569be7521dd7cb42e74589a308"
name = "github.com/lbryio/errors.go"
packages = ["."]
pruneopts = ""
revision = "ad03d3cc6a5c27c94bbe3412cf7de0eae1a9bd7c"
[[projects]]
branch = "master"
digest = "1:76fd7507e6014c598a01f1b3d558774d2a3114c438403bc98123870d2aecec62"
name = "github.com/lbryio/lbry.go"
packages = [
"crypto",
"errors",
"null",
"querytools",
"stop",
"util",
]
pruneopts = ""
revision = "e2c96944fc485d3ab5e164da78f8439a94c5aa85"
[[projects]]
branch = "master"
digest = "1:cabf2bf5e49edfe0c34cb9c6a256f2a99e6cc8c5e660855c8f3dafe1f81d5dcd"
name = "github.com/lyoshenka/bencode"
packages = ["."]
pruneopts = ""
revision = "b7abd7672df533e627eddbf3a5a529786e8bda7f"
[[projects]]
digest = "1:f0bad0fece0fb73c6ea249c18d8e80ffbe86be0457715b04463068f04686cf39"
name = "github.com/miekg/dns"
packages = ["."]
pruneopts = ""
revision = "5a2b9fab83ff0f8bfc99684bd5f43a37abe560f1"
version = "v1.0.8"
[[projects]]
digest = "1:ba3b2eb0ae6fd3deac4386c02fe9d2279c9520738eb9db2f0667e74d5c7a0a61"
name = "github.com/nlopes/slack"
packages = ["."]
pruneopts = ""
revision = "0db1d5eae1116bf7c8ed96c6749acfbf4daaec3e"
version = "v0.3.0"
[[projects]]
branch = "master"
digest = "1:af967afd3cbc6b0145937f4dcab78bcd93e7b2f2b618fb2bcaf7069ad5c638fa"
name = "github.com/phayes/freeport"
packages = ["."]
pruneopts = ""
revision = "e27662a4a9d6b2083dfd7e7b5d0e30985daca925"
[[projects]]
branch = "master"
digest = "1:6ee36f2cea425916d81fdaaf983469fc18f91b3cf090cfe90fa0a9d85b8bfab7"
name = "github.com/sean-/seed"
packages = ["."]
pruneopts = ""
revision = "e2103e2c35297fb7e17febb81e49b312087a2372"
[[projects]]
branch = "master"
digest = "1:65d0c541c450f6f8bb9654a3f1938407a835bf41faade00bb0fa1416de215d00"
name = "github.com/sebdah/goldie"
packages = ["."]
pruneopts = ""
revision = "8784dd1ab561dcf43d141f6678e9e41f3d0dff55"
[[projects]]
branch = "master"
digest = "1:1747c026a603e3c9f33f238e1d1390df2c8f48876b6bcb7a9c52c7b479e040f4"
name = "github.com/sirupsen/logrus"
packages = ["."]
pruneopts = ""
revision = "d329d24db4313262a3b0a24d8aeb1dc4bd294fb0"
[[projects]]
digest = "1:d0b38ba6da419a6d4380700218eeec8623841d44a856bb57369c172fbf692ab4"
name = "github.com/spf13/cast"
packages = ["."]
pruneopts = ""
revision = "8965335b8c7107321228e3e3702cab9832751bac"
version = "v1.2.0"
[[projects]]
branch = "master"
digest = "1:b5212c335a490d958a6b1b5b48901b46e682b82fa9af3a238fa88df6eaa60873"
name = "github.com/spf13/cobra"
packages = ["."]
pruneopts = ""
revision = "7c4570c3ebeb8129a1f7456d0908a8b676b6f9f1"
[[projects]]
digest = "1:8e243c568f36b09031ec18dff5f7d2769dcf5ca4d624ea511c8e3197dc3d352d"
name = "github.com/spf13/pflag"
packages = ["."]
pruneopts = ""
revision = "583c0c0531f06d5278b7d917446061adc344b5cd"
version = "v1.0.1"
[[projects]]
branch = "master"
digest = "1:f05efcac20bea32e1fcefde9a3cbefb07e02053666c4a67681ad18c8efc682d3"
name = "github.com/uber-go/atomic"
packages = ["."]
pruneopts = ""
revision = "ca680462431f8c7f96686d91b1d9e5203c4b075b"
[[projects]]
branch = "master"
digest = "1:16b935c128f178647036048862a21e8bfd66d1e83fb19787a8b356bdcf0de899"
name = "golang.org/x/crypto"
packages = [
"ed25519",
"ed25519/internal/edwards25519",
"ripemd160",
"sha3",
"ssh/terminal",
]
pruneopts = ""
revision = "f027049dab0ad238e394a753dba2d14753473a04"
[[projects]]
branch = "master"
digest = "1:15f2fc8cc79d90b0d4d712f04bd3eb3a3856ff3dd6b610a1425113ead3501610"
name = "golang.org/x/net"
packages = [
"bpf",
"context",
"internal/iana",
"internal/socket",
"ipv4",
"ipv6",
]
pruneopts = ""
revision = "19491d39cadbd9cd33f26ca22cc89ba4ba38251c"
[[projects]]
branch = "master"
digest = "1:e8f649ecfae7835a0a27ef39fd2180f6d3c12bc422e2ae55cd611f0b283b3e6e"
name = "golang.org/x/sys"
packages = [
"unix",
"windows",
]
pruneopts = ""
revision = "0718ef2ef256118d53a01598f179001ec2af7626"
[[projects]]
branch = "master"
digest = "1:55a681cb66f28755765fa5fa5104cbd8dc85c55c02d206f9f89566451e3fe1aa"
name = "golang.org/x/time"
packages = ["rate"]
pruneopts = ""
revision = "fbb02b2291d28baffd63558aa44b4b56f178d650"
[[projects]]
digest = "1:eede11c81b63c8f6fd06ef24ba0a640dc077196ec9b7a58ecde03c82eee2f151"
name = "google.golang.org/appengine"
packages = ["cloudsql"]
pruneopts = ""
revision = "b1f26356af11148e710935ed1ac8a7f5702c7612"
version = "v1.1.0"
[[projects]]
digest = "1:05eca53b271663de74078b5484b1995a8d56668a51434a698dc5d0863035d575"
name = "gopkg.in/nullbio/null.v6"
packages = ["convert"]
pruneopts = ""
revision = "40264a2e6b7972d183906cf17663983c23231c82"
version = "v6.3"
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
input-imports = [
"github.com/aws/aws-sdk-go/aws",
"github.com/aws/aws-sdk-go/aws/awserr",
"github.com/aws/aws-sdk-go/aws/credentials",
"github.com/aws/aws-sdk-go/aws/session",
"github.com/aws/aws-sdk-go/service/s3",
"github.com/aws/aws-sdk-go/service/s3/s3manager",
"github.com/davecgh/go-spew/spew",
"github.com/go-sql-driver/mysql",
"github.com/gorilla/mux",
"github.com/gorilla/rpc/v2",
"github.com/gorilla/rpc/v2/json",
"github.com/hashicorp/serf/serf",
"github.com/johntdyer/slackrus",
"github.com/lbryio/errors.go",
"github.com/lbryio/lbry.go/crypto",
"github.com/lbryio/lbry.go/errors",
"github.com/lbryio/lbry.go/querytools",
"github.com/lbryio/lbry.go/stop",
"github.com/lbryio/lbry.go/util",
"github.com/lyoshenka/bencode",
"github.com/phayes/freeport",
"github.com/sebdah/goldie",
"github.com/sirupsen/logrus",
"github.com/spf13/cast",
"github.com/spf13/cobra",
"github.com/uber-go/atomic",
"golang.org/x/time/rate",
]
solver-name = "gps-cdcl"
solver-version = 1
-39
View File
@@ -1,39 +0,0 @@
[[constraint]]
name = "github.com/aws/aws-sdk-go"
branch = "master"
[[constraint]]
name = "github.com/go-sql-driver/mysql"
branch = "master"
[[constraint]]
name = "github.com/hashicorp/serf"
branch = "master"
[[constraint]]
name = "github.com/lbryio/lbry.go"
branch = "master"
[[constraint]]
name = "github.com/sirupsen/logrus"
branch = "master"
[[constraint]]
name = "github.com/spf13/cobra"
branch = "master"
[[constraint]]
branch = "master"
name = "github.com/lyoshenka/bencode"
[[constraint]]
branch = "master"
name = "github.com/uber-go/atomic"
[[constraint]]
branch = "master"
name = "github.com/phayes/freeport"
[[constraint]]
branch = "master"
name = "github.com/johntdyer/slackrus"
+9 -11
View File
@@ -1,27 +1,25 @@
BINARY=prism-bin
DIR = $(shell cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)
VENDOR_DIR = vendor
BIN_DIR = ${DIR}/bin
IMPORT_PATH = github.com/lbryio/reflector.go
VERSION = $(shell git --git-dir=${DIR}/.git describe --dirty --always --long --abbrev=7)
LDFLAGS = -ldflags "-X ${IMPORT_PATH}/meta.Version=${VERSION}"
LDFLAGS = -ldflags "-X ${IMPORT_PATH}/meta.Version=${VERSION} -X ${IMPORT_PATH}/meta.Time=$(shell date +%s)"
.PHONY: build dep clean test
.PHONY: build clean test lint
.DEFAULT_GOAL: build
build: dep
CGO_ENABLED=0 go build ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${DIR}/${BINARY} main.go
dep: | $(VENDOR_DIR)
$(VENDOR_DIR):
go get github.com/golang/dep/cmd/dep && dep ensure
build:
mkdir -p ${BIN_DIR} && CGO_ENABLED=0 go build ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/${BINARY} main.go
clean:
if [ -f ${DIR}/${BINARY} ]; then rm ${DIR}/${BINARY}; fi
if [ -f ${BIN_DIR}/${BINARY} ]; then rm ${BIN_DIR}/${BINARY}; fi
test:
go test ./... -v -cover
lint:
go get github.com/alecthomas/gometalinter && gometalinter --install && gometalinter ./...
+3 -3
View File
@@ -6,9 +6,9 @@ import (
"sort"
"time"
"github.com/lbryio/lbry.go/crypto"
"github.com/lbryio/lbry.go/errors"
"github.com/lbryio/lbry.go/stop"
"github.com/lbryio/lbry.go/extras/crypto"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/stop"
"github.com/hashicorp/serf/serf"
log "github.com/sirupsen/logrus"
+1 -1
View File
@@ -6,7 +6,7 @@ import (
"strconv"
"syscall"
"github.com/lbryio/lbry.go/crypto"
"github.com/lbryio/lbry.go/extras/crypto"
"github.com/lbryio/reflector.go/cluster"
log "github.com/sirupsen/logrus"
+6 -6
View File
@@ -1,7 +1,6 @@
package cmd
import (
"log"
"net"
"os"
"os/signal"
@@ -9,15 +8,16 @@ import (
"syscall"
"time"
"github.com/lbryio/reflector.go/dht"
"github.com/lbryio/reflector.go/dht/bits"
"github.com/lbryio/lbry.go/dht"
"github.com/lbryio/lbry.go/dht/bits"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
var dhtNodeID string
var dhtPort int
var dhtRpcPort int
var dhtRPCPort int
var dhtSeeds []string
func init() {
@@ -30,7 +30,7 @@ func init() {
}
cmd.PersistentFlags().StringVar(&dhtNodeID, "nodeID", "", "nodeID in hex")
cmd.PersistentFlags().IntVar(&dhtPort, "port", 4567, "Port to start DHT on")
cmd.PersistentFlags().IntVar(&dhtRpcPort, "rpcPort", 0, "Port to listen for rpc commands on")
cmd.PersistentFlags().IntVar(&dhtRPCPort, "rpcPort", 0, "Port to listen for rpc commands on")
cmd.PersistentFlags().StringSliceVar(&dhtSeeds, "seeds", []string{}, "Addresses of seed nodes")
rootCmd.AddCommand(cmd)
}
@@ -56,7 +56,7 @@ func dhtCmd(cmd *cobra.Command, args []string) {
dhtConf := dht.NewStandardConfig()
dhtConf.Address = "0.0.0.0:" + strconv.Itoa(dhtPort)
dhtConf.RPCPort = dhtRpcPort
dhtConf.RPCPort = dhtRPCPort
if len(dhtSeeds) > 0 {
dhtConf.SeedNodes = dhtSeeds
}
+48 -11
View File
@@ -5,10 +5,13 @@ import (
"os/signal"
"strconv"
"syscall"
"time"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/meta"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/peer"
"github.com/lbryio/reflector.go/store"
log "github.com/sirupsen/logrus"
@@ -25,17 +28,48 @@ func init() {
}
func reflectorCmd(cmd *cobra.Command, args []string) {
log.Printf("reflector version %s", meta.Version)
db := new(db.SQL)
err := db.Connect(globalConfig.DBConn)
if err != nil {
log.Fatal(err)
}
log.Printf("reflector version %s, built %s", meta.Version, meta.BuildTime.Format(time.RFC3339))
// flip this flag to false when doing db maintenance. uploads will not work (as reflector server wont be running)
// but downloads will still work straight from s3
useDB := false
s3 := store.NewS3BlobStore(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
combo := store.NewDBBackedS3Store(s3, db)
reflectorServer := reflector.NewServer(combo)
err = reflectorServer.Start(":" + strconv.Itoa(reflector.DefaultPort))
var err error
var blobStore store.BlobStore = s3
var reflectorServer *reflector.Server
if useDB {
db := new(db.SQL)
err = db.Connect(globalConfig.DBConn)
if err != nil {
log.Fatal(err)
}
blobStore = store.NewDBBackedS3Store(s3, db)
reflectorServer = reflector.NewServer(blobStore)
reflectorServer.Timeout = 3 * time.Minute
if globalConfig.SlackHookURL != "" {
reflectorServer.StatLogger = log.StandardLogger()
reflectorServer.StatReportFrequency = 1 * time.Hour
}
reflectorServer.EnableBlocklist = true
err = reflectorServer.Start(":" + strconv.Itoa(reflector.DefaultPort))
if err != nil {
log.Fatal(err)
}
}
peerServer := peer.NewServer(blobStore)
if globalConfig.SlackHookURL != "" {
peerServer.StatLogger = log.StandardLogger()
peerServer.StatReportFrequency = 1 * time.Hour
}
err = peerServer.Start(":5567")
if err != nil {
log.Fatal(err)
}
@@ -43,5 +77,8 @@ func reflectorCmd(cmd *cobra.Command, args []string) {
interruptChan := make(chan os.Signal, 1)
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
<-interruptChan
reflectorServer.Shutdown()
peerServer.Shutdown()
if reflectorServer != nil {
reflectorServer.Shutdown()
}
}
+14 -3
View File
@@ -5,9 +5,10 @@ import (
"io/ioutil"
"os"
"github.com/lbryio/lbry.go/errors"
"github.com/lbryio/lbry.go/util"
"github.com/lbryio/reflector.go/dht"
"github.com/lbryio/lbry.go/dht"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/util"
"github.com/lbryio/reflector.go/updater"
"github.com/johntdyer/slackrus"
"github.com/sirupsen/logrus"
@@ -22,6 +23,8 @@ type Config struct {
BucketName string `json:"bucket_name"`
DBConn string `json:"db_conn"`
SlackHookURL string `json:"slack_hook_url"`
UpdateBinURL string `json:"update_bin_url"`
UpdateCmd string `json:"update_cmd"`
}
var verbose []string
@@ -102,6 +105,14 @@ func preRun(cmd *cobra.Command, args []string) {
logrus.AddHook(hook)
debugLogger.AddHook(hook)
}
if globalConfig.UpdateBinURL != "" {
if globalConfig.UpdateCmd == "" {
logrus.Warnln("update_cmd is empty in conf file")
}
logrus.Println("starting update checker")
go updater.Run(globalConfig.UpdateBinURL, globalConfig.UpdateCmd)
}
}
func checkErr(err error) {
+43
View File
@@ -0,0 +1,43 @@
package cmd
import (
"crypto/rand"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/lbry.go/stream"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
func init() {
var cmd = &cobra.Command{
Use: "sendblob ADDRESS:PORT",
Short: "Send a random blob to a reflector server",
Args: cobra.ExactArgs(1),
Run: sendBlobCmd,
}
rootCmd.AddCommand(cmd)
}
func sendBlobCmd(cmd *cobra.Command, args []string) {
addr := args[0]
c := reflector.Client{}
err := c.Connect(addr)
if err != nil {
log.Fatal("error connecting client to server: ", err)
}
blob := make(stream.Blob, 1024)
_, err = rand.Read(blob)
if err != nil {
log.Fatal("failed to make random blob: ", err)
}
err = c.SendBlob(blob)
if err != nil {
log.Error(err)
}
}
+2 -2
View File
@@ -7,10 +7,10 @@ import (
"strings"
"syscall"
"github.com/lbryio/lbry.go/dht"
"github.com/lbryio/lbry.go/dht/bits"
"github.com/lbryio/reflector.go/cluster"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/dht"
"github.com/lbryio/reflector.go/dht/bits"
"github.com/lbryio/reflector.go/peer"
"github.com/lbryio/reflector.go/prism"
"github.com/lbryio/reflector.go/reflector"
+58
View File
@@ -0,0 +1,58 @@
package cmd
import (
"fmt"
"os"
"os/signal"
"strconv"
"syscall"
"time"
"github.com/lbryio/reflector.go/meta"
"github.com/lbryio/reflector.go/peer"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/store"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
func init() {
var cmd = &cobra.Command{
Use: "test",
Short: "Test things",
Run: testCmd,
}
rootCmd.AddCommand(cmd)
}
func testCmd(cmd *cobra.Command, args []string) {
log.Printf("reflector version %s", meta.Version)
memStore := &store.MemoryBlobStore{}
reflectorServer := reflector.NewServer(memStore)
reflectorServer.Timeout = 3 * time.Minute
err := reflectorServer.Start(":" + strconv.Itoa(reflector.DefaultPort))
if err != nil {
log.Fatal(err)
}
peerServer := peer.NewServer(memStore)
err = peerServer.Start(":" + strconv.Itoa(reflector.DefaultPort+1))
if err != nil {
log.Fatal(err)
}
interruptChan := make(chan os.Signal, 1)
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
<-interruptChan
peerServer.Shutdown()
reflectorServer.Shutdown()
fmt.Println("Blobs in store")
for hash, blob := range memStore.Debug() {
fmt.Printf("%s: %d bytes\n", hash, len(blob))
}
}
+8 -226
View File
@@ -1,45 +1,20 @@
package cmd
import (
"encoding/json"
"io/ioutil"
"os"
"os/signal"
"path"
"sync"
"syscall"
"time"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/peer"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/stop"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
var uploadWorkers int
var uploadSkipExistsCheck bool
const (
sdInc = 1
blobInc = 2
errInc = 3
)
type uploaderParams struct {
workerWG *sync.WaitGroup
counterWG *sync.WaitGroup
stopper *stop.Group
pathChan chan string
countChan chan int
sdCount int
blobCount int
errCount int
}
func init() {
var cmd = &cobra.Command{
Use: "upload PATH",
@@ -53,216 +28,23 @@ func init() {
}
func uploadCmd(cmd *cobra.Command, args []string) {
startTime := time.Now()
db := new(db.SQL)
err := db.Connect(globalConfig.DBConn)
checkErr(err)
params := uploaderParams{
workerWG: &sync.WaitGroup{},
counterWG: &sync.WaitGroup{},
pathChan: make(chan string),
countChan: make(chan int),
stopper: stop.New()}
st := store.NewDBBackedS3Store(
store.NewS3BlobStore(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName),
db)
setInterrupt(params.stopper)
uploader := reflector.NewUploader(db, st, uploadWorkers, uploadSkipExistsCheck)
paths, err := getPaths(args[0])
checkErr(err)
totalCount := len(paths)
hashes := make([]string, len(paths))
for i, p := range paths {
hashes[i] = path.Base(p)
}
log.Println("checking for existing blobs")
exists := make(map[string]bool)
if !uploadSkipExistsCheck {
exists, err = db.HasBlobs(hashes)
checkErr(err)
}
existsCount := len(exists)
log.Printf("%d new blobs to upload", totalCount-existsCount)
startUploadWorkers(&params)
params.counterWG.Add(1)
go func() {
defer params.counterWG.Done()
runCountReceiver(&params, startTime, totalCount, existsCount)
}()
Upload:
for _, f := range paths {
if exists[path.Base(f)] {
continue
}
select {
case params.pathChan <- f:
case <-params.stopper.Ch():
log.Warnln("Caught interrupt, quitting at first opportunity...")
break Upload
}
}
close(params.pathChan)
params.workerWG.Wait()
close(params.countChan)
params.counterWG.Wait()
params.stopper.Stop()
log.Println("SUMMARY")
log.Printf("%d blobs total", totalCount)
log.Printf("%d SD blobs uploaded", params.sdCount)
log.Printf("%d content blobs uploaded", params.blobCount)
log.Printf("%d blobs already stored", existsCount)
log.Printf("%d errors encountered", params.errCount)
}
func isJSON(data []byte) bool {
var js json.RawMessage
return json.Unmarshal(data, &js) == nil
}
func newBlobStore() *store.DBBackedS3Store {
db := new(db.SQL)
err := db.Connect(globalConfig.DBConn)
checkErr(err)
s3 := store.NewS3BlobStore(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
return store.NewDBBackedS3Store(s3, db)
}
func setInterrupt(stopper *stop.Group) {
interruptChan := make(chan os.Signal, 1)
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
go func() {
<-interruptChan
stopper.Stop()
uploader.Stop()
}()
}
func startUploadWorkers(params *uploaderParams) {
for i := 0; i < uploadWorkers; i++ {
params.workerWG.Add(1)
go func(i int) {
defer params.workerWG.Done()
defer func(i int) {
log.Printf("worker %d quitting", i)
}(i)
blobStore := newBlobStore()
launchFileUploader(params, blobStore, i)
}(i)
}
}
func launchFileUploader(params *uploaderParams, blobStore *store.DBBackedS3Store, worker int) {
for {
select {
case <-params.stopper.Ch():
return
case filepath, ok := <-params.pathChan:
if !ok {
return
}
blob, err := ioutil.ReadFile(filepath)
checkErr(err)
hash := peer.GetBlobHash(blob)
if hash != path.Base(filepath) {
log.Errorf("worker %d: file name does not match hash (%s != %s), skipping", worker, filepath, hash)
select {
case params.countChan <- errInc:
case <-params.stopper.Ch():
}
continue
}
if isJSON(blob) {
log.Printf("worker %d: PUTTING SD BLOB %s", worker, hash)
err := blobStore.PutSD(hash, blob)
if err != nil {
log.Error("PutSD Error: ", err)
}
select {
case params.countChan <- sdInc:
case <-params.stopper.Ch():
}
} else {
log.Printf("worker %d: putting %s", worker, hash)
err = blobStore.Put(hash, blob)
if err != nil {
log.Error("put Blob Error: ", err)
}
select {
case params.countChan <- blobInc:
case <-params.stopper.Ch():
}
}
}
}
}
func runCountReceiver(params *uploaderParams, startTime time.Time, totalCount int, existsCount int) {
for {
select {
case <-params.stopper.Ch():
return
case countType, ok := <-params.countChan:
if !ok {
return
}
switch countType {
case sdInc:
params.sdCount++
case blobInc:
params.blobCount++
case errInc:
params.errCount++
}
}
if (params.sdCount+params.blobCount)%50 == 0 {
log.Printf("%d of %d done (%s elapsed, %.3fs per blob)", params.sdCount+params.blobCount, totalCount-existsCount, time.Since(startTime).String(), time.Since(startTime).Seconds()/float64(params.sdCount+params.blobCount))
}
}
}
func getPaths(path string) ([]string, error) {
info, err := os.Stat(path)
if err != nil {
return nil, err
}
if info.Mode().IsRegular() {
return []string{path}, nil
}
f, err := os.Open(path)
if err != nil {
return nil, err
}
files, err := f.Readdir(-1)
if err != nil {
return nil, err
}
err = f.Close()
if err != nil {
return nil, err
}
var filenames []string
for _, file := range files {
if !file.IsDir() {
filenames = append(filenames, path+"/"+file.Name())
}
}
return filenames, nil
err = uploader.Upload(args[0])
checkErr(err)
}
+3 -3
View File
@@ -1,10 +1,10 @@
package cmd
import (
"log"
"fmt"
"time"
"github.com/lbryio/reflector.go/meta"
"github.com/spf13/cobra"
)
@@ -18,5 +18,5 @@ func init() {
}
func versionCmd(cmd *cobra.Command, args []string) {
log.Printf("version %s\n", meta.Version)
fmt.Printf("version %s (built %s)\n", meta.Version, meta.BuildTime.Format(time.RFC3339))
}
+8
View File
@@ -0,0 +1,8 @@
{
"aws_id": "YOUR-AWS-ID",
"aws_secret": "YOUR-AWS-SECRET",
"bucket_region": "YOUR-BUCKET-REGION",
"bucket_name": "YOUR-BUCKET-NAME",
"db_conn": "USER:PASSWORD@tcp(localhost:3306)/DBNAME",
"slack_hook_url": ""
}
+322 -112
View File
@@ -3,12 +3,14 @@ package db
import (
"context"
"database/sql"
"time"
"github.com/lbryio/lbry.go/errors"
"github.com/lbryio/lbry.go/querytools"
"github.com/lbryio/reflector.go/dht/bits"
"github.com/lbryio/lbry.go/dht/bits"
"github.com/lbryio/lbry.go/extras/errors"
qt "github.com/lbryio/lbry.go/extras/query"
_ "github.com/go-sql-driver/mysql" // blank import for db driver
"github.com/go-sql-driver/mysql"
_ "github.com/go-sql-driver/mysql" // blank import for db driver ensures its imported even if its not used
log "github.com/sirupsen/logrus"
)
@@ -19,7 +21,7 @@ type SdBlob struct {
Length int `json:"length"`
BlobNum int `json:"blob_num"`
BlobHash string `json:"blob_hash,omitempty"`
Iv string `json:"iv"`
IV string `json:"iv"`
} `json:"blobs"`
StreamType string `json:"stream_type"`
Key string `json:"key"`
@@ -33,7 +35,7 @@ type SQL struct {
}
func logQuery(query string, args ...interface{}) {
s, err := querytools.InterpolateParams(query, args...)
s, err := qt.InterpolateParams(query, args...)
if err != nil {
log.Errorln(err)
} else {
@@ -53,6 +55,8 @@ func (s *SQL) Connect(dsn string) error {
return errors.Err(err)
}
s.conn.SetMaxIdleConns(12)
return errors.Err(s.conn.Ping())
}
@@ -62,44 +66,64 @@ func (s *SQL) AddBlob(hash string, length int, isStored bool) error {
return errors.Err("not connected")
}
return withTx(s.conn, func(tx *sql.Tx) error {
return addBlob(tx, hash, length, isStored)
})
_, err := s.insertBlob(hash, length, isStored)
return err
}
func addBlob(tx *sql.Tx, hash string, length int, isStored bool) error {
func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error) {
if length <= 0 {
return errors.Err("length must be positive")
return 0, errors.Err("length must be positive")
}
err := execTx(tx,
blobID, err := s.exec(
"INSERT INTO blob_ (hash, is_stored, length) VALUES (?,?,?) ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored))",
[]interface{}{hash, isStored, length},
hash, isStored, length,
)
if err != nil {
return errors.Err(err)
return 0, err
}
return nil
if blobID == 0 {
err = s.conn.QueryRow("SELECT id FROM blob_ WHERE hash = ?", hash).Scan(&blobID)
if err != nil {
return 0, errors.Err(err)
}
if blobID == 0 {
return 0, errors.Err("blob ID is 0 even after INSERTing and SELECTing")
}
}
return blobID, nil
}
func (s *SQL) insertStream(hash, sdHash string, sdBlobID int64) (int64, error) {
streamID, err := s.exec(
"INSERT IGNORE INTO stream (hash, sd_hash, sd_blob_id) VALUES (?,?, ?)",
hash, sdHash, sdBlobID,
)
if err != nil {
return 0, errors.Err(err)
}
if streamID == 0 {
err = s.conn.QueryRow("SELECT id FROM stream WHERE sd_blob_id = ?", sdBlobID).Scan(&streamID)
if err != nil {
return 0, errors.Err(err)
}
if streamID == 0 {
return 0, errors.Err("stream ID is 0 even after INSERTing and SELECTing")
}
}
return streamID, nil
}
// HasBlob checks if the database contains the blob information.
func (s *SQL) HasBlob(hash string) (bool, error) {
if s.conn == nil {
return false, errors.Err("not connected")
exists, err := s.HasBlobs([]string{hash})
if err != nil {
return false, err
}
query := "SELECT EXISTS(SELECT 1 FROM blob_ WHERE hash = ? AND is_stored = ?)"
args := []interface{}{hash, true}
logQuery(query, args...)
row := s.conn.QueryRow(query, args...)
exists := false
err := row.Scan(&exists)
return exists, errors.Err(err)
return exists[hash], nil
}
// HasBlobs checks if the database contains the set of blobs and returns a bool map.
@@ -110,7 +134,7 @@ func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
var hash string
exists := make(map[string]bool)
maxBatchSize := 100
maxBatchSize := 10000
doneIndex := 0
for len(hashes) > doneIndex {
@@ -121,7 +145,7 @@ func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
log.Debugf("getting hashes[%d:%d] of %d", doneIndex, sliceEnd, len(hashes))
batch := hashes[doneIndex:sliceEnd]
query := "SELECT hash FROM blob_ WHERE is_stored = ? && hash IN (" + querytools.Qs(len(batch)) + ")"
query := "SELECT hash FROM blob_ WHERE is_stored = ? && hash IN (" + qt.Qs(len(batch)) + ")"
args := make([]interface{}, len(batch)+1)
args[0] = true
for i := range batch {
@@ -130,106 +154,169 @@ func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
logQuery(query, args...)
rows, err := s.conn.Query(query, args...)
if err != nil {
closeRows(rows)
return exists, err
}
for rows.Next() {
err := rows.Scan(&hash)
err := func() error {
startTime := time.Now()
rows, err := s.conn.Query(query, args...)
log.Debugf("hashes query took %s", time.Since(startTime))
if err != nil {
closeRows(rows)
return exists, err
return errors.Err(err)
}
exists[hash] = true
}
defer closeRows(rows)
err = rows.Err()
for rows.Next() {
err := rows.Scan(&hash)
if err != nil {
return errors.Err(err)
}
exists[hash] = true
}
err = rows.Err()
if err != nil {
return errors.Err(err)
}
doneIndex += len(batch)
return nil
}()
if err != nil {
closeRows(rows)
return exists, err
return nil, err
}
closeRows(rows)
doneIndex += len(batch)
}
return exists, nil
}
// HasFullStream checks if the full stream has been uploaded (i.e. if we have the sd blob and all the content blobs)
func (s *SQL) HasFullStream(sdHash string) (bool, error) {
if s.conn == nil {
return false, errors.Err("not connected")
// Delete will remove the blob from the db
func (s *SQL) Delete(hash string) error {
_, err := s.exec("DELETE FROM stream WHERE sd_hash = ?", hash)
if err != nil {
return errors.Err(err)
}
query := `SELECT EXISTS(
SELECT 1 FROM stream s
LEFT JOIN stream_blob sb ON s.hash = sb.stream_hash
LEFT JOIN blob_ b ON b.hash = sb.blob_hash
WHERE s.sd_hash = ?
GROUP BY s.sd_hash
HAVING min(b.is_stored = 1)
);`
_, err = s.exec("DELETE FROM blob_ WHERE hash = ?", hash)
return errors.Err(err)
}
// Block will mark a blob as blocked
func (s *SQL) Block(hash string) error {
query := "INSERT IGNORE INTO blocked SET hash = ?"
args := []interface{}{hash}
logQuery(query, args...)
_, err := s.conn.Exec(query, args...)
return errors.Err(err)
}
// GetBlocked will return a list of blocked hashes
func (s *SQL) GetBlocked() (map[string]bool, error) {
query := "SELECT hash FROM blocked"
logQuery(query)
rows, err := s.conn.Query(query)
if err != nil {
return nil, errors.Err(err)
}
defer closeRows(rows)
blocked := make(map[string]bool)
var hash string
for rows.Next() {
err := rows.Scan(&hash)
if err != nil {
return nil, errors.Err(err)
}
blocked[hash] = true
}
err = rows.Err()
if err != nil {
return nil, errors.Err(err)
}
return blocked, nil
}
// MissingBlobsForKnownStream returns missing blobs for an existing stream
// WARNING: if the stream does NOT exist, no blob hashes will be returned, which looks
// like no blobs are missing
func (s *SQL) MissingBlobsForKnownStream(sdHash string) ([]string, error) {
if s.conn == nil {
return nil, errors.Err("not connected")
}
query := `
SELECT b.hash FROM blob_ b
INNER JOIN stream_blob sb ON b.hash = sb.blob_hash
INNER JOIN stream s ON s.hash = sb.stream_hash AND s.sd_hash = ?
WHERE b.is_stored = 0
`
args := []interface{}{sdHash}
logQuery(query, args...)
row := s.conn.QueryRow(query, args...)
rows, err := s.conn.Query(query, args...)
if err != nil {
return nil, errors.Err(err)
}
defer closeRows(rows)
exists := false
err := row.Scan(&exists)
var missingBlobs []string
var hash string
return exists, errors.Err(err)
for rows.Next() {
err := rows.Scan(&hash)
if err != nil {
return nil, errors.Err(err)
}
missingBlobs = append(missingBlobs, hash)
}
err = rows.Err()
if err != nil {
return nil, errors.Err(err)
}
return missingBlobs, errors.Err(err)
}
// AddSDBlob takes the SD Hash number of blobs and the set of blobs. In a single db tx it inserts the sdblob information
// into a stream, and inserts the associated blobs' information in the database. If a blob fails the transaction is
// rolled back and error(s) are returned.
// AddSDBlob insert the SD blob and all the content blobs. The content blobs are marked as "not stored",
// but they are tracked so reflector knows what it is missing.
func (s *SQL) AddSDBlob(sdHash string, sdBlobLength int, sdBlob SdBlob) error {
if s.conn == nil {
return errors.Err("not connected")
}
return withTx(s.conn, func(tx *sql.Tx) error {
// insert sd blob
err := addBlob(tx, sdHash, sdBlobLength, true)
sdBlobID, err := s.insertBlob(sdHash, sdBlobLength, true)
if err != nil {
return err
}
streamID, err := s.insertStream(sdBlob.StreamHash, sdHash, sdBlobID)
if err != nil {
return err
}
// insert content blobs and connect them to stream
for _, contentBlob := range sdBlob.Blobs {
if contentBlob.BlobHash == "" {
// null terminator blob
continue
}
blobID, err := s.insertBlob(contentBlob.BlobHash, contentBlob.Length, false)
if err != nil {
return err
}
// insert stream
err = execTx(tx,
"INSERT IGNORE INTO stream (hash, sd_hash) VALUES (?,?)",
[]interface{}{sdBlob.StreamHash, sdHash},
_, err = s.exec(
"INSERT IGNORE INTO stream_blob (stream_id, stream_hash, blob_id, blob_hash, num) VALUES (?,?,?,?,?)",
streamID, sdBlob.StreamHash, blobID, contentBlob.BlobHash, contentBlob.BlobNum,
)
if err != nil {
return errors.Err(err)
}
// insert content blobs and connect them to stream
for _, contentBlob := range sdBlob.Blobs {
if contentBlob.BlobHash == "" {
// null terminator blob
continue
}
err := addBlob(tx, contentBlob.BlobHash, contentBlob.Length, false)
if err != nil {
return err
}
err = execTx(tx,
"INSERT IGNORE INTO stream_blob (stream_hash, blob_hash, num) VALUES (?,?,?)",
[]interface{}{sdBlob.StreamHash, contentBlob.BlobHash, contentBlob.BlobNum},
)
if err != nil {
return errors.Err(err)
}
}
return nil
})
}
return nil
}
// GetHashRange gets the smallest and biggest hashes in the db
@@ -347,38 +434,161 @@ func closeRows(rows *sql.Rows) {
}
}
func execTx(tx *sql.Tx, query string, args []interface{}) error {
func (s *SQL) exec(query string, args ...interface{}) (int64, error) {
logQuery(query, args...)
_, err := tx.Exec(query, args...)
return errors.Err(err)
attempt, maxAttempts := 0, 3
Retry:
attempt++
result, err := s.conn.Exec(query, args...)
if isLockTimeoutError(err) {
if attempt <= maxAttempts {
//Error 1205: Lock wait timeout exceeded; try restarting transaction
goto Retry
}
err = errors.Prefix("Lock timeout for query "+query, err)
}
if err != nil {
return 0, errors.Err(err)
}
lastID, err := result.LastInsertId()
return lastID, errors.Err(err)
}
func isLockTimeoutError(err error) bool {
e, ok := err.(*mysql.MySQLError)
return ok && e != nil && e.Number == 1205
}
/* SQL schema
in prod, set tx_isolation to READ-COMMITTED to improve db performance
make sure you use latin1 or utf8 charset, NOT utf8mb4. that's a waste of space.
todo: could add UNIQUE KEY (stream_hash, num) to stream_blob ...
CREATE TABLE blob_ (
id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT UNIQUE,
hash char(96) NOT NULL,
is_stored TINYINT(1) NOT NULL DEFAULT 0,
length bigint(20) unsigned DEFAULT NULL,
PRIMARY KEY (hash)
PRIMARY KEY (id),
UNIQUE KEY blob_hash_idx (hash)
);
CREATE TABLE stream (
id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT UNIQUE,
hash char(96) NOT NULL,
sd_hash char(96) NOT NULL,
PRIMARY KEY (hash),
KEY sd_hash_idx (sd_hash),
FOREIGN KEY (sd_hash) REFERENCES blob_ (hash) ON DELETE RESTRICT ON UPDATE CASCADE
sd_blob_id BIGINT UNSIGNED NOT NULL,
PRIMARY KEY (id),
UNIQUE KEY stream_hash_idx (hash),
KEY stream_sd_blob_id_idx (sd_blob_id),
FOREIGN KEY (sd_blob_id) REFERENCES blob_ (id) ON DELETE RESTRICT ON UPDATE CASCADE
);
CREATE TABLE stream_blob (
stream_hash char(96) NOT NULL,
blob_hash char(96) NOT NULL,
stream_id BIGINT UNSIGNED NOT NULL,
blob_id BIGINT UNSIGNED NOT NULL,
num int NOT NULL,
PRIMARY KEY (stream_hash, blob_hash),
FOREIGN KEY (stream_hash) REFERENCES stream (hash) ON DELETE CASCADE ON UPDATE CASCADE,
FOREIGN KEY (blob_hash) REFERENCES blob_ (hash) ON DELETE CASCADE ON UPDATE CASCADE
PRIMARY KEY (stream_id, blob_id),
KEY stream_blob_blob_id_idx (blob_id),
FOREIGN KEY (stream_id) REFERENCES stream (id) ON DELETE CASCADE ON UPDATE CASCADE,
FOREIGN KEY (blob_id) REFERENCES blob_ (id) ON DELETE CASCADE ON UPDATE CASCADE
);
could add UNIQUE KEY (stream_hash, num) to stream_blob ...
CREATE TABLE blocked (
hash char(96) NOT NULL,
PRIMARY KEY (hash)
);
FOR THE MIGRATION, USE THESE TRIGGERS
CREATE TRIGGER tgr_stream_insert AFTER INSERT ON stream FOR EACH ROW INSERT INTO stream_new SET id = NEW.id, hash = NEW.hash, sd_blob_id = (SELECT id from blob_ where hash = NEW.sd_hash);
CREATE TRIGGER tgr_stream_delete AFTER DELETE ON stream FOR EACH ROW DELETE FROM stream_new WHERE id = OLD.id;
CREATE TRIGGER tgr_blob_insert AFTER INSERT ON blob_ FOR EACH ROW INSERT INTO blob_new SET id = NEW.id, hash = NEW.hash, is_stored = NEW.is_stored, length = NEW.length;
CREATE TRIGGER tgr_blob_update AFTER UPDATE ON blob_ FOR EACH ROW UPDATE blob_new SET hash = NEW.hash, is_stored = NEW.is_stored, length = NEW.length WHERE id = NEW.id;
CREATE TRIGGER tgr_blob_delete AFTER DELETE ON blob_ FOR EACH ROW DELETE FROM blob_new WHERE id = OLD.id;
CREATE TRIGGER tgr_stream_blob_insert AFTER INSERT ON stream_blob FOR EACH ROW INSERT INTO stream_blob_new SET stream_id = NEW.stream_id, blob_id = NEW.blob_id, num = NEW.num;
CREATE TRIGGER tgr_stream_blob_delete AFTER DELETE ON stream_blob FOR EACH ROW DELETE FROM stream_blob_new WHERE stream_id = OLD.stream_id AND blob_id = OLD.blob_id;
DROP PROCEDURE IF EXISTS copyblobs;
DELIMITER $$
CREATE PROCEDURE copyblobs()
BEGIN
DECLARE first_trigger_blob_id INT DEFAULT 124802284; # ID of first blob that was copied using the triggers. dont copy anything after that.
DECLARE i INT DEFAULT 0;
DECLARE minid BIGINT UNSIGNED DEFAULT 0;
SELECT min(id) INTO minid FROM blob_ WHERE id < first_trigger_blob_id AND id > (SELECT coalesce(max(id),0) from blob_new where id < first_trigger_blob_id);
wloop: WHILE minid is not null DO
#IF (i >= 100) THEN
# LEAVE wloop;
#END IF;
SET i = i + 1;
IF (i % 5000 = 0) THEN
SELECT concat('loop ', i, ', id ', minid) as progress;
END IF;
IF (i % 10 = 1) THEN # we start our loops on 1, like normal people
START TRANSACTION;
END IF;
INSERT INTO blob_new (id, hash, is_stored, length) SELECT id, hash, is_stored, length from blob_ where id = minid;
IF (i % 10 = 0) THEN
COMMIT;
END IF;
SELECT min(id) INTO minid FROM blob_ WHERE id < first_trigger_blob_id AND id > minid;
END WHILE wloop;
COMMIT;
END$$
DELIMITER ;
DROP PROCEDURE IF EXISTS copystreams;
DELIMITER $$
CREATE PROCEDURE copystreams()
BEGIN
DECLARE first_trigger_stream_id INT DEFAULT 1465749; # ID of first stream that was copied using the triggers. dont copy anything after that.
DECLARE i INT DEFAULT 0;
DECLARE minid BIGINT UNSIGNED DEFAULT 0;
DECLARE streamhash char(96);
SELECT min(id) INTO minid FROM stream WHERE id < first_trigger_stream_id AND id > (SELECT coalesce(max(id),0) from stream_new where id < first_trigger_stream_id);
wloop: WHILE minid is not null DO
#IF (i >= 10) THEN
# LEAVE wloop;
#END IF;
SET i = i + 1;
IF (i % 5000 = 0) THEN
SELECT concat('loop ', i, ', id ', minid) as progress;
END IF;
IF (i % 10 = 1) THEN # we start our loops on 1, like normal people
START TRANSACTION;
END IF;
SELECT hash INTO streamhash FROM stream WHERE id = minid;
INSERT INTO stream_new (id, hash, sd_blob_id) SELECT s.id, s.hash, b.id FROM stream s INNER JOIN blob_ b ON s.sd_hash = b.hash WHERE s.id = minid;
INSERT INTO stream_blob_new SELECT minid, b.id, sb.num FROM stream_blob sb INNER JOIN blob_ b ON sb.blob_hash = b.hash WHERE sb.stream_hash = streamhash;
IF (i % 10 = 0) THEN
COMMIT;
END IF;
SELECT min(id) INTO minid FROM stream WHERE id < first_trigger_stream_id AND id > minid;
END WHILE wloop;
COMMIT;
END$$
DELIMITER ;
*/
-399
View File
@@ -1,399 +0,0 @@
package bits
import (
"crypto/rand"
"encoding/hex"
"math/big"
"strconv"
"strings"
"github.com/lbryio/lbry.go/errors"
"github.com/lyoshenka/bencode"
)
// TODO: http://roaringbitmap.org/
const (
NumBytes = 48 // bytes
NumBits = NumBytes * 8
)
// Bitmap is a generalized representation of an identifier or data that can be sorted, compared fast. Used by the DHT
// package as a way to handle the unique identifiers of a DHT node.
type Bitmap [NumBytes]byte
func (b Bitmap) RawString() string {
return string(b[:])
}
func (b Bitmap) String() string {
return b.Hex()
}
// BString returns the bitmap as a string of 0s and 1s
func (b Bitmap) BString() string {
var s string
for _, byte := range b {
s += strconv.FormatInt(int64(byte), 2)
}
return s
}
// Hex returns a hexadecimal representation of the bitmap.
func (b Bitmap) Hex() string {
return hex.EncodeToString(b[:])
}
// HexShort returns a hexadecimal representation of the first 4 bytes.
func (b Bitmap) HexShort() string {
return hex.EncodeToString(b[:4])
}
// HexSimplified returns the hexadecimal representation with all leading 0's removed
func (b Bitmap) HexSimplified() string {
simple := strings.TrimLeft(b.Hex(), "0")
if simple == "" {
simple = "0"
}
return simple
}
func (b Bitmap) Big() *big.Int {
i := new(big.Int)
i.SetString(b.Hex(), 16)
return i
}
// Cmp compares b and other and returns:
//
// -1 if b < other
// 0 if b == other
// +1 if b > other
//
func (b Bitmap) Cmp(other Bitmap) int {
for k := range b {
if b[k] < other[k] {
return -1
} else if b[k] > other[k] {
return 1
}
}
return 0
}
// Closer returns true if dist(b,x) < dist(b,y)
func (b Bitmap) Closer(x, y Bitmap) bool {
return x.Xor(b).Cmp(y.Xor(b)) < 0
}
// Equals returns true if every byte in bitmap are equal, false otherwise
func (b Bitmap) Equals(other Bitmap) bool {
return b.Cmp(other) == 0
}
// Copy returns a duplicate value for the bitmap.
func (b Bitmap) Copy() Bitmap {
var ret Bitmap
copy(ret[:], b[:])
return ret
}
// Xor returns a diff bitmap. If they are equal, the returned bitmap will be all 0's. If 100% unique the returned
// bitmap will be all 1's.
func (b Bitmap) Xor(other Bitmap) Bitmap {
var ret Bitmap
for k := range b {
ret[k] = b[k] ^ other[k]
}
return ret
}
// And returns a comparison bitmap, that for each byte returns the AND true table result
func (b Bitmap) And(other Bitmap) Bitmap {
var ret Bitmap
for k := range b {
ret[k] = b[k] & other[k]
}
return ret
}
// Or returns a comparison bitmap, that for each byte returns the OR true table result
func (b Bitmap) Or(other Bitmap) Bitmap {
var ret Bitmap
for k := range b {
ret[k] = b[k] | other[k]
}
return ret
}
// Not returns a complimentary bitmap that is an inverse. So b.NOT.NOT = b
func (b Bitmap) Not() Bitmap {
var ret Bitmap
for k := range b {
ret[k] = ^b[k]
}
return ret
}
func (b Bitmap) add(other Bitmap) (Bitmap, bool) {
var ret Bitmap
carry := false
for i := NumBits - 1; i >= 0; i-- {
bBit := getBit(b[:], i)
oBit := getBit(other[:], i)
setBit(ret[:], i, bBit != oBit != carry)
carry = (bBit && oBit) || (bBit && carry) || (oBit && carry)
}
return ret, carry
}
// Add returns a bitmap that treats both bitmaps as numbers and adding them together. Since the size of a bitmap is
// limited, an overflow is possible when adding bitmaps.
func (b Bitmap) Add(other Bitmap) Bitmap {
ret, carry := b.add(other)
if carry {
panic("overflow in bitmap addition. limited to " + strconv.Itoa(NumBits) + " bits.")
}
return ret
}
// Sub returns a bitmap that treats both bitmaps as numbers and subtracts then via the inverse of the other and adding
// then together a + (-b). Negative bitmaps are not supported so other must be greater than this.
func (b Bitmap) Sub(other Bitmap) Bitmap {
if b.Cmp(other) < 0 {
// ToDo: Why is this not supported? Should it say not implemented? BitMap might have a generic use case outside of dht.
panic("negative bitmaps not supported")
}
complement, _ := other.Not().add(FromShortHexP("1"))
ret, _ := b.add(complement)
return ret
}
// Get returns the binary bit at the position passed.
func (b Bitmap) Get(n int) bool {
return getBit(b[:], n)
}
// Set sets the binary bit at the position passed.
func (b Bitmap) Set(n int, one bool) Bitmap {
ret := b.Copy()
setBit(ret[:], n, one)
return ret
}
// PrefixLen returns the number of leading 0 bits
func (b Bitmap) PrefixLen() int {
for i := range b {
for j := 0; j < 8; j++ {
if (b[i]>>uint8(7-j))&0x1 != 0 {
return i*8 + j
}
}
}
return NumBits
}
// Prefix returns a copy of b with the first n bits set to 1 (if `one` is true) or 0 (if `one` is false)
// https://stackoverflow.com/a/23192263/182709
func (b Bitmap) Prefix(n int, one bool) Bitmap {
ret := b.Copy()
Outer:
for i := range ret {
for j := 0; j < 8; j++ {
if i*8+j < n {
if one {
ret[i] |= 1 << uint(7-j)
} else {
ret[i] &= ^(1 << uint(7-j))
}
} else {
break Outer
}
}
}
return ret
}
// Suffix returns a copy of b with the last n bits set to 1 (if `one` is true) or 0 (if `one` is false)
// https://stackoverflow.com/a/23192263/182709
func (b Bitmap) Suffix(n int, one bool) Bitmap {
ret := b.Copy()
Outer:
for i := len(ret) - 1; i >= 0; i-- {
for j := 7; j >= 0; j-- {
if i*8+j >= NumBits-n {
if one {
ret[i] |= 1 << uint(7-j)
} else {
ret[i] &= ^(1 << uint(7-j))
}
} else {
break Outer
}
}
}
return ret
}
// MarshalBencode implements the Marshaller(bencode)/Message interface.
func (b Bitmap) MarshalBencode() ([]byte, error) {
str := string(b[:])
return bencode.EncodeBytes(str)
}
// UnmarshalBencode implements the Marshaller(bencode)/Message interface.
func (b *Bitmap) UnmarshalBencode(encoded []byte) error {
var str string
err := bencode.DecodeBytes(encoded, &str)
if err != nil {
return err
}
if len(str) != NumBytes {
return errors.Err("invalid bitmap length")
}
copy(b[:], str)
return nil
}
// FromBytes returns a bitmap as long as the byte array is of a specific length specified in the parameters.
func FromBytes(data []byte) (Bitmap, error) {
var bmp Bitmap
if len(data) != len(bmp) {
return bmp, errors.Err("invalid bitmap of length %d", len(data))
}
copy(bmp[:], data)
return bmp, nil
}
// FromBytesP returns a bitmap as long as the byte array is of a specific length specified in the parameters
// otherwise it wil panic.
func FromBytesP(data []byte) Bitmap {
bmp, err := FromBytes(data)
if err != nil {
panic(err)
}
return bmp
}
//FromString returns a bitmap by converting the string to bytes and creating from bytes as long as the byte array
// is of a specific length specified in the parameters
func FromString(data string) (Bitmap, error) {
return FromBytes([]byte(data))
}
//FromStringP returns a bitmap by converting the string to bytes and creating from bytes as long as the byte array
// is of a specific length specified in the parameters otherwise it wil panic.
func FromStringP(data string) Bitmap {
bmp, err := FromString(data)
if err != nil {
panic(err)
}
return bmp
}
//FromHex returns a bitmap by converting the hex string to bytes and creating from bytes as long as the byte array
// is of a specific length specified in the parameters
func FromHex(hexStr string) (Bitmap, error) {
decoded, err := hex.DecodeString(hexStr)
if err != nil {
return Bitmap{}, errors.Err(err)
}
return FromBytes(decoded)
}
//FromHexP returns a bitmap by converting the hex string to bytes and creating from bytes as long as the byte array
// is of a specific length specified in the parameters otherwise it wil panic.
func FromHexP(hexStr string) Bitmap {
bmp, err := FromHex(hexStr)
if err != nil {
panic(err)
}
return bmp
}
//FromShortHex returns a bitmap by converting the hex string to bytes, adding the leading zeros prefix to the
// hex string and creating from bytes as long as the byte array is of a specific length specified in the parameters
func FromShortHex(hexStr string) (Bitmap, error) {
return FromHex(strings.Repeat("0", NumBytes*2-len(hexStr)) + hexStr)
}
//FromShortHexP returns a bitmap by converting the hex string to bytes, adding the leading zeros prefix to the
// hex string and creating from bytes as long as the byte array is of a specific length specified in the parameters
// otherwise it wil panic.
func FromShortHexP(hexStr string) Bitmap {
bmp, err := FromShortHex(hexStr)
if err != nil {
panic(err)
}
return bmp
}
func FromBigP(b *big.Int) Bitmap {
return FromShortHexP(b.Text(16))
}
// Max returns a bitmap with all bits set to 1
func MaxP() Bitmap {
return FromHexP(strings.Repeat("f", NumBytes*2))
}
// Rand generates a cryptographically random bitmap with the confines of the parameters specified.
func Rand() Bitmap {
var id Bitmap
_, err := rand.Read(id[:])
if err != nil {
panic(err)
}
return id
}
// RandInRangeP generates a cryptographically random bitmap and while it is greater than the high threshold
// bitmap will subtract the diff between high and low until it is no longer greater that the high.
func RandInRangeP(low, high Bitmap) Bitmap {
diff := high.Sub(low)
r := Rand()
for r.Cmp(diff) > 0 {
r = r.Sub(diff)
}
//ToDo - Adding the low at this point doesn't gurantee it will be within the range. Consider bitmaps as numbers and
// I have a range of 50-100. If get to say 60, and add 50, I would be at 110. Should protect against this?
return r.Add(low)
}
func getBit(b []byte, n int) bool {
i := n / 8
j := n % 8
return b[i]&(1<<uint(7-j)) > 0
}
func setBit(b []byte, n int, one bool) {
i := n / 8
j := n % 8
if one {
b[i] |= 1 << uint(7-j)
} else {
b[i] &= ^(1 << uint(7-j))
}
}
// CLosest returns the closest bitmap to target. if no bitmaps are provided, target itself is returned
func Closest(target Bitmap, bitmaps ...Bitmap) Bitmap {
if len(bitmaps) == 0 {
return target
}
var closest *Bitmap
for _, b := range bitmaps {
if closest == nil || target.Closer(b, *closest) {
closest = &b
}
}
return *closest
}
-386
View File
@@ -1,386 +0,0 @@
package bits
import (
"fmt"
"testing"
"github.com/lyoshenka/bencode"
)
func TestBitmap(t *testing.T) {
a := Bitmap{
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35,
36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,
}
b := Bitmap{
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35,
36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 47, 46,
}
c := Bitmap{
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1,
}
if !a.Equals(a) {
t.Error("bitmap does not equal itself")
}
if a.Equals(b) {
t.Error("bitmap equals another bitmap with different id")
}
if !a.Xor(b).Equals(c) {
t.Error(a.Xor(b))
}
if c.PrefixLen() != 375 {
t.Error(c.PrefixLen())
}
if b.Cmp(a) < 0 {
t.Error("bitmap fails Cmp test")
}
if a.Closer(c, b) || !a.Closer(b, c) || c.Closer(a, b) || c.Closer(b, c) {
t.Error("bitmap fails Closer test")
}
id := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
if FromHexP(id).Hex() != id {
t.Error(FromHexP(id).Hex())
}
}
func TestBitmap_GetBit(t *testing.T) {
tt := []struct {
bit int
expected bool
panic bool
}{
{bit: 383, expected: false, panic: false},
{bit: 382, expected: true, panic: false},
{bit: 381, expected: false, panic: false},
{bit: 380, expected: true, panic: false},
}
b := FromShortHexP("a")
for _, test := range tt {
actual := getBit(b[:], test.bit)
if test.expected != actual {
t.Errorf("getting bit %d of %s: expected %t, got %t", test.bit, b.HexSimplified(), test.expected, actual)
}
}
}
func TestBitmap_SetBit(t *testing.T) {
tt := []struct {
hex string
bit int
one bool
expected string
panic bool
}{
{hex: "0", bit: 383, one: true, expected: "1", panic: false},
{hex: "0", bit: 382, one: true, expected: "2", panic: false},
{hex: "0", bit: 381, one: true, expected: "4", panic: false},
{hex: "0", bit: 385, one: true, expected: "1", panic: true},
{hex: "0", bit: 384, one: true, expected: "1", panic: true},
}
for _, test := range tt {
expected := FromShortHexP(test.expected)
actual := FromShortHexP(test.hex)
if test.panic {
assertPanic(t, fmt.Sprintf("setting bit %d to %t", test.bit, test.one), func() { setBit(actual[:], test.bit, test.one) })
} else {
setBit(actual[:], test.bit, test.one)
if !expected.Equals(actual) {
t.Errorf("setting bit %d to %t: expected %s, got %s", test.bit, test.one, test.expected, actual.HexSimplified())
}
}
}
}
func TestBitmap_FromHexShort(t *testing.T) {
tt := []struct {
short string
long string
}{
{short: "", long: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
{short: "0", long: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
{short: "00000", long: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
{short: "9473745bc", long: "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000009473745bc"},
{short: "09473745bc", long: "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000009473745bc"},
{short: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
long: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"},
}
for _, test := range tt {
short := FromShortHexP(test.short)
long := FromHexP(test.long)
if !short.Equals(long) {
t.Errorf("short hex %s: expected %s, got %s", test.short, long.Hex(), short.Hex())
}
}
}
func TestBitmapMarshal(t *testing.T) {
b := FromStringP("123456789012345678901234567890123456789012345678")
encoded, err := bencode.EncodeBytes(b)
if err != nil {
t.Error(err)
}
if string(encoded) != "48:123456789012345678901234567890123456789012345678" {
t.Error("encoding does not match expected")
}
}
func TestBitmapMarshalEmbedded(t *testing.T) {
e := struct {
A string
B Bitmap
C int
}{
A: "1",
B: FromStringP("222222222222222222222222222222222222222222222222"),
C: 3,
}
encoded, err := bencode.EncodeBytes(e)
if err != nil {
t.Error(err)
}
if string(encoded) != "d1:A1:11:B48:2222222222222222222222222222222222222222222222221:Ci3ee" {
t.Error("encoding does not match expected")
}
}
func TestBitmapMarshalEmbedded2(t *testing.T) {
encoded, err := bencode.EncodeBytes([]interface{}{
FromStringP("333333333333333333333333333333333333333333333333"),
})
if err != nil {
t.Error(err)
}
if string(encoded) != "l48:333333333333333333333333333333333333333333333333e" {
t.Error("encoding does not match expected")
}
}
func TestBitmap_PrefixLen(t *testing.T) {
tt := []struct {
hex string
len int
}{
{len: 0, hex: "F00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
{len: 0, hex: "800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
{len: 1, hex: "700000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
{len: 1, hex: "400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
{len: 384, hex: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
{len: 383, hex: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"},
{len: 382, hex: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002"},
{len: 382, hex: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003"},
}
for _, test := range tt {
len := FromHexP(test.hex).PrefixLen()
if len != test.len {
t.Errorf("got prefix len %d; expected %d for %s", len, test.len, test.hex)
}
}
}
func TestBitmap_Prefix(t *testing.T) {
allOne := FromHexP("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")
zerosTT := []struct {
zeros int
expected string
}{
{zeros: -123, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
{zeros: 0, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
{zeros: 1, expected: "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
{zeros: 69, expected: "000000000000000007ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
{zeros: 383, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"},
{zeros: 384, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
{zeros: 400, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
}
for _, test := range zerosTT {
expected := FromHexP(test.expected)
actual := allOne.Prefix(test.zeros, false)
if !actual.Equals(expected) {
t.Errorf("%d zeros: got %s; expected %s", test.zeros, actual.Hex(), expected.Hex())
}
}
for i := 0; i < NumBits; i++ {
b := allOne.Prefix(i, false)
if b.PrefixLen() != i {
t.Errorf("got prefix len %d; expected %d for %s", b.PrefixLen(), i, b.Hex())
}
}
allZero := FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
onesTT := []struct {
ones int
expected string
}{
{ones: -123, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
{ones: 0, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
{ones: 1, expected: "800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
{ones: 69, expected: "fffffffffffffffff8000000000000000000000000000000000000000000000000000000000000000000000000000000"},
{ones: 383, expected: "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe"},
{ones: 384, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
{ones: 400, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
}
for _, test := range onesTT {
expected := FromHexP(test.expected)
actual := allZero.Prefix(test.ones, true)
if !actual.Equals(expected) {
t.Errorf("%d ones: got %s; expected %s", test.ones, actual.Hex(), expected.Hex())
}
}
}
func TestBitmap_Suffix(t *testing.T) {
allOne := FromHexP("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")
zerosTT := []struct {
zeros int
expected string
}{
{zeros: -123, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
{zeros: 0, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
{zeros: 1, expected: "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe"},
{zeros: 69, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe00000000000000000"},
{zeros: 383, expected: "800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
{zeros: 384, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
{zeros: 400, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
}
for _, test := range zerosTT {
expected := FromHexP(test.expected)
actual := allOne.Suffix(test.zeros, false)
if !actual.Equals(expected) {
t.Errorf("%d zeros: got %s; expected %s", test.zeros, actual.Hex(), expected.Hex())
}
}
for i := 0; i < NumBits; i++ {
b := allOne.Prefix(i, false)
if b.PrefixLen() != i {
t.Errorf("got prefix len %d; expected %d for %s", b.PrefixLen(), i, b.Hex())
}
}
allZero := FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
onesTT := []struct {
ones int
expected string
}{
{ones: -123, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
{ones: 0, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
{ones: 1, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"},
{ones: 69, expected: "0000000000000000000000000000000000000000000000000000000000000000000000000000001fffffffffffffffff"},
{ones: 383, expected: "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
{ones: 384, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
{ones: 400, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
}
for _, test := range onesTT {
expected := FromHexP(test.expected)
actual := allZero.Suffix(test.ones, true)
if !actual.Equals(expected) {
t.Errorf("%d ones: got %s; expected %s", test.ones, actual.Hex(), expected.Hex())
}
}
}
func TestBitmap_Add(t *testing.T) {
tt := []struct {
a, b, sum string
panic bool
}{
{"0", "0", "0", false},
{"0", "1", "1", false},
{"1", "0", "1", false},
{"1", "1", "2", false},
{"8", "4", "c", false},
{"1000", "0010", "1010", false},
{"1111", "1111", "2222", false},
{"ffff", "1", "10000", false},
{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", false},
{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "1", "", true},
}
for _, test := range tt {
a := FromShortHexP(test.a)
b := FromShortHexP(test.b)
expected := FromShortHexP(test.sum)
if test.panic {
assertPanic(t, fmt.Sprintf("adding %s and %s", test.a, test.b), func() { a.Add(b) })
} else {
actual := a.Add(b)
if !expected.Equals(actual) {
t.Errorf("adding %s and %s; expected %s, got %s", test.a, test.b, test.sum, actual.HexSimplified())
}
}
}
}
func TestBitmap_Sub(t *testing.T) {
tt := []struct {
a, b, sum string
panic bool
}{
{"0", "0", "0", false},
{"1", "0", "1", false},
{"1", "1", "0", false},
{"8", "4", "4", false},
{"f", "9", "6", false},
{"f", "e", "1", false},
{"10", "f", "1", false},
{"2222", "1111", "1111", false},
{"ffff", "1", "fffe", false},
{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", false},
{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0", false},
{"0", "1", "", true},
}
for _, test := range tt {
a := FromShortHexP(test.a)
b := FromShortHexP(test.b)
expected := FromShortHexP(test.sum)
if test.panic {
assertPanic(t, fmt.Sprintf("subtracting %s - %s", test.a, test.b), func() { a.Sub(b) })
} else {
actual := a.Sub(b)
if !expected.Equals(actual) {
t.Errorf("subtracting %s - %s; expected %s, got %s", test.a, test.b, test.sum, actual.HexSimplified())
}
}
}
}
func assertPanic(t *testing.T, text string, f func()) {
defer func() {
if r := recover(); r == nil {
t.Errorf("%s: did not panic as expected", text)
}
}()
f()
}
-65
View File
@@ -1,65 +0,0 @@
package bits
import (
"math/big"
"github.com/lbryio/errors.go"
)
// Range has a start and end
type Range struct {
Start Bitmap
End Bitmap
}
func MaxRange() Range {
return Range{
Start: Bitmap{},
End: MaxP(),
}
}
// IntervalP divides the range into `num` intervals and returns the `n`th one
// intervals are approximately the same size, but may not be exact because of rounding issues
// the first interval always starts at the beginning of the range, and the last interval always ends at the end
func (r Range) IntervalP(n, num int) Range {
if num < 1 || n < 1 || n > num {
panic(errors.Err("invalid interval %d of %d", n, num))
}
start := r.intervalStart(n, num)
end := r.End.Big()
if n < num {
end = r.intervalStart(n+1, num)
end.Sub(end, big.NewInt(1))
}
return Range{FromBigP(start), FromBigP(end)}
}
func (r Range) intervalStart(n, num int) *big.Int {
// formula:
// size = (end - start) / num
// rem = (end - start) % num
// intervalStart = rangeStart + (size * n-1) + ((rem * n-1) % num)
size := new(big.Int)
rem := new(big.Int)
size.Sub(r.End.Big(), r.Start.Big()).DivMod(size, big.NewInt(int64(num)), rem)
size.Mul(size, big.NewInt(int64(n-1)))
rem.Mul(rem, big.NewInt(int64(n-1))).Mod(rem, big.NewInt(int64(num)))
start := r.Start.Big()
start.Add(start, size).Add(start, rem)
return start
}
func (r Range) IntervalSize() *big.Int {
return (&big.Int{}).Sub(r.End.Big(), r.Start.Big())
}
func (r Range) Contains(b Bitmap) bool {
return r.Start.Cmp(b) <= 0 && r.End.Cmp(b) >= 0
}
-48
View File
@@ -1,48 +0,0 @@
package bits
import (
"math/big"
"testing"
)
func TestMaxRange(t *testing.T) {
start := FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
end := FromHexP("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")
r := MaxRange()
if !r.Start.Equals(start) {
t.Error("max range does not start at the beginning")
}
if !r.End.Equals(end) {
t.Error("max range does not end at the end")
}
}
func TestRange_IntervalP(t *testing.T) {
max := MaxRange()
numIntervals := 97
expectedAvg := (&big.Int{}).Div(max.IntervalSize(), big.NewInt(int64(numIntervals)))
maxDiff := big.NewInt(int64(numIntervals))
var lastEnd Bitmap
for i := 1; i <= numIntervals; i++ {
ival := max.IntervalP(i, numIntervals)
if i == 1 && !ival.Start.Equals(max.Start) {
t.Error("first interval does not start at 0")
}
if i == numIntervals && !ival.End.Equals(max.End) {
t.Error("last interval does not end at max")
}
if i > 1 && !ival.Start.Equals(lastEnd.Add(FromShortHexP("1"))) {
t.Errorf("interval %d of %d: last end was %s, this start is %s", i, numIntervals, lastEnd.Hex(), ival.Start.Hex())
}
if ival.IntervalSize().Cmp((&big.Int{}).Add(expectedAvg, maxDiff)) > 0 || ival.IntervalSize().Cmp((&big.Int{}).Sub(expectedAvg, maxDiff)) < 0 {
t.Errorf("interval %d of %d: interval size is outside the normal range", i, numIntervals)
}
lastEnd = ival.End
}
}
-212
View File
@@ -1,212 +0,0 @@
package dht
import (
"math/rand"
"net"
"sync"
"time"
"github.com/lbryio/reflector.go/dht/bits"
)
const (
bootstrapDefaultRefreshDuration = 15 * time.Minute
)
// BootstrapNode is a configured node setup for testing.
type BootstrapNode struct {
Node
initialPingInterval time.Duration
checkInterval time.Duration
nlock *sync.RWMutex
peers map[bits.Bitmap]*peer
nodeIDs []bits.Bitmap // necessary for efficient random ID selection
}
// NewBootstrapNode returns a BootstrapNode pointer.
func NewBootstrapNode(id bits.Bitmap, initialPingInterval, rePingInterval time.Duration) *BootstrapNode {
b := &BootstrapNode{
Node: *NewNode(id),
initialPingInterval: initialPingInterval,
checkInterval: rePingInterval,
nlock: &sync.RWMutex{},
peers: make(map[bits.Bitmap]*peer),
nodeIDs: make([]bits.Bitmap, 0),
}
b.requestHandler = b.handleRequest
return b
}
// Add manually adds a contact
func (b *BootstrapNode) Add(c Contact) {
b.upsert(c)
}
// Connect connects to the given connection and starts any background threads necessary
func (b *BootstrapNode) Connect(conn UDPConn) error {
err := b.Node.Connect(conn)
if err != nil {
return err
}
log.Infof("[%s] bootstrap: node connected", b.id.HexShort())
go func() {
t := time.NewTicker(b.checkInterval / 5)
for {
select {
case <-t.C:
b.check()
case <-b.grp.Ch():
return
}
}
}()
return nil
}
// upsert adds the contact to the list, or updates the lastPinged time
func (b *BootstrapNode) upsert(c Contact) {
b.nlock.Lock()
defer b.nlock.Unlock()
if peer, exists := b.peers[c.ID]; exists {
log.Debugf("[%s] bootstrap: touching contact %s", b.id.HexShort(), peer.Contact.ID.HexShort())
peer.Touch()
return
}
log.Debugf("[%s] bootstrap: adding new contact %s", b.id.HexShort(), c.ID.HexShort())
b.peers[c.ID] = &peer{c, b.id.Xor(c.ID), time.Now(), 0}
b.nodeIDs = append(b.nodeIDs, c.ID)
}
// remove removes the contact from the list
func (b *BootstrapNode) remove(c Contact) {
b.nlock.Lock()
defer b.nlock.Unlock()
_, exists := b.peers[c.ID]
if !exists {
return
}
log.Debugf("[%s] bootstrap: removing contact %s", b.id.HexShort(), c.ID.HexShort())
delete(b.peers, c.ID)
for i := range b.nodeIDs {
if b.nodeIDs[i].Equals(c.ID) {
b.nodeIDs = append(b.nodeIDs[:i], b.nodeIDs[i+1:]...)
break
}
}
}
// get returns up to `limit` random contacts from the list
func (b *BootstrapNode) get(limit int) []Contact {
b.nlock.RLock()
defer b.nlock.RUnlock()
if len(b.peers) < limit {
limit = len(b.peers)
}
ret := make([]Contact, limit)
for i, k := range randKeys(len(b.nodeIDs))[:limit] {
ret[i] = b.peers[b.nodeIDs[k]].Contact
}
return ret
}
// ping pings a node. if the node responds, it is added to the list. otherwise, it is removed
func (b *BootstrapNode) ping(c Contact) {
log.Debugf("[%s] bootstrap: pinging %s", b.id.HexShort(), c.ID.HexShort())
b.grp.Add(1)
defer b.grp.Done()
resCh := b.SendAsync(c, Request{Method: pingMethod})
var res *Response
select {
case res = <-resCh:
case <-b.grp.Ch():
return
}
if res != nil && res.Data == pingSuccessResponse {
b.upsert(c)
} else {
b.remove(c)
}
}
func (b *BootstrapNode) check() {
b.nlock.RLock()
defer b.nlock.RUnlock()
for i := range b.peers {
if !b.peers[i].ActiveInLast(b.checkInterval) {
go b.ping(b.peers[i].Contact)
}
}
}
// handleRequest handles the requests received from udp.
func (b *BootstrapNode) handleRequest(addr *net.UDPAddr, request Request) {
switch request.Method {
case pingMethod:
err := b.sendMessage(addr, Response{ID: request.ID, NodeID: b.id, Data: pingSuccessResponse})
if err != nil {
log.Error("error sending response message - ", err)
}
case findNodeMethod:
if request.Arg == nil {
log.Errorln("request is missing arg")
return
}
err := b.sendMessage(addr, Response{
ID: request.ID,
NodeID: b.id,
Contacts: b.get(bucketSize),
})
if err != nil {
log.Error("error sending 'findnodemethod' response message - ", err)
}
}
go func() {
b.nlock.RLock()
_, exists := b.peers[request.NodeID]
b.nlock.RUnlock()
if !exists {
log.Debugf("[%s] bootstrap: queuing %s to ping", b.id.HexShort(), request.NodeID.HexShort())
<-time.After(b.initialPingInterval)
b.nlock.RLock()
_, exists = b.peers[request.NodeID]
b.nlock.RUnlock()
if !exists {
b.ping(Contact{ID: request.NodeID, IP: addr.IP, Port: addr.Port})
}
}
}()
}
func randKeys(max int) []int {
keys := make([]int, max)
for k := range keys {
keys[k] = k
}
rand.Shuffle(max, func(i, j int) {
keys[i], keys[j] = keys[j], keys[i]
})
return keys
}
-24
View File
@@ -1,24 +0,0 @@
package dht
import (
"net"
"testing"
"github.com/lbryio/reflector.go/dht/bits"
)
func TestBootstrapPing(t *testing.T) {
b := NewBootstrapNode(bits.Rand(), 10, bootstrapDefaultRefreshDuration)
listener, err := net.ListenPacket(Network, "127.0.0.1:54320")
if err != nil {
panic(err)
}
err = b.Connect(listener.(*net.UDPConn))
if err != nil {
t.Error(err)
}
b.Shutdown()
}
-76
View File
@@ -1,76 +0,0 @@
package dht
import (
"strconv"
"time"
"github.com/lbryio/reflector.go/dht/bits"
peerproto "github.com/lbryio/reflector.go/peer"
)
const (
Network = "udp4"
DefaultPort = 4444
DefaultAnnounceRate = 10 // send at most this many announces per second
DefaultReannounceTime = 50 * time.Minute // should be a bit less than hash expiration time
// TODO: all these constants should be defaults, and should be used to set values in the standard Config. then the code should use values in the config
// TODO: alternatively, have a global Config for constants. at least that way tests can modify the values
alpha = 3 // this is the constant alpha in the spec
bucketSize = 8 // this is the constant k in the spec
nodeIDLength = bits.NumBytes // bytes. this is the constant B in the spec
messageIDLength = 20 // bytes.
udpRetry = 1
udpTimeout = 5 * time.Second
udpMaxMessageLength = 4096 // bytes. I think our longest message is ~676 bytes, so I rounded up to 1024
// scratch that. a findValue could return more than K results if a lot of nodes are storing that value, so we need more buffer
maxPeerFails = 3 // after this many failures, a peer is considered bad and will be removed from the routing table
//tExpire = 60 * time.Minute // the time after which a key/value pair expires; this is a time-to-live (TTL) from the original publication date
tRefresh = 1 * time.Hour // the time after which an otherwise unaccessed bucket must be refreshed
//tReplicate = 1 * time.Hour // the interval between Kademlia replication events, when a node is required to publish its entire database
//tNodeRefresh = 15 * time.Minute // the time after which a good node becomes questionable if it has not messaged us
compactNodeInfoLength = nodeIDLength + 6 // nodeID + 4 for IP + 2 for port
tokenSecretRotationInterval = 5 * time.Minute // how often the token-generating secret is rotated
)
// Config represents the configure of dht.
type Config struct {
// this node's address. format is `ip:port`
Address string
// the seed nodes through which we can join in dht network
SeedNodes []string
// the hex-encoded node id for this node. if string is empty, a random id will be generated
NodeID string
// print the state of the dht every X time
PrintState time.Duration
// the port that clients can use to download blobs using the LBRY peer protocol
PeerProtocolPort int
// if nonzero, an RPC server will listen to requests on this port and respond to them
RPCPort int
// the time after which the original publisher must reannounce a key/value pair
ReannounceTime time.Duration
// send at most this many announces per second
AnnounceRate int
// channel that will receive notifications about announcements
AnnounceNotificationCh chan announceNotification
}
// NewStandardConfig returns a Config pointer with default values.
func NewStandardConfig() *Config {
return &Config{
Address: "0.0.0.0:" + strconv.Itoa(DefaultPort),
SeedNodes: []string{
"lbrynet1.lbry.io:4444",
"lbrynet2.lbry.io:4444",
"lbrynet3.lbry.io:4444",
},
PeerProtocolPort: peerproto.DefaultPort,
ReannounceTime: DefaultReannounceTime,
AnnounceRate: DefaultAnnounceRate,
}
}
-118
View File
@@ -1,118 +0,0 @@
package dht
import (
"bytes"
"net"
"sort"
"strconv"
"github.com/lbryio/lbry.go/errors"
"github.com/lbryio/reflector.go/dht/bits"
"github.com/lyoshenka/bencode"
)
// TODO: if routing table is ever empty (aka the node is isolated), it should re-bootstrap
// Contact contains information for contacting another node on the network
type Contact struct {
ID bits.Bitmap
IP net.IP
Port int // the udp port used for the dht
PeerPort int // the tcp port a peer can be contacted on for blob requests
}
// Equals returns true if two contacts are the same.
func (c Contact) Equals(other Contact, checkID bool) bool {
return c.IP.Equal(other.IP) && c.Port == other.Port && (!checkID || c.ID == other.ID)
}
// Addr returns the address of the contact.
func (c Contact) Addr() *net.UDPAddr {
return &net.UDPAddr{IP: c.IP, Port: c.Port}
}
// String returns a short string representation of the contact
func (c Contact) String() string {
str := c.ID.HexShort() + "@" + c.Addr().String()
if c.PeerPort != 0 {
str += "(" + strconv.Itoa(c.PeerPort) + ")"
}
return str
}
// MarshalCompact returns a compact byteslice representation of the contact
// NOTE: The compact representation always uses the tcp PeerPort, not the udp Port. This is dumb, but that's how the python daemon does it
func (c Contact) MarshalCompact() ([]byte, error) {
if c.IP.To4() == nil {
return nil, errors.Err("ip not set")
}
if c.PeerPort < 0 || c.PeerPort > 65535 {
return nil, errors.Err("invalid port")
}
var buf bytes.Buffer
buf.Write(c.IP.To4())
buf.WriteByte(byte(c.PeerPort >> 8))
buf.WriteByte(byte(c.PeerPort))
buf.Write(c.ID[:])
if buf.Len() != compactNodeInfoLength {
return nil, errors.Err("i dont know how this happened")
}
return buf.Bytes(), nil
}
// UnmarshalCompact unmarshals the compact byteslice representation of a contact.
// NOTE: The compact representation always uses the tcp PeerPort, not the udp Port. This is dumb, but that's how the python daemon does it
func (c *Contact) UnmarshalCompact(b []byte) error {
if len(b) != compactNodeInfoLength {
return errors.Err("invalid compact length")
}
c.IP = net.IPv4(b[0], b[1], b[2], b[3]).To4()
c.PeerPort = int(uint16(b[5]) | uint16(b[4])<<8)
c.ID = bits.FromBytesP(b[6:])
return nil
}
// MarshalBencode returns the serialized byte slice representation of a contact.
func (c Contact) MarshalBencode() ([]byte, error) {
return bencode.EncodeBytes([]interface{}{c.ID, c.IP.String(), c.Port})
}
// UnmarshalBencode unmarshals the serialized byte slice into the appropriate fields of the contact.
func (c *Contact) UnmarshalBencode(b []byte) error {
var raw []bencode.RawMessage
err := bencode.DecodeBytes(b, &raw)
if err != nil {
return err
}
if len(raw) != 3 {
return errors.Err("contact must have 3 elements; got %d", len(raw))
}
err = bencode.DecodeBytes(raw[0], &c.ID)
if err != nil {
return err
}
var ipStr string
err = bencode.DecodeBytes(raw[1], &ipStr)
if err != nil {
return err
}
c.IP = net.ParseIP(ipStr).To4()
if c.IP == nil {
return errors.Err("invalid IP")
}
return bencode.DecodeBytes(raw[2], &c.Port)
}
func sortByDistance(contacts []Contact, target bits.Bitmap) {
sort.Slice(contacts, func(i, j int) bool {
return contacts[i].ID.Xor(target).Cmp(contacts[j].ID.Xor(target)) < 0
})
}
-31
View File
@@ -1,31 +0,0 @@
package dht
import (
"net"
"reflect"
"testing"
"github.com/lbryio/reflector.go/dht/bits"
)
func TestCompactEncoding(t *testing.T) {
c := Contact{
ID: bits.FromHexP("1c8aff71b99462464d9eeac639595ab99664be3482cb91a29d87467515c7d9158fe72aa1f1582dab07d8f8b5db277f41"),
IP: net.ParseIP("1.2.3.4"),
PeerPort: int(55<<8 + 66),
}
var compact []byte
compact, err := c.MarshalCompact()
if err != nil {
t.Fatal(err)
}
if len(compact) != compactNodeInfoLength {
t.Fatalf("got length of %d; expected %d", len(compact), compactNodeInfoLength)
}
if !reflect.DeepEqual(compact, append([]byte{1, 2, 3, 4, 55, 66}, c.ID[:]...)) {
t.Errorf("compact bytes not encoded correctly")
}
}
-232
View File
@@ -1,232 +0,0 @@
package dht
import (
"fmt"
"net"
"strings"
"time"
"github.com/lbryio/reflector.go/dht/bits"
"github.com/lbryio/lbry.go/errors"
"github.com/lbryio/lbry.go/stop"
"github.com/sirupsen/logrus"
"github.com/spf13/cast"
)
var log *logrus.Logger
func UseLogger(l *logrus.Logger) {
log = l
}
func init() {
log = logrus.StandardLogger()
//log.SetFormatter(&log.TextFormatter{ForceColors: true})
//log.SetLevel(log.DebugLevel)
}
// DHT represents a DHT node.
type DHT struct {
// config
conf *Config
// local contact
contact Contact
// node
node *Node
// stopGroup to shut down DHT
grp *stop.Group
// channel is closed when DHT joins network
joined chan struct{}
// cache for store tokens
tokenCache *tokenCache
// hashes that need to be put into the announce queue or removed from the queue
announceAddRemove chan queueEdit
}
// New returns a DHT pointer. If config is nil, then config will be set to the default config.
func New(config *Config) *DHT {
if config == nil {
config = NewStandardConfig()
}
d := &DHT{
conf: config,
grp: stop.New(),
joined: make(chan struct{}),
announceAddRemove: make(chan queueEdit),
}
return d
}
func (dht *DHT) connect(conn UDPConn) error {
contact, err := getContact(dht.conf.NodeID, dht.conf.Address)
if err != nil {
return err
}
dht.contact = contact
dht.node = NewNode(contact.ID)
dht.tokenCache = newTokenCache(dht.node, tokenSecretRotationInterval)
return dht.node.Connect(conn)
}
// Start starts the dht
func (dht *DHT) Start() error {
listener, err := net.ListenPacket(Network, dht.conf.Address)
if err != nil {
return errors.Err(err)
}
conn := listener.(*net.UDPConn)
err = dht.connect(conn)
if err != nil {
return err
}
dht.join()
log.Infof("[%s] DHT ready on %s (%d nodes found during join)",
dht.node.id.HexShort(), dht.contact.Addr().String(), dht.node.rt.Count())
dht.grp.Add(1)
go func() {
dht.runAnnouncer()
dht.grp.Done()
}()
if dht.conf.RPCPort > 0 {
dht.grp.Add(1)
go func() {
dht.runRPCServer(dht.conf.RPCPort)
dht.grp.Done()
}()
}
return nil
}
// join makes current node join the dht network.
func (dht *DHT) join() {
defer close(dht.joined) // if anyone's waiting for join to finish, they'll know its done
log.Infof("[%s] joining DHT network", dht.node.id.HexShort())
// ping nodes, which gets their real node IDs and adds them to the routing table
atLeastOneNodeResponded := false
for _, addr := range dht.conf.SeedNodes {
err := dht.Ping(addr)
if err != nil {
log.Error(errors.Prefix(fmt.Sprintf("[%s] join", dht.node.id.HexShort()), err))
} else {
atLeastOneNodeResponded = true
}
}
if !atLeastOneNodeResponded {
log.Errorf("[%s] join: no nodes responded to initial ping", dht.node.id.HexShort())
return
}
// now call iterativeFind on yourself
_, _, err := FindContacts(dht.node, dht.node.id, false, dht.grp.Child())
if err != nil {
log.Errorf("[%s] join: %s", dht.node.id.HexShort(), err.Error())
}
// TODO: after joining, refresh all buckets further away than our closest neighbor
// http://xlattice.sourceforge.net/components/protocol/kademlia/specs.html#join
}
// WaitUntilJoined blocks until the node joins the network.
func (dht *DHT) WaitUntilJoined() {
if dht.joined == nil {
panic("dht not initialized")
}
<-dht.joined
}
// Shutdown shuts down the dht
func (dht *DHT) Shutdown() {
log.Debugf("[%s] DHT shutting down", dht.node.id.HexShort())
dht.grp.StopAndWait()
dht.node.Shutdown()
log.Debugf("[%s] DHT stopped", dht.node.id.HexShort())
}
// Ping pings a given address, creates a temporary contact for sending a message, and returns an error if communication
// fails.
func (dht *DHT) Ping(addr string) error {
raddr, err := net.ResolveUDPAddr(Network, addr)
if err != nil {
return err
}
tmpNode := Contact{ID: bits.Rand(), IP: raddr.IP, Port: raddr.Port}
res := dht.node.Send(tmpNode, Request{Method: pingMethod}, SendOptions{skipIDCheck: true})
if res == nil {
return errors.Err("no response from node %s", addr)
}
return nil
}
// Get returns the list of nodes that have the blob for the given hash
func (dht *DHT) Get(hash bits.Bitmap) ([]Contact, error) {
contacts, found, err := FindContacts(dht.node, hash, true, dht.grp.Child())
if err != nil {
return nil, err
}
if found {
return contacts, nil
}
return nil, nil
}
// PrintState prints the current state of the DHT including address, nr outstanding transactions, stored hashes as well
// as current bucket information.
func (dht *DHT) PrintState() {
log.Printf("DHT node %s at %s", dht.contact.String(), time.Now().Format(time.RFC822Z))
log.Printf("Outstanding transactions: %d", dht.node.CountActiveTransactions())
log.Printf("Stored hashes: %d", dht.node.store.CountStoredHashes())
log.Printf("Buckets:")
for _, line := range strings.Split(dht.node.rt.BucketInfo(), "\n") {
log.Println(line)
}
}
func (dht DHT) ID() bits.Bitmap {
return dht.contact.ID
}
func getContact(nodeID, addr string) (Contact, error) {
var c Contact
if nodeID == "" {
c.ID = bits.Rand()
} else {
c.ID = bits.FromHexP(nodeID)
}
ip, port, err := net.SplitHostPort(addr)
if err != nil {
return c, errors.Err(err)
} else if ip == "" {
return c, errors.Err("address does not contain an IP")
} else if port == "" {
return c, errors.Err("address does not contain a port")
}
c.IP = net.ParseIP(ip)
if c.IP == nil {
return c, errors.Err("invalid ip")
}
c.Port, err = cast.ToIntE(port)
if err != nil {
return c, errors.Err(err)
}
return c, nil
}
-214
View File
@@ -1,214 +0,0 @@
package dht
import (
"container/ring"
"context"
"math"
"sync"
"time"
"github.com/lbryio/lbry.go/errors"
"github.com/lbryio/reflector.go/dht/bits"
"golang.org/x/time/rate"
)
type queueEdit struct {
hash bits.Bitmap
add bool
}
const (
announceStarted = "started"
announceFinishd = "finished"
)
type announceNotification struct {
hash bits.Bitmap
action string
err error
}
// Add adds the hash to the list of hashes this node is announcing
func (dht *DHT) Add(hash bits.Bitmap) {
dht.announceAddRemove <- queueEdit{hash: hash, add: true}
}
// Remove removes the hash from the list of hashes this node is announcing
func (dht *DHT) Remove(hash bits.Bitmap) {
dht.announceAddRemove <- queueEdit{hash: hash, add: false}
}
func (dht *DHT) runAnnouncer() {
type hashAndTime struct {
hash bits.Bitmap
lastAnnounce time.Time
}
var queue *ring.Ring
hashes := make(map[bits.Bitmap]*ring.Ring)
var announceNextHash <-chan time.Time
timer := time.NewTimer(math.MaxInt64)
timer.Stop()
limitCh := make(chan time.Time)
dht.grp.Add(1)
go func() {
defer dht.grp.Done()
limiter := rate.NewLimiter(rate.Limit(dht.conf.AnnounceRate), dht.conf.AnnounceRate)
for {
err := limiter.Wait(context.Background()) // TODO: should use grp.ctx somehow? so when grp is closed, wait returns
if err != nil {
log.Error(errors.Prefix("rate limiter", err))
continue
}
select {
case limitCh <- time.Now():
case <-dht.grp.Ch():
return
}
}
}()
maintenance := time.NewTicker(1 * time.Minute)
// TODO: work to space hash announces out so they aren't bunched up around the reannounce time. track time since last announce. if its been more than the ideal time (reannounce time / numhashes), start announcing hashes early
for {
select {
case <-dht.grp.Ch():
return
case <-maintenance.C:
maxAnnounce := dht.conf.AnnounceRate * int(dht.conf.ReannounceTime.Seconds())
if len(hashes) > maxAnnounce {
// TODO: send this to slack
log.Warnf("DHT has %d hashes, but can only announce %d hashes in the %s reannounce window. Raise the announce rate or spawn more nodes.",
len(hashes), maxAnnounce, dht.conf.ReannounceTime.String())
}
case change := <-dht.announceAddRemove:
if change.add {
if _, exists := hashes[change.hash]; exists {
continue
}
r := ring.New(1)
r.Value = hashAndTime{hash: change.hash}
if queue != nil {
queue.Prev().Link(r)
}
queue = r
hashes[change.hash] = r
announceNextHash = limitCh // announce next hash ASAP
} else {
r, exists := hashes[change.hash]
if !exists {
continue
}
delete(hashes, change.hash)
if len(hashes) == 0 {
queue = ring.New(0)
announceNextHash = nil // no hashes to announce, wait indefinitely
} else {
if r == queue {
queue = queue.Next() // don't lose our pointer
}
r.Prev().Link(r.Next())
}
}
case <-announceNextHash:
dht.grp.Add(1)
ht := queue.Value.(hashAndTime)
if !ht.lastAnnounce.IsZero() {
nextAnnounce := ht.lastAnnounce.Add(dht.conf.ReannounceTime)
if nextAnnounce.After(time.Now()) {
timer.Reset(time.Until(nextAnnounce))
announceNextHash = timer.C // wait until next hash should be announced
continue
}
}
if dht.conf.AnnounceNotificationCh != nil {
dht.conf.AnnounceNotificationCh <- announceNotification{
hash: ht.hash,
action: announceStarted,
}
}
go func(hash bits.Bitmap) {
defer dht.grp.Done()
err := dht.announce(hash)
if err != nil {
log.Error(errors.Prefix("announce", err))
}
if dht.conf.AnnounceNotificationCh != nil {
dht.conf.AnnounceNotificationCh <- announceNotification{
hash: ht.hash,
action: announceFinishd,
err: err,
}
}
}(ht.hash)
queue.Value = hashAndTime{hash: ht.hash, lastAnnounce: time.Now()}
queue = queue.Next()
announceNextHash = limitCh // announce next hash ASAP
}
}
}
// Announce announces to the DHT that this node has the blob for the given hash
func (dht *DHT) announce(hash bits.Bitmap) error {
contacts, _, err := FindContacts(dht.node, hash, false, dht.grp.Child())
if err != nil {
return err
}
// self-store if we found less than K contacts, or we're closer than the farthest contact
if len(contacts) < bucketSize {
contacts = append(contacts, dht.contact)
} else if hash.Closer(dht.node.id, contacts[bucketSize-1].ID) {
contacts[bucketSize-1] = dht.contact
}
wg := &sync.WaitGroup{}
for _, c := range contacts {
wg.Add(1)
go func(c Contact) {
dht.store(hash, c)
wg.Done()
}(c)
}
wg.Wait()
return nil
}
func (dht *DHT) store(hash bits.Bitmap, c Contact) {
if dht.contact.ID == c.ID {
// self-store
c.PeerPort = dht.conf.PeerProtocolPort
dht.node.Store(hash, c)
return
}
dht.node.SendAsync(c, Request{
Method: storeMethod,
StoreArgs: &storeArgs{
BlobHash: hash,
Value: storeArgsValue{
Token: dht.tokenCache.Get(c, hash, dht.grp.Ch()),
LbryID: dht.contact.ID,
Port: dht.conf.PeerProtocolPort,
},
},
})
}
-181
View File
@@ -1,181 +0,0 @@
package dht
import (
"net"
"sync"
"testing"
"time"
"github.com/lbryio/reflector.go/dht/bits"
)
func TestNodeFinder_FindNodes(t *testing.T) {
if testing.Short() {
t.Skip("skipping slow nodeFinder test")
}
bs, dhts := TestingCreateNetwork(t, 3, true, false)
defer func() {
for i := range dhts {
dhts[i].Shutdown()
}
bs.Shutdown()
}()
contacts, found, err := FindContacts(dhts[2].node, bits.Rand(), false, nil)
if err != nil {
t.Fatal(err)
}
if found {
t.Fatal("something was found, but it should not have been")
}
if len(contacts) != 3 {
t.Errorf("expected 3 node, found %d", len(contacts))
}
foundBootstrap := false
foundOne := false
foundTwo := false
for _, n := range contacts {
if n.ID.Equals(bs.id) {
foundBootstrap = true
}
if n.ID.Equals(dhts[0].node.id) {
foundOne = true
}
if n.ID.Equals(dhts[1].node.id) {
foundTwo = true
}
}
if !foundBootstrap {
t.Errorf("did not find bootstrap node %s", bs.id.Hex())
}
if !foundOne {
t.Errorf("did not find first node %s", dhts[0].node.id.Hex())
}
if !foundTwo {
t.Errorf("did not find second node %s", dhts[1].node.id.Hex())
}
}
func TestNodeFinder_FindNodes_NoBootstrap(t *testing.T) {
_, dhts := TestingCreateNetwork(t, 3, false, false)
defer func() {
for i := range dhts {
dhts[i].Shutdown()
}
}()
_, _, err := FindContacts(dhts[2].node, bits.Rand(), false, nil)
if err == nil {
t.Fatal("contact finder should have errored saying that there are no contacts in the routing table")
}
}
func TestNodeFinder_FindValue(t *testing.T) {
if testing.Short() {
t.Skip("skipping slow nodeFinder test")
}
bs, dhts := TestingCreateNetwork(t, 3, true, false)
defer func() {
for i := range dhts {
dhts[i].Shutdown()
}
bs.Shutdown()
}()
blobHashToFind := bits.Rand()
nodeToFind := Contact{ID: bits.Rand(), IP: net.IPv4(1, 2, 3, 4), Port: 5678}
dhts[0].node.store.Upsert(blobHashToFind, nodeToFind)
contacts, found, err := FindContacts(dhts[2].node, blobHashToFind, true, nil)
if err != nil {
t.Fatal(err)
}
if !found {
t.Fatal("node was not found")
}
if len(contacts) != 1 {
t.Fatalf("expected one node, found %d", len(contacts))
}
if !contacts[0].ID.Equals(nodeToFind.ID) {
t.Fatalf("found node id %s, expected %s", contacts[0].ID.Hex(), nodeToFind.ID.Hex())
}
}
func TestDHT_LargeDHT(t *testing.T) {
if testing.Short() {
t.Skip("skipping large DHT test")
}
nodes := 100
bs, dhts := TestingCreateNetwork(t, nodes, true, true)
defer func() {
for _, d := range dhts {
go d.Shutdown()
}
bs.Shutdown()
time.Sleep(1 * time.Second)
}()
wg := &sync.WaitGroup{}
ids := make([]bits.Bitmap, nodes)
for i := range ids {
ids[i] = bits.Rand()
wg.Add(1)
go func(index int) {
defer wg.Done()
err := dhts[index].announce(ids[index])
if err != nil {
t.Error("error announcing random bitmap - ", err)
}
}(i)
}
wg.Wait()
// check that each node is in at learst 1 other routing table
rtCounts := make(map[bits.Bitmap]int)
for _, d := range dhts {
for _, d2 := range dhts {
if d.node.id.Equals(d2.node.id) {
continue
}
c := d2.node.rt.GetClosest(d.node.id, 1)
if len(c) > 1 {
t.Error("rt returned more than one node when only one requested")
} else if len(c) == 1 && c[0].ID.Equals(d.node.id) {
rtCounts[d.node.id]++
}
}
}
for k, v := range rtCounts {
if v == 0 {
t.Errorf("%s was not in any routing tables", k.HexShort())
}
}
// check that each ID is stored by at least 3 nodes
storeCounts := make(map[bits.Bitmap]int)
for _, d := range dhts {
for _, id := range ids {
if len(d.node.store.Get(id)) > 0 {
storeCounts[id]++
}
}
}
for k, v := range storeCounts {
if v == 0 {
t.Errorf("%s was not stored by any nodes", k.HexShort())
}
}
}
File diff suppressed because it is too large. Load diff
-463
View File
@@ -1,463 +0,0 @@
package dht
import (
"crypto/rand"
"encoding/hex"
"reflect"
"strconv"
"strings"
"github.com/lbryio/lbry.go/errors"
"github.com/lbryio/reflector.go/dht/bits"
"github.com/lyoshenka/bencode"
"github.com/spf13/cast"
)
const (
pingMethod = "ping"
storeMethod = "store"
findNodeMethod = "findNode"
findValueMethod = "findValue"
)
const (
pingSuccessResponse = "pong"
storeSuccessResponse = "OK"
)
const (
requestType = 0
responseType = 1
errorType = 2
)
const (
// these are strings because bencode requires bytestring keys
headerTypeField = "0"
headerMessageIDField = "1" // message id is 20 bytes long
headerNodeIDField = "2" // node id is 48 bytes long
headerPayloadField = "3"
headerArgsField = "4"
contactsField = "contacts"
tokenField = "token"
protocolVersionField = "protocolVersion"
)
// Message is a DHT message
type Message interface {
bencode.Marshaler
}
type messageID [messageIDLength]byte
// HexShort returns the first 8 hex characters of the hex encoded message id.
func (m messageID) HexShort() string {
return hex.EncodeToString(m[:])[:8]
}
// UnmarshalBencode takes a byte slice and unmarshals the message id.
func (m *messageID) UnmarshalBencode(encoded []byte) error {
var str string
err := bencode.DecodeBytes(encoded, &str)
if err != nil {
return err
}
copy(m[:], str)
return nil
}
// MarshallBencode returns the encoded byte slice of the message id.
func (m messageID) MarshalBencode() ([]byte, error) {
str := string(m[:])
return bencode.EncodeBytes(str)
}
func newMessageID() messageID {
var m messageID
_, err := rand.Read(m[:])
if err != nil {
panic(err)
}
return m
}
// Request represents a DHT request message
type Request struct {
ID messageID
NodeID bits.Bitmap
Method string
Arg *bits.Bitmap
StoreArgs *storeArgs
ProtocolVersion int
}
// MarshalBencode returns the serialized byte slice representation of the request
func (r Request) MarshalBencode() ([]byte, error) {
var args interface{}
if r.StoreArgs != nil {
args = r.StoreArgs
} else if r.Arg != nil {
args = []bits.Bitmap{*r.Arg}
} else {
args = []string{} // request must always have keys 0-4, so we use an empty list for PING
}
return bencode.EncodeBytes(map[string]interface{}{
headerTypeField: requestType,
headerMessageIDField: r.ID,
headerNodeIDField: r.NodeID,
headerPayloadField: r.Method,
headerArgsField: args,
})
}
// UnmarshalBencode unmarshals the serialized byte slice into the appropriate fields of the request.
func (r *Request) UnmarshalBencode(b []byte) error {
var raw struct {
ID messageID `bencode:"1"`
NodeID bits.Bitmap `bencode:"2"`
Method string `bencode:"3"`
Args bencode.RawMessage `bencode:"4"`
}
err := bencode.DecodeBytes(b, &raw)
if err != nil {
return errors.Prefix("request unmarshal", err)
}
r.ID = raw.ID
r.NodeID = raw.NodeID
r.Method = raw.Method
if r.Method == storeMethod {
r.StoreArgs = &storeArgs{} // bencode wont find the unmarshaler on a null pointer. need to fix it.
err = bencode.DecodeBytes(raw.Args, &r.StoreArgs)
if err != nil {
return errors.Prefix("request unmarshal", err)
}
} else if len(raw.Args) > 2 { // 2 because an empty list is `le`
r.Arg, r.ProtocolVersion, err = processArgsAndProtoVersion(raw.Args)
if err != nil {
return errors.Prefix("request unmarshal", err)
}
}
return nil
}
func processArgsAndProtoVersion(raw bencode.RawMessage) (arg *bits.Bitmap, version int, err error) {
var args []bencode.RawMessage
err = bencode.DecodeBytes(raw, &args)
if err != nil {
return nil, 0, err
}
if len(args) == 0 {
return nil, 0, nil
}
var extras map[string]int
err = bencode.DecodeBytes(args[len(args)-1], &extras)
if err == nil {
if v, exists := extras[protocolVersionField]; exists {
version = v
args = args[:len(args)-1]
}
}
if len(args) > 0 {
var b bits.Bitmap
err = bencode.DecodeBytes(args[0], &b)
if err != nil {
return nil, 0, err
}
arg = &b
}
return arg, version, nil
}
func (r Request) argsDebug() string {
if r.StoreArgs != nil {
return r.StoreArgs.BlobHash.HexShort() + ", " + r.StoreArgs.Value.LbryID.HexShort() + ":" + strconv.Itoa(r.StoreArgs.Value.Port)
} else if r.Arg != nil {
return r.Arg.HexShort()
}
return ""
}
type storeArgsValue struct {
Token string `bencode:"token"`
LbryID bits.Bitmap `bencode:"lbryid"`
Port int `bencode:"port"`
}
type storeArgs struct {
BlobHash bits.Bitmap
Value storeArgsValue
NodeID bits.Bitmap // original publisher id? I think this is getting fixed in the new dht stuff
SelfStore bool // this is an int on the wire
}
// MarshalBencode returns the serialized byte slice representation of the storage arguments.
func (s storeArgs) MarshalBencode() ([]byte, error) {
encodedValue, err := bencode.EncodeString(s.Value)
if err != nil {
return nil, err
}
selfStoreStr := 0
if s.SelfStore {
selfStoreStr = 1
}
return bencode.EncodeBytes([]interface{}{
s.BlobHash,
bencode.RawMessage(encodedValue),
s.NodeID,
selfStoreStr,
})
}
// UnmarshalBencode unmarshals the serialized byte slice into the appropriate fields of the store arguments.
func (s *storeArgs) UnmarshalBencode(b []byte) error {
var argsInt []bencode.RawMessage
err := bencode.DecodeBytes(b, &argsInt)
if err != nil {
return errors.Prefix("storeArgs unmarshal", err)
}
if len(argsInt) != 4 {
return errors.Err("unexpected number of fields for store args. got " + cast.ToString(len(argsInt)))
}
err = bencode.DecodeBytes(argsInt[0], &s.BlobHash)
if err != nil {
return errors.Prefix("storeArgs unmarshal", err)
}
err = bencode.DecodeBytes(argsInt[1], &s.Value)
if err != nil {
return errors.Prefix("storeArgs unmarshal", err)
}
err = bencode.DecodeBytes(argsInt[2], &s.NodeID)
if err != nil {
return errors.Prefix("storeArgs unmarshal", err)
}
var selfStore int
err = bencode.DecodeBytes(argsInt[3], &selfStore)
if err != nil {
return errors.Prefix("storeArgs unmarshal", err)
}
if selfStore == 0 {
s.SelfStore = false
} else if selfStore == 1 {
s.SelfStore = true
} else {
return errors.Err("selfstore must be 1 or 0")
}
return nil
}
// Response represents a DHT response message
type Response struct {
ID messageID
NodeID bits.Bitmap
Data string
Contacts []Contact
FindValueKey string
Token string
ProtocolVersion int
}
func (r Response) argsDebug() string {
if r.Data != "" {
return r.Data
}
str := "contacts "
if r.FindValueKey != "" {
str = "value for " + hex.EncodeToString([]byte(r.FindValueKey))[:8] + " "
}
str += "|"
for _, c := range r.Contacts {
str += c.String() + ","
}
str = strings.TrimRight(str, ",") + "|"
if r.Token != "" {
str += " token: " + hex.EncodeToString([]byte(r.Token))[:8]
}
return str
}
// MarshalBencode returns the serialized byte slice representation of the response.
func (r Response) MarshalBencode() ([]byte, error) {
data := map[string]interface{}{
headerTypeField: responseType,
headerMessageIDField: r.ID,
headerNodeIDField: r.NodeID,
}
if r.Data != "" {
// ping or store
data[headerPayloadField] = r.Data
} else if r.FindValueKey != "" {
// findValue success
if r.Token == "" {
return nil, errors.Err("response to findValue must have a token")
}
var contacts [][]byte
for _, c := range r.Contacts {
compact, err := c.MarshalCompact()
if err != nil {
return nil, err
}
contacts = append(contacts, compact)
}
data[headerPayloadField] = map[string]interface{}{
r.FindValueKey: contacts,
tokenField: r.Token,
}
} else if r.Token != "" {
// findValue failure falling back to findNode
data[headerPayloadField] = map[string]interface{}{
contactsField: r.Contacts,
tokenField: r.Token,
}
} else {
// straight up findNode
data[headerPayloadField] = r.Contacts
}
return bencode.EncodeBytes(data)
}
// UnmarshalBencode unmarshals the serialized byte slice into the appropriate fields of the store arguments.
func (r *Response) UnmarshalBencode(b []byte) error {
var raw struct {
ID messageID `bencode:"1"`
NodeID bits.Bitmap `bencode:"2"`
Data bencode.RawMessage `bencode:"3"`
}
err := bencode.DecodeBytes(b, &raw)
if err != nil {
return err
}
r.ID = raw.ID
r.NodeID = raw.NodeID
// maybe data is a string (response to ping or store)?
err = bencode.DecodeBytes(raw.Data, &r.Data)
if err == nil {
return nil
}
// maybe data is a list of contacts (response to findNode)?
err = bencode.DecodeBytes(raw.Data, &r.Contacts)
if err == nil {
return nil
}
// it must be a response to findValue
var rawData map[string]bencode.RawMessage
err = bencode.DecodeBytes(raw.Data, &rawData)
if err != nil {
return err
}
if token, ok := rawData[tokenField]; ok {
err = bencode.DecodeBytes(token, &r.Token)
if err != nil {
return err
}
delete(rawData, tokenField) // so it doesnt mess up findValue key finding below
}
if protocolVersion, ok := rawData[protocolVersionField]; ok {
err = bencode.DecodeBytes(protocolVersion, &r.ProtocolVersion)
if err != nil {
return err
}
delete(rawData, protocolVersionField) // so it doesnt mess up findValue key finding below
}
if contacts, ok := rawData[contactsField]; ok {
err = bencode.DecodeBytes(contacts, &r.Contacts)
if err != nil {
return err
}
} else {
for k, v := range rawData {
r.FindValueKey = k
var compactContacts [][]byte
err = bencode.DecodeBytes(v, &compactContacts)
if err != nil {
return err
}
for _, compact := range compactContacts {
var c Contact
err = c.UnmarshalCompact(compact)
if err != nil {
return err
}
r.Contacts = append(r.Contacts, c)
}
break
}
}
return nil
}
// Error represents a DHT error response
type Error struct {
ID messageID
NodeID bits.Bitmap
ExceptionType string
Response []string
}
// MarshalBencode returns the serialized byte slice representation of an error message.
func (e Error) MarshalBencode() ([]byte, error) {
return bencode.EncodeBytes(map[string]interface{}{
headerTypeField: errorType,
headerMessageIDField: e.ID,
headerNodeIDField: e.NodeID,
headerPayloadField: e.ExceptionType,
headerArgsField: e.Response,
})
}
// UnmarshalBencode unmarshals the serialized byte slice into the appropriate fields of the error message.
func (e *Error) UnmarshalBencode(b []byte) error {
var raw struct {
ID messageID `bencode:"1"`
NodeID bits.Bitmap `bencode:"2"`
ExceptionType string `bencode:"3"`
Args interface{} `bencode:"4"`
}
err := bencode.DecodeBytes(b, &raw)
if err != nil {
return err
}
e.ID = raw.ID
e.NodeID = raw.NodeID
e.ExceptionType = raw.ExceptionType
if reflect.TypeOf(raw.Args).Kind() == reflect.Slice {
v := reflect.ValueOf(raw.Args)
for i := 0; i < v.Len(); i++ {
e.Response = append(e.Response, cast.ToString(v.Index(i).Interface()))
}
}
return nil
}
-223
View File
@@ -1,223 +0,0 @@
package dht
import (
"encoding/hex"
"net"
"reflect"
"strconv"
"strings"
"testing"
"github.com/davecgh/go-spew/spew"
"github.com/lbryio/reflector.go/dht/bits"
"github.com/lyoshenka/bencode"
)
func TestBencodeDecodeStoreArgs(t *testing.T) {
blobHash := "3214D6C2F77FCB5E8D5FC07EDAFBA614F031CE8B2EAB49F924F8143F6DFBADE048D918710072FB98AB1B52B58F4E1468"
lbryID := "7CE1B831DEC8689E44F80F547D2DEA171F6A625E1A4FF6C6165E645F953103DABEB068A622203F859C6C64658FD3AA3B"
port := hex.EncodeToString([]byte("3333"))
token := "17C2D8E1E48EF21567FE4AD5C8ED944B798D3B65AB58D0C9122AD6587D1B5FED472EA2CB12284CEFA1C21EFF302322BD"
nodeID := "7CE1B831DEC8689E44F80F547D2DEA171F6A625E1A4FF6C6165E645F953103DABEB068A622203F859C6C64658FD3AA3B"
selfStore := hex.EncodeToString([]byte("1"))
raw := "6C" + // start args list
"3438 3A " + blobHash + // blob hash
"64" + // start value dict
"363A6C6272796964 3438 3A " + lbryID + // lbry id
"343A706F7274 69 " + port + " 65" + // port
"353A746F6B656E 3438 3A " + token + // token
"65" + // end value dict
"3438 3A " + nodeID + // node id
"69 " + selfStore + " 65" + // self store (integer)
"65" // end args list
raw = strings.ToLower(strings.Replace(raw, " ", "", -1))
data, err := hex.DecodeString(raw)
if err != nil {
t.Error(err)
return
}
storeArgs := &storeArgs{}
err = bencode.DecodeBytes(data, storeArgs)
if err != nil {
t.Error(err)
}
if storeArgs.BlobHash.Hex() != strings.ToLower(blobHash) {
t.Error("blob hash mismatch")
}
if storeArgs.Value.LbryID.Hex() != strings.ToLower(lbryID) {
t.Error("lbryid mismatch")
}
if hex.EncodeToString([]byte(strconv.Itoa(storeArgs.Value.Port))) != port {
t.Error("port mismatch")
}
if hex.EncodeToString([]byte(storeArgs.Value.Token)) != strings.ToLower(token) {
t.Error("token mismatch")
}
if storeArgs.NodeID.Hex() != strings.ToLower(nodeID) {
t.Error("node id mismatch")
}
if !storeArgs.SelfStore {
t.Error("selfStore mismatch")
}
reencoded, err := bencode.EncodeBytes(storeArgs)
if err != nil {
t.Error(err)
} else if !reflect.DeepEqual(reencoded, data) {
t.Error("reencoded data does not match original")
spew.Dump(reencoded, data)
}
}
func TestBencodeFindNodesResponse(t *testing.T) {
res := Response{
ID: newMessageID(),
NodeID: bits.Rand(),
Contacts: []Contact{
{ID: bits.Rand(), IP: net.IPv4(1, 2, 3, 4).To4(), Port: 5678},
{ID: bits.Rand(), IP: net.IPv4(4, 3, 2, 1).To4(), Port: 8765},
},
}
encoded, err := bencode.EncodeBytes(res)
if err != nil {
t.Fatal(err)
}
var res2 Response
err = bencode.DecodeBytes(encoded, &res2)
if err != nil {
t.Fatal(err)
}
compareResponses(t, res, res2)
}
func TestBencodeFindValueResponse(t *testing.T) {
res := Response{
ID: newMessageID(),
NodeID: bits.Rand(),
FindValueKey: bits.Rand().RawString(),
Token: "arstarstarst",
Contacts: []Contact{
{ID: bits.Rand(), IP: net.IPv4(1, 2, 3, 4).To4(), PeerPort: 8765},
},
}
encoded, err := bencode.EncodeBytes(res)
if err != nil {
t.Fatal(err)
}
var res2 Response
err = bencode.DecodeBytes(encoded, &res2)
if err != nil {
t.Fatal(err)
}
compareResponses(t, res, res2)
}
func TestDecodeSomeStrings(t *testing.T) {
strs := []string{
"6469306569306569316532303a359ec440b8236dee0fb2500ebda9c4704ae6741469326534383ade269943996b4bef3ff41176668a0577f86aba7f1ea2996edd18f9c42430802c8085331345c5f0c44a7f352e2ba8ae59693365383a66696e644e6f64656934656c34383ade269943996b4bef3ff41176668a0577f86aba7f1ea2996edd18f9c42430802c8085331345c5f0c44a7f352e2ba8ae596565",
"64313a30693165313a3132303a31f27ec6174783c6f00e4b9ebccd49c6bdce300e313a3234383a7b3785cd962a56d4d4574262b1b63cc95d062c825b38f73602c7dc297c1e6b8259cdcdb5e7edef216d514e0c31ad8637313a3364353a746f6b656e34383a071fdf9835995889920e9beb876ebe8e36e5d44f5822684b549457405e22a5bf3fe40335743b459f22c347c0e736eba434383a7bcc7f8804db7d0d40fcb9c508547e53ee4f7d4000758105ecce099fc931d9cccfbbf9a30829777afcf41f2b4b0aeb716c35343a627db2b20d05b109195e7d06a855c392fda9a05faa878e69b18cc1212c35371ba5cad6231e049a460cdc263e7f4701afad45b288f48435343a424434ae0d05bfd6e1b98cc921b441fa8b560ac29038c37ec4907ae023617bf2bbc54abd3403226daf479798edcce69743ea6699e6ff35343a4462ba020d059ed3d31ceb3d1ffb2c025abf065a14516a8942607f9b3fad4b697ccc232766bdf471b6518a3ed716f2712171fd046c4335343a47413e2b0d05bf6244e5edacf43f65ef4cfd4b6eac380244894cc364ec23e8ededfb085999a211e20f8de146d9e081bbb42dc61d9ca035343a627db2b20d05b109195e7d06a855c392fda9a05faa878e69b18cc1212c35371ba5cad6231e049a460cdc263e7f4701afad45b288f48435343a4462ba020d059ed3d31ceb3d1ffb2c025abf065a14516a8942607f9b3fad4b697ccc232766bdf471b6518a3ed716f2712171fd046c4335343a47413e2b0d05bf6244e5edacf43f65ef4cfd4b6eac380244894cc364ec23e8ededfb085999a211e20f8de146d9e081bbb42dc61d9ca035343a627db2b20d05b109195e7d06a855c392fda9a05faa878e69b18cc1212c35371ba5cad6231e049a460cdc263e7f4701afad45b288f48435343a68fe5c3e0d05bfd6e1b98cc921b441fa8b560ac29038c37ec4907ae023617bf2bbc54abd3403226daf479798edcce69743ea6699e6ff35343a4462ba020d059ed3d31ceb3d1ffb2c025abf065a14516a8942607f9b3fad4b697ccc232766bdf471b6518a3ed716f2712171fd046c4335343a47413e2b0d05bf6244e5edacf43f65ef4cfd4b6eac380244894cc364ec23e8ededfb085999a211e20f8de146d9e081bbb42dc61d9ca035343a67d9a7ea0d05223c42ef5979c7c8f2444311f7a8266d64d3f98e9d0b21726bc2714a6fea5426c9a46a68c49ea13b54ccc324adf0783035343a68fe5c3e0d05bfd6e1b98cc921b441fa8b560ac29038c37ec4907ae023617bf2bbc54abd3403226daf479798edcce69743ea6699e6ff35343a47413e2b0d05bf6244e5edacf43f65ef4cfd4b6eac380244894cc364ec23e8ededfb085999a211e20f8de146d9e081bbb42dc61d9ca035343a4462ba020d059ed3d31ceb3d1ffb2c025abf065a14516a8942607f9b3fad4b697ccc232766bdf471b6518a3ed716f2712171fd046c4335343a67d9a7ea0d05223c42ef5979c7c8f2444311f7a8266d64d3f98e9d0b21726bc2714a6fea5426c9a46a68c49ea13b54ccc324adf0783035343a424434ae0d05bfd6e1b98cc921b441fa8b560ac29038c37ec4907ae023617bf2bbc54abd3403226daf479798edcce69743ea6699e6ff35343a4ace39c80d0541ae66325db4f1fce530476942d339f3219913f0a237726e18746a65ff7b3c5558c82c274a6f1ed10bbaf7c1305ca8e635343a47413e2b0d05bf6244e5edacf43f65ef4cfd4b6eac380244894cc364ec23e8ededfb085999a211e20f8de146d9e081bbb42dc61d9ca035343a4462ba020d059ed3d31ceb3d1ffb2c025abf065a14516a8942607f9b3fad4b697ccc232766bdf471b6518a3ed716f2712171fd046c4335343a627db2b20d05b109195e7d06a855c392fda9a05faa878e69b18cc1212c35371ba5cad6231e049a460cdc263e7f4701afad45b288f48435343a424434ae0d05bfd6e1b98cc921b441fa8b560ac29038c37ec4907ae023617bf2bbc54abd3403226daf479798edcce69743ea6699e6ff35343a4ace39c80d0541ae66325db4f1fce530476942d339f3219913f0a237726e18746a65ff7b3c5558c82c274a6f1ed10bbaf7c1305ca8e635343a47413e2b0d05bf6244e5edacf43f65ef4cfd4b6eac380244894cc364ec23e8ededfb085999a211e20f8de146d9e081bbb42dc61d9ca035343a4462ba020d059ed3d31ceb3d1ffb2c025abf065a14516a8942607f9b3fad4b697ccc232766bdf471b6518a3ed716f2712171fd046c4335343a67d9a7ea0d05223c42ef5979c7c8f2444311f7a8266d64d3f98e9d0b21726bc2714a6fea5426c9a46a68c49ea13b54ccc324adf0783035343a4ace39c80d0541ae66325db4f1fce530476942d339f3219913f0a237726e18746a65ff7b3c5558c82c274a6f1ed10bbaf7c1305ca8e635343a47413e2b0d05bf6244e5edacf43f65ef4cfd4b6eac380244894cc364ec23e8ededfb085999a211e20f8de146d9e081bbb42dc61d9ca035343a4462ba020d059ed3d31ceb3d1ffb2c025abf065a14516a8942607f9b3fad4b697ccc232766bdf471b6518a3ed716f2712171fd046c4335343a67d9a7ea0d05223c42ef5979c7c8f2444311f7a8266d64d3f98e9d0b21726bc2714a6fea5426c9a46a68c49ea13b54ccc324adf0783035343a68fe5c3e0d05bfd6e1b98cc921b441fa8b560ac29038c37ec4907ae023617bf2bbc54abd3403226daf479798edcce69743ea6699e6ff35343a4ace39c80d0541ae66325db4f1fce530476942d339f3219913f0a237726e18746a65ff7b3c5558c82c274a6f1ed10bbaf7c1305ca8e635343a47413e2b0d05bf6244e5edacf43f65ef4cfd4b6eac380244894cc364ec23e8ededfb085999a211e20f8de146d9e081bbb42dc61d9ca035343a68fe5c3e0d05bfd6e1b98cc921b441fa8b560ac29038c37ec4907ae023617bf2bbc54abd3403226daf479798edcce69743ea6699e6ff35343a4ace39c80d0541ae66325db4f1fce530476942d339f3219913f0a237726e18746a65ff7b3c5558c82c274a6f1ed10bbaf7c1305ca8e635343a47413e2b0d05bf6244e5edacf43f65ef4cfd4b6eac380244894cc364ec23e8ededfb085999a211e20f8de146d9e081bbb42dc61d9ca035343a4462ba020d059ed3d31ceb3d1ffb2c025abf065a14516a8942607f9b3fad4b697ccc232766bdf471b6518a3ed716f2712171fd046c4335343a67d9a7ea0d05223c42ef5979c7c8f2444311f7a8266d64d3f98e9d0b21726bc2714a6fea5426c9a46a68c49ea13b54ccc324adf0783035343a68fe5c3e0d05bfd6e1b98cc921b441fa8b560ac29038c37ec4907ae023617bf2bbc54abd3403226daf479798edcce69743ea6699e6ff35343a4ace39c80d0541ae66325db4f1fce530476942d339f3219913f0a237726e18746a65ff7b3c5558c82c274a6f1ed10bbaf7c1305ca8e635343a47413e2b0d05bf6244e5edacf43fLine truncated
"64313a30693165313a3132303aa19e41f1d887db705fbf0a58f120773069d6ff11313a3234383a6911c21c3b5ea6536f4fb170c87fdc8bf4201124c5fe5eeb5f0054ff48e899a1d6e089a30a12ba8683ebf79691d71439313a3364383a636f6e74616374736c6c34383a6dd3dedeec334bae70a3c5c1b58fef2d4a501af320a332cdd5db4b71945c0a90d11aa2013e8d0216258a643f42d9562631323a34362e32382e3230342e3738693434343465656c34383a7ec0edf204d940d21c17e1c979df1c94610711d7773ceda72a4aa925c5f7799bbb741caa8d6d877aa2776399f77c08a031333a36392e3131382e34332e313234693434343465656c34383a6ec15cd8b95718d010bdf9739fda1af9ded7ac8be7da5cced6dc7a08470c8109accc5bfb973e5c1dd711349c62a8a3c331323a34362e32382e3230342e3738693434343465656c34383a6d2603a690712d5a3d5addb228d31a622f798c64ab2d3576d2c3ab3c5a64cb863c05545f37aa384ce83003416ca5cea431323a34362e32382e3230342e3738693434343465656c34383a6c1d92202a7e74ec85dd4e3e1699303174c4b6460b171637bba0a1c068d72f389035e1ba8bbe70f6621c36d70d4d045031323a34362e32382e3230342e3738693434343465656c34383a6dac30fbe94c006515cffe4898bc534a105fd0403bfc3c8692c2751b7946949dbc9e0c9570477cf8f817d1efc9a0e02531343a31382e3232312e3139342e313933693434343465656c34383a6f2ee379c662ea9f6db86beed6023be300cd23b9f9c4610a195dcb0f258392ec77188854ea3a8c0fffe67844849eb6ca31333a32342e3131362e39302e323238693434343465656c34383a6c55b2dbb89010de9db88e0ad3510a79fe60983b394a9e66b0efe139a455973cdc4415beff82cbdfb63d98caad648ab031333a38382e3132392e3234352e33396934343434656565353a746f6b656e34383a51844b0d1e8a613c4ad783c03b323063d3a5ff063a640368d7754bcae277c22b2b06b46c3e5466c30acb773f77b686476565",
"64313a30693165313a3132303af693334af099a987a4b55af2727472b51c1990db313a3234383ab6928ff25778a7bbb5d258d3b3a06e26db1654f3d2efce8c26681d43f7237cdf2e359a4d309c4473d5d89ec99fb4f573313a3364353a746f6b656e34383a61a45a40080ea118361f919ebf09cdc9d2c63f476e7bc4d193463f52b91dc4b0db4ed1004e32d4fc273645421e58479034383abe463e24af015c0d1ba496b4b1911aa98c8b42119295bbcc5c36829a2e2ff8f149743cbbc0af2c39669cb0750fd778576c35343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e95656565",
"6469306569316569316532303a3e0929c88abb8fe1d718025efc7f4a3cd85de16269326534383a21b2e2d2996b4bef3ff41176668a0577f86aba7f1ea2996edd18f9c42430802c8085331345c5f0c44a7f352e2ba8ae596933656c6565",
}
for i, str := range strs {
raw, err := hex.DecodeString(str)
if err != nil {
t.Errorf("error hex-decoding string %d: %s", i, err)
continue
}
var decoded interface{}
err = bencode.DecodeBytes(raw, &decoded)
if err != nil {
t.Errorf("error bencode-decoding string %d: %s", i, err)
continue
}
//
//t.Error("TODO")
//continue
//
//spew.Dump(decoded)
}
}
func TestDecodeFindNodeResponseWithNoNodes(t *testing.T) {
raw, err := hex.DecodeString("6469306569316569316532303a3e0929c88abb8fe1d718025efc7f4a3cd85de16269326534383a21b2e2d2996b4bef3ff41176668a0577f86aba7f1ea2996edd18f9c42430802c8085331345c5f0c44a7f352e2ba8ae596933656c6565")
if err != nil {
t.Fatal(err)
}
response := Response{}
err = bencode.DecodeBytes(raw, &response)
//spew.Dump(response)
if err != nil {
t.Fatal(err)
}
}
func TestDecodeRequestWithProtocolVersion(t *testing.T) {
raw, err := hex.DecodeString("6469306569306569316532303a65e1b9afce87c44abc40b4bb466890d2797f0dd269326534383a1baf3dbba8746c7a739f35465e268ace823622c5e8ec1dd7d5d27af5795cfbc54f22a32fbd05d420f241266b3dd16831693365343a70696e676934656c6431353a70726f746f636f6c56657273696f6e693165656565")
if err != nil {
t.Fatal(err)
}
req := Request{}
err = bencode.DecodeBytes(raw, &req)
if err != nil {
t.Fatal(err)
}
if req.ProtocolVersion != 1 {
t.Error("protocol version was not detected correctly")
}
}
func TestDecodeResponseWithProtocolVersion(t *testing.T) {
raw, err := hex.DecodeString("6469306569316569316532303a2b96f5de8be1e86dc2332a50eb313c97848064db69326534383a8b8eb692658ea3d7e7828e80a3133d524c6f82aaff370efa759d5b87821035a32de06724cb099f01a819695f829dff7f6933656431353a70726f746f636f6c56657273696f6e693165353a746f6b656e34383af2368fe4fd06ede6631ad8b153f3b5d8db724f8d520f4291e992e206dd02a216fb7dfd9b81686f11b626e3840df65fb434383a89c5c3f9794b0b24a03406e3b74361edb9ae70828e4c133512fc75db0a2d312673cdd4e30eed37892a46692d2fe439f36c35343a12dd65af0d058cd7d10d122fbe2eb5ae31062b7480011be588f20cfe6807b1939c42eea639059fa6365bfccb56ef8c9e574f49ba035c656565")
if err != nil {
t.Fatal(err)
}
res := Response{}
err = bencode.DecodeBytes(raw, &res)
if err != nil {
t.Fatal(err)
}
if res.ProtocolVersion != 1 {
t.Error("protocol version was not detected correctly")
}
}
func compareResponses(t *testing.T, res, res2 Response) {
if res.ID != res2.ID {
t.Errorf("expected ID %s, got %s", res.ID, res2.ID)
}
if !res.NodeID.Equals(res2.NodeID) {
t.Errorf("expected NodeID %s, got %s", res.NodeID.Hex(), res2.NodeID.Hex())
}
if res.Data != res2.Data {
t.Errorf("expected Data %s, got %s", res.Data, res2.Data)
}
if res.FindValueKey != res2.FindValueKey {
t.Errorf("expected FindValueKey %s, got %s", res.FindValueKey, res2.FindValueKey)
}
if res.Token != res2.Token {
t.Errorf("expected Token %s, got %s", res.Token, res2.Token)
}
if !reflect.DeepEqual(res.Contacts, res2.Contacts) {
t.Errorf("expected FindNodeData %s, got %s", spew.Sdump(res.Contacts), spew.Sdump(res2.Contacts))
}
}
-474
View File
@@ -1,474 +0,0 @@
package dht
import (
"encoding/hex"
"net"
"strings"
"sync"
"time"
"github.com/lbryio/errors.go"
"github.com/lbryio/lbry.go/stop"
"github.com/lbryio/lbry.go/util"
"github.com/lbryio/reflector.go/dht/bits"
"github.com/davecgh/go-spew/spew"
"github.com/lyoshenka/bencode"
)
// packet represents the information receive from udp.
type packet struct {
data []byte
raddr *net.UDPAddr
}
// UDPConn allows using a mocked connection to test sending/receiving data
// TODO: stop mocking this and use the real thing
type UDPConn interface {
ReadFromUDP([]byte) (int, *net.UDPAddr, error)
WriteToUDP([]byte, *net.UDPAddr) (int, error)
SetReadDeadline(time.Time) error
SetWriteDeadline(time.Time) error
Close() error
}
// RequestHandlerFunc is exported handler for requests.
type RequestHandlerFunc func(addr *net.UDPAddr, request Request)
// Node is a type representation of a node on the network.
type Node struct {
// the node's id
id bits.Bitmap
// UDP connection for sending and receiving data
conn UDPConn
// true if we've closed the connection on purpose
connClosed bool
// token manager
tokens *tokenManager
// map of outstanding transactions + mutex
txLock *sync.RWMutex
transactions map[messageID]*transaction
// routing table
rt *routingTable
// data store
store *contactStore
// overrides for request handlers
requestHandler RequestHandlerFunc
// stop the node neatly and clean up after itself
grp *stop.Group
}
// NewNode returns an initialized Node's pointer.
func NewNode(id bits.Bitmap) *Node {
return &Node{
id: id,
rt: newRoutingTable(id),
store: newStore(),
txLock: &sync.RWMutex{},
transactions: make(map[messageID]*transaction),
grp: stop.New(),
tokens: &tokenManager{},
}
}
// Connect connects to the given connection and starts any background threads necessary
func (n *Node) Connect(conn UDPConn) error {
n.conn = conn
n.tokens.Start(tokenSecretRotationInterval)
go func() {
// stop tokens and close the connection when we're shutting down
<-n.grp.Ch()
n.tokens.Stop()
n.connClosed = true
err := n.conn.Close()
if err != nil {
log.Error("error closing node connection on shutdown - ", err)
}
}()
packets := make(chan packet)
n.grp.Add(1)
go func() {
defer n.grp.Done()
buf := make([]byte, udpMaxMessageLength)
for {
bytesRead, raddr, err := n.conn.ReadFromUDP(buf)
if err != nil {
if n.connClosed {
return
}
log.Errorf("udp read error: %v", err)
continue
} else if raddr == nil {
log.Errorf("udp read with no raddr")
continue
}
data := make([]byte, bytesRead)
copy(data, buf[:bytesRead]) // slices use the same underlying array, so we need a new one for each packet
select { // needs select here because packet consumer can quit and the packets channel gets filled up and blocks
case packets <- packet{data: data, raddr: raddr}:
case <-n.grp.Ch():
return
}
}
}()
n.grp.Add(1)
go func() {
defer n.grp.Done()
var pkt packet
for {
select {
case pkt = <-packets:
n.handlePacket(pkt)
case <-n.grp.Ch():
return
}
}
}()
// TODO: turn this back on when you're sure it works right
n.grp.Add(1)
go func() {
defer n.grp.Done()
n.startRoutingTableGrooming()
}()
return nil
}
// Shutdown shuts down the node
func (n *Node) Shutdown() {
log.Debugf("[%s] node shutting down", n.id.HexShort())
n.grp.StopAndWait()
log.Debugf("[%s] node stopped", n.id.HexShort())
}
// handlePacket handles packets received from udp.
func (n *Node) handlePacket(pkt packet) {
//log.Debugf("[%s] Received message from %s (%d bytes) %s", n.id.HexShort(), pkt.raddr.String(), len(pkt.data), hex.EncodeToString(pkt.data))
if !util.InSlice(string(pkt.data[0:5]), []string{"d1:0i", "di0ei"}) {
log.Errorf("[%s] data is not a well-formatted dict: (%d bytes) %s", n.id.HexShort(), len(pkt.data), hex.EncodeToString(pkt.data))
return
}
// the following is a bit of a hack, but it lets us avoid decoding every message twice
// it depends on the data being a dict with 0 as the first key (so it starts with "d1:0i") and the message type as the first value
// TODO: test this more thoroughly
switch pkt.data[5] {
case '0' + requestType:
request := Request{}
err := bencode.DecodeBytes(pkt.data, &request)
if err != nil {
log.Errorf("[%s] error decoding request from %s: %s: (%d bytes) %s", n.id.HexShort(), pkt.raddr.String(), err.Error(), len(pkt.data), hex.EncodeToString(pkt.data))
return
}
log.Debugf("[%s] query %s: received request from %s: %s(%s)", n.id.HexShort(), request.ID.HexShort(), request.NodeID.HexShort(), request.Method, request.argsDebug())
n.handleRequest(pkt.raddr, request)
case '0' + responseType:
response := Response{}
err := bencode.DecodeBytes(pkt.data, &response)
if err != nil {
log.Errorf("[%s] error decoding response from %s: %s: (%d bytes) %s", n.id.HexShort(), pkt.raddr.String(), err.Error(), len(pkt.data), hex.EncodeToString(pkt.data))
return
}
log.Debugf("[%s] query %s: received response from %s: %s", n.id.HexShort(), response.ID.HexShort(), response.NodeID.HexShort(), response.argsDebug())
n.handleResponse(pkt.raddr, response)
case '0' + errorType:
e := Error{}
err := bencode.DecodeBytes(pkt.data, &e)
if err != nil {
log.Errorf("[%s] error decoding error from %s: %s: (%d bytes) %s", n.id.HexShort(), pkt.raddr.String(), err.Error(), len(pkt.data), hex.EncodeToString(pkt.data))
return
}
log.Debugf("[%s] query %s: received error from %s: %s", n.id.HexShort(), e.ID.HexShort(), e.NodeID.HexShort(), e.ExceptionType)
n.handleError(pkt.raddr, e)
default:
log.Errorf("[%s] invalid message type: %s", n.id.HexShort(), string(pkt.data[5]))
return
}
}
// handleRequest handles the requests received from udp.
func (n *Node) handleRequest(addr *net.UDPAddr, request Request) {
if request.NodeID.Equals(n.id) {
log.Warn("ignoring self-request")
return
}
// if a handler is overridden, call it instead
if n.requestHandler != nil {
n.requestHandler(addr, request)
return
}
switch request.Method {
default:
//n.sendMessage(addr, Error{ID: request.ID, NodeID: n.id, ExceptionType: "invalid-request-method"})
log.Errorln("invalid request method")
return
case pingMethod:
err := n.sendMessage(addr, Response{ID: request.ID, NodeID: n.id, Data: pingSuccessResponse})
if err != nil {
log.Error("error sending 'pingmethod' response message - ", err)
}
case storeMethod:
// TODO: we should be sending the IP in the request, not just using the sender's IP
// TODO: should we be using StoreArgs.NodeID or StoreArgs.Value.LbryID ???
if n.tokens.Verify(request.StoreArgs.Value.Token, request.NodeID, addr) {
n.Store(request.StoreArgs.BlobHash, Contact{ID: request.StoreArgs.NodeID, IP: addr.IP, Port: addr.Port, PeerPort: request.StoreArgs.Value.Port})
err := n.sendMessage(addr, Response{ID: request.ID, NodeID: n.id, Data: storeSuccessResponse})
if err != nil {
log.Error("error sending 'storemethod' response message - ", err)
}
} else {
err := n.sendMessage(addr, Error{ID: request.ID, NodeID: n.id, ExceptionType: "invalid-token"})
if err != nil {
log.Error("error sending 'storemethod'response message for invalid-token - ", err)
}
}
case findNodeMethod:
if request.Arg == nil {
log.Errorln("request is missing arg")
return
}
err := n.sendMessage(addr, Response{
ID: request.ID,
NodeID: n.id,
Contacts: n.rt.GetClosest(*request.Arg, bucketSize),
})
if err != nil {
log.Error("error sending 'findnodemethod' response message - ", err)
}
case findValueMethod:
if request.Arg == nil {
log.Errorln("request is missing arg")
return
}
res := Response{
ID: request.ID,
NodeID: n.id,
Token: n.tokens.Get(request.NodeID, addr),
}
if contacts := n.store.Get(*request.Arg); len(contacts) > 0 {
res.FindValueKey = request.Arg.RawString()
res.Contacts = contacts
} else {
res.Contacts = n.rt.GetClosest(*request.Arg, bucketSize)
}
err := n.sendMessage(addr, res)
if err != nil {
log.Error("error sending 'findvaluemethod' response message - ", err)
}
}
// nodes that send us requests should not be inserted, only refreshed.
// the routing table must only contain "good" nodes, which are nodes that reply to our requests
// if a node is already good (aka in the table), its fine to refresh it
// http://www.bittorrent.org/beps/bep_0005.html#routing-table
n.rt.Fresh(Contact{ID: request.NodeID, IP: addr.IP, Port: addr.Port})
}
// handleResponse handles responses received from udp.
func (n *Node) handleResponse(addr *net.UDPAddr, response Response) {
tx := n.txFind(response.ID, Contact{ID: response.NodeID, IP: addr.IP, Port: addr.Port})
if tx != nil {
select {
case tx.res <- response:
default:
//log.Errorf("[%s] query %s: response received, but tx has no listener or multiple responses to the same tx", n.id.HexShort(), response.ID.HexShort())
}
}
n.rt.Update(Contact{ID: response.NodeID, IP: addr.IP, Port: addr.Port})
}
// handleError handles errors received from udp.
func (n *Node) handleError(addr *net.UDPAddr, e Error) {
spew.Dump(e)
n.rt.Fresh(Contact{ID: e.NodeID, IP: addr.IP, Port: addr.Port})
}
// send sends data to a udp address
func (n *Node) sendMessage(addr *net.UDPAddr, data Message) error {
encoded, err := bencode.EncodeBytes(data)
if err != nil {
return errors.Err(err)
}
if req, ok := data.(Request); ok {
log.Debugf("[%s] query %s: sending request to %s (%d bytes) %s(%s)",
n.id.HexShort(), req.ID.HexShort(), addr.String(), len(encoded), req.Method, req.argsDebug())
} else if res, ok := data.(Response); ok {
log.Debugf("[%s] query %s: sending response to %s (%d bytes) %s",
n.id.HexShort(), res.ID.HexShort(), addr.String(), len(encoded), res.argsDebug())
} else {
log.Debugf("[%s] (%d bytes) %s", n.id.HexShort(), len(encoded), spew.Sdump(data))
}
err = n.conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
if err != nil {
if n.connClosed {
return nil
}
log.Error("error setting write deadline - ", err)
}
_, err = n.conn.WriteToUDP(encoded, addr)
return errors.Err(err)
}
// transaction represents a single query to the dht. it stores the queried contact, the request, and the response channel
type transaction struct {
contact Contact
req Request
res chan Response
skipIDCheck bool
}
// insert adds a transaction to the manager.
func (n *Node) txInsert(tx *transaction) {
n.txLock.Lock()
defer n.txLock.Unlock()
n.transactions[tx.req.ID] = tx
}
// delete removes a transaction from the manager.
func (n *Node) txDelete(id messageID) {
n.txLock.Lock()
defer n.txLock.Unlock()
delete(n.transactions, id)
}
// Find finds a transaction for the given id and contact
func (n *Node) txFind(id messageID, c Contact) *transaction {
n.txLock.RLock()
defer n.txLock.RUnlock()
t, ok := n.transactions[id]
if !ok || !t.contact.Equals(c, !t.skipIDCheck) {
return nil
}
return t
}
// SendOptions controls the behavior of send calls
type SendOptions struct {
skipIDCheck bool
}
// SendAsync sends a transaction and returns a channel that will eventually contain the transaction response
// The response channel is closed when the transaction is completed or times out.
func (n *Node) SendAsync(contact Contact, req Request, options ...SendOptions) <-chan *Response {
ch := make(chan *Response, 1)
if contact.ID.Equals(n.id) {
log.Error("sending query to self")
close(ch)
return ch
}
go func() {
defer close(ch)
req.ID = newMessageID()
req.NodeID = n.id
tx := &transaction{
contact: contact,
req: req,
res: make(chan Response),
}
if len(options) > 0 && options[0].skipIDCheck {
tx.skipIDCheck = true
}
n.txInsert(tx)
defer n.txDelete(tx.req.ID)
for i := 0; i < udpRetry; i++ {
err := n.sendMessage(contact.Addr(), tx.req)
if err != nil {
if !strings.Contains(err.Error(), "use of closed network connection") { // this only happens on localhost. real UDP has no connections
log.Error("send error: ", err)
}
continue
}
select {
case res := <-tx.res:
ch <- &res
return
case <-n.grp.Ch():
return
case <-time.After(udpTimeout):
}
}
// notify routing table about a failure to respond
n.rt.Fail(tx.contact)
}()
return ch
}
// Send sends a transaction and blocks until the response is available. It returns a response, or nil
// if the transaction timed out.
func (n *Node) Send(contact Contact, req Request, options ...SendOptions) *Response {
return <-n.SendAsync(contact, req, options...)
}
// CountActiveTransactions returns the number of transactions in the manager
func (n *Node) CountActiveTransactions() int {
n.txLock.Lock()
defer n.txLock.Unlock()
return len(n.transactions)
}
func (n *Node) startRoutingTableGrooming() {
refreshTicker := time.NewTicker(tRefresh / 5) // how often to check for buckets that need to be refreshed
for {
select {
case <-refreshTicker.C:
RoutingTableRefresh(n, tRefresh, n.grp.Child())
case <-n.grp.Ch():
return
}
}
}
// Store stores a node contact in the node's contact store.
func (n *Node) Store(hash bits.Bitmap, c Contact) {
n.store.Upsert(hash, c)
}
//AddKnownNode adds a known-good node to the routing table
func (n *Node) AddKnownNode(c Contact) {
n.rt.Update(c)
}
-338
View File
@@ -1,338 +0,0 @@
package dht
import (
"sync"
"time"
"github.com/lbryio/lbry.go/crypto"
"github.com/lbryio/lbry.go/errors"
"github.com/lbryio/lbry.go/stop"
"github.com/lbryio/reflector.go/dht/bits"
"github.com/sirupsen/logrus"
"github.com/uber-go/atomic"
)
// TODO: iterativeFindValue may be stopping early. if it gets a response with one peer, it should keep going because other nodes may know about more peers that have that blob
// TODO: or, it should try a tcp handshake with peers as it finds them, to make sure they are still online and have the blob
var cfLog *logrus.Logger
func init() {
cfLog = logrus.StandardLogger()
}
func NodeFinderUseLogger(l *logrus.Logger) {
cfLog = l
}
type contactFinder struct {
findValue bool // true if we're using findValue
target bits.Bitmap
node *Node
grp *stop.Group
findValueMutex *sync.Mutex
findValueResult []Contact
activeContactsMutex *sync.Mutex
activeContacts []Contact
shortlistMutex *sync.Mutex
shortlist []Contact
shortlistAdded map[bits.Bitmap]bool
closestContactMutex *sync.RWMutex
closestContact *Contact
notGettingCloser *atomic.Bool
}
func FindContacts(node *Node, target bits.Bitmap, findValue bool, parentGrp *stop.Group) ([]Contact, bool, error) {
cf := &contactFinder{
node: node,
target: target,
findValue: findValue,
findValueMutex: &sync.Mutex{},
activeContactsMutex: &sync.Mutex{},
shortlistMutex: &sync.Mutex{},
shortlistAdded: make(map[bits.Bitmap]bool),
grp: stop.New(parentGrp),
closestContactMutex: &sync.RWMutex{},
notGettingCloser: atomic.NewBool(false),
}
return cf.Find()
}
func (cf *contactFinder) Stop() {
cf.grp.StopAndWait()
}
func (cf *contactFinder) Find() ([]Contact, bool, error) {
if cf.findValue {
cf.debug("starting iterativeFindValue")
} else {
cf.debug("starting iterativeFindNode")
}
cf.appendNewToShortlist(cf.node.rt.GetClosest(cf.target, alpha))
if len(cf.shortlist) == 0 {
return nil, false, errors.Err("[%s] find %s: no contacts in routing table", cf.node.id.HexShort(), cf.target.HexShort())
}
go cf.cycle(false)
timeout := 5 * time.Second
CycleLoop:
for {
select {
case <-time.After(timeout):
go cf.cycle(false)
case <-cf.grp.Ch():
break CycleLoop
}
}
// TODO: what to do if we have less than K active contacts, shortlist is empty, but we have other contacts in our routing table whom we have not contacted. prolly contact them
var contacts []Contact
var found bool
if cf.findValue && len(cf.findValueResult) > 0 {
contacts = cf.findValueResult
found = true
} else {
contacts = cf.activeContacts
if len(contacts) > bucketSize {
contacts = contacts[:bucketSize]
}
}
cf.Stop()
return contacts, found, nil
}
// cycle does a single cycle of sending alpha probes and checking results against closestNode
func (cf *contactFinder) cycle(bigCycle bool) {
cycleID := crypto.RandString(6)
if bigCycle {
cf.debug("LAUNCHING CYCLE %s, AND ITS A BIG CYCLE", cycleID)
} else {
cf.debug("LAUNCHING CYCLE %s", cycleID)
}
defer cf.debug("CYCLE %s DONE", cycleID)
cf.closestContactMutex.RLock()
closestContact := cf.closestContact
cf.closestContactMutex.RUnlock()
var wg sync.WaitGroup
ch := make(chan *Contact)
limit := alpha
if bigCycle {
limit = bucketSize
}
for i := 0; i < limit; i++ {
wg.Add(1)
go func() {
defer wg.Done()
ch <- cf.probe(cycleID)
}()
}
go func() {
wg.Wait()
close(ch)
}()
foundCloser := false
for {
c, more := <-ch
if !more {
break
}
if c != nil && (closestContact == nil || cf.target.Closer(c.ID, closestContact.ID)) {
if closestContact != nil {
cf.debug("|%s| best contact improved: %s -> %s", cycleID, closestContact.ID.HexShort(), c.ID.HexShort())
} else {
cf.debug("|%s| best contact starting at %s", cycleID, c.ID.HexShort())
}
foundCloser = true
closestContact = c
}
}
if cf.isSearchFinished() {
cf.grp.Stop()
return
}
if foundCloser {
cf.closestContactMutex.Lock()
// have to check again after locking in case other probes found a closer one in the meantime
if cf.closestContact == nil || cf.target.Closer(closestContact.ID, cf.closestContact.ID) {
cf.closestContact = closestContact
}
cf.closestContactMutex.Unlock()
go cf.cycle(false)
} else if !bigCycle {
cf.debug("|%s| no improvement, running big cycle", cycleID)
go cf.cycle(true)
} else {
// big cycle ran and there was no improvement, so we're done
cf.debug("|%s| big cycle ran, still no improvement", cycleID)
cf.notGettingCloser.Store(true)
}
}
// probe sends a single probe, updates the lists, and returns the closest contact it found
func (cf *contactFinder) probe(cycleID string) *Contact {
maybeContact := cf.popFromShortlist()
if maybeContact == nil {
cf.debug("|%s| no contacts in shortlist, returning", cycleID)
return nil
}
c := *maybeContact
if c.ID.Equals(cf.node.id) {
return nil
}
cf.debug("|%s| probe %s: launching", cycleID, c.ID.HexShort())
req := Request{Arg: &cf.target}
if cf.findValue {
req.Method = findValueMethod
} else {
req.Method = findNodeMethod
}
var res *Response
resCh := cf.node.SendAsync(c, req)
select {
case res = <-resCh:
case <-cf.grp.Ch():
cf.debug("|%s| probe %s: canceled", cycleID, c.ID.HexShort())
return nil
}
if res == nil {
cf.debug("|%s| probe %s: req canceled or timed out", cycleID, c.ID.HexShort())
return nil
}
if cf.findValue && res.FindValueKey != "" {
cf.debug("|%s| probe %s: got value", cycleID, c.ID.HexShort())
cf.findValueMutex.Lock()
cf.findValueResult = res.Contacts
cf.findValueMutex.Unlock()
cf.grp.Stop()
return nil
}
cf.debug("|%s| probe %s: got %s", cycleID, c.ID.HexShort(), res.argsDebug())
cf.insertIntoActiveList(c)
cf.appendNewToShortlist(res.Contacts)
cf.activeContactsMutex.Lock()
contacts := cf.activeContacts
if len(contacts) > bucketSize {
contacts = contacts[:bucketSize]
}
contactsStr := ""
for _, c := range contacts {
contactsStr += c.ID.HexShort() + ", "
}
cf.activeContactsMutex.Unlock()
return cf.closest(res.Contacts...)
}
// appendNewToShortlist appends any new contacts to the shortlist and sorts it by distance
// contacts that have already been added to the shortlist in the past are ignored
func (cf *contactFinder) appendNewToShortlist(contacts []Contact) {
cf.shortlistMutex.Lock()
defer cf.shortlistMutex.Unlock()
for _, c := range contacts {
if _, ok := cf.shortlistAdded[c.ID]; !ok {
cf.shortlist = append(cf.shortlist, c)
cf.shortlistAdded[c.ID] = true
}
}
sortByDistance(cf.shortlist, cf.target)
}
// popFromShortlist pops the first contact off the shortlist and returns it
func (cf *contactFinder) popFromShortlist() *Contact {
cf.shortlistMutex.Lock()
defer cf.shortlistMutex.Unlock()
if len(cf.shortlist) == 0 {
return nil
}
first := cf.shortlist[0]
cf.shortlist = cf.shortlist[1:]
return &first
}
// insertIntoActiveList inserts the contact into appropriate place in the list of active contacts (sorted by distance)
func (cf *contactFinder) insertIntoActiveList(contact Contact) {
cf.activeContactsMutex.Lock()
defer cf.activeContactsMutex.Unlock()
inserted := false
for i, n := range cf.activeContacts {
if cf.target.Closer(contact.ID, n.ID) {
cf.activeContacts = append(cf.activeContacts[:i], append([]Contact{contact}, cf.activeContacts[i:]...)...)
inserted = true
break
}
}
if !inserted {
cf.activeContacts = append(cf.activeContacts, contact)
}
}
// isSearchFinished returns true if the search is done and should be stopped
func (cf *contactFinder) isSearchFinished() bool {
if cf.findValue && len(cf.findValueResult) > 0 {
return true
}
select {
case <-cf.grp.Ch():
return true
default:
}
if cf.notGettingCloser.Load() {
return true
}
cf.activeContactsMutex.Lock()
defer cf.activeContactsMutex.Unlock()
return len(cf.activeContacts) >= bucketSize
}
func (cf *contactFinder) debug(format string, args ...interface{}) {
args = append([]interface{}{cf.node.id.HexShort()}, append([]interface{}{cf.target.HexShort()}, args...)...)
cfLog.Debugf("[%s] find %s: "+format, args...)
}
func (cf *contactFinder) closest(contacts ...Contact) *Contact {
if len(contacts) == 0 {
return nil
}
closest := contacts[0]
for _, c := range contacts {
if cf.target.Closer(c.ID, closest.ID) {
closest = c
}
}
return &closest
}
-422
View File
@@ -1,422 +0,0 @@
package dht
import (
"net"
"testing"
"time"
"github.com/lbryio/reflector.go/dht/bits"
"github.com/lyoshenka/bencode"
)
func TestPing(t *testing.T) {
dhtNodeID := bits.Rand()
testNodeID := bits.Rand()
conn := newTestUDPConn("127.0.0.1:21217")
dht := New(&Config{Address: "127.0.0.1:21216", NodeID: dhtNodeID.Hex()})
err := dht.connect(conn)
if err != nil {
t.Fatal(err)
}
defer dht.Shutdown()
messageID := newMessageID()
data, err := bencode.EncodeBytes(map[string]interface{}{
headerTypeField: requestType,
headerMessageIDField: messageID,
headerNodeIDField: testNodeID.RawString(),
headerPayloadField: "ping",
headerArgsField: []string{},
})
if err != nil {
panic(err)
}
conn.toRead <- testUDPPacket{addr: conn.addr, data: data}
timer := time.NewTimer(3 * time.Second)
select {
case <-timer.C:
t.Error("timeout")
case resp := <-conn.writes:
var response map[string]interface{}
err := bencode.DecodeBytes(resp.data, &response)
if err != nil {
t.Fatal(err)
}
if len(response) != 4 {
t.Errorf("expected 4 response fields, got %d", len(response))
}
_, ok := response[headerTypeField]
if !ok {
t.Error("missing type field")
} else {
rType, ok := response[headerTypeField].(int64)
if !ok {
t.Error("type is not an integer")
} else if rType != responseType {
t.Error("unexpected response type")
}
}
_, ok = response[headerMessageIDField]
if !ok {
t.Error("missing message id field")
} else {
rMessageID, ok := response[headerMessageIDField].(string)
if !ok {
t.Error("message ID is not a string")
} else if rMessageID != string(messageID[:]) {
t.Error("unexpected message ID")
}
}
_, ok = response[headerNodeIDField]
if !ok {
t.Error("missing node id field")
} else {
rNodeID, ok := response[headerNodeIDField].(string)
if !ok {
t.Error("node ID is not a string")
} else if rNodeID != dhtNodeID.RawString() {
t.Error("unexpected node ID")
}
}
_, ok = response[headerPayloadField]
if !ok {
t.Error("missing payload field")
} else {
rNodeID, ok := response[headerPayloadField].(string)
if !ok {
t.Error("payload is not a string")
} else if rNodeID != pingSuccessResponse {
t.Error("did not pong")
}
}
}
}
func TestStore(t *testing.T) {
dhtNodeID := bits.Rand()
testNodeID := bits.Rand()
conn := newTestUDPConn("127.0.0.1:21217")
dht := New(&Config{Address: "127.0.0.1:21216", NodeID: dhtNodeID.Hex()})
err := dht.connect(conn)
if err != nil {
t.Fatal(err)
}
defer dht.Shutdown()
messageID := newMessageID()
blobHashToStore := bits.Rand()
storeRequest := Request{
ID: messageID,
NodeID: testNodeID,
Method: storeMethod,
StoreArgs: &storeArgs{
BlobHash: blobHashToStore,
Value: storeArgsValue{
Token: dht.node.tokens.Get(testNodeID, conn.addr),
LbryID: testNodeID,
Port: 9999,
},
NodeID: testNodeID,
},
}
_ = "64 " + // start message
"313A30 693065" + // type: 0
"313A31 3230 3A 6EB490B5788B63F0F7E6D92352024D0CBDEC2D3A" + // message id
"313A32 3438 3A 7CE1B831DEC8689E44F80F547D2DEA171F6A625E1A4FF6C6165E645F953103DABEB068A622203F859C6C64658FD3AA3B" + // node id
"313A33 35 3A 73746F7265" + // method
"313A34 6C" + // start args list
"3438 3A 3214D6C2F77FCB5E8D5FC07EDAFBA614F031CE8B2EAB49F924F8143F6DFBADE048D918710072FB98AB1B52B58F4E1468" + // block hash
"64" + // start value dict
"363A6C6272796964 3438 3A 7CE1B831DEC8689E44F80F547D2DEA171F6A625E1A4FF6C6165E645F953103DABEB068A622203F859C6C64658FD3AA3B" + // lbry id
"343A706F7274 69 33333333 65" + // port
"353A746F6B656E 3438 3A 17C2D8E1E48EF21567FE4AD5C8ED944B798D3B65AB58D0C9122AD6587D1B5FED472EA2CB12284CEFA1C21EFF302322BD" + // token
"65" + // end value dict
"3438 3A 7CE1B831DEC8689E44F80F547D2DEA171F6A625E1A4FF6C6165E645F953103DABEB068A622203F859C6C64658FD3AA3B" + // node id
"693065" + // self store (integer)
"65" + // end args list
"65" // end message
data, err := bencode.EncodeBytes(storeRequest)
if err != nil {
t.Fatal(err)
}
conn.toRead <- testUDPPacket{addr: conn.addr, data: data}
timer := time.NewTimer(3 * time.Second)
var response map[string]interface{}
select {
case <-timer.C:
t.Fatal("timeout")
case resp := <-conn.writes:
err := bencode.DecodeBytes(resp.data, &response)
if err != nil {
t.Fatal(err)
}
}
verifyResponse(t, response, messageID, dhtNodeID.RawString())
_, ok := response[headerPayloadField]
if !ok {
t.Error("missing payload field")
} else {
rNodeID, ok := response[headerPayloadField].(string)
if !ok {
t.Error("payload is not a string")
} else if rNodeID != storeSuccessResponse {
t.Error("did not return OK")
}
}
if dht.node.store.CountStoredHashes() != 1 {
t.Error("dht store has wrong number of items")
}
items := dht.node.store.Get(blobHashToStore)
if len(items) != 1 {
t.Error("list created in store, but nothing in list")
}
if !items[0].ID.Equals(testNodeID) {
t.Error("wrong value stored")
}
}
func TestFindNode(t *testing.T) {
dhtNodeID := bits.Rand()
testNodeID := bits.Rand()
conn := newTestUDPConn("127.0.0.1:21217")
dht := New(&Config{Address: "127.0.0.1:21216", NodeID: dhtNodeID.Hex()})
err := dht.connect(conn)
if err != nil {
t.Fatal(err)
}
defer dht.Shutdown()
nodesToInsert := 3
var nodes []Contact
for i := 0; i < nodesToInsert; i++ {
n := Contact{ID: bits.Rand(), IP: net.ParseIP("127.0.0.1"), Port: 10000 + i}
nodes = append(nodes, n)
dht.node.rt.Update(n)
}
messageID := newMessageID()
blobHashToFind := bits.Rand()
request := Request{
ID: messageID,
NodeID: testNodeID,
Method: findNodeMethod,
Arg: &blobHashToFind,
}
data, err := bencode.EncodeBytes(request)
if err != nil {
t.Fatal(err)
}
conn.toRead <- testUDPPacket{addr: conn.addr, data: data}
timer := time.NewTimer(3 * time.Second)
var response map[string]interface{}
select {
case <-timer.C:
t.Fatal("timeout")
case resp := <-conn.writes:
err := bencode.DecodeBytes(resp.data, &response)
if err != nil {
t.Fatal(err)
}
}
verifyResponse(t, response, messageID, dhtNodeID.RawString())
_, ok := response[headerPayloadField]
if !ok {
t.Fatal("missing payload field")
}
contacts, ok := response[headerPayloadField].([]interface{})
if !ok {
t.Fatal("payload is not a list")
}
verifyContacts(t, contacts, nodes)
}
func TestFindValueExisting(t *testing.T) {
dhtNodeID := bits.Rand()
testNodeID := bits.Rand()
conn := newTestUDPConn("127.0.0.1:21217")
dht := New(&Config{Address: "127.0.0.1:21216", NodeID: dhtNodeID.Hex()})
err := dht.connect(conn)
if err != nil {
t.Fatal(err)
}
defer dht.Shutdown()
nodesToInsert := 3
for i := 0; i < nodesToInsert; i++ {
n := Contact{ID: bits.Rand(), IP: net.ParseIP("127.0.0.1"), Port: 10000 + i}
dht.node.rt.Update(n)
}
//data, _ := hex.DecodeString("64313a30693065313a3132303a7de8e57d34e316abbb5a8a8da50dcd1ad4c80e0f313a3234383a7ce1b831dec8689e44f80f547d2dea171f6a625e1a4ff6c6165e645f953103dabeb068a622203f859c6c64658fd3aa3b313a33393a66696e6456616c7565313a346c34383aa47624b8e7ee1e54df0c45e2eb858feb0b705bd2a78d8b739be31ba188f4bd6f56b371c51fecc5280d5fd26ba4168e966565")
messageID := newMessageID()
valueToFind := bits.Rand()
nodeToFind := Contact{ID: bits.Rand(), IP: net.ParseIP("1.2.3.4"), PeerPort: 1286}
dht.node.store.Upsert(valueToFind, nodeToFind)
dht.node.store.Upsert(valueToFind, nodeToFind)
dht.node.store.Upsert(valueToFind, nodeToFind)
request := Request{
ID: messageID,
NodeID: testNodeID,
Method: findValueMethod,
Arg: &valueToFind,
}
data, err := bencode.EncodeBytes(request)
if err != nil {
t.Fatal(err)
}
conn.toRead <- testUDPPacket{addr: conn.addr, data: data}
timer := time.NewTimer(3 * time.Second)
var response map[string]interface{}
select {
case <-timer.C:
t.Fatal("timeout")
case resp := <-conn.writes:
err := bencode.DecodeBytes(resp.data, &response)
if err != nil {
t.Fatal(err)
}
}
verifyResponse(t, response, messageID, dhtNodeID.RawString())
_, ok := response[headerPayloadField]
if !ok {
t.Fatal("missing payload field")
}
payload, ok := response[headerPayloadField].(map[string]interface{})
if !ok {
t.Fatal("payload is not a dictionary")
}
compactContacts, ok := payload[valueToFind.RawString()]
if !ok {
t.Fatal("payload is missing key for search value")
}
contacts, ok := compactContacts.([]interface{})
if !ok {
t.Fatal("search results are not a list")
}
verifyCompactContacts(t, contacts, []Contact{nodeToFind})
}
func TestFindValueFallbackToFindNode(t *testing.T) {
dhtNodeID := bits.Rand()
testNodeID := bits.Rand()
conn := newTestUDPConn("127.0.0.1:21217")
dht := New(&Config{Address: "127.0.0.1:21216", NodeID: dhtNodeID.Hex()})
err := dht.connect(conn)
if err != nil {
t.Fatal(err)
}
defer dht.Shutdown()
nodesToInsert := 3
var nodes []Contact
for i := 0; i < nodesToInsert; i++ {
n := Contact{ID: bits.Rand(), IP: net.ParseIP("127.0.0.1"), Port: 10000 + i}
nodes = append(nodes, n)
dht.node.rt.Update(n)
}
messageID := newMessageID()
valueToFind := bits.Rand()
request := Request{
ID: messageID,
NodeID: testNodeID,
Method: findValueMethod,
Arg: &valueToFind,
}
data, err := bencode.EncodeBytes(request)
if err != nil {
t.Fatal(err)
}
conn.toRead <- testUDPPacket{addr: conn.addr, data: data}
timer := time.NewTimer(3 * time.Second)
var response map[string]interface{}
select {
case <-timer.C:
t.Fatal("timeout")
case resp := <-conn.writes:
err := bencode.DecodeBytes(resp.data, &response)
if err != nil {
t.Fatal(err)
}
}
verifyResponse(t, response, messageID, dhtNodeID.RawString())
_, ok := response[headerPayloadField]
if !ok {
t.Fatal("missing payload field")
}
payload, ok := response[headerPayloadField].(map[string]interface{})
if !ok {
t.Fatal("payload is not a dictionary")
}
contactsList, ok := payload[contactsField]
if !ok {
t.Fatal("payload is missing 'contacts' key")
}
contacts, ok := contactsList.([]interface{})
if !ok {
t.Fatal("'contacts' is not a list")
}
verifyContacts(t, contacts, nodes)
}
-463
View File
@@ -1,463 +0,0 @@
package dht
import (
"encoding/json"
"fmt"
"net"
"strconv"
"strings"
"sync"
"time"
"github.com/lbryio/lbry.go/errors"
"github.com/lbryio/lbry.go/stop"
"github.com/lbryio/reflector.go/dht/bits"
)
// TODO: if routing table is ever empty (aka the node is isolated), it should re-bootstrap
// TODO: use a tree with bucket splitting instead of a fixed bucket list. include jack's optimization (see link in commit mesg)
// https://github.com/lbryio/lbry/pull/1211/commits/341b27b6d21ac027671d42458826d02735aaae41
// peer is a contact with extra information
type peer struct {
Contact Contact
Distance bits.Bitmap
LastActivity time.Time
// LastReplied time.Time
// LastRequested time.Time
// LastFailure time.Time
// SecondLastFailure time.Time
NumFailures int
//<lastPublished>,
//<originallyPublished>
// <originalPublisherID>
}
func (p *peer) Touch() {
p.LastActivity = time.Now()
p.NumFailures = 0
}
// ActiveSince returns whether a peer has responded in the last `d` duration
// this is used to check if the peer is "good", meaning that we believe the peer will respond to our requests
func (p *peer) ActiveInLast(d time.Duration) bool {
return time.Since(p.LastActivity) < d
}
// IsBad returns whether a peer is "bad", meaning that it has failed to respond to multiple pings in a row
func (p *peer) IsBad(maxFalures int) bool {
return p.NumFailures >= maxFalures
}
// Fail marks a peer as having failed to respond. It returns whether or not the peer should be removed from the routing table
func (p *peer) Fail() {
p.NumFailures++
}
type bucket struct {
lock *sync.RWMutex
peers []peer
lastUpdate time.Time
Range bits.Range // capitalized because `range` is a keyword
}
func newBucket(r bits.Range) *bucket {
return &bucket{
peers: make([]peer, 0, bucketSize),
lock: &sync.RWMutex{},
Range: r,
}
}
// Len returns the number of peers in the bucket
func (b bucket) Len() int {
b.lock.RLock()
defer b.lock.RUnlock()
return len(b.peers)
}
func (b bucket) Has(c Contact) bool {
b.lock.RLock()
defer b.lock.RUnlock()
for _, p := range b.peers {
if p.Contact.Equals(c, true) {
return true
}
}
return false
}
// Contacts returns a slice of the bucket's contacts
func (b bucket) Contacts() []Contact {
b.lock.RLock()
defer b.lock.RUnlock()
contacts := make([]Contact, len(b.peers))
for i := range b.peers {
contacts[i] = b.peers[i].Contact
}
return contacts
}
// UpdatePeer marks a contact as having been successfully contacted. if insertIfNew and the contact is does not exist yet, it is inserted
func (b *bucket) UpdatePeer(p peer, insertIfNew bool) error {
b.lock.Lock()
defer b.lock.Unlock()
if !b.Range.Contains(p.Distance) {
return errors.Err("this bucket range does not cover this peer")
}
peerIndex := find(p.Contact.ID, b.peers)
if peerIndex >= 0 {
b.lastUpdate = time.Now()
b.peers[peerIndex].Touch()
moveToBack(b.peers, peerIndex)
} else if insertIfNew {
hasRoom := true
if len(b.peers) >= bucketSize {
hasRoom = false
for i := range b.peers {
if b.peers[i].IsBad(maxPeerFails) {
// TODO: Ping contact first. Only remove if it does not respond
b.peers = append(b.peers[:i], b.peers[i+1:]...)
hasRoom = true
break
}
}
}
if hasRoom {
b.lastUpdate = time.Now()
p.Touch()
b.peers = append(b.peers, p)
}
}
return nil
}
// FailContact marks a contact as having failed, and removes it if it failed too many times
func (b *bucket) FailContact(id bits.Bitmap) {
b.lock.Lock()
defer b.lock.Unlock()
i := find(id, b.peers)
if i >= 0 {
// BEP5 says not to remove the contact until the bucket is full and you try to insert
b.peers[i].Fail()
}
}
// find returns the contact in the bucket, or nil if the bucket does not contain the contact
func find(id bits.Bitmap, peers []peer) int {
for i := range peers {
if peers[i].Contact.ID.Equals(id) {
return i
}
}
return -1
}
// NeedsRefresh returns true if bucket has not been updated in the last `refreshInterval`, false otherwise
func (b *bucket) NeedsRefresh(refreshInterval time.Duration) bool {
b.lock.RLock()
defer b.lock.RUnlock()
return time.Since(b.lastUpdate) > refreshInterval
}
func (b *bucket) Split() (*bucket, *bucket) {
b.lock.Lock()
defer b.lock.Unlock()
left := newBucket(b.Range.IntervalP(1, 2))
right := newBucket(b.Range.IntervalP(2, 2))
left.lastUpdate = b.lastUpdate
right.lastUpdate = b.lastUpdate
for _, p := range b.peers {
if left.Range.Contains(p.Distance) {
left.peers = append(left.peers, p)
} else {
right.peers = append(right.peers, p)
}
}
if len(b.peers) > 1 {
if len(left.peers) == 0 {
left, right = right.Split()
left.Range.Start = b.Range.Start
} else if len(right.peers) == 0 {
left, right = left.Split()
right.Range.End = b.Range.End
}
}
return left, right
}
type routingTable struct {
id bits.Bitmap
buckets []*bucket
mu *sync.RWMutex // this mutex is write-locked only when CHANGING THE NUMBER OF BUCKETS in the table
}
func newRoutingTable(id bits.Bitmap) *routingTable {
rt := routingTable{
id: id,
mu: &sync.RWMutex{},
}
rt.reset()
return &rt
}
func (rt *routingTable) reset() {
rt.mu.Lock()
defer rt.mu.Unlock()
rt.buckets = []*bucket{newBucket(bits.MaxRange())}
}
func (rt *routingTable) BucketInfo() string {
rt.mu.RLock()
defer rt.mu.RUnlock()
var bucketInfo []string
for i, b := range rt.buckets {
if b.Len() > 0 {
contacts := b.Contacts()
s := make([]string, len(contacts))
for j, c := range contacts {
s[j] = c.ID.HexShort()
}
bucketInfo = append(bucketInfo, fmt.Sprintf("bucket %d: (%d) %s", i, len(contacts), strings.Join(s, ", ")))
}
}
if len(bucketInfo) == 0 {
return "buckets are empty"
}
return strings.Join(bucketInfo, "\n")
}
// Update inserts or refreshes a contact
func (rt *routingTable) Update(c Contact) {
rt.mu.Lock() // write lock, because updates may cause bucket splits
defer rt.mu.Unlock()
b := rt.bucketFor(c.ID)
if rt.shouldSplit(b, c) {
left, right := b.Split()
for i := range rt.buckets {
if rt.buckets[i].Range.Start.Equals(left.Range.Start) {
rt.buckets = append(rt.buckets[:i], append([]*bucket{left, right}, rt.buckets[i+1:]...)...)
break
}
}
if left.Range.Contains(c.ID) {
b = left
} else {
b = right
}
}
err := b.UpdatePeer(peer{Contact: c, Distance: rt.id.Xor(c.ID)}, true)
if err != nil {
log.Error(err)
}
}
// Fresh refreshes a contact if its already in the routing table
func (rt *routingTable) Fresh(c Contact) {
rt.mu.RLock()
defer rt.mu.RUnlock()
err := rt.bucketFor(c.ID).UpdatePeer(peer{Contact: c, Distance: rt.id.Xor(c.ID)}, false)
if err != nil {
log.Error(err)
}
}
// FailContact marks a contact as having failed, and removes it if it failed too many times
func (rt *routingTable) Fail(c Contact) {
rt.mu.RLock()
defer rt.mu.RUnlock()
rt.bucketFor(c.ID).FailContact(c.ID)
}
// GetClosest returns the closest `limit` contacts from the routing table.
// This is a locking wrapper around getClosest()
func (rt *routingTable) GetClosest(target bits.Bitmap, limit int) []Contact {
rt.mu.RLock()
defer rt.mu.RUnlock()
return rt.getClosest(target, limit)
}
// getClosest returns the closest `limit` contacts from the routing table
func (rt *routingTable) getClosest(target bits.Bitmap, limit int) []Contact {
var contacts []Contact
for _, b := range rt.buckets {
contacts = append(contacts, b.Contacts()...)
}
sortByDistance(contacts, target)
if len(contacts) > limit {
contacts = contacts[:limit]
}
return contacts
}
// Count returns the number of contacts in the routing table
func (rt *routingTable) Count() int {
rt.mu.RLock()
defer rt.mu.RUnlock()
count := 0
for _, bucket := range rt.buckets {
count += bucket.Len()
}
return count
}
// Len returns the number of buckets in the routing table
func (rt *routingTable) Len() int {
rt.mu.RLock()
defer rt.mu.RUnlock()
return len(rt.buckets)
}
func (rt *routingTable) bucketFor(target bits.Bitmap) *bucket {
if rt.id.Equals(target) {
panic("routing table does not have a bucket for its own id")
}
distance := target.Xor(rt.id)
for _, b := range rt.buckets {
if b.Range.Contains(distance) {
return b
}
}
panic("target is not contained in any buckets")
}
func (rt *routingTable) shouldSplit(b *bucket, c Contact) bool {
if b.Has(c) {
return false
}
if b.Len() >= bucketSize {
if b.Range.Start.Equals(bits.Bitmap{}) { // this is the bucket covering our node id
return true
}
kClosest := rt.getClosest(rt.id, bucketSize)
kthClosest := kClosest[len(kClosest)-1]
if rt.id.Closer(c.ID, kthClosest.ID) {
return true
}
}
return false
}
//func (rt *routingTable) printBucketInfo() {
// fmt.Printf("there are %d contacts in %d buckets\n", rt.Count(), rt.Len())
// for i, b := range rt.buckets {
// fmt.Printf("bucket %d, %d contacts\n", i+1, len(b.peers))
// fmt.Printf(" start : %s\n", b.Range.Start.String())
// fmt.Printf(" stop : %s\n", b.Range.End.String())
// fmt.Println("")
// }
//}
func (rt *routingTable) GetIDsForRefresh(refreshInterval time.Duration) []bits.Bitmap {
var bitmaps []bits.Bitmap
for i, bucket := range rt.buckets {
if bucket.NeedsRefresh(refreshInterval) {
bitmaps = append(bitmaps, bits.Rand().Prefix(i, false))
}
}
return bitmaps
}
const rtContactSep = "-"
type rtSave struct {
ID string `json:"id"`
Contacts []string `json:"contacts"`
}
func (rt *routingTable) MarshalJSON() ([]byte, error) {
var data rtSave
data.ID = rt.id.Hex()
for _, b := range rt.buckets {
for _, c := range b.Contacts() {
data.Contacts = append(data.Contacts, strings.Join([]string{c.ID.Hex(), c.IP.String(), strconv.Itoa(c.Port)}, rtContactSep))
}
}
return json.Marshal(data)
}
func (rt *routingTable) UnmarshalJSON(b []byte) error {
var data rtSave
err := json.Unmarshal(b, &data)
if err != nil {
return err
}
rt.id, err = bits.FromHex(data.ID)
if err != nil {
return errors.Prefix("decoding ID", err)
}
rt.reset()
for _, s := range data.Contacts {
parts := strings.Split(s, rtContactSep)
if len(parts) != 3 {
return errors.Err("decoding contact %s: wrong number of parts", s)
}
var c Contact
c.ID, err = bits.FromHex(parts[0])
if err != nil {
return errors.Err("decoding contact %s: invalid ID: %s", s, err)
}
c.IP = net.ParseIP(parts[1])
if c.IP == nil {
return errors.Err("decoding contact %s: invalid IP", s)
}
c.Port, err = strconv.Atoi(parts[2])
if err != nil {
return errors.Err("decoding contact %s: invalid port: %s", s, err)
}
rt.Update(c)
}
return nil
}
// RoutingTableRefresh refreshes any buckets that need to be refreshed
func RoutingTableRefresh(n *Node, refreshInterval time.Duration, parentGrp *stop.Group) {
done := stop.New()
for _, id := range n.rt.GetIDsForRefresh(refreshInterval) {
done.Add(1)
go func(id bits.Bitmap) {
defer done.Done()
_, _, err := FindContacts(n, id, false, parentGrp)
if err != nil {
log.Error("error finding contact during routing table refresh - ", err)
}
}(id)
}
done.Wait()
done.Stop()
}
func moveToBack(peers []peer, index int) {
if index < 0 || len(peers) <= index+1 {
return
}
p := peers[index]
for i := index; i < len(peers)-1; i++ {
peers[i] = peers[i+1]
}
peers[len(peers)-1] = p
}
-328
View File
@@ -1,328 +0,0 @@
package dht
import (
"encoding/json"
"math/big"
"net"
"strconv"
"strings"
"testing"
"github.com/lbryio/reflector.go/dht/bits"
"github.com/sebdah/goldie"
)
func TestBucket_Split(t *testing.T) {
rt := newRoutingTable(bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"))
if len(rt.buckets) != 1 {
t.Errorf("there should only be one bucket so far")
}
if len(rt.buckets[0].peers) != 0 {
t.Errorf("there should be no contacts yet")
}
var tests = []struct {
name string
id bits.Bitmap
expectedBucketCount int
expectedTotalContacts int
}{
//fill first bucket
{"b1-one", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100"), 1, 1},
{"b1-two", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000200"), 1, 2},
{"b1-three", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000300"), 1, 3},
{"b1-four", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000400"), 1, 4},
{"b1-five", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000500"), 1, 5},
{"b1-six", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000600"), 1, 6},
{"b1-seven", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000700"), 1, 7},
{"b1-eight", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800"), 1, 8},
// split off second bucket and fill it
{"b2-one", bits.FromHexP("001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 9},
{"b2-two", bits.FromHexP("002000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 10},
{"b2-three", bits.FromHexP("003000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 11},
{"b2-four", bits.FromHexP("004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 12},
{"b2-five", bits.FromHexP("005000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 13},
{"b2-six", bits.FromHexP("006000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 14},
{"b2-seven", bits.FromHexP("007000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 15},
// at this point there are two buckets. the first has 7 contacts, the second has 8
// inserts into the second bucket should be skipped
{"dont-split", bits.FromHexP("009000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 15},
// ... unless the ID is closer than the kth-closest contact
{"split-kth-closest", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), 2, 16},
{"b3-two", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002"), 3, 17},
{"b3-three", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003"), 3, 18},
}
for i, testCase := range tests {
rt.Update(Contact{testCase.id, net.ParseIP("127.0.0.1"), 8000 + i, 0})
if len(rt.buckets) != testCase.expectedBucketCount {
t.Errorf("failed test case %s. there should be %d buckets, got %d", testCase.name, testCase.expectedBucketCount, len(rt.buckets))
}
if rt.Count() != testCase.expectedTotalContacts {
t.Errorf("failed test case %s. there should be %d contacts, got %d", testCase.name, testCase.expectedTotalContacts, rt.Count())
}
}
var testRanges = []struct {
id bits.Bitmap
expected int
}{
{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), 0},
{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000005"), 0},
{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000410"), 1},
{bits.FromHexP("0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007f0"), 1},
{bits.FromHexP("F00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800"), 2},
{bits.FromHexP("F00000000000000000000000000000000000000000000000000F00000000000000000000000000000000000000000000"), 2},
{bits.FromHexP("F0000000000000000000000000000000F0000000000000000000000000F0000000000000000000000000000000000000"), 2},
}
for _, tt := range testRanges {
bucket := bucketNumFor(rt, tt.id)
if bucket != tt.expected {
t.Errorf("bucketFor(%s, %s) => got %d, expected %d", tt.id.Hex(), rt.id.Hex(), bucket, tt.expected)
}
}
}
func bucketNumFor(rt *routingTable, target bits.Bitmap) int {
if rt.id.Equals(target) {
panic("routing table does not have a bucket for its own id")
}
distance := target.Xor(rt.id)
for i := range rt.buckets {
if rt.buckets[i].Range.Contains(distance) {
return i
}
}
panic("target is not contained in any buckets")
}
func TestBucket_Split_Continuous(t *testing.T) {
b := newBucket(bits.MaxRange())
left, right := b.Split()
if !left.Range.Start.Equals(b.Range.Start) {
t.Errorf("left bucket start does not align with original bucket start. got %s, expected %s", left.Range.Start, b.Range.Start)
}
if !right.Range.End.Equals(b.Range.End) {
t.Errorf("right bucket end does not align with original bucket end. got %s, expected %s", right.Range.End, b.Range.End)
}
leftEndNext := (&big.Int{}).Add(left.Range.End.Big(), big.NewInt(1))
if !bits.FromBigP(leftEndNext).Equals(right.Range.Start) {
t.Errorf("there's a gap between left bucket end and right bucket start. end is %s, start is %s", left.Range.End, right.Range.Start)
}
}
func TestBucket_Split_KthClosest_DoSplit(t *testing.T) {
rt := newRoutingTable(bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"))
// add 4 low IDs
rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), net.ParseIP("127.0.0.1"), 8001, 0})
rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002"), net.ParseIP("127.0.0.1"), 8002, 0})
rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003"), net.ParseIP("127.0.0.1"), 8003, 0})
rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004"), net.ParseIP("127.0.0.1"), 8004, 0})
// add 4 high IDs
rt.Update(Contact{bits.FromHexP("800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8001, 0})
rt.Update(Contact{bits.FromHexP("900000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8002, 0})
rt.Update(Contact{bits.FromHexP("a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8003, 0})
rt.Update(Contact{bits.FromHexP("b00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8004, 0})
// split the bucket and fill the high bucket
rt.Update(Contact{bits.FromHexP("c00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8005, 0})
rt.Update(Contact{bits.FromHexP("d00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8006, 0})
rt.Update(Contact{bits.FromHexP("e00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8007, 0})
rt.Update(Contact{bits.FromHexP("f00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8008, 0})
// add a high ID. it should split because the high ID is closer than the Kth closest ID
rt.Update(Contact{bits.FromHexP("910000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.1"), 8009, 0})
if len(rt.buckets) != 3 {
t.Errorf("expected 3 buckets, got %d", len(rt.buckets))
}
if rt.Count() != 13 {
t.Errorf("expected 13 contacts, got %d", rt.Count())
}
}
func TestBucket_Split_KthClosest_DontSplit(t *testing.T) {
rt := newRoutingTable(bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"))
// add 4 low IDs
rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), net.ParseIP("127.0.0.1"), 8001, 0})
rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002"), net.ParseIP("127.0.0.1"), 8002, 0})
rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003"), net.ParseIP("127.0.0.1"), 8003, 0})
rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004"), net.ParseIP("127.0.0.1"), 8004, 0})
// add 4 high IDs
rt.Update(Contact{bits.FromHexP("800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8001, 0})
rt.Update(Contact{bits.FromHexP("900000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8002, 0})
rt.Update(Contact{bits.FromHexP("a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8003, 0})
rt.Update(Contact{bits.FromHexP("b00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8004, 0})
// split the bucket and fill the high bucket
rt.Update(Contact{bits.FromHexP("c00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8005, 0})
rt.Update(Contact{bits.FromHexP("d00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8006, 0})
rt.Update(Contact{bits.FromHexP("e00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8007, 0})
rt.Update(Contact{bits.FromHexP("f00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8008, 0})
// add a really high ID. this should not split because its not closer than the Kth closest ID
rt.Update(Contact{bits.FromHexP("ffff00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.1"), 8009, 0})
if len(rt.buckets) != 2 {
t.Errorf("expected 2 buckets, got %d", len(rt.buckets))
}
if rt.Count() != 12 {
t.Errorf("expected 12 contacts, got %d", rt.Count())
}
}
func TestRoutingTable_GetClosest(t *testing.T) {
n1 := bits.FromHexP("FFFFFFFF0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
n2 := bits.FromHexP("FFFFFFF00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
n3 := bits.FromHexP("111111110000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
rt := newRoutingTable(n1)
rt.Update(Contact{n2, net.ParseIP("127.0.0.1"), 8001, 0})
rt.Update(Contact{n3, net.ParseIP("127.0.0.1"), 8002, 0})
contacts := rt.GetClosest(bits.FromHexP("222222220000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 1)
if len(contacts) != 1 {
t.Fail()
return
}
if !contacts[0].ID.Equals(n3) {
t.Error(contacts[0])
}
contacts = rt.GetClosest(n2, 10)
if len(contacts) != 2 {
t.Error(len(contacts))
return
}
if !contacts[0].ID.Equals(n2) {
t.Error(contacts[0])
}
if !contacts[1].ID.Equals(n3) {
t.Error(contacts[1])
}
}
func TestRoutingTable_GetClosest_Empty(t *testing.T) {
n1 := bits.FromShortHexP("1")
rt := newRoutingTable(n1)
contacts := rt.GetClosest(bits.FromShortHexP("a"), 3)
if len(contacts) != 0 {
t.Error("there shouldn't be any contacts")
return
}
}
func TestRoutingTable_Refresh(t *testing.T) {
t.Skip("TODO: test routing table refreshing")
}
func TestRoutingTable_MoveToBack(t *testing.T) {
tt := map[string]struct {
data []peer
index int
expected []peer
}{
"simpleMove": {
data: []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
index: 1,
expected: []peer{{NumFailures: 0}, {NumFailures: 2}, {NumFailures: 3}, {NumFailures: 1}},
},
"moveFirst": {
data: []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
index: 0,
expected: []peer{{NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}, {NumFailures: 0}},
},
"moveLast": {
data: []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
index: 3,
expected: []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
},
"largeIndex": {
data: []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
index: 27,
expected: []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
},
"negativeIndex": {
data: []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
index: -12,
expected: []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
},
}
for name, test := range tt {
moveToBack(test.data, test.index)
expected := make([]string, len(test.expected))
actual := make([]string, len(test.data))
for i := range actual {
actual[i] = strconv.Itoa(test.data[i].NumFailures)
expected[i] = strconv.Itoa(test.expected[i].NumFailures)
}
expJoin := strings.Join(expected, ",")
actJoin := strings.Join(actual, ",")
if actJoin != expJoin {
t.Errorf("%s failed: got %s; expected %s", name, actJoin, expJoin)
}
}
}
func TestRoutingTable_Save(t *testing.T) {
t.Skip("fix me")
id := bits.FromHexP("1c8aff71b99462464d9eeac639595ab99664be3482cb91a29d87467515c7d9158fe72aa1f1582dab07d8f8b5db277f41")
rt := newRoutingTable(id)
for i, b := range rt.buckets {
for j := 0; j < bucketSize; j++ {
toAdd := b.Range.Start.Add(bits.FromShortHexP(strconv.Itoa(j)))
if toAdd.Cmp(b.Range.End) <= 0 {
rt.Update(Contact{
ID: b.Range.Start.Add(bits.FromShortHexP(strconv.Itoa(j))),
IP: net.ParseIP("1.2.3." + strconv.Itoa(j)),
Port: 1 + i*bucketSize + j,
})
}
}
}
data, err := json.MarshalIndent(rt, "", " ")
if err != nil {
t.Error(err)
}
goldie.Assert(t, t.Name(), data)
}
func TestRoutingTable_Load_ID(t *testing.T) {
t.Skip("fix me")
id := "1c8aff71b99462464d9eeac639595ab99664be3482cb91a29d87467515c7d9158fe72aa1f1582dab07d8f8b5db277f41"
data := []byte(`{"id": "` + id + `","contacts": []}`)
rt := routingTable{}
err := json.Unmarshal(data, &rt)
if err != nil {
t.Error(err)
}
if rt.id.Hex() != id {
t.Error("id mismatch")
}
}
func TestRoutingTable_Load_Contacts(t *testing.T) {
t.Skip("TODO")
}
-187
View File
@@ -1,187 +0,0 @@
package dht
import (
"context"
"net"
"net/http"
"strconv"
"sync"
"github.com/lbryio/lbry.go/errors"
"github.com/lbryio/reflector.go/dht/bits"
"github.com/gorilla/mux"
rpc2 "github.com/gorilla/rpc/v2"
"github.com/gorilla/rpc/v2/json"
)
type rpcReceiver struct {
dht *DHT
}
type RpcPingArgs struct {
Address string
}
func (rpc *rpcReceiver) Ping(r *http.Request, args *RpcPingArgs, result *string) error {
if args.Address == "" {
return errors.Err("no address given")
}
err := rpc.dht.Ping(args.Address)
if err != nil {
return err
}
*result = pingSuccessResponse
return nil
}
type RpcFindArgs struct {
Key string
NodeID string
IP string
Port int
}
func (rpc *rpcReceiver) FindNode(r *http.Request, args *RpcFindArgs, result *[]Contact) error {
key, err := bits.FromHex(args.Key)
if err != nil {
return err
}
toQuery, err := bits.FromHex(args.NodeID)
if err != nil {
return err
}
c := Contact{ID: toQuery, IP: net.ParseIP(args.IP), Port: args.Port}
req := Request{Method: findNodeMethod, Arg: &key}
nodeResponse := rpc.dht.node.Send(c, req)
if nodeResponse != nil && nodeResponse.Contacts != nil {
*result = nodeResponse.Contacts
}
return nil
}
type RpcFindValueResult struct {
Contacts []Contact
Value string
}
func (rpc *rpcReceiver) FindValue(r *http.Request, args *RpcFindArgs, result *RpcFindValueResult) error {
key, err := bits.FromHex(args.Key)
if err != nil {
return err
}
toQuery, err := bits.FromHex(args.NodeID)
if err != nil {
return err
}
c := Contact{ID: toQuery, IP: net.ParseIP(args.IP), Port: args.Port}
req := Request{Arg: &key, Method: findValueMethod}
nodeResponse := rpc.dht.node.Send(c, req)
if nodeResponse != nil && nodeResponse.FindValueKey != "" {
*result = RpcFindValueResult{Value: nodeResponse.FindValueKey}
return nil
}
if nodeResponse != nil && nodeResponse.Contacts != nil {
*result = RpcFindValueResult{Contacts: nodeResponse.Contacts}
return nil
}
return errors.Err("not sure what happened")
}
type RpcIterativeFindValueArgs struct {
Key string
}
type RpcIterativeFindValueResult struct {
Contacts []Contact
FoundValue bool
}
func (rpc *rpcReceiver) IterativeFindValue(r *http.Request, args *RpcIterativeFindValueArgs, result *RpcIterativeFindValueResult) error {
key, err := bits.FromHex(args.Key)
if err != nil {
return err
}
foundContacts, found, err := FindContacts(rpc.dht.node, key, false, nil)
if err != nil {
return err
}
result.Contacts = foundContacts
result.FoundValue = found
return nil
}
type RpcBucketResponse struct {
Start string
End string
NumContacts int
Contacts []Contact
}
type RpcRoutingTableResponse struct {
NodeID string
NumBuckets int
Buckets []RpcBucketResponse
}
func (rpc *rpcReceiver) GetRoutingTable(r *http.Request, args *struct{}, result *RpcRoutingTableResponse) error {
result.NodeID = rpc.dht.node.id.String()
result.NumBuckets = len(rpc.dht.node.rt.buckets)
for _, b := range rpc.dht.node.rt.buckets {
result.Buckets = append(result.Buckets, RpcBucketResponse{
Start: b.Range.Start.String(),
End: b.Range.End.String(),
NumContacts: b.Len(),
Contacts: b.Contacts(),
})
}
return nil
}
func (rpc *rpcReceiver) AddKnownNode(r *http.Request, args *Contact, result *string) error {
rpc.dht.node.AddKnownNode(*args)
return nil
}
func (dht *DHT) runRPCServer(port int) {
addr := "0.0.0.0:" + strconv.Itoa(port)
s := rpc2.NewServer()
s.RegisterCodec(json.NewCodec(), "application/json")
s.RegisterCodec(json.NewCodec(), "application/json;charset=UTF-8")
err := s.RegisterService(&rpcReceiver{dht: dht}, "rpc")
if err != nil {
log.Error(errors.Prefix("registering rpc service", err))
return
}
handler := mux.NewRouter()
handler.Handle("/", s)
server := &http.Server{Addr: addr, Handler: handler}
wg := sync.WaitGroup{}
wg.Add(1)
go func() {
defer wg.Done()
log.Printf("RPC server listening on %s", addr)
err := server.ListenAndServe()
if err != nil && err != http.ErrServerClosed {
log.Error(err)
}
}()
<-dht.grp.Ch()
err = server.Shutdown(context.Background())
if err != nil {
log.Error(errors.Prefix("shutting down rpc service", err))
return
}
wg.Wait()
}
-62
View File
@@ -1,62 +0,0 @@
package dht
import (
"sync"
"github.com/lbryio/reflector.go/dht/bits"
)
// TODO: expire stored data after tExpire time
type contactStore struct {
// map of blob hashes to (map of node IDs to bools)
hashes map[bits.Bitmap]map[bits.Bitmap]bool
// stores the peers themselves, so they can be updated in one place
contacts map[bits.Bitmap]Contact
lock sync.RWMutex
}
func newStore() *contactStore {
return &contactStore{
hashes: make(map[bits.Bitmap]map[bits.Bitmap]bool),
contacts: make(map[bits.Bitmap]Contact),
}
}
func (s *contactStore) Upsert(blobHash bits.Bitmap, contact Contact) {
s.lock.Lock()
defer s.lock.Unlock()
if _, ok := s.hashes[blobHash]; !ok {
s.hashes[blobHash] = make(map[bits.Bitmap]bool)
}
s.hashes[blobHash][contact.ID] = true
s.contacts[contact.ID] = contact
}
func (s *contactStore) Get(blobHash bits.Bitmap) []Contact {
s.lock.RLock()
defer s.lock.RUnlock()
var contacts []Contact
if ids, ok := s.hashes[blobHash]; ok {
for id := range ids {
contact, ok := s.contacts[id]
if !ok {
panic("node id in IDs list, but not in nodeInfo")
}
contacts = append(contacts, contact)
}
}
return contacts
}
func (s *contactStore) RemoveTODO(contact Contact) {
// TODO: remove peer from everywhere
}
func (s *contactStore) CountStoredHashes() int {
s.lock.RLock()
defer s.lock.RUnlock()
return len(s.hashes)
}
-312
View File
@@ -1,312 +0,0 @@
package dht
import (
"net"
"strconv"
"strings"
"testing"
"time"
"github.com/lbryio/lbry.go/errors"
"github.com/lbryio/reflector.go/dht/bits"
)
var testingDHTIP = "127.0.0.1"
var testingDHTFirstPort = 21000
// TestingCreateNetwork initializes a testable DHT network with a specific number of nodes, with bootstrap and concurrent options.
func TestingCreateNetwork(t *testing.T, numNodes int, bootstrap, concurrent bool) (*BootstrapNode, []*DHT) {
var bootstrapNode *BootstrapNode
var seeds []string
if bootstrap {
bootstrapAddress := testingDHTIP + ":" + strconv.Itoa(testingDHTFirstPort)
seeds = []string{bootstrapAddress}
bootstrapNode = NewBootstrapNode(bits.Rand(), 0, bootstrapDefaultRefreshDuration)
listener, err := net.ListenPacket(Network, bootstrapAddress)
if err != nil {
panic(err)
}
err = bootstrapNode.Connect(listener.(*net.UDPConn))
if err != nil {
t.Error("error connecting bootstrap node - ", err)
}
}
if numNodes < 1 {
return bootstrapNode, nil
}
firstPort := testingDHTFirstPort + 1
dhts := make([]*DHT, numNodes)
for i := 0; i < numNodes; i++ {
c := NewStandardConfig()
c.NodeID = bits.Rand().Hex()
c.Address = testingDHTIP + ":" + strconv.Itoa(firstPort+i)
c.SeedNodes = seeds
dht := New(c)
go func() {
err := dht.Start()
if err != nil {
t.Error("error starting dht - ", err)
}
}()
if !concurrent {
dht.WaitUntilJoined()
}
dhts[i] = dht
}
if concurrent {
for _, d := range dhts {
d.WaitUntilJoined()
}
}
return bootstrapNode, dhts
}
type timeoutErr struct {
error
}
func (t timeoutErr) Timeout() bool {
return true
}
func (t timeoutErr) Temporary() bool {
return true
}
// TODO: just use a normal net.Conn instead of this mock conn
type testUDPPacket struct {
data []byte
addr *net.UDPAddr
}
type testUDPConn struct {
addr *net.UDPAddr
toRead chan testUDPPacket
writes chan testUDPPacket
readDeadline time.Time
}
func newTestUDPConn(addr string) *testUDPConn {
parts := strings.Split(addr, ":")
if len(parts) != 2 {
panic("addr needs ip and port")
}
port, err := strconv.Atoi(parts[1])
if err != nil {
panic(err)
}
return &testUDPConn{
addr: &net.UDPAddr{IP: net.IP(parts[0]), Port: port},
toRead: make(chan testUDPPacket),
writes: make(chan testUDPPacket),
}
}
func (t testUDPConn) ReadFromUDP(b []byte) (int, *net.UDPAddr, error) {
var timeoutCh <-chan time.Time
if !t.readDeadline.IsZero() {
timeoutCh = time.After(time.Until(t.readDeadline))
}
select {
case packet, ok := <-t.toRead:
if !ok {
return 0, nil, errors.Err("conn closed")
}
n := copy(b, packet.data)
return n, packet.addr, nil
case <-timeoutCh:
return 0, nil, timeoutErr{errors.Err("timeout")}
}
}
func (t testUDPConn) WriteToUDP(b []byte, addr *net.UDPAddr) (int, error) {
t.writes <- testUDPPacket{data: b, addr: addr}
return len(b), nil
}
func (t *testUDPConn) SetReadDeadline(tm time.Time) error {
t.readDeadline = tm
return nil
}
func (t *testUDPConn) SetWriteDeadline(tm time.Time) error {
return nil
}
func (t *testUDPConn) Close() error {
close(t.toRead)
t.writes = nil
return nil
}
func verifyResponse(t *testing.T, resp map[string]interface{}, id messageID, dhtNodeID string) {
if len(resp) != 4 {
t.Errorf("expected 4 response fields, got %d", len(resp))
}
_, ok := resp[headerTypeField]
if !ok {
t.Error("missing type field")
} else {
rType, ok := resp[headerTypeField].(int64)
if !ok {
t.Error("type is not an integer")
} else if rType != responseType {
t.Error("unexpected response type")
}
}
_, ok = resp[headerMessageIDField]
if !ok {
t.Error("missing message id field")
} else {
rMessageID, ok := resp[headerMessageIDField].(string)
if !ok {
t.Error("message ID is not a string")
} else if rMessageID != string(id[:]) {
t.Error("unexpected message ID")
}
if len(rMessageID) != messageIDLength {
t.Errorf("message ID should be %d chars long", messageIDLength)
}
}
_, ok = resp[headerNodeIDField]
if !ok {
t.Error("missing node id field")
} else {
rNodeID, ok := resp[headerNodeIDField].(string)
if !ok {
t.Error("node ID is not a string")
} else if rNodeID != dhtNodeID {
t.Error("unexpected node ID")
}
if len(rNodeID) != nodeIDLength {
t.Errorf("node ID should be %d chars long", nodeIDLength)
}
}
}
func verifyContacts(t *testing.T, contacts []interface{}, nodes []Contact) {
if len(contacts) != len(nodes) {
t.Errorf("got %d contacts; expected %d", len(contacts), len(nodes))
return
}
foundNodes := make(map[string]bool)
for _, c := range contacts {
contact, ok := c.([]interface{})
if !ok {
t.Error("contact is not a list")
return
}
if len(contact) != 3 {
t.Error("contact must be 3 items")
return
}
var currNode Contact
currNodeFound := false
id, ok := contact[0].(string)
if !ok {
t.Error("contact id is not a string")
} else {
if _, ok := foundNodes[id]; ok {
t.Errorf("contact %s appears multiple times", id)
continue
}
for _, n := range nodes {
if n.ID.RawString() == id {
currNode = n
currNodeFound = true
foundNodes[id] = true
break
}
}
if !currNodeFound {
t.Errorf("unexpected contact %s", id)
continue
}
}
ip, ok := contact[1].(string)
if !ok {
t.Error("contact IP is not a string")
} else if !currNode.IP.Equal(net.ParseIP(ip)) {
t.Errorf("contact IP mismatch. got %s; expected %s", ip, currNode.IP.String())
}
port, ok := contact[2].(int64)
if !ok {
t.Error("contact port is not an int")
} else if int(port) != currNode.Port {
t.Errorf("contact port mismatch. got %d; expected %d", port, currNode.Port)
}
}
}
func verifyCompactContacts(t *testing.T, contacts []interface{}, nodes []Contact) {
if len(contacts) != len(nodes) {
t.Errorf("got %d contacts; expected %d", len(contacts), len(nodes))
return
}
foundNodes := make(map[string]bool)
for _, c := range contacts {
compact, ok := c.(string)
if !ok {
t.Error("contact is not a string")
return
}
contact := Contact{}
err := contact.UnmarshalCompact([]byte(compact))
if err != nil {
t.Error(err)
return
}
var currNode Contact
currNodeFound := false
if _, ok := foundNodes[contact.ID.Hex()]; ok {
t.Errorf("contact %s appears multiple times", contact.ID.Hex())
continue
}
for _, n := range nodes {
if n.ID.Equals(contact.ID) {
currNode = n
currNodeFound = true
foundNodes[contact.ID.Hex()] = true
break
}
}
if !currNodeFound {
t.Errorf("unexpected contact %s", contact.ID.Hex())
continue
}
if !currNode.IP.Equal(contact.IP) {
t.Errorf("contact IP mismatch. got %s; expected %s", contact.IP.String(), currNode.IP.String())
}
if contact.Port != currNode.Port {
t.Errorf("contact port mismatch. got %d; expected %d", contact.Port, currNode.Port)
}
}
}
-71
View File
@@ -1,71 +0,0 @@
package dht
import (
"sync"
"time"
"github.com/lbryio/reflector.go/dht/bits"
"github.com/lbryio/lbry.go/stop"
)
// TODO: this should be moved out of dht and into node, and it should be completely hidden inside node. dht should not need to know about tokens
type tokenCacheEntry struct {
token string
receivedAt time.Time
}
type tokenCache struct {
node *Node
tokens map[string]tokenCacheEntry
expiration time.Duration
lock *sync.RWMutex
}
func newTokenCache(node *Node, expiration time.Duration) *tokenCache {
tc := &tokenCache{}
tc.node = node
tc.tokens = make(map[string]tokenCacheEntry)
tc.expiration = expiration
tc.lock = &sync.RWMutex{}
return tc
}
// TODO: if store fails, get new token. can happen if a node restarts but we have the token cached
func (tc *tokenCache) Get(c Contact, hash bits.Bitmap, cancelCh stop.Chan) string {
tc.lock.RLock()
token, exists := tc.tokens[c.String()]
tc.lock.RUnlock()
if exists && time.Since(token.receivedAt) < tc.expiration {
return token.token
}
resCh := tc.node.SendAsync(c, Request{
Method: findValueMethod,
Arg: &hash,
})
var res *Response
select {
case res = <-resCh:
case <-cancelCh:
return ""
}
if res == nil {
return ""
}
tc.lock.Lock()
tc.tokens[c.String()] = tokenCacheEntry{
token: res.Token,
receivedAt: time.Now(),
}
tc.lock.Unlock()
return res.Token
}
-78
View File
@@ -1,78 +0,0 @@
package dht
import (
"bytes"
"crypto/rand"
"crypto/sha256"
"net"
"strconv"
"sync"
"time"
"github.com/lbryio/lbry.go/stop"
"github.com/lbryio/reflector.go/dht/bits"
)
type tokenManager struct {
secret []byte
prevSecret []byte
lock *sync.RWMutex
stop *stop.Group
}
func (tm *tokenManager) Start(interval time.Duration) {
tm.secret = make([]byte, 64)
tm.prevSecret = make([]byte, 64)
tm.lock = &sync.RWMutex{}
tm.stop = stop.New()
tm.rotateSecret()
tm.stop.Add(1)
go func() {
defer tm.stop.Done()
tick := time.NewTicker(interval)
for {
select {
case <-tick.C:
tm.rotateSecret()
case <-tm.stop.Ch():
return
}
}
}()
}
func (tm *tokenManager) Stop() {
tm.stop.StopAndWait()
}
func (tm *tokenManager) Get(nodeID bits.Bitmap, addr *net.UDPAddr) string {
return genToken(tm.secret, nodeID, addr)
}
func (tm *tokenManager) Verify(token string, nodeID bits.Bitmap, addr *net.UDPAddr) bool {
return token == genToken(tm.secret, nodeID, addr) || token == genToken(tm.prevSecret, nodeID, addr)
}
func genToken(secret []byte, nodeID bits.Bitmap, addr *net.UDPAddr) string {
buf := bytes.Buffer{}
buf.Write(nodeID[:])
buf.Write(addr.IP)
buf.WriteString(strconv.Itoa(addr.Port))
buf.Write(secret)
t := sha256.Sum256(buf.Bytes())
return string(t[:])
}
func (tm *tokenManager) rotateSecret() {
tm.lock.Lock()
defer tm.lock.Unlock()
copy(tm.prevSecret, tm.secret)
_, err := rand.Read(tm.secret)
if err != nil {
panic(err)
}
}
+67
View File
@@ -0,0 +1,67 @@
module github.com/lbryio/reflector.go
require (
cloud.google.com/go v0.34.0 // indirect
github.com/armon/go-metrics v0.0.0-20180713145231-3c58d8115a78
github.com/aws/aws-sdk-go v1.16.11
github.com/btcsuite/btcd v0.0.0-20190109040709-5bda5314ca95 // indirect
github.com/btcsuite/btcutil v0.0.0-20180706230648-ab6388e0c60a
github.com/btcsuite/goleveldb v1.0.0 // indirect
github.com/davecgh/go-spew v1.1.1
github.com/go-errors/errors v1.0.1
github.com/go-ini/ini v1.41.0
github.com/go-sql-driver/mysql v0.0.0-20180719071942-99ff426eb706
github.com/golang/mock v1.2.0 // indirect
github.com/golang/protobuf v1.3.1
github.com/gorilla/context v1.1.1
github.com/gorilla/mux v1.6.2
github.com/gorilla/rpc v1.1.0
github.com/gorilla/websocket v1.4.0
github.com/hashicorp/errwrap v0.0.0-20180715044906-d6c0cd880357
github.com/hashicorp/go-immutable-radix v0.0.0-20180129170900-7f3cd4390caa
github.com/hashicorp/go-msgpack v0.0.0-20150518234257-fa3f63826f7c
github.com/hashicorp/go-multierror v0.0.0-20180717150148-3d5d8f294aa0
github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86
github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47
github.com/hashicorp/memberlist v0.1.0
github.com/hashicorp/serf v0.0.0-20180530155958-984a73625de3
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf
github.com/inconshreveable/mousetrap v1.0.0
github.com/jessevdk/go-flags v1.4.0 // indirect
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af
github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22
github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07
github.com/kkdai/bstream v0.0.0-20181106074824-b3251f7901ec // indirect
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c
github.com/lbryio/lbry.go v0.0.0-20190109223729-30c312501602
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5
github.com/miekg/dns v1.0.8
github.com/mitchellh/mapstructure v1.1.2 // indirect
github.com/nlopes/slack v0.4.0
github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6
github.com/pkg/errors v0.8.1 // indirect
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529
github.com/sebdah/goldie v0.0.0-20180424091453-8784dd1ab561
github.com/sergi/go-diff v1.0.0
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 // indirect
github.com/sirupsen/logrus v1.3.0
github.com/spf13/cast v1.3.0
github.com/spf13/cobra v0.0.0-20180722215644-7c4570c3ebeb
github.com/spf13/pflag v1.0.1
github.com/uber-go/atomic v1.3.2
golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc
golang.org/x/lint v0.0.0-20181217174547-8f45f776aaf1 // indirect
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e
golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890 // indirect
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 // indirect
golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c
golang.org/x/tools v0.0.0-20190109165630-d30e00c24034 // indirect
google.golang.org/appengine v1.4.0
google.golang.org/genproto v0.0.0-20190108161440-ae2f86662275 // indirect
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79
gopkg.in/yaml.v2 v2.2.2 // indirect
honnef.co/go/tools v0.0.0-20190109154334-5bcec433c8ea // indirect
)
+220
View File
@@ -0,0 +1,220 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
github.com/armon/go-metrics v0.0.0-20180713145231-3c58d8115a78 h1:mdRSArcFLfW0VoL34LZAKSz6LkkK4jFxVx2xYavACMg=
github.com/armon/go-metrics v0.0.0-20180713145231-3c58d8115a78/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
github.com/aws/aws-sdk-go v0.0.0-20180806205706-c0447dbaaf19 h1:2di6C9H5QJ+D5LPazymJz8s2kRd8YkbN7knV17yH1Yw=
github.com/aws/aws-sdk-go v0.0.0-20180806205706-c0447dbaaf19/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
github.com/aws/aws-sdk-go v1.16.11 h1:g/c7gJeVyHoXCxM2fddS85bPGVkBF8s2q8t3fyElegc=
github.com/aws/aws-sdk-go v1.16.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/btcsuite/btcd v0.0.0-20180531025944-86fed781132a/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ=
github.com/btcsuite/btcd v0.0.0-20190109040709-5bda5314ca95/go.mod h1:d3C0AkH6BRcvO8T0UEPu53cnw4IbV63x1bEjildYhO0=
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
github.com/btcsuite/btcutil v0.0.0-20180524032703-d4cc87b86016/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
github.com/btcsuite/btcutil v0.0.0-20180706230648-ab6388e0c60a h1:RQMUrEILyYJEoAT34XS/kLu40vC0+po/UfxrBBA4qZE=
github.com/btcsuite/btcutil v0.0.0-20180706230648-ab6388e0c60a/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg=
github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY=
github.com/btcsuite/goleveldb v1.0.0/go.mod h1:QiK9vBlgftBg6rWQIj6wFzbPfRjiykIEhBH4obrXJ/I=
github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-ini/ini v1.38.1 h1:hbtfM8emWUVo9GnXSloXYyFbXxZ+tG6sbepSStoe1FY=
github.com/go-ini/ini v1.38.1/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-ini/ini v1.38.2/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-ini/ini v1.41.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
github.com/go-sql-driver/mysql v0.0.0-20180719071942-99ff426eb706 h1:P3NPKb7qq581SeMCB+dU1SuCX1kQh8VoQ/4HmT2ftQY=
github.com/go-sql-driver/mysql v0.0.0-20180719071942-99ff426eb706/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
github.com/golang/lint v0.0.0-20181217174547-8f45f776aaf1/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk=
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/rpc v1.1.0 h1:marKfvVP0Gpd/jHlVBKCQ8RAoUPdX7K1Nuh6l1BNh7A=
github.com/gorilla/rpc v1.1.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ=
github.com/gorilla/websocket v1.2.0 h1:VJtLvh6VQym50czpZzx07z/kw9EgAxI3x1ZB8taTMQQ=
github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/hashicorp/errwrap v0.0.0-20180715044906-d6c0cd880357 h1:Rem2+U35z1QtPQc6r+WolF7yXiefXqDKyk+lN2pE164=
github.com/hashicorp/errwrap v0.0.0-20180715044906-d6c0cd880357/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-immutable-radix v0.0.0-20180129170900-7f3cd4390caa h1:0nA8i+6Rwqaq9xlpmVxxTwk6rxiEhX+E6Wh4vPNHiS8=
github.com/hashicorp/go-immutable-radix v0.0.0-20180129170900-7f3cd4390caa/go.mod h1:6ij3Z20p+OhOkCSrA0gImAWoHYQRGbnlcuk6XYTiaRw=
github.com/hashicorp/go-msgpack v0.0.0-20150518234257-fa3f63826f7c h1:BTAbnbegUIMB6xmQCwWE8yRzbA4XSpnZY5hvRJC188I=
github.com/hashicorp/go-msgpack v0.0.0-20150518234257-fa3f63826f7c/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
github.com/hashicorp/go-multierror v0.0.0-20180717150148-3d5d8f294aa0 h1:j30noezaCfvNLcdMYSvHLv81DxYRSt1grlpseG67vhU=
github.com/hashicorp/go-multierror v0.0.0-20180717150148-3d5d8f294aa0/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=
github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86 h1:7YOlAIO2YWnJZkQp7B5eFykaIY7C9JndqAFQyVV5BhM=
github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47 h1:UnszMmmmm5vLwWzDjTFVIkfhvWF1NdrmChl8L2NUDCw=
github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/memberlist v0.1.0 h1:qSsCiC0WYD39lbSitKNt40e30uorm2Ss/d4JGU1hzH8=
github.com/hashicorp/memberlist v0.1.0/go.mod h1:ncdBp14cuox2iFOq3kDiquKU6fqsTBc3W6JvZwjxxsE=
github.com/hashicorp/serf v0.0.0-20180530155958-984a73625de3 h1:NUr1hG6WO9sI1x8ofSimmpqfJ+rEHiHP/PLEA33rcfQ=
github.com/hashicorp/serf v0.0.0-20180530155958-984a73625de3/go.mod h1:h/Ru6tmZazX7WO/GDmwdpS975F019L4t5ng5IgwbNrE=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf h1:WfD7VjIE6z8dIvMsI4/s+1qr5EL+zoIGev1BQj1eoJ8=
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h1:hyb9oH7vZsitZCiBt0ZvifOrB+qc8PS5IiilCIb87rg=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Aowv3l/EQUlocDHW2Cp4G9WJVH7uyH8QFJE=
github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22 h1:jKUP9TQ0c7X3w6+IPyMit07RE42MtTWNd77sN2cHngQ=
github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22/go.mod h1:u0Jo4f2dNlTJeeOywkM6bLwxq6gC3pZ9rEFHn3AhTdk=
github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07 h1:+kBG/8rjCa6vxJZbUjAiE4MQmBEBYc8nLEb51frnvBY=
github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07/go.mod h1:j1kV/8f3jowErEq4XyeypkCdvg5EeHkf0YCKCcq5Ybo=
github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
github.com/kkdai/bstream v0.0.0-20181106074824-b3251f7901ec/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M=
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8=
github.com/lbryio/lbry.go v0.0.0-20180803110248-e2c96944fc48 h1:ojUrZuL8vqHxyhivNdpjDaIBkjzWktS9FOTTFkVG8yU=
github.com/lbryio/lbry.go v0.0.0-20180803110248-e2c96944fc48/go.mod h1:jyMyYNmA1t7GTkgYs6z2kMnDFTFzlGBr+IkG9LXHk9M=
github.com/lbryio/lbry.go v0.0.0-20190109223729-30c312501602 h1:HSv40ELfMPW2GIpyWwX2JaSAaYz5XW1GNweYc89F2kc=
github.com/lbryio/lbry.go v0.0.0-20190109223729-30c312501602/go.mod h1:YEuFJD/oHNra6BFy+NfuvS84Wg6RMWJFGtiCCCc6MmQ=
github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4=
github.com/lbryio/types v0.0.0-20171215152337-0a913ba650dd h1:5wQgwcaLqLMaFIPju2QHcCseSHlArxpQZ1szEGzrp6Y=
github.com/lbryio/types v0.0.0-20171215152337-0a913ba650dd/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
github.com/lbryio/types v0.0.0-20181001180206-594241d24e00 h1:1qRpd8lcyVigX+kYkwQL13gpOURyytgvxZtuIQfPPX8=
github.com/lbryio/types v0.0.0-20181001180206-594241d24e00/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c h1:m3O7561xBQ00lfUVayW4c6SnpVbUDQtPUwGcGYSUYQA=
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0=
github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg=
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 h1:mG83tLXWSRdcXMWfkoumVwhcCbf3jHF9QKv/m37BkM0=
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCWffGOaDcjkw1iB7W9DVLp6GXmfcJY/7YZCWPA4=
github.com/miekg/dns v1.0.8 h1:Zi8HNpze3NeRWH1PQV6O71YcvJRQ6j0lORO6DAEmAAI=
github.com/miekg/dns v1.0.8/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/mitchellh/mapstructure v0.0.0-20180511142126-bb74f1db0675/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/nlopes/slack v0.2.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM=
github.com/nlopes/slack v0.3.0 h1:jCxvaS8wC4Bb1jnbqZMjCDkOOgy4spvQWcrw/TF0L0E=
github.com/nlopes/slack v0.3.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM=
github.com/nlopes/slack v0.4.0 h1:OVnHm7lv5gGT5gkcHsZAyw++oHVFihbjWbL3UceUpiA=
github.com/nlopes/slack v0.4.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6 h1:2bae6N0SZjgzk+Zg8mzTsfmpwHXY9VBNp9UdjhaElA0=
github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/sebdah/goldie v0.0.0-20180424091453-8784dd1ab561 h1:IY+sDBJR/wRtsxq+626xJnt4Tw7/ROA9cDIR8MMhWyg=
github.com/sebdah/goldie v0.0.0-20180424091453-8784dd1ab561/go.mod h1:lvjGftC8oe7XPtyrOidaMi0rp5B9+XY/ZRUynGnuaxQ=
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/shopspring/decimal v0.0.0-20180607144847-19e3cb6c2930/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
github.com/sirupsen/logrus v0.0.0-20180523074243-ea8897e79973/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
github.com/sirupsen/logrus v0.0.0-20180731161355-d329d24db431 h1:tJ2phHULXperiIq6Cl3t4MLypicinjmlM3Y+lNEipuo=
github.com/sirupsen/logrus v0.0.0-20180731161355-d329d24db431/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME=
github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
github.com/spf13/cast v1.2.0 h1:HHl1DSRbEQN2i8tJmtS6ViPyHx35+p51amrdsiTCrkg=
github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.0-20180722215644-7c4570c3ebeb h1:9EsYJzSlhhaP+nYmMOcptMF2VEUH52jxPzt/TX14KWM=
github.com/spf13/cobra v0.0.0-20180722215644-7c4570c3ebeb/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4=
github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/uber-go/atomic v0.0.0-20180806045314-ca680462431f h1:vKpCeeburRE8ZXnuj9ptRqjm7WLD0O7ug28tRJuWM54=
github.com/uber-go/atomic v0.0.0-20180806045314-ca680462431f/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
github.com/uber-go/atomic v1.3.2 h1:Azu9lPBWRNKzYXSIwRfgRuDuS0YKsK4NFhiQv98gkxo=
github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20180807104621-f027049dab0a h1:PulT0Y50PcfTWomfsD39bSQyVrjjWdIuJKfyR4nOCJw=
golang.org/x/crypto v0.0.0-20180807104621-f027049dab0a/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc h1:F5tKCVGp+MUAHhKp5MZtGqAlGX3+oCsiL1Q629FL90M=
golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20181217174547-8f45f776aaf1/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180807145015-19491d39cadb h1:H5qWxL6CeEdaEGPqB5nnxJja1ULIcpuu91vIqAuGoJ0=
golang.org/x/net v0.0.0-20180807145015-19491d39cadb/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e h1:bRhVy7zSSasaqNksaRZiA5EEI+Ei4I1nO5Jh72wfHlg=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180807141123-0718ef2ef256 h1:tKa4dTsBBeG8RnHO9sDPtJYNJNOc4ilC49ePrvii4To=
golang.org/x/sys v0.0.0-20180807141123-0718ef2ef256/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb h1:1w588/yEchbPNpa9sEvOcMZYbWHedwJjg4VOAdDHWHk=
golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2IVY3KZs6p9mix0ziNYJM=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190109165630-d30e00c24034/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190108161440-ae2f86662275/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 h1:FpCr9V8wuOei4BAen+93HtVJ+XSi+KPbaPKm0Vj5R64=
gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79/go.mod h1:gWkaRU7CoXpezCBWfWjm3999QqS+1pYPXGbqQCTMzo8=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190109154334-5bcec433c8ea/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+7 -1
View File
@@ -1,7 +1,13 @@
package main
import "github.com/lbryio/reflector.go/cmd"
import (
"math/rand"
"time"
"github.com/lbryio/reflector.go/cmd"
)
func main() {
rand.Seed(time.Now().UnixNano())
cmd.Execute()
}
+17
View File
@@ -1,3 +1,20 @@
package meta
import (
"strconv"
"time"
)
var Version = ""
var Time = ""
var BuildTime time.Time
func init() {
if Time != "" {
t, err := strconv.Atoi(Time)
if err != nil {
return
}
BuildTime = time.Unix(int64(t), 0).UTC()
}
}
+135 -39
View File
@@ -2,19 +2,18 @@ package peer
import (
"bufio"
"crypto/sha512"
"encoding/hex"
"encoding/json"
"io"
"net"
"strings"
"time"
"github.com/lbryio/lbry.go/errors"
"github.com/lbryio/lbry.go/stop"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/store"
"github.com/davecgh/go-spew/spew"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/stop"
log "github.com/sirupsen/logrus"
)
@@ -27,10 +26,14 @@ const (
// Server is an instance of a peer server that houses the listener and store.
type Server struct {
StatLogger *log.Logger // logger to log stats
StatReportFrequency time.Duration // how often to log stats
store store.BlobStore
closed bool
grp *stop.Group
grp *stop.Group
stats *reflector.Stats
}
// NewServer returns an initialized Server pointer.
@@ -44,6 +47,7 @@ func NewServer(store store.BlobStore) *Server {
// Shutdown gracefully shuts down the peer server.
func (s *Server) Shutdown() {
log.Debug("shutting down peer server...")
s.stats.Shutdown()
s.grp.StopAndWait()
log.Debug("peer server stopped")
}
@@ -63,6 +67,11 @@ func (s *Server) Start(address string) error {
s.grp.Done()
}()
s.stats = reflector.NewStatLogger("DOWNLOAD", s.StatLogger, s.StatReportFrequency, s.grp.Child())
if s.StatLogger != nil && s.StatReportFrequency > 0 {
s.stats.Start()
}
return nil
}
@@ -82,7 +91,7 @@ func (s *Server) listenAndServe(listener net.Listener) {
if s.closed {
return
}
log.Error(err)
log.Error(errors.Prefix("accepting conn", err))
} else {
s.grp.Add(1)
go func() {
@@ -100,7 +109,7 @@ func (s *Server) handleConnection(conn net.Conn) {
}
}()
timeoutDuration := 5 * time.Second
timeoutDuration := 1 * time.Minute
for {
var request []byte
@@ -114,7 +123,7 @@ func (s *Server) handleConnection(conn net.Conn) {
request, err = readNextRequest(conn)
if err != nil {
if err != io.EOF {
log.Errorln(err)
s.logError(err)
}
return
}
@@ -124,33 +133,32 @@ func (s *Server) handleConnection(conn net.Conn) {
log.Error(errors.FullTrace(err))
}
if strings.Contains(string(request), `"requested_blobs"`) {
log.Debugln("received availability request")
response, err = s.handleAvailabilityRequest(request)
} else if strings.Contains(string(request), `"blob_data_payment_rate"`) {
log.Debugln("received rate negotiation request")
response, err = s.handlePaymentRateNegotiation(request)
} else if strings.Contains(string(request), `"requested_blob"`) {
log.Debugln("received blob request")
response, err = s.handleBlobRequest(request)
} else {
log.Errorln("invalid request")
spew.Dump(request)
return
}
response, err = s.handleCompositeRequest(request)
if err != nil {
log.Error(err)
return
}
err = conn.SetWriteDeadline(time.Now().Add(timeoutDuration))
if err != nil {
log.Error(errors.FullTrace(err))
}
n, err := conn.Write(response)
if err != nil {
log.Errorln(err)
if !strings.Contains(err.Error(), "connection reset by peer") { // means the other side closed the connection using TCP reset
s.logError(err)
}
return
} else if n != len(response) {
log.Errorln(io.ErrShortWrite)
return
}
err = conn.SetWriteDeadline(time.Time{})
if err != nil {
log.Error(errors.FullTrace(err))
}
}
}
@@ -197,7 +205,7 @@ func (s *Server) handleBlobRequest(data []byte) ([]byte, error) {
return []byte{}, err
}
log.Println("Sending blob " + request.RequestedBlob[:8])
log.Debugln("Sending blob " + request.RequestedBlob[:8])
blob, err := s.store.Get(request.RequestedBlob)
if err != nil {
@@ -205,7 +213,7 @@ func (s *Server) handleBlobRequest(data []byte) ([]byte, error) {
}
response, err := json.Marshal(blobResponse{IncomingBlob: incomingBlob{
BlobHash: GetBlobHash(blob),
BlobHash: reflector.BlobHash(blob),
Length: len(blob),
}})
if err != nil {
@@ -215,6 +223,91 @@ func (s *Server) handleBlobRequest(data []byte) ([]byte, error) {
return append(response, blob...), nil
}
func (s *Server) handleCompositeRequest(data []byte) ([]byte, error) {
var request compositeRequest
err := json.Unmarshal(data, &request)
if err != nil {
return []byte{}, err
}
response := compositeResponse{
LbrycrdAddress: LbrycrdAddress,
}
if len(request.RequestedBlobs) > 0 {
var availableBlobs []string
for _, blobHash := range request.RequestedBlobs {
exists, err := s.store.Has(blobHash)
if err != nil {
return []byte{}, err
}
if exists {
availableBlobs = append(availableBlobs, blobHash)
}
}
response.AvailableBlobs = availableBlobs
}
response.BlobDataPaymentRate = paymentRateAccepted
if request.BlobDataPaymentRate < 0 {
response.BlobDataPaymentRate = paymentRateTooLow
}
var blob []byte
if request.RequestedBlob != "" {
log.Debugln("Sending blob " + request.RequestedBlob[:8])
blob, err = s.store.Get(request.RequestedBlob)
if errors.Is(err, store.ErrBlobNotFound) {
response.IncomingBlob = incomingBlob{
Error: err.Error(),
}
} else if err != nil {
return []byte{}, err
} else {
response.IncomingBlob = incomingBlob{
BlobHash: reflector.BlobHash(blob),
Length: len(blob),
}
s.stats.AddBlob()
}
}
respData, err := json.Marshal(response)
if err != nil {
return []byte{}, err
}
return append(respData, blob...), nil
}
func (s *Server) logError(e error) {
if e == nil {
return
}
shouldLog := s.stats.AddError(e)
if shouldLog {
log.Errorln(errors.FullTrace(e))
}
return
// old stuff below. its here for posterity, because we're gonna have to deal with these errors someday for real
//err := errors.Wrap(e, 0)
// these happen because the peer protocol does not have a way to cancel blob downloads
// so the client will just close the connection if its in the middle of downloading a blob
// but receives the blob from a different peer first or simply goes offline (timeout)
//if strings.Contains(err.Error(), "connection reset by peer") ||
// strings.Contains(err.Error(), "i/o timeout") ||
// strings.Contains(err.Error(), "broken pipe") {
// return
//}
//
//log.Error(errors.FullTrace(e))
}
func readNextRequest(conn net.Conn) ([]byte, error) {
request := make([]byte, 0)
eof := false
@@ -224,7 +317,7 @@ func readNextRequest(conn net.Conn) ([]byte, error) {
chunk, err := buf.ReadBytes('}')
if err != nil {
if err != io.EOF {
log.Errorln("read error:", err)
//log.Errorln("readBytes error:", err) // logged by caller
return request, err
}
eof = true
@@ -241,7 +334,7 @@ func readNextRequest(conn net.Conn) ([]byte, error) {
}
// yes, this is how the peer protocol knows when the request finishes
if isValidJSON(request) {
if reflector.IsValidJSON(request) {
break
}
}
@@ -263,17 +356,6 @@ func readNextRequest(conn net.Conn) ([]byte, error) {
return request, nil
}
func isValidJSON(b []byte) bool {
var r json.RawMessage
return json.Unmarshal(b, &r) == nil
}
// GetBlobHash returns the sha512 hash hex encoded string of the blob byte slice.
func GetBlobHash(blob []byte) string {
hashBytes := sha512.Sum384(blob)
return hex.EncodeToString(hashBytes[:])
}
const (
maxRequestSize = 64 * (2 ^ 10) // 64kb
paymentRateAccepted = "RATE_ACCEPTED"
@@ -314,3 +396,17 @@ type incomingBlob struct {
type blobResponse struct {
IncomingBlob incomingBlob `json:"incoming_blob"`
}
type compositeRequest struct {
LbrycrdAddress bool `json:"lbrycrd_address"`
RequestedBlobs []string `json:"requested_blobs"`
BlobDataPaymentRate float64 `json:"blob_data_payment_rate"`
RequestedBlob string `json:"requested_blob"`
}
type compositeResponse struct {
LbrycrdAddress string `json:"lbrycrd_address,omitempty"`
AvailableBlobs []string `json:"available_blobs,omitempty"`
BlobDataPaymentRate string `json:"blob_data_payment_rate,omitempty"`
IncomingBlob incomingBlob `json:"incoming_blob,omitempty"`
}
+4 -4
View File
@@ -5,16 +5,16 @@ import (
"strconv"
"sync"
"github.com/lbryio/lbry.go/dht"
"github.com/lbryio/lbry.go/dht/bits"
"github.com/lbryio/reflector.go/cluster"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/dht"
"github.com/lbryio/reflector.go/dht/bits"
"github.com/lbryio/reflector.go/peer"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/errors"
"github.com/lbryio/lbry.go/stop"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/stop"
log "github.com/sirupsen/logrus"
)
+1 -1
View File
@@ -5,7 +5,7 @@ import (
"testing"
"github.com/davecgh/go-spew/spew"
"github.com/lbryio/reflector.go/dht/bits"
"github.com/lbryio/lbry.go/dht/bits"
)
func TestAnnounceRange(t *testing.T) {
+3 -3
View File
@@ -19,7 +19,7 @@ This project requires [Go v1.10](https://golang.org/doc/install) or higher.
go get -u github.com/lbryio/reflector.go
cd "$(go env GOPATH)/src/github.com/lbryio/reflector.go"
make
./prism
./bin/prism-bin
```
## Contributing
@@ -32,9 +32,9 @@ This project is MIT licensed.
## Security
We take security seriously. Please contact security@lbry.io regarding any security issues.
We take security seriously. Please contact security@lbry.com regarding any security issues.
Our PGP key is [here](https://keybase.io/lbry/key.asc) if you need it.
## Contact
The primary contact for this project is [@lyoshenka](https://github.com/lyoshenka) (grin@lbry.io)
The primary contact for this project is [@lyoshenka](https://github.com/lyoshenka) (grin@lbry.com)
+173
View File
@@ -0,0 +1,173 @@
package reflector
import (
"encoding/hex"
"encoding/json"
"net/http"
"strconv"
"strings"
"time"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/reflector.go/wallet"
"github.com/lbryio/lbry.go/extras/errors"
types1 "github.com/lbryio/types/v1/go"
types2 "github.com/lbryio/types/v2/go"
"github.com/golang/protobuf/proto"
log "github.com/sirupsen/logrus"
)
const blocklistURL = "https://api.lbry.io/file/list_blocked"
func (s *Server) enableBlocklist(b store.Blocklister) {
// TODO: updateBlocklist should be killed when server is shutting down
updateBlocklist(b)
t := time.NewTicker(12 * time.Hour)
for {
select {
case <-s.grp.Ch():
return
case <-t.C:
updateBlocklist(b)
}
}
}
func updateBlocklist(b store.Blocklister) {
values, err := blockedSdHashes()
if err != nil {
log.Error(err)
return
}
for _, v := range values {
if v.Err != nil {
continue
}
err = b.Block(v.Value)
if err != nil {
log.Error(err)
}
}
}
func blockedSdHashes() (map[string]valOrErr, error) {
resp, err := http.Get(blocklistURL)
if err != nil {
return nil, errors.Err(err)
}
defer func() {
err := resp.Body.Close()
if err != nil {
log.Errorln(err)
}
}()
var r struct {
Success bool `json:"success"`
Error string `json:"error"`
Data struct {
Outpoints []string `json:"outpoints"`
} `json:"data"`
}
if err = json.NewDecoder(resp.Body).Decode(&r); err != nil {
return nil, errors.Err(err)
}
if !r.Success {
return nil, errors.Prefix("list_blocked API call", r.Error)
}
return sdHashesForOutpoints(r.Data.Outpoints)
}
type valOrErr struct {
Value string
Err error
}
// sdHashesForOutpoints queries wallet server for the sd hashes in a given outpoints
func sdHashesForOutpoints(outpoints []string) (map[string]valOrErr, error) {
values := make(map[string]valOrErr)
node := wallet.NewNode()
defer node.Shutdown()
err := node.Connect([]string{
"lbryumx1.lbry.io:50001",
"lbryumx2.lbry.io:50001",
}, nil)
if err != nil {
return nil, err
}
for _, outpoint := range outpoints {
parts := strings.Split(outpoint, ":")
if len(parts) != 2 {
values[outpoint] = valOrErr{Err: errors.Err("invalid outpoint format")}
continue
}
nout, err := strconv.Atoi(parts[1])
if err != nil {
values[outpoint] = valOrErr{Err: errors.Prefix("invalid nout", err)}
continue
}
resp, err := node.GetClaimsInTx(parts[0])
if err != nil {
values[outpoint] = valOrErr{Err: err}
continue
}
var value []byte
for _, tx := range resp.Result {
if tx.Nout != nout {
continue
}
value, err = hex.DecodeString(tx.Value)
break
}
if err != nil {
values[outpoint] = valOrErr{Err: err}
continue
} else if value == nil {
values[outpoint] = valOrErr{Err: errors.Err("outpoint not found")}
continue
}
hash, err := hashFromClaim(value)
values[outpoint] = valOrErr{Value: hash, Err: err}
}
return values, nil
}
func hashFromClaim(value []byte) (string, error) {
claim := &types1.Claim{}
err := proto.Unmarshal(value, claim)
if err != nil {
return "", err
}
if claim.GetStream().GetSource().GetSourceType() == types1.Source_lbry_sd_hash && claim.GetStream().GetSource().GetSource() != nil {
return hex.EncodeToString(claim.GetStream().GetSource().GetSource()), nil
}
claim2 := &types2.Claim{}
err = proto.Unmarshal(value, claim2)
if err != nil {
return "", err
}
stream, ok := claim2.GetType().(*types2.Claim_Stream)
if !ok || stream == nil {
return "", errors.Err("not a stream claim")
}
return hex.EncodeToString(claim2.GetStream().GetSource().GetSdHash()), nil
}
+13 -12
View File
@@ -3,9 +3,10 @@ package reflector
import (
"encoding/json"
"net"
"strconv"
"github.com/lbryio/lbry.go/errors"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/stream"
log "github.com/sirupsen/logrus"
)
@@ -36,20 +37,18 @@ func (c *Client) Close() error {
}
// SendBlob sends a send blob request to the client.
func (c *Client) SendBlob(blob []byte) error {
func (c *Client) SendBlob(blob stream.Blob) error {
if !c.connected {
return errors.Err("not connected")
}
if len(blob) > maxBlobSize {
return errors.Err("blob must be at most " + strconv.Itoa(maxBlobSize) + " bytes")
} else if len(blob) == 0 {
return errors.Err("blob is empty")
if err := blob.ValidForSend(); err != nil {
return errors.Err(err)
}
blobHash := getBlobHash(blob)
blobHash := blob.HashHex()
sendRequest, err := json.Marshal(sendBlobRequest{
BlobSize: len(blob),
BlobSize: blob.Size(),
BlobHash: blobHash,
})
if err != nil {
@@ -70,7 +69,7 @@ func (c *Client) SendBlob(blob []byte) error {
}
if !sendResp.SendBlob {
return ErrBlobExists
return errors.Prefix(blobHash[:8], ErrBlobExists)
}
log.Println("Sending blob " + blobHash[:8])
@@ -97,7 +96,7 @@ func (c *Client) doHandshake(version int) error {
return errors.Err("not connected")
}
handshake, err := json.Marshal(handshakeRequestResponse{Version: version})
handshake, err := json.Marshal(handshakeRequestResponse{Version: &version})
if err != nil {
return err
}
@@ -111,7 +110,9 @@ func (c *Client) doHandshake(version int) error {
err = json.NewDecoder(c.conn).Decode(&resp)
if err != nil {
return err
} else if resp.Version != version {
} else if resp.Version == nil {
return errors.Err("invalid handshake")
} else if *resp.Version != version {
return errors.Err("handshake version mismatch")
}
+124 -47
View File
@@ -6,14 +6,16 @@ import (
"encoding/hex"
"encoding/json"
"io"
"io/ioutil"
"net"
"strconv"
"time"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/errors"
"github.com/lbryio/lbry.go/stop"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/stop"
"github.com/lbryio/lbry.go/stream"
log "github.com/sirupsen/logrus"
)
@@ -27,28 +29,36 @@ const (
network = "tcp4"
protocolVersion1 = 0
protocolVersion2 = 1
maxBlobSize = 2 * 1024 * 1024
maxBlobSize = stream.MaxBlobSize
)
// Server is and instance of the reflector server. It houses the blob store and listener.
type Server struct {
store store.BlobStore
timeout time.Duration // timeout to read or write next message
grp *stop.Group
Timeout time.Duration // timeout to read or write next message
StatLogger *log.Logger // logger to log stats
StatReportFrequency time.Duration // how often to log stats
EnableBlocklist bool // if true, blocklist checking and blob deletion will be enabled
store store.BlobStore
grp *stop.Group
stats *Stats
}
// NewServer returns an initialized reflector server pointer.
func NewServer(store store.BlobStore) *Server {
return &Server{
Timeout: DefaultTimeout,
store: store,
grp: stop.New(),
timeout: DefaultTimeout,
}
}
// Shutdown shuts down the reflector server gracefully.
func (s *Server) Shutdown() {
log.Println("shutting down reflector server...")
s.stats.Shutdown()
s.grp.StopAndWait()
log.Println("reflector server stopped")
}
@@ -77,6 +87,24 @@ func (s *Server) Start(address string) error {
s.grp.Done()
}()
s.stats = NewStatLogger("UPLOAD", s.StatLogger, s.StatReportFrequency, s.grp.Child())
if s.StatLogger != nil && s.StatReportFrequency > 0 {
s.stats.Start()
}
if s.EnableBlocklist {
if b, ok := s.store.(store.Blocklister); ok {
s.grp.Add(1)
go func() {
s.enableBlocklist(b)
s.grp.Done()
}()
} else {
//s.Shutdown()
return errors.Err("blocklist is enabled but blob store does not support blocklisting")
}
}
return nil
}
@@ -119,7 +147,7 @@ func (s *Server) handleConn(conn net.Conn) {
err := s.doHandshake(conn)
if err != nil {
if err == io.EOF || s.quitting() {
if errors.Is(err, io.EOF) || s.quitting() {
return
}
err := s.doError(conn, err)
@@ -132,7 +160,7 @@ func (s *Server) handleConn(conn net.Conn) {
for {
err = s.receiveBlob(conn)
if err != nil {
if err == io.EOF || s.quitting() {
if errors.Is(err, io.EOF) || s.quitting() {
return
}
err := s.doError(conn, err)
@@ -145,57 +173,80 @@ func (s *Server) handleConn(conn net.Conn) {
}
func (s *Server) doError(conn net.Conn, err error) error {
log.Errorln(errors.FullTrace(err))
shouldLog := s.stats.AddError(err)
if shouldLog {
log.Errorln(errors.FullTrace(err))
}
if e2, ok := err.(*json.SyntaxError); ok {
log.Errorf("syntax error at byte offset %d", e2.Offset)
}
resp, err := json.Marshal(errorResponse{Error: err.Error()})
if err != nil {
return err
}
return s.write(conn, resp)
//resp, err := json.Marshal(errorResponse{Error: err.Error()})
//if err != nil {
// return err
//}
//return s.write(conn, resp)
return nil
}
func (s *Server) receiveBlob(conn net.Conn) error {
var err error
blobSize, blobHash, isSdBlob, err := s.readBlobRequest(conn)
if err != nil {
return err
}
// fullStreamChecker can check if the full stream has been uploaded
type fullStreamChecker interface {
HasFullStream(string) (bool, error)
}
blobExists := false
if fsc, ok := s.store.(fullStreamChecker); ok && isSdBlob {
blobExists, err = fsc.HasFullStream(blobHash)
var wantsBlob bool
if bl, ok := s.store.(store.Blocklister); ok {
wantsBlob, err = bl.Wants(blobHash)
if err != nil {
return err
}
} else {
// if we can't confirm that we have the full stream, we have to say that the sd blob is missing. if we say we have it, they wont try to send any content blobs
blobExists, err = s.store.Has(blobHash)
blobExists, err := s.store.Has(blobHash)
if err != nil {
return err
}
wantsBlob = !blobExists
}
var neededBlobs []string
if isSdBlob && !wantsBlob {
if nbc, ok := s.store.(neededBlobChecker); ok {
neededBlobs, err = nbc.MissingBlobsForKnownStream(blobHash)
if err != nil {
return err
}
} else {
// if we can't check for blobs in a stream, we have to say that the sd blob is
// missing. if we say we have the sd blob, they wont try to send any content blobs
wantsBlob = true
}
}
err = s.sendBlobResponse(conn, wantsBlob, isSdBlob, neededBlobs)
if err != nil {
return err
}
err = s.sendBlobResponse(conn, blobExists, isSdBlob)
if err != nil {
return err
}
if blobExists {
if !wantsBlob {
return nil
}
blob, err := s.readRawBlob(conn, blobSize)
if err != nil {
return err
sendErr := s.sendTransferResponse(conn, false, isSdBlob)
if sendErr != nil {
return sendErr
}
return errors.Prefix("error reading blob "+blobHash[:8], err)
}
receivedBlobHash := getBlobHash(blob)
receivedBlobHash := BlobHash(blob)
if blobHash != receivedBlobHash {
sendErr := s.sendTransferResponse(conn, false, isSdBlob)
if sendErr != nil {
return sendErr
}
return errors.Err("hash of received blob data does not match hash from send request")
// this can also happen if the blob size is wrong, because the server will read the wrong number of bytes from the stream
}
@@ -211,6 +262,10 @@ func (s *Server) receiveBlob(conn net.Conn) error {
return err
}
s.stats.AddBlob()
if isSdBlob {
s.stats.AddStream()
}
return s.sendTransferResponse(conn, true, isSdBlob)
}
@@ -219,7 +274,9 @@ func (s *Server) doHandshake(conn net.Conn) error {
err := s.read(conn, &handshake)
if err != nil {
return err
} else if handshake.Version != protocolVersion1 && handshake.Version != protocolVersion2 {
} else if handshake.Version == nil {
return errors.Err("handshake is missing protocol version")
} else if *handshake.Version != protocolVersion1 && *handshake.Version != protocolVersion2 {
return errors.Err("protocol version not supported")
}
@@ -263,14 +320,14 @@ func (s *Server) readBlobRequest(conn net.Conn) (int, string, bool, error) {
return blobSize, blobHash, isSdBlob, nil
}
func (s *Server) sendBlobResponse(conn net.Conn, blobExists, isSdBlob bool) error {
func (s *Server) sendBlobResponse(conn net.Conn, shouldSendBlob, isSdBlob bool, neededBlobs []string) error {
var response []byte
var err error
if isSdBlob {
response, err = json.Marshal(sendSdBlobResponse{SendSdBlob: !blobExists})
response, err = json.Marshal(sendSdBlobResponse{SendSdBlob: shouldSendBlob, NeededBlobs: neededBlobs})
} else {
response, err = json.Marshal(sendBlobResponse{SendBlob: !blobExists})
response, err = json.Marshal(sendBlobResponse{SendBlob: shouldSendBlob})
}
if err != nil {
return err
@@ -296,16 +353,25 @@ func (s *Server) sendTransferResponse(conn net.Conn, receivedBlob, isSdBlob bool
}
func (s *Server) read(conn net.Conn, v interface{}) error {
err := conn.SetReadDeadline(time.Now().Add(s.timeout))
err := conn.SetReadDeadline(time.Now().Add(s.Timeout))
if err != nil {
return errors.Err(err)
}
return errors.Err(json.NewDecoder(conn).Decode(v))
dec := json.NewDecoder(conn)
err = dec.Decode(v)
if err != nil {
data, _ := ioutil.ReadAll(dec.Buffered())
if len(data) > 0 {
return errors.Err("%s. Data: %s", err.Error(), hex.EncodeToString(data))
}
return errors.Err(err)
}
return nil
}
func (s *Server) readRawBlob(conn net.Conn, blobSize int) ([]byte, error) {
err := conn.SetReadDeadline(time.Now().Add(s.timeout))
err := conn.SetReadDeadline(time.Now().Add(s.Timeout))
if err != nil {
return nil, errors.Err(err)
}
@@ -316,7 +382,7 @@ func (s *Server) readRawBlob(conn net.Conn, blobSize int) ([]byte, error) {
}
func (s *Server) write(conn net.Conn, b []byte) error {
err := conn.SetWriteDeadline(time.Now().Add(s.timeout))
err := conn.SetWriteDeadline(time.Now().Add(s.Timeout))
if err != nil {
return errors.Err(err)
}
@@ -337,17 +403,23 @@ func (s *Server) quitting() bool {
}
}
func getBlobHash(blob []byte) string {
// BlobHash returns the sha512 hash hex encoded string of the blob byte slice.
func BlobHash(blob []byte) string {
hashBytes := sha512.Sum384(blob)
return hex.EncodeToString(hashBytes[:])
}
type errorResponse struct {
Error string `json:"error"`
func IsValidJSON(b []byte) bool {
var r json.RawMessage
return json.Unmarshal(b, &r) == nil
}
//type errorResponse struct {
// Error string `json:"error"`
//}
type handshakeRequestResponse struct {
Version int `json:"version"`
Version *int `json:"version"`
}
type sendBlobRequest struct {
@@ -373,3 +445,8 @@ type blobTransferResponse struct {
type sdBlobTransferResponse struct {
ReceivedSdBlob bool `json:"received_sd_blob"`
}
// neededBlobChecker can check which blobs from a known stream are not uploaded yet
type neededBlobChecker interface {
MissingBlobsForKnownStream(string) ([]string, error)
}
+117 -22
View File
@@ -2,14 +2,17 @@ package reflector
import (
"crypto/rand"
"encoding/json"
"io"
"sort"
"strconv"
"testing"
"time"
"github.com/davecgh/go-spew/spew"
"github.com/lbryio/lbry.go/dht/bits"
"github.com/lbryio/reflector.go/store"
"github.com/davecgh/go-spew/spew"
"github.com/phayes/freeport"
)
@@ -62,11 +65,7 @@ func TestServer_MediumBlob(t *testing.T) {
t.Fatal("error connecting client to server", err)
}
blob := make([]byte, 1000)
_, err = rand.Read(blob)
if err != nil {
t.Fatal("failed to make random blob")
}
blob := randBlob(1000)
err = c.SendBlob(blob)
if err != nil {
@@ -84,11 +83,7 @@ func TestServer_FullBlob(t *testing.T) {
t.Fatal("error connecting client to server", err)
}
blob := make([]byte, maxBlobSize)
_, err = rand.Read(blob)
if err != nil {
t.Fatal("failed to make random blob")
}
blob := randBlob(maxBlobSize)
err = c.SendBlob(blob)
if err != nil {
@@ -106,11 +101,7 @@ func TestServer_TooBigBlob(t *testing.T) {
t.Fatal("error connecting client to server", err)
}
blob := make([]byte, maxBlobSize+1)
_, err = rand.Read(blob)
if err != nil {
t.Fatal("failed to make random blob")
}
blob := randBlob(maxBlobSize + 1)
err = c.SendBlob(blob)
if err == nil {
@@ -129,7 +120,7 @@ func TestServer_Timeout(t *testing.T) {
}
srv := NewServer(&store.MemoryBlobStore{})
srv.timeout = testTimeout
srv.Timeout = testTimeout
err = srv.Start("127.0.0.1:" + strconv.Itoa(port))
if err != nil {
t.Fatal(err)
@@ -144,11 +135,7 @@ func TestServer_Timeout(t *testing.T) {
time.Sleep(testTimeout * 2)
blob := make([]byte, 10)
_, err = rand.Read(blob)
if err != nil {
t.Fatal("failed to make random blob")
}
blob := randBlob(10)
err = c.SendBlob(blob)
t.Log(spew.Sdump(err))
@@ -156,3 +143,111 @@ func TestServer_Timeout(t *testing.T) {
t.Error("server should have timed out by now")
}
}
//func TestServer_InvalidJSONHandshake(t *testing.T) {
// srv, port := startServerOnRandomPort(t)
// defer srv.Shutdown()
//
// c := Client{}
// err := c.Connect(":" + strconv.Itoa(port))
// if err != nil {
// t.Fatal("error connecting client to server", err)
// }
//
// _, err = c.conn.Write([]byte(`{"stuff":4,tf}"`))
// if err == nil {
// t.Error("expected an error")
// }
//}
type mockPartialStore struct {
store.MemoryBlobStore
missing []string
}
func (m mockPartialStore) MissingBlobsForKnownStream(hash string) ([]string, error) {
return m.missing, nil
}
func TestServer_PartialUpload(t *testing.T) {
port, err := freeport.GetFreePort()
if err != nil {
t.Fatal(err)
}
sdHash := bits.Rand().String()
missing := make([]string, 4)
for i := range missing {
missing[i] = bits.Rand().String()
}
st := store.BlobStore(&mockPartialStore{missing: missing})
if _, ok := st.(neededBlobChecker); !ok {
t.Fatal("mock does not implement the relevant interface")
}
err = st.Put(sdHash, randBlob(10))
if err != nil {
t.Fatal(err)
}
srv := NewServer(st)
err = srv.Start("127.0.0.1:" + strconv.Itoa(port))
if err != nil {
t.Fatal(err)
}
defer srv.Shutdown()
c := Client{}
err = c.Connect(":" + strconv.Itoa(port))
if err != nil {
t.Fatal("error connecting client to server", err)
}
sendRequest, err := json.Marshal(sendBlobRequest{
SdBlobHash: sdHash,
SdBlobSize: len(sdHash),
})
if err != nil {
t.Fatal(err)
}
_, err = c.conn.Write(sendRequest)
if err != nil {
t.Fatal(err)
}
var sendResp sendSdBlobResponse
err = json.NewDecoder(c.conn).Decode(&sendResp)
if err != nil {
t.Fatal(err)
}
if sendResp.SendSdBlob {
t.Errorf("expected SendSdBlob = false, got true")
}
if len(sendResp.NeededBlobs) != len(missing) {
t.Fatalf("got %d needed blobs, expected %d", len(sendResp.NeededBlobs), len(missing))
}
sort.Strings(sendResp.NeededBlobs)
sort.Strings(missing)
for i := range missing {
if missing[i] != sendResp.NeededBlobs[i] {
t.Errorf("needed blobs mismatch: %s != %s", missing[i], sendResp.NeededBlobs[i])
}
}
}
func randBlob(size int) []byte {
//if size > maxBlobSize {
// panic("blob size too big")
//}
blob := make([]byte, size)
_, err := rand.Read(blob)
if err != nil {
panic("failed to make random blob")
}
return blob
}
+122
View File
@@ -0,0 +1,122 @@
package reflector
import (
"fmt"
"strings"
"sync"
"time"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/stop"
log "github.com/sirupsen/logrus"
)
// TODO: store daily stats too. and maybe other intervals
type Stats struct {
mu *sync.Mutex
blobs int
streams int
errors map[string]int
started bool
name string
logger *log.Logger
logFreq time.Duration
grp *stop.Group
}
func NewStatLogger(name string, logger *log.Logger, logFreq time.Duration, parentGrp *stop.Group) *Stats {
return &Stats{
mu: &sync.Mutex{},
grp: stop.New(parentGrp),
logger: logger,
logFreq: logFreq,
errors: make(map[string]int),
name: name,
}
}
func (s *Stats) Start() {
s.started = true
s.grp.Add(1)
go func() {
defer s.grp.Done()
s.runSlackLogger()
}()
}
func (s *Stats) Shutdown() {
if !s.started {
return
}
s.log()
s.grp.StopAndWait()
s.started = false
}
func (s *Stats) AddBlob() {
s.mu.Lock()
defer s.mu.Unlock()
s.blobs++
}
func (s *Stats) AddStream() {
s.mu.Lock()
defer s.mu.Unlock()
s.streams++
}
func (s *Stats) AddError(e error) (shouldLog bool) { // shouldLog is a hack, but whatever
if e == nil {
return
}
err := errors.Wrap(e, 0)
name := err.TypeName()
if strings.Contains(err.Error(), "i/o timeout") { // hit a read or write deadline
name = "i/o timeout"
} else if strings.Contains(err.Error(), "read: connection reset by peer") { // the other side closed the connection using TCP reset
name = "read conn reset"
} else if strings.Contains(err.Error(), "unexpected EOF") { // tried to read from closed pipe or socket
name = "unexpected EOF"
} else if strings.Contains(err.Error(), "write: broken pipe") { // tried to write to a pipe or socket that was closed by the peer
name = "write broken pipe"
} else {
shouldLog = true
}
s.mu.Lock()
defer s.mu.Unlock()
s.errors[name]++
return
}
func (s *Stats) runSlackLogger() {
t := time.NewTicker(s.logFreq)
for {
select {
case <-s.grp.Ch():
return
case <-t.C:
s.log()
}
}
}
func (s *Stats) log() {
s.mu.Lock()
blobs, streams := s.blobs, s.streams
s.blobs, s.streams = 0, 0
errStr := ""
for name, count := range s.errors {
errStr += fmt.Sprintf("%d %s, ", count, name)
delete(s.errors, name)
}
s.mu.Unlock()
if len(errStr) > 2 {
errStr = errStr[:len(errStr)-2] // trim last comma and space
}
s.logger.Printf("%s stats: %d blobs, %d streams, errors: %s", s.name, blobs, streams, errStr)
}
+256
View File
@@ -0,0 +1,256 @@
package reflector
import (
"io/ioutil"
"os"
"path"
"sync"
"time"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/stop"
log "github.com/sirupsen/logrus"
)
type increment int
const (
sdInc increment = iota + 1
blobInc
errInc
)
type Uploader struct {
db *db.SQL
store *store.DBBackedS3Store // could just be store.BlobStore interface
workers int
skipExistsCheck bool
stopper *stop.Group
countChan chan increment
count struct {
total, alreadyStored, sd, blob, err int
}
}
func NewUploader(db *db.SQL, store *store.DBBackedS3Store, workers int, skipExistsCheck bool) *Uploader {
return &Uploader{
db: db,
store: store,
workers: workers,
skipExistsCheck: skipExistsCheck,
stopper: stop.New(),
countChan: make(chan increment),
}
}
func (u *Uploader) Stop() {
log.Infoln("stopping uploader")
u.stopper.StopAndWait()
}
func (u *Uploader) Upload(dirOrFilePath string) error {
paths, err := getPaths(dirOrFilePath)
if err != nil {
return err
}
u.count.total = len(paths)
hashes := make([]string, len(paths))
for i, p := range paths {
hashes[i] = path.Base(p)
}
log.Infoln("checking for existing blobs")
var exists map[string]bool
if !u.skipExistsCheck {
exists, err = u.db.HasBlobs(hashes)
if err != nil {
return err
}
u.count.alreadyStored = len(exists)
}
log.Infof("%d new blobs to upload", u.count.total-u.count.alreadyStored)
workerWG := sync.WaitGroup{}
pathChan := make(chan string)
for i := 0; i < u.workers; i++ {
workerWG.Add(1)
go func(i int) {
defer workerWG.Done()
defer func(i int) { log.Debugf("worker %d quitting", i) }(i)
u.worker(pathChan)
}(i)
}
countWG := sync.WaitGroup{}
countWG.Add(1)
go func() {
defer countWG.Done()
u.counter()
}()
Upload:
for _, f := range paths {
if exists != nil && exists[path.Base(f)] {
continue
}
select {
case pathChan <- f:
case <-u.stopper.Ch():
break Upload
}
}
close(pathChan)
workerWG.Wait()
close(u.countChan)
countWG.Wait()
u.stopper.Stop()
log.Infoln("SUMMARY")
log.Infof("%d blobs total", u.count.total)
log.Infof("%d blobs already stored", u.count.alreadyStored)
log.Infof("%d SD blobs uploaded", u.count.sd)
log.Infof("%d content blobs uploaded", u.count.blob)
log.Infof("%d errors encountered", u.count.err)
return nil
}
// worker reads paths from a channel and uploads them
func (u *Uploader) worker(pathChan chan string) {
for {
select {
case <-u.stopper.Ch():
return
case filepath, ok := <-pathChan:
if !ok {
return
}
err := u.uploadBlob(filepath)
if err != nil {
log.Errorln(err)
}
}
}
}
// uploadBlob uploads a blob
func (u *Uploader) uploadBlob(filepath string) (err error) {
defer func() {
if err != nil {
u.inc(errInc)
}
}()
blob, err := ioutil.ReadFile(filepath)
if err != nil {
return err
}
hash := BlobHash(blob)
if hash != path.Base(filepath) {
return errors.Err("file name does not match hash (%s != %s), skipping", filepath, hash)
}
if IsValidJSON(blob) {
log.Debugf("Uploading SD blob %s", hash)
err := u.store.PutSD(hash, blob)
if err != nil {
return errors.Prefix("Uploading SD blob "+hash, err)
}
u.inc(sdInc)
} else {
log.Debugf("Uploading blob %s", hash)
err = u.store.Put(hash, blob)
if err != nil {
return errors.Prefix("Uploading blob "+hash, err)
}
u.inc(blobInc)
}
return nil
}
// counter updates the counts of how many sd blobs and content blobs were uploaded, and how many
// errors were encountered. It occasionally prints the upload progress to debug.
func (u *Uploader) counter() {
start := time.Now()
for {
select {
case <-u.stopper.Ch():
return
case incrementType, ok := <-u.countChan:
if !ok {
return
}
switch incrementType {
case sdInc:
u.count.sd++
case blobInc:
u.count.blob++
case errInc:
u.count.err++
}
}
if (u.count.sd+u.count.blob)%50 == 0 {
log.Infof("%d of %d done (%s elapsed, %.3fs per blob)", u.count.sd+u.count.blob, u.count.total-u.count.alreadyStored, time.Since(start).String(), time.Since(start).Seconds()/float64(u.count.sd+u.count.blob))
}
}
}
func (u *Uploader) inc(t increment) {
select {
case u.countChan <- t:
case <-u.stopper.Ch():
}
}
// getPaths returns the paths for files to upload. it takes a path to a file or a dir. for a file,
// it returns the full path to that file. for a dir, it returns the paths for all the files in the
// dir
func getPaths(dirOrFilePath string) ([]string, error) {
info, err := os.Stat(dirOrFilePath)
if err != nil {
return nil, errors.Err(err)
}
if info.Mode().IsRegular() {
return []string{dirOrFilePath}, nil
}
f, err := os.Open(dirOrFilePath)
if err != nil {
return nil, errors.Err(err)
}
files, err := f.Readdir(-1)
if err != nil {
return nil, errors.Err(err)
}
err = f.Close()
if err != nil {
return nil, errors.Err(err)
}
var filenames []string
for _, file := range files {
if !file.IsDir() {
filenames = append(filenames, dirOrFilePath+"/"+file.Name())
}
}
return filenames, nil
}
+113 -8
View File
@@ -2,15 +2,20 @@ package store
import (
"encoding/json"
"sync"
"github.com/lbryio/lbry.go/errors"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/lbry.go/extras/errors"
log "github.com/sirupsen/logrus"
)
// DBBackedS3Store is an instance of an S3 Store that is backed by a DB for what is stored.
type DBBackedS3Store struct {
s3 *S3BlobStore
db *db.SQL
s3 *S3BlobStore
db *db.SQL
blockedMu sync.RWMutex
blocked map[string]bool
}
// NewDBBackedS3Store returns an initialized store pointer.
@@ -18,12 +23,12 @@ func NewDBBackedS3Store(s3 *S3BlobStore, db *db.SQL) *DBBackedS3Store {
return &DBBackedS3Store{s3: s3, db: db}
}
// Has returns T/F or Error ( if the DB errors ) if store contains the blob.
// Has returns true if the blob is in the store
func (d *DBBackedS3Store) Has(hash string) (bool, error) {
return d.db.HasBlob(hash)
}
// Get returns the byte slice of the blob or an error.
// Get gets the blob
func (d *DBBackedS3Store) Get(hash string) ([]byte, error) {
return d.s3.Get(hash)
}
@@ -58,7 +63,107 @@ func (d *DBBackedS3Store) PutSD(hash string, blob []byte) error {
return d.db.AddSDBlob(hash, len(blob), blobContents)
}
// HasFullStream checks if the full stream has been uploaded (i.e. if we have the sd blob and all the content blobs)
func (d *DBBackedS3Store) HasFullStream(sdHash string) (bool, error) {
return d.db.HasFullStream(sdHash)
func (d *DBBackedS3Store) Delete(hash string) error {
err := d.s3.Delete(hash)
if err != nil {
return err
}
return d.db.Delete(hash)
}
// Block deletes the blob and prevents it from being uploaded in the future
func (d *DBBackedS3Store) Block(hash string) error {
if blocked, err := d.isBlocked(hash); blocked || err != nil {
return err
}
log.Debugf("blocking %s", hash)
err := d.db.Block(hash)
if err != nil {
return err
}
has, err := d.db.HasBlob(hash)
if err != nil {
return err
}
if has {
err = d.s3.Delete(hash)
if err != nil {
return err
}
err = d.db.Delete(hash)
if err != nil {
return err
}
}
return d.markBlocked(hash)
}
// Wants returns false if the hash exists or is blocked, true otherwise
func (d *DBBackedS3Store) Wants(hash string) (bool, error) {
blocked, err := d.isBlocked(hash)
if blocked || err != nil {
return false, err
}
has, err := d.Has(hash)
return !has, err
}
// MissingBlobsForKnownStream returns missing blobs for an existing stream
// WARNING: if the stream does NOT exist, no blob hashes will be returned, which looks
// like no blobs are missing
func (d *DBBackedS3Store) MissingBlobsForKnownStream(sdHash string) ([]string, error) {
return d.db.MissingBlobsForKnownStream(sdHash)
}
func (d *DBBackedS3Store) markBlocked(hash string) error {
err := d.initBlocked()
if err != nil {
return err
}
d.blockedMu.Lock()
defer d.blockedMu.Unlock()
d.blocked[hash] = true
return nil
}
func (d *DBBackedS3Store) isBlocked(hash string) (bool, error) {
err := d.initBlocked()
if err != nil {
return false, err
}
d.blockedMu.RLock()
defer d.blockedMu.RUnlock()
return d.blocked[hash], nil
}
func (d *DBBackedS3Store) initBlocked() error {
// first check without blocking since this is the most likely scenario
if d.blocked != nil {
return nil
}
d.blockedMu.Lock()
defer d.blockedMu.Unlock()
// check again in case of race condition
if d.blocked != nil {
return nil
}
var err error
d.blocked, err = d.db.GetBlocked()
return err
}
+19 -1
View File
@@ -5,7 +5,7 @@ import (
"os"
"path"
"github.com/lbryio/lbry.go/errors"
"github.com/lbryio/lbry.go/extras/errors"
)
// FileBlobStore is a local disk store.
@@ -95,3 +95,21 @@ func (f *FileBlobStore) Put(hash string, blob []byte) error {
func (f *FileBlobStore) PutSD(hash string, blob []byte) error {
return f.Put(hash, blob)
}
// Delete deletes the blob from the store
func (f *FileBlobStore) Delete(hash string) error {
err := f.initOnce()
if err != nil {
return err
}
has, err := f.Has(hash)
if err != nil {
return err
}
if !has {
return nil
}
return os.Remove(f.path(hash))
}
+12 -1
View File
@@ -1,6 +1,6 @@
package store
import "github.com/lbryio/lbry.go/errors"
import "github.com/lbryio/lbry.go/extras/errors"
// MemoryBlobStore is an in memory only blob store with no persistence.
type MemoryBlobStore struct {
@@ -41,3 +41,14 @@ func (m *MemoryBlobStore) Put(hash string, blob []byte) error {
func (m *MemoryBlobStore) PutSD(hash string, blob []byte) error {
return m.Put(hash, blob)
}
// Delete deletes the blob from the store
func (m *MemoryBlobStore) Delete(hash string) error {
delete(m.blobs, hash)
return nil
}
// Debug returns the blobs in memory. It's useful for testing and debugging.
func (m *MemoryBlobStore) Debug() map[string][]byte {
return m.blobs
}
+1 -1
View File
@@ -4,7 +4,7 @@ import (
"bytes"
"testing"
"github.com/lbryio/lbry.go/errors"
"github.com/lbryio/lbry.go/extras/errors"
)
func TestMemoryBlobStore_Put(t *testing.T) {
+21 -4
View File
@@ -5,7 +5,7 @@ import (
"net/http"
"time"
"github.com/lbryio/lbry.go/errors"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
@@ -120,9 +120,10 @@ func (s *S3BlobStore) Put(hash string, blob []byte) error {
}(time.Now())
_, err = s3manager.NewUploader(s.session).Upload(&s3manager.UploadInput{
Bucket: aws.String(s.bucket),
Key: aws.String(hash),
Body: bytes.NewBuffer(blob),
Bucket: aws.String(s.bucket),
Key: aws.String(hash),
Body: bytes.NewBuffer(blob),
StorageClass: aws.String(s3.StorageClassIntelligentTiering),
})
return err
@@ -133,3 +134,19 @@ func (s *S3BlobStore) PutSD(hash string, blob []byte) error {
//Todo - handle missing stream for consistency
return s.Put(hash, blob)
}
func (s *S3BlobStore) Delete(hash string) error {
err := s.initOnce()
if err != nil {
return err
}
log.Debugf("Deleting %s from S3", hash[:8])
_, err = s3.New(s.session).DeleteObject(&s3.DeleteObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(hash),
})
return err
}
+18 -5
View File
@@ -1,13 +1,26 @@
package store
import "github.com/lbryio/lbry.go/errors"
import "github.com/lbryio/lbry.go/extras/errors"
// BlobStore is an interface with methods for consistently handling blob storage.
type BlobStore interface {
Has(string) (bool, error)
Get(string) ([]byte, error)
Put(string, []byte) error
PutSD(string, []byte) error
// Does blob exist in the store
Has(hash string) (bool, error)
// Get the blob from the store
Get(hash string) ([]byte, error)
// Put the blob into the store
Put(hash string, blob []byte) error
// Put an SD blob into the store
PutSD(hash string, blob []byte) error
// Delete the blob from the store
Delete(hash string) error
}
type Blocklister interface {
// Block deletes the blob and prevents it from being uploaded in the future
Block(hash string) error
// Wants returns false if the hash exists or is blocked, true otherwise
Wants(hash string) (bool, error)
}
//ErrBlobNotFound is a standard error when a blob is not found in the store.
+74
View File
@@ -0,0 +1,74 @@
package updater
import (
"net/http"
"os/exec"
"strings"
"time"
"github.com/lbryio/reflector.go/meta"
"github.com/inconshreveable/go-update"
"github.com/sirupsen/logrus"
)
// TODO: this needs to be shut down cleanly
func Run(url, updateCmd string) {
t := time.NewTicker(1 * time.Minute)
for {
<-t.C
r, err := http.Head(url)
if err != nil {
logrus.Errorln(err)
continue
} else if r.StatusCode >= 300 {
logrus.Errorf("HEAD request returned status code %d", r.StatusCode)
continue
}
lm, err := time.Parse(time.RFC1123, r.Header.Get("Last-Modified"))
if err != nil {
logrus.Errorln(err)
continue
}
if !lm.After(meta.BuildTime.Add(5 * time.Minute)) { // buffer for compile and upload time
continue
}
logrus.Println("updating binary...")
err = updateBin(url)
if err != nil {
logrus.Errorln(err)
// todo: may need to recover here, if stuck without binary
continue
}
parts := strings.Fields(updateCmd)
err = exec.Command(parts[0], parts[1:]...).Start()
if err != nil {
logrus.Errorln(err)
continue
}
return // stop checking for updates in case restart takes a while
}
}
func updateBin(url string) error {
resp, err := http.Get(url)
if err != nil {
return err
}
defer func() {
err := resp.Body.Close()
if err != nil {
logrus.Errorln(err)
}
}()
return update.Apply(resp.Body, update.Options{})
}
+43
View File
@@ -0,0 +1,43 @@
package wallet
// ServerVersion returns the server's version.
// https://electrumx.readthedocs.io/en/latest/protocol-methods.html#server-version
func (n *Node) ServerVersion() (string, error) {
resp := &struct {
Result []string `json:"result"`
}{}
err := n.request("server.version", []string{"reflector.go", ProtocolVersion}, resp)
var v string
if len(resp.Result) >= 2 {
v = resp.Result[1]
}
return v, err
}
type GetClaimsInTxResp struct {
Jsonrpc string `json:"jsonrpc"`
ID int `json:"id"`
Result []struct {
Name string `json:"name"`
ClaimID string `json:"claim_id"`
Txid string `json:"txid"`
Nout int `json:"nout"`
Amount int `json:"amount"`
Depth int `json:"depth"`
Height int `json:"height"`
Value string `json:"value"`
ClaimSequence int `json:"claim_sequence"`
Address string `json:"address"`
Supports []interface{} `json:"supports"` // TODO: finish me
EffectiveAmount int `json:"effective_amount"`
ValidAtHeight int `json:"valid_at_height"`
} `json:"result"`
}
func (n *Node) GetClaimsInTx(txid string) (*GetClaimsInTxResp, error) {
var resp GetClaimsInTxResp
err := n.request("blockchain.claimtrie.getclaimsintx", []string{txid}, &resp)
return &resp, err
}
+240
View File
@@ -0,0 +1,240 @@
package wallet
// copied from https://github.com/d4l3k/go-electrum
import (
"crypto/tls"
"encoding/json"
"math/rand"
"net"
"sync"
"time"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/stop"
log "github.com/sirupsen/logrus"
"github.com/uber-go/atomic"
)
const (
ClientVersion = "0.0.1"
ProtocolVersion = "1.0"
)
var (
ErrNotImplemented = errors.Base("not implemented")
ErrNodeConnected = errors.Base("node already connected")
ErrConnectFailed = errors.Base("failed to connect")
ErrTimeout = errors.Base("timeout")
)
type response struct {
data []byte
err error
}
type Node struct {
transport *TCPTransport
nextId atomic.Uint32
grp *stop.Group
handlersMu *sync.RWMutex
handlers map[uint32]chan response
pushHandlersMu *sync.RWMutex
pushHandlers map[string][]chan response
timeout time.Duration
}
// NewNode creates a new node.
func NewNode() *Node {
return &Node{
handlers: make(map[uint32]chan response),
pushHandlers: make(map[string][]chan response),
handlersMu: &sync.RWMutex{},
pushHandlersMu: &sync.RWMutex{},
grp: stop.New(),
timeout: 1 * time.Second,
}
}
// Connect creates a new connection to the specified address.
func (n *Node) Connect(addrs []string, config *tls.Config) error {
if n.transport != nil {
return ErrNodeConnected
}
// shuffle addresses for load balancing
rand.Shuffle(len(addrs), func(i, j int) { addrs[i], addrs[j] = addrs[j], addrs[i] })
var err error
for _, addr := range addrs {
n.transport, err = NewTransport(addr, config)
if err == nil {
break
}
if e, ok := err.(*net.OpError); ok && e.Err.Error() == "no such host" {
// net.errNoSuchHost is not exported, so we have to string-match
continue
}
return err
}
if n.transport == nil {
return ErrConnectFailed
}
log.Debugf("wallet connected to %s", n.transport.conn.RemoteAddr())
n.grp.Add(1)
go func() {
defer n.grp.Done()
<-n.grp.Ch()
n.transport.Shutdown()
}()
n.grp.Add(1)
go func() {
defer n.grp.Done()
n.handleErrors()
}()
n.grp.Add(1)
go func() {
defer n.grp.Done()
n.listen()
}()
return nil
}
func (n *Node) Shutdown() {
n.grp.StopAndWait()
}
func (n *Node) handleErrors() {
for {
select {
case <-n.grp.Ch():
return
case err := <-n.transport.Errors():
n.err(err)
}
}
}
// err handles errors produced by the foreign node.
func (n *Node) err(err error) {
// TODO: Better error handling.
log.Error(err)
}
// listen processes messages from the server.
func (n *Node) listen() {
for {
select {
case <-n.grp.Ch():
return
case bytes := <-n.transport.Responses():
msg := &struct {
Id uint32 `json:"id"`
Method string `json:"method"`
Error struct {
Code int `json:"code"`
Message string `json:"message"`
} `json:"error"`
}{}
if err := json.Unmarshal(bytes, msg); err != nil {
n.err(err)
continue
}
r := response{}
if len(msg.Error.Message) > 0 {
r.err = errors.Base("%d: %s", msg.Error.Code, msg.Error.Message)
} else {
r.data = bytes
}
if len(msg.Method) > 0 {
n.pushHandlersMu.RLock()
handlers := n.pushHandlers[msg.Method]
n.pushHandlersMu.RUnlock()
for _, handler := range handlers {
select {
case handler <- r:
default:
}
}
}
n.handlersMu.RLock()
c, ok := n.handlers[msg.Id]
n.handlersMu.RUnlock()
if ok {
c <- r
}
}
}
}
// listenPush returns a channel of messages matching the method.
//func (n *Node) listenPush(method string) <-chan []byte {
// c := make(chan []byte, 1)
// n.pushHandlersMu.Lock()
// defer n.pushHandlersMu.Unlock()
// n.pushHandlers[method] = append(n.pushHandlers[method], c)
// return c
//}
// request makes a request to the server and unmarshals the response into v.
func (n *Node) request(method string, params []string, v interface{}) error {
msg := struct {
Id uint32 `json:"id"`
Method string `json:"method"`
Params []string `json:"params"`
}{
Id: n.nextId.Load(),
Method: method,
Params: params,
}
n.nextId.Inc()
bytes, err := json.Marshal(msg)
if err != nil {
return err
}
bytes = append(bytes, delimiter)
c := make(chan response, 1)
n.handlersMu.Lock()
n.handlers[msg.Id] = c
n.handlersMu.Unlock()
err = n.transport.Send(bytes)
if err != nil {
return err
}
var r response
select {
case r = <-c:
case <-time.After(n.timeout):
r = response{err: errors.Err(ErrTimeout)}
}
n.handlersMu.Lock()
delete(n.handlers, msg.Id)
n.handlersMu.Unlock()
if r.err != nil {
return r.err
}
return json.Unmarshal(r.data, v)
}
+101
View File
@@ -0,0 +1,101 @@
package wallet
// copied from https://github.com/d4l3k/go-electrum
import (
"bufio"
"crypto/tls"
"net"
"time"
"github.com/lbryio/lbry.go/extras/stop"
log "github.com/sirupsen/logrus"
)
type TCPTransport struct {
conn net.Conn
responses chan []byte
errors chan error
grp *stop.Group
}
func NewTransport(addr string, config *tls.Config) (*TCPTransport, error) {
var conn net.Conn
var err error
timeout := 5 * time.Second
if config != nil {
conn, err = tls.DialWithDialer(&net.Dialer{Timeout: timeout}, "tcp", addr, config)
} else {
conn, err = net.DialTimeout("tcp", addr, timeout)
}
if err != nil {
return nil, err
}
t := &TCPTransport{
conn: conn,
responses: make(chan []byte),
errors: make(chan error),
grp: stop.New(),
}
t.grp.Add(1)
go func() {
defer t.grp.Done()
<-t.grp.Ch()
t.close()
}()
t.grp.Add(1)
go func() {
t.grp.Done()
t.listen()
}()
return t, nil
}
const delimiter = byte('\n')
func (t *TCPTransport) listen() {
reader := bufio.NewReader(t.conn)
for {
line, err := reader.ReadBytes(delimiter)
if err != nil {
t.error(err)
return
}
log.Debugf("%s -> %s", t.conn.RemoteAddr(), line)
t.responses <- line
}
}
func (t *TCPTransport) Send(body []byte) error {
log.Debugf("%s <- %s", t.conn.RemoteAddr(), body)
_, err := t.conn.Write(body)
return err
}
func (t *TCPTransport) error(err error) {
select {
case t.errors <- err:
default:
}
}
func (t *TCPTransport) Responses() <-chan []byte { return t.responses }
func (t *TCPTransport) Errors() <-chan error { return t.errors }
func (t *TCPTransport) Shutdown() {
t.grp.StopAndWait()
}
func (t *TCPTransport) close() {
err := t.conn.Close()
if err != nil {
t.error(err)
}
}