Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0359d723c5 | ||
|
|
0cfc336be5 | ||
|
|
05acdcb9e0 | ||
|
|
9e2cccc51f | ||
|
|
1b79371f8b | ||
|
|
cbcc611196 | ||
|
|
68c0fe3b97 | ||
|
|
8bfe7e101d | ||
|
|
7b3cca45b4 | ||
|
|
ad710dfeea | ||
|
|
ebc52245f3 | ||
|
|
b098aa2dc8 | ||
|
|
ddd23f9a36 | ||
|
|
4928e430c0 | ||
|
|
d21b0867dd | ||
|
|
32a85a815e | ||
|
|
9cdac902bd | ||
|
|
8ec01bf1a7 | ||
|
|
ba8d1ba67b | ||
|
|
edecf4f988 | ||
|
|
77836f9c83 | ||
|
|
9716e41d63 | ||
|
|
7260388255 | ||
|
|
071a7907c1 | ||
|
|
185900582d | ||
|
|
03445476ec | ||
|
|
e5a9590cb6 | ||
|
|
de736b068d | ||
|
|
14fbd2ae27 | ||
|
|
4d81a43a8f | ||
|
|
b3f0d63b4d | ||
|
|
c880f0b80f | ||
|
|
085490e92b | ||
|
|
456fe53e01 | ||
|
|
778fc17adf | ||
|
|
e93c097fd9 | ||
|
|
0dfda70c70 | ||
|
|
6c082993cf | ||
|
|
08ed3c9f13 | ||
|
|
7f75602841 | ||
|
|
4d168ddefc | ||
|
|
b7abb77ea1 | ||
|
|
8c2a46752c | ||
|
|
0c8da4abe5 | ||
|
|
87680d806c | ||
|
|
e1c59b9b63 | ||
|
|
c79634706b | ||
|
|
f5d30b1a6e | ||
|
|
0177dd4ce0 | ||
|
|
5693529216 | ||
|
|
a1c2e92ca3 | ||
|
|
9c0554ef05 | ||
|
|
4e80f91a57 | ||
|
|
c211f83ba7 | ||
|
|
29d1ccf68c | ||
|
|
2f7d67794f | ||
|
|
4d8e7739d7 | ||
|
|
6fc0ceea2a | ||
|
|
ae0c7dd2bb | ||
|
|
4af5c2f4c6 | ||
|
|
def0a97f49 | ||
|
|
6dde793745 | ||
|
|
654cc44935 | ||
|
|
90d6d29452 | ||
|
|
b2272fef3a | ||
|
|
86f3e62aa8 | ||
|
|
e1b4f21e00 | ||
|
|
b4913ecedf | ||
|
|
63a574ec2f | ||
|
|
b8af3408e0 | ||
|
|
847089d0d6 | ||
|
|
170dfef3a8 | ||
|
|
2b458a6bd0 | ||
|
|
febfc51cb0 | ||
|
|
72be487262 | ||
|
|
94e7d81bd3 | ||
|
|
c6c779da39 | ||
|
|
2e101083e6 | ||
|
|
63aacd8a69 | ||
|
|
c03ae6487d | ||
|
|
0c4f455f0c | ||
|
|
af3e08c446 | ||
|
|
975bfe7fac | ||
|
|
b075d948bb | ||
|
|
2651a64dbb | ||
|
|
fa7150cf2b | ||
|
|
6c4db980c9 | ||
|
|
7adaa510fd | ||
|
|
64ed7304f6 | ||
|
|
5aefaf061e | ||
|
|
724ee47c8b | ||
|
|
caaec6fcb1 | ||
|
|
15984b8fd9 | ||
|
|
2be913b077 | ||
|
|
34c11b0a0e | ||
|
|
64acdc29c3 | ||
|
|
598773c90d | ||
|
|
766238fd7e | ||
|
|
ac5242f173 | ||
|
|
215103cb33 | ||
|
|
ed3622d0a6 | ||
|
|
848fce5afa | ||
|
|
e37eeba0c9 | ||
|
|
7da49a4ccb | ||
|
|
7b02ace5e2 | ||
|
|
5fb67b32db | ||
|
|
a0c9ed2ace | ||
|
|
998b082a06 | ||
|
|
36d4156e2a | ||
|
|
74925ebba2 | ||
|
|
6f95b3395f | ||
|
|
dff00e2317 | ||
|
|
9a5d9d7ff5 | ||
|
|
5794c57898 | ||
|
|
35c713a26e | ||
|
|
6fb0620091 | ||
|
|
03df751bc7 | ||
|
|
c902858958 | ||
|
|
84fabdd5f4 | ||
|
|
f5cad15f84 | ||
|
|
dd3d0ae42c | ||
|
|
0b565852b8 | ||
|
|
ff13d7b2f7 | ||
|
|
7f5a89fa5a | ||
|
|
704e15f8c1 | ||
|
|
5eb1f13b54 | ||
|
|
176e05714e | ||
|
|
eefd84b02d | ||
|
|
af2742c34f | ||
|
|
2cf4acdb59 | ||
|
|
044e2fe5d7 | ||
|
|
fb77bf621e | ||
|
|
e70b9af3e4 | ||
|
|
5df05cf46f | ||
|
|
7bddcf01b8 | ||
|
|
659a6e73cc | ||
|
|
aaae3ffa5b | ||
|
|
131fed28d2 | ||
|
|
72571236ab | ||
|
|
560e180e36 | ||
|
|
070c378dfd | ||
|
|
124d4065c2 | ||
|
|
f131c1f35b | ||
|
|
7a3225434e | ||
|
|
3608971f0b | ||
|
|
c9fa04043c | ||
|
|
c6b53792c8 | ||
|
|
69fa06420b | ||
|
|
5cb1365903 | ||
|
|
e430c2fd40 | ||
|
|
66024716ac | ||
|
|
f043516a14 | ||
|
|
c3db95a6c1 | ||
|
|
08c93d44fd | ||
|
|
b02e80d472 | ||
|
|
f1875454cc | ||
|
|
a80e0f5b0d | ||
|
|
2b3581a692 | ||
|
|
a0f78028cc | ||
|
|
a084330055 | ||
|
|
ea80ed6506 | ||
|
|
4b335ed692 | ||
|
|
6118dde36c | ||
|
|
8364d3fc54 | ||
|
|
de0ccd4da7 | ||
|
|
31f9346027 | ||
|
|
5658fe4607 | ||
|
|
fc5f5ff7d3 | ||
|
|
e8b98bc862 | ||
|
|
f458529c74 | ||
|
|
150b1f6f1f | ||
|
|
47f28002ff | ||
|
|
beff466d18 | ||
|
|
694bda105c | ||
|
|
34ca7847d0 | ||
|
|
5c91051b78 | ||
|
|
df4f42db82 | ||
|
|
41d758ef5c | ||
|
|
264390a2b2 | ||
|
|
fdcc41829a | ||
|
|
09c7718f30 | ||
|
|
8a5f57b14f | ||
|
|
90997b9918 | ||
|
|
e0da2674a1 | ||
|
|
a80599413c | ||
|
|
3ffe7a10c7 | ||
|
|
1bf3cb81b3 | ||
|
|
4a9f127ecc | ||
|
|
de1fb63a1c | ||
|
|
e98794e125 | ||
|
|
fb0004bac4 | ||
|
|
1f12b1d5d8 | ||
|
|
bc95ca61d5 | ||
|
|
be69c2f05c | ||
|
|
4a902597df | ||
|
|
dd528f03f6 | ||
|
|
e6ba61fce2 | ||
|
|
e03abad012 | ||
|
|
dde93a1fe6 | ||
|
|
4a5a148843 | ||
|
|
cb6443fa1e | ||
|
|
1a1f991e48 | ||
|
|
d1063bd54e | ||
|
|
d291c063ec | ||
|
|
a939529d19 | ||
|
|
cf34547b8a | ||
|
|
6631ad325f | ||
|
|
df266f6194 | ||
|
|
b3b581c00e | ||
|
|
18b15674d9 | ||
|
|
780e899e90 | ||
|
|
5c5643749e | ||
|
|
11e50a6022 | ||
|
|
1603b3bb22 | ||
|
|
5465527faf | ||
|
|
5d8a2d697c | ||
|
|
50089481fb | ||
|
|
d6cf9e9e63 | ||
|
|
dc6dd8d12b | ||
|
|
ad8d623863 | ||
|
|
d89b58c52a | ||
|
|
2aa40850b0 | ||
|
|
ca24c2be28 | ||
|
|
52127eee7c | ||
|
|
95eb94f5a7 | ||
|
|
834733b675 | ||
|
|
86a553b876 | ||
|
|
e5438713ce | ||
|
|
f6c4b7d36b | ||
|
|
661c20a21d | ||
|
|
69f1e0f4ca | ||
|
|
2ca83139df | ||
|
|
0af6d65d40 | ||
|
|
a8230db802 | ||
|
|
24f885e268 | ||
|
|
acb9840871 | ||
|
|
a745bafc58 | ||
|
|
bf9e90f14c | ||
|
|
a98d1e1217 | ||
|
|
36ee7e8d1f | ||
|
|
c1e8e7481f | ||
|
|
cb669eb1a7 | ||
|
|
08df3b167c | ||
|
|
1a6b862c96 | ||
|
|
ce3d4403db | ||
|
|
949ea2f2d8 | ||
|
|
390d2b1eec | ||
|
|
557c595e90 | ||
|
|
7d8772ed63 | ||
|
|
80f3b3437a | ||
|
|
68caed8a62 | ||
|
|
3823ee7a4e | ||
|
|
5916ab4efd | ||
|
|
227b362296 | ||
|
|
8f280bf52f | ||
|
|
f30401d7a6 | ||
|
|
83bb1791b4 | ||
|
|
3e006f1571 | ||
|
|
2497f354bd | ||
|
|
db83ebe434 | ||
|
|
a2d51cee3d | ||
|
|
47425c2c50 | ||
|
|
594a45a271 | ||
|
|
fd39b09e95 | ||
|
|
fce8beea4d | ||
|
|
96e0c2e2b2 | ||
|
|
bb5df9896a | ||
|
|
accd79c81f | ||
|
|
32dae2704f | ||
|
|
2ed1e23228 | ||
|
|
d61f7c892f | ||
|
|
027409fe40 | ||
|
|
38401bf8d6 | ||
|
|
825e699114 | ||
|
|
6509c7dc52 | ||
|
|
c088a65041 | ||
|
|
c7643fb5da | ||
|
|
c3ab5dd2a5 | ||
|
|
2047fe6c05 | ||
|
|
6166ff37cf | ||
|
|
970585c75d | ||
|
|
71549c0dea | ||
|
|
46ae5a502a | ||
|
|
fb2f807e6a | ||
|
|
ad1eea1e8e | ||
|
|
dfb77d3547 | ||
|
|
96427136ef | ||
|
|
c649636eeb | ||
|
|
2e81b1ab03 | ||
|
|
fa2ec38d3e | ||
|
|
2cacdaf22d | ||
|
|
3b269050af | ||
|
|
59ff828794 | ||
|
|
4569d04522 | ||
|
|
7a618f4228 | ||
|
|
0d996d7415 | ||
|
|
2744080c6b | ||
|
|
41b513439f | ||
|
|
31daec7054 | ||
|
|
00feaaf76c | ||
|
|
a98990f573 | ||
|
|
5788f5aa55 | ||
|
|
922a9402aa | ||
|
|
05e0a506b4 | ||
|
|
a8fe33ecf4 | ||
|
|
030ed4914d | ||
|
|
e03810903a | ||
|
|
b0b2d21a08 | ||
|
|
557988d2cb | ||
|
|
6acb99c74d | ||
|
|
d77f5d17f3 | ||
|
|
609ebdb74e | ||
|
|
5baa66f0aa | ||
|
|
75ad143d84 | ||
|
|
14d6d32a41 | ||
|
|
e3ae0ef5c9 | ||
|
|
db791e26ef | ||
|
|
dc104dd17b | ||
|
|
e67e3dbf85 | ||
|
|
e14a1f36cf | ||
|
|
11ebfb822b | ||
|
|
4293e772e2 | ||
|
|
686ac662cc | ||
|
|
747a6ec2b2 | ||
|
|
046868c4e0 | ||
|
|
53d3eea8fb | ||
|
|
61e83d86de | ||
|
|
9fb824790b | ||
|
|
6fd0526376 | ||
|
|
cd236708b0 | ||
|
|
83921b8c2d | ||
|
|
c042fe08c3 | ||
|
|
59e92ccf88 | ||
|
|
cb386de4c7 | ||
|
|
146adcf7fd | ||
|
|
5d844fc3ea | ||
|
|
4ed52e20d7 | ||
|
|
7ab2d355d5 | ||
|
|
064035f1d8 | ||
|
|
e158fa1bb2 | ||
|
|
25ec93a6b9 | ||
|
|
22f42d5559 | ||
|
|
473e7e3b07 | ||
|
|
d7a4b34d8c | ||
|
|
605afd1d6c | ||
|
|
204198bf9d | ||
|
|
fe9cf091fc | ||
|
|
65df7bc627 | ||
|
|
75886211b1 | ||
|
|
391f983630 | ||
|
|
44cf4d085c | ||
|
|
bd8a35e366 | ||
|
|
4284c3b1f9 | ||
|
|
4c8eda783f |
No files matched your search
@@ -0,0 +1,37 @@
|
||||
name: Go
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: 1.25.x
|
||||
|
||||
- name: Build linux
|
||||
run: make linux
|
||||
|
||||
- name: Build macos
|
||||
run: make macos
|
||||
|
||||
- name: Test
|
||||
run: make test
|
||||
|
||||
- name: Lint
|
||||
run: make lint
|
||||
|
||||
- name: retrieve all tags
|
||||
run: git fetch --prune --unshallow --tags
|
||||
|
||||
- name: Print changes since last version
|
||||
run: git log $(git describe --tags --abbrev=0)..HEAD --no-merges --oneline
|
||||
@@ -0,0 +1,62 @@
|
||||
name: release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "*.*.*"
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: 1.25.x
|
||||
|
||||
- name: Build linux
|
||||
run: make linux
|
||||
|
||||
- name: Build macos
|
||||
run: make macos
|
||||
|
||||
- name: Test
|
||||
run: make test
|
||||
|
||||
- name: Lint
|
||||
run: make lint
|
||||
|
||||
- name: Zip macos
|
||||
run: zip -r reflector_darwin_amd64.zip ./dist/darwin_amd64
|
||||
|
||||
- name: Zip linux
|
||||
run: zip -r reflector_linux_amd64.zip ./dist/linux_amd64
|
||||
|
||||
- name: retrieve all tags
|
||||
run: git fetch --prune --unshallow --tags
|
||||
|
||||
- name: Generate Changelog
|
||||
run: git log $(git describe --tags --abbrev=0 @^)..@ --no-merges --oneline > ${{ github.workspace }}-CHANGELOG.txt
|
||||
|
||||
- name: upload to github releases
|
||||
uses: softprops/action-gh-release@v1
|
||||
with:
|
||||
files: |
|
||||
./reflector_linux_amd64.zip
|
||||
./reflector_darwin_amd64.zip
|
||||
body_path: ${{ github.workspace }}-CHANGELOG.txt
|
||||
|
||||
# - name: Login to DockerHub
|
||||
# uses: docker/login-action@v2
|
||||
# with:
|
||||
# username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
# password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
# - name: Generate docker image
|
||||
# run: make image
|
||||
|
||||
# - name: Docker push
|
||||
# run: make publish_image
|
||||
+10
-2
@@ -1,4 +1,12 @@
|
||||
/vendor
|
||||
/blobs
|
||||
/config.json*
|
||||
/prism-bin
|
||||
/dist
|
||||
/bin
|
||||
|
||||
blobcache.yaml
|
||||
|
||||
reflector.yaml
|
||||
|
||||
upload.yaml
|
||||
|
||||
remotereflector.yaml
|
||||
@@ -0,0 +1,57 @@
|
||||
run:
|
||||
timeout: 5m
|
||||
tests: false
|
||||
modules-download-mode: readonly
|
||||
|
||||
linters-settings:
|
||||
gci:
|
||||
# Checks that no inline Comments are present.
|
||||
# Default: false
|
||||
no-inline-comments: false
|
||||
|
||||
# Checks that no prefix Comments(comment lines above an import) are present.
|
||||
# Default: false
|
||||
no-prefix-comments: false
|
||||
|
||||
# Skip generated files
|
||||
skip-generated: true
|
||||
|
||||
# Use custom order
|
||||
custom-order: true
|
||||
|
||||
sections:
|
||||
- standard
|
||||
- prefix(github.com/lbryio/reflector.go)
|
||||
- prefix(github.com/OdyseeTeam,github.com/lbryio)
|
||||
- default
|
||||
gofmt:
|
||||
simplify: true
|
||||
govet:
|
||||
enable-all: true
|
||||
misspell:
|
||||
locale: US
|
||||
revive:
|
||||
rules:
|
||||
- name: exported
|
||||
disabled: true
|
||||
staticcheck:
|
||||
checks: ["all"]
|
||||
|
||||
linters:
|
||||
enable:
|
||||
- errcheck
|
||||
- gosimple
|
||||
- govet
|
||||
- ineffassign
|
||||
- gci
|
||||
- gofmt
|
||||
- goimports
|
||||
- misspell
|
||||
- revive
|
||||
- unconvert
|
||||
- whitespace
|
||||
- typecheck
|
||||
disable:
|
||||
- unused
|
||||
- unparam
|
||||
- staticcheck
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"Disable": ["golint"],
|
||||
"Vendor": true
|
||||
}
|
||||
-82
@@ -1,82 +0,0 @@
|
||||
os: linux
|
||||
dist: trusty
|
||||
language: go
|
||||
|
||||
# Only the last two Go releases are supported by the Go team with security
|
||||
# updates. Any versions older than that should be considered deprecated.
|
||||
# Don't bother testing with them. tip builds your code with the latest
|
||||
# development version of Go. This can warn you that your code will break
|
||||
# in the next version of Go. Don't worry! Later we declare that test runs
|
||||
# are allowed to fail on Go tip.
|
||||
matrix:
|
||||
include:
|
||||
- go: 1.10.2
|
||||
env: DEPLOY=true # dont deploy tip build
|
||||
- go: master
|
||||
env: DEPLOY=false # dont deploy tip build
|
||||
allow_failures:
|
||||
- go: master
|
||||
# Don't wait for tip tests to finish. Mark the test run green if the
|
||||
# tests pass on the stable versions of Go.
|
||||
fast_finish: true
|
||||
|
||||
# Don't email me the results of the test runs.
|
||||
notifications:
|
||||
email: false
|
||||
|
||||
# Skip the install step. Don't `go get` dependencies. Only build with the
|
||||
# code in vendor/
|
||||
install: true
|
||||
|
||||
# Anything in before_script that returns a nonzero exit code will
|
||||
# flunk the build and immediately stop. It's sorta like having
|
||||
# set -e enabled in bash.
|
||||
before_script:
|
||||
# All the .go files, excluding vendor/ and model (auto generated)
|
||||
- GO_FILES=$(find . -iname '*.go' ! -iname '*_test.go' -type f | grep -v /vendor/ )
|
||||
- go get golang.org/x/tools/cmd/goimports # Used in build script for generated files
|
||||
- go get github.com/golang/lint/golint # Linter
|
||||
- go get honnef.co/go/tools/cmd/megacheck # Badass static analyzer/linter
|
||||
- go get github.com/jgautheron/gocyclo # Check against high complexity
|
||||
- go get github.com/mdempsky/unconvert # Identifies unnecessary type conversions
|
||||
- go get github.com/kisielk/errcheck # Checks for unhandled errors
|
||||
- go get github.com/opennota/check/cmd/varcheck # Checks for unused vars
|
||||
- go get github.com/opennota/check/cmd/structcheck # Checks for unused fields in structs
|
||||
|
||||
|
||||
|
||||
# script always run to completion (set +e). All of these code checks are must haves
|
||||
# in a modern Go project.
|
||||
script:
|
||||
# Build Prism successfully
|
||||
- make
|
||||
# Fail if a .go file hasn't been formatted with gofmt
|
||||
- test -z $(gofmt -s -l $GO_FILES)
|
||||
# Run unit tests
|
||||
- go test ./...
|
||||
# Checks for unused vars and fields on structs
|
||||
- varcheck ./...
|
||||
- structcheck ./...
|
||||
# go vet is the official Go static analyzer
|
||||
- go vet ./...
|
||||
# forbid code with huge functions
|
||||
#- gocyclo -ignore "_test.go" -avg -over 19 $GO_FILES
|
||||
# checks for unhandled errors
|
||||
- errcheck ./...
|
||||
# "go vet on steroids" + linter - ignore autogen code
|
||||
- megacheck -simple.exit-non-zero=true ./...
|
||||
# check for unnecessary conversions - ignore autogen code
|
||||
- unconvert ./...
|
||||
# one last linter - ignore autogen code
|
||||
#- golint -set_exit_status $(go list ./... | grep -v /vendor/ )
|
||||
|
||||
deploy:
|
||||
- provider: releases
|
||||
file: prism-bin
|
||||
skip_cleanup: true
|
||||
on:
|
||||
repo: lbryio/reflector.go
|
||||
tags: true
|
||||
condition: "$DEPLOY = true"
|
||||
api_key:
|
||||
secure: epAlhp3SUr8hhISarJ22n6tRw2TEa4s4oNFIvJUb5HGECVp1SYN7ao0ln5NoNLmfJS60pi911i/kMhhi21/uhZ0kCYlEhhIE2pc1zsiAxK9L9ENCssJ205HfVbe5grhwskLGzgjhU9OznO8WtmyOPWXr0it8M8RCTjx6rEC0A33Id3WMYyhP938Sj9CxEYeH4KS8wFvBXkgBVtrgaYwRTCIROFddHFXOb9jyNhqQ1RbfKtllsVtQhVk5WMlomheBNSS4vr6WMS4X4+2okFqnLtiSn1wrn5I/94UQbnrI1juVnQj0K+j32EyQbAOt4T2cLW3GtG0jhaYKyNMT9ycDCdVACPSDELlHWjeyoes9bnhUFftm6kDbQxwA1UsTF1yG8tMKXxBSmYyoT7qDloi6pBifZMrFXL61uTs6yhVB9LS/2oqg4sc0Ne87bRcn4OxsBeVCe3kbBHDTR/NTyF2gNPtRvgMAWULxTVcUm9VYdO0IWvAig5g4Row0DnFzEquD6CzezbRWD9WyZyV/AFyYHeeQ2PO7jTw0/3M7aDX33Fuhh34lehzmrC03cfgD/wZW+spxozIcQCYdiJqVw+u+/NvbNr0kkFzE9zW26JEmUFTyDvKxvnza1Kwtww3EgH6zaOL8r4yVbb54rePRvLw7pl93zlfJnEB2MCPqJOY5ZpU=
|
||||
@@ -0,0 +1,9 @@
|
||||
FROM alpine
|
||||
EXPOSE 8080
|
||||
|
||||
RUN mkdir /app
|
||||
WORKDIR /app
|
||||
COPY dist/linux_amd64/prism-bin ./prism
|
||||
RUN chmod +x prism
|
||||
|
||||
ENTRYPOINT [ "/app/prism" ]
|
||||
Generated
-435
@@ -1,435 +0,0 @@
|
||||
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
|
||||
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:d64110a78451e373c5a952d2625323dbbe3bfe41c67f9652ea9668a6ceb4f645"
|
||||
name = "github.com/armon/go-metrics"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "3c58d8115a78a6879e5df75ae900846768d36895"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:c0b6dbbb56a745020d5939bdde2197241a1c6109f226cca57b16f46916be5e94"
|
||||
name = "github.com/aws/aws-sdk-go"
|
||||
packages = [
|
||||
"aws",
|
||||
"aws/awserr",
|
||||
"aws/awsutil",
|
||||
"aws/client",
|
||||
"aws/client/metadata",
|
||||
"aws/corehandlers",
|
||||
"aws/credentials",
|
||||
"aws/credentials/ec2rolecreds",
|
||||
"aws/credentials/endpointcreds",
|
||||
"aws/credentials/stscreds",
|
||||
"aws/csm",
|
||||
"aws/defaults",
|
||||
"aws/ec2metadata",
|
||||
"aws/endpoints",
|
||||
"aws/request",
|
||||
"aws/session",
|
||||
"aws/signer/v4",
|
||||
"internal/sdkio",
|
||||
"internal/sdkrand",
|
||||
"internal/sdkuri",
|
||||
"internal/shareddefaults",
|
||||
"private/protocol",
|
||||
"private/protocol/eventstream",
|
||||
"private/protocol/eventstream/eventstreamapi",
|
||||
"private/protocol/query",
|
||||
"private/protocol/query/queryutil",
|
||||
"private/protocol/rest",
|
||||
"private/protocol/restxml",
|
||||
"private/protocol/xml/xmlutil",
|
||||
"service/s3",
|
||||
"service/s3/s3iface",
|
||||
"service/s3/s3manager",
|
||||
"service/sts",
|
||||
]
|
||||
pruneopts = ""
|
||||
revision = "c0447dbaaf195bb477fd2d511b8e4665e04b9017"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:e250643be8120824556f39df6ef128fc2be490fc96e0cb64b1a8ecf96bbe3ce6"
|
||||
name = "github.com/btcsuite/btcutil"
|
||||
packages = ["base58"]
|
||||
pruneopts = ""
|
||||
revision = "ab6388e0c60ae4834a1f57511e20c17b5f78be4b"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:0a39ec8bf5629610a4bc7873a92039ee509246da3cef1a0ea60f1ed7e5f9cea5"
|
||||
name = "github.com/davecgh/go-spew"
|
||||
packages = ["spew"]
|
||||
pruneopts = ""
|
||||
revision = "346938d642f2ec3594ed81d874461961cd0faa76"
|
||||
version = "v1.1.0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:968d8903d598e3fae738325d3410f33f07ea6a2b9ee5591e9c262ee37df6845a"
|
||||
name = "github.com/go-errors/errors"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "a6af135bd4e28680facf08a3d206b454abc877a4"
|
||||
version = "v1.0.1"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:d19c78214e03e297e9e30d2eb11892f731358b2951f2a5c7374658a156373e4c"
|
||||
name = "github.com/go-ini/ini"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "358ee7663966325963d4e8b2e1fbd570c5195153"
|
||||
version = "v1.38.1"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:27b11ca1ad214ead955ff5480e8575e74c5df4e4dc02b04256a8d92131e1d3ad"
|
||||
name = "github.com/go-sql-driver/mysql"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "99ff426eb706cffe92ff3d058e168b278cabf7c7"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:dbbeb8ddb0be949954c8157ee8439c2adfd8dc1c9510eb44a6e58cb68c3dce28"
|
||||
name = "github.com/gorilla/context"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "08b5f424b9271eedf6f9f0ce86cb9396ed337a42"
|
||||
version = "v1.1.1"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:c2c8666b4836c81a1d247bdf21c6a6fc1ab586538ab56f74437c2e0df5c375e1"
|
||||
name = "github.com/gorilla/mux"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "e3702bed27f0d39777b0b37b664b6280e8ef8fbf"
|
||||
version = "v1.6.2"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:b4817bdb0da3054166de058111943ac58b315aead2fed4ee838625a4e304f74c"
|
||||
name = "github.com/gorilla/rpc"
|
||||
packages = [
|
||||
"v2",
|
||||
"v2/json",
|
||||
]
|
||||
pruneopts = ""
|
||||
revision = "22c016f3df3febe0c1f6727598b6389507e03a18"
|
||||
version = "v1.1.0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:fe1b4d4cbe48c0d55507c55f8663aa4185576cc58fa0c8be03bb8f19dfe17a9c"
|
||||
name = "github.com/gorilla/websocket"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "ea4d1f681babbce9545c9c5f3d5194a789c89f5b"
|
||||
version = "v1.2.0"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:4fe55793760295fbef367890352b720784243e0ad19b5ee242519a4682bb9ef8"
|
||||
name = "github.com/hashicorp/errwrap"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "d6c0cd88035724dd42e0f335ae30161c20575ecc"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:4423ee95d6ee30bb22f680445c58889bb5b91e1b955405bf34374a053784a8a2"
|
||||
name = "github.com/hashicorp/go-immutable-radix"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "7f3cd4390caab3250a57f30efdb2a65dd7649ecf"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:6a611e691e739173805cb54019b5c39bb9d46455526dff31e0e6fe3aaca52776"
|
||||
name = "github.com/hashicorp/go-msgpack"
|
||||
packages = ["codec"]
|
||||
pruneopts = ""
|
||||
revision = "fa3f63826f7c23912c15263591e65d54d080b458"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:deaebb4a98ca748bbad7eb653f3a675749500020823a086448ffcd7ba6b8b02d"
|
||||
name = "github.com/hashicorp/go-multierror"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "3d5d8f294aa03d8e98859feac328afbdf1ae0703"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:74f54e6ef2339f1de1e8c4b6674442118bd89e619b2fbd949ef2337330067994"
|
||||
name = "github.com/hashicorp/go-sockaddr"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "6d291a969b86c4b633730bfc6b8b9d64c3aafed9"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:9c776d7d9c54b7ed89f119e449983c3f24c0023e75001d6092442412ebca6b94"
|
||||
name = "github.com/hashicorp/golang-lru"
|
||||
packages = ["simplelru"]
|
||||
pruneopts = ""
|
||||
revision = "0fb14efe8c47ae851c0034ed7a448854d3d34cf3"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:d7ce65372f495908f80fc1f80f4dab5d763d9a1de544abd95aa719e4262d0dd5"
|
||||
name = "github.com/hashicorp/memberlist"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "ce8abaa0c60c2d6bee7219f5ddf500e0a1457b28"
|
||||
version = "v0.1.0"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:4fd01ac3766b886665cfd335cc63819ec4e4538dcc1180c05d6edc089619962c"
|
||||
name = "github.com/hashicorp/serf"
|
||||
packages = [
|
||||
"coordinate",
|
||||
"serf",
|
||||
]
|
||||
pruneopts = ""
|
||||
revision = "984a73625de3138f44deb38d00878fab39eb6447"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:870d441fe217b8e689d7949fef6e43efbc787e50f200cb1e70dbca9204a1d6be"
|
||||
name = "github.com/inconshreveable/mousetrap"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
|
||||
version = "v1.0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:4f767a115bc8e08576f6d38ab73c376fc1b1cd3bb5041171c9e8668cc7739b52"
|
||||
name = "github.com/jmespath/go-jmespath"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "0b12d6b5"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:1e4cb5c6d4e92e2fb34f1545e071fb61e81c71fc0c325445db19b9f3981cb343"
|
||||
name = "github.com/johntdyer/slack-go"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "95fac1160b220c5abcf8b0ef88e9c3cb213c09f4"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:132aae3fa5ad407b53f57cf6ebe274e414ec22d1700899023319931fa90b63e2"
|
||||
name = "github.com/johntdyer/slackrus"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "f7aae3243a0786c5a974bce71ed951c459876e64"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:65bb92382468ce7a1ee218b5f352cd0927d358569be7521dd7cb42e74589a308"
|
||||
name = "github.com/lbryio/errors.go"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "ad03d3cc6a5c27c94bbe3412cf7de0eae1a9bd7c"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:76fd7507e6014c598a01f1b3d558774d2a3114c438403bc98123870d2aecec62"
|
||||
name = "github.com/lbryio/lbry.go"
|
||||
packages = [
|
||||
"crypto",
|
||||
"errors",
|
||||
"null",
|
||||
"querytools",
|
||||
"stop",
|
||||
"util",
|
||||
]
|
||||
pruneopts = ""
|
||||
revision = "e2c96944fc485d3ab5e164da78f8439a94c5aa85"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:cabf2bf5e49edfe0c34cb9c6a256f2a99e6cc8c5e660855c8f3dafe1f81d5dcd"
|
||||
name = "github.com/lyoshenka/bencode"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "b7abd7672df533e627eddbf3a5a529786e8bda7f"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:f0bad0fece0fb73c6ea249c18d8e80ffbe86be0457715b04463068f04686cf39"
|
||||
name = "github.com/miekg/dns"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "5a2b9fab83ff0f8bfc99684bd5f43a37abe560f1"
|
||||
version = "v1.0.8"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:ba3b2eb0ae6fd3deac4386c02fe9d2279c9520738eb9db2f0667e74d5c7a0a61"
|
||||
name = "github.com/nlopes/slack"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "0db1d5eae1116bf7c8ed96c6749acfbf4daaec3e"
|
||||
version = "v0.3.0"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:af967afd3cbc6b0145937f4dcab78bcd93e7b2f2b618fb2bcaf7069ad5c638fa"
|
||||
name = "github.com/phayes/freeport"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "e27662a4a9d6b2083dfd7e7b5d0e30985daca925"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:6ee36f2cea425916d81fdaaf983469fc18f91b3cf090cfe90fa0a9d85b8bfab7"
|
||||
name = "github.com/sean-/seed"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "e2103e2c35297fb7e17febb81e49b312087a2372"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:65d0c541c450f6f8bb9654a3f1938407a835bf41faade00bb0fa1416de215d00"
|
||||
name = "github.com/sebdah/goldie"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "8784dd1ab561dcf43d141f6678e9e41f3d0dff55"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:1747c026a603e3c9f33f238e1d1390df2c8f48876b6bcb7a9c52c7b479e040f4"
|
||||
name = "github.com/sirupsen/logrus"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "d329d24db4313262a3b0a24d8aeb1dc4bd294fb0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:d0b38ba6da419a6d4380700218eeec8623841d44a856bb57369c172fbf692ab4"
|
||||
name = "github.com/spf13/cast"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "8965335b8c7107321228e3e3702cab9832751bac"
|
||||
version = "v1.2.0"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:b5212c335a490d958a6b1b5b48901b46e682b82fa9af3a238fa88df6eaa60873"
|
||||
name = "github.com/spf13/cobra"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "7c4570c3ebeb8129a1f7456d0908a8b676b6f9f1"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:8e243c568f36b09031ec18dff5f7d2769dcf5ca4d624ea511c8e3197dc3d352d"
|
||||
name = "github.com/spf13/pflag"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "583c0c0531f06d5278b7d917446061adc344b5cd"
|
||||
version = "v1.0.1"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:f05efcac20bea32e1fcefde9a3cbefb07e02053666c4a67681ad18c8efc682d3"
|
||||
name = "github.com/uber-go/atomic"
|
||||
packages = ["."]
|
||||
pruneopts = ""
|
||||
revision = "ca680462431f8c7f96686d91b1d9e5203c4b075b"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:16b935c128f178647036048862a21e8bfd66d1e83fb19787a8b356bdcf0de899"
|
||||
name = "golang.org/x/crypto"
|
||||
packages = [
|
||||
"ed25519",
|
||||
"ed25519/internal/edwards25519",
|
||||
"ripemd160",
|
||||
"sha3",
|
||||
"ssh/terminal",
|
||||
]
|
||||
pruneopts = ""
|
||||
revision = "f027049dab0ad238e394a753dba2d14753473a04"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:15f2fc8cc79d90b0d4d712f04bd3eb3a3856ff3dd6b610a1425113ead3501610"
|
||||
name = "golang.org/x/net"
|
||||
packages = [
|
||||
"bpf",
|
||||
"context",
|
||||
"internal/iana",
|
||||
"internal/socket",
|
||||
"ipv4",
|
||||
"ipv6",
|
||||
]
|
||||
pruneopts = ""
|
||||
revision = "19491d39cadbd9cd33f26ca22cc89ba4ba38251c"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:e8f649ecfae7835a0a27ef39fd2180f6d3c12bc422e2ae55cd611f0b283b3e6e"
|
||||
name = "golang.org/x/sys"
|
||||
packages = [
|
||||
"unix",
|
||||
"windows",
|
||||
]
|
||||
pruneopts = ""
|
||||
revision = "0718ef2ef256118d53a01598f179001ec2af7626"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:55a681cb66f28755765fa5fa5104cbd8dc85c55c02d206f9f89566451e3fe1aa"
|
||||
name = "golang.org/x/time"
|
||||
packages = ["rate"]
|
||||
pruneopts = ""
|
||||
revision = "fbb02b2291d28baffd63558aa44b4b56f178d650"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:eede11c81b63c8f6fd06ef24ba0a640dc077196ec9b7a58ecde03c82eee2f151"
|
||||
name = "google.golang.org/appengine"
|
||||
packages = ["cloudsql"]
|
||||
pruneopts = ""
|
||||
revision = "b1f26356af11148e710935ed1ac8a7f5702c7612"
|
||||
version = "v1.1.0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:05eca53b271663de74078b5484b1995a8d56668a51434a698dc5d0863035d575"
|
||||
name = "gopkg.in/nullbio/null.v6"
|
||||
packages = ["convert"]
|
||||
pruneopts = ""
|
||||
revision = "40264a2e6b7972d183906cf17663983c23231c82"
|
||||
version = "v6.3"
|
||||
|
||||
[solve-meta]
|
||||
analyzer-name = "dep"
|
||||
analyzer-version = 1
|
||||
input-imports = [
|
||||
"github.com/aws/aws-sdk-go/aws",
|
||||
"github.com/aws/aws-sdk-go/aws/awserr",
|
||||
"github.com/aws/aws-sdk-go/aws/credentials",
|
||||
"github.com/aws/aws-sdk-go/aws/session",
|
||||
"github.com/aws/aws-sdk-go/service/s3",
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager",
|
||||
"github.com/davecgh/go-spew/spew",
|
||||
"github.com/go-sql-driver/mysql",
|
||||
"github.com/gorilla/mux",
|
||||
"github.com/gorilla/rpc/v2",
|
||||
"github.com/gorilla/rpc/v2/json",
|
||||
"github.com/hashicorp/serf/serf",
|
||||
"github.com/johntdyer/slackrus",
|
||||
"github.com/lbryio/errors.go",
|
||||
"github.com/lbryio/lbry.go/crypto",
|
||||
"github.com/lbryio/lbry.go/errors",
|
||||
"github.com/lbryio/lbry.go/querytools",
|
||||
"github.com/lbryio/lbry.go/stop",
|
||||
"github.com/lbryio/lbry.go/util",
|
||||
"github.com/lyoshenka/bencode",
|
||||
"github.com/phayes/freeport",
|
||||
"github.com/sebdah/goldie",
|
||||
"github.com/sirupsen/logrus",
|
||||
"github.com/spf13/cast",
|
||||
"github.com/spf13/cobra",
|
||||
"github.com/uber-go/atomic",
|
||||
"golang.org/x/time/rate",
|
||||
]
|
||||
solver-name = "gps-cdcl"
|
||||
solver-version = 1
|
||||
-39
@@ -1,39 +0,0 @@
|
||||
[[constraint]]
|
||||
name = "github.com/aws/aws-sdk-go"
|
||||
branch = "master"
|
||||
|
||||
[[constraint]]
|
||||
name = "github.com/go-sql-driver/mysql"
|
||||
branch = "master"
|
||||
|
||||
[[constraint]]
|
||||
name = "github.com/hashicorp/serf"
|
||||
branch = "master"
|
||||
|
||||
[[constraint]]
|
||||
name = "github.com/lbryio/lbry.go"
|
||||
branch = "master"
|
||||
|
||||
[[constraint]]
|
||||
name = "github.com/sirupsen/logrus"
|
||||
branch = "master"
|
||||
|
||||
[[constraint]]
|
||||
name = "github.com/spf13/cobra"
|
||||
branch = "master"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/lyoshenka/bencode"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/uber-go/atomic"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/phayes/freeport"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/johntdyer/slackrus"
|
||||
@@ -1,6 +1,6 @@
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2016-2018 LBRY Inc
|
||||
Copyright (c) 2016-2020 LBRY Inc
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish,distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||
|
||||
|
||||
@@ -1,27 +1,37 @@
|
||||
version := $(shell git describe --dirty --always --long --abbrev=7)
|
||||
commit := $(shell git rev-parse --short HEAD)
|
||||
commit_long := $(shell git rev-parse HEAD)
|
||||
branch := $(shell git rev-parse --abbrev-ref HEAD)
|
||||
curTime := $(shell date +%s)
|
||||
|
||||
BINARY=prism-bin
|
||||
|
||||
DIR = $(shell cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)
|
||||
VENDOR_DIR = vendor
|
||||
IMPORT_PATH = github.com/lbryio/reflector.go
|
||||
LDFLAGS="-X ${IMPORT_PATH}/meta.version=$(version) -X ${IMPORT_PATH}/meta.commit=$(commit) -X ${IMPORT_PATH}/meta.commitLong=$(commit_long) -X ${IMPORT_PATH}/meta.branch=$(branch) -X '${IMPORT_PATH}/meta.Time=$(curTime)'"
|
||||
DIR = $(shell cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)
|
||||
BIN_DIR = $(DIR)/dist
|
||||
|
||||
VERSION = $(shell git --git-dir=${DIR}/.git describe --dirty --always --long --abbrev=7)
|
||||
LDFLAGS = -ldflags "-X ${IMPORT_PATH}/meta.Version=${VERSION}"
|
||||
|
||||
|
||||
.PHONY: build dep clean test
|
||||
.DEFAULT_GOAL: build
|
||||
|
||||
|
||||
build: dep
|
||||
CGO_ENABLED=0 go build ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${DIR}/${BINARY} main.go
|
||||
|
||||
dep: | $(VENDOR_DIR)
|
||||
|
||||
$(VENDOR_DIR):
|
||||
go get github.com/golang/dep/cmd/dep && dep ensure
|
||||
|
||||
clean:
|
||||
if [ -f ${DIR}/${BINARY} ]; then rm ${DIR}/${BINARY}; fi
|
||||
.DEFAULT_GOAL := linux
|
||||
|
||||
.PHONY: test
|
||||
test:
|
||||
go test ./... -v -cover
|
||||
go test -cover -v ./...
|
||||
|
||||
.PHONY: lint
|
||||
lint:
|
||||
./scripts/lint.sh
|
||||
|
||||
.PHONY: imports
|
||||
imports:
|
||||
gci write --skip-generated --custom-order -s standard -s "prefix(github.com/lbryio/reflector.go)" -s "prefix(github.com/OdyseeTeam,github.com/lbryio)" -s default .
|
||||
|
||||
.PHONY: linux
|
||||
linux:
|
||||
GOARCH=amd64 CGO_ENABLED=0 GOOS=linux go build -ldflags ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/linux_amd64/${BINARY}
|
||||
|
||||
.PHONY: macos
|
||||
macos:
|
||||
GOARCH=amd64 GOOS=darwin go build -ldflags ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/darwin_amd64/${BINARY}
|
||||
|
||||
.PHONY: image
|
||||
image:
|
||||
docker buildx build -t lbry/reflector:$(version) -t lbry/reflector:latest --platform linux/amd64 .
|
||||
+9
-10
@@ -1,14 +1,14 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"io"
|
||||
baselog "log"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/crypto"
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/lbry.go/stop"
|
||||
"github.com/lbryio/lbry.go/v2/extras/crypto"
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
|
||||
"github.com/hashicorp/serf/serf"
|
||||
log "github.com/sirupsen/logrus"
|
||||
@@ -23,13 +23,12 @@ const (
|
||||
type Cluster struct {
|
||||
OnMembershipChange func(n, total int)
|
||||
|
||||
s *serf.Serf
|
||||
eventCh chan serf.Event
|
||||
stop *stop.Group
|
||||
name string
|
||||
port int
|
||||
seedAddr string
|
||||
|
||||
s *serf.Serf
|
||||
eventCh chan serf.Event
|
||||
stop *stop.Group
|
||||
port int
|
||||
}
|
||||
|
||||
// New returns a new Cluster instance that is not connected.
|
||||
@@ -52,7 +51,7 @@ func (c *Cluster) Connect() error {
|
||||
conf.MemberlistConfig.AdvertisePort = c.port
|
||||
conf.NodeName = c.name
|
||||
|
||||
nullLogger := baselog.New(ioutil.Discard, "", 0)
|
||||
nullLogger := baselog.New(io.Discard, "", 0)
|
||||
conf.Logger = nullLogger
|
||||
|
||||
c.eventCh = make(chan serf.Event)
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"syscall"
|
||||
|
||||
"github.com/lbryio/reflector.go/config"
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func init() {
|
||||
var cmd = &cobra.Command{
|
||||
Use: "blobcache",
|
||||
Short: "Run blobcache server",
|
||||
Run: blobcacheCmd,
|
||||
}
|
||||
|
||||
cmd.Flags().IntVar(&metricsPort, "metrics-port", 2112, "The port reflector will use for prometheus metrics")
|
||||
cmd.Flags().BoolVar(&disableBlocklist, "disable-blocklist", false, "Disable blocklist watching/updating")
|
||||
|
||||
rootCmd.AddCommand(cmd)
|
||||
}
|
||||
|
||||
func blobcacheCmd(cmd *cobra.Command, args []string) {
|
||||
store, err := config.LoadStores(conf, "blobcache")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer store.Shutdown()
|
||||
|
||||
servers, err := config.LoadServers(store, conf, "blobcache")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
for _, s := range servers {
|
||||
err = s.Start()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer s.Shutdown()
|
||||
}
|
||||
|
||||
metricsServer := metrics.NewServer(":"+strconv.Itoa(metricsPort), "/metrics")
|
||||
metricsServer.Start()
|
||||
defer metricsServer.Shutdown()
|
||||
|
||||
interruptChan := make(chan os.Signal, 1)
|
||||
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
|
||||
<-interruptChan
|
||||
}
|
||||
+2
-1
@@ -6,9 +6,10 @@ import (
|
||||
"strconv"
|
||||
"syscall"
|
||||
|
||||
"github.com/lbryio/lbry.go/crypto"
|
||||
"github.com/lbryio/reflector.go/cluster"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/crypto"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/schema/stake"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/gogo/protobuf/jsonpb"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func init() {
|
||||
var cmd = &cobra.Command{
|
||||
Use: "decode VALUE",
|
||||
Short: "Decode a claim value",
|
||||
Args: cobra.ExactArgs(1),
|
||||
Run: decodeCmd,
|
||||
}
|
||||
rootCmd.AddCommand(cmd)
|
||||
}
|
||||
|
||||
func decodeCmd(cmd *cobra.Command, args []string) {
|
||||
c, err := stake.DecodeClaimHex(args[0], "")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
m := jsonpb.Marshaler{Indent: " "}
|
||||
|
||||
if stream := c.Claim.GetStream(); stream != nil {
|
||||
json, err := m.MarshalToString(stream)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
fmt.Println(json)
|
||||
fmt.Printf("SD hash as hex: %s\n", hex.EncodeToString(stream.GetSource().GetSdHash()))
|
||||
} else if channel := c.Claim.GetChannel(); channel != nil {
|
||||
json, err := m.MarshalToString(channel)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
fmt.Println(json)
|
||||
} else if repost := c.Claim.GetRepost(); repost != nil {
|
||||
json, err := m.MarshalToString(repost)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
fmt.Println(json)
|
||||
} else {
|
||||
spew.Dump(c)
|
||||
}
|
||||
}
|
||||
+6
-6
@@ -1,7 +1,6 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"log"
|
||||
"net"
|
||||
"os"
|
||||
"os/signal"
|
||||
@@ -9,15 +8,16 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/dht"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
"github.com/lbryio/lbry.go/v2/dht"
|
||||
"github.com/lbryio/lbry.go/v2/dht/bits"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var dhtNodeID string
|
||||
var dhtPort int
|
||||
var dhtRpcPort int
|
||||
var dhtRPCPort int
|
||||
var dhtSeeds []string
|
||||
|
||||
func init() {
|
||||
@@ -30,7 +30,7 @@ func init() {
|
||||
}
|
||||
cmd.PersistentFlags().StringVar(&dhtNodeID, "nodeID", "", "nodeID in hex")
|
||||
cmd.PersistentFlags().IntVar(&dhtPort, "port", 4567, "Port to start DHT on")
|
||||
cmd.PersistentFlags().IntVar(&dhtRpcPort, "rpcPort", 0, "Port to listen for rpc commands on")
|
||||
cmd.PersistentFlags().IntVar(&dhtRPCPort, "rpcPort", 0, "Port to listen for rpc commands on")
|
||||
cmd.PersistentFlags().StringSliceVar(&dhtSeeds, "seeds", []string{}, "Addresses of seed nodes")
|
||||
rootCmd.AddCommand(cmd)
|
||||
}
|
||||
@@ -56,7 +56,7 @@ func dhtCmd(cmd *cobra.Command, args []string) {
|
||||
|
||||
dhtConf := dht.NewStandardConfig()
|
||||
dhtConf.Address = "0.0.0.0:" + strconv.Itoa(dhtPort)
|
||||
dhtConf.RPCPort = dhtRpcPort
|
||||
dhtConf.RPCPort = dhtRPCPort
|
||||
if len(dhtSeeds) > 0 {
|
||||
dhtConf.SeedNodes = dhtSeeds
|
||||
}
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func init() {
|
||||
var cmd = &cobra.Command{
|
||||
Use: "getstream ADDRESS:PORT SDHASH",
|
||||
Short: "Get a stream from a reflector server",
|
||||
Args: cobra.ExactArgs(2),
|
||||
Run: getStreamCmd,
|
||||
}
|
||||
rootCmd.AddCommand(cmd)
|
||||
}
|
||||
|
||||
func getStreamCmd(cmd *cobra.Command, args []string) {
|
||||
addr := args[0]
|
||||
sdHash := args[1]
|
||||
|
||||
s := store.NewCachingStore(store.CachingParams{
|
||||
Name: "getstream",
|
||||
Cache: store.NewPeerStore(store.PeerParams{
|
||||
Name: "getstream",
|
||||
Address: addr,
|
||||
Timeout: 30 * time.Second,
|
||||
}),
|
||||
Origin: store.NewDiskStore(store.DiskParams{
|
||||
Name: "getstream",
|
||||
MountPoint: "/tmp/lbry_downloaded_blobs",
|
||||
ShardingSize: 2,
|
||||
}),
|
||||
})
|
||||
|
||||
wd, err := os.Getwd()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
var sd stream.SDBlob
|
||||
|
||||
sdb, _, err := s.Get(sdHash)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
err = sd.FromBlob(sdb)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
filename := sd.SuggestedFileName
|
||||
if filename == "" {
|
||||
filename = "stream_" + time.Now().Format("20060102_150405")
|
||||
}
|
||||
|
||||
f, err := os.Create(wd + "/" + filename)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
for i := 0; i < len(sd.BlobInfos)-1; i++ {
|
||||
b, _, err := s.Get(hex.EncodeToString(sd.BlobInfos[i].BlobHash))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
data, err := b.Plaintext(sd.Key, sd.BlobInfos[i].IV)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = f.Write(data)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"crypto/sha512"
|
||||
"encoding/hex"
|
||||
"os"
|
||||
"path"
|
||||
"runtime"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/meta"
|
||||
"github.com/lbryio/reflector.go/store/speedwalk"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var threads int
|
||||
|
||||
func init() {
|
||||
var cmd = &cobra.Command{
|
||||
Use: "check-integrity",
|
||||
Short: "check blobs integrity for a given path",
|
||||
Run: integrityCheckCmd,
|
||||
}
|
||||
cmd.Flags().StringVar(&diskStorePath, "store-path", "", "path of the store where all blobs are cached")
|
||||
cmd.Flags().IntVar(&threads, "threads", runtime.NumCPU()-1, "number of concurrent threads to process blobs")
|
||||
rootCmd.AddCommand(cmd)
|
||||
}
|
||||
|
||||
func integrityCheckCmd(cmd *cobra.Command, args []string) {
|
||||
log.Printf("reflector %s", meta.VersionString())
|
||||
if diskStorePath == "" {
|
||||
log.Fatal("store-path must be defined")
|
||||
}
|
||||
|
||||
blobs, err := speedwalk.AllFiles(diskStorePath, true)
|
||||
if err != nil {
|
||||
log.Fatalf("error while reading blobs from disk %s", errors.FullTrace(err))
|
||||
}
|
||||
tasks := make(chan string, len(blobs))
|
||||
done := make(chan bool)
|
||||
processed := new(int32)
|
||||
go produce(tasks, blobs)
|
||||
cpus := runtime.NumCPU()
|
||||
for i := 0; i < cpus-1; i++ {
|
||||
go consume(i, tasks, done, len(blobs), processed)
|
||||
}
|
||||
<-done
|
||||
}
|
||||
|
||||
func produce(tasks chan<- string, blobs []string) {
|
||||
for _, b := range blobs {
|
||||
tasks <- b
|
||||
}
|
||||
close(tasks)
|
||||
}
|
||||
|
||||
func consume(worker int, tasks <-chan string, done chan<- bool, totalTasks int, processed *int32) {
|
||||
start := time.Now()
|
||||
|
||||
for b := range tasks {
|
||||
processedSoFar := atomic.AddInt32(processed, 1)
|
||||
if worker == 0 {
|
||||
remaining := int32(totalTasks) - processedSoFar
|
||||
timePerBlob := time.Since(start).Microseconds() / int64(processedSoFar)
|
||||
remainingTime := time.Duration(int64(remaining)*timePerBlob) * time.Microsecond
|
||||
log.Infof("[T%d] %d/%d blobs processed so far. ETA: %s", worker, processedSoFar, totalTasks, remainingTime.String())
|
||||
}
|
||||
blobPath := path.Join(diskStorePath, b[:2], b)
|
||||
blob, err := os.ReadFile(blobPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
continue
|
||||
}
|
||||
log.Errorf("[Worker %d] Error looking up blob %s: %s", worker, b, err.Error())
|
||||
continue
|
||||
}
|
||||
hashBytes := sha512.Sum384(blob)
|
||||
readHash := hex.EncodeToString(hashBytes[:])
|
||||
if readHash != b {
|
||||
log.Infof("[%s] found a broken blob while reading from disk. Actual hash: %s", b, readHash)
|
||||
err := os.Remove(blobPath)
|
||||
if err != nil {
|
||||
log.Errorf("Error while deleting broken blob %s: %s", b, err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
done <- true
|
||||
}
|
||||
-44
@@ -1,44 +0,0 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"syscall"
|
||||
|
||||
"github.com/lbryio/reflector.go/db"
|
||||
"github.com/lbryio/reflector.go/peer"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func init() {
|
||||
var cmd = &cobra.Command{
|
||||
Use: "peer",
|
||||
Short: "Run peer server",
|
||||
Run: peerCmd,
|
||||
}
|
||||
rootCmd.AddCommand(cmd)
|
||||
}
|
||||
|
||||
func peerCmd(cmd *cobra.Command, args []string) {
|
||||
db := new(db.SQL)
|
||||
err := db.Connect(globalConfig.DBConn)
|
||||
checkErr(err)
|
||||
|
||||
s3 := store.NewS3BlobStore(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
|
||||
combo := store.NewDBBackedS3Store(s3, db)
|
||||
peerServer := peer.NewServer(combo)
|
||||
|
||||
err = peerServer.Start(":" + strconv.Itoa(peer.DefaultPort))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
interruptChan := make(chan os.Signal, 1)
|
||||
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
|
||||
<-interruptChan
|
||||
peerServer.Shutdown()
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"github.com/lbryio/reflector.go/db"
|
||||
"github.com/lbryio/reflector.go/meta"
|
||||
"github.com/lbryio/reflector.go/store/speedwalk"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var (
|
||||
diskStorePath string
|
||||
)
|
||||
|
||||
func init() {
|
||||
var cmd = &cobra.Command{
|
||||
Use: "populate-db",
|
||||
Short: "populate local database with blobs from a disk storage",
|
||||
Run: populateDbCmd,
|
||||
}
|
||||
cmd.Flags().StringVar(&diskStorePath, "store-path", "",
|
||||
"path of the store where all blobs are cached")
|
||||
rootCmd.AddCommand(cmd)
|
||||
}
|
||||
|
||||
func populateDbCmd(cmd *cobra.Command, args []string) {
|
||||
log.Printf("reflector %s", meta.VersionString())
|
||||
if diskStorePath == "" {
|
||||
log.Fatal("store-path must be defined")
|
||||
}
|
||||
localDb := &db.SQL{
|
||||
SoftDelete: true,
|
||||
TrackingLevel: db.TrackAccessBlobs,
|
||||
LogQueries: log.GetLevel() == log.DebugLevel,
|
||||
}
|
||||
err := localDb.Connect("reflector:reflector@tcp(localhost:3306)/reflector")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
blobs, err := speedwalk.AllFiles(diskStorePath, true)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
err = localDb.AddBlobs(blobs)
|
||||
if err != nil {
|
||||
log.Errorf("error while storing to db: %s", errors.FullTrace(err))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/lbryio/reflector.go/publish"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/lbrycrd"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func init() {
|
||||
var cmd = &cobra.Command{
|
||||
Use: "publish FILE",
|
||||
Short: "Publish a file",
|
||||
Args: cobra.ExactArgs(1),
|
||||
Run: publishCmd,
|
||||
}
|
||||
cmd.Flags().String("name", "", "Claim name")
|
||||
cmd.Flags().String("title", "", "Title of the content")
|
||||
cmd.Flags().String("description", "", "Description of the content")
|
||||
cmd.Flags().String("author", "", "Content author")
|
||||
cmd.Flags().String("tags", "", "Comma-separated list of tags")
|
||||
cmd.Flags().Int64("release-time", 0, "original public release of content, seconds since UNIX epoch")
|
||||
rootCmd.AddCommand(cmd)
|
||||
}
|
||||
|
||||
func publishCmd(cmd *cobra.Command, args []string) {
|
||||
var err error
|
||||
|
||||
claimName := mustGetFlagString(cmd, "name")
|
||||
if claimName == "" {
|
||||
log.Errorln("--name required")
|
||||
return
|
||||
}
|
||||
|
||||
path := args[0]
|
||||
|
||||
client, err := lbrycrd.NewWithDefaultURL(nil)
|
||||
checkErr(err)
|
||||
|
||||
tx, txid, err := publish.Publish(
|
||||
client,
|
||||
path,
|
||||
claimName,
|
||||
"bSzpgkTnAoiT2YAhUShPpfpajPESfNXVTu",
|
||||
publish.Details{
|
||||
Title: mustGetFlagString(cmd, "title"),
|
||||
Description: mustGetFlagString(cmd, "description"),
|
||||
Author: mustGetFlagString(cmd, "author"),
|
||||
Tags: nil,
|
||||
ReleaseTime: mustGetFlagInt64(cmd, "release-time"),
|
||||
},
|
||||
"reflector.lbry.com:5566",
|
||||
)
|
||||
checkErr(err)
|
||||
|
||||
decoded, err := publish.Decode(client, tx)
|
||||
checkErr(err)
|
||||
|
||||
fmt.Printf("TX: %s\n\n", decoded)
|
||||
fmt.Printf("TXID: %s\n", txid.String())
|
||||
}
|
||||
+40
-11
@@ -5,43 +5,72 @@ import (
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/db"
|
||||
"github.com/lbryio/reflector.go/meta"
|
||||
"github.com/lbryio/reflector.go/config"
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var (
|
||||
//port configuration
|
||||
receiverPort int
|
||||
metricsPort int
|
||||
|
||||
//flags configuration
|
||||
disableBlocklist bool
|
||||
)
|
||||
|
||||
func init() {
|
||||
var cmd = &cobra.Command{
|
||||
Use: "reflector",
|
||||
Short: "Run reflector server",
|
||||
Run: reflectorCmd,
|
||||
}
|
||||
|
||||
cmd.Flags().IntVar(&metricsPort, "metrics-port", 2112, "The port reflector will use for prometheus metrics")
|
||||
cmd.Flags().IntVar(&receiverPort, "receiver-port", 5566, "The port reflector will receive content from")
|
||||
cmd.Flags().BoolVar(&disableBlocklist, "disable-blocklist", false, "Disable blocklist watching/updating")
|
||||
|
||||
rootCmd.AddCommand(cmd)
|
||||
}
|
||||
|
||||
func reflectorCmd(cmd *cobra.Command, args []string) {
|
||||
log.Printf("reflector version %s", meta.Version)
|
||||
db := new(db.SQL)
|
||||
err := db.Connect(globalConfig.DBConn)
|
||||
store, err := config.LoadStores(conf, "reflector")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer store.Shutdown()
|
||||
|
||||
servers, err := config.LoadServers(store, conf, "reflector")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
for _, s := range servers {
|
||||
err = s.Start()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer s.Shutdown()
|
||||
}
|
||||
|
||||
s3 := store.NewS3BlobStore(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
|
||||
combo := store.NewDBBackedS3Store(s3, db)
|
||||
reflectorServer := reflector.NewServer(combo)
|
||||
err = reflectorServer.Start(":" + strconv.Itoa(reflector.DefaultPort))
|
||||
reflectorServer := reflector.NewIngestionServer(store)
|
||||
reflectorServer.Timeout = 3 * time.Minute
|
||||
reflectorServer.EnableBlocklist = !disableBlocklist
|
||||
err = reflectorServer.Start(":" + strconv.Itoa(receiverPort))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer reflectorServer.Shutdown()
|
||||
|
||||
metricsServer := metrics.NewServer(":"+strconv.Itoa(metricsPort), "/metrics")
|
||||
metricsServer.Start()
|
||||
defer metricsServer.Shutdown()
|
||||
|
||||
interruptChan := make(chan os.Signal, 1)
|
||||
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
|
||||
<-interruptChan
|
||||
reflectorServer.Shutdown()
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/lbryio/reflector.go/wallet"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func init() {
|
||||
var cmd = &cobra.Command{
|
||||
Use: "resolve ADDRESS:PORT URL",
|
||||
Short: "Resolve a URL",
|
||||
Args: cobra.ExactArgs(2),
|
||||
Run: resolveCmd,
|
||||
}
|
||||
rootCmd.AddCommand(cmd)
|
||||
}
|
||||
|
||||
func resolveCmd(cmd *cobra.Command, args []string) {
|
||||
addr := args[0]
|
||||
url := args[1]
|
||||
|
||||
node := wallet.NewNode()
|
||||
defer node.Shutdown()
|
||||
err := node.Connect([]string{addr}, nil)
|
||||
checkErr(err)
|
||||
|
||||
output, err := node.Resolve(url)
|
||||
checkErr(err)
|
||||
|
||||
claim, err := node.GetClaimInTx(hex.EncodeToString(rev(output.GetTxHash())), int(output.GetNout()))
|
||||
checkErr(err)
|
||||
|
||||
jsonClaim, err := json.MarshalIndent(claim, "", " ")
|
||||
checkErr(err)
|
||||
|
||||
fmt.Println(string(jsonClaim))
|
||||
}
|
||||
|
||||
func rev(b []byte) []byte {
|
||||
r := make([]byte, len(b))
|
||||
for left, right := 0, len(b)-1; left < right; left, right = left+1, right-1 {
|
||||
r[left], r[right] = b[right], b[left]
|
||||
}
|
||||
return r
|
||||
}
|
||||
+73
-39
@@ -1,17 +1,15 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/lbry.go/util"
|
||||
"github.com/lbryio/reflector.go/dht"
|
||||
"github.com/lbryio/lbry.go/v2/dht"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
|
||||
"github.com/johntdyer/slackrus"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
// Config is the base configuration for Prism when no sub commands are used.
|
||||
@@ -20,8 +18,12 @@ type Config struct {
|
||||
AwsSecret string `json:"aws_secret"`
|
||||
BucketRegion string `json:"bucket_region"`
|
||||
BucketName string `json:"bucket_name"`
|
||||
S3Endpoint string `json:"s3_endpoint"`
|
||||
DBConn string `json:"db_conn"`
|
||||
SlackHookURL string `json:"slack_hook_url"`
|
||||
SlackChannel string `json:"slack_channel"`
|
||||
UpdateBinURL string `json:"update_bin_url"`
|
||||
UpdateCmd string `json:"update_cmd"`
|
||||
}
|
||||
|
||||
var verbose []string
|
||||
@@ -33,7 +35,8 @@ const (
|
||||
)
|
||||
|
||||
var conf string
|
||||
var globalConfig Config
|
||||
|
||||
//var globalConfig Config
|
||||
|
||||
var rootCmd = &cobra.Command{
|
||||
Use: "prism",
|
||||
@@ -46,7 +49,7 @@ var rootCmd = &cobra.Command{
|
||||
|
||||
func init() {
|
||||
rootCmd.PersistentFlags().StringSliceVarP(&verbose, "verbose", "v", []string{}, "Verbose logging for specific components")
|
||||
rootCmd.PersistentFlags().StringVar(&conf, "conf", "config.json", "Path to config")
|
||||
rootCmd.PersistentFlags().StringVar(&conf, "conf-dir", "./", "Path to config directory")
|
||||
}
|
||||
|
||||
// Execute adds all child commands to the root command and sets flags appropriately.
|
||||
@@ -65,8 +68,11 @@ func preRun(cmd *cobra.Command, args []string) {
|
||||
debugLogger.SetOutput(os.Stderr)
|
||||
|
||||
if util.InSlice(verboseAll, verbose) {
|
||||
logrus.Info("global verbose logging enabled")
|
||||
logrus.SetLevel(logrus.DebugLevel)
|
||||
verbose = []string{verboseDHT, verboseNodeFinder}
|
||||
} else if len(verbose) > 0 {
|
||||
logrus.Infof("verbose logging enabled for: %s", strings.Join(verbose, ", "))
|
||||
}
|
||||
|
||||
for _, debugType := range verbose {
|
||||
@@ -78,30 +84,40 @@ func preRun(cmd *cobra.Command, args []string) {
|
||||
}
|
||||
}
|
||||
|
||||
var err error
|
||||
//var err error
|
||||
if conf == "" {
|
||||
logrus.Errorln("--conf flag required")
|
||||
os.Exit(1)
|
||||
} else if conf != "none" {
|
||||
globalConfig, err = loadConfig(conf)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
//else if conf != "none" {
|
||||
// globalConfig, err = loadConfig(conf)
|
||||
// if err != nil {
|
||||
// logrus.Error(err)
|
||||
// os.Exit(1)
|
||||
// }
|
||||
//}
|
||||
viper.AddConfigPath(conf)
|
||||
|
||||
if globalConfig.SlackHookURL != "" {
|
||||
hook := &slackrus.SlackrusHook{
|
||||
HookURL: globalConfig.SlackHookURL,
|
||||
AcceptedLevels: slackrus.LevelThreshold(logrus.InfoLevel),
|
||||
Channel: "#reflector-logs",
|
||||
//IconEmoji: ":ghost:",
|
||||
//Username: "reflector.go",
|
||||
}
|
||||
//logrus.SetFormatter(&logrus.JSONFormatter{})
|
||||
logrus.AddHook(hook)
|
||||
debugLogger.AddHook(hook)
|
||||
}
|
||||
//if globalConfig.SlackHookURL != "" {
|
||||
// hook := &slackrus.SlackrusHook{
|
||||
// HookURL: globalConfig.SlackHookURL,
|
||||
// AcceptedLevels: slackrus.LevelThreshold(logrus.InfoLevel),
|
||||
// Channel: globalConfig.SlackChannel,
|
||||
// //IconEmoji: ":ghost:",
|
||||
// //Username: "reflector.go",
|
||||
// }
|
||||
// //logrus.SetFormatter(&logrus.JSONFormatter{})
|
||||
// logrus.AddHook(hook)
|
||||
// debugLogger.AddHook(hook)
|
||||
//}
|
||||
|
||||
//if globalConfig.UpdateBinURL != "" {
|
||||
// if globalConfig.UpdateCmd == "" {
|
||||
// logrus.Warnln("update_cmd is empty in conf file")
|
||||
// }
|
||||
// logrus.Println("starting update checker")
|
||||
// go updater.Run(globalConfig.UpdateBinURL, globalConfig.UpdateCmd)
|
||||
//}
|
||||
}
|
||||
|
||||
func checkErr(err error) {
|
||||
@@ -122,17 +138,35 @@ func argFuncs(funcs ...cobra.PositionalArgs) cobra.PositionalArgs {
|
||||
}
|
||||
}
|
||||
|
||||
func loadConfig(path string) (Config, error) {
|
||||
var c Config
|
||||
//func loadConfig(path string) (Config, error) {
|
||||
// var c Config
|
||||
//
|
||||
// raw, err := os.ReadFile(path)
|
||||
// if err != nil {
|
||||
// if os.IsNotExist(err) {
|
||||
// return c, errors.Err("config file not found")
|
||||
// }
|
||||
// return c, errors.Err(err)
|
||||
// }
|
||||
//
|
||||
// err = json.Unmarshal(raw, &c)
|
||||
// return c, errors.Err(err)
|
||||
//}
|
||||
|
||||
raw, err := ioutil.ReadFile(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return c, errors.Err("config file not found")
|
||||
}
|
||||
return c, err
|
||||
}
|
||||
|
||||
err = json.Unmarshal(raw, &c)
|
||||
return c, err
|
||||
func mustGetFlagString(cmd *cobra.Command, name string) string {
|
||||
v, err := cmd.Flags().GetString(name)
|
||||
checkErr(err)
|
||||
return v
|
||||
}
|
||||
|
||||
func mustGetFlagInt64(cmd *cobra.Command, name string) int64 {
|
||||
v, err := cmd.Flags().GetInt64(name)
|
||||
checkErr(err)
|
||||
return v
|
||||
}
|
||||
|
||||
//func mustGetFlagBool(cmd *cobra.Command, name string) bool {
|
||||
// v, err := cmd.Flags().GetBool(name)
|
||||
// checkErr(err)
|
||||
// return v
|
||||
//}
|
||||
+160
@@ -0,0 +1,160 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path"
|
||||
"syscall"
|
||||
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func init() {
|
||||
var cmd = &cobra.Command{
|
||||
Use: "send ADDRESS:PORT PATH",
|
||||
Short: "Send a file to a reflector",
|
||||
Args: cobra.ExactArgs(2),
|
||||
Run: sendCmd,
|
||||
}
|
||||
cmd.PersistentFlags().String("sd-cache", "", "path to dir where sd blobs will be cached")
|
||||
rootCmd.AddCommand(cmd)
|
||||
}
|
||||
|
||||
// todo: if retrying a large file is slow, we can add the ability to seek ahead in the file so we're not
|
||||
// re-uploading blobs that already exist
|
||||
|
||||
var hackyReflector reflector.Client
|
||||
|
||||
func sendCmd(cmd *cobra.Command, args []string) {
|
||||
reflectorAddress := args[0]
|
||||
err := hackyReflector.Connect(reflectorAddress)
|
||||
checkErr(err)
|
||||
defer func() { _ = hackyReflector.Close() }()
|
||||
|
||||
filePath := args[1]
|
||||
file, err := os.Open(filePath)
|
||||
checkErr(err)
|
||||
defer func() { _ = file.Close() }()
|
||||
sdCachePath := ""
|
||||
sdCacheDir := mustGetFlagString(cmd, "sd-cache")
|
||||
if sdCacheDir != "" {
|
||||
if _, statErr := os.Stat(sdCacheDir); os.IsNotExist(statErr) {
|
||||
err = os.MkdirAll(sdCacheDir, 0777)
|
||||
checkErr(err)
|
||||
}
|
||||
sdCachePath = path.Join(sdCacheDir, filePath+".sdblob")
|
||||
}
|
||||
|
||||
var enc *stream.Encoder
|
||||
|
||||
if sdCachePath != "" {
|
||||
if _, statErr := os.Stat(sdCachePath); !os.IsNotExist(statErr) {
|
||||
var sdBlob []byte
|
||||
sdBlob, err = os.ReadFile(sdCachePath)
|
||||
checkErr(err)
|
||||
cachedSDBlob := &stream.SDBlob{}
|
||||
err = cachedSDBlob.FromBlob(sdBlob)
|
||||
checkErr(err)
|
||||
enc = stream.NewEncoderFromSD(file, cachedSDBlob)
|
||||
}
|
||||
}
|
||||
if enc == nil {
|
||||
enc = stream.NewEncoder(file)
|
||||
}
|
||||
|
||||
exitCode := 0
|
||||
|
||||
var killed bool
|
||||
interruptChan := make(chan os.Signal, 1)
|
||||
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
|
||||
go func() {
|
||||
sig := <-interruptChan
|
||||
fmt.Printf("caught %s, exiting...\n", sig.String())
|
||||
killed = true
|
||||
exitCode = 1
|
||||
}()
|
||||
|
||||
for {
|
||||
if killed {
|
||||
break
|
||||
}
|
||||
|
||||
var b stream.Blob
|
||||
b, err = enc.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Printf("error reading next blob: %v\n", err)
|
||||
exitCode = 1
|
||||
break
|
||||
}
|
||||
|
||||
err = hackyReflect(b, false)
|
||||
if err != nil {
|
||||
fmt.Printf("error reflecting blob %s: %v\n", b.HashHex()[:8], err)
|
||||
exitCode = 1
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
sd := enc.SDBlob()
|
||||
//sd.StreamName = filepath.Base(filePath)
|
||||
//sd.SuggestedFileName = filepath.Base(filePath)
|
||||
err = os.WriteFile(sdCachePath, sd.ToBlob(), 0666)
|
||||
if err != nil {
|
||||
fmt.Printf("error saving sd blob: %v\n", err)
|
||||
fmt.Println(sd.ToJson())
|
||||
exitCode = 1
|
||||
}
|
||||
|
||||
if killed {
|
||||
os.Exit(exitCode)
|
||||
}
|
||||
|
||||
if reflectorAddress != "" {
|
||||
err = hackyReflect(sd.ToBlob(), true)
|
||||
if err != nil {
|
||||
fmt.Printf("error reflecting sd blob %s: %v\n", sd.HashHex()[:8], err)
|
||||
exitCode = 1
|
||||
}
|
||||
}
|
||||
|
||||
ret := struct {
|
||||
SDHash string `json:"sd_hash"`
|
||||
SourceHash string `json:"source_hash"`
|
||||
}{
|
||||
SDHash: sd.HashHex(),
|
||||
SourceHash: hex.EncodeToString(enc.SourceHash()),
|
||||
}
|
||||
|
||||
j, err := json.MarshalIndent(ret, "", " ")
|
||||
checkErr(err)
|
||||
fmt.Println(string(j))
|
||||
os.Exit(exitCode)
|
||||
}
|
||||
|
||||
func hackyReflect(b stream.Blob, sd bool) error {
|
||||
var err error
|
||||
if sd {
|
||||
err = hackyReflector.SendSDBlob(b)
|
||||
} else {
|
||||
err = hackyReflector.SendBlob(b)
|
||||
}
|
||||
|
||||
if errors.Is(err, reflector.ErrBlobExists) {
|
||||
//fmt.Printf("%s already reflected\n", b.HashHex()[:8])
|
||||
return nil
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"os"
|
||||
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func init() {
|
||||
var cmd = &cobra.Command{
|
||||
Use: "sendblob ADDRESS:PORT [PATH]",
|
||||
Short: "Send a random blob to a reflector server",
|
||||
Args: cobra.RangeArgs(1, 2),
|
||||
Run: sendBlobCmd,
|
||||
}
|
||||
rootCmd.AddCommand(cmd)
|
||||
}
|
||||
|
||||
func sendBlobCmd(cmd *cobra.Command, args []string) {
|
||||
addr := args[0]
|
||||
var path string
|
||||
if len(args) >= 2 {
|
||||
path = args[1]
|
||||
}
|
||||
|
||||
c := reflector.Client{}
|
||||
err := c.Connect(addr)
|
||||
if err != nil {
|
||||
log.Fatal("error connecting client to server: ", err)
|
||||
}
|
||||
|
||||
if path == "" {
|
||||
blob := make(stream.Blob, 1024)
|
||||
_, err = rand.Read(blob)
|
||||
if err != nil {
|
||||
log.Fatal("failed to make random blob: ", err)
|
||||
}
|
||||
|
||||
err = c.SendBlob(blob)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
file, err := os.Open(path)
|
||||
checkErr(err)
|
||||
defer func() { _ = file.Close() }()
|
||||
s, err := stream.New(file)
|
||||
checkErr(err)
|
||||
|
||||
sdBlob := &stream.SDBlob{}
|
||||
err = sdBlob.FromBlob(s[0])
|
||||
checkErr(err)
|
||||
|
||||
for i, b := range s {
|
||||
if i == 0 {
|
||||
err = c.SendSDBlob(b)
|
||||
} else {
|
||||
err = c.SendBlob(b)
|
||||
}
|
||||
checkErr(err)
|
||||
}
|
||||
}
|
||||
@@ -1,99 +0,0 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"github.com/lbryio/reflector.go/cluster"
|
||||
"github.com/lbryio/reflector.go/db"
|
||||
"github.com/lbryio/reflector.go/dht"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
"github.com/lbryio/reflector.go/peer"
|
||||
"github.com/lbryio/reflector.go/prism"
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
const (
|
||||
startNewCluster = "new"
|
||||
)
|
||||
|
||||
var (
|
||||
startClusterPort int
|
||||
startPeerPort int
|
||||
startReflectorPort int
|
||||
startDhtPort int
|
||||
startDhtSeeds []string
|
||||
startHashRange string
|
||||
)
|
||||
|
||||
func init() {
|
||||
var cmd = &cobra.Command{
|
||||
Use: `start [cluster-address|"new"]`,
|
||||
Short: "Runs full prism application with cluster, dht, peer server, and reflector server.",
|
||||
Run: startCmd,
|
||||
Args: cobra.ExactArgs(1),
|
||||
}
|
||||
cmd.PersistentFlags().IntVar(&startClusterPort, "cluster-port", cluster.DefaultPort, "Port that cluster listens on")
|
||||
cmd.PersistentFlags().IntVar(&startPeerPort, "peer-port", peer.DefaultPort, "Port to start peer protocol on")
|
||||
cmd.PersistentFlags().IntVar(&startReflectorPort, "reflector-port", reflector.DefaultPort, "Port to start reflector protocol on")
|
||||
cmd.PersistentFlags().IntVar(&startDhtPort, "dht-port", dht.DefaultPort, "Port that dht will listen on")
|
||||
cmd.PersistentFlags().StringSliceVar(&startDhtSeeds, "dht-seeds", []string{}, "Comma-separated list of dht seed nodes (addr:port,addr:port,...)")
|
||||
|
||||
cmd.PersistentFlags().StringVar(&startHashRange, "hash-range", "", "Limit on range of hashes to announce (start-end)")
|
||||
|
||||
rootCmd.AddCommand(cmd)
|
||||
}
|
||||
|
||||
func startCmd(cmd *cobra.Command, args []string) {
|
||||
db := new(db.SQL)
|
||||
err := db.Connect(globalConfig.DBConn)
|
||||
checkErr(err)
|
||||
s3 := store.NewS3BlobStore(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
|
||||
comboStore := store.NewDBBackedS3Store(s3, db)
|
||||
|
||||
conf := prism.DefaultConf()
|
||||
|
||||
// TODO: args we need:
|
||||
// minNodes - minimum number of nodes before announcing starts. otherwise first node will try to announce all the blobs in the db
|
||||
// or maybe we should do maxHashesPerNode?
|
||||
// in either case, this should not kill the cluster, but should only limit announces (and notify when some hashes are being left unannounced)
|
||||
|
||||
if args[0] != startNewCluster {
|
||||
conf.ClusterSeedAddr = args[0]
|
||||
}
|
||||
|
||||
conf.DB = db
|
||||
conf.Blobs = comboStore
|
||||
conf.DhtAddress = "0.0.0.0:" + strconv.Itoa(startDhtPort)
|
||||
conf.DhtSeedNodes = startDhtSeeds
|
||||
conf.ClusterPort = startClusterPort
|
||||
conf.PeerPort = startPeerPort
|
||||
conf.ReflectorPort = startReflectorPort
|
||||
|
||||
if startHashRange != "" {
|
||||
hashRange := strings.Split(startHashRange, "-")
|
||||
if len(hashRange) != 2 {
|
||||
log.Fatal("invalid hash range")
|
||||
}
|
||||
r := bits.Range{Start: bits.FromShortHexP(hashRange[0]), End: bits.FromShortHexP(hashRange[1])}
|
||||
conf.HashRange = &r
|
||||
}
|
||||
|
||||
p := prism.New(conf)
|
||||
err = p.Start()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
interruptChan := make(chan os.Signal, 1)
|
||||
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
|
||||
<-interruptChan
|
||||
p.Shutdown()
|
||||
}
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/meta"
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
"github.com/lbryio/reflector.go/server/peer"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func init() {
|
||||
var cmd = &cobra.Command{
|
||||
Use: "test",
|
||||
Short: "Test things",
|
||||
Run: testCmd,
|
||||
}
|
||||
rootCmd.AddCommand(cmd)
|
||||
}
|
||||
|
||||
func testCmd(cmd *cobra.Command, args []string) {
|
||||
log.Printf("reflector %s", meta.VersionString())
|
||||
|
||||
memStore := store.NewMemStore(store.MemParams{Name: "test"})
|
||||
|
||||
reflectorServer := reflector.NewIngestionServer(memStore)
|
||||
reflectorServer.Timeout = 3 * time.Minute
|
||||
|
||||
err := reflectorServer.Start(":" + strconv.Itoa(reflector.DefaultPort))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
peerServer := peer.NewServer(memStore, fmt.Sprintf(":%d", reflector.DefaultPort+1))
|
||||
err = peerServer.Start()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
interruptChan := make(chan os.Signal, 1)
|
||||
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
|
||||
<-interruptChan
|
||||
peerServer.Shutdown()
|
||||
reflectorServer.Shutdown()
|
||||
|
||||
fmt.Println("Blobs in store")
|
||||
for hash, blob := range memStore.Debug() {
|
||||
fmt.Printf("%s: %d bytes\n", hash, len(blob))
|
||||
}
|
||||
}
|
||||
+16
-229
@@ -1,20 +1,12 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/db"
|
||||
"github.com/lbryio/reflector.go/peer"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/stop"
|
||||
"github.com/lbryio/reflector.go/config"
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
@@ -22,23 +14,7 @@ import (
|
||||
|
||||
var uploadWorkers int
|
||||
var uploadSkipExistsCheck bool
|
||||
|
||||
const (
|
||||
sdInc = 1
|
||||
blobInc = 2
|
||||
errInc = 3
|
||||
)
|
||||
|
||||
type uploaderParams struct {
|
||||
workerWG *sync.WaitGroup
|
||||
counterWG *sync.WaitGroup
|
||||
stopper *stop.Group
|
||||
pathChan chan string
|
||||
countChan chan int
|
||||
sdCount int
|
||||
blobCount int
|
||||
errCount int
|
||||
}
|
||||
var uploadDeleteBlobsAfterUpload bool
|
||||
|
||||
func init() {
|
||||
var cmd = &cobra.Command{
|
||||
@@ -49,220 +25,31 @@ func init() {
|
||||
}
|
||||
cmd.PersistentFlags().IntVar(&uploadWorkers, "workers", 1, "How many worker threads to run at once")
|
||||
cmd.PersistentFlags().BoolVar(&uploadSkipExistsCheck, "skipExistsCheck", false, "Dont check if blobs exist before uploading")
|
||||
cmd.PersistentFlags().BoolVar(&uploadDeleteBlobsAfterUpload, "deleteBlobsAfterUpload", false, "Delete blobs after uploading them")
|
||||
rootCmd.AddCommand(cmd)
|
||||
}
|
||||
|
||||
func uploadCmd(cmd *cobra.Command, args []string) {
|
||||
startTime := time.Now()
|
||||
db := new(db.SQL)
|
||||
err := db.Connect(globalConfig.DBConn)
|
||||
checkErr(err)
|
||||
store, err := config.LoadStores(conf, "upload")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer store.Shutdown()
|
||||
|
||||
params := uploaderParams{
|
||||
workerWG: &sync.WaitGroup{},
|
||||
counterWG: &sync.WaitGroup{},
|
||||
pathChan: make(chan string),
|
||||
countChan: make(chan int),
|
||||
stopper: stop.New()}
|
||||
|
||||
setInterrupt(params.stopper)
|
||||
|
||||
paths, err := getPaths(args[0])
|
||||
checkErr(err)
|
||||
|
||||
totalCount := len(paths)
|
||||
|
||||
hashes := make([]string, len(paths))
|
||||
for i, p := range paths {
|
||||
hashes[i] = path.Base(p)
|
||||
databaseConn, err := config.LoadDatabase(conf, "upload")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
log.Println("checking for existing blobs")
|
||||
uploader := reflector.NewUploader(databaseConn, store, uploadWorkers, uploadSkipExistsCheck, uploadDeleteBlobsAfterUpload)
|
||||
|
||||
exists := make(map[string]bool)
|
||||
if !uploadSkipExistsCheck {
|
||||
exists, err = db.HasBlobs(hashes)
|
||||
checkErr(err)
|
||||
}
|
||||
existsCount := len(exists)
|
||||
|
||||
log.Printf("%d new blobs to upload", totalCount-existsCount)
|
||||
|
||||
startUploadWorkers(¶ms)
|
||||
params.counterWG.Add(1)
|
||||
go func() {
|
||||
defer params.counterWG.Done()
|
||||
runCountReceiver(¶ms, startTime, totalCount, existsCount)
|
||||
}()
|
||||
|
||||
Upload:
|
||||
for _, f := range paths {
|
||||
if exists[path.Base(f)] {
|
||||
continue
|
||||
}
|
||||
|
||||
select {
|
||||
case params.pathChan <- f:
|
||||
case <-params.stopper.Ch():
|
||||
log.Warnln("Caught interrupt, quitting at first opportunity...")
|
||||
break Upload
|
||||
}
|
||||
}
|
||||
|
||||
close(params.pathChan)
|
||||
params.workerWG.Wait()
|
||||
close(params.countChan)
|
||||
params.counterWG.Wait()
|
||||
params.stopper.Stop()
|
||||
|
||||
log.Println("SUMMARY")
|
||||
log.Printf("%d blobs total", totalCount)
|
||||
log.Printf("%d SD blobs uploaded", params.sdCount)
|
||||
log.Printf("%d content blobs uploaded", params.blobCount)
|
||||
log.Printf("%d blobs already stored", existsCount)
|
||||
log.Printf("%d errors encountered", params.errCount)
|
||||
}
|
||||
|
||||
func isJSON(data []byte) bool {
|
||||
var js json.RawMessage
|
||||
return json.Unmarshal(data, &js) == nil
|
||||
}
|
||||
|
||||
func newBlobStore() *store.DBBackedS3Store {
|
||||
db := new(db.SQL)
|
||||
err := db.Connect(globalConfig.DBConn)
|
||||
checkErr(err)
|
||||
|
||||
s3 := store.NewS3BlobStore(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
|
||||
return store.NewDBBackedS3Store(s3, db)
|
||||
}
|
||||
|
||||
func setInterrupt(stopper *stop.Group) {
|
||||
interruptChan := make(chan os.Signal, 1)
|
||||
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
|
||||
go func() {
|
||||
<-interruptChan
|
||||
stopper.Stop()
|
||||
uploader.Stop()
|
||||
}()
|
||||
}
|
||||
|
||||
func startUploadWorkers(params *uploaderParams) {
|
||||
for i := 0; i < uploadWorkers; i++ {
|
||||
params.workerWG.Add(1)
|
||||
go func(i int) {
|
||||
defer params.workerWG.Done()
|
||||
defer func(i int) {
|
||||
log.Printf("worker %d quitting", i)
|
||||
}(i)
|
||||
|
||||
blobStore := newBlobStore()
|
||||
launchFileUploader(params, blobStore, i)
|
||||
}(i)
|
||||
}
|
||||
}
|
||||
|
||||
func launchFileUploader(params *uploaderParams, blobStore *store.DBBackedS3Store, worker int) {
|
||||
for {
|
||||
select {
|
||||
case <-params.stopper.Ch():
|
||||
return
|
||||
case filepath, ok := <-params.pathChan:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
blob, err := ioutil.ReadFile(filepath)
|
||||
checkErr(err)
|
||||
|
||||
hash := peer.GetBlobHash(blob)
|
||||
if hash != path.Base(filepath) {
|
||||
log.Errorf("worker %d: file name does not match hash (%s != %s), skipping", worker, filepath, hash)
|
||||
select {
|
||||
case params.countChan <- errInc:
|
||||
case <-params.stopper.Ch():
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if isJSON(blob) {
|
||||
log.Printf("worker %d: PUTTING SD BLOB %s", worker, hash)
|
||||
err := blobStore.PutSD(hash, blob)
|
||||
if err != nil {
|
||||
log.Error("PutSD Error: ", err)
|
||||
}
|
||||
select {
|
||||
case params.countChan <- sdInc:
|
||||
case <-params.stopper.Ch():
|
||||
}
|
||||
} else {
|
||||
log.Printf("worker %d: putting %s", worker, hash)
|
||||
err = blobStore.Put(hash, blob)
|
||||
if err != nil {
|
||||
log.Error("put Blob Error: ", err)
|
||||
}
|
||||
select {
|
||||
case params.countChan <- blobInc:
|
||||
case <-params.stopper.Ch():
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func runCountReceiver(params *uploaderParams, startTime time.Time, totalCount int, existsCount int) {
|
||||
for {
|
||||
select {
|
||||
case <-params.stopper.Ch():
|
||||
return
|
||||
case countType, ok := <-params.countChan:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
switch countType {
|
||||
case sdInc:
|
||||
params.sdCount++
|
||||
case blobInc:
|
||||
params.blobCount++
|
||||
case errInc:
|
||||
params.errCount++
|
||||
}
|
||||
}
|
||||
if (params.sdCount+params.blobCount)%50 == 0 {
|
||||
log.Printf("%d of %d done (%s elapsed, %.3fs per blob)", params.sdCount+params.blobCount, totalCount-existsCount, time.Since(startTime).String(), time.Since(startTime).Seconds()/float64(params.sdCount+params.blobCount))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func getPaths(path string) ([]string, error) {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if info.Mode().IsRegular() {
|
||||
return []string{path}, nil
|
||||
}
|
||||
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
files, err := f.Readdir(-1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = f.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var filenames []string
|
||||
for _, file := range files {
|
||||
if !file.IsDir() {
|
||||
filenames = append(filenames, path+"/"+file.Name())
|
||||
}
|
||||
}
|
||||
|
||||
return filenames, nil
|
||||
err = uploader.Upload(args[0])
|
||||
checkErr(err)
|
||||
}
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"log"
|
||||
"fmt"
|
||||
|
||||
"github.com/lbryio/reflector.go/meta"
|
||||
|
||||
@@ -18,5 +18,5 @@ func init() {
|
||||
}
|
||||
|
||||
func versionCmd(cmd *cobra.Command, args []string) {
|
||||
log.Printf("version %s\n", meta.Version)
|
||||
fmt.Println(meta.FullName())
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"aws_id": "YOUR-AWS-ID",
|
||||
"aws_secret": "YOUR-AWS-SECRET",
|
||||
"bucket_region": "YOUR-BUCKET-REGION",
|
||||
"bucket_name": "YOUR-BUCKET-NAME",
|
||||
"db_conn": "USER:PASSWORD@tcp(localhost:3306)/DBNAME",
|
||||
"slack_hook_url": ""
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/lbryio/reflector.go/db"
|
||||
"github.com/lbryio/reflector.go/server"
|
||||
"github.com/lbryio/reflector.go/server/http"
|
||||
"github.com/lbryio/reflector.go/server/http3"
|
||||
"github.com/lbryio/reflector.go/server/peer"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
func LoadStores(path, file string) (store.BlobStore, error) {
|
||||
v := viper.New()
|
||||
v.SetConfigType("yaml")
|
||||
v.AddConfigPath(path)
|
||||
v.SetConfigName(file)
|
||||
err := v.ReadInConfig()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
storeViper := v.Sub("store")
|
||||
for storeType := range storeViper.AllSettings() {
|
||||
factory, exists := store.Factories[storeType]
|
||||
if !exists {
|
||||
return nil, errors.Err("unknown store type: %s", storeType)
|
||||
}
|
||||
storeConfig := storeViper.Sub(storeType)
|
||||
s, err := factory(storeConfig)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
//we only expect 1 store as the root, so let's return it
|
||||
return s, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func LoadServers(store store.BlobStore, path, file string) ([]server.BlobServer, error) {
|
||||
v := viper.New()
|
||||
v.SetConfigType("yaml")
|
||||
v.AddConfigPath(path)
|
||||
v.SetConfigName(file)
|
||||
|
||||
err := v.ReadInConfig()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
servers := make([]server.BlobServer, 0)
|
||||
serversViper := v.Sub("servers")
|
||||
for serverType := range serversViper.AllSettings() {
|
||||
var cfg server.BlobServerConfig
|
||||
err := serversViper.Sub(serverType).Unmarshal(&cfg)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
switch serverType {
|
||||
case "http":
|
||||
servers = append(servers, http.NewServer(store, cfg.MaxConcurrentRequests, cfg.EdgeToken, fmt.Sprintf("%s:%d", cfg.Address, cfg.Port)))
|
||||
case "http3":
|
||||
servers = append(servers, http3.NewServer(store, cfg.MaxConcurrentRequests, fmt.Sprintf("%s:%d", cfg.Address, cfg.Port)))
|
||||
case "peer":
|
||||
servers = append(servers, peer.NewServer(store, fmt.Sprintf("%s:%d", cfg.Address, cfg.Port)))
|
||||
default:
|
||||
return nil, errors.Err("unknown server type: %s", serverType)
|
||||
}
|
||||
}
|
||||
return servers, nil
|
||||
}
|
||||
|
||||
func LoadDatabase(path, file string) (*db.SQL, error) {
|
||||
v := viper.New()
|
||||
v.SetConfigType("yaml")
|
||||
v.AddConfigPath(path)
|
||||
v.SetConfigName(file)
|
||||
|
||||
err := v.ReadInConfig()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
dbConfig := v.Sub("database")
|
||||
if dbConfig == nil {
|
||||
return nil, errors.Err("db config not found")
|
||||
}
|
||||
user := dbConfig.GetString("user")
|
||||
password := dbConfig.GetString("password")
|
||||
host := dbConfig.GetString("host")
|
||||
port := dbConfig.GetInt("port")
|
||||
database := dbConfig.GetString("database")
|
||||
logQueries := dbConfig.GetBool("log_queries")
|
||||
accessTracking := dbConfig.GetInt("access_tracking")
|
||||
|
||||
if user == "" || password == "" || host == "" || port == 0 || database == "" {
|
||||
return nil, errors.Err("db config is missing required fields")
|
||||
}
|
||||
dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", user, password, host, port, database)
|
||||
dbInstance := &db.SQL{
|
||||
TrackingLevel: db.AccessTrackingLevel(accessTracking),
|
||||
SoftDelete: true,
|
||||
LogQueries: logQueries || log.GetLevel() == log.DebugLevel,
|
||||
}
|
||||
err = dbInstance.Connect(dsn)
|
||||
return dbInstance, err
|
||||
}
|
||||
@@ -3,41 +3,70 @@ package db
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/lbry.go/querytools"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
"github.com/lbryio/lbry.go/v2/dht/bits"
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
qt "github.com/lbryio/lbry.go/v2/extras/query"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
_ "github.com/go-sql-driver/mysql" // blank import for db driver
|
||||
"github.com/go-sql-driver/mysql"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/volatiletech/null/v8"
|
||||
"go.uber.org/atomic"
|
||||
)
|
||||
|
||||
// SdBlob is a special blob that contains information on the rest of the blobs in the stream
|
||||
type SdBlob struct {
|
||||
StreamName string `json:"stream_name"`
|
||||
Blobs []struct {
|
||||
Length int `json:"length"`
|
||||
BlobNum int `json:"blob_num"`
|
||||
BlobHash string `json:"blob_hash,omitempty"`
|
||||
Iv string `json:"iv"`
|
||||
} `json:"blobs"`
|
||||
StreamName string `json:"stream_name"`
|
||||
StreamType string `json:"stream_type"`
|
||||
Key string `json:"key"`
|
||||
SuggestedFileName string `json:"suggested_file_name"`
|
||||
StreamHash string `json:"stream_hash"`
|
||||
Blobs []struct {
|
||||
BlobHash string `json:"blob_hash,omitempty"`
|
||||
IV string `json:"iv"`
|
||||
Length int `json:"length"`
|
||||
BlobNum int `json:"blob_num"`
|
||||
} `json:"blobs"`
|
||||
}
|
||||
|
||||
type AccessTrackingLevel int
|
||||
|
||||
const (
|
||||
TrackAccessNone AccessTrackingLevel = iota // Don't track accesses
|
||||
TrackAccessStreams // Track accesses at the stream level
|
||||
TrackAccessBlobs // Track accesses at the blob level
|
||||
)
|
||||
|
||||
// SQL implements the DB interface
|
||||
type SQL struct {
|
||||
conn *sql.DB
|
||||
|
||||
// Track the approx last time a blob or stream was accessed
|
||||
TrackingLevel AccessTrackingLevel
|
||||
|
||||
// Instead of deleting a blob, marked it as not stored in the db
|
||||
SoftDelete bool
|
||||
|
||||
// Log executed queries. qt.InterpolateParams is cpu-heavy. This avoids that call if not needed.
|
||||
LogQueries bool
|
||||
}
|
||||
|
||||
func logQuery(query string, args ...interface{}) {
|
||||
s, err := querytools.InterpolateParams(query, args...)
|
||||
func (s *SQL) logQuery(query string, args ...interface{}) {
|
||||
if !s.LogQueries {
|
||||
return
|
||||
}
|
||||
|
||||
qStr, err := qt.InterpolateParams(query, args...)
|
||||
if err != nil {
|
||||
log.Errorln(err)
|
||||
} else {
|
||||
log.Debugln(s)
|
||||
log.Debugln(qStr)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -53,6 +82,8 @@ func (s *SQL) Connect(dsn string) error {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
s.conn.SetMaxIdleConns(12)
|
||||
|
||||
return errors.Err(s.conn.Ping())
|
||||
}
|
||||
|
||||
@@ -62,55 +93,240 @@ func (s *SQL) AddBlob(hash string, length int, isStored bool) error {
|
||||
return errors.Err("not connected")
|
||||
}
|
||||
|
||||
return withTx(s.conn, func(tx *sql.Tx) error {
|
||||
return addBlob(tx, hash, length, isStored)
|
||||
})
|
||||
_, err := s.insertBlob(hash, length, isStored)
|
||||
return err
|
||||
}
|
||||
|
||||
func addBlob(tx *sql.Tx, hash string, length int, isStored bool) error {
|
||||
if length <= 0 {
|
||||
return errors.Err("length must be positive")
|
||||
// AddBlobs adds blobs to the database.
|
||||
func (s *SQL) AddBlobs(hash []string) error {
|
||||
if s.conn == nil {
|
||||
return errors.Err("not connected")
|
||||
}
|
||||
|
||||
err := execTx(tx,
|
||||
"INSERT INTO blob_ (hash, is_stored, length) VALUES (?,?,?) ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored))",
|
||||
[]interface{}{hash, isStored, length},
|
||||
batch := 10000
|
||||
totalBlobs := int64(len(hash))
|
||||
work := make(chan []string, 1000)
|
||||
stopper := stop.New()
|
||||
var totalInserted atomic.Int64
|
||||
start := time.Now()
|
||||
|
||||
go func() {
|
||||
for i := 0; i < len(hash); i += batch {
|
||||
j := i + batch
|
||||
if j > len(hash) {
|
||||
j = len(hash)
|
||||
}
|
||||
work <- hash[i:j]
|
||||
}
|
||||
log.Infof("done loading %d hashes in the work queue", len(hash))
|
||||
close(work)
|
||||
}()
|
||||
|
||||
for i := 0; i < runtime.NumCPU(); i++ {
|
||||
stopper.Add(1)
|
||||
go func(worker int) {
|
||||
log.Infof("starting worker %d", worker)
|
||||
defer stopper.Done()
|
||||
for hashes := range work {
|
||||
inserted := totalInserted.Load()
|
||||
remaining := totalBlobs - inserted
|
||||
if inserted > 0 {
|
||||
timePerBlob := time.Since(start).Microseconds() / inserted
|
||||
remainingTime := time.Duration(remaining*timePerBlob) * time.Microsecond
|
||||
log.Infof("[T%d] processing batch of %d items. ETA: %s", worker, len(hashes), remainingTime.String())
|
||||
}
|
||||
err := s.insertBlobs(hashes) // Process the batch.
|
||||
if err != nil {
|
||||
log.Errorf("error while inserting batch: %s", errors.FullTrace(err))
|
||||
}
|
||||
totalInserted.Add(int64(len(hashes)))
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
|
||||
stopper.Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *SQL) insertBlobs(hashes []string) error {
|
||||
var (
|
||||
q string
|
||||
//args []interface{}
|
||||
)
|
||||
dayAgo := time.Now().AddDate(0, 0, -1).Format("2006-01-02 15:04:05")
|
||||
q = "insert into blob_ (hash, is_stored, length, last_accessed_at) values "
|
||||
for _, hash := range hashes {
|
||||
// prepared statements slow everything down by a lot due to reflection
|
||||
// for this specific instance we'll go ahead and hardcode the query to make it go faster
|
||||
q += fmt.Sprintf("('%s',1,%d,'%s'),", hash, stream.MaxBlobSize, dayAgo)
|
||||
//args = append(args, hash, true, stream.MaxBlobSize, dayAgo)
|
||||
}
|
||||
q = strings.TrimSuffix(q, ",")
|
||||
_, err := s.exec(q)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// HasBlob checks if the database contains the blob information.
|
||||
func (s *SQL) HasBlob(hash string) (bool, error) {
|
||||
if s.conn == nil {
|
||||
return false, errors.Err("not connected")
|
||||
func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error) {
|
||||
if length <= 0 {
|
||||
return 0, errors.Err("length must be positive")
|
||||
}
|
||||
|
||||
query := "SELECT EXISTS(SELECT 1 FROM blob_ WHERE hash = ? AND is_stored = ?)"
|
||||
args := []interface{}{hash, true}
|
||||
var (
|
||||
q string
|
||||
args []interface{}
|
||||
)
|
||||
if s.TrackingLevel == TrackAccessBlobs {
|
||||
args = []interface{}{hash, isStored, length, time.Now()}
|
||||
q = "INSERT INTO blob_ (hash, is_stored, length, last_accessed_at) VALUES (" + qt.Qs(len(args)) + ") ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored)), last_accessed_at = VALUES(last_accessed_at)"
|
||||
} else {
|
||||
args = []interface{}{hash, isStored, length}
|
||||
q = "INSERT INTO blob_ (hash, is_stored, length) VALUES (" + qt.Qs(len(args)) + ") ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored))"
|
||||
}
|
||||
|
||||
logQuery(query, args...)
|
||||
blobID, err := s.exec(q, args...)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
row := s.conn.QueryRow(query, args...)
|
||||
if blobID == 0 {
|
||||
err = s.conn.QueryRow("SELECT id FROM blob_ WHERE hash = ?", hash).Scan(&blobID)
|
||||
if err != nil {
|
||||
return 0, errors.Err(err)
|
||||
}
|
||||
if blobID == 0 {
|
||||
return 0, errors.Err("blob ID is 0 even after INSERTing and SELECTing")
|
||||
}
|
||||
|
||||
exists := false
|
||||
err := row.Scan(&exists)
|
||||
if s.TrackingLevel == TrackAccessBlobs {
|
||||
err := s.touchBlobs([]uint64{uint64(blobID)})
|
||||
if err != nil {
|
||||
return 0, errors.Err(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return exists, errors.Err(err)
|
||||
return blobID, nil
|
||||
}
|
||||
|
||||
func (s *SQL) insertStream(hash string, sdBlobID int64) (int64, error) {
|
||||
var (
|
||||
q string
|
||||
args []interface{}
|
||||
)
|
||||
|
||||
if s.TrackingLevel == TrackAccessStreams {
|
||||
args = []interface{}{hash, sdBlobID, time.Now()}
|
||||
q = "INSERT IGNORE INTO stream (hash, sd_blob_id, last_accessed_at) VALUES (" + qt.Qs(len(args)) + ")"
|
||||
} else {
|
||||
args = []interface{}{hash, sdBlobID}
|
||||
q = "INSERT IGNORE INTO stream (hash, sd_blob_id) VALUES (" + qt.Qs(len(args)) + ")"
|
||||
}
|
||||
|
||||
streamID, err := s.exec(q, args...)
|
||||
if err != nil {
|
||||
return 0, errors.Err(err)
|
||||
}
|
||||
|
||||
if streamID == 0 {
|
||||
err = s.conn.QueryRow("SELECT id FROM stream WHERE sd_blob_id = ?", sdBlobID).Scan(&streamID)
|
||||
if err != nil {
|
||||
return 0, errors.Err(err)
|
||||
}
|
||||
if streamID == 0 {
|
||||
return 0, errors.Err("stream ID is 0 even after INSERTing and SELECTing")
|
||||
}
|
||||
|
||||
if s.TrackingLevel == TrackAccessStreams {
|
||||
err := s.touchStreams([]uint64{uint64(streamID)})
|
||||
if err != nil {
|
||||
return 0, errors.Err(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return streamID, nil
|
||||
}
|
||||
|
||||
// HasBlob checks if the database contains the blob information.
|
||||
func (s *SQL) HasBlob(hash string, touch bool) (bool, error) {
|
||||
exists, err := s.HasBlobs([]string{hash}, touch)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return exists[hash], nil
|
||||
}
|
||||
|
||||
// HasBlobs checks if the database contains the set of blobs and returns a bool map.
|
||||
func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
|
||||
if s.conn == nil {
|
||||
return nil, errors.Err("not connected")
|
||||
func (s *SQL) HasBlobs(hashes []string, touch bool) (map[string]bool, error) {
|
||||
exists, idsNeedingTouch, err := s.hasBlobs(hashes)
|
||||
|
||||
if touch {
|
||||
if s.TrackingLevel == TrackAccessBlobs {
|
||||
_ = s.touchBlobs(idsNeedingTouch)
|
||||
} else if s.TrackingLevel == TrackAccessStreams {
|
||||
_ = s.touchStreams(idsNeedingTouch)
|
||||
}
|
||||
}
|
||||
|
||||
var hash string
|
||||
return exists, err
|
||||
}
|
||||
|
||||
func (s *SQL) touchBlobs(blobIDs []uint64) error {
|
||||
if len(blobIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
query := "UPDATE blob_ SET last_accessed_at = ? WHERE id IN (" + qt.Qs(len(blobIDs)) + ")"
|
||||
args := make([]interface{}, len(blobIDs)+1)
|
||||
args[0] = time.Now()
|
||||
for i := range blobIDs {
|
||||
args[i+1] = blobIDs[i]
|
||||
}
|
||||
|
||||
startTime := time.Now()
|
||||
_, err := s.exec(query, args...)
|
||||
log.Debugf("touched %d blobs and took %s", len(blobIDs), time.Since(startTime))
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
func (s *SQL) touchStreams(streamIDs []uint64) error {
|
||||
if len(streamIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
query := "UPDATE stream SET last_accessed_at = ? WHERE id IN (" + qt.Qs(len(streamIDs)) + ")"
|
||||
args := make([]interface{}, len(streamIDs)+1)
|
||||
args[0] = time.Now()
|
||||
for i := range streamIDs {
|
||||
args[i+1] = streamIDs[i]
|
||||
}
|
||||
|
||||
startTime := time.Now()
|
||||
_, err := s.exec(query, args...)
|
||||
log.Debugf("touched %d streams and took %s", len(streamIDs), time.Since(startTime))
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
func (s *SQL) hasBlobs(hashes []string) (map[string]bool, []uint64, error) {
|
||||
if s.conn == nil {
|
||||
return nil, nil, errors.Err("not connected")
|
||||
}
|
||||
|
||||
var (
|
||||
hash string
|
||||
blobID uint64
|
||||
streamID null.Uint64
|
||||
lastAccessedAt null.Time
|
||||
)
|
||||
|
||||
var needsTouch []uint64
|
||||
exists := make(map[string]bool)
|
||||
maxBatchSize := 100
|
||||
|
||||
touchDeadline := time.Now().Add(-6 * time.Hour) // touch blob if last accessed before this time
|
||||
maxBatchSize := 10000
|
||||
doneIndex := 0
|
||||
|
||||
for len(hashes) > doneIndex {
|
||||
@@ -121,115 +337,256 @@ func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
|
||||
log.Debugf("getting hashes[%d:%d] of %d", doneIndex, sliceEnd, len(hashes))
|
||||
batch := hashes[doneIndex:sliceEnd]
|
||||
|
||||
query := "SELECT hash FROM blob_ WHERE is_stored = ? && hash IN (" + querytools.Qs(len(batch)) + ")"
|
||||
args := make([]interface{}, len(batch)+1)
|
||||
args[0] = true
|
||||
var query string
|
||||
if s.TrackingLevel == TrackAccessBlobs {
|
||||
query = `SELECT b.hash, b.id, NULL, b.last_accessed_at
|
||||
FROM blob_ b
|
||||
WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)`
|
||||
} else if s.TrackingLevel == TrackAccessStreams {
|
||||
query = `SELECT b.hash, b.id, s.id, s.last_accessed_at
|
||||
FROM blob_ b
|
||||
LEFT JOIN stream_blob sb ON b.id = sb.blob_id
|
||||
INNER JOIN stream s on (sb.stream_id = s.id or s.sd_blob_id = b.id)
|
||||
WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)`
|
||||
} else {
|
||||
query = `SELECT b.hash, b.id, NULL, NULL
|
||||
FROM blob_ b
|
||||
WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)`
|
||||
}
|
||||
|
||||
args := make([]interface{}, len(batch))
|
||||
for i := range batch {
|
||||
args[i+1] = batch[i]
|
||||
args[i] = batch[i]
|
||||
}
|
||||
|
||||
logQuery(query, args...)
|
||||
s.logQuery(query, args...)
|
||||
|
||||
rows, err := s.conn.Query(query, args...)
|
||||
if err != nil {
|
||||
closeRows(rows)
|
||||
return exists, err
|
||||
}
|
||||
|
||||
for rows.Next() {
|
||||
err := rows.Scan(&hash)
|
||||
err := func() error {
|
||||
startTime := time.Now()
|
||||
rows, err := s.conn.Query(query, args...)
|
||||
log.Debugf("hashes query took %s", time.Since(startTime))
|
||||
if err != nil {
|
||||
closeRows(rows)
|
||||
return exists, err
|
||||
return errors.Err(err)
|
||||
}
|
||||
exists[hash] = true
|
||||
}
|
||||
defer closeRows(rows)
|
||||
|
||||
err = rows.Err()
|
||||
for rows.Next() {
|
||||
err = rows.Scan(&hash, &blobID, &streamID, &lastAccessedAt)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
exists[hash] = true
|
||||
if !lastAccessedAt.Valid || lastAccessedAt.Time.Before(touchDeadline) {
|
||||
if s.TrackingLevel == TrackAccessBlobs {
|
||||
needsTouch = append(needsTouch, blobID)
|
||||
} else if s.TrackingLevel == TrackAccessStreams && !streamID.IsZero() {
|
||||
needsTouch = append(needsTouch, streamID.Uint64)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
err = rows.Err()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
doneIndex += len(batch)
|
||||
return nil
|
||||
}()
|
||||
if err != nil {
|
||||
closeRows(rows)
|
||||
return exists, err
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
closeRows(rows)
|
||||
doneIndex += len(batch)
|
||||
}
|
||||
|
||||
return exists, nil
|
||||
return exists, needsTouch, nil
|
||||
}
|
||||
|
||||
// HasFullStream checks if the full stream has been uploaded (i.e. if we have the sd blob and all the content blobs)
|
||||
func (s *SQL) HasFullStream(sdHash string) (bool, error) {
|
||||
if s.conn == nil {
|
||||
return false, errors.Err("not connected")
|
||||
// Delete will remove (or soft-delete) the blob from the db
|
||||
// NOTE: If SoftDelete is enabled, streams will never be deleted
|
||||
func (s *SQL) Delete(hash string) error {
|
||||
if s.SoftDelete {
|
||||
_, err := s.exec("UPDATE blob_ SET is_stored = 0 WHERE hash = ?", hash)
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
query := `SELECT EXISTS(
|
||||
SELECT 1 FROM stream s
|
||||
LEFT JOIN stream_blob sb ON s.hash = sb.stream_hash
|
||||
LEFT JOIN blob_ b ON b.hash = sb.blob_hash
|
||||
WHERE s.sd_hash = ?
|
||||
GROUP BY s.sd_hash
|
||||
HAVING min(b.is_stored = 1)
|
||||
);`
|
||||
_, err := s.exec("DELETE FROM stream WHERE sd_blob_id = (SELECT id FROM blob_ WHERE hash = ?)", hash)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
_, err = s.exec("DELETE FROM blob_ WHERE hash = ?", hash)
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
// LeastRecentlyAccessedHashes gets the least recently accessed blobs
|
||||
func (s *SQL) LeastRecentlyAccessedHashes(maxBlobs int) ([]string, error) {
|
||||
if s.conn == nil {
|
||||
return nil, errors.Err("not connected")
|
||||
}
|
||||
|
||||
if s.TrackingLevel != TrackAccessBlobs {
|
||||
return nil, errors.Err("blob access tracking is disabled")
|
||||
}
|
||||
|
||||
query := "SELECT hash from blob_ where is_stored = 1 order by last_accessed_at limit ?"
|
||||
s.logQuery(query, maxBlobs)
|
||||
|
||||
rows, err := s.conn.Query(query, maxBlobs)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
defer closeRows(rows)
|
||||
|
||||
blobs := make([]string, 0, maxBlobs)
|
||||
for rows.Next() {
|
||||
var hash string
|
||||
err := rows.Scan(&hash)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
blobs = append(blobs, hash)
|
||||
}
|
||||
|
||||
return blobs, nil
|
||||
}
|
||||
|
||||
func (s *SQL) Count() (int, error) {
|
||||
if s.conn == nil {
|
||||
return 0, errors.Err("not connected")
|
||||
}
|
||||
|
||||
query := "SELECT count(id) from blob_"
|
||||
if s.SoftDelete {
|
||||
query += " where is_stored = 1"
|
||||
}
|
||||
s.logQuery(query)
|
||||
|
||||
var count int
|
||||
err := s.conn.QueryRow(query).Scan(&count)
|
||||
return count, errors.Err(err)
|
||||
}
|
||||
|
||||
// Block will mark a blob as blocked
|
||||
func (s *SQL) Block(hash string) error {
|
||||
query := "INSERT IGNORE INTO blocked SET hash = ?"
|
||||
args := []interface{}{hash}
|
||||
s.logQuery(query, args...)
|
||||
_, err := s.conn.Exec(query, args...)
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
// GetBlocked will return a list of blocked hashes
|
||||
func (s *SQL) GetBlocked() (map[string]bool, error) {
|
||||
query := "SELECT hash FROM blocked"
|
||||
s.logQuery(query)
|
||||
rows, err := s.conn.Query(query)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
defer closeRows(rows)
|
||||
|
||||
blocked := make(map[string]bool)
|
||||
|
||||
var hash string
|
||||
for rows.Next() {
|
||||
err = rows.Scan(&hash)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
blocked[hash] = true
|
||||
}
|
||||
|
||||
err = rows.Err()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
return blocked, nil
|
||||
}
|
||||
|
||||
// MissingBlobsForKnownStream returns missing blobs for an existing stream
|
||||
// WARNING: if the stream does NOT exist, no blob hashes will be returned, which looks
|
||||
// like no blobs are missing
|
||||
func (s *SQL) MissingBlobsForKnownStream(sdHash string) ([]string, error) {
|
||||
if s.conn == nil {
|
||||
return nil, errors.Err("not connected")
|
||||
}
|
||||
|
||||
query := `
|
||||
SELECT b.hash FROM blob_ b
|
||||
INNER JOIN stream_blob sb ON b.id = sb.blob_id
|
||||
INNER JOIN stream s ON s.id = sb.stream_id
|
||||
INNER JOIN blob_ sdb ON sdb.id = s.sd_blob_id AND sdb.hash = ?
|
||||
WHERE b.is_stored = 0
|
||||
`
|
||||
args := []interface{}{sdHash}
|
||||
|
||||
logQuery(query, args...)
|
||||
s.logQuery(query, args...)
|
||||
|
||||
row := s.conn.QueryRow(query, args...)
|
||||
rows, err := s.conn.Query(query, args...)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
defer closeRows(rows)
|
||||
|
||||
exists := false
|
||||
err := row.Scan(&exists)
|
||||
var missingBlobs []string
|
||||
var hash string
|
||||
|
||||
return exists, errors.Err(err)
|
||||
for rows.Next() {
|
||||
err = rows.Scan(&hash)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
missingBlobs = append(missingBlobs, hash)
|
||||
}
|
||||
|
||||
err = rows.Err()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
return missingBlobs, errors.Err(err)
|
||||
}
|
||||
|
||||
// AddSDBlob takes the SD Hash number of blobs and the set of blobs. In a single db tx it inserts the sdblob information
|
||||
// into a stream, and inserts the associated blobs' information in the database. If a blob fails the transaction is
|
||||
// rolled back and error(s) are returned.
|
||||
// AddSDBlob insert the SD blob and all the content blobs. The content blobs are marked as "not stored",
|
||||
// but they are tracked so reflector knows what it is missing.
|
||||
func (s *SQL) AddSDBlob(sdHash string, sdBlobLength int, sdBlob SdBlob) error {
|
||||
if s.conn == nil {
|
||||
return errors.Err("not connected")
|
||||
}
|
||||
|
||||
return withTx(s.conn, func(tx *sql.Tx) error {
|
||||
// insert sd blob
|
||||
err := addBlob(tx, sdHash, sdBlobLength, true)
|
||||
sdBlobID, err := s.insertBlob(sdHash, sdBlobLength, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
streamID, err := s.insertStream(sdBlob.StreamHash, sdBlobID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// insert content blobs and connect them to stream
|
||||
for _, contentBlob := range sdBlob.Blobs {
|
||||
if contentBlob.BlobHash == "" {
|
||||
// null terminator blob
|
||||
continue
|
||||
}
|
||||
|
||||
blobID, err := s.insertBlob(contentBlob.BlobHash, contentBlob.Length, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// insert stream
|
||||
err = execTx(tx,
|
||||
"INSERT IGNORE INTO stream (hash, sd_hash) VALUES (?,?)",
|
||||
[]interface{}{sdBlob.StreamHash, sdHash},
|
||||
args := []interface{}{streamID, blobID, contentBlob.BlobNum}
|
||||
_, err = s.exec(
|
||||
"INSERT IGNORE INTO stream_blob (stream_id, blob_id, num) VALUES ("+qt.Qs(len(args))+")",
|
||||
args...,
|
||||
)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
// insert content blobs and connect them to stream
|
||||
for _, contentBlob := range sdBlob.Blobs {
|
||||
if contentBlob.BlobHash == "" {
|
||||
// null terminator blob
|
||||
continue
|
||||
}
|
||||
|
||||
err := addBlob(tx, contentBlob.BlobHash, contentBlob.Length, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = execTx(tx,
|
||||
"INSERT IGNORE INTO stream_blob (stream_hash, blob_hash, num) VALUES (?,?,?)",
|
||||
[]interface{}{sdBlob.StreamHash, contentBlob.BlobHash, contentBlob.BlobNum},
|
||||
)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetHashRange gets the smallest and biggest hashes in the db
|
||||
@@ -243,7 +600,7 @@ func (s *SQL) GetHashRange() (string, string, error) {
|
||||
|
||||
query := "SELECT MIN(hash), MAX(hash) from blob_"
|
||||
|
||||
logQuery(query)
|
||||
s.logQuery(query)
|
||||
|
||||
err := s.conn.QueryRow(query).Scan(&min, &max)
|
||||
return min, max, err
|
||||
@@ -267,7 +624,7 @@ func (s *SQL) GetStoredHashesInRange(ctx context.Context, start, end bits.Bitmap
|
||||
query := "SELECT hash FROM blob_ WHERE hash >= ? AND hash <= ? AND is_stored = 1"
|
||||
args := []interface{}{start.Hex(), end.Hex()}
|
||||
|
||||
logQuery(query, args...)
|
||||
s.logQuery(query, args...)
|
||||
|
||||
rows, err := s.conn.Query(query, args...)
|
||||
defer closeRows(rows)
|
||||
@@ -279,7 +636,7 @@ func (s *SQL) GetStoredHashesInRange(ctx context.Context, start, end bits.Bitmap
|
||||
var hash string
|
||||
ScanLoop:
|
||||
for rows.Next() {
|
||||
err := rows.Scan(&hash)
|
||||
err = rows.Scan(&hash)
|
||||
if err != nil {
|
||||
ech <- err
|
||||
return
|
||||
@@ -347,38 +704,77 @@ func closeRows(rows *sql.Rows) {
|
||||
}
|
||||
}
|
||||
|
||||
func execTx(tx *sql.Tx, query string, args []interface{}) error {
|
||||
logQuery(query, args...)
|
||||
_, err := tx.Exec(query, args...)
|
||||
return errors.Err(err)
|
||||
func (s *SQL) exec(query string, args ...interface{}) (int64, error) {
|
||||
s.logQuery(query, args...)
|
||||
attempt, maxAttempts := 0, 3
|
||||
Retry:
|
||||
attempt++
|
||||
result, err := s.conn.Exec(query, args...)
|
||||
if isLockTimeoutError(err) {
|
||||
if attempt <= maxAttempts {
|
||||
//Error 1205: Lock wait timeout exceeded; try restarting transaction
|
||||
goto Retry
|
||||
}
|
||||
err = errors.Prefix("Lock timeout for query "+query, err)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return 0, errors.Err(err)
|
||||
}
|
||||
|
||||
lastID, err := result.LastInsertId()
|
||||
return lastID, errors.Err(err)
|
||||
}
|
||||
|
||||
func isLockTimeoutError(err error) bool {
|
||||
e, ok := err.(*mysql.MySQLError)
|
||||
return ok && e != nil && e.Number == 1205
|
||||
}
|
||||
|
||||
/* SQL schema
|
||||
|
||||
in prod, set tx_isolation to READ-COMMITTED to improve db performance
|
||||
make sure you use latin1 or utf8 charset, NOT utf8mb4. that's a waste of space.
|
||||
|
||||
todo: could add UNIQUE KEY (stream_hash, num) to stream_blob ...
|
||||
|
||||
CREATE TABLE blob_ (
|
||||
id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT UNIQUE,
|
||||
hash char(96) NOT NULL,
|
||||
is_stored TINYINT(1) NOT NULL DEFAULT 0,
|
||||
length bigint(20) unsigned DEFAULT NULL,
|
||||
PRIMARY KEY (hash)
|
||||
last_accessed_at TIMESTAMP NULL DEFAULT NULL,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY blob_hash_idx (hash),
|
||||
KEY `blob_last_accessed_idx` (`last_accessed_at`),
|
||||
KEY `is_stored_idx` (`is_stored`)
|
||||
);
|
||||
|
||||
CREATE TABLE stream (
|
||||
id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT UNIQUE,
|
||||
hash char(96) NOT NULL,
|
||||
sd_hash char(96) NOT NULL,
|
||||
PRIMARY KEY (hash),
|
||||
KEY sd_hash_idx (sd_hash),
|
||||
FOREIGN KEY (sd_hash) REFERENCES blob_ (hash) ON DELETE RESTRICT ON UPDATE CASCADE
|
||||
sd_blob_id BIGINT UNSIGNED NOT NULL,
|
||||
last_accessed_at TIMESTAMP NULL DEFAULT NULL,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY stream_hash_idx (hash),
|
||||
KEY stream_sd_blob_id_idx (sd_blob_id),
|
||||
KEY last_accessed_at_idx (last_accessed_at),
|
||||
FOREIGN KEY (sd_blob_id) REFERENCES blob_ (id) ON DELETE RESTRICT ON UPDATE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE stream_blob (
|
||||
stream_hash char(96) NOT NULL,
|
||||
blob_hash char(96) NOT NULL,
|
||||
stream_id BIGINT UNSIGNED NOT NULL,
|
||||
blob_id BIGINT UNSIGNED NOT NULL,
|
||||
num int NOT NULL,
|
||||
PRIMARY KEY (stream_hash, blob_hash),
|
||||
FOREIGN KEY (stream_hash) REFERENCES stream (hash) ON DELETE CASCADE ON UPDATE CASCADE,
|
||||
FOREIGN KEY (blob_hash) REFERENCES blob_ (hash) ON DELETE CASCADE ON UPDATE CASCADE
|
||||
PRIMARY KEY (stream_id, blob_id),
|
||||
KEY stream_blob_blob_id_idx (blob_id),
|
||||
FOREIGN KEY (stream_id) REFERENCES stream (id) ON DELETE CASCADE ON UPDATE CASCADE,
|
||||
FOREIGN KEY (blob_id) REFERENCES blob_ (id) ON DELETE CASCADE ON UPDATE CASCADE
|
||||
);
|
||||
|
||||
could add UNIQUE KEY (stream_hash, num) to stream_blob ...
|
||||
CREATE TABLE blocked (
|
||||
hash char(96) NOT NULL,
|
||||
PRIMARY KEY (hash)
|
||||
);
|
||||
|
||||
*/
|
||||
@@ -1,399 +0,0 @@
|
||||
package bits
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"math/big"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
|
||||
"github.com/lyoshenka/bencode"
|
||||
)
|
||||
|
||||
// TODO: http://roaringbitmap.org/
|
||||
|
||||
const (
|
||||
NumBytes = 48 // bytes
|
||||
NumBits = NumBytes * 8
|
||||
)
|
||||
|
||||
// Bitmap is a generalized representation of an identifier or data that can be sorted, compared fast. Used by the DHT
|
||||
// package as a way to handle the unique identifiers of a DHT node.
|
||||
type Bitmap [NumBytes]byte
|
||||
|
||||
func (b Bitmap) RawString() string {
|
||||
return string(b[:])
|
||||
}
|
||||
|
||||
func (b Bitmap) String() string {
|
||||
return b.Hex()
|
||||
}
|
||||
|
||||
// BString returns the bitmap as a string of 0s and 1s
|
||||
func (b Bitmap) BString() string {
|
||||
var s string
|
||||
for _, byte := range b {
|
||||
s += strconv.FormatInt(int64(byte), 2)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// Hex returns a hexadecimal representation of the bitmap.
|
||||
func (b Bitmap) Hex() string {
|
||||
return hex.EncodeToString(b[:])
|
||||
}
|
||||
|
||||
// HexShort returns a hexadecimal representation of the first 4 bytes.
|
||||
func (b Bitmap) HexShort() string {
|
||||
return hex.EncodeToString(b[:4])
|
||||
}
|
||||
|
||||
// HexSimplified returns the hexadecimal representation with all leading 0's removed
|
||||
func (b Bitmap) HexSimplified() string {
|
||||
simple := strings.TrimLeft(b.Hex(), "0")
|
||||
if simple == "" {
|
||||
simple = "0"
|
||||
}
|
||||
return simple
|
||||
}
|
||||
|
||||
func (b Bitmap) Big() *big.Int {
|
||||
i := new(big.Int)
|
||||
i.SetString(b.Hex(), 16)
|
||||
return i
|
||||
}
|
||||
|
||||
// Cmp compares b and other and returns:
|
||||
//
|
||||
// -1 if b < other
|
||||
// 0 if b == other
|
||||
// +1 if b > other
|
||||
//
|
||||
func (b Bitmap) Cmp(other Bitmap) int {
|
||||
for k := range b {
|
||||
if b[k] < other[k] {
|
||||
return -1
|
||||
} else if b[k] > other[k] {
|
||||
return 1
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// Closer returns true if dist(b,x) < dist(b,y)
|
||||
func (b Bitmap) Closer(x, y Bitmap) bool {
|
||||
return x.Xor(b).Cmp(y.Xor(b)) < 0
|
||||
}
|
||||
|
||||
// Equals returns true if every byte in bitmap are equal, false otherwise
|
||||
func (b Bitmap) Equals(other Bitmap) bool {
|
||||
return b.Cmp(other) == 0
|
||||
}
|
||||
|
||||
// Copy returns a duplicate value for the bitmap.
|
||||
func (b Bitmap) Copy() Bitmap {
|
||||
var ret Bitmap
|
||||
copy(ret[:], b[:])
|
||||
return ret
|
||||
}
|
||||
|
||||
// Xor returns a diff bitmap. If they are equal, the returned bitmap will be all 0's. If 100% unique the returned
|
||||
// bitmap will be all 1's.
|
||||
func (b Bitmap) Xor(other Bitmap) Bitmap {
|
||||
var ret Bitmap
|
||||
for k := range b {
|
||||
ret[k] = b[k] ^ other[k]
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
// And returns a comparison bitmap, that for each byte returns the AND true table result
|
||||
func (b Bitmap) And(other Bitmap) Bitmap {
|
||||
var ret Bitmap
|
||||
for k := range b {
|
||||
ret[k] = b[k] & other[k]
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
// Or returns a comparison bitmap, that for each byte returns the OR true table result
|
||||
func (b Bitmap) Or(other Bitmap) Bitmap {
|
||||
var ret Bitmap
|
||||
for k := range b {
|
||||
ret[k] = b[k] | other[k]
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
// Not returns a complimentary bitmap that is an inverse. So b.NOT.NOT = b
|
||||
func (b Bitmap) Not() Bitmap {
|
||||
var ret Bitmap
|
||||
for k := range b {
|
||||
ret[k] = ^b[k]
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
func (b Bitmap) add(other Bitmap) (Bitmap, bool) {
|
||||
var ret Bitmap
|
||||
carry := false
|
||||
for i := NumBits - 1; i >= 0; i-- {
|
||||
bBit := getBit(b[:], i)
|
||||
oBit := getBit(other[:], i)
|
||||
setBit(ret[:], i, bBit != oBit != carry)
|
||||
carry = (bBit && oBit) || (bBit && carry) || (oBit && carry)
|
||||
}
|
||||
return ret, carry
|
||||
}
|
||||
|
||||
// Add returns a bitmap that treats both bitmaps as numbers and adding them together. Since the size of a bitmap is
|
||||
// limited, an overflow is possible when adding bitmaps.
|
||||
func (b Bitmap) Add(other Bitmap) Bitmap {
|
||||
ret, carry := b.add(other)
|
||||
if carry {
|
||||
panic("overflow in bitmap addition. limited to " + strconv.Itoa(NumBits) + " bits.")
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
// Sub returns a bitmap that treats both bitmaps as numbers and subtracts then via the inverse of the other and adding
|
||||
// then together a + (-b). Negative bitmaps are not supported so other must be greater than this.
|
||||
func (b Bitmap) Sub(other Bitmap) Bitmap {
|
||||
if b.Cmp(other) < 0 {
|
||||
// ToDo: Why is this not supported? Should it say not implemented? BitMap might have a generic use case outside of dht.
|
||||
panic("negative bitmaps not supported")
|
||||
}
|
||||
complement, _ := other.Not().add(FromShortHexP("1"))
|
||||
ret, _ := b.add(complement)
|
||||
return ret
|
||||
}
|
||||
|
||||
// Get returns the binary bit at the position passed.
|
||||
func (b Bitmap) Get(n int) bool {
|
||||
return getBit(b[:], n)
|
||||
}
|
||||
|
||||
// Set sets the binary bit at the position passed.
|
||||
func (b Bitmap) Set(n int, one bool) Bitmap {
|
||||
ret := b.Copy()
|
||||
setBit(ret[:], n, one)
|
||||
return ret
|
||||
}
|
||||
|
||||
// PrefixLen returns the number of leading 0 bits
|
||||
func (b Bitmap) PrefixLen() int {
|
||||
for i := range b {
|
||||
for j := 0; j < 8; j++ {
|
||||
if (b[i]>>uint8(7-j))&0x1 != 0 {
|
||||
return i*8 + j
|
||||
}
|
||||
}
|
||||
}
|
||||
return NumBits
|
||||
}
|
||||
|
||||
// Prefix returns a copy of b with the first n bits set to 1 (if `one` is true) or 0 (if `one` is false)
|
||||
// https://stackoverflow.com/a/23192263/182709
|
||||
func (b Bitmap) Prefix(n int, one bool) Bitmap {
|
||||
ret := b.Copy()
|
||||
|
||||
Outer:
|
||||
for i := range ret {
|
||||
for j := 0; j < 8; j++ {
|
||||
if i*8+j < n {
|
||||
if one {
|
||||
ret[i] |= 1 << uint(7-j)
|
||||
} else {
|
||||
ret[i] &= ^(1 << uint(7-j))
|
||||
}
|
||||
} else {
|
||||
break Outer
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ret
|
||||
}
|
||||
|
||||
// Suffix returns a copy of b with the last n bits set to 1 (if `one` is true) or 0 (if `one` is false)
|
||||
// https://stackoverflow.com/a/23192263/182709
|
||||
func (b Bitmap) Suffix(n int, one bool) Bitmap {
|
||||
ret := b.Copy()
|
||||
|
||||
Outer:
|
||||
for i := len(ret) - 1; i >= 0; i-- {
|
||||
for j := 7; j >= 0; j-- {
|
||||
if i*8+j >= NumBits-n {
|
||||
if one {
|
||||
ret[i] |= 1 << uint(7-j)
|
||||
} else {
|
||||
ret[i] &= ^(1 << uint(7-j))
|
||||
}
|
||||
} else {
|
||||
break Outer
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ret
|
||||
}
|
||||
|
||||
// MarshalBencode implements the Marshaller(bencode)/Message interface.
|
||||
func (b Bitmap) MarshalBencode() ([]byte, error) {
|
||||
str := string(b[:])
|
||||
return bencode.EncodeBytes(str)
|
||||
}
|
||||
|
||||
// UnmarshalBencode implements the Marshaller(bencode)/Message interface.
|
||||
func (b *Bitmap) UnmarshalBencode(encoded []byte) error {
|
||||
var str string
|
||||
err := bencode.DecodeBytes(encoded, &str)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(str) != NumBytes {
|
||||
return errors.Err("invalid bitmap length")
|
||||
}
|
||||
copy(b[:], str)
|
||||
return nil
|
||||
}
|
||||
|
||||
// FromBytes returns a bitmap as long as the byte array is of a specific length specified in the parameters.
|
||||
func FromBytes(data []byte) (Bitmap, error) {
|
||||
var bmp Bitmap
|
||||
|
||||
if len(data) != len(bmp) {
|
||||
return bmp, errors.Err("invalid bitmap of length %d", len(data))
|
||||
}
|
||||
|
||||
copy(bmp[:], data)
|
||||
return bmp, nil
|
||||
}
|
||||
|
||||
// FromBytesP returns a bitmap as long as the byte array is of a specific length specified in the parameters
|
||||
// otherwise it wil panic.
|
||||
func FromBytesP(data []byte) Bitmap {
|
||||
bmp, err := FromBytes(data)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return bmp
|
||||
}
|
||||
|
||||
//FromString returns a bitmap by converting the string to bytes and creating from bytes as long as the byte array
|
||||
// is of a specific length specified in the parameters
|
||||
func FromString(data string) (Bitmap, error) {
|
||||
return FromBytes([]byte(data))
|
||||
}
|
||||
|
||||
//FromStringP returns a bitmap by converting the string to bytes and creating from bytes as long as the byte array
|
||||
// is of a specific length specified in the parameters otherwise it wil panic.
|
||||
func FromStringP(data string) Bitmap {
|
||||
bmp, err := FromString(data)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return bmp
|
||||
}
|
||||
|
||||
//FromHex returns a bitmap by converting the hex string to bytes and creating from bytes as long as the byte array
|
||||
// is of a specific length specified in the parameters
|
||||
func FromHex(hexStr string) (Bitmap, error) {
|
||||
decoded, err := hex.DecodeString(hexStr)
|
||||
if err != nil {
|
||||
return Bitmap{}, errors.Err(err)
|
||||
}
|
||||
return FromBytes(decoded)
|
||||
}
|
||||
|
||||
//FromHexP returns a bitmap by converting the hex string to bytes and creating from bytes as long as the byte array
|
||||
// is of a specific length specified in the parameters otherwise it wil panic.
|
||||
func FromHexP(hexStr string) Bitmap {
|
||||
bmp, err := FromHex(hexStr)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return bmp
|
||||
}
|
||||
|
||||
//FromShortHex returns a bitmap by converting the hex string to bytes, adding the leading zeros prefix to the
|
||||
// hex string and creating from bytes as long as the byte array is of a specific length specified in the parameters
|
||||
func FromShortHex(hexStr string) (Bitmap, error) {
|
||||
return FromHex(strings.Repeat("0", NumBytes*2-len(hexStr)) + hexStr)
|
||||
}
|
||||
|
||||
//FromShortHexP returns a bitmap by converting the hex string to bytes, adding the leading zeros prefix to the
|
||||
// hex string and creating from bytes as long as the byte array is of a specific length specified in the parameters
|
||||
// otherwise it wil panic.
|
||||
func FromShortHexP(hexStr string) Bitmap {
|
||||
bmp, err := FromShortHex(hexStr)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return bmp
|
||||
}
|
||||
|
||||
func FromBigP(b *big.Int) Bitmap {
|
||||
return FromShortHexP(b.Text(16))
|
||||
}
|
||||
|
||||
// Max returns a bitmap with all bits set to 1
|
||||
func MaxP() Bitmap {
|
||||
return FromHexP(strings.Repeat("f", NumBytes*2))
|
||||
}
|
||||
|
||||
// Rand generates a cryptographically random bitmap with the confines of the parameters specified.
|
||||
func Rand() Bitmap {
|
||||
var id Bitmap
|
||||
_, err := rand.Read(id[:])
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
// RandInRangeP generates a cryptographically random bitmap and while it is greater than the high threshold
|
||||
// bitmap will subtract the diff between high and low until it is no longer greater that the high.
|
||||
func RandInRangeP(low, high Bitmap) Bitmap {
|
||||
diff := high.Sub(low)
|
||||
r := Rand()
|
||||
for r.Cmp(diff) > 0 {
|
||||
r = r.Sub(diff)
|
||||
}
|
||||
//ToDo - Adding the low at this point doesn't gurantee it will be within the range. Consider bitmaps as numbers and
|
||||
// I have a range of 50-100. If get to say 60, and add 50, I would be at 110. Should protect against this?
|
||||
return r.Add(low)
|
||||
}
|
||||
|
||||
func getBit(b []byte, n int) bool {
|
||||
i := n / 8
|
||||
j := n % 8
|
||||
return b[i]&(1<<uint(7-j)) > 0
|
||||
}
|
||||
|
||||
func setBit(b []byte, n int, one bool) {
|
||||
i := n / 8
|
||||
j := n % 8
|
||||
if one {
|
||||
b[i] |= 1 << uint(7-j)
|
||||
} else {
|
||||
b[i] &= ^(1 << uint(7-j))
|
||||
}
|
||||
}
|
||||
|
||||
// CLosest returns the closest bitmap to target. if no bitmaps are provided, target itself is returned
|
||||
func Closest(target Bitmap, bitmaps ...Bitmap) Bitmap {
|
||||
if len(bitmaps) == 0 {
|
||||
return target
|
||||
}
|
||||
|
||||
var closest *Bitmap
|
||||
for _, b := range bitmaps {
|
||||
if closest == nil || target.Closer(b, *closest) {
|
||||
closest = &b
|
||||
}
|
||||
}
|
||||
return *closest
|
||||
}
|
||||
@@ -1,386 +0,0 @@
|
||||
package bits
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/lyoshenka/bencode"
|
||||
)
|
||||
|
||||
func TestBitmap(t *testing.T) {
|
||||
a := Bitmap{
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
|
||||
12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
|
||||
24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35,
|
||||
36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,
|
||||
}
|
||||
b := Bitmap{
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
|
||||
12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
|
||||
24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35,
|
||||
36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 47, 46,
|
||||
}
|
||||
c := Bitmap{
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1,
|
||||
}
|
||||
|
||||
if !a.Equals(a) {
|
||||
t.Error("bitmap does not equal itself")
|
||||
}
|
||||
if a.Equals(b) {
|
||||
t.Error("bitmap equals another bitmap with different id")
|
||||
}
|
||||
|
||||
if !a.Xor(b).Equals(c) {
|
||||
t.Error(a.Xor(b))
|
||||
}
|
||||
|
||||
if c.PrefixLen() != 375 {
|
||||
t.Error(c.PrefixLen())
|
||||
}
|
||||
|
||||
if b.Cmp(a) < 0 {
|
||||
t.Error("bitmap fails Cmp test")
|
||||
}
|
||||
|
||||
if a.Closer(c, b) || !a.Closer(b, c) || c.Closer(a, b) || c.Closer(b, c) {
|
||||
t.Error("bitmap fails Closer test")
|
||||
}
|
||||
|
||||
id := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
|
||||
if FromHexP(id).Hex() != id {
|
||||
t.Error(FromHexP(id).Hex())
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmap_GetBit(t *testing.T) {
|
||||
tt := []struct {
|
||||
bit int
|
||||
expected bool
|
||||
panic bool
|
||||
}{
|
||||
{bit: 383, expected: false, panic: false},
|
||||
{bit: 382, expected: true, panic: false},
|
||||
{bit: 381, expected: false, panic: false},
|
||||
{bit: 380, expected: true, panic: false},
|
||||
}
|
||||
|
||||
b := FromShortHexP("a")
|
||||
|
||||
for _, test := range tt {
|
||||
actual := getBit(b[:], test.bit)
|
||||
if test.expected != actual {
|
||||
t.Errorf("getting bit %d of %s: expected %t, got %t", test.bit, b.HexSimplified(), test.expected, actual)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmap_SetBit(t *testing.T) {
|
||||
tt := []struct {
|
||||
hex string
|
||||
bit int
|
||||
one bool
|
||||
expected string
|
||||
panic bool
|
||||
}{
|
||||
{hex: "0", bit: 383, one: true, expected: "1", panic: false},
|
||||
{hex: "0", bit: 382, one: true, expected: "2", panic: false},
|
||||
{hex: "0", bit: 381, one: true, expected: "4", panic: false},
|
||||
{hex: "0", bit: 385, one: true, expected: "1", panic: true},
|
||||
{hex: "0", bit: 384, one: true, expected: "1", panic: true},
|
||||
}
|
||||
|
||||
for _, test := range tt {
|
||||
expected := FromShortHexP(test.expected)
|
||||
actual := FromShortHexP(test.hex)
|
||||
if test.panic {
|
||||
assertPanic(t, fmt.Sprintf("setting bit %d to %t", test.bit, test.one), func() { setBit(actual[:], test.bit, test.one) })
|
||||
} else {
|
||||
setBit(actual[:], test.bit, test.one)
|
||||
if !expected.Equals(actual) {
|
||||
t.Errorf("setting bit %d to %t: expected %s, got %s", test.bit, test.one, test.expected, actual.HexSimplified())
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmap_FromHexShort(t *testing.T) {
|
||||
tt := []struct {
|
||||
short string
|
||||
long string
|
||||
}{
|
||||
{short: "", long: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{short: "0", long: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{short: "00000", long: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{short: "9473745bc", long: "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000009473745bc"},
|
||||
{short: "09473745bc", long: "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000009473745bc"},
|
||||
{short: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
||||
long: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"},
|
||||
}
|
||||
|
||||
for _, test := range tt {
|
||||
short := FromShortHexP(test.short)
|
||||
long := FromHexP(test.long)
|
||||
if !short.Equals(long) {
|
||||
t.Errorf("short hex %s: expected %s, got %s", test.short, long.Hex(), short.Hex())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmapMarshal(t *testing.T) {
|
||||
b := FromStringP("123456789012345678901234567890123456789012345678")
|
||||
encoded, err := bencode.EncodeBytes(b)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
if string(encoded) != "48:123456789012345678901234567890123456789012345678" {
|
||||
t.Error("encoding does not match expected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmapMarshalEmbedded(t *testing.T) {
|
||||
e := struct {
|
||||
A string
|
||||
B Bitmap
|
||||
C int
|
||||
}{
|
||||
A: "1",
|
||||
B: FromStringP("222222222222222222222222222222222222222222222222"),
|
||||
C: 3,
|
||||
}
|
||||
|
||||
encoded, err := bencode.EncodeBytes(e)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
if string(encoded) != "d1:A1:11:B48:2222222222222222222222222222222222222222222222221:Ci3ee" {
|
||||
t.Error("encoding does not match expected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmapMarshalEmbedded2(t *testing.T) {
|
||||
encoded, err := bencode.EncodeBytes([]interface{}{
|
||||
FromStringP("333333333333333333333333333333333333333333333333"),
|
||||
})
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
if string(encoded) != "l48:333333333333333333333333333333333333333333333333e" {
|
||||
t.Error("encoding does not match expected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmap_PrefixLen(t *testing.T) {
|
||||
tt := []struct {
|
||||
hex string
|
||||
len int
|
||||
}{
|
||||
{len: 0, hex: "F00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{len: 0, hex: "800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{len: 1, hex: "700000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{len: 1, hex: "400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{len: 384, hex: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{len: 383, hex: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"},
|
||||
{len: 382, hex: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002"},
|
||||
{len: 382, hex: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003"},
|
||||
}
|
||||
|
||||
for _, test := range tt {
|
||||
len := FromHexP(test.hex).PrefixLen()
|
||||
if len != test.len {
|
||||
t.Errorf("got prefix len %d; expected %d for %s", len, test.len, test.hex)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmap_Prefix(t *testing.T) {
|
||||
allOne := FromHexP("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")
|
||||
|
||||
zerosTT := []struct {
|
||||
zeros int
|
||||
expected string
|
||||
}{
|
||||
{zeros: -123, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
|
||||
{zeros: 0, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
|
||||
{zeros: 1, expected: "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
|
||||
{zeros: 69, expected: "000000000000000007ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
|
||||
{zeros: 383, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"},
|
||||
{zeros: 384, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{zeros: 400, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
}
|
||||
|
||||
for _, test := range zerosTT {
|
||||
expected := FromHexP(test.expected)
|
||||
actual := allOne.Prefix(test.zeros, false)
|
||||
if !actual.Equals(expected) {
|
||||
t.Errorf("%d zeros: got %s; expected %s", test.zeros, actual.Hex(), expected.Hex())
|
||||
}
|
||||
}
|
||||
|
||||
for i := 0; i < NumBits; i++ {
|
||||
b := allOne.Prefix(i, false)
|
||||
if b.PrefixLen() != i {
|
||||
t.Errorf("got prefix len %d; expected %d for %s", b.PrefixLen(), i, b.Hex())
|
||||
}
|
||||
}
|
||||
|
||||
allZero := FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
|
||||
|
||||
onesTT := []struct {
|
||||
ones int
|
||||
expected string
|
||||
}{
|
||||
{ones: -123, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{ones: 0, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{ones: 1, expected: "800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{ones: 69, expected: "fffffffffffffffff8000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{ones: 383, expected: "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe"},
|
||||
{ones: 384, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
|
||||
{ones: 400, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
|
||||
}
|
||||
|
||||
for _, test := range onesTT {
|
||||
expected := FromHexP(test.expected)
|
||||
actual := allZero.Prefix(test.ones, true)
|
||||
if !actual.Equals(expected) {
|
||||
t.Errorf("%d ones: got %s; expected %s", test.ones, actual.Hex(), expected.Hex())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmap_Suffix(t *testing.T) {
|
||||
allOne := FromHexP("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")
|
||||
|
||||
zerosTT := []struct {
|
||||
zeros int
|
||||
expected string
|
||||
}{
|
||||
{zeros: -123, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
|
||||
{zeros: 0, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
|
||||
{zeros: 1, expected: "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe"},
|
||||
{zeros: 69, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe00000000000000000"},
|
||||
{zeros: 383, expected: "800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{zeros: 384, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{zeros: 400, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
}
|
||||
|
||||
for _, test := range zerosTT {
|
||||
expected := FromHexP(test.expected)
|
||||
actual := allOne.Suffix(test.zeros, false)
|
||||
if !actual.Equals(expected) {
|
||||
t.Errorf("%d zeros: got %s; expected %s", test.zeros, actual.Hex(), expected.Hex())
|
||||
}
|
||||
}
|
||||
|
||||
for i := 0; i < NumBits; i++ {
|
||||
b := allOne.Prefix(i, false)
|
||||
if b.PrefixLen() != i {
|
||||
t.Errorf("got prefix len %d; expected %d for %s", b.PrefixLen(), i, b.Hex())
|
||||
}
|
||||
}
|
||||
|
||||
allZero := FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
|
||||
|
||||
onesTT := []struct {
|
||||
ones int
|
||||
expected string
|
||||
}{
|
||||
{ones: -123, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{ones: 0, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"},
|
||||
{ones: 1, expected: "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"},
|
||||
{ones: 69, expected: "0000000000000000000000000000000000000000000000000000000000000000000000000000001fffffffffffffffff"},
|
||||
{ones: 383, expected: "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
|
||||
{ones: 384, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
|
||||
{ones: 400, expected: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
|
||||
}
|
||||
|
||||
for _, test := range onesTT {
|
||||
expected := FromHexP(test.expected)
|
||||
actual := allZero.Suffix(test.ones, true)
|
||||
if !actual.Equals(expected) {
|
||||
t.Errorf("%d ones: got %s; expected %s", test.ones, actual.Hex(), expected.Hex())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmap_Add(t *testing.T) {
|
||||
tt := []struct {
|
||||
a, b, sum string
|
||||
panic bool
|
||||
}{
|
||||
{"0", "0", "0", false},
|
||||
{"0", "1", "1", false},
|
||||
{"1", "0", "1", false},
|
||||
{"1", "1", "2", false},
|
||||
{"8", "4", "c", false},
|
||||
{"1000", "0010", "1010", false},
|
||||
{"1111", "1111", "2222", false},
|
||||
{"ffff", "1", "10000", false},
|
||||
{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", false},
|
||||
{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "1", "", true},
|
||||
}
|
||||
|
||||
for _, test := range tt {
|
||||
a := FromShortHexP(test.a)
|
||||
b := FromShortHexP(test.b)
|
||||
expected := FromShortHexP(test.sum)
|
||||
if test.panic {
|
||||
assertPanic(t, fmt.Sprintf("adding %s and %s", test.a, test.b), func() { a.Add(b) })
|
||||
} else {
|
||||
actual := a.Add(b)
|
||||
if !expected.Equals(actual) {
|
||||
t.Errorf("adding %s and %s; expected %s, got %s", test.a, test.b, test.sum, actual.HexSimplified())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmap_Sub(t *testing.T) {
|
||||
tt := []struct {
|
||||
a, b, sum string
|
||||
panic bool
|
||||
}{
|
||||
{"0", "0", "0", false},
|
||||
{"1", "0", "1", false},
|
||||
{"1", "1", "0", false},
|
||||
{"8", "4", "4", false},
|
||||
{"f", "9", "6", false},
|
||||
{"f", "e", "1", false},
|
||||
{"10", "f", "1", false},
|
||||
{"2222", "1111", "1111", false},
|
||||
{"ffff", "1", "fffe", false},
|
||||
{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", false},
|
||||
{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0", false},
|
||||
{"0", "1", "", true},
|
||||
}
|
||||
|
||||
for _, test := range tt {
|
||||
a := FromShortHexP(test.a)
|
||||
b := FromShortHexP(test.b)
|
||||
expected := FromShortHexP(test.sum)
|
||||
if test.panic {
|
||||
assertPanic(t, fmt.Sprintf("subtracting %s - %s", test.a, test.b), func() { a.Sub(b) })
|
||||
} else {
|
||||
actual := a.Sub(b)
|
||||
if !expected.Equals(actual) {
|
||||
t.Errorf("subtracting %s - %s; expected %s, got %s", test.a, test.b, test.sum, actual.HexSimplified())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertPanic(t *testing.T, text string, f func()) {
|
||||
defer func() {
|
||||
if r := recover(); r == nil {
|
||||
t.Errorf("%s: did not panic as expected", text)
|
||||
}
|
||||
}()
|
||||
f()
|
||||
}
|
||||
@@ -1,65 +0,0 @@
|
||||
package bits
|
||||
|
||||
import (
|
||||
"math/big"
|
||||
|
||||
"github.com/lbryio/errors.go"
|
||||
)
|
||||
|
||||
// Range has a start and end
|
||||
type Range struct {
|
||||
Start Bitmap
|
||||
End Bitmap
|
||||
}
|
||||
|
||||
func MaxRange() Range {
|
||||
return Range{
|
||||
Start: Bitmap{},
|
||||
End: MaxP(),
|
||||
}
|
||||
}
|
||||
|
||||
// IntervalP divides the range into `num` intervals and returns the `n`th one
|
||||
// intervals are approximately the same size, but may not be exact because of rounding issues
|
||||
// the first interval always starts at the beginning of the range, and the last interval always ends at the end
|
||||
func (r Range) IntervalP(n, num int) Range {
|
||||
if num < 1 || n < 1 || n > num {
|
||||
panic(errors.Err("invalid interval %d of %d", n, num))
|
||||
}
|
||||
|
||||
start := r.intervalStart(n, num)
|
||||
end := r.End.Big()
|
||||
if n < num {
|
||||
end = r.intervalStart(n+1, num)
|
||||
end.Sub(end, big.NewInt(1))
|
||||
}
|
||||
|
||||
return Range{FromBigP(start), FromBigP(end)}
|
||||
}
|
||||
|
||||
func (r Range) intervalStart(n, num int) *big.Int {
|
||||
// formula:
|
||||
// size = (end - start) / num
|
||||
// rem = (end - start) % num
|
||||
// intervalStart = rangeStart + (size * n-1) + ((rem * n-1) % num)
|
||||
|
||||
size := new(big.Int)
|
||||
rem := new(big.Int)
|
||||
size.Sub(r.End.Big(), r.Start.Big()).DivMod(size, big.NewInt(int64(num)), rem)
|
||||
|
||||
size.Mul(size, big.NewInt(int64(n-1)))
|
||||
rem.Mul(rem, big.NewInt(int64(n-1))).Mod(rem, big.NewInt(int64(num)))
|
||||
|
||||
start := r.Start.Big()
|
||||
start.Add(start, size).Add(start, rem)
|
||||
|
||||
return start
|
||||
}
|
||||
|
||||
func (r Range) IntervalSize() *big.Int {
|
||||
return (&big.Int{}).Sub(r.End.Big(), r.Start.Big())
|
||||
}
|
||||
|
||||
func (r Range) Contains(b Bitmap) bool {
|
||||
return r.Start.Cmp(b) <= 0 && r.End.Cmp(b) >= 0
|
||||
}
|
||||
@@ -1,48 +0,0 @@
|
||||
package bits
|
||||
|
||||
import (
|
||||
"math/big"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMaxRange(t *testing.T) {
|
||||
start := FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
|
||||
end := FromHexP("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")
|
||||
r := MaxRange()
|
||||
|
||||
if !r.Start.Equals(start) {
|
||||
t.Error("max range does not start at the beginning")
|
||||
}
|
||||
if !r.End.Equals(end) {
|
||||
t.Error("max range does not end at the end")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRange_IntervalP(t *testing.T) {
|
||||
max := MaxRange()
|
||||
|
||||
numIntervals := 97
|
||||
expectedAvg := (&big.Int{}).Div(max.IntervalSize(), big.NewInt(int64(numIntervals)))
|
||||
maxDiff := big.NewInt(int64(numIntervals))
|
||||
|
||||
var lastEnd Bitmap
|
||||
|
||||
for i := 1; i <= numIntervals; i++ {
|
||||
ival := max.IntervalP(i, numIntervals)
|
||||
if i == 1 && !ival.Start.Equals(max.Start) {
|
||||
t.Error("first interval does not start at 0")
|
||||
}
|
||||
if i == numIntervals && !ival.End.Equals(max.End) {
|
||||
t.Error("last interval does not end at max")
|
||||
}
|
||||
if i > 1 && !ival.Start.Equals(lastEnd.Add(FromShortHexP("1"))) {
|
||||
t.Errorf("interval %d of %d: last end was %s, this start is %s", i, numIntervals, lastEnd.Hex(), ival.Start.Hex())
|
||||
}
|
||||
|
||||
if ival.IntervalSize().Cmp((&big.Int{}).Add(expectedAvg, maxDiff)) > 0 || ival.IntervalSize().Cmp((&big.Int{}).Sub(expectedAvg, maxDiff)) < 0 {
|
||||
t.Errorf("interval %d of %d: interval size is outside the normal range", i, numIntervals)
|
||||
}
|
||||
|
||||
lastEnd = ival.End
|
||||
}
|
||||
}
|
||||
@@ -1,212 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"math/rand"
|
||||
"net"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
)
|
||||
|
||||
const (
|
||||
bootstrapDefaultRefreshDuration = 15 * time.Minute
|
||||
)
|
||||
|
||||
// BootstrapNode is a configured node setup for testing.
|
||||
type BootstrapNode struct {
|
||||
Node
|
||||
|
||||
initialPingInterval time.Duration
|
||||
checkInterval time.Duration
|
||||
|
||||
nlock *sync.RWMutex
|
||||
peers map[bits.Bitmap]*peer
|
||||
nodeIDs []bits.Bitmap // necessary for efficient random ID selection
|
||||
}
|
||||
|
||||
// NewBootstrapNode returns a BootstrapNode pointer.
|
||||
func NewBootstrapNode(id bits.Bitmap, initialPingInterval, rePingInterval time.Duration) *BootstrapNode {
|
||||
b := &BootstrapNode{
|
||||
Node: *NewNode(id),
|
||||
|
||||
initialPingInterval: initialPingInterval,
|
||||
checkInterval: rePingInterval,
|
||||
|
||||
nlock: &sync.RWMutex{},
|
||||
peers: make(map[bits.Bitmap]*peer),
|
||||
nodeIDs: make([]bits.Bitmap, 0),
|
||||
}
|
||||
|
||||
b.requestHandler = b.handleRequest
|
||||
|
||||
return b
|
||||
}
|
||||
|
||||
// Add manually adds a contact
|
||||
func (b *BootstrapNode) Add(c Contact) {
|
||||
b.upsert(c)
|
||||
}
|
||||
|
||||
// Connect connects to the given connection and starts any background threads necessary
|
||||
func (b *BootstrapNode) Connect(conn UDPConn) error {
|
||||
err := b.Node.Connect(conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Infof("[%s] bootstrap: node connected", b.id.HexShort())
|
||||
|
||||
go func() {
|
||||
t := time.NewTicker(b.checkInterval / 5)
|
||||
for {
|
||||
select {
|
||||
case <-t.C:
|
||||
b.check()
|
||||
case <-b.grp.Ch():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// upsert adds the contact to the list, or updates the lastPinged time
|
||||
func (b *BootstrapNode) upsert(c Contact) {
|
||||
b.nlock.Lock()
|
||||
defer b.nlock.Unlock()
|
||||
|
||||
if peer, exists := b.peers[c.ID]; exists {
|
||||
log.Debugf("[%s] bootstrap: touching contact %s", b.id.HexShort(), peer.Contact.ID.HexShort())
|
||||
peer.Touch()
|
||||
return
|
||||
}
|
||||
|
||||
log.Debugf("[%s] bootstrap: adding new contact %s", b.id.HexShort(), c.ID.HexShort())
|
||||
b.peers[c.ID] = &peer{c, b.id.Xor(c.ID), time.Now(), 0}
|
||||
b.nodeIDs = append(b.nodeIDs, c.ID)
|
||||
}
|
||||
|
||||
// remove removes the contact from the list
|
||||
func (b *BootstrapNode) remove(c Contact) {
|
||||
b.nlock.Lock()
|
||||
defer b.nlock.Unlock()
|
||||
|
||||
_, exists := b.peers[c.ID]
|
||||
if !exists {
|
||||
return
|
||||
}
|
||||
|
||||
log.Debugf("[%s] bootstrap: removing contact %s", b.id.HexShort(), c.ID.HexShort())
|
||||
delete(b.peers, c.ID)
|
||||
for i := range b.nodeIDs {
|
||||
if b.nodeIDs[i].Equals(c.ID) {
|
||||
b.nodeIDs = append(b.nodeIDs[:i], b.nodeIDs[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// get returns up to `limit` random contacts from the list
|
||||
func (b *BootstrapNode) get(limit int) []Contact {
|
||||
b.nlock.RLock()
|
||||
defer b.nlock.RUnlock()
|
||||
|
||||
if len(b.peers) < limit {
|
||||
limit = len(b.peers)
|
||||
}
|
||||
|
||||
ret := make([]Contact, limit)
|
||||
for i, k := range randKeys(len(b.nodeIDs))[:limit] {
|
||||
ret[i] = b.peers[b.nodeIDs[k]].Contact
|
||||
}
|
||||
|
||||
return ret
|
||||
}
|
||||
|
||||
// ping pings a node. if the node responds, it is added to the list. otherwise, it is removed
|
||||
func (b *BootstrapNode) ping(c Contact) {
|
||||
log.Debugf("[%s] bootstrap: pinging %s", b.id.HexShort(), c.ID.HexShort())
|
||||
b.grp.Add(1)
|
||||
defer b.grp.Done()
|
||||
|
||||
resCh := b.SendAsync(c, Request{Method: pingMethod})
|
||||
|
||||
var res *Response
|
||||
|
||||
select {
|
||||
case res = <-resCh:
|
||||
case <-b.grp.Ch():
|
||||
return
|
||||
}
|
||||
|
||||
if res != nil && res.Data == pingSuccessResponse {
|
||||
b.upsert(c)
|
||||
} else {
|
||||
b.remove(c)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *BootstrapNode) check() {
|
||||
b.nlock.RLock()
|
||||
defer b.nlock.RUnlock()
|
||||
|
||||
for i := range b.peers {
|
||||
if !b.peers[i].ActiveInLast(b.checkInterval) {
|
||||
go b.ping(b.peers[i].Contact)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// handleRequest handles the requests received from udp.
|
||||
func (b *BootstrapNode) handleRequest(addr *net.UDPAddr, request Request) {
|
||||
switch request.Method {
|
||||
case pingMethod:
|
||||
err := b.sendMessage(addr, Response{ID: request.ID, NodeID: b.id, Data: pingSuccessResponse})
|
||||
if err != nil {
|
||||
log.Error("error sending response message - ", err)
|
||||
}
|
||||
case findNodeMethod:
|
||||
if request.Arg == nil {
|
||||
log.Errorln("request is missing arg")
|
||||
return
|
||||
}
|
||||
|
||||
err := b.sendMessage(addr, Response{
|
||||
ID: request.ID,
|
||||
NodeID: b.id,
|
||||
Contacts: b.get(bucketSize),
|
||||
})
|
||||
if err != nil {
|
||||
log.Error("error sending 'findnodemethod' response message - ", err)
|
||||
}
|
||||
}
|
||||
|
||||
go func() {
|
||||
b.nlock.RLock()
|
||||
_, exists := b.peers[request.NodeID]
|
||||
b.nlock.RUnlock()
|
||||
if !exists {
|
||||
log.Debugf("[%s] bootstrap: queuing %s to ping", b.id.HexShort(), request.NodeID.HexShort())
|
||||
<-time.After(b.initialPingInterval)
|
||||
b.nlock.RLock()
|
||||
_, exists = b.peers[request.NodeID]
|
||||
b.nlock.RUnlock()
|
||||
if !exists {
|
||||
b.ping(Contact{ID: request.NodeID, IP: addr.IP, Port: addr.Port})
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func randKeys(max int) []int {
|
||||
keys := make([]int, max)
|
||||
for k := range keys {
|
||||
keys[k] = k
|
||||
}
|
||||
rand.Shuffle(max, func(i, j int) {
|
||||
keys[i], keys[j] = keys[j], keys[i]
|
||||
})
|
||||
return keys
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"net"
|
||||
"testing"
|
||||
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
)
|
||||
|
||||
func TestBootstrapPing(t *testing.T) {
|
||||
b := NewBootstrapNode(bits.Rand(), 10, bootstrapDefaultRefreshDuration)
|
||||
|
||||
listener, err := net.ListenPacket(Network, "127.0.0.1:54320")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
err = b.Connect(listener.(*net.UDPConn))
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
b.Shutdown()
|
||||
}
|
||||
@@ -1,76 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
peerproto "github.com/lbryio/reflector.go/peer"
|
||||
)
|
||||
|
||||
const (
|
||||
Network = "udp4"
|
||||
DefaultPort = 4444
|
||||
|
||||
DefaultAnnounceRate = 10 // send at most this many announces per second
|
||||
DefaultReannounceTime = 50 * time.Minute // should be a bit less than hash expiration time
|
||||
|
||||
// TODO: all these constants should be defaults, and should be used to set values in the standard Config. then the code should use values in the config
|
||||
// TODO: alternatively, have a global Config for constants. at least that way tests can modify the values
|
||||
alpha = 3 // this is the constant alpha in the spec
|
||||
bucketSize = 8 // this is the constant k in the spec
|
||||
nodeIDLength = bits.NumBytes // bytes. this is the constant B in the spec
|
||||
messageIDLength = 20 // bytes.
|
||||
|
||||
udpRetry = 1
|
||||
udpTimeout = 5 * time.Second
|
||||
udpMaxMessageLength = 4096 // bytes. I think our longest message is ~676 bytes, so I rounded up to 1024
|
||||
// scratch that. a findValue could return more than K results if a lot of nodes are storing that value, so we need more buffer
|
||||
|
||||
maxPeerFails = 3 // after this many failures, a peer is considered bad and will be removed from the routing table
|
||||
//tExpire = 60 * time.Minute // the time after which a key/value pair expires; this is a time-to-live (TTL) from the original publication date
|
||||
tRefresh = 1 * time.Hour // the time after which an otherwise unaccessed bucket must be refreshed
|
||||
//tReplicate = 1 * time.Hour // the interval between Kademlia replication events, when a node is required to publish its entire database
|
||||
//tNodeRefresh = 15 * time.Minute // the time after which a good node becomes questionable if it has not messaged us
|
||||
|
||||
compactNodeInfoLength = nodeIDLength + 6 // nodeID + 4 for IP + 2 for port
|
||||
|
||||
tokenSecretRotationInterval = 5 * time.Minute // how often the token-generating secret is rotated
|
||||
)
|
||||
|
||||
// Config represents the configure of dht.
|
||||
type Config struct {
|
||||
// this node's address. format is `ip:port`
|
||||
Address string
|
||||
// the seed nodes through which we can join in dht network
|
||||
SeedNodes []string
|
||||
// the hex-encoded node id for this node. if string is empty, a random id will be generated
|
||||
NodeID string
|
||||
// print the state of the dht every X time
|
||||
PrintState time.Duration
|
||||
// the port that clients can use to download blobs using the LBRY peer protocol
|
||||
PeerProtocolPort int
|
||||
// if nonzero, an RPC server will listen to requests on this port and respond to them
|
||||
RPCPort int
|
||||
// the time after which the original publisher must reannounce a key/value pair
|
||||
ReannounceTime time.Duration
|
||||
// send at most this many announces per second
|
||||
AnnounceRate int
|
||||
// channel that will receive notifications about announcements
|
||||
AnnounceNotificationCh chan announceNotification
|
||||
}
|
||||
|
||||
// NewStandardConfig returns a Config pointer with default values.
|
||||
func NewStandardConfig() *Config {
|
||||
return &Config{
|
||||
Address: "0.0.0.0:" + strconv.Itoa(DefaultPort),
|
||||
SeedNodes: []string{
|
||||
"lbrynet1.lbry.io:4444",
|
||||
"lbrynet2.lbry.io:4444",
|
||||
"lbrynet3.lbry.io:4444",
|
||||
},
|
||||
PeerProtocolPort: peerproto.DefaultPort,
|
||||
ReannounceTime: DefaultReannounceTime,
|
||||
AnnounceRate: DefaultAnnounceRate,
|
||||
}
|
||||
}
|
||||
-118
@@ -1,118 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net"
|
||||
"sort"
|
||||
"strconv"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
|
||||
"github.com/lyoshenka/bencode"
|
||||
)
|
||||
|
||||
// TODO: if routing table is ever empty (aka the node is isolated), it should re-bootstrap
|
||||
|
||||
// Contact contains information for contacting another node on the network
|
||||
type Contact struct {
|
||||
ID bits.Bitmap
|
||||
IP net.IP
|
||||
Port int // the udp port used for the dht
|
||||
PeerPort int // the tcp port a peer can be contacted on for blob requests
|
||||
}
|
||||
|
||||
// Equals returns true if two contacts are the same.
|
||||
func (c Contact) Equals(other Contact, checkID bool) bool {
|
||||
return c.IP.Equal(other.IP) && c.Port == other.Port && (!checkID || c.ID == other.ID)
|
||||
}
|
||||
|
||||
// Addr returns the address of the contact.
|
||||
func (c Contact) Addr() *net.UDPAddr {
|
||||
return &net.UDPAddr{IP: c.IP, Port: c.Port}
|
||||
}
|
||||
|
||||
// String returns a short string representation of the contact
|
||||
func (c Contact) String() string {
|
||||
str := c.ID.HexShort() + "@" + c.Addr().String()
|
||||
if c.PeerPort != 0 {
|
||||
str += "(" + strconv.Itoa(c.PeerPort) + ")"
|
||||
}
|
||||
return str
|
||||
}
|
||||
|
||||
// MarshalCompact returns a compact byteslice representation of the contact
|
||||
// NOTE: The compact representation always uses the tcp PeerPort, not the udp Port. This is dumb, but that's how the python daemon does it
|
||||
func (c Contact) MarshalCompact() ([]byte, error) {
|
||||
if c.IP.To4() == nil {
|
||||
return nil, errors.Err("ip not set")
|
||||
}
|
||||
if c.PeerPort < 0 || c.PeerPort > 65535 {
|
||||
return nil, errors.Err("invalid port")
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
buf.Write(c.IP.To4())
|
||||
buf.WriteByte(byte(c.PeerPort >> 8))
|
||||
buf.WriteByte(byte(c.PeerPort))
|
||||
buf.Write(c.ID[:])
|
||||
|
||||
if buf.Len() != compactNodeInfoLength {
|
||||
return nil, errors.Err("i dont know how this happened")
|
||||
}
|
||||
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
// UnmarshalCompact unmarshals the compact byteslice representation of a contact.
|
||||
// NOTE: The compact representation always uses the tcp PeerPort, not the udp Port. This is dumb, but that's how the python daemon does it
|
||||
func (c *Contact) UnmarshalCompact(b []byte) error {
|
||||
if len(b) != compactNodeInfoLength {
|
||||
return errors.Err("invalid compact length")
|
||||
}
|
||||
c.IP = net.IPv4(b[0], b[1], b[2], b[3]).To4()
|
||||
c.PeerPort = int(uint16(b[5]) | uint16(b[4])<<8)
|
||||
c.ID = bits.FromBytesP(b[6:])
|
||||
return nil
|
||||
}
|
||||
|
||||
// MarshalBencode returns the serialized byte slice representation of a contact.
|
||||
func (c Contact) MarshalBencode() ([]byte, error) {
|
||||
return bencode.EncodeBytes([]interface{}{c.ID, c.IP.String(), c.Port})
|
||||
}
|
||||
|
||||
// UnmarshalBencode unmarshals the serialized byte slice into the appropriate fields of the contact.
|
||||
func (c *Contact) UnmarshalBencode(b []byte) error {
|
||||
var raw []bencode.RawMessage
|
||||
err := bencode.DecodeBytes(b, &raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(raw) != 3 {
|
||||
return errors.Err("contact must have 3 elements; got %d", len(raw))
|
||||
}
|
||||
|
||||
err = bencode.DecodeBytes(raw[0], &c.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var ipStr string
|
||||
err = bencode.DecodeBytes(raw[1], &ipStr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.IP = net.ParseIP(ipStr).To4()
|
||||
if c.IP == nil {
|
||||
return errors.Err("invalid IP")
|
||||
}
|
||||
|
||||
return bencode.DecodeBytes(raw[2], &c.Port)
|
||||
}
|
||||
|
||||
func sortByDistance(contacts []Contact, target bits.Bitmap) {
|
||||
sort.Slice(contacts, func(i, j int) bool {
|
||||
return contacts[i].ID.Xor(target).Cmp(contacts[j].ID.Xor(target)) < 0
|
||||
})
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"net"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
)
|
||||
|
||||
func TestCompactEncoding(t *testing.T) {
|
||||
c := Contact{
|
||||
ID: bits.FromHexP("1c8aff71b99462464d9eeac639595ab99664be3482cb91a29d87467515c7d9158fe72aa1f1582dab07d8f8b5db277f41"),
|
||||
IP: net.ParseIP("1.2.3.4"),
|
||||
PeerPort: int(55<<8 + 66),
|
||||
}
|
||||
|
||||
var compact []byte
|
||||
compact, err := c.MarshalCompact()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(compact) != compactNodeInfoLength {
|
||||
t.Fatalf("got length of %d; expected %d", len(compact), compactNodeInfoLength)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(compact, append([]byte{1, 2, 3, 4, 55, 66}, c.ID[:]...)) {
|
||||
t.Errorf("compact bytes not encoded correctly")
|
||||
}
|
||||
}
|
||||
-232
@@ -1,232 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/lbry.go/stop"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cast"
|
||||
)
|
||||
|
||||
var log *logrus.Logger
|
||||
|
||||
func UseLogger(l *logrus.Logger) {
|
||||
log = l
|
||||
}
|
||||
|
||||
func init() {
|
||||
log = logrus.StandardLogger()
|
||||
//log.SetFormatter(&log.TextFormatter{ForceColors: true})
|
||||
//log.SetLevel(log.DebugLevel)
|
||||
}
|
||||
|
||||
// DHT represents a DHT node.
|
||||
type DHT struct {
|
||||
// config
|
||||
conf *Config
|
||||
// local contact
|
||||
contact Contact
|
||||
// node
|
||||
node *Node
|
||||
// stopGroup to shut down DHT
|
||||
grp *stop.Group
|
||||
// channel is closed when DHT joins network
|
||||
joined chan struct{}
|
||||
// cache for store tokens
|
||||
tokenCache *tokenCache
|
||||
// hashes that need to be put into the announce queue or removed from the queue
|
||||
announceAddRemove chan queueEdit
|
||||
}
|
||||
|
||||
// New returns a DHT pointer. If config is nil, then config will be set to the default config.
|
||||
func New(config *Config) *DHT {
|
||||
if config == nil {
|
||||
config = NewStandardConfig()
|
||||
}
|
||||
|
||||
d := &DHT{
|
||||
conf: config,
|
||||
grp: stop.New(),
|
||||
joined: make(chan struct{}),
|
||||
announceAddRemove: make(chan queueEdit),
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func (dht *DHT) connect(conn UDPConn) error {
|
||||
contact, err := getContact(dht.conf.NodeID, dht.conf.Address)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dht.contact = contact
|
||||
dht.node = NewNode(contact.ID)
|
||||
dht.tokenCache = newTokenCache(dht.node, tokenSecretRotationInterval)
|
||||
|
||||
return dht.node.Connect(conn)
|
||||
}
|
||||
|
||||
// Start starts the dht
|
||||
func (dht *DHT) Start() error {
|
||||
listener, err := net.ListenPacket(Network, dht.conf.Address)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
conn := listener.(*net.UDPConn)
|
||||
|
||||
err = dht.connect(conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dht.join()
|
||||
log.Infof("[%s] DHT ready on %s (%d nodes found during join)",
|
||||
dht.node.id.HexShort(), dht.contact.Addr().String(), dht.node.rt.Count())
|
||||
|
||||
dht.grp.Add(1)
|
||||
go func() {
|
||||
dht.runAnnouncer()
|
||||
dht.grp.Done()
|
||||
}()
|
||||
|
||||
if dht.conf.RPCPort > 0 {
|
||||
dht.grp.Add(1)
|
||||
go func() {
|
||||
dht.runRPCServer(dht.conf.RPCPort)
|
||||
dht.grp.Done()
|
||||
}()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// join makes current node join the dht network.
|
||||
func (dht *DHT) join() {
|
||||
defer close(dht.joined) // if anyone's waiting for join to finish, they'll know its done
|
||||
|
||||
log.Infof("[%s] joining DHT network", dht.node.id.HexShort())
|
||||
|
||||
// ping nodes, which gets their real node IDs and adds them to the routing table
|
||||
atLeastOneNodeResponded := false
|
||||
for _, addr := range dht.conf.SeedNodes {
|
||||
err := dht.Ping(addr)
|
||||
if err != nil {
|
||||
log.Error(errors.Prefix(fmt.Sprintf("[%s] join", dht.node.id.HexShort()), err))
|
||||
} else {
|
||||
atLeastOneNodeResponded = true
|
||||
}
|
||||
}
|
||||
|
||||
if !atLeastOneNodeResponded {
|
||||
log.Errorf("[%s] join: no nodes responded to initial ping", dht.node.id.HexShort())
|
||||
return
|
||||
}
|
||||
|
||||
// now call iterativeFind on yourself
|
||||
_, _, err := FindContacts(dht.node, dht.node.id, false, dht.grp.Child())
|
||||
if err != nil {
|
||||
log.Errorf("[%s] join: %s", dht.node.id.HexShort(), err.Error())
|
||||
}
|
||||
|
||||
// TODO: after joining, refresh all buckets further away than our closest neighbor
|
||||
// http://xlattice.sourceforge.net/components/protocol/kademlia/specs.html#join
|
||||
}
|
||||
|
||||
// WaitUntilJoined blocks until the node joins the network.
|
||||
func (dht *DHT) WaitUntilJoined() {
|
||||
if dht.joined == nil {
|
||||
panic("dht not initialized")
|
||||
}
|
||||
<-dht.joined
|
||||
}
|
||||
|
||||
// Shutdown shuts down the dht
|
||||
func (dht *DHT) Shutdown() {
|
||||
log.Debugf("[%s] DHT shutting down", dht.node.id.HexShort())
|
||||
dht.grp.StopAndWait()
|
||||
dht.node.Shutdown()
|
||||
log.Debugf("[%s] DHT stopped", dht.node.id.HexShort())
|
||||
}
|
||||
|
||||
// Ping pings a given address, creates a temporary contact for sending a message, and returns an error if communication
|
||||
// fails.
|
||||
func (dht *DHT) Ping(addr string) error {
|
||||
raddr, err := net.ResolveUDPAddr(Network, addr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tmpNode := Contact{ID: bits.Rand(), IP: raddr.IP, Port: raddr.Port}
|
||||
res := dht.node.Send(tmpNode, Request{Method: pingMethod}, SendOptions{skipIDCheck: true})
|
||||
if res == nil {
|
||||
return errors.Err("no response from node %s", addr)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get returns the list of nodes that have the blob for the given hash
|
||||
func (dht *DHT) Get(hash bits.Bitmap) ([]Contact, error) {
|
||||
contacts, found, err := FindContacts(dht.node, hash, true, dht.grp.Child())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if found {
|
||||
return contacts, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// PrintState prints the current state of the DHT including address, nr outstanding transactions, stored hashes as well
|
||||
// as current bucket information.
|
||||
func (dht *DHT) PrintState() {
|
||||
log.Printf("DHT node %s at %s", dht.contact.String(), time.Now().Format(time.RFC822Z))
|
||||
log.Printf("Outstanding transactions: %d", dht.node.CountActiveTransactions())
|
||||
log.Printf("Stored hashes: %d", dht.node.store.CountStoredHashes())
|
||||
log.Printf("Buckets:")
|
||||
for _, line := range strings.Split(dht.node.rt.BucketInfo(), "\n") {
|
||||
log.Println(line)
|
||||
}
|
||||
}
|
||||
|
||||
func (dht DHT) ID() bits.Bitmap {
|
||||
return dht.contact.ID
|
||||
}
|
||||
|
||||
func getContact(nodeID, addr string) (Contact, error) {
|
||||
var c Contact
|
||||
if nodeID == "" {
|
||||
c.ID = bits.Rand()
|
||||
} else {
|
||||
c.ID = bits.FromHexP(nodeID)
|
||||
}
|
||||
|
||||
ip, port, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return c, errors.Err(err)
|
||||
} else if ip == "" {
|
||||
return c, errors.Err("address does not contain an IP")
|
||||
} else if port == "" {
|
||||
return c, errors.Err("address does not contain a port")
|
||||
}
|
||||
|
||||
c.IP = net.ParseIP(ip)
|
||||
if c.IP == nil {
|
||||
return c, errors.Err("invalid ip")
|
||||
}
|
||||
|
||||
c.Port, err = cast.ToIntE(port)
|
||||
if err != nil {
|
||||
return c, errors.Err(err)
|
||||
}
|
||||
|
||||
return c, nil
|
||||
}
|
||||
@@ -1,214 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"container/ring"
|
||||
"context"
|
||||
"math"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
type queueEdit struct {
|
||||
hash bits.Bitmap
|
||||
add bool
|
||||
}
|
||||
|
||||
const (
|
||||
announceStarted = "started"
|
||||
announceFinishd = "finished"
|
||||
)
|
||||
|
||||
type announceNotification struct {
|
||||
hash bits.Bitmap
|
||||
action string
|
||||
err error
|
||||
}
|
||||
|
||||
// Add adds the hash to the list of hashes this node is announcing
|
||||
func (dht *DHT) Add(hash bits.Bitmap) {
|
||||
dht.announceAddRemove <- queueEdit{hash: hash, add: true}
|
||||
}
|
||||
|
||||
// Remove removes the hash from the list of hashes this node is announcing
|
||||
func (dht *DHT) Remove(hash bits.Bitmap) {
|
||||
dht.announceAddRemove <- queueEdit{hash: hash, add: false}
|
||||
}
|
||||
|
||||
func (dht *DHT) runAnnouncer() {
|
||||
type hashAndTime struct {
|
||||
hash bits.Bitmap
|
||||
lastAnnounce time.Time
|
||||
}
|
||||
|
||||
var queue *ring.Ring
|
||||
hashes := make(map[bits.Bitmap]*ring.Ring)
|
||||
|
||||
var announceNextHash <-chan time.Time
|
||||
timer := time.NewTimer(math.MaxInt64)
|
||||
timer.Stop()
|
||||
|
||||
limitCh := make(chan time.Time)
|
||||
dht.grp.Add(1)
|
||||
go func() {
|
||||
defer dht.grp.Done()
|
||||
limiter := rate.NewLimiter(rate.Limit(dht.conf.AnnounceRate), dht.conf.AnnounceRate)
|
||||
for {
|
||||
err := limiter.Wait(context.Background()) // TODO: should use grp.ctx somehow? so when grp is closed, wait returns
|
||||
if err != nil {
|
||||
log.Error(errors.Prefix("rate limiter", err))
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case limitCh <- time.Now():
|
||||
case <-dht.grp.Ch():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
maintenance := time.NewTicker(1 * time.Minute)
|
||||
|
||||
// TODO: work to space hash announces out so they aren't bunched up around the reannounce time. track time since last announce. if its been more than the ideal time (reannounce time / numhashes), start announcing hashes early
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-dht.grp.Ch():
|
||||
return
|
||||
|
||||
case <-maintenance.C:
|
||||
maxAnnounce := dht.conf.AnnounceRate * int(dht.conf.ReannounceTime.Seconds())
|
||||
if len(hashes) > maxAnnounce {
|
||||
// TODO: send this to slack
|
||||
log.Warnf("DHT has %d hashes, but can only announce %d hashes in the %s reannounce window. Raise the announce rate or spawn more nodes.",
|
||||
len(hashes), maxAnnounce, dht.conf.ReannounceTime.String())
|
||||
}
|
||||
|
||||
case change := <-dht.announceAddRemove:
|
||||
if change.add {
|
||||
if _, exists := hashes[change.hash]; exists {
|
||||
continue
|
||||
}
|
||||
|
||||
r := ring.New(1)
|
||||
r.Value = hashAndTime{hash: change.hash}
|
||||
if queue != nil {
|
||||
queue.Prev().Link(r)
|
||||
}
|
||||
queue = r
|
||||
hashes[change.hash] = r
|
||||
announceNextHash = limitCh // announce next hash ASAP
|
||||
} else {
|
||||
r, exists := hashes[change.hash]
|
||||
if !exists {
|
||||
continue
|
||||
}
|
||||
|
||||
delete(hashes, change.hash)
|
||||
|
||||
if len(hashes) == 0 {
|
||||
queue = ring.New(0)
|
||||
announceNextHash = nil // no hashes to announce, wait indefinitely
|
||||
} else {
|
||||
if r == queue {
|
||||
queue = queue.Next() // don't lose our pointer
|
||||
}
|
||||
r.Prev().Link(r.Next())
|
||||
}
|
||||
}
|
||||
|
||||
case <-announceNextHash:
|
||||
dht.grp.Add(1)
|
||||
ht := queue.Value.(hashAndTime)
|
||||
|
||||
if !ht.lastAnnounce.IsZero() {
|
||||
nextAnnounce := ht.lastAnnounce.Add(dht.conf.ReannounceTime)
|
||||
if nextAnnounce.After(time.Now()) {
|
||||
timer.Reset(time.Until(nextAnnounce))
|
||||
announceNextHash = timer.C // wait until next hash should be announced
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if dht.conf.AnnounceNotificationCh != nil {
|
||||
dht.conf.AnnounceNotificationCh <- announceNotification{
|
||||
hash: ht.hash,
|
||||
action: announceStarted,
|
||||
}
|
||||
}
|
||||
|
||||
go func(hash bits.Bitmap) {
|
||||
defer dht.grp.Done()
|
||||
err := dht.announce(hash)
|
||||
if err != nil {
|
||||
log.Error(errors.Prefix("announce", err))
|
||||
}
|
||||
|
||||
if dht.conf.AnnounceNotificationCh != nil {
|
||||
dht.conf.AnnounceNotificationCh <- announceNotification{
|
||||
hash: ht.hash,
|
||||
action: announceFinishd,
|
||||
err: err,
|
||||
}
|
||||
}
|
||||
}(ht.hash)
|
||||
|
||||
queue.Value = hashAndTime{hash: ht.hash, lastAnnounce: time.Now()}
|
||||
queue = queue.Next()
|
||||
announceNextHash = limitCh // announce next hash ASAP
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Announce announces to the DHT that this node has the blob for the given hash
|
||||
func (dht *DHT) announce(hash bits.Bitmap) error {
|
||||
contacts, _, err := FindContacts(dht.node, hash, false, dht.grp.Child())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// self-store if we found less than K contacts, or we're closer than the farthest contact
|
||||
if len(contacts) < bucketSize {
|
||||
contacts = append(contacts, dht.contact)
|
||||
} else if hash.Closer(dht.node.id, contacts[bucketSize-1].ID) {
|
||||
contacts[bucketSize-1] = dht.contact
|
||||
}
|
||||
|
||||
wg := &sync.WaitGroup{}
|
||||
for _, c := range contacts {
|
||||
wg.Add(1)
|
||||
go func(c Contact) {
|
||||
dht.store(hash, c)
|
||||
wg.Done()
|
||||
}(c)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (dht *DHT) store(hash bits.Bitmap, c Contact) {
|
||||
if dht.contact.ID == c.ID {
|
||||
// self-store
|
||||
c.PeerPort = dht.conf.PeerProtocolPort
|
||||
dht.node.Store(hash, c)
|
||||
return
|
||||
}
|
||||
|
||||
dht.node.SendAsync(c, Request{
|
||||
Method: storeMethod,
|
||||
StoreArgs: &storeArgs{
|
||||
BlobHash: hash,
|
||||
Value: storeArgsValue{
|
||||
Token: dht.tokenCache.Get(c, hash, dht.grp.Ch()),
|
||||
LbryID: dht.contact.ID,
|
||||
Port: dht.conf.PeerProtocolPort,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
-181
@@ -1,181 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"net"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
)
|
||||
|
||||
func TestNodeFinder_FindNodes(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping slow nodeFinder test")
|
||||
}
|
||||
|
||||
bs, dhts := TestingCreateNetwork(t, 3, true, false)
|
||||
defer func() {
|
||||
for i := range dhts {
|
||||
dhts[i].Shutdown()
|
||||
}
|
||||
bs.Shutdown()
|
||||
}()
|
||||
|
||||
contacts, found, err := FindContacts(dhts[2].node, bits.Rand(), false, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if found {
|
||||
t.Fatal("something was found, but it should not have been")
|
||||
}
|
||||
|
||||
if len(contacts) != 3 {
|
||||
t.Errorf("expected 3 node, found %d", len(contacts))
|
||||
}
|
||||
|
||||
foundBootstrap := false
|
||||
foundOne := false
|
||||
foundTwo := false
|
||||
|
||||
for _, n := range contacts {
|
||||
if n.ID.Equals(bs.id) {
|
||||
foundBootstrap = true
|
||||
}
|
||||
if n.ID.Equals(dhts[0].node.id) {
|
||||
foundOne = true
|
||||
}
|
||||
if n.ID.Equals(dhts[1].node.id) {
|
||||
foundTwo = true
|
||||
}
|
||||
}
|
||||
|
||||
if !foundBootstrap {
|
||||
t.Errorf("did not find bootstrap node %s", bs.id.Hex())
|
||||
}
|
||||
if !foundOne {
|
||||
t.Errorf("did not find first node %s", dhts[0].node.id.Hex())
|
||||
}
|
||||
if !foundTwo {
|
||||
t.Errorf("did not find second node %s", dhts[1].node.id.Hex())
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodeFinder_FindNodes_NoBootstrap(t *testing.T) {
|
||||
_, dhts := TestingCreateNetwork(t, 3, false, false)
|
||||
defer func() {
|
||||
for i := range dhts {
|
||||
dhts[i].Shutdown()
|
||||
}
|
||||
}()
|
||||
|
||||
_, _, err := FindContacts(dhts[2].node, bits.Rand(), false, nil)
|
||||
if err == nil {
|
||||
t.Fatal("contact finder should have errored saying that there are no contacts in the routing table")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodeFinder_FindValue(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping slow nodeFinder test")
|
||||
}
|
||||
|
||||
bs, dhts := TestingCreateNetwork(t, 3, true, false)
|
||||
defer func() {
|
||||
for i := range dhts {
|
||||
dhts[i].Shutdown()
|
||||
}
|
||||
bs.Shutdown()
|
||||
}()
|
||||
|
||||
blobHashToFind := bits.Rand()
|
||||
nodeToFind := Contact{ID: bits.Rand(), IP: net.IPv4(1, 2, 3, 4), Port: 5678}
|
||||
dhts[0].node.store.Upsert(blobHashToFind, nodeToFind)
|
||||
|
||||
contacts, found, err := FindContacts(dhts[2].node, blobHashToFind, true, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if !found {
|
||||
t.Fatal("node was not found")
|
||||
}
|
||||
|
||||
if len(contacts) != 1 {
|
||||
t.Fatalf("expected one node, found %d", len(contacts))
|
||||
}
|
||||
|
||||
if !contacts[0].ID.Equals(nodeToFind.ID) {
|
||||
t.Fatalf("found node id %s, expected %s", contacts[0].ID.Hex(), nodeToFind.ID.Hex())
|
||||
}
|
||||
}
|
||||
|
||||
func TestDHT_LargeDHT(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping large DHT test")
|
||||
}
|
||||
|
||||
nodes := 100
|
||||
bs, dhts := TestingCreateNetwork(t, nodes, true, true)
|
||||
defer func() {
|
||||
for _, d := range dhts {
|
||||
go d.Shutdown()
|
||||
}
|
||||
bs.Shutdown()
|
||||
time.Sleep(1 * time.Second)
|
||||
}()
|
||||
|
||||
wg := &sync.WaitGroup{}
|
||||
ids := make([]bits.Bitmap, nodes)
|
||||
for i := range ids {
|
||||
ids[i] = bits.Rand()
|
||||
wg.Add(1)
|
||||
go func(index int) {
|
||||
defer wg.Done()
|
||||
err := dhts[index].announce(ids[index])
|
||||
if err != nil {
|
||||
t.Error("error announcing random bitmap - ", err)
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
// check that each node is in at learst 1 other routing table
|
||||
rtCounts := make(map[bits.Bitmap]int)
|
||||
for _, d := range dhts {
|
||||
for _, d2 := range dhts {
|
||||
if d.node.id.Equals(d2.node.id) {
|
||||
continue
|
||||
}
|
||||
c := d2.node.rt.GetClosest(d.node.id, 1)
|
||||
if len(c) > 1 {
|
||||
t.Error("rt returned more than one node when only one requested")
|
||||
} else if len(c) == 1 && c[0].ID.Equals(d.node.id) {
|
||||
rtCounts[d.node.id]++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for k, v := range rtCounts {
|
||||
if v == 0 {
|
||||
t.Errorf("%s was not in any routing tables", k.HexShort())
|
||||
}
|
||||
}
|
||||
|
||||
// check that each ID is stored by at least 3 nodes
|
||||
storeCounts := make(map[bits.Bitmap]int)
|
||||
for _, d := range dhts {
|
||||
for _, id := range ids {
|
||||
if len(d.node.store.Get(id)) > 0 {
|
||||
storeCounts[id]++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for k, v := range storeCounts {
|
||||
if v == 0 {
|
||||
t.Errorf("%s was not stored by any nodes", k.HexShort())
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
-463
@@ -1,463 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
|
||||
"github.com/lyoshenka/bencode"
|
||||
"github.com/spf13/cast"
|
||||
)
|
||||
|
||||
const (
|
||||
pingMethod = "ping"
|
||||
storeMethod = "store"
|
||||
findNodeMethod = "findNode"
|
||||
findValueMethod = "findValue"
|
||||
)
|
||||
|
||||
const (
|
||||
pingSuccessResponse = "pong"
|
||||
storeSuccessResponse = "OK"
|
||||
)
|
||||
|
||||
const (
|
||||
requestType = 0
|
||||
responseType = 1
|
||||
errorType = 2
|
||||
)
|
||||
|
||||
const (
|
||||
// these are strings because bencode requires bytestring keys
|
||||
headerTypeField = "0"
|
||||
headerMessageIDField = "1" // message id is 20 bytes long
|
||||
headerNodeIDField = "2" // node id is 48 bytes long
|
||||
headerPayloadField = "3"
|
||||
headerArgsField = "4"
|
||||
contactsField = "contacts"
|
||||
tokenField = "token"
|
||||
protocolVersionField = "protocolVersion"
|
||||
)
|
||||
|
||||
// Message is a DHT message
|
||||
type Message interface {
|
||||
bencode.Marshaler
|
||||
}
|
||||
|
||||
type messageID [messageIDLength]byte
|
||||
|
||||
// HexShort returns the first 8 hex characters of the hex encoded message id.
|
||||
func (m messageID) HexShort() string {
|
||||
return hex.EncodeToString(m[:])[:8]
|
||||
}
|
||||
|
||||
// UnmarshalBencode takes a byte slice and unmarshals the message id.
|
||||
func (m *messageID) UnmarshalBencode(encoded []byte) error {
|
||||
var str string
|
||||
err := bencode.DecodeBytes(encoded, &str)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
copy(m[:], str)
|
||||
return nil
|
||||
}
|
||||
|
||||
// MarshallBencode returns the encoded byte slice of the message id.
|
||||
func (m messageID) MarshalBencode() ([]byte, error) {
|
||||
str := string(m[:])
|
||||
return bencode.EncodeBytes(str)
|
||||
}
|
||||
|
||||
func newMessageID() messageID {
|
||||
var m messageID
|
||||
_, err := rand.Read(m[:])
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// Request represents a DHT request message
|
||||
type Request struct {
|
||||
ID messageID
|
||||
NodeID bits.Bitmap
|
||||
Method string
|
||||
Arg *bits.Bitmap
|
||||
StoreArgs *storeArgs
|
||||
ProtocolVersion int
|
||||
}
|
||||
|
||||
// MarshalBencode returns the serialized byte slice representation of the request
|
||||
func (r Request) MarshalBencode() ([]byte, error) {
|
||||
var args interface{}
|
||||
if r.StoreArgs != nil {
|
||||
args = r.StoreArgs
|
||||
} else if r.Arg != nil {
|
||||
args = []bits.Bitmap{*r.Arg}
|
||||
} else {
|
||||
args = []string{} // request must always have keys 0-4, so we use an empty list for PING
|
||||
}
|
||||
return bencode.EncodeBytes(map[string]interface{}{
|
||||
headerTypeField: requestType,
|
||||
headerMessageIDField: r.ID,
|
||||
headerNodeIDField: r.NodeID,
|
||||
headerPayloadField: r.Method,
|
||||
headerArgsField: args,
|
||||
})
|
||||
}
|
||||
|
||||
// UnmarshalBencode unmarshals the serialized byte slice into the appropriate fields of the request.
|
||||
func (r *Request) UnmarshalBencode(b []byte) error {
|
||||
var raw struct {
|
||||
ID messageID `bencode:"1"`
|
||||
NodeID bits.Bitmap `bencode:"2"`
|
||||
Method string `bencode:"3"`
|
||||
Args bencode.RawMessage `bencode:"4"`
|
||||
}
|
||||
err := bencode.DecodeBytes(b, &raw)
|
||||
if err != nil {
|
||||
return errors.Prefix("request unmarshal", err)
|
||||
}
|
||||
|
||||
r.ID = raw.ID
|
||||
r.NodeID = raw.NodeID
|
||||
r.Method = raw.Method
|
||||
|
||||
if r.Method == storeMethod {
|
||||
r.StoreArgs = &storeArgs{} // bencode wont find the unmarshaler on a null pointer. need to fix it.
|
||||
err = bencode.DecodeBytes(raw.Args, &r.StoreArgs)
|
||||
if err != nil {
|
||||
return errors.Prefix("request unmarshal", err)
|
||||
}
|
||||
} else if len(raw.Args) > 2 { // 2 because an empty list is `le`
|
||||
r.Arg, r.ProtocolVersion, err = processArgsAndProtoVersion(raw.Args)
|
||||
if err != nil {
|
||||
return errors.Prefix("request unmarshal", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func processArgsAndProtoVersion(raw bencode.RawMessage) (arg *bits.Bitmap, version int, err error) {
|
||||
var args []bencode.RawMessage
|
||||
err = bencode.DecodeBytes(raw, &args)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
if len(args) == 0 {
|
||||
return nil, 0, nil
|
||||
}
|
||||
|
||||
var extras map[string]int
|
||||
err = bencode.DecodeBytes(args[len(args)-1], &extras)
|
||||
if err == nil {
|
||||
if v, exists := extras[protocolVersionField]; exists {
|
||||
version = v
|
||||
args = args[:len(args)-1]
|
||||
}
|
||||
}
|
||||
|
||||
if len(args) > 0 {
|
||||
var b bits.Bitmap
|
||||
err = bencode.DecodeBytes(args[0], &b)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
arg = &b
|
||||
}
|
||||
|
||||
return arg, version, nil
|
||||
}
|
||||
|
||||
func (r Request) argsDebug() string {
|
||||
if r.StoreArgs != nil {
|
||||
return r.StoreArgs.BlobHash.HexShort() + ", " + r.StoreArgs.Value.LbryID.HexShort() + ":" + strconv.Itoa(r.StoreArgs.Value.Port)
|
||||
} else if r.Arg != nil {
|
||||
return r.Arg.HexShort()
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type storeArgsValue struct {
|
||||
Token string `bencode:"token"`
|
||||
LbryID bits.Bitmap `bencode:"lbryid"`
|
||||
Port int `bencode:"port"`
|
||||
}
|
||||
|
||||
type storeArgs struct {
|
||||
BlobHash bits.Bitmap
|
||||
Value storeArgsValue
|
||||
NodeID bits.Bitmap // original publisher id? I think this is getting fixed in the new dht stuff
|
||||
SelfStore bool // this is an int on the wire
|
||||
}
|
||||
|
||||
// MarshalBencode returns the serialized byte slice representation of the storage arguments.
|
||||
func (s storeArgs) MarshalBencode() ([]byte, error) {
|
||||
encodedValue, err := bencode.EncodeString(s.Value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
selfStoreStr := 0
|
||||
if s.SelfStore {
|
||||
selfStoreStr = 1
|
||||
}
|
||||
|
||||
return bencode.EncodeBytes([]interface{}{
|
||||
s.BlobHash,
|
||||
bencode.RawMessage(encodedValue),
|
||||
s.NodeID,
|
||||
selfStoreStr,
|
||||
})
|
||||
}
|
||||
|
||||
// UnmarshalBencode unmarshals the serialized byte slice into the appropriate fields of the store arguments.
|
||||
func (s *storeArgs) UnmarshalBencode(b []byte) error {
|
||||
var argsInt []bencode.RawMessage
|
||||
err := bencode.DecodeBytes(b, &argsInt)
|
||||
if err != nil {
|
||||
return errors.Prefix("storeArgs unmarshal", err)
|
||||
}
|
||||
|
||||
if len(argsInt) != 4 {
|
||||
return errors.Err("unexpected number of fields for store args. got " + cast.ToString(len(argsInt)))
|
||||
}
|
||||
|
||||
err = bencode.DecodeBytes(argsInt[0], &s.BlobHash)
|
||||
if err != nil {
|
||||
return errors.Prefix("storeArgs unmarshal", err)
|
||||
}
|
||||
|
||||
err = bencode.DecodeBytes(argsInt[1], &s.Value)
|
||||
if err != nil {
|
||||
return errors.Prefix("storeArgs unmarshal", err)
|
||||
}
|
||||
|
||||
err = bencode.DecodeBytes(argsInt[2], &s.NodeID)
|
||||
if err != nil {
|
||||
return errors.Prefix("storeArgs unmarshal", err)
|
||||
}
|
||||
|
||||
var selfStore int
|
||||
err = bencode.DecodeBytes(argsInt[3], &selfStore)
|
||||
if err != nil {
|
||||
return errors.Prefix("storeArgs unmarshal", err)
|
||||
}
|
||||
if selfStore == 0 {
|
||||
s.SelfStore = false
|
||||
} else if selfStore == 1 {
|
||||
s.SelfStore = true
|
||||
} else {
|
||||
return errors.Err("selfstore must be 1 or 0")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Response represents a DHT response message
|
||||
type Response struct {
|
||||
ID messageID
|
||||
NodeID bits.Bitmap
|
||||
Data string
|
||||
Contacts []Contact
|
||||
FindValueKey string
|
||||
Token string
|
||||
ProtocolVersion int
|
||||
}
|
||||
|
||||
func (r Response) argsDebug() string {
|
||||
if r.Data != "" {
|
||||
return r.Data
|
||||
}
|
||||
|
||||
str := "contacts "
|
||||
if r.FindValueKey != "" {
|
||||
str = "value for " + hex.EncodeToString([]byte(r.FindValueKey))[:8] + " "
|
||||
}
|
||||
|
||||
str += "|"
|
||||
for _, c := range r.Contacts {
|
||||
str += c.String() + ","
|
||||
}
|
||||
str = strings.TrimRight(str, ",") + "|"
|
||||
|
||||
if r.Token != "" {
|
||||
str += " token: " + hex.EncodeToString([]byte(r.Token))[:8]
|
||||
}
|
||||
|
||||
return str
|
||||
}
|
||||
|
||||
// MarshalBencode returns the serialized byte slice representation of the response.
|
||||
func (r Response) MarshalBencode() ([]byte, error) {
|
||||
data := map[string]interface{}{
|
||||
headerTypeField: responseType,
|
||||
headerMessageIDField: r.ID,
|
||||
headerNodeIDField: r.NodeID,
|
||||
}
|
||||
|
||||
if r.Data != "" {
|
||||
// ping or store
|
||||
data[headerPayloadField] = r.Data
|
||||
} else if r.FindValueKey != "" {
|
||||
// findValue success
|
||||
if r.Token == "" {
|
||||
return nil, errors.Err("response to findValue must have a token")
|
||||
}
|
||||
|
||||
var contacts [][]byte
|
||||
for _, c := range r.Contacts {
|
||||
compact, err := c.MarshalCompact()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
contacts = append(contacts, compact)
|
||||
}
|
||||
data[headerPayloadField] = map[string]interface{}{
|
||||
r.FindValueKey: contacts,
|
||||
tokenField: r.Token,
|
||||
}
|
||||
} else if r.Token != "" {
|
||||
// findValue failure falling back to findNode
|
||||
data[headerPayloadField] = map[string]interface{}{
|
||||
contactsField: r.Contacts,
|
||||
tokenField: r.Token,
|
||||
}
|
||||
} else {
|
||||
// straight up findNode
|
||||
data[headerPayloadField] = r.Contacts
|
||||
}
|
||||
|
||||
return bencode.EncodeBytes(data)
|
||||
}
|
||||
|
||||
// UnmarshalBencode unmarshals the serialized byte slice into the appropriate fields of the store arguments.
|
||||
func (r *Response) UnmarshalBencode(b []byte) error {
|
||||
var raw struct {
|
||||
ID messageID `bencode:"1"`
|
||||
NodeID bits.Bitmap `bencode:"2"`
|
||||
Data bencode.RawMessage `bencode:"3"`
|
||||
}
|
||||
err := bencode.DecodeBytes(b, &raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
r.ID = raw.ID
|
||||
r.NodeID = raw.NodeID
|
||||
|
||||
// maybe data is a string (response to ping or store)?
|
||||
err = bencode.DecodeBytes(raw.Data, &r.Data)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// maybe data is a list of contacts (response to findNode)?
|
||||
err = bencode.DecodeBytes(raw.Data, &r.Contacts)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// it must be a response to findValue
|
||||
var rawData map[string]bencode.RawMessage
|
||||
err = bencode.DecodeBytes(raw.Data, &rawData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if token, ok := rawData[tokenField]; ok {
|
||||
err = bencode.DecodeBytes(token, &r.Token)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
delete(rawData, tokenField) // so it doesnt mess up findValue key finding below
|
||||
}
|
||||
|
||||
if protocolVersion, ok := rawData[protocolVersionField]; ok {
|
||||
err = bencode.DecodeBytes(protocolVersion, &r.ProtocolVersion)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
delete(rawData, protocolVersionField) // so it doesnt mess up findValue key finding below
|
||||
}
|
||||
|
||||
if contacts, ok := rawData[contactsField]; ok {
|
||||
err = bencode.DecodeBytes(contacts, &r.Contacts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
for k, v := range rawData {
|
||||
r.FindValueKey = k
|
||||
var compactContacts [][]byte
|
||||
err = bencode.DecodeBytes(v, &compactContacts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, compact := range compactContacts {
|
||||
var c Contact
|
||||
err = c.UnmarshalCompact(compact)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r.Contacts = append(r.Contacts, c)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Error represents a DHT error response
|
||||
type Error struct {
|
||||
ID messageID
|
||||
NodeID bits.Bitmap
|
||||
ExceptionType string
|
||||
Response []string
|
||||
}
|
||||
|
||||
// MarshalBencode returns the serialized byte slice representation of an error message.
|
||||
func (e Error) MarshalBencode() ([]byte, error) {
|
||||
return bencode.EncodeBytes(map[string]interface{}{
|
||||
headerTypeField: errorType,
|
||||
headerMessageIDField: e.ID,
|
||||
headerNodeIDField: e.NodeID,
|
||||
headerPayloadField: e.ExceptionType,
|
||||
headerArgsField: e.Response,
|
||||
})
|
||||
}
|
||||
|
||||
// UnmarshalBencode unmarshals the serialized byte slice into the appropriate fields of the error message.
|
||||
func (e *Error) UnmarshalBencode(b []byte) error {
|
||||
var raw struct {
|
||||
ID messageID `bencode:"1"`
|
||||
NodeID bits.Bitmap `bencode:"2"`
|
||||
ExceptionType string `bencode:"3"`
|
||||
Args interface{} `bencode:"4"`
|
||||
}
|
||||
err := bencode.DecodeBytes(b, &raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
e.ID = raw.ID
|
||||
e.NodeID = raw.NodeID
|
||||
e.ExceptionType = raw.ExceptionType
|
||||
|
||||
if reflect.TypeOf(raw.Args).Kind() == reflect.Slice {
|
||||
v := reflect.ValueOf(raw.Args)
|
||||
for i := 0; i < v.Len(); i++ {
|
||||
e.Response = append(e.Response, cast.ToString(v.Index(i).Interface()))
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,223 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"net"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
"github.com/lyoshenka/bencode"
|
||||
)
|
||||
|
||||
func TestBencodeDecodeStoreArgs(t *testing.T) {
|
||||
blobHash := "3214D6C2F77FCB5E8D5FC07EDAFBA614F031CE8B2EAB49F924F8143F6DFBADE048D918710072FB98AB1B52B58F4E1468"
|
||||
lbryID := "7CE1B831DEC8689E44F80F547D2DEA171F6A625E1A4FF6C6165E645F953103DABEB068A622203F859C6C64658FD3AA3B"
|
||||
port := hex.EncodeToString([]byte("3333"))
|
||||
token := "17C2D8E1E48EF21567FE4AD5C8ED944B798D3B65AB58D0C9122AD6587D1B5FED472EA2CB12284CEFA1C21EFF302322BD"
|
||||
nodeID := "7CE1B831DEC8689E44F80F547D2DEA171F6A625E1A4FF6C6165E645F953103DABEB068A622203F859C6C64658FD3AA3B"
|
||||
selfStore := hex.EncodeToString([]byte("1"))
|
||||
|
||||
raw := "6C" + // start args list
|
||||
"3438 3A " + blobHash + // blob hash
|
||||
"64" + // start value dict
|
||||
"363A6C6272796964 3438 3A " + lbryID + // lbry id
|
||||
"343A706F7274 69 " + port + " 65" + // port
|
||||
"353A746F6B656E 3438 3A " + token + // token
|
||||
"65" + // end value dict
|
||||
"3438 3A " + nodeID + // node id
|
||||
"69 " + selfStore + " 65" + // self store (integer)
|
||||
"65" // end args list
|
||||
|
||||
raw = strings.ToLower(strings.Replace(raw, " ", "", -1))
|
||||
|
||||
data, err := hex.DecodeString(raw)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
storeArgs := &storeArgs{}
|
||||
err = bencode.DecodeBytes(data, storeArgs)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
if storeArgs.BlobHash.Hex() != strings.ToLower(blobHash) {
|
||||
t.Error("blob hash mismatch")
|
||||
}
|
||||
if storeArgs.Value.LbryID.Hex() != strings.ToLower(lbryID) {
|
||||
t.Error("lbryid mismatch")
|
||||
}
|
||||
if hex.EncodeToString([]byte(strconv.Itoa(storeArgs.Value.Port))) != port {
|
||||
t.Error("port mismatch")
|
||||
}
|
||||
if hex.EncodeToString([]byte(storeArgs.Value.Token)) != strings.ToLower(token) {
|
||||
t.Error("token mismatch")
|
||||
}
|
||||
if storeArgs.NodeID.Hex() != strings.ToLower(nodeID) {
|
||||
t.Error("node id mismatch")
|
||||
}
|
||||
if !storeArgs.SelfStore {
|
||||
t.Error("selfStore mismatch")
|
||||
}
|
||||
|
||||
reencoded, err := bencode.EncodeBytes(storeArgs)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
} else if !reflect.DeepEqual(reencoded, data) {
|
||||
t.Error("reencoded data does not match original")
|
||||
spew.Dump(reencoded, data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBencodeFindNodesResponse(t *testing.T) {
|
||||
res := Response{
|
||||
ID: newMessageID(),
|
||||
NodeID: bits.Rand(),
|
||||
Contacts: []Contact{
|
||||
{ID: bits.Rand(), IP: net.IPv4(1, 2, 3, 4).To4(), Port: 5678},
|
||||
{ID: bits.Rand(), IP: net.IPv4(4, 3, 2, 1).To4(), Port: 8765},
|
||||
},
|
||||
}
|
||||
|
||||
encoded, err := bencode.EncodeBytes(res)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var res2 Response
|
||||
err = bencode.DecodeBytes(encoded, &res2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
compareResponses(t, res, res2)
|
||||
}
|
||||
|
||||
func TestBencodeFindValueResponse(t *testing.T) {
|
||||
res := Response{
|
||||
ID: newMessageID(),
|
||||
NodeID: bits.Rand(),
|
||||
FindValueKey: bits.Rand().RawString(),
|
||||
Token: "arstarstarst",
|
||||
Contacts: []Contact{
|
||||
{ID: bits.Rand(), IP: net.IPv4(1, 2, 3, 4).To4(), PeerPort: 8765},
|
||||
},
|
||||
}
|
||||
|
||||
encoded, err := bencode.EncodeBytes(res)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var res2 Response
|
||||
err = bencode.DecodeBytes(encoded, &res2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
compareResponses(t, res, res2)
|
||||
}
|
||||
|
||||
func TestDecodeSomeStrings(t *testing.T) {
|
||||
strs := []string{
|
||||
"6469306569306569316532303a359ec440b8236dee0fb2500ebda9c4704ae6741469326534383ade269943996b4bef3ff41176668a0577f86aba7f1ea2996edd18f9c42430802c8085331345c5f0c44a7f352e2ba8ae59693365383a66696e644e6f64656934656c34383ade269943996b4bef3ff41176668a0577f86aba7f1ea2996edd18f9c42430802c8085331345c5f0c44a7f352e2ba8ae596565",
|
||||
"64313a30693165313a3132303a31f27ec6174783c6f00e4b9ebccd49c6bdce300e313a3234383a7b3785cd962a56d4d4574262b1b63cc95d062c825b38f73602c7dc297c1e6b8259cdcdb5e7edef216d514e0c31ad8637313a3364353a746f6b656e34383a071fdf9835995889920e9beb876ebe8e36e5d44f5822684b549457405e22a5bf3fe40335743b459f22c347c0e736eba434383a7bcc7f8804db7d0d40fcb9c508547e53ee4f7d4000758105ecce099fc931d9cccfbbf9a30829777afcf41f2b4b0aeb716c35343a627db2b20d05b109195e7d06a855c392fda9a05faa878e69b18cc1212c35371ba5cad6231e049a460cdc263e7f4701afad45b288f48435343a424434ae0d05bfd6e1b98cc921b441fa8b560ac29038c37ec4907ae023617bf2bbc54abd3403226daf479798edcce69743ea6699e6ff35343a4462ba020d059ed3d31ceb3d1ffb2c025abf065a14516a8942607f9b3fad4b697ccc232766bdf471b6518a3ed716f2712171fd046c4335343a47413e2b0d05bf6244e5edacf43f65ef4cfd4b6eac380244894cc364ec23e8ededfb085999a211e20f8de146d9e081bbb42dc61d9ca035343a627db2b20d05b109195e7d06a855c392fda9a05faa878e69b18cc1212c35371ba5cad6231e049a460cdc263e7f4701afad45b288f48435343a4462ba020d059ed3d31ceb3d1ffb2c025abf065a14516a8942607f9b3fad4b697ccc232766bdf471b6518a3ed716f2712171fd046c4335343a47413e2b0d05bf6244e5edacf43f65ef4cfd4b6eac380244894cc364ec23e8ededfb085999a211e20f8de146d9e081bbb42dc61d9ca035343a627db2b20d05b109195e7d06a855c392fda9a05faa878e69b18cc1212c35371ba5cad6231e049a460cdc263e7f4701afad45b288f48435343a68fe5c3e0d05bfd6e1b98cc921b441fa8b560ac29038c37ec4907ae023617bf2bbc54abd3403226daf479798edcce69743ea6699e6ff35343a4462ba020d059ed3d31ceb3d1ffb2c025abf065a14516a8942607f9b3fad4b697ccc232766bdf471b6518a3ed716f2712171fd046c4335343a47413e2b0d05bf6244e5edacf43f65ef4cfd4b6eac380244894cc364ec23e8ededfb085999a211e20f8de146d9e081bbb42dc61d9ca035343a67d9a7ea0d05223c42ef5979c7c8f2444311f7a8266d64d3f98e9d0b21726bc2714a6fea5426c9a46a68c49ea13b54ccc324adf0783035343a68fe5c3e0d05bfd6e1b98cc921b441fa8b560ac29038c37ec4907ae023617bf2bbc54abd3403226daf479798edcce69743ea6699e6ff35343a47413e2b0d05bf6244e5edacf43f65ef4cfd4b6eac380244894cc364ec23e8ededfb085999a211e20f8de146d9e081bbb42dc61d9ca035343a4462ba020d059ed3d31ceb3d1ffb2c025abf065a14516a8942607f9b3fad4b697ccc232766bdf471b6518a3ed716f2712171fd046c4335343a67d9a7ea0d05223c42ef5979c7c8f2444311f7a8266d64d3f98e9d0b21726bc2714a6fea5426c9a46a68c49ea13b54ccc324adf0783035343a424434ae0d05bfd6e1b98cc921b441fa8b560ac29038c37ec4907ae023617bf2bbc54abd3403226daf479798edcce69743ea6699e6ff35343a4ace39c80d0541ae66325db4f1fce530476942d339f3219913f0a237726e18746a65ff7b3c5558c82c274a6f1ed10bbaf7c1305ca8e635343a47413e2b0d05bf6244e5edacf43f65ef4cfd4b6eac380244894cc364ec23e8ededfb085999a211e20f8de146d9e081bbb42dc61d9ca035343a4462ba020d059ed3d31ceb3d1ffb2c025abf065a14516a8942607f9b3fad4b697ccc232766bdf471b6518a3ed716f2712171fd046c4335343a627db2b20d05b109195e7d06a855c392fda9a05faa878e69b18cc1212c35371ba5cad6231e049a460cdc263e7f4701afad45b288f48435343a424434ae0d05bfd6e1b98cc921b441fa8b560ac29038c37ec4907ae023617bf2bbc54abd3403226daf479798edcce69743ea6699e6ff35343a4ace39c80d0541ae66325db4f1fce530476942d339f3219913f0a237726e18746a65ff7b3c5558c82c274a6f1ed10bbaf7c1305ca8e635343a47413e2b0d05bf6244e5edacf43f65ef4cfd4b6eac380244894cc364ec23e8ededfb085999a211e20f8de146d9e081bbb42dc61d9ca035343a4462ba020d059ed3d31ceb3d1ffb2c025abf065a14516a8942607f9b3fad4b697ccc232766bdf471b6518a3ed716f2712171fd046c4335343a67d9a7ea0d05223c42ef5979c7c8f2444311f7a8266d64d3f98e9d0b21726bc2714a6fea5426c9a46a68c49ea13b54ccc324adf0783035343a4ace39c80d0541ae66325db4f1fce530476942d339f3219913f0a237726e18746a65ff7b3c5558c82c274a6f1ed10bbaf7c1305ca8e635343a47413e2b0d05bf6244e5edacf43f65ef4cfd4b6eac380244894cc364ec23e8ededfb085999a211e20f8de146d9e081bbb42dc61d9ca035343a4462ba020d059ed3d31ceb3d1ffb2c025abf065a14516a8942607f9b3fad4b697ccc232766bdf471b6518a3ed716f2712171fd046c4335343a67d9a7ea0d05223c42ef5979c7c8f2444311f7a8266d64d3f98e9d0b21726bc2714a6fea5426c9a46a68c49ea13b54ccc324adf0783035343a68fe5c3e0d05bfd6e1b98cc921b441fa8b560ac29038c37ec4907ae023617bf2bbc54abd3403226daf479798edcce69743ea6699e6ff35343a4ace39c80d0541ae66325db4f1fce530476942d339f3219913f0a237726e18746a65ff7b3c5558c82c274a6f1ed10bbaf7c1305ca8e635343a47413e2b0d05bf6244e5edacf43f65ef4cfd4b6eac380244894cc364ec23e8ededfb085999a211e20f8de146d9e081bbb42dc61d9ca035343a68fe5c3e0d05bfd6e1b98cc921b441fa8b560ac29038c37ec4907ae023617bf2bbc54abd3403226daf479798edcce69743ea6699e6ff35343a4ace39c80d0541ae66325db4f1fce530476942d339f3219913f0a237726e18746a65ff7b3c5558c82c274a6f1ed10bbaf7c1305ca8e635343a47413e2b0d05bf6244e5edacf43f65ef4cfd4b6eac380244894cc364ec23e8ededfb085999a211e20f8de146d9e081bbb42dc61d9ca035343a4462ba020d059ed3d31ceb3d1ffb2c025abf065a14516a8942607f9b3fad4b697ccc232766bdf471b6518a3ed716f2712171fd046c4335343a67d9a7ea0d05223c42ef5979c7c8f2444311f7a8266d64d3f98e9d0b21726bc2714a6fea5426c9a46a68c49ea13b54ccc324adf0783035343a68fe5c3e0d05bfd6e1b98cc921b441fa8b560ac29038c37ec4907ae023617bf2bbc54abd3403226daf479798edcce69743ea6699e6ff35343a4ace39c80d0541ae66325db4f1fce530476942d339f3219913f0a237726e18746a65ff7b3c5558c82c274a6f1ed10bbaf7c1305ca8e635343a47413e2b0d05bf6244e5edacf43fLine truncated
|
||||
"64313a30693165313a3132303aa19e41f1d887db705fbf0a58f120773069d6ff11313a3234383a6911c21c3b5ea6536f4fb170c87fdc8bf4201124c5fe5eeb5f0054ff48e899a1d6e089a30a12ba8683ebf79691d71439313a3364383a636f6e74616374736c6c34383a6dd3dedeec334bae70a3c5c1b58fef2d4a501af320a332cdd5db4b71945c0a90d11aa2013e8d0216258a643f42d9562631323a34362e32382e3230342e3738693434343465656c34383a7ec0edf204d940d21c17e1c979df1c94610711d7773ceda72a4aa925c5f7799bbb741caa8d6d877aa2776399f77c08a031333a36392e3131382e34332e313234693434343465656c34383a6ec15cd8b95718d010bdf9739fda1af9ded7ac8be7da5cced6dc7a08470c8109accc5bfb973e5c1dd711349c62a8a3c331323a34362e32382e3230342e3738693434343465656c34383a6d2603a690712d5a3d5addb228d31a622f798c64ab2d3576d2c3ab3c5a64cb863c05545f37aa384ce83003416ca5cea431323a34362e32382e3230342e3738693434343465656c34383a6c1d92202a7e74ec85dd4e3e1699303174c4b6460b171637bba0a1c068d72f389035e1ba8bbe70f6621c36d70d4d045031323a34362e32382e3230342e3738693434343465656c34383a6dac30fbe94c006515cffe4898bc534a105fd0403bfc3c8692c2751b7946949dbc9e0c9570477cf8f817d1efc9a0e02531343a31382e3232312e3139342e313933693434343465656c34383a6f2ee379c662ea9f6db86beed6023be300cd23b9f9c4610a195dcb0f258392ec77188854ea3a8c0fffe67844849eb6ca31333a32342e3131362e39302e323238693434343465656c34383a6c55b2dbb89010de9db88e0ad3510a79fe60983b394a9e66b0efe139a455973cdc4415beff82cbdfb63d98caad648ab031333a38382e3132392e3234352e33396934343434656565353a746f6b656e34383a51844b0d1e8a613c4ad783c03b323063d3a5ff063a640368d7754bcae277c22b2b06b46c3e5466c30acb773f77b686476565",
|
||||
"64313a30693165313a3132303af693334af099a987a4b55af2727472b51c1990db313a3234383ab6928ff25778a7bbb5d258d3b3a06e26db1654f3d2efce8c26681d43f7237cdf2e359a4d309c4473d5d89ec99fb4f573313a3364353a746f6b656e34383a61a45a40080ea118361f919ebf09cdc9d2c63f476e7bc4d193463f52b91dc4b0db4ed1004e32d4fc273645421e58479034383abe463e24af015c0d1ba496b4b1911aa98c8b42119295bbcc5c36829a2e2ff8f149743cbbc0af2c39669cb0750fd778576c35343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e9535343a239911670d05234aa57f90985a97e6e67a5131a4f8fe7dfccaacb4707734294b8ad774c92eb424671aa533e76c3083da796b4db30e95656565",
|
||||
"6469306569316569316532303a3e0929c88abb8fe1d718025efc7f4a3cd85de16269326534383a21b2e2d2996b4bef3ff41176668a0577f86aba7f1ea2996edd18f9c42430802c8085331345c5f0c44a7f352e2ba8ae596933656c6565",
|
||||
}
|
||||
|
||||
for i, str := range strs {
|
||||
raw, err := hex.DecodeString(str)
|
||||
if err != nil {
|
||||
t.Errorf("error hex-decoding string %d: %s", i, err)
|
||||
continue
|
||||
}
|
||||
|
||||
var decoded interface{}
|
||||
err = bencode.DecodeBytes(raw, &decoded)
|
||||
if err != nil {
|
||||
t.Errorf("error bencode-decoding string %d: %s", i, err)
|
||||
continue
|
||||
}
|
||||
//
|
||||
//t.Error("TODO")
|
||||
//continue
|
||||
//
|
||||
//spew.Dump(decoded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeFindNodeResponseWithNoNodes(t *testing.T) {
|
||||
raw, err := hex.DecodeString("6469306569316569316532303a3e0929c88abb8fe1d718025efc7f4a3cd85de16269326534383a21b2e2d2996b4bef3ff41176668a0577f86aba7f1ea2996edd18f9c42430802c8085331345c5f0c44a7f352e2ba8ae596933656c6565")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
response := Response{}
|
||||
err = bencode.DecodeBytes(raw, &response)
|
||||
//spew.Dump(response)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeRequestWithProtocolVersion(t *testing.T) {
|
||||
raw, err := hex.DecodeString("6469306569306569316532303a65e1b9afce87c44abc40b4bb466890d2797f0dd269326534383a1baf3dbba8746c7a739f35465e268ace823622c5e8ec1dd7d5d27af5795cfbc54f22a32fbd05d420f241266b3dd16831693365343a70696e676934656c6431353a70726f746f636f6c56657273696f6e693165656565")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
req := Request{}
|
||||
err = bencode.DecodeBytes(raw, &req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if req.ProtocolVersion != 1 {
|
||||
t.Error("protocol version was not detected correctly")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeResponseWithProtocolVersion(t *testing.T) {
|
||||
raw, err := hex.DecodeString("6469306569316569316532303a2b96f5de8be1e86dc2332a50eb313c97848064db69326534383a8b8eb692658ea3d7e7828e80a3133d524c6f82aaff370efa759d5b87821035a32de06724cb099f01a819695f829dff7f6933656431353a70726f746f636f6c56657273696f6e693165353a746f6b656e34383af2368fe4fd06ede6631ad8b153f3b5d8db724f8d520f4291e992e206dd02a216fb7dfd9b81686f11b626e3840df65fb434383a89c5c3f9794b0b24a03406e3b74361edb9ae70828e4c133512fc75db0a2d312673cdd4e30eed37892a46692d2fe439f36c35343a12dd65af0d058cd7d10d122fbe2eb5ae31062b7480011be588f20cfe6807b1939c42eea639059fa6365bfccb56ef8c9e574f49ba035c656565")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
res := Response{}
|
||||
err = bencode.DecodeBytes(raw, &res)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if res.ProtocolVersion != 1 {
|
||||
t.Error("protocol version was not detected correctly")
|
||||
}
|
||||
}
|
||||
|
||||
func compareResponses(t *testing.T, res, res2 Response) {
|
||||
if res.ID != res2.ID {
|
||||
t.Errorf("expected ID %s, got %s", res.ID, res2.ID)
|
||||
}
|
||||
if !res.NodeID.Equals(res2.NodeID) {
|
||||
t.Errorf("expected NodeID %s, got %s", res.NodeID.Hex(), res2.NodeID.Hex())
|
||||
}
|
||||
if res.Data != res2.Data {
|
||||
t.Errorf("expected Data %s, got %s", res.Data, res2.Data)
|
||||
}
|
||||
if res.FindValueKey != res2.FindValueKey {
|
||||
t.Errorf("expected FindValueKey %s, got %s", res.FindValueKey, res2.FindValueKey)
|
||||
}
|
||||
if res.Token != res2.Token {
|
||||
t.Errorf("expected Token %s, got %s", res.Token, res2.Token)
|
||||
}
|
||||
if !reflect.DeepEqual(res.Contacts, res2.Contacts) {
|
||||
t.Errorf("expected FindNodeData %s, got %s", spew.Sdump(res.Contacts), spew.Sdump(res2.Contacts))
|
||||
}
|
||||
}
|
||||
-474
@@ -1,474 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/errors.go"
|
||||
"github.com/lbryio/lbry.go/stop"
|
||||
"github.com/lbryio/lbry.go/util"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/lyoshenka/bencode"
|
||||
)
|
||||
|
||||
// packet represents the information receive from udp.
|
||||
type packet struct {
|
||||
data []byte
|
||||
raddr *net.UDPAddr
|
||||
}
|
||||
|
||||
// UDPConn allows using a mocked connection to test sending/receiving data
|
||||
// TODO: stop mocking this and use the real thing
|
||||
type UDPConn interface {
|
||||
ReadFromUDP([]byte) (int, *net.UDPAddr, error)
|
||||
WriteToUDP([]byte, *net.UDPAddr) (int, error)
|
||||
SetReadDeadline(time.Time) error
|
||||
SetWriteDeadline(time.Time) error
|
||||
Close() error
|
||||
}
|
||||
|
||||
// RequestHandlerFunc is exported handler for requests.
|
||||
type RequestHandlerFunc func(addr *net.UDPAddr, request Request)
|
||||
|
||||
// Node is a type representation of a node on the network.
|
||||
type Node struct {
|
||||
// the node's id
|
||||
id bits.Bitmap
|
||||
// UDP connection for sending and receiving data
|
||||
conn UDPConn
|
||||
// true if we've closed the connection on purpose
|
||||
connClosed bool
|
||||
// token manager
|
||||
tokens *tokenManager
|
||||
|
||||
// map of outstanding transactions + mutex
|
||||
txLock *sync.RWMutex
|
||||
transactions map[messageID]*transaction
|
||||
|
||||
// routing table
|
||||
rt *routingTable
|
||||
// data store
|
||||
store *contactStore
|
||||
|
||||
// overrides for request handlers
|
||||
requestHandler RequestHandlerFunc
|
||||
|
||||
// stop the node neatly and clean up after itself
|
||||
grp *stop.Group
|
||||
}
|
||||
|
||||
// NewNode returns an initialized Node's pointer.
|
||||
func NewNode(id bits.Bitmap) *Node {
|
||||
return &Node{
|
||||
id: id,
|
||||
rt: newRoutingTable(id),
|
||||
store: newStore(),
|
||||
|
||||
txLock: &sync.RWMutex{},
|
||||
transactions: make(map[messageID]*transaction),
|
||||
|
||||
grp: stop.New(),
|
||||
tokens: &tokenManager{},
|
||||
}
|
||||
}
|
||||
|
||||
// Connect connects to the given connection and starts any background threads necessary
|
||||
func (n *Node) Connect(conn UDPConn) error {
|
||||
n.conn = conn
|
||||
|
||||
n.tokens.Start(tokenSecretRotationInterval)
|
||||
|
||||
go func() {
|
||||
// stop tokens and close the connection when we're shutting down
|
||||
<-n.grp.Ch()
|
||||
n.tokens.Stop()
|
||||
n.connClosed = true
|
||||
err := n.conn.Close()
|
||||
if err != nil {
|
||||
log.Error("error closing node connection on shutdown - ", err)
|
||||
}
|
||||
}()
|
||||
|
||||
packets := make(chan packet)
|
||||
|
||||
n.grp.Add(1)
|
||||
go func() {
|
||||
defer n.grp.Done()
|
||||
|
||||
buf := make([]byte, udpMaxMessageLength)
|
||||
|
||||
for {
|
||||
bytesRead, raddr, err := n.conn.ReadFromUDP(buf)
|
||||
if err != nil {
|
||||
if n.connClosed {
|
||||
return
|
||||
}
|
||||
log.Errorf("udp read error: %v", err)
|
||||
continue
|
||||
} else if raddr == nil {
|
||||
log.Errorf("udp read with no raddr")
|
||||
continue
|
||||
}
|
||||
|
||||
data := make([]byte, bytesRead)
|
||||
copy(data, buf[:bytesRead]) // slices use the same underlying array, so we need a new one for each packet
|
||||
|
||||
select { // needs select here because packet consumer can quit and the packets channel gets filled up and blocks
|
||||
case packets <- packet{data: data, raddr: raddr}:
|
||||
case <-n.grp.Ch():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
n.grp.Add(1)
|
||||
go func() {
|
||||
defer n.grp.Done()
|
||||
|
||||
var pkt packet
|
||||
|
||||
for {
|
||||
select {
|
||||
case pkt = <-packets:
|
||||
n.handlePacket(pkt)
|
||||
case <-n.grp.Ch():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// TODO: turn this back on when you're sure it works right
|
||||
n.grp.Add(1)
|
||||
go func() {
|
||||
defer n.grp.Done()
|
||||
n.startRoutingTableGrooming()
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Shutdown shuts down the node
|
||||
func (n *Node) Shutdown() {
|
||||
log.Debugf("[%s] node shutting down", n.id.HexShort())
|
||||
n.grp.StopAndWait()
|
||||
log.Debugf("[%s] node stopped", n.id.HexShort())
|
||||
}
|
||||
|
||||
// handlePacket handles packets received from udp.
|
||||
func (n *Node) handlePacket(pkt packet) {
|
||||
//log.Debugf("[%s] Received message from %s (%d bytes) %s", n.id.HexShort(), pkt.raddr.String(), len(pkt.data), hex.EncodeToString(pkt.data))
|
||||
|
||||
if !util.InSlice(string(pkt.data[0:5]), []string{"d1:0i", "di0ei"}) {
|
||||
log.Errorf("[%s] data is not a well-formatted dict: (%d bytes) %s", n.id.HexShort(), len(pkt.data), hex.EncodeToString(pkt.data))
|
||||
return
|
||||
}
|
||||
|
||||
// the following is a bit of a hack, but it lets us avoid decoding every message twice
|
||||
// it depends on the data being a dict with 0 as the first key (so it starts with "d1:0i") and the message type as the first value
|
||||
// TODO: test this more thoroughly
|
||||
|
||||
switch pkt.data[5] {
|
||||
case '0' + requestType:
|
||||
request := Request{}
|
||||
err := bencode.DecodeBytes(pkt.data, &request)
|
||||
if err != nil {
|
||||
log.Errorf("[%s] error decoding request from %s: %s: (%d bytes) %s", n.id.HexShort(), pkt.raddr.String(), err.Error(), len(pkt.data), hex.EncodeToString(pkt.data))
|
||||
return
|
||||
}
|
||||
log.Debugf("[%s] query %s: received request from %s: %s(%s)", n.id.HexShort(), request.ID.HexShort(), request.NodeID.HexShort(), request.Method, request.argsDebug())
|
||||
n.handleRequest(pkt.raddr, request)
|
||||
|
||||
case '0' + responseType:
|
||||
response := Response{}
|
||||
err := bencode.DecodeBytes(pkt.data, &response)
|
||||
if err != nil {
|
||||
log.Errorf("[%s] error decoding response from %s: %s: (%d bytes) %s", n.id.HexShort(), pkt.raddr.String(), err.Error(), len(pkt.data), hex.EncodeToString(pkt.data))
|
||||
return
|
||||
}
|
||||
log.Debugf("[%s] query %s: received response from %s: %s", n.id.HexShort(), response.ID.HexShort(), response.NodeID.HexShort(), response.argsDebug())
|
||||
n.handleResponse(pkt.raddr, response)
|
||||
|
||||
case '0' + errorType:
|
||||
e := Error{}
|
||||
err := bencode.DecodeBytes(pkt.data, &e)
|
||||
if err != nil {
|
||||
log.Errorf("[%s] error decoding error from %s: %s: (%d bytes) %s", n.id.HexShort(), pkt.raddr.String(), err.Error(), len(pkt.data), hex.EncodeToString(pkt.data))
|
||||
return
|
||||
}
|
||||
log.Debugf("[%s] query %s: received error from %s: %s", n.id.HexShort(), e.ID.HexShort(), e.NodeID.HexShort(), e.ExceptionType)
|
||||
n.handleError(pkt.raddr, e)
|
||||
|
||||
default:
|
||||
log.Errorf("[%s] invalid message type: %s", n.id.HexShort(), string(pkt.data[5]))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// handleRequest handles the requests received from udp.
|
||||
func (n *Node) handleRequest(addr *net.UDPAddr, request Request) {
|
||||
if request.NodeID.Equals(n.id) {
|
||||
log.Warn("ignoring self-request")
|
||||
return
|
||||
}
|
||||
|
||||
// if a handler is overridden, call it instead
|
||||
if n.requestHandler != nil {
|
||||
n.requestHandler(addr, request)
|
||||
return
|
||||
}
|
||||
|
||||
switch request.Method {
|
||||
default:
|
||||
//n.sendMessage(addr, Error{ID: request.ID, NodeID: n.id, ExceptionType: "invalid-request-method"})
|
||||
log.Errorln("invalid request method")
|
||||
return
|
||||
case pingMethod:
|
||||
err := n.sendMessage(addr, Response{ID: request.ID, NodeID: n.id, Data: pingSuccessResponse})
|
||||
if err != nil {
|
||||
log.Error("error sending 'pingmethod' response message - ", err)
|
||||
}
|
||||
case storeMethod:
|
||||
// TODO: we should be sending the IP in the request, not just using the sender's IP
|
||||
// TODO: should we be using StoreArgs.NodeID or StoreArgs.Value.LbryID ???
|
||||
if n.tokens.Verify(request.StoreArgs.Value.Token, request.NodeID, addr) {
|
||||
n.Store(request.StoreArgs.BlobHash, Contact{ID: request.StoreArgs.NodeID, IP: addr.IP, Port: addr.Port, PeerPort: request.StoreArgs.Value.Port})
|
||||
|
||||
err := n.sendMessage(addr, Response{ID: request.ID, NodeID: n.id, Data: storeSuccessResponse})
|
||||
if err != nil {
|
||||
log.Error("error sending 'storemethod' response message - ", err)
|
||||
}
|
||||
} else {
|
||||
err := n.sendMessage(addr, Error{ID: request.ID, NodeID: n.id, ExceptionType: "invalid-token"})
|
||||
if err != nil {
|
||||
log.Error("error sending 'storemethod'response message for invalid-token - ", err)
|
||||
}
|
||||
}
|
||||
case findNodeMethod:
|
||||
if request.Arg == nil {
|
||||
log.Errorln("request is missing arg")
|
||||
return
|
||||
}
|
||||
err := n.sendMessage(addr, Response{
|
||||
ID: request.ID,
|
||||
NodeID: n.id,
|
||||
Contacts: n.rt.GetClosest(*request.Arg, bucketSize),
|
||||
})
|
||||
if err != nil {
|
||||
log.Error("error sending 'findnodemethod' response message - ", err)
|
||||
}
|
||||
|
||||
case findValueMethod:
|
||||
if request.Arg == nil {
|
||||
log.Errorln("request is missing arg")
|
||||
return
|
||||
}
|
||||
|
||||
res := Response{
|
||||
ID: request.ID,
|
||||
NodeID: n.id,
|
||||
Token: n.tokens.Get(request.NodeID, addr),
|
||||
}
|
||||
|
||||
if contacts := n.store.Get(*request.Arg); len(contacts) > 0 {
|
||||
res.FindValueKey = request.Arg.RawString()
|
||||
res.Contacts = contacts
|
||||
} else {
|
||||
res.Contacts = n.rt.GetClosest(*request.Arg, bucketSize)
|
||||
}
|
||||
|
||||
err := n.sendMessage(addr, res)
|
||||
if err != nil {
|
||||
log.Error("error sending 'findvaluemethod' response message - ", err)
|
||||
}
|
||||
}
|
||||
|
||||
// nodes that send us requests should not be inserted, only refreshed.
|
||||
// the routing table must only contain "good" nodes, which are nodes that reply to our requests
|
||||
// if a node is already good (aka in the table), its fine to refresh it
|
||||
// http://www.bittorrent.org/beps/bep_0005.html#routing-table
|
||||
n.rt.Fresh(Contact{ID: request.NodeID, IP: addr.IP, Port: addr.Port})
|
||||
}
|
||||
|
||||
// handleResponse handles responses received from udp.
|
||||
func (n *Node) handleResponse(addr *net.UDPAddr, response Response) {
|
||||
tx := n.txFind(response.ID, Contact{ID: response.NodeID, IP: addr.IP, Port: addr.Port})
|
||||
if tx != nil {
|
||||
select {
|
||||
case tx.res <- response:
|
||||
default:
|
||||
//log.Errorf("[%s] query %s: response received, but tx has no listener or multiple responses to the same tx", n.id.HexShort(), response.ID.HexShort())
|
||||
}
|
||||
}
|
||||
|
||||
n.rt.Update(Contact{ID: response.NodeID, IP: addr.IP, Port: addr.Port})
|
||||
}
|
||||
|
||||
// handleError handles errors received from udp.
|
||||
func (n *Node) handleError(addr *net.UDPAddr, e Error) {
|
||||
spew.Dump(e)
|
||||
n.rt.Fresh(Contact{ID: e.NodeID, IP: addr.IP, Port: addr.Port})
|
||||
}
|
||||
|
||||
// send sends data to a udp address
|
||||
func (n *Node) sendMessage(addr *net.UDPAddr, data Message) error {
|
||||
encoded, err := bencode.EncodeBytes(data)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
if req, ok := data.(Request); ok {
|
||||
log.Debugf("[%s] query %s: sending request to %s (%d bytes) %s(%s)",
|
||||
n.id.HexShort(), req.ID.HexShort(), addr.String(), len(encoded), req.Method, req.argsDebug())
|
||||
} else if res, ok := data.(Response); ok {
|
||||
log.Debugf("[%s] query %s: sending response to %s (%d bytes) %s",
|
||||
n.id.HexShort(), res.ID.HexShort(), addr.String(), len(encoded), res.argsDebug())
|
||||
} else {
|
||||
log.Debugf("[%s] (%d bytes) %s", n.id.HexShort(), len(encoded), spew.Sdump(data))
|
||||
}
|
||||
|
||||
err = n.conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
|
||||
if err != nil {
|
||||
if n.connClosed {
|
||||
return nil
|
||||
}
|
||||
log.Error("error setting write deadline - ", err)
|
||||
}
|
||||
|
||||
_, err = n.conn.WriteToUDP(encoded, addr)
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
// transaction represents a single query to the dht. it stores the queried contact, the request, and the response channel
|
||||
type transaction struct {
|
||||
contact Contact
|
||||
req Request
|
||||
res chan Response
|
||||
skipIDCheck bool
|
||||
}
|
||||
|
||||
// insert adds a transaction to the manager.
|
||||
func (n *Node) txInsert(tx *transaction) {
|
||||
n.txLock.Lock()
|
||||
defer n.txLock.Unlock()
|
||||
n.transactions[tx.req.ID] = tx
|
||||
}
|
||||
|
||||
// delete removes a transaction from the manager.
|
||||
func (n *Node) txDelete(id messageID) {
|
||||
n.txLock.Lock()
|
||||
defer n.txLock.Unlock()
|
||||
delete(n.transactions, id)
|
||||
}
|
||||
|
||||
// Find finds a transaction for the given id and contact
|
||||
func (n *Node) txFind(id messageID, c Contact) *transaction {
|
||||
n.txLock.RLock()
|
||||
defer n.txLock.RUnlock()
|
||||
|
||||
t, ok := n.transactions[id]
|
||||
if !ok || !t.contact.Equals(c, !t.skipIDCheck) {
|
||||
return nil
|
||||
}
|
||||
|
||||
return t
|
||||
}
|
||||
|
||||
// SendOptions controls the behavior of send calls
|
||||
type SendOptions struct {
|
||||
skipIDCheck bool
|
||||
}
|
||||
|
||||
// SendAsync sends a transaction and returns a channel that will eventually contain the transaction response
|
||||
// The response channel is closed when the transaction is completed or times out.
|
||||
func (n *Node) SendAsync(contact Contact, req Request, options ...SendOptions) <-chan *Response {
|
||||
ch := make(chan *Response, 1)
|
||||
|
||||
if contact.ID.Equals(n.id) {
|
||||
log.Error("sending query to self")
|
||||
close(ch)
|
||||
return ch
|
||||
}
|
||||
|
||||
go func() {
|
||||
defer close(ch)
|
||||
|
||||
req.ID = newMessageID()
|
||||
req.NodeID = n.id
|
||||
tx := &transaction{
|
||||
contact: contact,
|
||||
req: req,
|
||||
res: make(chan Response),
|
||||
}
|
||||
|
||||
if len(options) > 0 && options[0].skipIDCheck {
|
||||
tx.skipIDCheck = true
|
||||
}
|
||||
|
||||
n.txInsert(tx)
|
||||
defer n.txDelete(tx.req.ID)
|
||||
|
||||
for i := 0; i < udpRetry; i++ {
|
||||
err := n.sendMessage(contact.Addr(), tx.req)
|
||||
if err != nil {
|
||||
if !strings.Contains(err.Error(), "use of closed network connection") { // this only happens on localhost. real UDP has no connections
|
||||
log.Error("send error: ", err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
select {
|
||||
case res := <-tx.res:
|
||||
ch <- &res
|
||||
return
|
||||
case <-n.grp.Ch():
|
||||
return
|
||||
case <-time.After(udpTimeout):
|
||||
}
|
||||
}
|
||||
|
||||
// notify routing table about a failure to respond
|
||||
n.rt.Fail(tx.contact)
|
||||
}()
|
||||
|
||||
return ch
|
||||
}
|
||||
|
||||
// Send sends a transaction and blocks until the response is available. It returns a response, or nil
|
||||
// if the transaction timed out.
|
||||
func (n *Node) Send(contact Contact, req Request, options ...SendOptions) *Response {
|
||||
return <-n.SendAsync(contact, req, options...)
|
||||
}
|
||||
|
||||
// CountActiveTransactions returns the number of transactions in the manager
|
||||
func (n *Node) CountActiveTransactions() int {
|
||||
n.txLock.Lock()
|
||||
defer n.txLock.Unlock()
|
||||
return len(n.transactions)
|
||||
}
|
||||
|
||||
func (n *Node) startRoutingTableGrooming() {
|
||||
refreshTicker := time.NewTicker(tRefresh / 5) // how often to check for buckets that need to be refreshed
|
||||
for {
|
||||
select {
|
||||
case <-refreshTicker.C:
|
||||
RoutingTableRefresh(n, tRefresh, n.grp.Child())
|
||||
case <-n.grp.Ch():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Store stores a node contact in the node's contact store.
|
||||
func (n *Node) Store(hash bits.Bitmap, c Contact) {
|
||||
n.store.Upsert(hash, c)
|
||||
}
|
||||
|
||||
//AddKnownNode adds a known-good node to the routing table
|
||||
func (n *Node) AddKnownNode(c Contact) {
|
||||
n.rt.Update(c)
|
||||
}
|
||||
@@ -1,338 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/crypto"
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/lbry.go/stop"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/uber-go/atomic"
|
||||
)
|
||||
|
||||
// TODO: iterativeFindValue may be stopping early. if it gets a response with one peer, it should keep going because other nodes may know about more peers that have that blob
|
||||
// TODO: or, it should try a tcp handshake with peers as it finds them, to make sure they are still online and have the blob
|
||||
|
||||
var cfLog *logrus.Logger
|
||||
|
||||
func init() {
|
||||
cfLog = logrus.StandardLogger()
|
||||
}
|
||||
|
||||
func NodeFinderUseLogger(l *logrus.Logger) {
|
||||
cfLog = l
|
||||
}
|
||||
|
||||
type contactFinder struct {
|
||||
findValue bool // true if we're using findValue
|
||||
target bits.Bitmap
|
||||
node *Node
|
||||
|
||||
grp *stop.Group
|
||||
|
||||
findValueMutex *sync.Mutex
|
||||
findValueResult []Contact
|
||||
|
||||
activeContactsMutex *sync.Mutex
|
||||
activeContacts []Contact
|
||||
|
||||
shortlistMutex *sync.Mutex
|
||||
shortlist []Contact
|
||||
shortlistAdded map[bits.Bitmap]bool
|
||||
|
||||
closestContactMutex *sync.RWMutex
|
||||
closestContact *Contact
|
||||
notGettingCloser *atomic.Bool
|
||||
}
|
||||
|
||||
func FindContacts(node *Node, target bits.Bitmap, findValue bool, parentGrp *stop.Group) ([]Contact, bool, error) {
|
||||
cf := &contactFinder{
|
||||
node: node,
|
||||
target: target,
|
||||
findValue: findValue,
|
||||
findValueMutex: &sync.Mutex{},
|
||||
activeContactsMutex: &sync.Mutex{},
|
||||
shortlistMutex: &sync.Mutex{},
|
||||
shortlistAdded: make(map[bits.Bitmap]bool),
|
||||
grp: stop.New(parentGrp),
|
||||
closestContactMutex: &sync.RWMutex{},
|
||||
notGettingCloser: atomic.NewBool(false),
|
||||
}
|
||||
|
||||
return cf.Find()
|
||||
}
|
||||
|
||||
func (cf *contactFinder) Stop() {
|
||||
cf.grp.StopAndWait()
|
||||
}
|
||||
|
||||
func (cf *contactFinder) Find() ([]Contact, bool, error) {
|
||||
if cf.findValue {
|
||||
cf.debug("starting iterativeFindValue")
|
||||
} else {
|
||||
cf.debug("starting iterativeFindNode")
|
||||
}
|
||||
|
||||
cf.appendNewToShortlist(cf.node.rt.GetClosest(cf.target, alpha))
|
||||
if len(cf.shortlist) == 0 {
|
||||
return nil, false, errors.Err("[%s] find %s: no contacts in routing table", cf.node.id.HexShort(), cf.target.HexShort())
|
||||
}
|
||||
|
||||
go cf.cycle(false)
|
||||
timeout := 5 * time.Second
|
||||
CycleLoop:
|
||||
for {
|
||||
select {
|
||||
case <-time.After(timeout):
|
||||
go cf.cycle(false)
|
||||
case <-cf.grp.Ch():
|
||||
break CycleLoop
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: what to do if we have less than K active contacts, shortlist is empty, but we have other contacts in our routing table whom we have not contacted. prolly contact them
|
||||
|
||||
var contacts []Contact
|
||||
var found bool
|
||||
if cf.findValue && len(cf.findValueResult) > 0 {
|
||||
contacts = cf.findValueResult
|
||||
found = true
|
||||
} else {
|
||||
contacts = cf.activeContacts
|
||||
if len(contacts) > bucketSize {
|
||||
contacts = contacts[:bucketSize]
|
||||
}
|
||||
}
|
||||
|
||||
cf.Stop()
|
||||
return contacts, found, nil
|
||||
}
|
||||
|
||||
// cycle does a single cycle of sending alpha probes and checking results against closestNode
|
||||
func (cf *contactFinder) cycle(bigCycle bool) {
|
||||
cycleID := crypto.RandString(6)
|
||||
if bigCycle {
|
||||
cf.debug("LAUNCHING CYCLE %s, AND ITS A BIG CYCLE", cycleID)
|
||||
} else {
|
||||
cf.debug("LAUNCHING CYCLE %s", cycleID)
|
||||
}
|
||||
defer cf.debug("CYCLE %s DONE", cycleID)
|
||||
|
||||
cf.closestContactMutex.RLock()
|
||||
closestContact := cf.closestContact
|
||||
cf.closestContactMutex.RUnlock()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
ch := make(chan *Contact)
|
||||
|
||||
limit := alpha
|
||||
if bigCycle {
|
||||
limit = bucketSize
|
||||
}
|
||||
|
||||
for i := 0; i < limit; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
ch <- cf.probe(cycleID)
|
||||
}()
|
||||
}
|
||||
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(ch)
|
||||
}()
|
||||
|
||||
foundCloser := false
|
||||
for {
|
||||
c, more := <-ch
|
||||
if !more {
|
||||
break
|
||||
}
|
||||
if c != nil && (closestContact == nil || cf.target.Closer(c.ID, closestContact.ID)) {
|
||||
if closestContact != nil {
|
||||
cf.debug("|%s| best contact improved: %s -> %s", cycleID, closestContact.ID.HexShort(), c.ID.HexShort())
|
||||
} else {
|
||||
cf.debug("|%s| best contact starting at %s", cycleID, c.ID.HexShort())
|
||||
}
|
||||
foundCloser = true
|
||||
closestContact = c
|
||||
}
|
||||
}
|
||||
|
||||
if cf.isSearchFinished() {
|
||||
cf.grp.Stop()
|
||||
return
|
||||
}
|
||||
|
||||
if foundCloser {
|
||||
cf.closestContactMutex.Lock()
|
||||
// have to check again after locking in case other probes found a closer one in the meantime
|
||||
if cf.closestContact == nil || cf.target.Closer(closestContact.ID, cf.closestContact.ID) {
|
||||
cf.closestContact = closestContact
|
||||
}
|
||||
cf.closestContactMutex.Unlock()
|
||||
go cf.cycle(false)
|
||||
} else if !bigCycle {
|
||||
cf.debug("|%s| no improvement, running big cycle", cycleID)
|
||||
go cf.cycle(true)
|
||||
} else {
|
||||
// big cycle ran and there was no improvement, so we're done
|
||||
cf.debug("|%s| big cycle ran, still no improvement", cycleID)
|
||||
cf.notGettingCloser.Store(true)
|
||||
}
|
||||
}
|
||||
|
||||
// probe sends a single probe, updates the lists, and returns the closest contact it found
|
||||
func (cf *contactFinder) probe(cycleID string) *Contact {
|
||||
maybeContact := cf.popFromShortlist()
|
||||
if maybeContact == nil {
|
||||
cf.debug("|%s| no contacts in shortlist, returning", cycleID)
|
||||
return nil
|
||||
}
|
||||
|
||||
c := *maybeContact
|
||||
|
||||
if c.ID.Equals(cf.node.id) {
|
||||
return nil
|
||||
}
|
||||
|
||||
cf.debug("|%s| probe %s: launching", cycleID, c.ID.HexShort())
|
||||
|
||||
req := Request{Arg: &cf.target}
|
||||
if cf.findValue {
|
||||
req.Method = findValueMethod
|
||||
} else {
|
||||
req.Method = findNodeMethod
|
||||
}
|
||||
|
||||
var res *Response
|
||||
resCh := cf.node.SendAsync(c, req)
|
||||
select {
|
||||
case res = <-resCh:
|
||||
case <-cf.grp.Ch():
|
||||
cf.debug("|%s| probe %s: canceled", cycleID, c.ID.HexShort())
|
||||
return nil
|
||||
}
|
||||
|
||||
if res == nil {
|
||||
cf.debug("|%s| probe %s: req canceled or timed out", cycleID, c.ID.HexShort())
|
||||
return nil
|
||||
}
|
||||
|
||||
if cf.findValue && res.FindValueKey != "" {
|
||||
cf.debug("|%s| probe %s: got value", cycleID, c.ID.HexShort())
|
||||
cf.findValueMutex.Lock()
|
||||
cf.findValueResult = res.Contacts
|
||||
cf.findValueMutex.Unlock()
|
||||
cf.grp.Stop()
|
||||
return nil
|
||||
}
|
||||
|
||||
cf.debug("|%s| probe %s: got %s", cycleID, c.ID.HexShort(), res.argsDebug())
|
||||
cf.insertIntoActiveList(c)
|
||||
cf.appendNewToShortlist(res.Contacts)
|
||||
|
||||
cf.activeContactsMutex.Lock()
|
||||
contacts := cf.activeContacts
|
||||
if len(contacts) > bucketSize {
|
||||
contacts = contacts[:bucketSize]
|
||||
}
|
||||
contactsStr := ""
|
||||
for _, c := range contacts {
|
||||
contactsStr += c.ID.HexShort() + ", "
|
||||
}
|
||||
cf.activeContactsMutex.Unlock()
|
||||
|
||||
return cf.closest(res.Contacts...)
|
||||
}
|
||||
|
||||
// appendNewToShortlist appends any new contacts to the shortlist and sorts it by distance
|
||||
// contacts that have already been added to the shortlist in the past are ignored
|
||||
func (cf *contactFinder) appendNewToShortlist(contacts []Contact) {
|
||||
cf.shortlistMutex.Lock()
|
||||
defer cf.shortlistMutex.Unlock()
|
||||
|
||||
for _, c := range contacts {
|
||||
if _, ok := cf.shortlistAdded[c.ID]; !ok {
|
||||
cf.shortlist = append(cf.shortlist, c)
|
||||
cf.shortlistAdded[c.ID] = true
|
||||
}
|
||||
}
|
||||
|
||||
sortByDistance(cf.shortlist, cf.target)
|
||||
}
|
||||
|
||||
// popFromShortlist pops the first contact off the shortlist and returns it
|
||||
func (cf *contactFinder) popFromShortlist() *Contact {
|
||||
cf.shortlistMutex.Lock()
|
||||
defer cf.shortlistMutex.Unlock()
|
||||
|
||||
if len(cf.shortlist) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
first := cf.shortlist[0]
|
||||
cf.shortlist = cf.shortlist[1:]
|
||||
return &first
|
||||
}
|
||||
|
||||
// insertIntoActiveList inserts the contact into appropriate place in the list of active contacts (sorted by distance)
|
||||
func (cf *contactFinder) insertIntoActiveList(contact Contact) {
|
||||
cf.activeContactsMutex.Lock()
|
||||
defer cf.activeContactsMutex.Unlock()
|
||||
|
||||
inserted := false
|
||||
for i, n := range cf.activeContacts {
|
||||
if cf.target.Closer(contact.ID, n.ID) {
|
||||
cf.activeContacts = append(cf.activeContacts[:i], append([]Contact{contact}, cf.activeContacts[i:]...)...)
|
||||
inserted = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !inserted {
|
||||
cf.activeContacts = append(cf.activeContacts, contact)
|
||||
}
|
||||
}
|
||||
|
||||
// isSearchFinished returns true if the search is done and should be stopped
|
||||
func (cf *contactFinder) isSearchFinished() bool {
|
||||
if cf.findValue && len(cf.findValueResult) > 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
select {
|
||||
case <-cf.grp.Ch():
|
||||
return true
|
||||
default:
|
||||
}
|
||||
|
||||
if cf.notGettingCloser.Load() {
|
||||
return true
|
||||
}
|
||||
|
||||
cf.activeContactsMutex.Lock()
|
||||
defer cf.activeContactsMutex.Unlock()
|
||||
return len(cf.activeContacts) >= bucketSize
|
||||
}
|
||||
|
||||
func (cf *contactFinder) debug(format string, args ...interface{}) {
|
||||
args = append([]interface{}{cf.node.id.HexShort()}, append([]interface{}{cf.target.HexShort()}, args...)...)
|
||||
cfLog.Debugf("[%s] find %s: "+format, args...)
|
||||
}
|
||||
|
||||
func (cf *contactFinder) closest(contacts ...Contact) *Contact {
|
||||
if len(contacts) == 0 {
|
||||
return nil
|
||||
}
|
||||
closest := contacts[0]
|
||||
for _, c := range contacts {
|
||||
if cf.target.Closer(c.ID, closest.ID) {
|
||||
closest = c
|
||||
}
|
||||
}
|
||||
return &closest
|
||||
}
|
||||
@@ -1,422 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"net"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
"github.com/lyoshenka/bencode"
|
||||
)
|
||||
|
||||
func TestPing(t *testing.T) {
|
||||
dhtNodeID := bits.Rand()
|
||||
testNodeID := bits.Rand()
|
||||
|
||||
conn := newTestUDPConn("127.0.0.1:21217")
|
||||
|
||||
dht := New(&Config{Address: "127.0.0.1:21216", NodeID: dhtNodeID.Hex()})
|
||||
|
||||
err := dht.connect(conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer dht.Shutdown()
|
||||
|
||||
messageID := newMessageID()
|
||||
|
||||
data, err := bencode.EncodeBytes(map[string]interface{}{
|
||||
headerTypeField: requestType,
|
||||
headerMessageIDField: messageID,
|
||||
headerNodeIDField: testNodeID.RawString(),
|
||||
headerPayloadField: "ping",
|
||||
headerArgsField: []string{},
|
||||
})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
conn.toRead <- testUDPPacket{addr: conn.addr, data: data}
|
||||
timer := time.NewTimer(3 * time.Second)
|
||||
|
||||
select {
|
||||
case <-timer.C:
|
||||
t.Error("timeout")
|
||||
case resp := <-conn.writes:
|
||||
var response map[string]interface{}
|
||||
err := bencode.DecodeBytes(resp.data, &response)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(response) != 4 {
|
||||
t.Errorf("expected 4 response fields, got %d", len(response))
|
||||
}
|
||||
|
||||
_, ok := response[headerTypeField]
|
||||
if !ok {
|
||||
t.Error("missing type field")
|
||||
} else {
|
||||
rType, ok := response[headerTypeField].(int64)
|
||||
if !ok {
|
||||
t.Error("type is not an integer")
|
||||
} else if rType != responseType {
|
||||
t.Error("unexpected response type")
|
||||
}
|
||||
}
|
||||
|
||||
_, ok = response[headerMessageIDField]
|
||||
if !ok {
|
||||
t.Error("missing message id field")
|
||||
} else {
|
||||
rMessageID, ok := response[headerMessageIDField].(string)
|
||||
if !ok {
|
||||
t.Error("message ID is not a string")
|
||||
} else if rMessageID != string(messageID[:]) {
|
||||
t.Error("unexpected message ID")
|
||||
}
|
||||
}
|
||||
|
||||
_, ok = response[headerNodeIDField]
|
||||
if !ok {
|
||||
t.Error("missing node id field")
|
||||
} else {
|
||||
rNodeID, ok := response[headerNodeIDField].(string)
|
||||
if !ok {
|
||||
t.Error("node ID is not a string")
|
||||
} else if rNodeID != dhtNodeID.RawString() {
|
||||
t.Error("unexpected node ID")
|
||||
}
|
||||
}
|
||||
|
||||
_, ok = response[headerPayloadField]
|
||||
if !ok {
|
||||
t.Error("missing payload field")
|
||||
} else {
|
||||
rNodeID, ok := response[headerPayloadField].(string)
|
||||
if !ok {
|
||||
t.Error("payload is not a string")
|
||||
} else if rNodeID != pingSuccessResponse {
|
||||
t.Error("did not pong")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStore(t *testing.T) {
|
||||
dhtNodeID := bits.Rand()
|
||||
testNodeID := bits.Rand()
|
||||
|
||||
conn := newTestUDPConn("127.0.0.1:21217")
|
||||
|
||||
dht := New(&Config{Address: "127.0.0.1:21216", NodeID: dhtNodeID.Hex()})
|
||||
|
||||
err := dht.connect(conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer dht.Shutdown()
|
||||
|
||||
messageID := newMessageID()
|
||||
blobHashToStore := bits.Rand()
|
||||
|
||||
storeRequest := Request{
|
||||
ID: messageID,
|
||||
NodeID: testNodeID,
|
||||
Method: storeMethod,
|
||||
StoreArgs: &storeArgs{
|
||||
BlobHash: blobHashToStore,
|
||||
Value: storeArgsValue{
|
||||
Token: dht.node.tokens.Get(testNodeID, conn.addr),
|
||||
LbryID: testNodeID,
|
||||
Port: 9999,
|
||||
},
|
||||
NodeID: testNodeID,
|
||||
},
|
||||
}
|
||||
|
||||
_ = "64 " + // start message
|
||||
"313A30 693065" + // type: 0
|
||||
"313A31 3230 3A 6EB490B5788B63F0F7E6D92352024D0CBDEC2D3A" + // message id
|
||||
"313A32 3438 3A 7CE1B831DEC8689E44F80F547D2DEA171F6A625E1A4FF6C6165E645F953103DABEB068A622203F859C6C64658FD3AA3B" + // node id
|
||||
"313A33 35 3A 73746F7265" + // method
|
||||
"313A34 6C" + // start args list
|
||||
"3438 3A 3214D6C2F77FCB5E8D5FC07EDAFBA614F031CE8B2EAB49F924F8143F6DFBADE048D918710072FB98AB1B52B58F4E1468" + // block hash
|
||||
"64" + // start value dict
|
||||
"363A6C6272796964 3438 3A 7CE1B831DEC8689E44F80F547D2DEA171F6A625E1A4FF6C6165E645F953103DABEB068A622203F859C6C64658FD3AA3B" + // lbry id
|
||||
"343A706F7274 69 33333333 65" + // port
|
||||
"353A746F6B656E 3438 3A 17C2D8E1E48EF21567FE4AD5C8ED944B798D3B65AB58D0C9122AD6587D1B5FED472EA2CB12284CEFA1C21EFF302322BD" + // token
|
||||
"65" + // end value dict
|
||||
"3438 3A 7CE1B831DEC8689E44F80F547D2DEA171F6A625E1A4FF6C6165E645F953103DABEB068A622203F859C6C64658FD3AA3B" + // node id
|
||||
"693065" + // self store (integer)
|
||||
"65" + // end args list
|
||||
"65" // end message
|
||||
|
||||
data, err := bencode.EncodeBytes(storeRequest)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
conn.toRead <- testUDPPacket{addr: conn.addr, data: data}
|
||||
timer := time.NewTimer(3 * time.Second)
|
||||
|
||||
var response map[string]interface{}
|
||||
select {
|
||||
case <-timer.C:
|
||||
t.Fatal("timeout")
|
||||
case resp := <-conn.writes:
|
||||
err := bencode.DecodeBytes(resp.data, &response)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
verifyResponse(t, response, messageID, dhtNodeID.RawString())
|
||||
|
||||
_, ok := response[headerPayloadField]
|
||||
if !ok {
|
||||
t.Error("missing payload field")
|
||||
} else {
|
||||
rNodeID, ok := response[headerPayloadField].(string)
|
||||
if !ok {
|
||||
t.Error("payload is not a string")
|
||||
} else if rNodeID != storeSuccessResponse {
|
||||
t.Error("did not return OK")
|
||||
}
|
||||
}
|
||||
|
||||
if dht.node.store.CountStoredHashes() != 1 {
|
||||
t.Error("dht store has wrong number of items")
|
||||
}
|
||||
|
||||
items := dht.node.store.Get(blobHashToStore)
|
||||
if len(items) != 1 {
|
||||
t.Error("list created in store, but nothing in list")
|
||||
}
|
||||
if !items[0].ID.Equals(testNodeID) {
|
||||
t.Error("wrong value stored")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindNode(t *testing.T) {
|
||||
dhtNodeID := bits.Rand()
|
||||
testNodeID := bits.Rand()
|
||||
|
||||
conn := newTestUDPConn("127.0.0.1:21217")
|
||||
|
||||
dht := New(&Config{Address: "127.0.0.1:21216", NodeID: dhtNodeID.Hex()})
|
||||
|
||||
err := dht.connect(conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer dht.Shutdown()
|
||||
|
||||
nodesToInsert := 3
|
||||
var nodes []Contact
|
||||
for i := 0; i < nodesToInsert; i++ {
|
||||
n := Contact{ID: bits.Rand(), IP: net.ParseIP("127.0.0.1"), Port: 10000 + i}
|
||||
nodes = append(nodes, n)
|
||||
dht.node.rt.Update(n)
|
||||
}
|
||||
|
||||
messageID := newMessageID()
|
||||
blobHashToFind := bits.Rand()
|
||||
|
||||
request := Request{
|
||||
ID: messageID,
|
||||
NodeID: testNodeID,
|
||||
Method: findNodeMethod,
|
||||
Arg: &blobHashToFind,
|
||||
}
|
||||
|
||||
data, err := bencode.EncodeBytes(request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
conn.toRead <- testUDPPacket{addr: conn.addr, data: data}
|
||||
timer := time.NewTimer(3 * time.Second)
|
||||
|
||||
var response map[string]interface{}
|
||||
select {
|
||||
case <-timer.C:
|
||||
t.Fatal("timeout")
|
||||
case resp := <-conn.writes:
|
||||
err := bencode.DecodeBytes(resp.data, &response)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
verifyResponse(t, response, messageID, dhtNodeID.RawString())
|
||||
|
||||
_, ok := response[headerPayloadField]
|
||||
if !ok {
|
||||
t.Fatal("missing payload field")
|
||||
}
|
||||
|
||||
contacts, ok := response[headerPayloadField].([]interface{})
|
||||
if !ok {
|
||||
t.Fatal("payload is not a list")
|
||||
}
|
||||
|
||||
verifyContacts(t, contacts, nodes)
|
||||
}
|
||||
|
||||
func TestFindValueExisting(t *testing.T) {
|
||||
dhtNodeID := bits.Rand()
|
||||
testNodeID := bits.Rand()
|
||||
|
||||
conn := newTestUDPConn("127.0.0.1:21217")
|
||||
|
||||
dht := New(&Config{Address: "127.0.0.1:21216", NodeID: dhtNodeID.Hex()})
|
||||
|
||||
err := dht.connect(conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer dht.Shutdown()
|
||||
|
||||
nodesToInsert := 3
|
||||
for i := 0; i < nodesToInsert; i++ {
|
||||
n := Contact{ID: bits.Rand(), IP: net.ParseIP("127.0.0.1"), Port: 10000 + i}
|
||||
dht.node.rt.Update(n)
|
||||
}
|
||||
|
||||
//data, _ := hex.DecodeString("64313a30693065313a3132303a7de8e57d34e316abbb5a8a8da50dcd1ad4c80e0f313a3234383a7ce1b831dec8689e44f80f547d2dea171f6a625e1a4ff6c6165e645f953103dabeb068a622203f859c6c64658fd3aa3b313a33393a66696e6456616c7565313a346c34383aa47624b8e7ee1e54df0c45e2eb858feb0b705bd2a78d8b739be31ba188f4bd6f56b371c51fecc5280d5fd26ba4168e966565")
|
||||
|
||||
messageID := newMessageID()
|
||||
valueToFind := bits.Rand()
|
||||
|
||||
nodeToFind := Contact{ID: bits.Rand(), IP: net.ParseIP("1.2.3.4"), PeerPort: 1286}
|
||||
dht.node.store.Upsert(valueToFind, nodeToFind)
|
||||
dht.node.store.Upsert(valueToFind, nodeToFind)
|
||||
dht.node.store.Upsert(valueToFind, nodeToFind)
|
||||
|
||||
request := Request{
|
||||
ID: messageID,
|
||||
NodeID: testNodeID,
|
||||
Method: findValueMethod,
|
||||
Arg: &valueToFind,
|
||||
}
|
||||
|
||||
data, err := bencode.EncodeBytes(request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
conn.toRead <- testUDPPacket{addr: conn.addr, data: data}
|
||||
timer := time.NewTimer(3 * time.Second)
|
||||
|
||||
var response map[string]interface{}
|
||||
select {
|
||||
case <-timer.C:
|
||||
t.Fatal("timeout")
|
||||
case resp := <-conn.writes:
|
||||
err := bencode.DecodeBytes(resp.data, &response)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
verifyResponse(t, response, messageID, dhtNodeID.RawString())
|
||||
|
||||
_, ok := response[headerPayloadField]
|
||||
if !ok {
|
||||
t.Fatal("missing payload field")
|
||||
}
|
||||
|
||||
payload, ok := response[headerPayloadField].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatal("payload is not a dictionary")
|
||||
}
|
||||
|
||||
compactContacts, ok := payload[valueToFind.RawString()]
|
||||
if !ok {
|
||||
t.Fatal("payload is missing key for search value")
|
||||
}
|
||||
|
||||
contacts, ok := compactContacts.([]interface{})
|
||||
if !ok {
|
||||
t.Fatal("search results are not a list")
|
||||
}
|
||||
|
||||
verifyCompactContacts(t, contacts, []Contact{nodeToFind})
|
||||
}
|
||||
|
||||
func TestFindValueFallbackToFindNode(t *testing.T) {
|
||||
dhtNodeID := bits.Rand()
|
||||
testNodeID := bits.Rand()
|
||||
|
||||
conn := newTestUDPConn("127.0.0.1:21217")
|
||||
|
||||
dht := New(&Config{Address: "127.0.0.1:21216", NodeID: dhtNodeID.Hex()})
|
||||
|
||||
err := dht.connect(conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer dht.Shutdown()
|
||||
|
||||
nodesToInsert := 3
|
||||
var nodes []Contact
|
||||
for i := 0; i < nodesToInsert; i++ {
|
||||
n := Contact{ID: bits.Rand(), IP: net.ParseIP("127.0.0.1"), Port: 10000 + i}
|
||||
nodes = append(nodes, n)
|
||||
dht.node.rt.Update(n)
|
||||
}
|
||||
|
||||
messageID := newMessageID()
|
||||
valueToFind := bits.Rand()
|
||||
|
||||
request := Request{
|
||||
ID: messageID,
|
||||
NodeID: testNodeID,
|
||||
Method: findValueMethod,
|
||||
Arg: &valueToFind,
|
||||
}
|
||||
|
||||
data, err := bencode.EncodeBytes(request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
conn.toRead <- testUDPPacket{addr: conn.addr, data: data}
|
||||
timer := time.NewTimer(3 * time.Second)
|
||||
|
||||
var response map[string]interface{}
|
||||
select {
|
||||
case <-timer.C:
|
||||
t.Fatal("timeout")
|
||||
case resp := <-conn.writes:
|
||||
err := bencode.DecodeBytes(resp.data, &response)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
verifyResponse(t, response, messageID, dhtNodeID.RawString())
|
||||
|
||||
_, ok := response[headerPayloadField]
|
||||
if !ok {
|
||||
t.Fatal("missing payload field")
|
||||
}
|
||||
|
||||
payload, ok := response[headerPayloadField].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatal("payload is not a dictionary")
|
||||
}
|
||||
|
||||
contactsList, ok := payload[contactsField]
|
||||
if !ok {
|
||||
t.Fatal("payload is missing 'contacts' key")
|
||||
}
|
||||
|
||||
contacts, ok := contactsList.([]interface{})
|
||||
if !ok {
|
||||
t.Fatal("'contacts' is not a list")
|
||||
}
|
||||
|
||||
verifyContacts(t, contacts, nodes)
|
||||
}
|
||||
@@ -1,463 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/lbry.go/stop"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
)
|
||||
|
||||
// TODO: if routing table is ever empty (aka the node is isolated), it should re-bootstrap
|
||||
|
||||
// TODO: use a tree with bucket splitting instead of a fixed bucket list. include jack's optimization (see link in commit mesg)
|
||||
// https://github.com/lbryio/lbry/pull/1211/commits/341b27b6d21ac027671d42458826d02735aaae41
|
||||
|
||||
// peer is a contact with extra information
|
||||
type peer struct {
|
||||
Contact Contact
|
||||
Distance bits.Bitmap
|
||||
LastActivity time.Time
|
||||
// LastReplied time.Time
|
||||
// LastRequested time.Time
|
||||
// LastFailure time.Time
|
||||
// SecondLastFailure time.Time
|
||||
NumFailures int
|
||||
|
||||
//<lastPublished>,
|
||||
//<originallyPublished>
|
||||
// <originalPublisherID>
|
||||
}
|
||||
|
||||
func (p *peer) Touch() {
|
||||
p.LastActivity = time.Now()
|
||||
p.NumFailures = 0
|
||||
}
|
||||
|
||||
// ActiveSince returns whether a peer has responded in the last `d` duration
|
||||
// this is used to check if the peer is "good", meaning that we believe the peer will respond to our requests
|
||||
func (p *peer) ActiveInLast(d time.Duration) bool {
|
||||
return time.Since(p.LastActivity) < d
|
||||
}
|
||||
|
||||
// IsBad returns whether a peer is "bad", meaning that it has failed to respond to multiple pings in a row
|
||||
func (p *peer) IsBad(maxFalures int) bool {
|
||||
return p.NumFailures >= maxFalures
|
||||
}
|
||||
|
||||
// Fail marks a peer as having failed to respond. It returns whether or not the peer should be removed from the routing table
|
||||
func (p *peer) Fail() {
|
||||
p.NumFailures++
|
||||
}
|
||||
|
||||
type bucket struct {
|
||||
lock *sync.RWMutex
|
||||
peers []peer
|
||||
lastUpdate time.Time
|
||||
Range bits.Range // capitalized because `range` is a keyword
|
||||
}
|
||||
|
||||
func newBucket(r bits.Range) *bucket {
|
||||
return &bucket{
|
||||
peers: make([]peer, 0, bucketSize),
|
||||
lock: &sync.RWMutex{},
|
||||
Range: r,
|
||||
}
|
||||
}
|
||||
|
||||
// Len returns the number of peers in the bucket
|
||||
func (b bucket) Len() int {
|
||||
b.lock.RLock()
|
||||
defer b.lock.RUnlock()
|
||||
return len(b.peers)
|
||||
}
|
||||
|
||||
func (b bucket) Has(c Contact) bool {
|
||||
b.lock.RLock()
|
||||
defer b.lock.RUnlock()
|
||||
for _, p := range b.peers {
|
||||
if p.Contact.Equals(c, true) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Contacts returns a slice of the bucket's contacts
|
||||
func (b bucket) Contacts() []Contact {
|
||||
b.lock.RLock()
|
||||
defer b.lock.RUnlock()
|
||||
contacts := make([]Contact, len(b.peers))
|
||||
for i := range b.peers {
|
||||
contacts[i] = b.peers[i].Contact
|
||||
}
|
||||
return contacts
|
||||
}
|
||||
|
||||
// UpdatePeer marks a contact as having been successfully contacted. if insertIfNew and the contact is does not exist yet, it is inserted
|
||||
func (b *bucket) UpdatePeer(p peer, insertIfNew bool) error {
|
||||
b.lock.Lock()
|
||||
defer b.lock.Unlock()
|
||||
|
||||
if !b.Range.Contains(p.Distance) {
|
||||
return errors.Err("this bucket range does not cover this peer")
|
||||
}
|
||||
|
||||
peerIndex := find(p.Contact.ID, b.peers)
|
||||
if peerIndex >= 0 {
|
||||
b.lastUpdate = time.Now()
|
||||
b.peers[peerIndex].Touch()
|
||||
moveToBack(b.peers, peerIndex)
|
||||
} else if insertIfNew {
|
||||
hasRoom := true
|
||||
|
||||
if len(b.peers) >= bucketSize {
|
||||
hasRoom = false
|
||||
for i := range b.peers {
|
||||
if b.peers[i].IsBad(maxPeerFails) {
|
||||
// TODO: Ping contact first. Only remove if it does not respond
|
||||
b.peers = append(b.peers[:i], b.peers[i+1:]...)
|
||||
hasRoom = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if hasRoom {
|
||||
b.lastUpdate = time.Now()
|
||||
p.Touch()
|
||||
b.peers = append(b.peers, p)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// FailContact marks a contact as having failed, and removes it if it failed too many times
|
||||
func (b *bucket) FailContact(id bits.Bitmap) {
|
||||
b.lock.Lock()
|
||||
defer b.lock.Unlock()
|
||||
i := find(id, b.peers)
|
||||
if i >= 0 {
|
||||
// BEP5 says not to remove the contact until the bucket is full and you try to insert
|
||||
b.peers[i].Fail()
|
||||
}
|
||||
}
|
||||
|
||||
// find returns the contact in the bucket, or nil if the bucket does not contain the contact
|
||||
func find(id bits.Bitmap, peers []peer) int {
|
||||
for i := range peers {
|
||||
if peers[i].Contact.ID.Equals(id) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// NeedsRefresh returns true if bucket has not been updated in the last `refreshInterval`, false otherwise
|
||||
func (b *bucket) NeedsRefresh(refreshInterval time.Duration) bool {
|
||||
b.lock.RLock()
|
||||
defer b.lock.RUnlock()
|
||||
return time.Since(b.lastUpdate) > refreshInterval
|
||||
}
|
||||
|
||||
func (b *bucket) Split() (*bucket, *bucket) {
|
||||
b.lock.Lock()
|
||||
defer b.lock.Unlock()
|
||||
|
||||
left := newBucket(b.Range.IntervalP(1, 2))
|
||||
right := newBucket(b.Range.IntervalP(2, 2))
|
||||
left.lastUpdate = b.lastUpdate
|
||||
right.lastUpdate = b.lastUpdate
|
||||
|
||||
for _, p := range b.peers {
|
||||
if left.Range.Contains(p.Distance) {
|
||||
left.peers = append(left.peers, p)
|
||||
} else {
|
||||
right.peers = append(right.peers, p)
|
||||
}
|
||||
}
|
||||
|
||||
if len(b.peers) > 1 {
|
||||
if len(left.peers) == 0 {
|
||||
left, right = right.Split()
|
||||
left.Range.Start = b.Range.Start
|
||||
} else if len(right.peers) == 0 {
|
||||
left, right = left.Split()
|
||||
right.Range.End = b.Range.End
|
||||
}
|
||||
}
|
||||
|
||||
return left, right
|
||||
}
|
||||
|
||||
type routingTable struct {
|
||||
id bits.Bitmap
|
||||
buckets []*bucket
|
||||
mu *sync.RWMutex // this mutex is write-locked only when CHANGING THE NUMBER OF BUCKETS in the table
|
||||
}
|
||||
|
||||
func newRoutingTable(id bits.Bitmap) *routingTable {
|
||||
rt := routingTable{
|
||||
id: id,
|
||||
mu: &sync.RWMutex{},
|
||||
}
|
||||
rt.reset()
|
||||
return &rt
|
||||
}
|
||||
|
||||
func (rt *routingTable) reset() {
|
||||
rt.mu.Lock()
|
||||
defer rt.mu.Unlock()
|
||||
rt.buckets = []*bucket{newBucket(bits.MaxRange())}
|
||||
}
|
||||
|
||||
func (rt *routingTable) BucketInfo() string {
|
||||
rt.mu.RLock()
|
||||
defer rt.mu.RUnlock()
|
||||
|
||||
var bucketInfo []string
|
||||
for i, b := range rt.buckets {
|
||||
if b.Len() > 0 {
|
||||
contacts := b.Contacts()
|
||||
s := make([]string, len(contacts))
|
||||
for j, c := range contacts {
|
||||
s[j] = c.ID.HexShort()
|
||||
}
|
||||
bucketInfo = append(bucketInfo, fmt.Sprintf("bucket %d: (%d) %s", i, len(contacts), strings.Join(s, ", ")))
|
||||
}
|
||||
}
|
||||
if len(bucketInfo) == 0 {
|
||||
return "buckets are empty"
|
||||
}
|
||||
return strings.Join(bucketInfo, "\n")
|
||||
}
|
||||
|
||||
// Update inserts or refreshes a contact
|
||||
func (rt *routingTable) Update(c Contact) {
|
||||
rt.mu.Lock() // write lock, because updates may cause bucket splits
|
||||
defer rt.mu.Unlock()
|
||||
|
||||
b := rt.bucketFor(c.ID)
|
||||
|
||||
if rt.shouldSplit(b, c) {
|
||||
left, right := b.Split()
|
||||
|
||||
for i := range rt.buckets {
|
||||
if rt.buckets[i].Range.Start.Equals(left.Range.Start) {
|
||||
rt.buckets = append(rt.buckets[:i], append([]*bucket{left, right}, rt.buckets[i+1:]...)...)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if left.Range.Contains(c.ID) {
|
||||
b = left
|
||||
} else {
|
||||
b = right
|
||||
}
|
||||
}
|
||||
|
||||
err := b.UpdatePeer(peer{Contact: c, Distance: rt.id.Xor(c.ID)}, true)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Fresh refreshes a contact if its already in the routing table
|
||||
func (rt *routingTable) Fresh(c Contact) {
|
||||
rt.mu.RLock()
|
||||
defer rt.mu.RUnlock()
|
||||
err := rt.bucketFor(c.ID).UpdatePeer(peer{Contact: c, Distance: rt.id.Xor(c.ID)}, false)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
// FailContact marks a contact as having failed, and removes it if it failed too many times
|
||||
func (rt *routingTable) Fail(c Contact) {
|
||||
rt.mu.RLock()
|
||||
defer rt.mu.RUnlock()
|
||||
rt.bucketFor(c.ID).FailContact(c.ID)
|
||||
}
|
||||
|
||||
// GetClosest returns the closest `limit` contacts from the routing table.
|
||||
// This is a locking wrapper around getClosest()
|
||||
func (rt *routingTable) GetClosest(target bits.Bitmap, limit int) []Contact {
|
||||
rt.mu.RLock()
|
||||
defer rt.mu.RUnlock()
|
||||
return rt.getClosest(target, limit)
|
||||
}
|
||||
|
||||
// getClosest returns the closest `limit` contacts from the routing table
|
||||
func (rt *routingTable) getClosest(target bits.Bitmap, limit int) []Contact {
|
||||
var contacts []Contact
|
||||
for _, b := range rt.buckets {
|
||||
contacts = append(contacts, b.Contacts()...)
|
||||
}
|
||||
|
||||
sortByDistance(contacts, target)
|
||||
if len(contacts) > limit {
|
||||
contacts = contacts[:limit]
|
||||
}
|
||||
|
||||
return contacts
|
||||
}
|
||||
|
||||
// Count returns the number of contacts in the routing table
|
||||
func (rt *routingTable) Count() int {
|
||||
rt.mu.RLock()
|
||||
defer rt.mu.RUnlock()
|
||||
count := 0
|
||||
for _, bucket := range rt.buckets {
|
||||
count += bucket.Len()
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
// Len returns the number of buckets in the routing table
|
||||
func (rt *routingTable) Len() int {
|
||||
rt.mu.RLock()
|
||||
defer rt.mu.RUnlock()
|
||||
return len(rt.buckets)
|
||||
}
|
||||
|
||||
func (rt *routingTable) bucketFor(target bits.Bitmap) *bucket {
|
||||
if rt.id.Equals(target) {
|
||||
panic("routing table does not have a bucket for its own id")
|
||||
}
|
||||
distance := target.Xor(rt.id)
|
||||
for _, b := range rt.buckets {
|
||||
if b.Range.Contains(distance) {
|
||||
return b
|
||||
}
|
||||
}
|
||||
panic("target is not contained in any buckets")
|
||||
}
|
||||
|
||||
func (rt *routingTable) shouldSplit(b *bucket, c Contact) bool {
|
||||
if b.Has(c) {
|
||||
return false
|
||||
}
|
||||
if b.Len() >= bucketSize {
|
||||
if b.Range.Start.Equals(bits.Bitmap{}) { // this is the bucket covering our node id
|
||||
return true
|
||||
}
|
||||
kClosest := rt.getClosest(rt.id, bucketSize)
|
||||
kthClosest := kClosest[len(kClosest)-1]
|
||||
if rt.id.Closer(c.ID, kthClosest.ID) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
//func (rt *routingTable) printBucketInfo() {
|
||||
// fmt.Printf("there are %d contacts in %d buckets\n", rt.Count(), rt.Len())
|
||||
// for i, b := range rt.buckets {
|
||||
// fmt.Printf("bucket %d, %d contacts\n", i+1, len(b.peers))
|
||||
// fmt.Printf(" start : %s\n", b.Range.Start.String())
|
||||
// fmt.Printf(" stop : %s\n", b.Range.End.String())
|
||||
// fmt.Println("")
|
||||
// }
|
||||
//}
|
||||
|
||||
func (rt *routingTable) GetIDsForRefresh(refreshInterval time.Duration) []bits.Bitmap {
|
||||
var bitmaps []bits.Bitmap
|
||||
for i, bucket := range rt.buckets {
|
||||
if bucket.NeedsRefresh(refreshInterval) {
|
||||
bitmaps = append(bitmaps, bits.Rand().Prefix(i, false))
|
||||
}
|
||||
}
|
||||
return bitmaps
|
||||
}
|
||||
|
||||
const rtContactSep = "-"
|
||||
|
||||
type rtSave struct {
|
||||
ID string `json:"id"`
|
||||
Contacts []string `json:"contacts"`
|
||||
}
|
||||
|
||||
func (rt *routingTable) MarshalJSON() ([]byte, error) {
|
||||
var data rtSave
|
||||
data.ID = rt.id.Hex()
|
||||
for _, b := range rt.buckets {
|
||||
for _, c := range b.Contacts() {
|
||||
data.Contacts = append(data.Contacts, strings.Join([]string{c.ID.Hex(), c.IP.String(), strconv.Itoa(c.Port)}, rtContactSep))
|
||||
}
|
||||
}
|
||||
return json.Marshal(data)
|
||||
}
|
||||
|
||||
func (rt *routingTable) UnmarshalJSON(b []byte) error {
|
||||
var data rtSave
|
||||
err := json.Unmarshal(b, &data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rt.id, err = bits.FromHex(data.ID)
|
||||
if err != nil {
|
||||
return errors.Prefix("decoding ID", err)
|
||||
}
|
||||
rt.reset()
|
||||
|
||||
for _, s := range data.Contacts {
|
||||
parts := strings.Split(s, rtContactSep)
|
||||
if len(parts) != 3 {
|
||||
return errors.Err("decoding contact %s: wrong number of parts", s)
|
||||
}
|
||||
var c Contact
|
||||
c.ID, err = bits.FromHex(parts[0])
|
||||
if err != nil {
|
||||
return errors.Err("decoding contact %s: invalid ID: %s", s, err)
|
||||
}
|
||||
c.IP = net.ParseIP(parts[1])
|
||||
if c.IP == nil {
|
||||
return errors.Err("decoding contact %s: invalid IP", s)
|
||||
}
|
||||
c.Port, err = strconv.Atoi(parts[2])
|
||||
if err != nil {
|
||||
return errors.Err("decoding contact %s: invalid port: %s", s, err)
|
||||
}
|
||||
rt.Update(c)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// RoutingTableRefresh refreshes any buckets that need to be refreshed
|
||||
func RoutingTableRefresh(n *Node, refreshInterval time.Duration, parentGrp *stop.Group) {
|
||||
done := stop.New()
|
||||
|
||||
for _, id := range n.rt.GetIDsForRefresh(refreshInterval) {
|
||||
done.Add(1)
|
||||
go func(id bits.Bitmap) {
|
||||
defer done.Done()
|
||||
_, _, err := FindContacts(n, id, false, parentGrp)
|
||||
if err != nil {
|
||||
log.Error("error finding contact during routing table refresh - ", err)
|
||||
}
|
||||
}(id)
|
||||
}
|
||||
|
||||
done.Wait()
|
||||
done.Stop()
|
||||
}
|
||||
|
||||
func moveToBack(peers []peer, index int) {
|
||||
if index < 0 || len(peers) <= index+1 {
|
||||
return
|
||||
}
|
||||
p := peers[index]
|
||||
for i := index; i < len(peers)-1; i++ {
|
||||
peers[i] = peers[i+1]
|
||||
}
|
||||
peers[len(peers)-1] = p
|
||||
}
|
||||
@@ -1,328 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"math/big"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
|
||||
"github.com/sebdah/goldie"
|
||||
)
|
||||
|
||||
func TestBucket_Split(t *testing.T) {
|
||||
rt := newRoutingTable(bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"))
|
||||
if len(rt.buckets) != 1 {
|
||||
t.Errorf("there should only be one bucket so far")
|
||||
}
|
||||
if len(rt.buckets[0].peers) != 0 {
|
||||
t.Errorf("there should be no contacts yet")
|
||||
}
|
||||
|
||||
var tests = []struct {
|
||||
name string
|
||||
id bits.Bitmap
|
||||
expectedBucketCount int
|
||||
expectedTotalContacts int
|
||||
}{
|
||||
//fill first bucket
|
||||
{"b1-one", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100"), 1, 1},
|
||||
{"b1-two", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000200"), 1, 2},
|
||||
{"b1-three", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000300"), 1, 3},
|
||||
{"b1-four", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000400"), 1, 4},
|
||||
{"b1-five", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000500"), 1, 5},
|
||||
{"b1-six", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000600"), 1, 6},
|
||||
{"b1-seven", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000700"), 1, 7},
|
||||
{"b1-eight", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800"), 1, 8},
|
||||
|
||||
// split off second bucket and fill it
|
||||
{"b2-one", bits.FromHexP("001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 9},
|
||||
{"b2-two", bits.FromHexP("002000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 10},
|
||||
{"b2-three", bits.FromHexP("003000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 11},
|
||||
{"b2-four", bits.FromHexP("004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 12},
|
||||
{"b2-five", bits.FromHexP("005000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 13},
|
||||
{"b2-six", bits.FromHexP("006000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 14},
|
||||
{"b2-seven", bits.FromHexP("007000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 15},
|
||||
|
||||
// at this point there are two buckets. the first has 7 contacts, the second has 8
|
||||
|
||||
// inserts into the second bucket should be skipped
|
||||
{"dont-split", bits.FromHexP("009000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 2, 15},
|
||||
|
||||
// ... unless the ID is closer than the kth-closest contact
|
||||
{"split-kth-closest", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), 2, 16},
|
||||
|
||||
{"b3-two", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002"), 3, 17},
|
||||
{"b3-three", bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003"), 3, 18},
|
||||
}
|
||||
|
||||
for i, testCase := range tests {
|
||||
rt.Update(Contact{testCase.id, net.ParseIP("127.0.0.1"), 8000 + i, 0})
|
||||
|
||||
if len(rt.buckets) != testCase.expectedBucketCount {
|
||||
t.Errorf("failed test case %s. there should be %d buckets, got %d", testCase.name, testCase.expectedBucketCount, len(rt.buckets))
|
||||
}
|
||||
if rt.Count() != testCase.expectedTotalContacts {
|
||||
t.Errorf("failed test case %s. there should be %d contacts, got %d", testCase.name, testCase.expectedTotalContacts, rt.Count())
|
||||
}
|
||||
}
|
||||
|
||||
var testRanges = []struct {
|
||||
id bits.Bitmap
|
||||
expected int
|
||||
}{
|
||||
{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), 0},
|
||||
{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000005"), 0},
|
||||
{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000410"), 1},
|
||||
{bits.FromHexP("0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007f0"), 1},
|
||||
{bits.FromHexP("F00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800"), 2},
|
||||
{bits.FromHexP("F00000000000000000000000000000000000000000000000000F00000000000000000000000000000000000000000000"), 2},
|
||||
{bits.FromHexP("F0000000000000000000000000000000F0000000000000000000000000F0000000000000000000000000000000000000"), 2},
|
||||
}
|
||||
|
||||
for _, tt := range testRanges {
|
||||
bucket := bucketNumFor(rt, tt.id)
|
||||
if bucket != tt.expected {
|
||||
t.Errorf("bucketFor(%s, %s) => got %d, expected %d", tt.id.Hex(), rt.id.Hex(), bucket, tt.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func bucketNumFor(rt *routingTable, target bits.Bitmap) int {
|
||||
if rt.id.Equals(target) {
|
||||
panic("routing table does not have a bucket for its own id")
|
||||
}
|
||||
distance := target.Xor(rt.id)
|
||||
for i := range rt.buckets {
|
||||
if rt.buckets[i].Range.Contains(distance) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
panic("target is not contained in any buckets")
|
||||
}
|
||||
|
||||
func TestBucket_Split_Continuous(t *testing.T) {
|
||||
b := newBucket(bits.MaxRange())
|
||||
|
||||
left, right := b.Split()
|
||||
|
||||
if !left.Range.Start.Equals(b.Range.Start) {
|
||||
t.Errorf("left bucket start does not align with original bucket start. got %s, expected %s", left.Range.Start, b.Range.Start)
|
||||
}
|
||||
|
||||
if !right.Range.End.Equals(b.Range.End) {
|
||||
t.Errorf("right bucket end does not align with original bucket end. got %s, expected %s", right.Range.End, b.Range.End)
|
||||
}
|
||||
|
||||
leftEndNext := (&big.Int{}).Add(left.Range.End.Big(), big.NewInt(1))
|
||||
if !bits.FromBigP(leftEndNext).Equals(right.Range.Start) {
|
||||
t.Errorf("there's a gap between left bucket end and right bucket start. end is %s, start is %s", left.Range.End, right.Range.Start)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBucket_Split_KthClosest_DoSplit(t *testing.T) {
|
||||
rt := newRoutingTable(bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"))
|
||||
|
||||
// add 4 low IDs
|
||||
rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), net.ParseIP("127.0.0.1"), 8001, 0})
|
||||
rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002"), net.ParseIP("127.0.0.1"), 8002, 0})
|
||||
rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003"), net.ParseIP("127.0.0.1"), 8003, 0})
|
||||
rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004"), net.ParseIP("127.0.0.1"), 8004, 0})
|
||||
|
||||
// add 4 high IDs
|
||||
rt.Update(Contact{bits.FromHexP("800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8001, 0})
|
||||
rt.Update(Contact{bits.FromHexP("900000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8002, 0})
|
||||
rt.Update(Contact{bits.FromHexP("a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8003, 0})
|
||||
rt.Update(Contact{bits.FromHexP("b00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8004, 0})
|
||||
|
||||
// split the bucket and fill the high bucket
|
||||
rt.Update(Contact{bits.FromHexP("c00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8005, 0})
|
||||
rt.Update(Contact{bits.FromHexP("d00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8006, 0})
|
||||
rt.Update(Contact{bits.FromHexP("e00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8007, 0})
|
||||
rt.Update(Contact{bits.FromHexP("f00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8008, 0})
|
||||
|
||||
// add a high ID. it should split because the high ID is closer than the Kth closest ID
|
||||
rt.Update(Contact{bits.FromHexP("910000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.1"), 8009, 0})
|
||||
|
||||
if len(rt.buckets) != 3 {
|
||||
t.Errorf("expected 3 buckets, got %d", len(rt.buckets))
|
||||
}
|
||||
if rt.Count() != 13 {
|
||||
t.Errorf("expected 13 contacts, got %d", rt.Count())
|
||||
}
|
||||
}
|
||||
|
||||
func TestBucket_Split_KthClosest_DontSplit(t *testing.T) {
|
||||
rt := newRoutingTable(bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"))
|
||||
|
||||
// add 4 low IDs
|
||||
rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), net.ParseIP("127.0.0.1"), 8001, 0})
|
||||
rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002"), net.ParseIP("127.0.0.1"), 8002, 0})
|
||||
rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003"), net.ParseIP("127.0.0.1"), 8003, 0})
|
||||
rt.Update(Contact{bits.FromHexP("000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004"), net.ParseIP("127.0.0.1"), 8004, 0})
|
||||
|
||||
// add 4 high IDs
|
||||
rt.Update(Contact{bits.FromHexP("800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8001, 0})
|
||||
rt.Update(Contact{bits.FromHexP("900000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8002, 0})
|
||||
rt.Update(Contact{bits.FromHexP("a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8003, 0})
|
||||
rt.Update(Contact{bits.FromHexP("b00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8004, 0})
|
||||
|
||||
// split the bucket and fill the high bucket
|
||||
rt.Update(Contact{bits.FromHexP("c00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8005, 0})
|
||||
rt.Update(Contact{bits.FromHexP("d00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8006, 0})
|
||||
rt.Update(Contact{bits.FromHexP("e00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8007, 0})
|
||||
rt.Update(Contact{bits.FromHexP("f00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.2"), 8008, 0})
|
||||
|
||||
// add a really high ID. this should not split because its not closer than the Kth closest ID
|
||||
rt.Update(Contact{bits.FromHexP("ffff00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), net.ParseIP("127.0.0.1"), 8009, 0})
|
||||
|
||||
if len(rt.buckets) != 2 {
|
||||
t.Errorf("expected 2 buckets, got %d", len(rt.buckets))
|
||||
}
|
||||
if rt.Count() != 12 {
|
||||
t.Errorf("expected 12 contacts, got %d", rt.Count())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoutingTable_GetClosest(t *testing.T) {
|
||||
n1 := bits.FromHexP("FFFFFFFF0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
|
||||
n2 := bits.FromHexP("FFFFFFF00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
|
||||
n3 := bits.FromHexP("111111110000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
|
||||
rt := newRoutingTable(n1)
|
||||
rt.Update(Contact{n2, net.ParseIP("127.0.0.1"), 8001, 0})
|
||||
rt.Update(Contact{n3, net.ParseIP("127.0.0.1"), 8002, 0})
|
||||
|
||||
contacts := rt.GetClosest(bits.FromHexP("222222220000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), 1)
|
||||
if len(contacts) != 1 {
|
||||
t.Fail()
|
||||
return
|
||||
}
|
||||
if !contacts[0].ID.Equals(n3) {
|
||||
t.Error(contacts[0])
|
||||
}
|
||||
contacts = rt.GetClosest(n2, 10)
|
||||
if len(contacts) != 2 {
|
||||
t.Error(len(contacts))
|
||||
return
|
||||
}
|
||||
if !contacts[0].ID.Equals(n2) {
|
||||
t.Error(contacts[0])
|
||||
}
|
||||
if !contacts[1].ID.Equals(n3) {
|
||||
t.Error(contacts[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoutingTable_GetClosest_Empty(t *testing.T) {
|
||||
n1 := bits.FromShortHexP("1")
|
||||
rt := newRoutingTable(n1)
|
||||
|
||||
contacts := rt.GetClosest(bits.FromShortHexP("a"), 3)
|
||||
if len(contacts) != 0 {
|
||||
t.Error("there shouldn't be any contacts")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoutingTable_Refresh(t *testing.T) {
|
||||
t.Skip("TODO: test routing table refreshing")
|
||||
}
|
||||
|
||||
func TestRoutingTable_MoveToBack(t *testing.T) {
|
||||
tt := map[string]struct {
|
||||
data []peer
|
||||
index int
|
||||
expected []peer
|
||||
}{
|
||||
"simpleMove": {
|
||||
data: []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
|
||||
index: 1,
|
||||
expected: []peer{{NumFailures: 0}, {NumFailures: 2}, {NumFailures: 3}, {NumFailures: 1}},
|
||||
},
|
||||
"moveFirst": {
|
||||
data: []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
|
||||
index: 0,
|
||||
expected: []peer{{NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}, {NumFailures: 0}},
|
||||
},
|
||||
"moveLast": {
|
||||
data: []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
|
||||
index: 3,
|
||||
expected: []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
|
||||
},
|
||||
"largeIndex": {
|
||||
data: []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
|
||||
index: 27,
|
||||
expected: []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
|
||||
},
|
||||
"negativeIndex": {
|
||||
data: []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
|
||||
index: -12,
|
||||
expected: []peer{{NumFailures: 0}, {NumFailures: 1}, {NumFailures: 2}, {NumFailures: 3}},
|
||||
},
|
||||
}
|
||||
|
||||
for name, test := range tt {
|
||||
moveToBack(test.data, test.index)
|
||||
expected := make([]string, len(test.expected))
|
||||
actual := make([]string, len(test.data))
|
||||
for i := range actual {
|
||||
actual[i] = strconv.Itoa(test.data[i].NumFailures)
|
||||
expected[i] = strconv.Itoa(test.expected[i].NumFailures)
|
||||
}
|
||||
|
||||
expJoin := strings.Join(expected, ",")
|
||||
actJoin := strings.Join(actual, ",")
|
||||
|
||||
if actJoin != expJoin {
|
||||
t.Errorf("%s failed: got %s; expected %s", name, actJoin, expJoin)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoutingTable_Save(t *testing.T) {
|
||||
t.Skip("fix me")
|
||||
id := bits.FromHexP("1c8aff71b99462464d9eeac639595ab99664be3482cb91a29d87467515c7d9158fe72aa1f1582dab07d8f8b5db277f41")
|
||||
rt := newRoutingTable(id)
|
||||
|
||||
for i, b := range rt.buckets {
|
||||
for j := 0; j < bucketSize; j++ {
|
||||
toAdd := b.Range.Start.Add(bits.FromShortHexP(strconv.Itoa(j)))
|
||||
if toAdd.Cmp(b.Range.End) <= 0 {
|
||||
rt.Update(Contact{
|
||||
ID: b.Range.Start.Add(bits.FromShortHexP(strconv.Itoa(j))),
|
||||
IP: net.ParseIP("1.2.3." + strconv.Itoa(j)),
|
||||
Port: 1 + i*bucketSize + j,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
data, err := json.MarshalIndent(rt, "", " ")
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
goldie.Assert(t, t.Name(), data)
|
||||
}
|
||||
|
||||
func TestRoutingTable_Load_ID(t *testing.T) {
|
||||
t.Skip("fix me")
|
||||
id := "1c8aff71b99462464d9eeac639595ab99664be3482cb91a29d87467515c7d9158fe72aa1f1582dab07d8f8b5db277f41"
|
||||
data := []byte(`{"id": "` + id + `","contacts": []}`)
|
||||
|
||||
rt := routingTable{}
|
||||
err := json.Unmarshal(data, &rt)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if rt.id.Hex() != id {
|
||||
t.Error("id mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoutingTable_Load_Contacts(t *testing.T) {
|
||||
t.Skip("TODO")
|
||||
}
|
||||
-187
@@ -1,187 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"sync"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
rpc2 "github.com/gorilla/rpc/v2"
|
||||
"github.com/gorilla/rpc/v2/json"
|
||||
)
|
||||
|
||||
type rpcReceiver struct {
|
||||
dht *DHT
|
||||
}
|
||||
|
||||
type RpcPingArgs struct {
|
||||
Address string
|
||||
}
|
||||
|
||||
func (rpc *rpcReceiver) Ping(r *http.Request, args *RpcPingArgs, result *string) error {
|
||||
if args.Address == "" {
|
||||
return errors.Err("no address given")
|
||||
}
|
||||
|
||||
err := rpc.dht.Ping(args.Address)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
*result = pingSuccessResponse
|
||||
return nil
|
||||
}
|
||||
|
||||
type RpcFindArgs struct {
|
||||
Key string
|
||||
NodeID string
|
||||
IP string
|
||||
Port int
|
||||
}
|
||||
|
||||
func (rpc *rpcReceiver) FindNode(r *http.Request, args *RpcFindArgs, result *[]Contact) error {
|
||||
key, err := bits.FromHex(args.Key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
toQuery, err := bits.FromHex(args.NodeID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
c := Contact{ID: toQuery, IP: net.ParseIP(args.IP), Port: args.Port}
|
||||
req := Request{Method: findNodeMethod, Arg: &key}
|
||||
|
||||
nodeResponse := rpc.dht.node.Send(c, req)
|
||||
if nodeResponse != nil && nodeResponse.Contacts != nil {
|
||||
*result = nodeResponse.Contacts
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type RpcFindValueResult struct {
|
||||
Contacts []Contact
|
||||
Value string
|
||||
}
|
||||
|
||||
func (rpc *rpcReceiver) FindValue(r *http.Request, args *RpcFindArgs, result *RpcFindValueResult) error {
|
||||
key, err := bits.FromHex(args.Key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
toQuery, err := bits.FromHex(args.NodeID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c := Contact{ID: toQuery, IP: net.ParseIP(args.IP), Port: args.Port}
|
||||
req := Request{Arg: &key, Method: findValueMethod}
|
||||
|
||||
nodeResponse := rpc.dht.node.Send(c, req)
|
||||
if nodeResponse != nil && nodeResponse.FindValueKey != "" {
|
||||
*result = RpcFindValueResult{Value: nodeResponse.FindValueKey}
|
||||
return nil
|
||||
}
|
||||
if nodeResponse != nil && nodeResponse.Contacts != nil {
|
||||
*result = RpcFindValueResult{Contacts: nodeResponse.Contacts}
|
||||
return nil
|
||||
}
|
||||
|
||||
return errors.Err("not sure what happened")
|
||||
}
|
||||
|
||||
type RpcIterativeFindValueArgs struct {
|
||||
Key string
|
||||
}
|
||||
|
||||
type RpcIterativeFindValueResult struct {
|
||||
Contacts []Contact
|
||||
FoundValue bool
|
||||
}
|
||||
|
||||
func (rpc *rpcReceiver) IterativeFindValue(r *http.Request, args *RpcIterativeFindValueArgs, result *RpcIterativeFindValueResult) error {
|
||||
key, err := bits.FromHex(args.Key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
foundContacts, found, err := FindContacts(rpc.dht.node, key, false, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result.Contacts = foundContacts
|
||||
result.FoundValue = found
|
||||
return nil
|
||||
}
|
||||
|
||||
type RpcBucketResponse struct {
|
||||
Start string
|
||||
End string
|
||||
NumContacts int
|
||||
Contacts []Contact
|
||||
}
|
||||
|
||||
type RpcRoutingTableResponse struct {
|
||||
NodeID string
|
||||
NumBuckets int
|
||||
Buckets []RpcBucketResponse
|
||||
}
|
||||
|
||||
func (rpc *rpcReceiver) GetRoutingTable(r *http.Request, args *struct{}, result *RpcRoutingTableResponse) error {
|
||||
result.NodeID = rpc.dht.node.id.String()
|
||||
result.NumBuckets = len(rpc.dht.node.rt.buckets)
|
||||
for _, b := range rpc.dht.node.rt.buckets {
|
||||
result.Buckets = append(result.Buckets, RpcBucketResponse{
|
||||
Start: b.Range.Start.String(),
|
||||
End: b.Range.End.String(),
|
||||
NumContacts: b.Len(),
|
||||
Contacts: b.Contacts(),
|
||||
})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (rpc *rpcReceiver) AddKnownNode(r *http.Request, args *Contact, result *string) error {
|
||||
rpc.dht.node.AddKnownNode(*args)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (dht *DHT) runRPCServer(port int) {
|
||||
addr := "0.0.0.0:" + strconv.Itoa(port)
|
||||
|
||||
s := rpc2.NewServer()
|
||||
s.RegisterCodec(json.NewCodec(), "application/json")
|
||||
s.RegisterCodec(json.NewCodec(), "application/json;charset=UTF-8")
|
||||
err := s.RegisterService(&rpcReceiver{dht: dht}, "rpc")
|
||||
if err != nil {
|
||||
log.Error(errors.Prefix("registering rpc service", err))
|
||||
return
|
||||
}
|
||||
|
||||
handler := mux.NewRouter()
|
||||
handler.Handle("/", s)
|
||||
server := &http.Server{Addr: addr, Handler: handler}
|
||||
|
||||
wg := sync.WaitGroup{}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
log.Printf("RPC server listening on %s", addr)
|
||||
err := server.ListenAndServe()
|
||||
if err != nil && err != http.ErrServerClosed {
|
||||
log.Error(err)
|
||||
}
|
||||
}()
|
||||
|
||||
<-dht.grp.Ch()
|
||||
err = server.Shutdown(context.Background())
|
||||
if err != nil {
|
||||
log.Error(errors.Prefix("shutting down rpc service", err))
|
||||
return
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
)
|
||||
|
||||
// TODO: expire stored data after tExpire time
|
||||
|
||||
type contactStore struct {
|
||||
// map of blob hashes to (map of node IDs to bools)
|
||||
hashes map[bits.Bitmap]map[bits.Bitmap]bool
|
||||
// stores the peers themselves, so they can be updated in one place
|
||||
contacts map[bits.Bitmap]Contact
|
||||
lock sync.RWMutex
|
||||
}
|
||||
|
||||
func newStore() *contactStore {
|
||||
return &contactStore{
|
||||
hashes: make(map[bits.Bitmap]map[bits.Bitmap]bool),
|
||||
contacts: make(map[bits.Bitmap]Contact),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *contactStore) Upsert(blobHash bits.Bitmap, contact Contact) {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
|
||||
if _, ok := s.hashes[blobHash]; !ok {
|
||||
s.hashes[blobHash] = make(map[bits.Bitmap]bool)
|
||||
}
|
||||
s.hashes[blobHash][contact.ID] = true
|
||||
s.contacts[contact.ID] = contact
|
||||
}
|
||||
|
||||
func (s *contactStore) Get(blobHash bits.Bitmap) []Contact {
|
||||
s.lock.RLock()
|
||||
defer s.lock.RUnlock()
|
||||
|
||||
var contacts []Contact
|
||||
if ids, ok := s.hashes[blobHash]; ok {
|
||||
for id := range ids {
|
||||
contact, ok := s.contacts[id]
|
||||
if !ok {
|
||||
panic("node id in IDs list, but not in nodeInfo")
|
||||
}
|
||||
contacts = append(contacts, contact)
|
||||
}
|
||||
}
|
||||
return contacts
|
||||
}
|
||||
|
||||
func (s *contactStore) RemoveTODO(contact Contact) {
|
||||
// TODO: remove peer from everywhere
|
||||
}
|
||||
|
||||
func (s *contactStore) CountStoredHashes() int {
|
||||
s.lock.RLock()
|
||||
defer s.lock.RUnlock()
|
||||
return len(s.hashes)
|
||||
}
|
||||
-312
@@ -1,312 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
)
|
||||
|
||||
var testingDHTIP = "127.0.0.1"
|
||||
var testingDHTFirstPort = 21000
|
||||
|
||||
// TestingCreateNetwork initializes a testable DHT network with a specific number of nodes, with bootstrap and concurrent options.
|
||||
func TestingCreateNetwork(t *testing.T, numNodes int, bootstrap, concurrent bool) (*BootstrapNode, []*DHT) {
|
||||
var bootstrapNode *BootstrapNode
|
||||
var seeds []string
|
||||
|
||||
if bootstrap {
|
||||
bootstrapAddress := testingDHTIP + ":" + strconv.Itoa(testingDHTFirstPort)
|
||||
seeds = []string{bootstrapAddress}
|
||||
bootstrapNode = NewBootstrapNode(bits.Rand(), 0, bootstrapDefaultRefreshDuration)
|
||||
listener, err := net.ListenPacket(Network, bootstrapAddress)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
err = bootstrapNode.Connect(listener.(*net.UDPConn))
|
||||
if err != nil {
|
||||
t.Error("error connecting bootstrap node - ", err)
|
||||
}
|
||||
}
|
||||
|
||||
if numNodes < 1 {
|
||||
return bootstrapNode, nil
|
||||
}
|
||||
|
||||
firstPort := testingDHTFirstPort + 1
|
||||
dhts := make([]*DHT, numNodes)
|
||||
|
||||
for i := 0; i < numNodes; i++ {
|
||||
c := NewStandardConfig()
|
||||
c.NodeID = bits.Rand().Hex()
|
||||
c.Address = testingDHTIP + ":" + strconv.Itoa(firstPort+i)
|
||||
c.SeedNodes = seeds
|
||||
dht := New(c)
|
||||
|
||||
go func() {
|
||||
err := dht.Start()
|
||||
if err != nil {
|
||||
t.Error("error starting dht - ", err)
|
||||
}
|
||||
}()
|
||||
if !concurrent {
|
||||
dht.WaitUntilJoined()
|
||||
}
|
||||
dhts[i] = dht
|
||||
}
|
||||
|
||||
if concurrent {
|
||||
for _, d := range dhts {
|
||||
d.WaitUntilJoined()
|
||||
}
|
||||
}
|
||||
|
||||
return bootstrapNode, dhts
|
||||
}
|
||||
|
||||
type timeoutErr struct {
|
||||
error
|
||||
}
|
||||
|
||||
func (t timeoutErr) Timeout() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (t timeoutErr) Temporary() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// TODO: just use a normal net.Conn instead of this mock conn
|
||||
|
||||
type testUDPPacket struct {
|
||||
data []byte
|
||||
addr *net.UDPAddr
|
||||
}
|
||||
|
||||
type testUDPConn struct {
|
||||
addr *net.UDPAddr
|
||||
toRead chan testUDPPacket
|
||||
writes chan testUDPPacket
|
||||
|
||||
readDeadline time.Time
|
||||
}
|
||||
|
||||
func newTestUDPConn(addr string) *testUDPConn {
|
||||
parts := strings.Split(addr, ":")
|
||||
if len(parts) != 2 {
|
||||
panic("addr needs ip and port")
|
||||
}
|
||||
port, err := strconv.Atoi(parts[1])
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return &testUDPConn{
|
||||
addr: &net.UDPAddr{IP: net.IP(parts[0]), Port: port},
|
||||
toRead: make(chan testUDPPacket),
|
||||
writes: make(chan testUDPPacket),
|
||||
}
|
||||
}
|
||||
|
||||
func (t testUDPConn) ReadFromUDP(b []byte) (int, *net.UDPAddr, error) {
|
||||
var timeoutCh <-chan time.Time
|
||||
if !t.readDeadline.IsZero() {
|
||||
timeoutCh = time.After(time.Until(t.readDeadline))
|
||||
}
|
||||
|
||||
select {
|
||||
case packet, ok := <-t.toRead:
|
||||
if !ok {
|
||||
return 0, nil, errors.Err("conn closed")
|
||||
}
|
||||
n := copy(b, packet.data)
|
||||
return n, packet.addr, nil
|
||||
case <-timeoutCh:
|
||||
return 0, nil, timeoutErr{errors.Err("timeout")}
|
||||
}
|
||||
}
|
||||
|
||||
func (t testUDPConn) WriteToUDP(b []byte, addr *net.UDPAddr) (int, error) {
|
||||
t.writes <- testUDPPacket{data: b, addr: addr}
|
||||
return len(b), nil
|
||||
}
|
||||
|
||||
func (t *testUDPConn) SetReadDeadline(tm time.Time) error {
|
||||
t.readDeadline = tm
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *testUDPConn) SetWriteDeadline(tm time.Time) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *testUDPConn) Close() error {
|
||||
close(t.toRead)
|
||||
t.writes = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
func verifyResponse(t *testing.T, resp map[string]interface{}, id messageID, dhtNodeID string) {
|
||||
if len(resp) != 4 {
|
||||
t.Errorf("expected 4 response fields, got %d", len(resp))
|
||||
}
|
||||
|
||||
_, ok := resp[headerTypeField]
|
||||
if !ok {
|
||||
t.Error("missing type field")
|
||||
} else {
|
||||
rType, ok := resp[headerTypeField].(int64)
|
||||
if !ok {
|
||||
t.Error("type is not an integer")
|
||||
} else if rType != responseType {
|
||||
t.Error("unexpected response type")
|
||||
}
|
||||
}
|
||||
|
||||
_, ok = resp[headerMessageIDField]
|
||||
if !ok {
|
||||
t.Error("missing message id field")
|
||||
} else {
|
||||
rMessageID, ok := resp[headerMessageIDField].(string)
|
||||
if !ok {
|
||||
t.Error("message ID is not a string")
|
||||
} else if rMessageID != string(id[:]) {
|
||||
t.Error("unexpected message ID")
|
||||
}
|
||||
if len(rMessageID) != messageIDLength {
|
||||
t.Errorf("message ID should be %d chars long", messageIDLength)
|
||||
}
|
||||
}
|
||||
|
||||
_, ok = resp[headerNodeIDField]
|
||||
if !ok {
|
||||
t.Error("missing node id field")
|
||||
} else {
|
||||
rNodeID, ok := resp[headerNodeIDField].(string)
|
||||
if !ok {
|
||||
t.Error("node ID is not a string")
|
||||
} else if rNodeID != dhtNodeID {
|
||||
t.Error("unexpected node ID")
|
||||
}
|
||||
if len(rNodeID) != nodeIDLength {
|
||||
t.Errorf("node ID should be %d chars long", nodeIDLength)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func verifyContacts(t *testing.T, contacts []interface{}, nodes []Contact) {
|
||||
if len(contacts) != len(nodes) {
|
||||
t.Errorf("got %d contacts; expected %d", len(contacts), len(nodes))
|
||||
return
|
||||
}
|
||||
|
||||
foundNodes := make(map[string]bool)
|
||||
|
||||
for _, c := range contacts {
|
||||
contact, ok := c.([]interface{})
|
||||
if !ok {
|
||||
t.Error("contact is not a list")
|
||||
return
|
||||
}
|
||||
|
||||
if len(contact) != 3 {
|
||||
t.Error("contact must be 3 items")
|
||||
return
|
||||
}
|
||||
|
||||
var currNode Contact
|
||||
currNodeFound := false
|
||||
|
||||
id, ok := contact[0].(string)
|
||||
if !ok {
|
||||
t.Error("contact id is not a string")
|
||||
} else {
|
||||
if _, ok := foundNodes[id]; ok {
|
||||
t.Errorf("contact %s appears multiple times", id)
|
||||
continue
|
||||
}
|
||||
for _, n := range nodes {
|
||||
if n.ID.RawString() == id {
|
||||
currNode = n
|
||||
currNodeFound = true
|
||||
foundNodes[id] = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !currNodeFound {
|
||||
t.Errorf("unexpected contact %s", id)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
ip, ok := contact[1].(string)
|
||||
if !ok {
|
||||
t.Error("contact IP is not a string")
|
||||
} else if !currNode.IP.Equal(net.ParseIP(ip)) {
|
||||
t.Errorf("contact IP mismatch. got %s; expected %s", ip, currNode.IP.String())
|
||||
}
|
||||
|
||||
port, ok := contact[2].(int64)
|
||||
if !ok {
|
||||
t.Error("contact port is not an int")
|
||||
} else if int(port) != currNode.Port {
|
||||
t.Errorf("contact port mismatch. got %d; expected %d", port, currNode.Port)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func verifyCompactContacts(t *testing.T, contacts []interface{}, nodes []Contact) {
|
||||
if len(contacts) != len(nodes) {
|
||||
t.Errorf("got %d contacts; expected %d", len(contacts), len(nodes))
|
||||
return
|
||||
}
|
||||
|
||||
foundNodes := make(map[string]bool)
|
||||
|
||||
for _, c := range contacts {
|
||||
compact, ok := c.(string)
|
||||
if !ok {
|
||||
t.Error("contact is not a string")
|
||||
return
|
||||
}
|
||||
|
||||
contact := Contact{}
|
||||
err := contact.UnmarshalCompact([]byte(compact))
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
var currNode Contact
|
||||
currNodeFound := false
|
||||
|
||||
if _, ok := foundNodes[contact.ID.Hex()]; ok {
|
||||
t.Errorf("contact %s appears multiple times", contact.ID.Hex())
|
||||
continue
|
||||
}
|
||||
for _, n := range nodes {
|
||||
if n.ID.Equals(contact.ID) {
|
||||
currNode = n
|
||||
currNodeFound = true
|
||||
foundNodes[contact.ID.Hex()] = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !currNodeFound {
|
||||
t.Errorf("unexpected contact %s", contact.ID.Hex())
|
||||
continue
|
||||
}
|
||||
|
||||
if !currNode.IP.Equal(contact.IP) {
|
||||
t.Errorf("contact IP mismatch. got %s; expected %s", contact.IP.String(), currNode.IP.String())
|
||||
}
|
||||
|
||||
if contact.Port != currNode.Port {
|
||||
t.Errorf("contact port mismatch. got %d; expected %d", contact.Port, currNode.Port)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,71 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
|
||||
"github.com/lbryio/lbry.go/stop"
|
||||
)
|
||||
|
||||
// TODO: this should be moved out of dht and into node, and it should be completely hidden inside node. dht should not need to know about tokens
|
||||
|
||||
type tokenCacheEntry struct {
|
||||
token string
|
||||
receivedAt time.Time
|
||||
}
|
||||
|
||||
type tokenCache struct {
|
||||
node *Node
|
||||
tokens map[string]tokenCacheEntry
|
||||
expiration time.Duration
|
||||
lock *sync.RWMutex
|
||||
}
|
||||
|
||||
func newTokenCache(node *Node, expiration time.Duration) *tokenCache {
|
||||
tc := &tokenCache{}
|
||||
tc.node = node
|
||||
tc.tokens = make(map[string]tokenCacheEntry)
|
||||
tc.expiration = expiration
|
||||
tc.lock = &sync.RWMutex{}
|
||||
return tc
|
||||
}
|
||||
|
||||
// TODO: if store fails, get new token. can happen if a node restarts but we have the token cached
|
||||
|
||||
func (tc *tokenCache) Get(c Contact, hash bits.Bitmap, cancelCh stop.Chan) string {
|
||||
tc.lock.RLock()
|
||||
token, exists := tc.tokens[c.String()]
|
||||
tc.lock.RUnlock()
|
||||
|
||||
if exists && time.Since(token.receivedAt) < tc.expiration {
|
||||
return token.token
|
||||
}
|
||||
|
||||
resCh := tc.node.SendAsync(c, Request{
|
||||
Method: findValueMethod,
|
||||
Arg: &hash,
|
||||
})
|
||||
|
||||
var res *Response
|
||||
|
||||
select {
|
||||
case res = <-resCh:
|
||||
case <-cancelCh:
|
||||
return ""
|
||||
}
|
||||
|
||||
if res == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
tc.lock.Lock()
|
||||
tc.tokens[c.String()] = tokenCacheEntry{
|
||||
token: res.Token,
|
||||
receivedAt: time.Now(),
|
||||
}
|
||||
tc.lock.Unlock()
|
||||
|
||||
return res.Token
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
package dht
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"net"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/stop"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
)
|
||||
|
||||
type tokenManager struct {
|
||||
secret []byte
|
||||
prevSecret []byte
|
||||
lock *sync.RWMutex
|
||||
stop *stop.Group
|
||||
}
|
||||
|
||||
func (tm *tokenManager) Start(interval time.Duration) {
|
||||
tm.secret = make([]byte, 64)
|
||||
tm.prevSecret = make([]byte, 64)
|
||||
tm.lock = &sync.RWMutex{}
|
||||
tm.stop = stop.New()
|
||||
|
||||
tm.rotateSecret()
|
||||
|
||||
tm.stop.Add(1)
|
||||
go func() {
|
||||
defer tm.stop.Done()
|
||||
tick := time.NewTicker(interval)
|
||||
for {
|
||||
select {
|
||||
case <-tick.C:
|
||||
tm.rotateSecret()
|
||||
case <-tm.stop.Ch():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (tm *tokenManager) Stop() {
|
||||
tm.stop.StopAndWait()
|
||||
}
|
||||
|
||||
func (tm *tokenManager) Get(nodeID bits.Bitmap, addr *net.UDPAddr) string {
|
||||
return genToken(tm.secret, nodeID, addr)
|
||||
}
|
||||
|
||||
func (tm *tokenManager) Verify(token string, nodeID bits.Bitmap, addr *net.UDPAddr) bool {
|
||||
return token == genToken(tm.secret, nodeID, addr) || token == genToken(tm.prevSecret, nodeID, addr)
|
||||
}
|
||||
|
||||
func genToken(secret []byte, nodeID bits.Bitmap, addr *net.UDPAddr) string {
|
||||
buf := bytes.Buffer{}
|
||||
buf.Write(nodeID[:])
|
||||
buf.Write(addr.IP)
|
||||
buf.WriteString(strconv.Itoa(addr.Port))
|
||||
buf.Write(secret)
|
||||
t := sha256.Sum256(buf.Bytes())
|
||||
return string(t[:])
|
||||
}
|
||||
|
||||
func (tm *tokenManager) rotateSecret() {
|
||||
tm.lock.Lock()
|
||||
defer tm.lock.Unlock()
|
||||
|
||||
copy(tm.prevSecret, tm.secret)
|
||||
|
||||
_, err := rand.Read(tm.secret)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
module github.com/lbryio/reflector.go
|
||||
|
||||
go 1.25
|
||||
|
||||
replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.0
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.6
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.6
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.18
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0
|
||||
github.com/bluele/gcache v0.0.2
|
||||
github.com/brk0v/directio v0.0.0-20241105172640-ae9d82eb8fee
|
||||
github.com/btcsuite/btcd v0.24.2
|
||||
github.com/btcsuite/btcutil v1.0.2
|
||||
github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
|
||||
github.com/ekyoung/gin-nice-recovery v0.0.0-20160510022553-1654dca486db
|
||||
github.com/gin-gonic/gin v1.11.0
|
||||
github.com/go-sql-driver/mysql v1.9.3
|
||||
github.com/gogo/protobuf v1.3.2
|
||||
github.com/golang/protobuf v1.5.4
|
||||
github.com/google/gops v0.3.28
|
||||
github.com/gorilla/mux v1.8.1
|
||||
github.com/hashicorp/serf v0.10.2
|
||||
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf
|
||||
github.com/karrick/godirwalk v1.17.0
|
||||
github.com/lbryio/chainquery v1.9.1-0.20240927170248-48c092515dea
|
||||
github.com/lbryio/lbry.go/v2 v2.7.2-0.20230307181431-a01aa6dc0629
|
||||
github.com/lbryio/types v0.0.0-20220224142228-73610f6654a6
|
||||
github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5
|
||||
github.com/prometheus/client_golang v1.23.2
|
||||
github.com/quic-go/quic-go v0.58.0
|
||||
github.com/sirupsen/logrus v1.9.3
|
||||
github.com/spf13/cast v1.10.0
|
||||
github.com/spf13/cobra v1.10.2
|
||||
github.com/spf13/viper v1.21.0
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/volatiletech/null/v8 v8.1.2
|
||||
go.uber.org/atomic v1.11.0
|
||||
golang.org/x/sync v0.19.0
|
||||
)
|
||||
|
||||
require (
|
||||
filippo.io/edwards25519 v1.1.0 // indirect
|
||||
github.com/armon/go-metrics v0.4.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 // indirect
|
||||
github.com/aws/smithy-go v1.24.0 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/btcsuite/btclog v1.0.0 // indirect
|
||||
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd // indirect
|
||||
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 // indirect
|
||||
github.com/bytedance/sonic v1.14.0 // indirect
|
||||
github.com/bytedance/sonic/loader v0.3.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/cloudwego/base64x v0.1.6 // indirect
|
||||
github.com/friendsofgo/errors v0.9.2 // indirect
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.8 // indirect
|
||||
github.com/gin-contrib/sse v1.1.0 // indirect
|
||||
github.com/go-errors/errors v1.5.1 // indirect
|
||||
github.com/go-ini/ini v1.67.0 // indirect
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||
github.com/go-playground/validator/v10 v10.27.0 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
||||
github.com/goccy/go-json v0.10.5 // indirect
|
||||
github.com/goccy/go-yaml v1.18.0 // indirect
|
||||
github.com/gofrs/uuid v4.4.0+incompatible // indirect
|
||||
github.com/google/btree v1.1.2 // indirect
|
||||
github.com/gorilla/rpc v1.2.0 // indirect
|
||||
github.com/gorilla/websocket v1.5.3 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
|
||||
github.com/hashicorp/go-metrics v0.5.4 // indirect
|
||||
github.com/hashicorp/go-msgpack/v2 v2.1.2 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/hashicorp/go-sockaddr v1.0.5 // indirect
|
||||
github.com/hashicorp/golang-lru v1.0.2 // indirect
|
||||
github.com/hashicorp/memberlist v0.5.2 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
|
||||
github.com/leodido/go-urn v1.4.0 // indirect
|
||||
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/miekg/dns v1.1.56 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/common v0.66.1 // indirect
|
||||
github.com/prometheus/procfs v0.16.1 // indirect
|
||||
github.com/quic-go/qpack v0.6.0 // indirect
|
||||
github.com/sagikazarmark/locafero v0.11.0 // indirect
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
|
||||
github.com/shopspring/decimal v1.4.0 // indirect
|
||||
github.com/slack-go/slack v0.16.0 // indirect
|
||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
|
||||
github.com/spf13/afero v1.15.0 // indirect
|
||||
github.com/spf13/pflag v1.0.10 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
||||
github.com/ugorji/go/codec v1.3.0 // indirect
|
||||
github.com/volatiletech/inflect v0.0.1 // indirect
|
||||
github.com/volatiletech/randomize v0.0.1 // indirect
|
||||
github.com/volatiletech/strmangle v0.0.8 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.2 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/arch v0.20.0 // indirect
|
||||
golang.org/x/crypto v0.41.0 // indirect
|
||||
golang.org/x/mod v0.27.0 // indirect
|
||||
golang.org/x/net v0.43.0 // indirect
|
||||
golang.org/x/sys v0.35.0 // indirect
|
||||
golang.org/x/text v0.28.0 // indirect
|
||||
golang.org/x/time v0.8.0 // indirect
|
||||
golang.org/x/tools v0.36.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
|
||||
google.golang.org/protobuf v1.36.9 // indirect
|
||||
gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
@@ -0,0 +1,488 @@
|
||||
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
|
||||
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
|
||||
github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
|
||||
github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
|
||||
github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA=
|
||||
github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.0 h1:tNvqh1s+v0vFYdA1xq0aOJH+Y5cRyZ5upu6roPgPKd4=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.0/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.6 h1:hFLBGUKjmLAekvi1evLi5hVvFQtSo3GYwi+Bx4lpJf8=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.6/go.mod h1:lcUL/gcd8WyjCrMnxez5OXkO3/rwcNmvfno62tnXNcI=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.6 h1:F9vWao2TwjV2MyiyVS+duza0NIRtAslgLUM0vTA1ZaE=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.6/go.mod h1:SgHzKjEVsdQr6Opor0ihgWtkWdfRAIwxYzSJ8O85VHY=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 h1:80+uETIWS1BqjnN9uJ0dBUaETh+P1XwFy5vwHwK5r9k=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16/go.mod h1:wOOsYuxYuB/7FlnVtzeBYRcjSRtQpAW0hCP7tIULMwo=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.18 h1:9vWXHtaepwoAl/UuKzxwgOoJDXPCC3hvgNMfcmdS2Tk=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.18/go.mod h1:sKuUZ+MwUTuJbYvZ8pK0x10LvgcJK3Y4rmh63YBekwk=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 h1:rgGwPzb82iBYSvHMHXc8h9mRoOUBZIGFgKb9qniaZZc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16/go.mod h1:L/UxsGeKpGoIj6DxfhOWHWQ/kGKcd4I1VncE4++IyKA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 h1:1jtGzuV7c82xnqOVfx2F0xmJcOw5374L7N6juGW6x6U=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16/go.mod h1:M2E5OQf+XLe+SZGmmpaI2yy+J326aFf6/+54PoxSANc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 h1:CjMzUs78RDDv4ROu3JnJn/Ig1r6ZD7/T2DXLLRpejic=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16/go.mod h1:uVW4OLBqbJXSHJYA9svT9BluSvvwbzLQ2Crf6UPzR3c=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 h1:DIBqIrJ7hv+e4CmIk2z3pyKT+3B6qVMgRsawHiR3qso=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7/go.mod h1:vLm00xmBke75UmpNvOcZQ/Q30ZFjbczeLFqGx5urmGo=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 h1:oHjJHeUy0ImIV0bsrX0X91GkV5nJAyv1l1CC9lnO0TI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16/go.mod h1:iRSNGgOYmiYwSCXxXaKb9HfOEj40+oTKn8pTxMlYkRM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 h1:NSbvS17MlI2lurYgXnCOLvCFX38sBW4eiVER7+kkgsU=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16/go.mod h1:SwT8Tmqd4sA6G1qaGdzWCJN99bUmPGHfRwwq3G5Qb+A=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0 h1:MIWra+MSq53CFaXXAywB2qg9YvVZifkk6vEGl/1Qor0=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0/go.mod h1:79S2BdqCJpScXZA2y+cpZuocWsjGjJINyXnOsf5DTz8=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 h1:HpI7aMmJ+mm1wkSHIA2t5EaFFv5EFYXePW30p1EIrbQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.4/go.mod h1:C5RdGMYGlfM0gYq/tifqgn4EbyX99V15P2V3R+VHbQU=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 h1:aM/Q24rIlS3bRAhTyFurowU8A0SMyGDtEOY/l/s/1Uw=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.8/go.mod h1:+fWt2UHSb4kS7Pu8y+BMBvJF0EWx+4H0hzNwtDNRTrg=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 h1:AHDr0DaHIAo8c9t1emrzAlVDFp+iMMKnPdYy6XO4MCE=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12/go.mod h1:GQ73XawFFiWxyWXMHWfhiomvP3tXtdNar/fi8z18sx0=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 h1:SciGFVNZ4mHdm7gpD1dgZYnCuVdX1s+lFTg4+4DOy70=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.5/go.mod h1:iW40X4QBmUxdP+fZNOpfmkdMZqsovezbAeO+Ubiv2pk=
|
||||
github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk=
|
||||
github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw=
|
||||
github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0=
|
||||
github.com/brk0v/directio v0.0.0-20241105172640-ae9d82eb8fee h1:gwVgfLo8dvQMauElELV1b1kwMuUh2ETzX6weEbdAKgA=
|
||||
github.com/brk0v/directio v0.0.0-20241105172640-ae9d82eb8fee/go.mod h1:M/KA3XJG5PJaApPiv4gWNsgcSJquOQTqumZNLyYE0KM=
|
||||
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
|
||||
github.com/btcsuite/btclog v1.0.0 h1:sEkpKJMmfGiyZjADwEIgB1NSwMyfdD1FB8v6+w1T0Ns=
|
||||
github.com/btcsuite/btclog v1.0.0/go.mod h1:w7xnGOhwT3lmrS4H3b/D1XAXxvh+tbhUm8xeHN2y3TQ=
|
||||
github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
|
||||
github.com/btcsuite/btcutil v1.0.2 h1:9iZ1Terx9fMIOtq1VrwdqfsATL9MC2l8ZrUY6YZ2uts=
|
||||
github.com/btcsuite/btcutil v1.0.2/go.mod h1:j9HUFwoQRsZL3V4n+qG+CUnEGHOarIxfC3Le2Yhbcts=
|
||||
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd h1:R/opQEbFEy9JGkIguV40SvRY1uliPX8ifOvi6ICsFCw=
|
||||
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg=
|
||||
github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY=
|
||||
github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
|
||||
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3SkEwmHoWBmX1DNXhXZqlTpq6s4tyJGc=
|
||||
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
|
||||
github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
|
||||
github.com/bytedance/sonic v1.14.0 h1:/OfKt8HFw0kh2rj8N0F6C/qPGRESq0BbaNZgcNXXzQQ=
|
||||
github.com/bytedance/sonic v1.14.0/go.mod h1:WoEbx8WTcFJfzCe0hbmyTGrfjt8PzNEBdxlNUO24NhA=
|
||||
github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA=
|
||||
github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI=
|
||||
github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 h1:6lhrsTEnloDPXyeZBvSYvQf8u86jbKehZPVDDlkgDl4=
|
||||
github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M=
|
||||
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
|
||||
github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
|
||||
github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M=
|
||||
github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||
github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/ekyoung/gin-nice-recovery v0.0.0-20160510022553-1654dca486db h1:oZ4U9IqO8NS+61OmGTBi8vopzqTRxwQeogyBHdrhjbc=
|
||||
github.com/ekyoung/gin-nice-recovery v0.0.0-20160510022553-1654dca486db/go.mod h1:Pk7/9x6tyChFTkahDvLBQMlvdsWvfC+yU8HTT5VD314=
|
||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||
github.com/friendsofgo/errors v0.9.2 h1:X6NYxef4efCBdwI7BgS820zFaN7Cphrmb+Pljdzjtgk=
|
||||
github.com/friendsofgo/errors v0.9.2/go.mod h1:yCvFW5AkDIL9qn7suHVLiI/gH228n7PC4Pn44IGoTOI=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
|
||||
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
||||
github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM=
|
||||
github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8=
|
||||
github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w=
|
||||
github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM=
|
||||
github.com/gin-gonic/gin v1.11.0 h1:OW/6PLjyusp2PPXtyxKHU0RbX6I/l28FTdDlae5ueWk=
|
||||
github.com/gin-gonic/gin v1.11.0/go.mod h1:+iq/FyxlGzII0KHiBGjuNn4UNENUlKbGlNmc+W50Dls=
|
||||
github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk=
|
||||
github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
|
||||
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
|
||||
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
|
||||
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
|
||||
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
|
||||
github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
|
||||
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
|
||||
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
|
||||
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
|
||||
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
|
||||
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
|
||||
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
|
||||
github.com/go-playground/validator/v10 v10.27.0 h1:w8+XrWVMhGkxOaaowyKH35gFydVHOvC0/uWoy2Fzwn4=
|
||||
github.com/go-playground/validator/v10 v10.27.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo=
|
||||
github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
|
||||
github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/go-test/deep v1.0.4 h1:u2CU3YKy9I2pmu9pX0eq50wCgjfGIt539SqR7FbHiho=
|
||||
github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
|
||||
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||
github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw=
|
||||
github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
|
||||
github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
|
||||
github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA=
|
||||
github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
|
||||
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
|
||||
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
|
||||
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
|
||||
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
|
||||
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
|
||||
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
|
||||
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||
github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU=
|
||||
github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/gops v0.3.28 h1:2Xr57tqKAmQYRAfG12E+yLcoa2Y42UJo2lOrUFL9ark=
|
||||
github.com/google/gops v0.3.28/go.mod h1:6f6+Nl8LcHrzJwi8+p0ii+vmBFSlB4f8cOOkTJ7sk4c=
|
||||
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
|
||||
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
|
||||
github.com/gorilla/rpc v1.2.0 h1:WvvdC2lNeT1SP32zrIce5l0ECBfbAlmrmSBsuc57wfk=
|
||||
github.com/gorilla/rpc v1.2.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ=
|
||||
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
|
||||
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
|
||||
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
|
||||
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
|
||||
github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc=
|
||||
github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
|
||||
github.com/hashicorp/go-metrics v0.5.4 h1:8mmPiIJkTPPEbAiV97IxdAGNdRdaWwVap1BU6elejKY=
|
||||
github.com/hashicorp/go-metrics v0.5.4/go.mod h1:CG5yz4NZ/AI/aQt9Ucm/vdBnbh7fvmv4lxZ350i+QQI=
|
||||
github.com/hashicorp/go-msgpack/v2 v2.1.2 h1:4Ee8FTp834e+ewB71RDrQ0VKpyFdrKOjvYtnQ/ltVj0=
|
||||
github.com/hashicorp/go-msgpack/v2 v2.1.2/go.mod h1:upybraOAblm4S7rx0+jeNy+CWWhzywQsSRV5033mMu4=
|
||||
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
|
||||
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
|
||||
github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
|
||||
github.com/hashicorp/go-sockaddr v1.0.5 h1:dvk7TIXCZpmfOlM+9mlcrWmWjw/wlKT+VDq2wMvfPJU=
|
||||
github.com/hashicorp/go-sockaddr v1.0.5/go.mod h1:uoUUmtwU7n9Dv3O4SNLeFvg0SxQ3lyjsj6+CCykpaxI=
|
||||
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE=
|
||||
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c=
|
||||
github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
|
||||
github.com/hashicorp/memberlist v0.5.2 h1:rJoNPWZ0juJBgqn48gjy59K5H4rNgvUoM1kUD7bXiuI=
|
||||
github.com/hashicorp/memberlist v0.5.2/go.mod h1:Ri9p/tRShbjYnpNf4FFPXG7wxEGY4Nrcn6E7jrVa//4=
|
||||
github.com/hashicorp/serf v0.10.2 h1:m5IORhuNSjaxeljg5DeQVDlQyVkhRIjJDimbkCa8aAc=
|
||||
github.com/hashicorp/serf v0.10.2/go.mod h1:T1CmSGfSeGfnfNy/w0odXQUR1rfECGd2Qdsp84DjOiY=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf h1:WfD7VjIE6z8dIvMsI4/s+1qr5EL+zoIGev1BQj1eoJ8=
|
||||
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h1:hyb9oH7vZsitZCiBt0ZvifOrB+qc8PS5IiilCIb87rg=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
|
||||
github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
|
||||
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
||||
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
|
||||
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
|
||||
github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwSWoI=
|
||||
github.com/karrick/godirwalk v1.17.0/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
|
||||
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
|
||||
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
|
||||
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/lbryio/chainquery v1.9.1-0.20240927170248-48c092515dea h1:Rr5hL3GibxiP221+3tBX7Ep9+13jszLXdqftM2oEPFc=
|
||||
github.com/lbryio/chainquery v1.9.1-0.20240927170248-48c092515dea/go.mod h1:dzrrwADJ2CGlI66MyRoWKWX1NmewTiQ3II4fpaTJjZ4=
|
||||
github.com/lbryio/lbry.go/v2 v2.7.2-0.20230307181431-a01aa6dc0629 h1:klpHPQ5iERUhczdITuKUpYuUZrWDGWb3zlAv3qYgc+o=
|
||||
github.com/lbryio/lbry.go/v2 v2.7.2-0.20230307181431-a01aa6dc0629/go.mod h1:JTkXBAVK8iHNcYmffbLzQ7IFKd/+/oBQGIwiG53bbqw=
|
||||
github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19 h1:/zWD8dVIl7bV1TdJWqPqy9tpqixzX2Qxgit48h3hQcY=
|
||||
github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
|
||||
github.com/lbryio/types v0.0.0-20220224142228-73610f6654a6 h1:IhL9D2QfDWhLNDQpZ3Uiiw0gZEUYeLBS6uDqOd59G5o=
|
||||
github.com/lbryio/types v0.0.0-20220224142228-73610f6654a6/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
|
||||
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
|
||||
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
|
||||
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 h1:mG83tLXWSRdcXMWfkoumVwhcCbf3jHF9QKv/m37BkM0=
|
||||
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCWffGOaDcjkw1iB7W9DVLp6GXmfcJY/7YZCWPA4=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
github.com/miekg/dns v1.1.56 h1:5imZaSeoRNvpM9SzWNhEcP9QliKiz20/dA2QabIGVnE=
|
||||
github.com/miekg/dns v1.1.56/go.mod h1:cRm6Oo2C8TY9ZS/TqsSrseAcncm74lfK5G+ikN2SWWY=
|
||||
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
|
||||
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
|
||||
github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||
github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 h1:Ii+DKncOVM8Cu1Hc+ETb5K+23HdAMvESYE3ZJ5b5cMI=
|
||||
github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
|
||||
github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
|
||||
github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
|
||||
github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
|
||||
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
|
||||
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
|
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
||||
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
|
||||
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
|
||||
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
|
||||
github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
|
||||
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
|
||||
github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
|
||||
github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
|
||||
github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
|
||||
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
|
||||
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
|
||||
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
|
||||
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
|
||||
github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
|
||||
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
|
||||
github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
|
||||
github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
|
||||
github.com/quic-go/quic-go v0.58.0 h1:ggY2pvZaVdB9EyojxL1p+5mptkuHyX5MOSv4dgWF4Ug=
|
||||
github.com/quic-go/quic-go v0.58.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU=
|
||||
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
|
||||
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
|
||||
github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
|
||||
github.com/sebdah/goldie v1.0.0 h1:9GNhIat69MSlz/ndaBg48vl9dF5fI+NBB6kfOxgfkMc=
|
||||
github.com/sebdah/goldie v1.0.0/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4=
|
||||
github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
|
||||
github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
|
||||
github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
|
||||
github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
|
||||
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
|
||||
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
github.com/slack-go/slack v0.16.0 h1:khp/WCFv+Hb/B/AJaAwvcxKun0hM6grN0bUZ8xG60P8=
|
||||
github.com/slack-go/slack v0.16.0/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw=
|
||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw=
|
||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U=
|
||||
github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
|
||||
github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
|
||||
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
|
||||
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
|
||||
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
|
||||
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
|
||||
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
|
||||
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
|
||||
github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
|
||||
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
|
||||
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
|
||||
github.com/ugorji/go/codec v1.3.0 h1:Qd2W2sQawAfG8XSvzwhBeoGq71zXOC/Q1E9y/wUcsUA=
|
||||
github.com/ugorji/go/codec v1.3.0/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4=
|
||||
github.com/volatiletech/inflect v0.0.1 h1:2a6FcMQyhmPZcLa+uet3VJ8gLn/9svWhJxJYwvE8KsU=
|
||||
github.com/volatiletech/inflect v0.0.1/go.mod h1:IBti31tG6phkHitLlr5j7shC5SOo//x0AjDzaJU1PLA=
|
||||
github.com/volatiletech/null/v8 v8.1.2 h1:kiTiX1PpwvuugKwfvUNX/SU/5A2KGZMXfGD0DUHdKEI=
|
||||
github.com/volatiletech/null/v8 v8.1.2/go.mod h1:98DbwNoKEpRrYtGjWFctievIfm4n4MxG0A6EBUcoS5g=
|
||||
github.com/volatiletech/randomize v0.0.1 h1:eE5yajattWqTB2/eN8df4dw+8jwAzBtbdo5sbWC4nMk=
|
||||
github.com/volatiletech/randomize v0.0.1/go.mod h1:GN3U0QYqfZ9FOJ67bzax1cqZ5q2xuj2mXrXBjWaRTlY=
|
||||
github.com/volatiletech/strmangle v0.0.1/go.mod h1:F6RA6IkB5vq0yTG4GQ0UsbbRcl3ni9P76i+JrTBKFFg=
|
||||
github.com/volatiletech/strmangle v0.0.8 h1:UZkTDFIjZcL1Lk4BXhGsxcyXxNcWuM5ZwdzZc0sJcWg=
|
||||
github.com/volatiletech/strmangle v0.0.8/go.mod h1:ycDvbDkjDvhC0NUU8w3fWwl5JEMTV56vTKXzR3GeR+0=
|
||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
|
||||
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko=
|
||||
go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o=
|
||||
go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
|
||||
go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/arch v0.20.0 h1:dx1zTU0MAE98U+TQ8BLl7XsJbgze2WnNKF/8tGp/Q6c=
|
||||
golang.org/x/arch v0.20.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk=
|
||||
golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4=
|
||||
golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc=
|
||||
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ=
|
||||
golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE=
|
||||
golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
|
||||
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
|
||||
golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
|
||||
golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
|
||||
golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg=
|
||||
golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg=
|
||||
golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY=
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
|
||||
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
|
||||
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.36.9 h1:w2gp2mA27hUeUzj9Ex9FBjsBm40zfaDtEWow293U7Iw=
|
||||
google.golang.org/protobuf v1.36.9/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 h1:FpCr9V8wuOei4BAen+93HtVJ+XSi+KPbaPKm0Vj5R64=
|
||||
gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79/go.mod h1:gWkaRU7CoXpezCBWfWjm3999QqS+1pYPXGbqQCTMzo8=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
|
||||
gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
|
||||
@@ -0,0 +1,344 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
ee "github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type Server struct {
|
||||
srv *http.Server
|
||||
stop *stop.Stopper
|
||||
}
|
||||
|
||||
func NewServer(address string, path string) *Server {
|
||||
h := http.NewServeMux()
|
||||
h.Handle(path, promhttp.Handler())
|
||||
return &Server{
|
||||
srv: &http.Server{
|
||||
Addr: address,
|
||||
Handler: h,
|
||||
//https://blog.cloudflare.com/the-complete-guide-to-golang-net-http-timeouts/
|
||||
//https://blog.cloudflare.com/exposing-go-on-the-internet/
|
||||
ReadTimeout: 5 * time.Second,
|
||||
WriteTimeout: 10 * time.Second,
|
||||
IdleTimeout: 120 * time.Second,
|
||||
},
|
||||
stop: stop.New(),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) Start() {
|
||||
s.stop.Add(1)
|
||||
go func() {
|
||||
defer s.stop.Done()
|
||||
err := s.srv.ListenAndServe()
|
||||
if err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
log.Error(err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (s *Server) Shutdown() {
|
||||
_ = s.srv.Shutdown(context.Background())
|
||||
s.stop.StopAndWait()
|
||||
}
|
||||
|
||||
const (
|
||||
ns = "reflector"
|
||||
subsystemCache = "cache"
|
||||
subsystemITTT = "ittt"
|
||||
|
||||
labelDirection = "direction"
|
||||
labelErrorType = "error_type"
|
||||
|
||||
DirectionUpload = "upload" // to reflector
|
||||
DirectionDownload = "download" // from reflector
|
||||
|
||||
LabelCacheType = "cache_type"
|
||||
LabelOrigin = "origin"
|
||||
LabelComponent = "component"
|
||||
LabelSource = "source"
|
||||
|
||||
errConnReset = "conn_reset"
|
||||
errReadConnReset = "read_conn_reset"
|
||||
errWriteConnReset = "write_conn_reset"
|
||||
errReadConnTimedOut = "read_conn_timed_out"
|
||||
errNoNetworkActivity = "no_network_activity"
|
||||
errWriteConnTimedOut = "write_conn_timed_out"
|
||||
errWriteBrokenPipe = "write_broken_pipe"
|
||||
errEPipe = "e_pipe"
|
||||
errETimedout = "e_timedout"
|
||||
errIOTimeout = "io_timeout"
|
||||
errUnexpectedEOF = "unexpected_eof"
|
||||
errUnexpectedEOFStr = "unexpected_eof_str"
|
||||
errJSONSyntax = "json_syntax"
|
||||
errBlobTooBig = "blob_too_big"
|
||||
errInvalidPeerJSON = "invalid_peer_json"
|
||||
errInvalidPeerData = "invalid_peer_data"
|
||||
errRequestTooLarge = "request_too_large"
|
||||
errDeadlineExceeded = "deadline_exceeded"
|
||||
errHashMismatch = "hash_mismatch"
|
||||
errProtectedBlob = "protected_blob"
|
||||
errInvalidBlobHash = "invalid_blob_hash"
|
||||
errZeroByteBlob = "zero_byte_blob"
|
||||
errInvalidCharacter = "invalid_character"
|
||||
errBlobNotFound = "blob_not_found"
|
||||
errNoErr = "no_error"
|
||||
errQuicProto = "quic_protocol_violation"
|
||||
errOther = "other"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrorCount = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "error_total",
|
||||
Help: "Total number of errors",
|
||||
}, []string{labelDirection, labelErrorType})
|
||||
|
||||
BlobDownloadCount = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "blob_download_total",
|
||||
Help: "Total number of blobs downloaded from reflector",
|
||||
})
|
||||
PeerDownloadCount = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "peer_download_total",
|
||||
Help: "Total number of blobs downloaded from reflector through tcp protocol",
|
||||
})
|
||||
Http3DownloadCount = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "http3_blob_download_total",
|
||||
Help: "Total number of blobs downloaded from reflector through QUIC protocol",
|
||||
})
|
||||
HttpDownloadCount = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "http_blob_download_total",
|
||||
Help: "Total number of blobs downloaded from reflector through HTTP protocol",
|
||||
})
|
||||
|
||||
CacheHitCount = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Subsystem: subsystemCache,
|
||||
Name: "hit_total",
|
||||
Help: "Total number of blobs retrieved from the cache storage",
|
||||
}, []string{LabelCacheType, LabelComponent})
|
||||
ItttHitCount = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Subsystem: subsystemITTT,
|
||||
Name: "hits_total",
|
||||
Help: "Total number of blobs retrieved from the this/that storage",
|
||||
}, []string{LabelOrigin})
|
||||
CacheMissCount = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Subsystem: subsystemCache,
|
||||
Name: "miss_total",
|
||||
Help: "Total number of blobs retrieved from origin rather than cache storage",
|
||||
}, []string{LabelCacheType, LabelComponent})
|
||||
CacheOriginRequestsCount = promauto.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: ns,
|
||||
Subsystem: subsystemCache,
|
||||
Name: "origin_requests_total",
|
||||
Help: "How many Get requests are in flight from the cache to the origin",
|
||||
}, []string{LabelCacheType, LabelComponent})
|
||||
//during thundering-herd situations, the metric below should be a lot smaller than the metric above
|
||||
CacheWaitingRequestsCount = promauto.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: ns,
|
||||
Subsystem: subsystemCache,
|
||||
Name: "waiting_requests_total",
|
||||
Help: "How many cache requests are waiting for an in-flight origin request",
|
||||
}, []string{LabelCacheType, LabelComponent})
|
||||
CacheLRUEvictCount = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Subsystem: subsystemCache,
|
||||
Name: "evict_total",
|
||||
Help: "Count of blobs evicted from cache",
|
||||
}, []string{LabelCacheType, LabelComponent})
|
||||
CacheRetrievalSpeed = promauto.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: ns,
|
||||
Name: "speed_mbps",
|
||||
Help: "Speed of blob retrieval from cache or from origin",
|
||||
}, []string{LabelCacheType, LabelComponent, LabelSource})
|
||||
|
||||
BlobUploadCount = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "blob_upload_total",
|
||||
Help: "Total number of blobs uploaded to reflector",
|
||||
})
|
||||
SDBlobUploadCount = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "sdblob_upload_total",
|
||||
Help: "Total number of SD blobs (and therefore streams) uploaded to reflector",
|
||||
})
|
||||
|
||||
MtrInBytesTcp = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "tcp_in_bytes",
|
||||
Help: "Total number of bytes downloaded through TCP",
|
||||
})
|
||||
MtrOutBytesTcp = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "tcp_out_bytes",
|
||||
Help: "Total number of bytes streamed out through TCP",
|
||||
})
|
||||
MtrInBytesUdp = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "udp_in_bytes",
|
||||
Help: "Total number of bytes downloaded through UDP",
|
||||
})
|
||||
MtrInBytesHttp = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "http_in_bytes",
|
||||
Help: "Total number of bytes downloaded through HTTP",
|
||||
})
|
||||
MtrOutBytesUdp = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "udp_out_bytes",
|
||||
Help: "Total number of bytes streamed out through UDP",
|
||||
})
|
||||
MtrOutBytesHttp = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "http_out_bytes",
|
||||
Help: "Total number of bytes streamed out through UDP",
|
||||
})
|
||||
MtrInBytesReflector = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "reflector_in_bytes",
|
||||
Help: "Total number of incoming bytes (from users)",
|
||||
})
|
||||
MtrOutBytesReflector = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "s3_out_bytes",
|
||||
Help: "Total number of outgoing bytes (to S3)",
|
||||
})
|
||||
MtrInBytesUpstream = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "upstream_in_bytes",
|
||||
Help: "Total number of incoming bytes (from Upstream)",
|
||||
})
|
||||
MtrInBytesS3 = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "s3_in_bytes",
|
||||
Help: "Total number of incoming bytes (from S3-CF)",
|
||||
})
|
||||
Http3BlobReqQueue = promauto.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: ns,
|
||||
Name: "http3_blob_request_queue_size",
|
||||
Help: "Blob requests of https queue size",
|
||||
})
|
||||
HttpBlobReqQueue = promauto.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: ns,
|
||||
Name: "http_blob_request_queue_size",
|
||||
Help: "Blob requests queue size of the HTTP protocol",
|
||||
})
|
||||
RoutinesQueue = promauto.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: ns,
|
||||
Name: "routines",
|
||||
Help: "routines running by type",
|
||||
}, []string{"package", "kind"})
|
||||
)
|
||||
|
||||
func CacheLabels(name, component string) prometheus.Labels {
|
||||
return prometheus.Labels{
|
||||
LabelCacheType: name,
|
||||
LabelComponent: component,
|
||||
}
|
||||
}
|
||||
func ItttLabels(orig string) prometheus.Labels {
|
||||
return prometheus.Labels{
|
||||
LabelOrigin: orig,
|
||||
}
|
||||
}
|
||||
|
||||
func TrackError(direction string, e error) (shouldLog bool) { // shouldLog is a hack, but whatever
|
||||
if e == nil {
|
||||
return
|
||||
}
|
||||
|
||||
err := ee.Wrap(e, 0)
|
||||
errType := errOther
|
||||
if strings.Contains(err.Error(), "i/o timeout") {
|
||||
errType = errIOTimeout
|
||||
} else if errors.Is(e, syscall.ECONNRESET) {
|
||||
// Looks like we're getting this when direction == "download", but read_conn_reset and
|
||||
// write_conn_reset when its "upload"
|
||||
errType = errConnReset
|
||||
} else if errors.Is(e, context.DeadlineExceeded) {
|
||||
errType = errDeadlineExceeded
|
||||
} else if strings.Contains(err.Error(), "read: connection reset by peer") { // the other side closed the connection using TCP reset
|
||||
errType = errReadConnReset
|
||||
} else if strings.Contains(err.Error(), "write: connection reset by peer") { // the other side closed the connection using TCP reset
|
||||
errType = errWriteConnReset
|
||||
} else if errors.Is(e, syscall.ETIMEDOUT) {
|
||||
errType = errETimedout
|
||||
} else if strings.Contains(err.Error(), "read: connection timed out") { // the other side closed the connection using TCP reset
|
||||
//log.Warnln("read conn timed out is not the same as ETIMEDOUT")
|
||||
errType = errReadConnTimedOut
|
||||
} else if strings.Contains(err.Error(), "NO_ERROR: No recent network activity") { // the other side closed the QUIC connection
|
||||
//log.Warnln("read conn timed out is not the same as ETIMEDOUT")
|
||||
errType = errNoNetworkActivity
|
||||
} else if strings.Contains(err.Error(), "write: connection timed out") {
|
||||
errType = errWriteConnTimedOut
|
||||
} else if errors.Is(e, io.ErrUnexpectedEOF) {
|
||||
errType = errUnexpectedEOF
|
||||
} else if strings.Contains(err.Error(), "unexpected EOF") { // tried to read from closed pipe or socket
|
||||
errType = errUnexpectedEOFStr
|
||||
} else if errors.Is(e, syscall.EPIPE) {
|
||||
errType = errEPipe
|
||||
} else if strings.Contains(err.Error(), "write: broken pipe") { // tried to write to a pipe or socket that was closed by the peer
|
||||
// I believe this is the same as EPipe when direction == "download", but not for upload
|
||||
errType = errWriteBrokenPipe
|
||||
//} else if errors.Is(e, reflector.ErrBlobTooBig) { # this creates a circular import
|
||||
// errType = errBlobTooBig
|
||||
} else if strings.Contains(err.Error(), "blob must be at most") {
|
||||
//log.Warnln("blob must be at most X bytes is not the same as ErrBlobTooBig")
|
||||
errType = errBlobTooBig
|
||||
} else if strings.Contains(err.Error(), "invalid json request") {
|
||||
errType = errInvalidPeerJSON
|
||||
} else if strings.Contains(err.Error(), "Invalid data") {
|
||||
errType = errInvalidPeerData
|
||||
} else if strings.Contains(err.Error(), "request is too large") {
|
||||
errType = errRequestTooLarge
|
||||
} else if strings.Contains(err.Error(), "Invalid blob hash length") {
|
||||
errType = errInvalidBlobHash
|
||||
} else if strings.Contains(err.Error(), "hash of received blob data does not match hash from send request") {
|
||||
errType = errHashMismatch
|
||||
} else if strings.Contains(err.Error(), "blob not found") {
|
||||
errType = errBlobNotFound
|
||||
} else if strings.Contains(err.Error(), "requested blob is protected") {
|
||||
errType = errProtectedBlob
|
||||
} else if strings.Contains(err.Error(), "0-byte blob received") {
|
||||
errType = errZeroByteBlob
|
||||
} else if strings.Contains(err.Error(), "PROTOCOL_VIOLATION: tried to retire connection") {
|
||||
errType = errQuicProto
|
||||
} else if strings.Contains(err.Error(), "invalid character") {
|
||||
errType = errInvalidCharacter
|
||||
} else if _, ok := e.(*json.SyntaxError); ok {
|
||||
errType = errJSONSyntax
|
||||
} else if strings.Contains(err.Error(), "NO_ERROR") {
|
||||
errType = errNoErr
|
||||
} else {
|
||||
log.Warnf("error '%s' for direction '%s' is not being tracked", err.TypeName(), direction)
|
||||
shouldLog = true
|
||||
}
|
||||
|
||||
ErrorCount.With(map[string]string{
|
||||
labelDirection: direction,
|
||||
labelErrorType: errType,
|
||||
}).Inc()
|
||||
|
||||
return
|
||||
}
|
||||
+347
@@ -0,0 +1,347 @@
|
||||
package lite_db
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
qt "github.com/lbryio/lbry.go/v2/extras/query"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/volatiletech/null/v8"
|
||||
)
|
||||
|
||||
// SdBlob is a special blob that contains information on the rest of the blobs in the stream
|
||||
type SdBlob struct {
|
||||
StreamName string `json:"stream_name"`
|
||||
StreamType string `json:"stream_type"`
|
||||
Key string `json:"key"`
|
||||
SuggestedFileName string `json:"suggested_file_name"`
|
||||
StreamHash string `json:"stream_hash"`
|
||||
Blobs []struct {
|
||||
BlobHash string `json:"blob_hash,omitempty"`
|
||||
IV string `json:"iv"`
|
||||
Length int `json:"length"`
|
||||
BlobNum int `json:"blob_num"`
|
||||
} `json:"blobs"`
|
||||
}
|
||||
|
||||
// SQL implements the DB interface
|
||||
type SQL struct {
|
||||
conn *sql.DB
|
||||
|
||||
TrackAccessTime bool
|
||||
}
|
||||
|
||||
func logQuery(query string, args ...interface{}) {
|
||||
s, err := qt.InterpolateParams(query, args...)
|
||||
if err != nil {
|
||||
log.Errorln(err)
|
||||
} else {
|
||||
log.Debugln(s)
|
||||
}
|
||||
}
|
||||
|
||||
// Connect will create a connection to the database
|
||||
func (s *SQL) Connect(dsn string) error {
|
||||
var err error
|
||||
// interpolateParams is necessary. otherwise uploading a stream with thousands of blobs
|
||||
// will hit MySQL's max_prepared_stmt_count limit because the prepared statements are all
|
||||
// opened inside a transaction. closing them manually doesn't seem to help
|
||||
dsn += "?parseTime=1&collation=utf8mb4_unicode_ci&interpolateParams=1"
|
||||
s.conn, err = sql.Open("mysql", dsn)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
s.conn.SetMaxIdleConns(12)
|
||||
|
||||
return errors.Err(s.conn.Ping())
|
||||
}
|
||||
|
||||
// AddBlob adds a blob to the database.
|
||||
func (s *SQL) AddBlob(hash string, length int) error {
|
||||
if s.conn == nil {
|
||||
return errors.Err("not connected")
|
||||
}
|
||||
|
||||
_, err := s.insertBlob(hash, length)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *SQL) insertBlob(hash string, length int) (int64, error) {
|
||||
if length <= 0 {
|
||||
return 0, errors.Err("length must be positive")
|
||||
}
|
||||
const isStored = true
|
||||
now := time.Now()
|
||||
args := []interface{}{hash, isStored, length, now}
|
||||
blobID, err := s.exec(
|
||||
"INSERT INTO blob_ (hash, is_stored, length, last_accessed_at) VALUES ("+qt.Qs(len(args))+") ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored)), last_accessed_at=VALUES(last_accessed_at)",
|
||||
args...,
|
||||
)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if blobID == 0 {
|
||||
err = s.conn.QueryRow("SELECT id FROM blob_ WHERE hash = ?", hash).Scan(&blobID)
|
||||
if err != nil {
|
||||
return 0, errors.Err(err)
|
||||
}
|
||||
if blobID == 0 {
|
||||
return 0, errors.Err("blob ID is 0 even after INSERTing and SELECTing")
|
||||
}
|
||||
}
|
||||
|
||||
return blobID, nil
|
||||
}
|
||||
|
||||
// HasBlob checks if the database contains the blob information.
|
||||
func (s *SQL) HasBlob(hash string) (bool, error) {
|
||||
exists, err := s.HasBlobs([]string{hash})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return exists[hash], nil
|
||||
}
|
||||
|
||||
// HasBlobs checks if the database contains the set of blobs and returns a bool map.
|
||||
func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
|
||||
exists, streamsNeedingTouch, err := s.hasBlobs(hashes)
|
||||
_ = s.touch(streamsNeedingTouch)
|
||||
return exists, err
|
||||
}
|
||||
|
||||
func (s *SQL) touch(blobIDs []uint64) error {
|
||||
if len(blobIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
query := "UPDATE blob_ SET last_accessed_at = ? WHERE id IN (" + qt.Qs(len(blobIDs)) + ")"
|
||||
args := make([]interface{}, len(blobIDs)+1)
|
||||
args[0] = time.Now()
|
||||
for i := range blobIDs {
|
||||
args[i+1] = blobIDs[i]
|
||||
}
|
||||
|
||||
startTime := time.Now()
|
||||
_, err := s.exec(query, args...)
|
||||
log.Debugf("blobs access query touched %d blobs and took %s", len(blobIDs), time.Since(startTime))
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
func (s *SQL) hasBlobs(hashes []string) (map[string]bool, []uint64, error) {
|
||||
if s.conn == nil {
|
||||
return nil, nil, errors.Err("not connected")
|
||||
}
|
||||
|
||||
var (
|
||||
hash string
|
||||
blobID uint64
|
||||
lastAccessedAt null.Time
|
||||
)
|
||||
|
||||
var needsTouch []uint64
|
||||
exists := make(map[string]bool)
|
||||
|
||||
touchDeadline := time.Now().AddDate(0, 0, -1) // touch blob if last accessed before this time
|
||||
maxBatchSize := 10000
|
||||
doneIndex := 0
|
||||
|
||||
for len(hashes) > doneIndex {
|
||||
sliceEnd := doneIndex + maxBatchSize
|
||||
if sliceEnd > len(hashes) {
|
||||
sliceEnd = len(hashes)
|
||||
}
|
||||
log.Debugf("getting hashes[%d:%d] of %d", doneIndex, sliceEnd, len(hashes))
|
||||
batch := hashes[doneIndex:sliceEnd]
|
||||
|
||||
// TODO: this query doesn't work for SD blobs, which are not in the stream_blob table
|
||||
|
||||
query := `SELECT hash, id, last_accessed_at
|
||||
FROM blob_
|
||||
WHERE is_stored = ? and hash IN (` + qt.Qs(len(batch)) + `)`
|
||||
args := make([]interface{}, len(batch)+1)
|
||||
args[0] = true
|
||||
for i := range batch {
|
||||
args[i+1] = batch[i]
|
||||
}
|
||||
|
||||
logQuery(query, args...)
|
||||
|
||||
err := func() error {
|
||||
startTime := time.Now()
|
||||
rows, err := s.conn.Query(query, args...)
|
||||
log.Debugf("hashes query took %s", time.Since(startTime))
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer closeRows(rows)
|
||||
|
||||
for rows.Next() {
|
||||
err = rows.Scan(&hash, &blobID, &lastAccessedAt)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
exists[hash] = true
|
||||
if s.TrackAccessTime && (!lastAccessedAt.Valid || lastAccessedAt.Time.Before(touchDeadline)) {
|
||||
needsTouch = append(needsTouch, blobID)
|
||||
}
|
||||
}
|
||||
|
||||
err = rows.Err()
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
doneIndex += len(batch)
|
||||
return nil
|
||||
}()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return exists, needsTouch, nil
|
||||
}
|
||||
|
||||
// Delete will remove the blob from the db
|
||||
func (s *SQL) Delete(hash string) error {
|
||||
_, err := s.exec("UPDATE blob_ set is_stored = ? WHERE hash = ?", 0, hash)
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
// AddSDBlob insert the SD blob and all the content blobs. The content blobs are marked as "not stored",
|
||||
// but they are tracked so reflector knows what it is missing.
|
||||
func (s *SQL) AddSDBlob(sdHash string, sdBlobLength int) error {
|
||||
if s.conn == nil {
|
||||
return errors.Err("not connected")
|
||||
}
|
||||
|
||||
_, err := s.insertBlob(sdHash, sdBlobLength)
|
||||
return err
|
||||
}
|
||||
|
||||
// GetHashRange gets the smallest and biggest hashes in the db
|
||||
func (s *SQL) GetLRUBlobs(maxBlobs int) ([]string, error) {
|
||||
if s.conn == nil {
|
||||
return nil, errors.Err("not connected")
|
||||
}
|
||||
|
||||
query := "SELECT hash from blob_ where is_stored = ? order by last_accessed_at limit ?"
|
||||
const isStored = true
|
||||
logQuery(query, isStored, maxBlobs)
|
||||
rows, err := s.conn.Query(query, isStored, maxBlobs)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
defer closeRows(rows)
|
||||
blobs := make([]string, 0, maxBlobs)
|
||||
for rows.Next() {
|
||||
var hash string
|
||||
err := rows.Scan(&hash)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
blobs = append(blobs, hash)
|
||||
}
|
||||
return blobs, nil
|
||||
}
|
||||
|
||||
func (s *SQL) AllBlobs() ([]string, error) {
|
||||
if s.conn == nil {
|
||||
return nil, errors.Err("not connected")
|
||||
}
|
||||
|
||||
query := "SELECT hash from blob_ where is_stored = ?" //TODO: maybe sorting them makes more sense?
|
||||
const isStored = true
|
||||
logQuery(query, isStored)
|
||||
rows, err := s.conn.Query(query, isStored)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
defer closeRows(rows)
|
||||
totalBlobs, err := s.BlobsCount()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
blobs := make([]string, 0, totalBlobs)
|
||||
for rows.Next() {
|
||||
var hash string
|
||||
err := rows.Scan(&hash)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
blobs = append(blobs, hash)
|
||||
}
|
||||
return blobs, nil
|
||||
}
|
||||
|
||||
func (s *SQL) BlobsCount() (int, error) {
|
||||
if s.conn == nil {
|
||||
return 0, errors.Err("not connected")
|
||||
}
|
||||
|
||||
query := "SELECT count(id) from blob_ where is_stored = ?" //TODO: maybe sorting them makes more sense?
|
||||
const isStored = true
|
||||
logQuery(query, isStored)
|
||||
var count int
|
||||
err := s.conn.QueryRow(query, isStored).Scan(&count)
|
||||
return count, errors.Err(err)
|
||||
}
|
||||
|
||||
func closeRows(rows *sql.Rows) {
|
||||
if rows != nil {
|
||||
err := rows.Close()
|
||||
if err != nil {
|
||||
log.Error("error closing rows: ", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *SQL) exec(query string, args ...interface{}) (int64, error) {
|
||||
logQuery(query, args...)
|
||||
attempt, maxAttempts := 0, 3
|
||||
Retry:
|
||||
attempt++
|
||||
result, err := s.conn.Exec(query, args...)
|
||||
if isLockTimeoutError(err) {
|
||||
if attempt <= maxAttempts {
|
||||
//Error 1205: Lock wait timeout exceeded; try restarting transaction
|
||||
goto Retry
|
||||
}
|
||||
err = errors.Prefix("Lock timeout for query "+query, err)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return 0, errors.Err(err)
|
||||
}
|
||||
|
||||
lastID, err := result.LastInsertId()
|
||||
return lastID, errors.Err(err)
|
||||
}
|
||||
|
||||
func isLockTimeoutError(err error) bool {
|
||||
e, ok := err.(*mysql.MySQLError)
|
||||
return ok && e != nil && e.Number == 1205
|
||||
}
|
||||
|
||||
/* SQL schema
|
||||
|
||||
in prod make sure you use latin1 or utf8 charset, NOT utf8mb4. that's a waste of space.
|
||||
|
||||
CREATE TABLE `blob_` (
|
||||
`id` bigint unsigned NOT NULL AUTO_INCREMENT,
|
||||
`hash` char(96) NOT NULL,
|
||||
`is_stored` tinyint(1) NOT NULL DEFAULT '0',
|
||||
`length` bigint unsigned DEFAULT NULL,
|
||||
`last_accessed_at` datetime DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `id` (`id`),
|
||||
UNIQUE KEY `blob_hash_idx` (`hash`),
|
||||
KEY `blob_last_accessed_idx` (`last_accessed_at`)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=latin1
|
||||
|
||||
*/
|
||||
@@ -1,7 +1,15 @@
|
||||
package main
|
||||
|
||||
import "github.com/lbryio/reflector.go/cmd"
|
||||
import (
|
||||
"github.com/lbryio/reflector.go/cmd"
|
||||
|
||||
"github.com/google/gops/agent"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := agent.Listen(agent.Options{}); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
cmd.Execute()
|
||||
}
|
||||
+56
-1
@@ -1,3 +1,58 @@
|
||||
package meta
|
||||
|
||||
var Version = ""
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
name = "prism-bin"
|
||||
version = "unknown"
|
||||
commit = "unknown"
|
||||
commitLong = "unknown"
|
||||
branch = "unknown"
|
||||
Time = "unknown"
|
||||
BuildTime time.Time
|
||||
)
|
||||
|
||||
// Name returns main application name
|
||||
func Name() string {
|
||||
return name
|
||||
}
|
||||
|
||||
// Version returns current application version
|
||||
func Version() string {
|
||||
return version
|
||||
}
|
||||
|
||||
// FullName returns current app version, commit and build time
|
||||
func FullName() string {
|
||||
return fmt.Sprintf(
|
||||
`Name: %v
|
||||
Version: %v
|
||||
branch: %v
|
||||
commit: %v
|
||||
commit long: %v
|
||||
build date: %v`, Name(), Version(), branch, commit, commitLong, BuildTime.String())
|
||||
}
|
||||
|
||||
func init() {
|
||||
if Time != "" {
|
||||
t, err := strconv.Atoi(Time)
|
||||
if err == nil {
|
||||
BuildTime = time.Unix(int64(t), 0).UTC()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func VersionString() string {
|
||||
var buildTime string
|
||||
if BuildTime.IsZero() {
|
||||
buildTime = "<now>"
|
||||
} else {
|
||||
buildTime = BuildTime.Format(time.RFC3339)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("version %s, built %s", version, buildTime)
|
||||
}
|
||||
-316
@@ -1,316 +0,0 @@
|
||||
package peer
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"crypto/sha512"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/lbry.go/stop"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const (
|
||||
// DefaultPort is the port the peer server listens on if not passed in.
|
||||
DefaultPort = 3333
|
||||
// LbrycrdAddress to be used when paying for data. Not implemented yet.
|
||||
LbrycrdAddress = "bJxKvpD96kaJLriqVajZ7SaQTsWWyrGQct"
|
||||
)
|
||||
|
||||
// Server is an instance of a peer server that houses the listener and store.
|
||||
type Server struct {
|
||||
store store.BlobStore
|
||||
closed bool
|
||||
|
||||
grp *stop.Group
|
||||
}
|
||||
|
||||
// NewServer returns an initialized Server pointer.
|
||||
func NewServer(store store.BlobStore) *Server {
|
||||
return &Server{
|
||||
store: store,
|
||||
grp: stop.New(),
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown gracefully shuts down the peer server.
|
||||
func (s *Server) Shutdown() {
|
||||
log.Debug("shutting down peer server...")
|
||||
s.grp.StopAndWait()
|
||||
log.Debug("peer server stopped")
|
||||
}
|
||||
|
||||
// Start starts the server listener to handle connections.
|
||||
func (s *Server) Start(address string) error {
|
||||
log.Println("peer listening on " + address)
|
||||
l, err := net.Listen("tcp4", address)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
go s.listenForShutdown(l)
|
||||
s.grp.Add(1)
|
||||
go func() {
|
||||
s.listenAndServe(l)
|
||||
s.grp.Done()
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) listenForShutdown(listener net.Listener) {
|
||||
<-s.grp.Ch()
|
||||
s.closed = true
|
||||
err := listener.Close()
|
||||
if err != nil {
|
||||
log.Error("error closing listener for peer server - ", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) listenAndServe(listener net.Listener) {
|
||||
for {
|
||||
conn, err := listener.Accept()
|
||||
if err != nil {
|
||||
if s.closed {
|
||||
return
|
||||
}
|
||||
log.Error(err)
|
||||
} else {
|
||||
s.grp.Add(1)
|
||||
go func() {
|
||||
s.handleConnection(conn)
|
||||
s.grp.Done()
|
||||
}()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) handleConnection(conn net.Conn) {
|
||||
defer func() {
|
||||
if err := conn.Close(); err != nil {
|
||||
log.Error(errors.Prefix("closing peer conn", err))
|
||||
}
|
||||
}()
|
||||
|
||||
timeoutDuration := 5 * time.Second
|
||||
|
||||
for {
|
||||
var request []byte
|
||||
var response []byte
|
||||
|
||||
err := conn.SetReadDeadline(time.Now().Add(timeoutDuration))
|
||||
if err != nil {
|
||||
log.Error(errors.FullTrace(err))
|
||||
}
|
||||
|
||||
request, err = readNextRequest(conn)
|
||||
if err != nil {
|
||||
if err != io.EOF {
|
||||
log.Errorln(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
err = conn.SetReadDeadline(time.Time{})
|
||||
if err != nil {
|
||||
log.Error(errors.FullTrace(err))
|
||||
}
|
||||
|
||||
if strings.Contains(string(request), `"requested_blobs"`) {
|
||||
log.Debugln("received availability request")
|
||||
response, err = s.handleAvailabilityRequest(request)
|
||||
} else if strings.Contains(string(request), `"blob_data_payment_rate"`) {
|
||||
log.Debugln("received rate negotiation request")
|
||||
response, err = s.handlePaymentRateNegotiation(request)
|
||||
} else if strings.Contains(string(request), `"requested_blob"`) {
|
||||
log.Debugln("received blob request")
|
||||
response, err = s.handleBlobRequest(request)
|
||||
} else {
|
||||
log.Errorln("invalid request")
|
||||
spew.Dump(request)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
n, err := conn.Write(response)
|
||||
if err != nil {
|
||||
log.Errorln(err)
|
||||
return
|
||||
} else if n != len(response) {
|
||||
log.Errorln(io.ErrShortWrite)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) handleAvailabilityRequest(data []byte) ([]byte, error) {
|
||||
var request availabilityRequest
|
||||
err := json.Unmarshal(data, &request)
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
|
||||
availableBlobs := []string{}
|
||||
for _, blobHash := range request.RequestedBlobs {
|
||||
exists, err := s.store.Has(blobHash)
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
if exists {
|
||||
availableBlobs = append(availableBlobs, blobHash)
|
||||
}
|
||||
}
|
||||
|
||||
return json.Marshal(availabilityResponse{LbrycrdAddress: LbrycrdAddress, AvailableBlobs: availableBlobs})
|
||||
}
|
||||
|
||||
func (s *Server) handlePaymentRateNegotiation(data []byte) ([]byte, error) {
|
||||
var request paymentRateRequest
|
||||
err := json.Unmarshal(data, &request)
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
|
||||
offerReply := paymentRateAccepted
|
||||
if request.BlobDataPaymentRate < 0 {
|
||||
offerReply = paymentRateTooLow
|
||||
}
|
||||
|
||||
return json.Marshal(paymentRateResponse{BlobDataPaymentRate: offerReply})
|
||||
}
|
||||
|
||||
func (s *Server) handleBlobRequest(data []byte) ([]byte, error) {
|
||||
var request blobRequest
|
||||
err := json.Unmarshal(data, &request)
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
|
||||
log.Println("Sending blob " + request.RequestedBlob[:8])
|
||||
|
||||
blob, err := s.store.Get(request.RequestedBlob)
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
|
||||
response, err := json.Marshal(blobResponse{IncomingBlob: incomingBlob{
|
||||
BlobHash: GetBlobHash(blob),
|
||||
Length: len(blob),
|
||||
}})
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
|
||||
return append(response, blob...), nil
|
||||
}
|
||||
|
||||
func readNextRequest(conn net.Conn) ([]byte, error) {
|
||||
request := make([]byte, 0)
|
||||
eof := false
|
||||
buf := bufio.NewReader(conn)
|
||||
|
||||
for {
|
||||
chunk, err := buf.ReadBytes('}')
|
||||
if err != nil {
|
||||
if err != io.EOF {
|
||||
log.Errorln("read error:", err)
|
||||
return request, err
|
||||
}
|
||||
eof = true
|
||||
}
|
||||
|
||||
//log.Debugln("got", len(chunk), "bytes.")
|
||||
//spew.Dump(chunk)
|
||||
|
||||
if len(chunk) > 0 {
|
||||
request = append(request, chunk...)
|
||||
|
||||
if len(request) > maxRequestSize {
|
||||
return request, errRequestTooLarge
|
||||
}
|
||||
|
||||
// yes, this is how the peer protocol knows when the request finishes
|
||||
if isValidJSON(request) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if eof {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
//log.Debugln("total size:", len(request))
|
||||
//if len(request) > 0 {
|
||||
// spew.Dump(request)
|
||||
//}
|
||||
|
||||
if len(request) == 0 && eof {
|
||||
return []byte{}, io.EOF
|
||||
}
|
||||
|
||||
return request, nil
|
||||
}
|
||||
|
||||
func isValidJSON(b []byte) bool {
|
||||
var r json.RawMessage
|
||||
return json.Unmarshal(b, &r) == nil
|
||||
}
|
||||
|
||||
// GetBlobHash returns the sha512 hash hex encoded string of the blob byte slice.
|
||||
func GetBlobHash(blob []byte) string {
|
||||
hashBytes := sha512.Sum384(blob)
|
||||
return hex.EncodeToString(hashBytes[:])
|
||||
}
|
||||
|
||||
const (
|
||||
maxRequestSize = 64 * (2 ^ 10) // 64kb
|
||||
paymentRateAccepted = "RATE_ACCEPTED"
|
||||
paymentRateTooLow = "RATE_TOO_LOW"
|
||||
//ToDo: paymentRateUnset is not used but exists in the protocol.
|
||||
//paymentRateUnset = "RATE_UNSET"
|
||||
)
|
||||
|
||||
var errRequestTooLarge = errors.Base("request is too large")
|
||||
|
||||
type availabilityRequest struct {
|
||||
LbrycrdAddress bool `json:"lbrycrd_address"`
|
||||
RequestedBlobs []string `json:"requested_blobs"`
|
||||
}
|
||||
|
||||
type availabilityResponse struct {
|
||||
LbrycrdAddress string `json:"lbrycrd_address"`
|
||||
AvailableBlobs []string `json:"available_blobs"`
|
||||
}
|
||||
|
||||
type paymentRateRequest struct {
|
||||
BlobDataPaymentRate float64 `json:"blob_data_payment_rate"`
|
||||
}
|
||||
|
||||
type paymentRateResponse struct {
|
||||
BlobDataPaymentRate string `json:"blob_data_payment_rate"`
|
||||
}
|
||||
|
||||
type blobRequest struct {
|
||||
RequestedBlob string `json:"requested_blob"`
|
||||
}
|
||||
|
||||
type incomingBlob struct {
|
||||
Error string `json:"error,omitempty"`
|
||||
BlobHash string `json:"blob_hash"`
|
||||
Length int `json:"length"`
|
||||
}
|
||||
type blobResponse struct {
|
||||
IncomingBlob incomingBlob `json:"incoming_blob"`
|
||||
}
|
||||
+18
-21
@@ -2,38 +2,35 @@ package prism
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"sync"
|
||||
|
||||
"github.com/lbryio/reflector.go/cluster"
|
||||
"github.com/lbryio/reflector.go/db"
|
||||
"github.com/lbryio/reflector.go/dht"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
"github.com/lbryio/reflector.go/peer"
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
"github.com/lbryio/reflector.go/server/peer"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/lbry.go/stop"
|
||||
"github.com/lbryio/lbry.go/v2/dht"
|
||||
"github.com/lbryio/lbry.go/v2/dht/bits"
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
PeerPort int
|
||||
ReflectorPort int
|
||||
|
||||
DhtAddress string
|
||||
DhtSeedNodes []string
|
||||
|
||||
ClusterPort int
|
||||
ClusterSeedAddr string
|
||||
|
||||
// limit the range of hashes to announce. useful for testing
|
||||
HashRange *bits.Range
|
||||
|
||||
DB *db.SQL
|
||||
Blobs store.BlobStore
|
||||
// limit the range of hashes to announce. useful for testing
|
||||
HashRange *bits.Range
|
||||
DB *db.SQL
|
||||
DhtAddress string
|
||||
ClusterSeedAddr string
|
||||
DhtSeedNodes []string
|
||||
PeerPort int
|
||||
ReflectorPort int
|
||||
ClusterPort int
|
||||
}
|
||||
|
||||
// DefaultConf returns a default config
|
||||
@@ -78,8 +75,8 @@ func New(conf *Config) *Prism {
|
||||
db: conf.DB,
|
||||
dht: d,
|
||||
cluster: c,
|
||||
peer: peer.NewServer(conf.Blobs),
|
||||
reflector: reflector.NewServer(conf.Blobs),
|
||||
peer: peer.NewServer(conf.Blobs, fmt.Sprintf(":%d", conf.PeerPort)),
|
||||
reflector: reflector.NewIngestionServer(conf.Blobs),
|
||||
|
||||
grp: stop.New(),
|
||||
}
|
||||
@@ -107,7 +104,7 @@ func (p *Prism) Start() error {
|
||||
return errors.Err("blobs required in conf")
|
||||
}
|
||||
|
||||
err = p.peer.Start(":" + strconv.Itoa(p.conf.PeerPort))
|
||||
err = p.peer.Start()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
+2
-1
@@ -4,8 +4,9 @@ import (
|
||||
"math/big"
|
||||
"testing"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/dht/bits"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/lbryio/reflector.go/dht/bits"
|
||||
)
|
||||
|
||||
func TestAnnounceRange(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
package publish
|
||||
|
||||
import "strings"
|
||||
|
||||
func guessMimeType(ext string) (string, string) {
|
||||
if ext == "" {
|
||||
return "application/octet-stream", "binary"
|
||||
}
|
||||
|
||||
ext = strings.ToLower(strings.TrimLeft(strings.TrimSpace(ext), "."))
|
||||
|
||||
types := map[string]struct{ mime, t string }{
|
||||
"a": {"application/octet-stream", "binary"},
|
||||
"ai": {"application/postscript", "image"},
|
||||
"aif": {"audio/x-aiff", "audio"},
|
||||
"aifc": {"audio/x-aiff", "audio"},
|
||||
"aiff": {"audio/x-aiff", "audio"},
|
||||
"au": {"audio/basic", "audio"},
|
||||
"avi": {"video/x-msvideo", "video"},
|
||||
"bat": {"text/plain", "document"},
|
||||
"bcpio": {"application/x-bcpio", "binary"},
|
||||
"bin": {"application/octet-stream", "binary"},
|
||||
"bmp": {"image/bmp", "image"},
|
||||
"c": {"text/plain", "document"},
|
||||
"cdf": {"application/x-netcdf", "binary"},
|
||||
"cpio": {"application/x-cpio", "binary"},
|
||||
"csh": {"application/x-csh", "binary"},
|
||||
"css": {"text/css", "document"},
|
||||
"csv": {"text/csv", "document"},
|
||||
"dll": {"application/octet-stream", "binary"},
|
||||
"doc": {"application/msword", "document"},
|
||||
"dot": {"application/msword", "document"},
|
||||
"dvi": {"application/x-dvi", "binary"},
|
||||
"eml": {"message/rfc822", "document"},
|
||||
"eps": {"application/postscript", "document"},
|
||||
"epub": {"application/epub+zip", "document"},
|
||||
"etx": {"text/x-setext", "document"},
|
||||
"exe": {"application/octet-stream", "binary"},
|
||||
"gif": {"image/gif", "image"},
|
||||
"gtar": {"application/x-gtar", "binary"},
|
||||
"h": {"text/plain", "document"},
|
||||
"hdf": {"application/x-hdf", "binary"},
|
||||
"htm": {"text/html", "document"},
|
||||
"html": {"text/html", "document"},
|
||||
"ico": {"image/vnd.microsoft.icon", "image"},
|
||||
"ief": {"image/ief", "image"},
|
||||
"iges": {"model/iges", "model"},
|
||||
"jpe": {"image/jpeg", "image"},
|
||||
"jpeg": {"image/jpeg", "image"},
|
||||
"jpg": {"image/jpeg", "image"},
|
||||
"js": {"application/javascript", "document"},
|
||||
"json": {"application/json", "document"},
|
||||
"ksh": {"text/plain", "document"},
|
||||
"latex": {"application/x-latex", "binary"},
|
||||
"m1v": {"video/mpeg", "video"},
|
||||
"m3u": {"application/vnd.apple.mpegurl", "audio"},
|
||||
"m3u8": {"application/vnd.apple.mpegurl", "audio"},
|
||||
"man": {"application/x-troff-man", "document"},
|
||||
"markdown": {"text/markdown", "document"},
|
||||
"md": {"text/markdown", "document"},
|
||||
"me": {"application/x-troff-me", "binary"},
|
||||
"mht": {"message/rfc822", "document"},
|
||||
"mhtml": {"message/rfc822", "document"},
|
||||
"mif": {"application/x-mif", "binary"},
|
||||
"mov": {"video/quicktime", "video"},
|
||||
"movie": {"video/x-sgi-movie", "video"},
|
||||
"mp2": {"audio/mpeg", "audio"},
|
||||
"mp3": {"audio/mpeg", "audio"},
|
||||
"mp4": {"video/mp4", "video"},
|
||||
"mpa": {"video/mpeg", "video"},
|
||||
"mpe": {"video/mpeg", "video"},
|
||||
"mpeg": {"video/mpeg", "video"},
|
||||
"mpg": {"video/mpeg", "video"},
|
||||
"ms": {"application/x-troff-ms", "binary"},
|
||||
"nc": {"application/x-netcdf", "binary"},
|
||||
"nws": {"message/rfc822", "document"},
|
||||
"o": {"application/octet-stream", "binary"},
|
||||
"obj": {"application/octet-stream", "model"},
|
||||
"oda": {"application/oda", "binary"},
|
||||
"p12": {"application/x-pkcs12", "binary"},
|
||||
"p7c": {"application/pkcs7-mime", "binary"},
|
||||
"pbm": {"image/x-portable-bitmap", "image"},
|
||||
"pdf": {"application/pdf", "document"},
|
||||
"pfx": {"application/x-pkcs12", "binary"},
|
||||
"pgm": {"image/x-portable-graymap", "image"},
|
||||
"pl": {"text/plain", "document"},
|
||||
"png": {"image/png", "image"},
|
||||
"pnm": {"image/x-portable-anymap", "image"},
|
||||
"pot": {"application/vnd.ms-powerpoint", "document"},
|
||||
"ppa": {"application/vnd.ms-powerpoint", "document"},
|
||||
"ppm": {"image/x-portable-pixmap", "image"},
|
||||
"pps": {"application/vnd.ms-powerpoint", "document"},
|
||||
"ppt": {"application/vnd.ms-powerpoint", "document"},
|
||||
"ps": {"application/postscript", "document"},
|
||||
"pwz": {"application/vnd.ms-powerpoint", "document"},
|
||||
"py": {"text/x-python", "document"},
|
||||
"pyc": {"application/x-python-code", "binary"},
|
||||
"pyo": {"application/x-python-code", "binary"},
|
||||
"qt": {"video/quicktime", "video"},
|
||||
"ra": {"audio/x-pn-realaudio", "audio"},
|
||||
"ram": {"application/x-pn-realaudio", "audio"},
|
||||
"ras": {"image/x-cmu-raster", "image"},
|
||||
"rdf": {"application/xml", "binary"},
|
||||
"rgb": {"image/x-rgb", "image"},
|
||||
"roff": {"application/x-troff", "binary"},
|
||||
"rtx": {"text/richtext", "document"},
|
||||
"sgm": {"text/x-sgml", "document"},
|
||||
"sgml": {"text/x-sgml", "document"},
|
||||
"sh": {"application/x-sh", "document"},
|
||||
"shar": {"application/x-shar", "binary"},
|
||||
"snd": {"audio/basic", "audio"},
|
||||
"so": {"application/octet-stream", "binary"},
|
||||
"src": {"application/x-wais-source", "binary"},
|
||||
"stl": {"model/stl", "model"},
|
||||
"sv4cpio": {"application/x-sv4cpio", "binary"},
|
||||
"sv4crc": {"application/x-sv4crc", "binary"},
|
||||
"svg": {"image/svg+xml", "image"},
|
||||
"swf": {"application/x-shockwave-flash", "binary"},
|
||||
"t": {"application/x-troff", "binary"},
|
||||
"tar": {"application/x-tar", "binary"},
|
||||
"tcl": {"application/x-tcl", "binary"},
|
||||
"tex": {"application/x-tex", "binary"},
|
||||
"texi": {"application/x-texinfo", "binary"},
|
||||
"texinfo": {"application/x-texinfo", "binary"},
|
||||
"tif": {"image/tiff", "image"},
|
||||
"tiff": {"image/tiff", "image"},
|
||||
"tr": {"application/x-troff", "binary"},
|
||||
"tsv": {"text/tab-separated-values", "document"},
|
||||
"txt": {"text/plain", "document"},
|
||||
"ustar": {"application/x-ustar", "binary"},
|
||||
"vcf": {"text/x-vcard", "document"},
|
||||
"wav": {"audio/x-wav", "audio"},
|
||||
"webm": {"video/webm", "video"},
|
||||
"wiz": {"application/msword", "document"},
|
||||
"wsdl": {"application/xml", "document"},
|
||||
"xbm": {"image/x-xbitmap", "image"},
|
||||
"xlb": {"application/vnd.ms-excel", "document"},
|
||||
"xls": {"application/vnd.ms-excel", "document"},
|
||||
"xml": {"text/xml", "document"},
|
||||
"xpdl": {"application/xml", "document"},
|
||||
"xpm": {"image/x-xpixmap", "image"},
|
||||
"xsl": {"application/xml", "document"},
|
||||
"xwd": {"image/x-xwindowdump", "image"},
|
||||
"zip": {"application/zip", "binary"},
|
||||
|
||||
// These are non-standard types, commonly found in the wild.
|
||||
"cbr": {"application/vnd.comicbook-rar", "document"},
|
||||
"cbz": {"application/vnd.comicbook+zip", "document"},
|
||||
"flac": {"audio/flac", "audio"},
|
||||
"lbry": {"application/x-ext-lbry", "document"},
|
||||
"m4v": {"video/m4v", "video"},
|
||||
"mid": {"audio/midi", "audio"},
|
||||
"midi": {"audio/midi", "audio"},
|
||||
"mkv": {"video/x-matroska", "video"},
|
||||
"mobi": {"application/x-mobipocket-ebook", "document"},
|
||||
"oga": {"audio/ogg", "audio"},
|
||||
"ogv": {"video/ogg", "video"},
|
||||
"pct": {"image/pict", "image"},
|
||||
"pic": {"image/pict", "image"},
|
||||
"pict": {"image/pict", "image"},
|
||||
"prc": {"application/x-mobipocket-ebook", "document"},
|
||||
"rtf": {"application/rtf", "document"},
|
||||
"xul": {"text/xul", "document"},
|
||||
|
||||
// microsoft is special and has its own "standard"
|
||||
// https://docs.microsoft.com/en-us/windows/desktop/wmp/file-name-extensions
|
||||
"wmv": {"video/x-ms-wmv", "video"},
|
||||
}
|
||||
|
||||
if data, ok := types[ext]; ok {
|
||||
return data.mime, data.t
|
||||
}
|
||||
return "application/x-ext-" + ext, "binary"
|
||||
}
|
||||
@@ -0,0 +1,291 @@
|
||||
package publish
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/lbrycrd"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
pb "github.com/lbryio/types/v2/go"
|
||||
|
||||
"github.com/btcsuite/btcd/btcjson"
|
||||
"github.com/btcsuite/btcd/chaincfg/chainhash"
|
||||
"github.com/btcsuite/btcd/txscript"
|
||||
"github.com/btcsuite/btcd/wire"
|
||||
"github.com/btcsuite/btcutil"
|
||||
"github.com/golang/protobuf/proto"
|
||||
)
|
||||
|
||||
/* TODO:
|
||||
import cert from wallet
|
||||
get all utxos from chainquery
|
||||
create transaction
|
||||
sign it with the channel
|
||||
track state of utxos across publishes from this channel so that we can just do one query to get utxos
|
||||
prioritize only confirmed utxos
|
||||
|
||||
Handling all the issues we handle currently with lbrynet:
|
||||
"Couldn't find private key for id",
|
||||
"You already have a stream claim published under the name",
|
||||
"Cannot publish using channel",
|
||||
"txn-mempool-conflict",
|
||||
"too-long-mempool-chain",
|
||||
"Missing inputs",
|
||||
"Not enough funds to cover this transaction",
|
||||
*/
|
||||
|
||||
type Details struct {
|
||||
Title string
|
||||
Description string
|
||||
Author string
|
||||
Tags []string
|
||||
ReleaseTime int64
|
||||
}
|
||||
|
||||
func Publish(client *lbrycrd.Client, path, name, address string, details Details, reflectorAddress string) (*wire.MsgTx, *chainhash.Hash, error) {
|
||||
if name == "" {
|
||||
return nil, nil, errors.Err("name required")
|
||||
}
|
||||
|
||||
//TODO: sign claim if publishing into channel
|
||||
|
||||
addr, err := btcutil.DecodeAddress(address, &lbrycrd.MainNetParams)
|
||||
if errors.Is(err, btcutil.ErrUnknownAddressType) {
|
||||
return nil, nil, errors.Err(`unknown address type. here's what you need to make this work:
|
||||
- deprecatedrpc=validateaddress" and "deprecatedrpc=signrawtransaction" in your lbrycrd.conf
|
||||
- github.com/btcsuite/btcd pinned to hash 306aecffea32
|
||||
- github.com/btcsuite/btcutil pinned to 4c204d697803
|
||||
- github.com/lbryio/lbry.go/v2 (make sure you have v2 at the end)`)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
amount := 0.01
|
||||
changeAddr := addr // TODO: fix this? or maybe its fine?
|
||||
tx, err := baseTx(client, amount, changeAddr)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
st, stPB, err := makeStream(path)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
stPB.Author = details.Author
|
||||
stPB.ReleaseTime = details.ReleaseTime
|
||||
|
||||
claim := &pb.Claim{
|
||||
Title: details.Title,
|
||||
Description: details.Description,
|
||||
Type: &pb.Claim_Stream{Stream: stPB},
|
||||
}
|
||||
|
||||
err = addClaimToTx(tx, claim, name, amount, addr)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// sign and send
|
||||
signedTx, allInputsSigned, err := client.SignRawTransaction(tx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if !allInputsSigned {
|
||||
return nil, nil, errors.Err("not all inputs for the tx could be signed")
|
||||
}
|
||||
|
||||
err = reflect(st, reflectorAddress)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
txid, err := client.SendRawTransaction(signedTx, false)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return signedTx, txid, nil
|
||||
}
|
||||
|
||||
// TODO: lots of assumptions. hardcoded values need to be passed in or calculated
|
||||
func baseTx(client *lbrycrd.Client, amount float64, changeAddress btcutil.Address) (*wire.MsgTx, error) {
|
||||
txFee := 0.0002 // TODO: estimate this better?
|
||||
|
||||
inputs, total, err := coinChooser(client, amount+txFee)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
change := total - amount - txFee
|
||||
|
||||
// create base raw tx
|
||||
addresses := make(map[btcutil.Address]btcutil.Amount)
|
||||
//changeAddr, err := client.GetNewAddress("")
|
||||
changeAmount, err := btcutil.NewAmount(change)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
addresses[changeAddress] = changeAmount
|
||||
lockTime := int64(0)
|
||||
return client.CreateRawTransaction(inputs, addresses, &lockTime)
|
||||
}
|
||||
|
||||
func coinChooser(client *lbrycrd.Client, amount float64) ([]btcjson.TransactionInput, float64, error) {
|
||||
utxos, err := client.ListUnspentMin(1)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
sort.Slice(utxos, func(i, j int) bool { return utxos[i].Amount < utxos[j].Amount })
|
||||
|
||||
var utxo btcjson.ListUnspentResult
|
||||
for _, u := range utxos {
|
||||
if u.Spendable && u.Amount >= amount {
|
||||
utxo = u
|
||||
break
|
||||
}
|
||||
}
|
||||
if utxo.TxID == "" {
|
||||
return nil, 0, errors.Err("not enough utxos to create tx")
|
||||
}
|
||||
|
||||
return []btcjson.TransactionInput{{Txid: utxo.TxID, Vout: utxo.Vout}}, utxo.Amount, nil
|
||||
}
|
||||
|
||||
func addClaimToTx(tx *wire.MsgTx, claim *pb.Claim, name string, amount float64, claimAddress btcutil.Address) error {
|
||||
claimBytes, err := proto.Marshal(claim)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
claimBytes = append([]byte{0}, claimBytes...) // version 0 = no channel sig
|
||||
|
||||
amt, err := btcutil.NewAmount(amount)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
script, err := getClaimPayoutScript(name, claimBytes, claimAddress)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tx.AddTxOut(wire.NewTxOut(int64(amt), script))
|
||||
return nil
|
||||
}
|
||||
|
||||
func Decode(client *lbrycrd.Client, tx *wire.MsgTx) (string, error) {
|
||||
buf := bytes.NewBuffer(make([]byte, 0, tx.SerializeSize()))
|
||||
if err := tx.Serialize(buf); err != nil {
|
||||
return "", errors.Err(err)
|
||||
}
|
||||
//txHex := hex.EncodeToString(buf.Bytes())
|
||||
//spew.Dump(txHex)
|
||||
decoded, err := client.DecodeRawTransaction(buf.Bytes())
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
data, err := json.MarshalIndent(decoded, "", " ")
|
||||
return string(data), err
|
||||
}
|
||||
|
||||
func reflect(st stream.Stream, reflectorAddress string) error {
|
||||
// upload blobs to reflector
|
||||
c := reflector.Client{}
|
||||
err := c.Connect(reflectorAddress)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
for i, b := range st {
|
||||
if i == 0 {
|
||||
err = c.SendSDBlob(b)
|
||||
} else {
|
||||
err = c.SendBlob(b)
|
||||
}
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func makeStream(path string) (stream.Stream, *pb.Stream, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, nil, errors.Err(err)
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
enc := stream.NewEncoder(file)
|
||||
|
||||
s, err := enc.Stream()
|
||||
if err != nil {
|
||||
return nil, nil, errors.Err(err)
|
||||
}
|
||||
|
||||
streamProto := &pb.Stream{
|
||||
Source: &pb.Source{
|
||||
SdHash: enc.SDBlob().Hash(),
|
||||
Name: filepath.Base(file.Name()),
|
||||
Size: uint64(enc.SourceLen()),
|
||||
Hash: enc.SourceHash(),
|
||||
},
|
||||
}
|
||||
|
||||
mimeType, category := guessMimeType(filepath.Ext(file.Name()))
|
||||
streamProto.Source.MediaType = mimeType
|
||||
|
||||
switch category {
|
||||
case "video":
|
||||
//t, err := streamVideoMetadata(path)
|
||||
//if err != nil {
|
||||
// return nil, nil, err
|
||||
//}
|
||||
streamProto.Type = &pb.Stream_Video{}
|
||||
case "audio":
|
||||
streamProto.Type = &pb.Stream_Audio{}
|
||||
case "image":
|
||||
streamProto.Type = &pb.Stream_Image{}
|
||||
}
|
||||
|
||||
return s, streamProto, nil
|
||||
}
|
||||
|
||||
func getClaimPayoutScript(name string, value []byte, address btcutil.Address) ([]byte, error) {
|
||||
//OP_CLAIM_NAME <name> <value> OP_2DROP OP_DROP OP_DUP OP_HASH160 <address> OP_EQUALVERIFY OP_CHECKSIG
|
||||
|
||||
pkscript, err := txscript.PayToAddrScript(address)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
return txscript.NewScriptBuilder().
|
||||
AddOp(txscript.OP_NOP6). //OP_CLAIM_NAME
|
||||
AddData([]byte(name)). //<name>
|
||||
AddData(value). //<value>
|
||||
AddOp(txscript.OP_2DROP). //OP_2DROP
|
||||
AddOp(txscript.OP_DROP). //OP_DROP
|
||||
AddOps(pkscript). //OP_DUP OP_HASH160 <address> OP_EQUALVERIFY OP_CHECKSIG
|
||||
Script()
|
||||
}
|
||||
|
||||
//func streamVideoMetadata(path string) (*pb.Stream_Video, error) {
|
||||
// mi, err := mediainfo.GetMediaInfo(path)
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
// return &pb.Stream_Video{
|
||||
// Video: &pb.Video{
|
||||
// Duration: uint32(mi.General.Duration / 1000),
|
||||
// Height: uint32(mi.Video.Height),
|
||||
// Width: uint32(mi.Video.Width),
|
||||
// },
|
||||
// }, nil
|
||||
//}
|
||||
@@ -0,0 +1,59 @@
|
||||
package publish
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
)
|
||||
|
||||
func LoadWallet(r io.Reader) (WalletFile, error) {
|
||||
var w WalletFile
|
||||
err := json.NewDecoder(r).Decode(&w)
|
||||
return w, err
|
||||
}
|
||||
|
||||
type WalletFile struct {
|
||||
Name string `json:"name"`
|
||||
Accounts []Account `json:"accounts"`
|
||||
Preferences WalletPrefs `json:"preferences"`
|
||||
Version int `json:"version"`
|
||||
}
|
||||
|
||||
type Account struct {
|
||||
Certificates map[string]string `json:"certificates"`
|
||||
Ledger string `json:"ledger"`
|
||||
Name string `json:"name"`
|
||||
PrivateKey string `json:"private_key"`
|
||||
PublicKey string `json:"public_key"`
|
||||
Seed string `json:"seed"`
|
||||
AddressGenerator AddressGenerator `json:"address_generator"`
|
||||
ModifiedOn float64 `json:"modified_on"`
|
||||
Encrypted bool `json:"encrypted"`
|
||||
}
|
||||
|
||||
type AddressGenerator struct {
|
||||
Name string `json:"name"`
|
||||
Change AddressGenParams `json:"change"` // should "change" and "receiving" be replaced with a map[string]AddressGenParams?
|
||||
Receiving AddressGenParams `json:"receiving"`
|
||||
}
|
||||
|
||||
type AddressGenParams struct {
|
||||
Gap int `json:"gap"`
|
||||
MaximumUsesPerAddress int `json:"maximum_uses_per_address"`
|
||||
}
|
||||
|
||||
type WalletPrefs struct {
|
||||
Shared struct {
|
||||
Value struct {
|
||||
Type string `json:"type"`
|
||||
Version string `json:"version"`
|
||||
Value struct {
|
||||
Blocked []interface{} `json:"blocked"`
|
||||
Subscriptions []string `json:"subscriptions"`
|
||||
Tags []string `json:"tags"`
|
||||
AppWelcomeVersion int `json:"app_welcome_version"`
|
||||
Sharing3P bool `json:"sharing_3P"`
|
||||
} `json:"value"`
|
||||
} `json:"value"`
|
||||
Ts float64 `json:"ts"`
|
||||
} `json:"shared"`
|
||||
}
|
||||
@@ -1,40 +1,189 @@
|
||||
# Reflector
|
||||
|
||||
A reflector cluster to accept LBRY content for hosting en masse, rehost the content, and make money on data fees (TODO).
|
||||
This code includes Go implementations of the LBRY peer protocol, reflector protocol, and DHT.
|
||||
Production-ready blob reflection, distribution, and caching for Odysee.
|
||||
|
||||
## Installation
|
||||
This repository provides the components used in production:
|
||||
- Reflector ingestion server (command name: `reflector`)
|
||||
- Blob cache/edge server (`blobcache`)
|
||||
- Uploader to object storage (`upload`)
|
||||
|
||||
coming soon
|
||||
Other commands exist in the tree for historical/legacy reasons and are not supported.
|
||||
|
||||
## Usage
|
||||
## How it works (at a glance)
|
||||
- Ingestion (reflector): accepts uploaded blobs, persists them to object storage (e.g., S3/Wasabi) and tracks state in MySQL.
|
||||
- Distribution: serves blobs over HTTP/HTTP3/Peer. Blobcaches can be deployed in front of the origin to reduce latency and egress.
|
||||
- Caching (blobcache): layered disk caches backed by HTTP(S) origins (e.g., S3 endpoints), with optional local DB metadata for capacity/eviction.
|
||||
|
||||
coming soon
|
||||
All services are started by the `prism` binary and are configured via YAML files loaded from a configuration directory.
|
||||
|
||||
## Running from Source
|
||||
## Supported commands
|
||||
The following are the only supported commands for production use:
|
||||
|
||||
This project requires [Go v1.10](https://golang.org/doc/install) or higher.
|
||||
- Reflector ingestion: `prism reflector`
|
||||
- Flags: `--receiver-port` (default 5566), `--metrics-port` (default 2112), `--disable-blocklist`
|
||||
- Loads `reflector.yaml` from the config directory.
|
||||
|
||||
```
|
||||
go get -u github.com/lbryio/reflector.go
|
||||
cd "$(go env GOPATH)/src/github.com/lbryio/reflector.go"
|
||||
make
|
||||
./prism
|
||||
- Blob cache: `prism blobcache`
|
||||
- Flags: `--metrics-port` (default 2112), `--disable-blocklist`
|
||||
- Loads `blobcache.yaml` from the config directory.
|
||||
|
||||
- Uploader: `prism upload PATH`
|
||||
- Flags: `--workers`, `--skipExistsCheck`, `--deleteBlobsAfterUpload`
|
||||
- Loads `upload.yaml` from the config directory.
|
||||
|
||||
Global flag for all commands:
|
||||
- `--conf-dir` (default `./`): directory containing YAML config files.
|
||||
|
||||
## Configuration
|
||||
Configuration is per-command. The loader reads `<command>.yaml` from `--conf-dir`.
|
||||
|
||||
Common sections:
|
||||
- `servers`: enables HTTP/HTTP3/Peer servers. Keys: `http`, `http3`, `peer`. Each accepts:
|
||||
- `port` (int)
|
||||
- `max_concurrent_requests` (int, http/http3)
|
||||
- `edge_token` (string, http)
|
||||
- `address` (string, optional; bind address, omit for all interfaces)
|
||||
- `store`: defines the storage topology using composable stores. Frequently used:
|
||||
- `proxied-s3`: production pattern with a `writer` (DB-backed -> S3/multiwriter) and a `reader` (caching -> disk + HTTP origins).
|
||||
- `caching`: layered cache with a `cache` (often `db_backed` -> `disk`) and an `origin` chain (`http`, `http3`, or `ittt` fan-in).
|
||||
- `s3`, `disk`, `multiwriter`, `db_backed`, `http`, `http3`, `peer`, `upstream` are also available building blocks.
|
||||
|
||||
### Minimal examples
|
||||
Reflector – conf-dir contains `reflector.yaml`:
|
||||
```yaml
|
||||
servers:
|
||||
http:
|
||||
port: 5569
|
||||
max_concurrent_requests: 200
|
||||
http3:
|
||||
port: 5568
|
||||
max_concurrent_requests: 200
|
||||
peer:
|
||||
port: 5567
|
||||
store:
|
||||
proxied-s3:
|
||||
name: s3_read_proxy
|
||||
writer:
|
||||
db_backed:
|
||||
user: reflector
|
||||
password: reflector
|
||||
database: reflector
|
||||
host: localhost
|
||||
port: 3306
|
||||
access_tracking: 1
|
||||
soft_deletes: true
|
||||
store:
|
||||
s3:
|
||||
name: primary
|
||||
aws_id: YOUR_KEY
|
||||
aws_secret: YOUR_SECRET
|
||||
region: us-east-1
|
||||
bucket: blobs-bucket
|
||||
endpoint: https://s3.yourendpoint.tv
|
||||
reader:
|
||||
caching:
|
||||
cache:
|
||||
disk:
|
||||
name: local_cache
|
||||
mount_point: /mnt/reflector/cache
|
||||
sharding_size: 2
|
||||
origin:
|
||||
http:
|
||||
endpoint: https://s3.yourendpoint.tv/blobs-bucket/
|
||||
sharding_size: 4
|
||||
```
|
||||
|
||||
## Contributing
|
||||
Blobcache – conf-dir contains `blobcache.yaml`:
|
||||
```yaml
|
||||
servers:
|
||||
http:
|
||||
port: 5569
|
||||
max_concurrent_requests: 200
|
||||
http3:
|
||||
port: 5568
|
||||
max_concurrent_requests: 200
|
||||
peer:
|
||||
port: 5567
|
||||
store:
|
||||
caching:
|
||||
cache:
|
||||
db_backed:
|
||||
user: reflector
|
||||
password: reflector
|
||||
database: reflector
|
||||
host: localhost
|
||||
port: 3306
|
||||
has_cap: true
|
||||
max_size: 500GB
|
||||
store:
|
||||
disk:
|
||||
name: blobcache
|
||||
mount_point: /mnt/blobcache/cache
|
||||
sharding_size: 2
|
||||
origin:
|
||||
http:
|
||||
endpoint: https://s3.yourendpoint.tv/blobs-bucket/
|
||||
sharding_size: 4
|
||||
```
|
||||
|
||||
coming soon
|
||||
Uploader – conf-dir contains `upload.yaml` (points to the same writer/backend as reflector):
|
||||
```yaml
|
||||
database:
|
||||
user: reflector
|
||||
password: reflector
|
||||
database: reflector
|
||||
host: localhost
|
||||
port: 3306
|
||||
store:
|
||||
proxied-s3:
|
||||
writer:
|
||||
db_backed:
|
||||
user: reflector
|
||||
password: reflector
|
||||
database: reflector
|
||||
host: localhost
|
||||
port: 3306
|
||||
store:
|
||||
s3:
|
||||
aws_id: YOUR_KEY
|
||||
aws_secret: YOUR_SECRET
|
||||
region: us-east-1
|
||||
bucket: blobs-bucket
|
||||
endpoint: https://s3.yourendpoint.tv
|
||||
```
|
||||
|
||||
## License
|
||||
## Quick start
|
||||
1) Build
|
||||
- Requires Go 1.25+
|
||||
- `make` (binaries in `dist/<platform>/prism-bin`)
|
||||
|
||||
This project is MIT licensed.
|
||||
2) Run a local blobcache
|
||||
```bash
|
||||
./dist/linux_amd64/prism-bin --conf-dir=./ blobcache
|
||||
```
|
||||
Place your `blobcache.yaml` in the `--conf-dir` directory.
|
||||
|
||||
3) Run reflector ingestion
|
||||
```bash
|
||||
./dist/linux_amd64/prism-bin --conf-dir=./ reflector --receiver-port=5566 --metrics-port=2112
|
||||
```
|
||||
|
||||
4) Upload blobs
|
||||
```bash
|
||||
./dist/linux_amd64/prism-bin --conf-dir=./ upload /path/to/blobs \
|
||||
--workers=4 --skipExistsCheck
|
||||
```
|
||||
|
||||
## Notes
|
||||
- Only reflector, blobcache, and upload are supported. All other commands are legacy and may be removed in the future.
|
||||
- Metrics are exposed on the configured `--metrics-port` at `/metrics` (Prometheus format).
|
||||
- MySQL is required when using DB-backed stores (e.g., ingestion writer, capacity-aware caches).
|
||||
|
||||
## Security
|
||||
If you discover a security issue, please email security@lbry.com. Our PGP key is available at https://lbry.com/faq/pgp-key.
|
||||
|
||||
We take security seriously. Please contact security@lbry.io regarding any security issues.
|
||||
Our PGP key is [here](https://keybase.io/lbry/key.asc) if you need it.
|
||||
## License
|
||||
MIT License. See LICENSE.
|
||||
|
||||
## Contact
|
||||
|
||||
The primary contact for this project is [@lyoshenka](https://github.com/lyoshenka) (grin@lbry.io)
|
||||
The primary contact for this project is [@Nikooo777](https://github.com/Nikooo777)
|
||||
@@ -0,0 +1,159 @@
|
||||
package reflector
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
"github.com/lbryio/reflector.go/wallet"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const blocklistURL = "https://api.lbry.com/file/list_blocked"
|
||||
|
||||
func (s *Server) enableBlocklist(b store.Blocklister) {
|
||||
walletServers := []string{
|
||||
"a-hub1.odysee.com:50001",
|
||||
"b-hub1.odysee.com:50001",
|
||||
"c-hub1.odysee.com:50001",
|
||||
"s-hub1.odysee.com:50001",
|
||||
}
|
||||
|
||||
updateBlocklist(b, walletServers, s.grp.Ch())
|
||||
t := time.NewTicker(12 * time.Hour)
|
||||
for {
|
||||
select {
|
||||
case <-s.grp.Ch():
|
||||
return
|
||||
case <-t.C:
|
||||
updateBlocklist(b, walletServers, s.grp.Ch())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func updateBlocklist(b store.Blocklister, walletServers []string, stopper stop.Chan) {
|
||||
log.Debugf("blocklist update starting")
|
||||
values, err := blockedSdHashes(walletServers, stopper)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
for name, v := range values {
|
||||
if v.Err != nil {
|
||||
log.Error(errors.FullTrace(errors.Err("blocklist: %s: %s", name, v.Err)))
|
||||
continue
|
||||
}
|
||||
|
||||
err = b.Block(v.Value)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
}
|
||||
log.Debugf("blocklist update done")
|
||||
}
|
||||
|
||||
func blockedSdHashes(walletServers []string, stopper stop.Chan) (map[string]valOrErr, error) {
|
||||
client := http.Client{Timeout: 1 * time.Second}
|
||||
resp, err := client.Get(blocklistURL)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
defer func() {
|
||||
closeErr := resp.Body.Close()
|
||||
if closeErr != nil {
|
||||
log.Errorln(errors.Err(closeErr))
|
||||
}
|
||||
}()
|
||||
|
||||
var r struct {
|
||||
Error string `json:"error"`
|
||||
Data struct {
|
||||
Outpoints []string `json:"outpoints"`
|
||||
} `json:"data"`
|
||||
Success bool `json:"success"`
|
||||
}
|
||||
|
||||
if err = json.NewDecoder(resp.Body).Decode(&r); err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
if !r.Success {
|
||||
return nil, errors.Prefix("list_blocked API call", r.Error)
|
||||
}
|
||||
|
||||
return sdHashesForOutpoints(walletServers, r.Data.Outpoints, stopper)
|
||||
}
|
||||
|
||||
type valOrErr struct {
|
||||
Err error
|
||||
Value string
|
||||
}
|
||||
|
||||
// sdHashesForOutpoints queries wallet server for the sd hashes in a given outpoints
|
||||
func sdHashesForOutpoints(walletServers, outpoints []string, stopper stop.Chan) (map[string]valOrErr, error) {
|
||||
values := make(map[string]valOrErr)
|
||||
|
||||
node := wallet.NewNode()
|
||||
err := node.Connect(walletServers, nil)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
done := make(chan bool)
|
||||
metrics.RoutinesQueue.WithLabelValues("reflector", "sdhashesforoutput").Inc()
|
||||
go func() {
|
||||
defer metrics.RoutinesQueue.WithLabelValues("reflector", "sdhashesforoutput").Dec()
|
||||
select {
|
||||
case <-done:
|
||||
case <-stopper:
|
||||
}
|
||||
node.Shutdown()
|
||||
}()
|
||||
|
||||
OutpointLoop:
|
||||
for _, outpoint := range outpoints {
|
||||
select {
|
||||
case <-stopper:
|
||||
break OutpointLoop
|
||||
default:
|
||||
}
|
||||
|
||||
parts := strings.Split(outpoint, ":")
|
||||
if len(parts) != 2 {
|
||||
values[outpoint] = valOrErr{Err: errors.Err("invalid outpoint format")}
|
||||
continue
|
||||
}
|
||||
|
||||
nout, err := strconv.Atoi(parts[1])
|
||||
if err != nil {
|
||||
values[outpoint] = valOrErr{Err: errors.Prefix("invalid nout", err)}
|
||||
continue
|
||||
}
|
||||
|
||||
claim, err := node.GetClaimInTx(parts[0], nout)
|
||||
if err != nil {
|
||||
values[outpoint] = valOrErr{Err: err}
|
||||
continue
|
||||
}
|
||||
|
||||
hash := hex.EncodeToString(claim.GetStream().GetSource().GetSdHash())
|
||||
values[outpoint] = valOrErr{Value: hash, Err: nil}
|
||||
}
|
||||
|
||||
select {
|
||||
case done <- true:
|
||||
default: // in case of race where stopper got stopped right after loop finished
|
||||
}
|
||||
|
||||
return values, nil
|
||||
}
|
||||
+69
-33
@@ -2,11 +2,11 @@ package reflector
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log"
|
||||
"net"
|
||||
"strconv"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
)
|
||||
|
||||
// ErrBlobExists is a default error for when a blob already exists on the reflector server.
|
||||
@@ -35,23 +35,36 @@ func (c *Client) Close() error {
|
||||
return c.conn.Close()
|
||||
}
|
||||
|
||||
// SendBlob sends a send blob request to the client.
|
||||
func (c *Client) SendBlob(blob []byte) error {
|
||||
// SendBlob sends a blob to the server.
|
||||
func (c *Client) SendBlob(blob stream.Blob) error {
|
||||
return c.sendBlob(blob, false)
|
||||
}
|
||||
|
||||
// SendSDBlob sends an SD blob request to the server.
|
||||
func (c *Client) SendSDBlob(blob stream.Blob) error {
|
||||
return c.sendBlob(blob, true)
|
||||
}
|
||||
|
||||
// sendBlob does the actual blob sending
|
||||
func (c *Client) sendBlob(blob stream.Blob, isSDBlob bool) error {
|
||||
if !c.connected {
|
||||
return errors.Err("not connected")
|
||||
}
|
||||
|
||||
if len(blob) > maxBlobSize {
|
||||
return errors.Err("blob must be at most " + strconv.Itoa(maxBlobSize) + " bytes")
|
||||
} else if len(blob) == 0 {
|
||||
return errors.Err("blob is empty")
|
||||
if err := blob.ValidForSend(); err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
blobHash := getBlobHash(blob)
|
||||
sendRequest, err := json.Marshal(sendBlobRequest{
|
||||
BlobSize: len(blob),
|
||||
BlobHash: blobHash,
|
||||
})
|
||||
blobHash := blob.HashHex()
|
||||
var req sendBlobRequest
|
||||
if isSDBlob {
|
||||
req.SdBlobSize = blob.Size()
|
||||
req.SdBlobHash = blobHash
|
||||
} else {
|
||||
req.BlobSize = blob.Size()
|
||||
req.BlobHash = blobHash
|
||||
}
|
||||
sendRequest, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -63,30 +76,51 @@ func (c *Client) SendBlob(blob []byte) error {
|
||||
|
||||
dec := json.NewDecoder(c.conn)
|
||||
|
||||
var sendResp sendBlobResponse
|
||||
err = dec.Decode(&sendResp)
|
||||
if err != nil {
|
||||
return err
|
||||
if isSDBlob {
|
||||
var sendResp sendSdBlobResponse
|
||||
err = dec.Decode(&sendResp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !sendResp.SendSdBlob {
|
||||
return errors.Prefix(blobHash[:8], ErrBlobExists)
|
||||
}
|
||||
log.Println("Sending SD blob " + blobHash[:8])
|
||||
} else {
|
||||
var sendResp sendBlobResponse
|
||||
err = dec.Decode(&sendResp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !sendResp.SendBlob {
|
||||
return errors.Prefix(blobHash[:8], ErrBlobExists)
|
||||
}
|
||||
log.Println("Sending blob " + blobHash[:8])
|
||||
}
|
||||
|
||||
if !sendResp.SendBlob {
|
||||
return ErrBlobExists
|
||||
}
|
||||
|
||||
log.Println("Sending blob " + blobHash[:8])
|
||||
|
||||
_, err = c.conn.Write(blob)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var transferResp blobTransferResponse
|
||||
err = dec.Decode(&transferResp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !transferResp.ReceivedBlob {
|
||||
return errors.Err("server did not received blob")
|
||||
if isSDBlob {
|
||||
var transferResp sdBlobTransferResponse
|
||||
err = dec.Decode(&transferResp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !transferResp.ReceivedSdBlob {
|
||||
return errors.Err("server did not received SD blob")
|
||||
}
|
||||
} else {
|
||||
var transferResp blobTransferResponse
|
||||
err = dec.Decode(&transferResp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !transferResp.ReceivedBlob {
|
||||
return errors.Err("server did not received blob")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -97,7 +131,7 @@ func (c *Client) doHandshake(version int) error {
|
||||
return errors.Err("not connected")
|
||||
}
|
||||
|
||||
handshake, err := json.Marshal(handshakeRequestResponse{Version: version})
|
||||
handshake, err := json.Marshal(handshakeRequestResponse{Version: &version})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -111,7 +145,9 @@ func (c *Client) doHandshake(version int) error {
|
||||
err = json.NewDecoder(c.conn).Decode(&resp)
|
||||
if err != nil {
|
||||
return err
|
||||
} else if resp.Version != version {
|
||||
} else if resp.Version == nil {
|
||||
return errors.Err("invalid handshake")
|
||||
} else if *resp.Version != version {
|
||||
return errors.Err("handshake version mismatch")
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
package reflector
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
|
||||
"github.com/bluele/gcache"
|
||||
"golang.org/x/sync/singleflight"
|
||||
)
|
||||
|
||||
const protectedListURL = "https://direct.api.odysee.com/file/list_protected"
|
||||
|
||||
type ProtectedContent struct {
|
||||
SDHash string `json:"sd_hash"`
|
||||
ClaimID string `json:"claim_id"`
|
||||
}
|
||||
|
||||
var protectedCache = gcache.New(10).Expiration(2 * time.Minute).Build()
|
||||
|
||||
func GetProtectedContent() (interface{}, error) {
|
||||
cachedVal, err := protectedCache.Get("protected")
|
||||
if err == nil && cachedVal != nil {
|
||||
return cachedVal.(map[string]bool), nil
|
||||
}
|
||||
|
||||
method := "GET"
|
||||
var r struct {
|
||||
Error string `json:"error"`
|
||||
Data []ProtectedContent `json:"data"`
|
||||
Success bool `json:"success"`
|
||||
}
|
||||
|
||||
// Bound the request to avoid hanging the entire request path.
|
||||
// Without a timeout, a slow or unreachable endpoint can block
|
||||
// singleflight callers indefinitely and stall HTTP handlers.
|
||||
client := &http.Client{Timeout: 5 * time.Second}
|
||||
req, err := http.NewRequest(method, protectedListURL, nil)
|
||||
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
defer func() { _ = res.Body.Close() }()
|
||||
if res.StatusCode != http.StatusOK {
|
||||
return nil, errors.Err("unexpected status code %d", res.StatusCode)
|
||||
}
|
||||
if err = json.NewDecoder(res.Body).Decode(&r); err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
if !r.Success {
|
||||
return nil, errors.Prefix("file/list_protected API call", r.Error)
|
||||
}
|
||||
|
||||
protectedMap := make(map[string]bool, len(r.Data))
|
||||
for _, pc := range r.Data {
|
||||
protectedMap[pc.SDHash] = true
|
||||
}
|
||||
err = protectedCache.Set("protected", protectedMap)
|
||||
if err != nil {
|
||||
return protectedMap, errors.Err(err)
|
||||
}
|
||||
return protectedMap, nil
|
||||
}
|
||||
|
||||
var sf = singleflight.Group{}
|
||||
|
||||
func IsProtected(sdHash string) bool {
|
||||
val, err, _ := sf.Do("protected", GetProtectedContent)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
cachedMap, ok := val.(map[string]bool)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
return cachedMap[sdHash]
|
||||
}
|
||||
+132
-55
@@ -7,13 +7,14 @@ import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/lbry.go/stop"
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
@@ -27,22 +28,28 @@ const (
|
||||
network = "tcp4"
|
||||
protocolVersion1 = 0
|
||||
protocolVersion2 = 1
|
||||
maxBlobSize = 2 * 1024 * 1024
|
||||
maxBlobSize = stream.MaxBlobSize
|
||||
)
|
||||
|
||||
var ErrBlobTooBig = errors.Base("blob must be at most %d bytes", maxBlobSize)
|
||||
|
||||
// Server is and instance of the reflector server. It houses the blob store and listener.
|
||||
type Server struct {
|
||||
store store.BlobStore
|
||||
timeout time.Duration // timeout to read or write next message
|
||||
grp *stop.Group
|
||||
Timeout time.Duration // timeout to read or write next message
|
||||
|
||||
EnableBlocklist bool // if true, blocklist checking and blob deletion will be enabled
|
||||
|
||||
//underlyingStore store.BlobStore
|
||||
//outerStore store.BlobStore
|
||||
}
|
||||
|
||||
// NewServer returns an initialized reflector server pointer.
|
||||
func NewServer(store store.BlobStore) *Server {
|
||||
func NewIngestionServer(store store.BlobStore) *Server {
|
||||
return &Server{
|
||||
Timeout: DefaultTimeout,
|
||||
store: store,
|
||||
grp: stop.New(),
|
||||
timeout: DefaultTimeout,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -53,16 +60,17 @@ func (s *Server) Shutdown() {
|
||||
log.Println("reflector server stopped")
|
||||
}
|
||||
|
||||
//Start starts the server to handle connections.
|
||||
// Start starts the server to handle connections.
|
||||
func (s *Server) Start(address string) error {
|
||||
l, err := net.Listen(network, address)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
log.Println("reflector listening on " + address)
|
||||
|
||||
s.grp.Add(1)
|
||||
metrics.RoutinesQueue.WithLabelValues("reflector", "listener").Inc()
|
||||
go func() {
|
||||
defer metrics.RoutinesQueue.WithLabelValues("reflector", "listener").Dec()
|
||||
<-s.grp.Ch()
|
||||
err := l.Close()
|
||||
if err != nil {
|
||||
@@ -72,11 +80,28 @@ func (s *Server) Start(address string) error {
|
||||
}()
|
||||
|
||||
s.grp.Add(1)
|
||||
metrics.RoutinesQueue.WithLabelValues("reflector", "start").Inc()
|
||||
go func() {
|
||||
defer metrics.RoutinesQueue.WithLabelValues("reflector", "start").Dec()
|
||||
s.listenAndServe(l)
|
||||
s.grp.Done()
|
||||
}()
|
||||
|
||||
if s.EnableBlocklist {
|
||||
if b, ok := s.store.(store.Blocklister); ok {
|
||||
s.grp.Add(1)
|
||||
metrics.RoutinesQueue.WithLabelValues("reflector", "enableblocklist").Inc()
|
||||
go func() {
|
||||
defer metrics.RoutinesQueue.WithLabelValues("reflector", "enableblocklist").Dec()
|
||||
s.enableBlocklist(b)
|
||||
s.grp.Done()
|
||||
}()
|
||||
} else {
|
||||
//s.Shutdown()
|
||||
return errors.Err("blocklist is enabled but blob store does not support blocklisting")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -90,7 +115,9 @@ func (s *Server) listenAndServe(listener net.Listener) {
|
||||
log.Error(err)
|
||||
} else {
|
||||
s.grp.Add(1)
|
||||
metrics.RoutinesQueue.WithLabelValues("reflector", "server-listenandserve").Inc()
|
||||
go func() {
|
||||
defer metrics.RoutinesQueue.WithLabelValues("reflector", "server-listenandserve").Dec()
|
||||
s.handleConn(conn)
|
||||
s.grp.Done()
|
||||
}()
|
||||
@@ -105,7 +132,9 @@ func (s *Server) handleConn(conn net.Conn) {
|
||||
close(connNeedsClosing)
|
||||
}()
|
||||
s.grp.Add(1)
|
||||
metrics.RoutinesQueue.WithLabelValues("reflector", "server-handleconn").Inc()
|
||||
go func() {
|
||||
defer metrics.RoutinesQueue.WithLabelValues("reflector", "server-handleconn").Dec()
|
||||
defer s.grp.Done()
|
||||
select {
|
||||
case <-connNeedsClosing:
|
||||
@@ -119,10 +148,10 @@ func (s *Server) handleConn(conn net.Conn) {
|
||||
|
||||
err := s.doHandshake(conn)
|
||||
if err != nil {
|
||||
if err == io.EOF || s.quitting() {
|
||||
if errors.Is(err, io.EOF) || s.quitting() {
|
||||
return
|
||||
}
|
||||
err := s.doError(conn, err)
|
||||
err = s.doError(conn, err)
|
||||
if err != nil {
|
||||
log.Error(errors.Prefix("sending handshake error", err))
|
||||
}
|
||||
@@ -132,10 +161,10 @@ func (s *Server) handleConn(conn net.Conn) {
|
||||
for {
|
||||
err = s.receiveBlob(conn)
|
||||
if err != nil {
|
||||
if err == io.EOF || s.quitting() {
|
||||
if errors.Is(err, io.EOF) || s.quitting() {
|
||||
return
|
||||
}
|
||||
err := s.doError(conn, err)
|
||||
err = s.doError(conn, err)
|
||||
if err != nil {
|
||||
log.Error(errors.Prefix("sending blob receive error", err))
|
||||
}
|
||||
@@ -145,57 +174,84 @@ func (s *Server) handleConn(conn net.Conn) {
|
||||
}
|
||||
|
||||
func (s *Server) doError(conn net.Conn, err error) error {
|
||||
log.Errorln(errors.FullTrace(err))
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
shouldLog := metrics.TrackError(metrics.DirectionUpload, err)
|
||||
if shouldLog {
|
||||
log.Errorln(errors.FullTrace(err))
|
||||
}
|
||||
if e2, ok := err.(*json.SyntaxError); ok {
|
||||
log.Errorf("syntax error at byte offset %d", e2.Offset)
|
||||
}
|
||||
resp, err := json.Marshal(errorResponse{Error: err.Error()})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.write(conn, resp)
|
||||
//resp, err := json.Marshal(errorResponse{Error: err.Error()})
|
||||
//if err != nil {
|
||||
// return err
|
||||
//}
|
||||
//return s.write(conn, resp)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) receiveBlob(conn net.Conn) error {
|
||||
var err error
|
||||
|
||||
blobSize, blobHash, isSdBlob, err := s.readBlobRequest(conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// fullStreamChecker can check if the full stream has been uploaded
|
||||
type fullStreamChecker interface {
|
||||
HasFullStream(string) (bool, error)
|
||||
}
|
||||
|
||||
blobExists := false
|
||||
if fsc, ok := s.store.(fullStreamChecker); ok && isSdBlob {
|
||||
blobExists, err = fsc.HasFullStream(blobHash)
|
||||
var wantsBlob bool
|
||||
if bl, ok := s.store.(store.Blocklister); ok {
|
||||
wantsBlob, err = bl.Wants(blobHash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
// if we can't confirm that we have the full stream, we have to say that the sd blob is missing. if we say we have it, they wont try to send any content blobs
|
||||
var blobExists bool
|
||||
blobExists, err = s.store.Has(blobHash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
wantsBlob = !blobExists
|
||||
}
|
||||
|
||||
var neededBlobs []string
|
||||
|
||||
if isSdBlob && !wantsBlob {
|
||||
if nbc, ok := s.store.(store.NeededBlobChecker); ok {
|
||||
neededBlobs, err = nbc.MissingBlobsForKnownStream(blobHash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
// if we can't check for blobs in a stream, we have to say that the sd blob is
|
||||
// missing. if we say we have the sd blob, they won't try to send any content blobs
|
||||
wantsBlob = true
|
||||
}
|
||||
}
|
||||
|
||||
err = s.sendBlobResponse(conn, wantsBlob, isSdBlob, neededBlobs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = s.sendBlobResponse(conn, blobExists, isSdBlob)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if blobExists {
|
||||
if !wantsBlob {
|
||||
return nil
|
||||
}
|
||||
|
||||
blob, err := s.readRawBlob(conn, blobSize)
|
||||
if err != nil {
|
||||
return err
|
||||
sendErr := s.sendTransferResponse(conn, false, isSdBlob)
|
||||
if sendErr != nil {
|
||||
return sendErr
|
||||
}
|
||||
return errors.Prefix("error reading blob "+blobHash[:8], err)
|
||||
}
|
||||
|
||||
receivedBlobHash := getBlobHash(blob)
|
||||
receivedBlobHash := BlobHash(blob)
|
||||
if blobHash != receivedBlobHash {
|
||||
sendErr := s.sendTransferResponse(conn, false, isSdBlob)
|
||||
if sendErr != nil {
|
||||
return sendErr
|
||||
}
|
||||
return errors.Err("hash of received blob data does not match hash from send request")
|
||||
// this can also happen if the blob size is wrong, because the server will read the wrong number of bytes from the stream
|
||||
}
|
||||
@@ -210,7 +266,11 @@ func (s *Server) receiveBlob(conn net.Conn) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
metrics.MtrInBytesReflector.Add(float64(len(blob)))
|
||||
metrics.BlobUploadCount.Inc()
|
||||
if isSdBlob {
|
||||
metrics.SDBlobUploadCount.Inc()
|
||||
}
|
||||
return s.sendTransferResponse(conn, true, isSdBlob)
|
||||
}
|
||||
|
||||
@@ -219,7 +279,9 @@ func (s *Server) doHandshake(conn net.Conn) error {
|
||||
err := s.read(conn, &handshake)
|
||||
if err != nil {
|
||||
return err
|
||||
} else if handshake.Version != protocolVersion1 && handshake.Version != protocolVersion2 {
|
||||
} else if handshake.Version == nil {
|
||||
return errors.Err("handshake is missing protocol version")
|
||||
} else if *handshake.Version != protocolVersion1 && *handshake.Version != protocolVersion2 {
|
||||
return errors.Err("protocol version not supported")
|
||||
}
|
||||
|
||||
@@ -254,7 +316,7 @@ func (s *Server) readBlobRequest(conn net.Conn) (int, string, bool, error) {
|
||||
return blobSize, blobHash, isSdBlob, errors.Err("blob hash is empty")
|
||||
}
|
||||
if blobSize > maxBlobSize {
|
||||
return blobSize, blobHash, isSdBlob, errors.Err("blob must be at most " + strconv.Itoa(maxBlobSize) + " bytes")
|
||||
return blobSize, blobHash, isSdBlob, errors.Err(ErrBlobTooBig)
|
||||
}
|
||||
if blobSize == 0 {
|
||||
return blobSize, blobHash, isSdBlob, errors.Err("0-byte blob received")
|
||||
@@ -263,14 +325,14 @@ func (s *Server) readBlobRequest(conn net.Conn) (int, string, bool, error) {
|
||||
return blobSize, blobHash, isSdBlob, nil
|
||||
}
|
||||
|
||||
func (s *Server) sendBlobResponse(conn net.Conn, blobExists, isSdBlob bool) error {
|
||||
func (s *Server) sendBlobResponse(conn net.Conn, shouldSendBlob, isSdBlob bool, neededBlobs []string) error {
|
||||
var response []byte
|
||||
var err error
|
||||
|
||||
if isSdBlob {
|
||||
response, err = json.Marshal(sendSdBlobResponse{SendSdBlob: !blobExists})
|
||||
response, err = json.Marshal(sendSdBlobResponse{SendSdBlob: shouldSendBlob, NeededBlobs: neededBlobs})
|
||||
} else {
|
||||
response, err = json.Marshal(sendBlobResponse{SendBlob: !blobExists})
|
||||
response, err = json.Marshal(sendBlobResponse{SendBlob: shouldSendBlob})
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -296,16 +358,25 @@ func (s *Server) sendTransferResponse(conn net.Conn, receivedBlob, isSdBlob bool
|
||||
}
|
||||
|
||||
func (s *Server) read(conn net.Conn, v interface{}) error {
|
||||
err := conn.SetReadDeadline(time.Now().Add(s.timeout))
|
||||
err := conn.SetReadDeadline(time.Now().Add(s.Timeout))
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
return errors.Err(json.NewDecoder(conn).Decode(v))
|
||||
dec := json.NewDecoder(conn)
|
||||
err = dec.Decode(v)
|
||||
if err != nil {
|
||||
data, _ := io.ReadAll(dec.Buffered())
|
||||
if len(data) > 0 {
|
||||
return errors.Err("%s. Data: %s", err.Error(), hex.EncodeToString(data))
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) readRawBlob(conn net.Conn, blobSize int) ([]byte, error) {
|
||||
err := conn.SetReadDeadline(time.Now().Add(s.timeout))
|
||||
err := conn.SetReadDeadline(time.Now().Add(s.Timeout))
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
@@ -316,7 +387,7 @@ func (s *Server) readRawBlob(conn net.Conn, blobSize int) ([]byte, error) {
|
||||
}
|
||||
|
||||
func (s *Server) write(conn net.Conn, b []byte) error {
|
||||
err := conn.SetWriteDeadline(time.Now().Add(s.timeout))
|
||||
err := conn.SetWriteDeadline(time.Now().Add(s.Timeout))
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
@@ -337,23 +408,29 @@ func (s *Server) quitting() bool {
|
||||
}
|
||||
}
|
||||
|
||||
func getBlobHash(blob []byte) string {
|
||||
// BlobHash returns the sha512 hash hex encoded string of the blob byte slice.
|
||||
func BlobHash(blob []byte) string {
|
||||
hashBytes := sha512.Sum384(blob)
|
||||
return hex.EncodeToString(hashBytes[:])
|
||||
}
|
||||
|
||||
type errorResponse struct {
|
||||
Error string `json:"error"`
|
||||
func IsValidJSON(b []byte) bool {
|
||||
var r json.RawMessage
|
||||
return json.Unmarshal(b, &r) == nil
|
||||
}
|
||||
|
||||
//type errorResponse struct {
|
||||
// Error string `json:"error"`
|
||||
//}
|
||||
|
||||
type handshakeRequestResponse struct {
|
||||
Version int `json:"version"`
|
||||
Version *int `json:"version"`
|
||||
}
|
||||
|
||||
type sendBlobRequest struct {
|
||||
BlobHash string `json:"blob_hash,omitempty"`
|
||||
BlobSize int `json:"blob_size,omitempty"`
|
||||
SdBlobHash string `json:"sd_blob_hash,omitempty"`
|
||||
BlobSize int `json:"blob_size,omitempty"`
|
||||
SdBlobSize int `json:"sd_blob_size,omitempty"`
|
||||
}
|
||||
|
||||
@@ -362,8 +439,8 @@ type sendBlobResponse struct {
|
||||
}
|
||||
|
||||
type sendSdBlobResponse struct {
|
||||
SendSdBlob bool `json:"send_sd_blob"`
|
||||
NeededBlobs []string `json:"needed_blobs,omitempty"`
|
||||
SendSdBlob bool `json:"send_sd_blob"`
|
||||
}
|
||||
|
||||
type blobTransferResponse struct {
|
||||
|
||||
+120
-24
@@ -2,14 +2,18 @@ package reflector
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"sort"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/dht/bits"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/phayes/freeport"
|
||||
)
|
||||
|
||||
@@ -19,7 +23,7 @@ func startServerOnRandomPort(t *testing.T) (*Server, int) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
srv := NewServer(&store.MemoryBlobStore{})
|
||||
srv := NewIngestionServer(store.NewMemStore(store.MemParams{Name: "test"}))
|
||||
err = srv.Start("127.0.0.1:" + strconv.Itoa(port))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -62,11 +66,7 @@ func TestServer_MediumBlob(t *testing.T) {
|
||||
t.Fatal("error connecting client to server", err)
|
||||
}
|
||||
|
||||
blob := make([]byte, 1000)
|
||||
_, err = rand.Read(blob)
|
||||
if err != nil {
|
||||
t.Fatal("failed to make random blob")
|
||||
}
|
||||
blob := randBlob(1000)
|
||||
|
||||
err = c.SendBlob(blob)
|
||||
if err != nil {
|
||||
@@ -84,11 +84,7 @@ func TestServer_FullBlob(t *testing.T) {
|
||||
t.Fatal("error connecting client to server", err)
|
||||
}
|
||||
|
||||
blob := make([]byte, maxBlobSize)
|
||||
_, err = rand.Read(blob)
|
||||
if err != nil {
|
||||
t.Fatal("failed to make random blob")
|
||||
}
|
||||
blob := randBlob(maxBlobSize)
|
||||
|
||||
err = c.SendBlob(blob)
|
||||
if err != nil {
|
||||
@@ -106,11 +102,7 @@ func TestServer_TooBigBlob(t *testing.T) {
|
||||
t.Fatal("error connecting client to server", err)
|
||||
}
|
||||
|
||||
blob := make([]byte, maxBlobSize+1)
|
||||
_, err = rand.Read(blob)
|
||||
if err != nil {
|
||||
t.Fatal("failed to make random blob")
|
||||
}
|
||||
blob := randBlob(maxBlobSize + 1)
|
||||
|
||||
err = c.SendBlob(blob)
|
||||
if err == nil {
|
||||
@@ -128,8 +120,8 @@ func TestServer_Timeout(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
srv := NewServer(&store.MemoryBlobStore{})
|
||||
srv.timeout = testTimeout
|
||||
srv := NewIngestionServer(store.NewMemStore(store.MemParams{Name: "test"}))
|
||||
srv.Timeout = testTimeout
|
||||
err = srv.Start("127.0.0.1:" + strconv.Itoa(port))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -144,11 +136,7 @@ func TestServer_Timeout(t *testing.T) {
|
||||
|
||||
time.Sleep(testTimeout * 2)
|
||||
|
||||
blob := make([]byte, 10)
|
||||
_, err = rand.Read(blob)
|
||||
if err != nil {
|
||||
t.Fatal("failed to make random blob")
|
||||
}
|
||||
blob := randBlob(10)
|
||||
|
||||
err = c.SendBlob(blob)
|
||||
t.Log(spew.Sdump(err))
|
||||
@@ -156,3 +144,111 @@ func TestServer_Timeout(t *testing.T) {
|
||||
t.Error("server should have timed out by now")
|
||||
}
|
||||
}
|
||||
|
||||
//func TestServer_InvalidJSONHandshake(t *testing.T) {
|
||||
// srv, port := startServerOnRandomPort(t)
|
||||
// defer srv.Shutdown()
|
||||
//
|
||||
// c := Client{}
|
||||
// err := c.Connect(":" + strconv.Itoa(port))
|
||||
// if err != nil {
|
||||
// t.Fatal("error connecting client to server", err)
|
||||
// }
|
||||
//
|
||||
// _, err = c.conn.Write([]byte(`{"stuff":4,tf}"`))
|
||||
// if err == nil {
|
||||
// t.Error("expected an error")
|
||||
// }
|
||||
//}
|
||||
|
||||
type mockPartialStore struct {
|
||||
*store.MemStore
|
||||
missing []string
|
||||
}
|
||||
|
||||
func (m mockPartialStore) MissingBlobsForKnownStream(hash string) ([]string, error) {
|
||||
return m.missing, nil
|
||||
}
|
||||
|
||||
func TestServer_PartialUpload(t *testing.T) {
|
||||
port, err := freeport.GetFreePort()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
sdHash := bits.Rand().String()
|
||||
missing := make([]string, 4)
|
||||
for i := range missing {
|
||||
missing[i] = bits.Rand().String()
|
||||
}
|
||||
|
||||
st := store.BlobStore(&mockPartialStore{MemStore: store.NewMemStore(store.MemParams{Name: "test"}), missing: missing})
|
||||
if _, ok := st.(store.NeededBlobChecker); !ok {
|
||||
t.Fatal("mock does not implement the relevant interface")
|
||||
}
|
||||
err = st.Put(sdHash, randBlob(10))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
srv := NewIngestionServer(st)
|
||||
err = srv.Start("127.0.0.1:" + strconv.Itoa(port))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer srv.Shutdown()
|
||||
|
||||
c := Client{}
|
||||
err = c.Connect(":" + strconv.Itoa(port))
|
||||
if err != nil {
|
||||
t.Fatal("error connecting client to server", err)
|
||||
}
|
||||
|
||||
sendRequest, err := json.Marshal(sendBlobRequest{
|
||||
SdBlobHash: sdHash,
|
||||
SdBlobSize: len(sdHash),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = c.conn.Write(sendRequest)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var sendResp sendSdBlobResponse
|
||||
err = json.NewDecoder(c.conn).Decode(&sendResp)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if sendResp.SendSdBlob {
|
||||
t.Errorf("expected SendSdBlob = false, got true")
|
||||
}
|
||||
|
||||
if len(sendResp.NeededBlobs) != len(missing) {
|
||||
t.Fatalf("got %d needed blobs, expected %d", len(sendResp.NeededBlobs), len(missing))
|
||||
}
|
||||
|
||||
sort.Strings(sendResp.NeededBlobs)
|
||||
sort.Strings(missing)
|
||||
|
||||
for i := range missing {
|
||||
if missing[i] != sendResp.NeededBlobs[i] {
|
||||
t.Errorf("needed blobs mismatch: %s != %s", missing[i], sendResp.NeededBlobs[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func randBlob(size int) []byte {
|
||||
//if size > maxBlobSize {
|
||||
// panic("blob size too big")
|
||||
//}
|
||||
blob := make([]byte, size)
|
||||
_, err := rand.Read(blob)
|
||||
if err != nil {
|
||||
panic("failed to make random blob")
|
||||
}
|
||||
return blob
|
||||
}
|
||||
@@ -0,0 +1,269 @@
|
||||
package reflector
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/db"
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type increment int
|
||||
|
||||
const (
|
||||
sdInc increment = iota + 1
|
||||
blobInc
|
||||
errInc
|
||||
)
|
||||
|
||||
type Summary struct {
|
||||
Total, AlreadyStored, Sd, Blob, Err int
|
||||
}
|
||||
|
||||
type Uploader struct {
|
||||
db *db.SQL
|
||||
store store.BlobStore
|
||||
stopper *stop.Group
|
||||
countChan chan increment
|
||||
count Summary
|
||||
workers int
|
||||
skipExistsCheck bool
|
||||
deleteBlobsAfterUpload bool
|
||||
}
|
||||
|
||||
func NewUploader(db *db.SQL, store store.BlobStore, workers int, skipExistsCheck, deleteBlobsAfterUpload bool) *Uploader {
|
||||
return &Uploader{
|
||||
db: db,
|
||||
store: store,
|
||||
workers: workers,
|
||||
skipExistsCheck: skipExistsCheck,
|
||||
deleteBlobsAfterUpload: deleteBlobsAfterUpload,
|
||||
stopper: stop.New(),
|
||||
countChan: make(chan increment),
|
||||
}
|
||||
}
|
||||
|
||||
func (u *Uploader) Stop() {
|
||||
log.Infoln("stopping uploader")
|
||||
u.stopper.StopAndWait()
|
||||
}
|
||||
|
||||
func (u *Uploader) Upload(dirOrFilePath string) error {
|
||||
paths, err := getPaths(dirOrFilePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
u.count.Total = len(paths)
|
||||
|
||||
hashes := make([]string, len(paths))
|
||||
for i, p := range paths {
|
||||
hashes[i] = path.Base(p)
|
||||
}
|
||||
|
||||
log.Debug("checking for existing blobs")
|
||||
|
||||
var exists map[string]bool
|
||||
if !u.skipExistsCheck {
|
||||
exists, err = u.db.HasBlobs(hashes, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
u.count.AlreadyStored = len(exists)
|
||||
}
|
||||
|
||||
log.Debugf("%d new blobs to upload", u.count.Total-u.count.AlreadyStored)
|
||||
|
||||
workerWG := sync.WaitGroup{}
|
||||
pathChan := make(chan string)
|
||||
|
||||
for i := 0; i < u.workers; i++ {
|
||||
workerWG.Add(1)
|
||||
metrics.RoutinesQueue.WithLabelValues("reflector", "upload").Inc()
|
||||
go func(i int) {
|
||||
defer metrics.RoutinesQueue.WithLabelValues("reflector", "upload").Dec()
|
||||
defer workerWG.Done()
|
||||
defer func(i int) { log.Debugf("worker %d quitting", i) }(i)
|
||||
u.worker(pathChan)
|
||||
}(i)
|
||||
}
|
||||
|
||||
countWG := sync.WaitGroup{}
|
||||
countWG.Add(1)
|
||||
metrics.RoutinesQueue.WithLabelValues("reflector", "uploader").Inc()
|
||||
go func() {
|
||||
defer metrics.RoutinesQueue.WithLabelValues("reflector", "uploader").Dec()
|
||||
defer countWG.Done()
|
||||
u.counter()
|
||||
}()
|
||||
|
||||
Upload:
|
||||
for _, f := range paths {
|
||||
if exists != nil && exists[path.Base(f)] {
|
||||
continue
|
||||
}
|
||||
|
||||
select {
|
||||
case pathChan <- f:
|
||||
case <-u.stopper.Ch():
|
||||
break Upload
|
||||
}
|
||||
}
|
||||
|
||||
close(pathChan)
|
||||
workerWG.Wait()
|
||||
close(u.countChan)
|
||||
countWG.Wait()
|
||||
u.stopper.Stop()
|
||||
|
||||
log.Debugf(
|
||||
"upload stats: %d blobs total, %d already stored, %d SD blobs uploaded, %d content blobs uploaded, %d errors",
|
||||
u.count.Total, u.count.AlreadyStored, u.count.Sd, u.count.Blob, u.count.Err,
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
// worker reads paths from a channel, uploads them, and optionally deletes them
|
||||
func (u *Uploader) worker(pathChan chan string) {
|
||||
for {
|
||||
select {
|
||||
case <-u.stopper.Ch():
|
||||
return
|
||||
case filepath, ok := <-pathChan:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
err := u.uploadBlob(filepath)
|
||||
if err != nil {
|
||||
log.Errorln(err)
|
||||
} else if u.deleteBlobsAfterUpload {
|
||||
err = os.Remove(filepath)
|
||||
if err != nil {
|
||||
log.Errorln(errors.Prefix("deleting blob", err))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// uploadBlob uploads a blob
|
||||
func (u *Uploader) uploadBlob(filepath string) (err error) {
|
||||
defer func() {
|
||||
if err != nil {
|
||||
u.inc(errInc)
|
||||
}
|
||||
}()
|
||||
|
||||
blob, err := os.ReadFile(filepath)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
hash := BlobHash(blob)
|
||||
if hash != path.Base(filepath) {
|
||||
return errors.Err("file name does not match hash (%s != %s), skipping", filepath, hash)
|
||||
}
|
||||
|
||||
if IsValidJSON(blob) {
|
||||
log.Debugf("uploading SD blob %s", hash)
|
||||
err = u.store.PutSD(hash, blob)
|
||||
if err != nil {
|
||||
return errors.Prefix("uploading SD blob "+hash, err)
|
||||
}
|
||||
u.inc(sdInc)
|
||||
} else {
|
||||
log.Debugf("uploading blob %s", hash)
|
||||
err = u.store.Put(hash, blob)
|
||||
if err != nil {
|
||||
return errors.Prefix("uploading blob "+hash, err)
|
||||
}
|
||||
u.inc(blobInc)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// counter updates the counts of how many sd blobs and content blobs were uploaded, and how many
|
||||
// errors were encountered. It occasionally prints the upload progress to debug.
|
||||
func (u *Uploader) counter() {
|
||||
start := time.Now()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-u.stopper.Ch():
|
||||
return
|
||||
case incrementType, ok := <-u.countChan:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
switch incrementType {
|
||||
case sdInc:
|
||||
u.count.Sd++
|
||||
case blobInc:
|
||||
u.count.Blob++
|
||||
case errInc:
|
||||
u.count.Err++
|
||||
}
|
||||
}
|
||||
if (u.count.Sd+u.count.Blob)%50 == 0 {
|
||||
log.Debugf("%d of %d done (%s elapsed, %.3fs per blob)", u.count.Sd+u.count.Blob, u.count.Total-u.count.AlreadyStored, time.Since(start).String(), time.Since(start).Seconds()/float64(u.count.Sd+u.count.Blob))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (u *Uploader) GetSummary() Summary {
|
||||
return u.count
|
||||
}
|
||||
|
||||
func (u *Uploader) inc(t increment) {
|
||||
select {
|
||||
case u.countChan <- t:
|
||||
case <-u.stopper.Ch():
|
||||
}
|
||||
}
|
||||
|
||||
// getPaths returns the paths for files to upload. it takes a path to a file or a dir. for a file,
|
||||
// it returns the full path to that file. for a dir, it returns the paths for all the files in the
|
||||
// dir
|
||||
func getPaths(dirOrFilePath string) ([]string, error) {
|
||||
info, err := os.Stat(dirOrFilePath)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
if info.Mode().IsRegular() {
|
||||
return []string{dirOrFilePath}, nil
|
||||
}
|
||||
|
||||
f, err := os.Open(dirOrFilePath)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
files, err := f.Readdir(-1)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
err = f.Close()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
var filenames []string
|
||||
for _, file := range files {
|
||||
if !file.IsDir() {
|
||||
filenames = append(filenames, dirOrFilePath+"/"+file.Name())
|
||||
}
|
||||
}
|
||||
|
||||
return filenames, nil
|
||||
}
|
||||
Executable
+16
@@ -0,0 +1,16 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
cd "$SCRIPT_DIR/.."
|
||||
|
||||
if ! command -v golangci-lint &> /dev/null; then
|
||||
echo "Installing golangci-lint..."
|
||||
go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
|
||||
else
|
||||
echo "Ensuring golangci-lint is up to date..."
|
||||
go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
|
||||
fi
|
||||
|
||||
echo "Running golangci-lint..."
|
||||
golangci-lint run --config .golangci.yml ./...
|
||||
@@ -0,0 +1,107 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func (s *Server) getBlob(c *gin.Context) {
|
||||
waiter := &sync.WaitGroup{}
|
||||
waiter.Add(1)
|
||||
enqueue(&blobRequest{c: c, finished: waiter})
|
||||
waiter.Wait()
|
||||
}
|
||||
|
||||
func (s *Server) HandleGetBlob(c *gin.Context) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Errorf("Recovered from panic: %v", r)
|
||||
}
|
||||
}()
|
||||
start := time.Now()
|
||||
hash := c.Query("hash")
|
||||
edgeToken := c.Query("edge_token")
|
||||
|
||||
if reflector.IsProtected(hash) && edgeToken != s.edgeToken {
|
||||
_ = c.Error(errors.Err("requested blob is protected"))
|
||||
c.String(http.StatusForbidden, "requested blob is protected")
|
||||
return
|
||||
}
|
||||
if s.missesCache.Has(hash) {
|
||||
serialized, err := shared.NewBlobTrace(time.Since(start), "http").Serialize()
|
||||
c.Header("Via", serialized)
|
||||
if err != nil {
|
||||
_ = c.Error(errors.Err(err))
|
||||
c.String(http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
c.AbortWithStatus(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
blob, trace, err := s.store.Get(hash)
|
||||
if err != nil {
|
||||
serialized, serializeErr := trace.Serialize()
|
||||
if serializeErr != nil {
|
||||
_ = c.Error(errors.Prefix(serializeErr.Error(), err))
|
||||
c.String(http.StatusInternalServerError, errors.Prefix(serializeErr.Error(), err).Error())
|
||||
return
|
||||
}
|
||||
c.Header("Via", serialized)
|
||||
|
||||
if errors.Is(err, store.ErrBlobNotFound) {
|
||||
_ = s.missesCache.Set(hash, true)
|
||||
c.AbortWithStatus(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
_ = c.Error(err)
|
||||
c.String(http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
serialized, err := trace.Serialize()
|
||||
if err != nil {
|
||||
_ = c.Error(err)
|
||||
c.String(http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
metrics.MtrOutBytesHttp.Add(float64(len(blob)))
|
||||
metrics.BlobDownloadCount.Inc()
|
||||
metrics.HttpDownloadCount.Inc()
|
||||
c.Header("Via", serialized)
|
||||
c.Header("Content-Disposition", "filename="+hash)
|
||||
c.Header("Content-Length", strconv.Itoa(len(blob)))
|
||||
c.Data(http.StatusOK, "application/octet-stream", blob)
|
||||
}
|
||||
|
||||
func (s *Server) hasBlob(c *gin.Context) {
|
||||
hash := c.Query("hash")
|
||||
has, err := s.store.Has(hash)
|
||||
if err != nil {
|
||||
_ = c.Error(err)
|
||||
c.String(http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
if has {
|
||||
c.Status(http.StatusNoContent)
|
||||
return
|
||||
}
|
||||
c.Status(http.StatusNotFound)
|
||||
}
|
||||
|
||||
func (s *Server) recoveryHandler(c *gin.Context, err interface{}) {
|
||||
c.JSON(500, gin.H{
|
||||
"title": "Error",
|
||||
"err": err,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
|
||||
"github.com/bluele/gcache"
|
||||
nice "github.com/ekyoung/gin-nice-recovery"
|
||||
"github.com/gin-gonic/gin"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// Server is an instance of a peer server that houses the listener and store.
|
||||
type Server struct {
|
||||
store store.BlobStore
|
||||
grp *stop.Group
|
||||
missesCache gcache.Cache
|
||||
edgeToken string
|
||||
address string
|
||||
concurrentRequests int
|
||||
}
|
||||
|
||||
// NewServer returns an initialized Server pointer.
|
||||
func NewServer(store store.BlobStore, requestQueueSize int, edgeToken string, address string) *Server {
|
||||
return &Server{
|
||||
store: store,
|
||||
grp: stop.New(),
|
||||
concurrentRequests: requestQueueSize,
|
||||
missesCache: gcache.New(2000).Expiration(5 * time.Minute).ARC().Build(),
|
||||
edgeToken: edgeToken,
|
||||
address: address,
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown gracefully shuts down the peer server.
|
||||
func (s *Server) Shutdown() {
|
||||
log.Debug("shutting down HTTP server")
|
||||
s.grp.StopAndWait()
|
||||
log.Debug("HTTP server stopped")
|
||||
}
|
||||
|
||||
// Start starts the server listener to handle connections.
|
||||
func (s *Server) Start() error {
|
||||
gin.SetMode(gin.ReleaseMode)
|
||||
router := gin.New()
|
||||
router.Use(gin.Logger())
|
||||
// Install nice.Recovery, passing the handler to call after recovery
|
||||
router.Use(nice.Recovery(s.recoveryHandler))
|
||||
router.GET("/blob", s.getBlob)
|
||||
router.HEAD("/blob", s.hasBlob)
|
||||
srv := &http.Server{
|
||||
Addr: s.address,
|
||||
Handler: router,
|
||||
}
|
||||
go s.listenForShutdown(srv)
|
||||
go InitWorkers(s, s.concurrentRequests)
|
||||
// Initializing the server in a goroutine so that
|
||||
// it won't block the graceful shutdown handling below
|
||||
s.grp.Add(1)
|
||||
go func() {
|
||||
defer s.grp.Done()
|
||||
log.Println("HTTP server listening on " + s.address)
|
||||
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||
log.Fatalf("listen: %s\n", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) listenForShutdown(listener *http.Server) {
|
||||
<-s.grp.Ch()
|
||||
// The context is used to inform the server it has 5 seconds to finish
|
||||
// the request it is currently handling
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if err := listener.Shutdown(ctx); err != nil {
|
||||
log.Fatal("Server forced to shutdown:", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
type blobRequest struct {
|
||||
c *gin.Context
|
||||
finished *sync.WaitGroup
|
||||
}
|
||||
|
||||
var getReqCh = make(chan *blobRequest, 20000)
|
||||
|
||||
func InitWorkers(server *Server, workers int) {
|
||||
stopper := stop.New(server.grp)
|
||||
for i := 0; i < workers; i++ {
|
||||
metrics.RoutinesQueue.WithLabelValues("http", "worker").Inc()
|
||||
go func(worker int) {
|
||||
defer metrics.RoutinesQueue.WithLabelValues("http", "worker").Dec()
|
||||
for {
|
||||
select {
|
||||
case <-stopper.Ch():
|
||||
case r := <-getReqCh:
|
||||
process(server, r)
|
||||
metrics.HttpBlobReqQueue.Dec()
|
||||
}
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
}
|
||||
|
||||
func enqueue(b *blobRequest) {
|
||||
metrics.HttpBlobReqQueue.Inc()
|
||||
getReqCh <- b
|
||||
}
|
||||
|
||||
func process(server *Server, r *blobRequest) {
|
||||
server.HandleGetBlob(r.c)
|
||||
r.finished.Done()
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
package http3
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
"github.com/quic-go/quic-go/http3"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// Client is an instance of a client connected to a server.
|
||||
type Client struct {
|
||||
conn *http.Client
|
||||
roundTripper *http3.Transport
|
||||
ServerAddr string
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
// Close closes the connection with the client.
|
||||
func (c *Client) Close() error {
|
||||
c.conn.CloseIdleConnections()
|
||||
return c.roundTripper.Close()
|
||||
}
|
||||
|
||||
// GetStream gets a stream
|
||||
func (c *Client) GetStream(sdHash string, blobCache store.BlobStore) (stream.Stream, error) {
|
||||
var sd stream.SDBlob
|
||||
|
||||
b, _, err := c.GetBlob(sdHash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = sd.FromBlob(b)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s := make(stream.Stream, len(sd.BlobInfos)+1-1) // +1 for sd blob, -1 for last null blob
|
||||
s[0] = b
|
||||
|
||||
for i := 0; i < len(sd.BlobInfos)-1; i++ {
|
||||
var trace shared.BlobTrace
|
||||
s[i+1], trace, err = c.GetBlob(hex.EncodeToString(sd.BlobInfos[i].BlobHash))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
log.Debug(trace.String())
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// HasBlob checks if the blob is available
|
||||
func (c *Client) HasBlob(hash string) (bool, error) {
|
||||
resp, err := c.conn.Get(fmt.Sprintf("https://%s/has/%s", c.ServerAddr, hash))
|
||||
if err != nil {
|
||||
return false, errors.Err(err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode == http.StatusOK {
|
||||
return true, nil
|
||||
}
|
||||
if resp.StatusCode == http.StatusNotFound {
|
||||
return false, nil
|
||||
}
|
||||
return false, errors.Err("non 200 status code returned: %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
// GetBlob gets a blob
|
||||
func (c *Client) GetBlob(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
start := time.Now()
|
||||
resp, err := c.conn.Get(fmt.Sprintf("https://%s/get/%s?trace=true", c.ServerAddr, hash))
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err(err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode == http.StatusNotFound {
|
||||
fmt.Printf("%s blob not found %d\n", hash, resp.StatusCode)
|
||||
return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err(store.ErrBlobNotFound)
|
||||
} else if resp.StatusCode != http.StatusOK {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err("non 200 status code returned: %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
tmp := getBuffer()
|
||||
defer putBuffer(tmp)
|
||||
serialized := resp.Header.Get("Via")
|
||||
trace := shared.NewBlobTrace(time.Since(start), "http3")
|
||||
if serialized != "" {
|
||||
var parsedTrace *shared.BlobTrace
|
||||
parsedTrace, err = shared.Deserialize(serialized)
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), "http3"), err
|
||||
}
|
||||
trace = *parsedTrace
|
||||
}
|
||||
written, err := io.Copy(tmp, resp.Body)
|
||||
if err != nil {
|
||||
return nil, trace.Stack(time.Since(start), "http3"), errors.Err(err)
|
||||
}
|
||||
|
||||
blob := make([]byte, written)
|
||||
copy(blob, tmp.Bytes())
|
||||
|
||||
metrics.MtrInBytesUdp.Add(float64(len(blob)))
|
||||
|
||||
return blob, trace.Stack(time.Since(start), "http3"), nil
|
||||
}
|
||||
|
||||
// buffer pool to reduce GC
|
||||
// https://www.captaincodeman.com/2017/06/02/golang-buffer-pool-gotcha
|
||||
var buffers = sync.Pool{
|
||||
// New is called when a new instance is needed
|
||||
New: func() interface{} {
|
||||
buf := make([]byte, 0, stream.MaxBlobSize)
|
||||
return bytes.NewBuffer(buf)
|
||||
},
|
||||
}
|
||||
|
||||
// getBuffer fetches a buffer from the pool
|
||||
func getBuffer() *bytes.Buffer {
|
||||
return buffers.Get().(*bytes.Buffer)
|
||||
}
|
||||
|
||||
// putBuffer returns a buffer to the pool
|
||||
func putBuffer(buf *bytes.Buffer) {
|
||||
buf.Reset()
|
||||
buffers.Put(buf)
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
package http3
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/quic-go/quic-go"
|
||||
"github.com/quic-go/quic-go/http3"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// Server is an instance of a peer server that houses the listener and store.
|
||||
type Server struct {
|
||||
store store.BlobStore
|
||||
grp *stop.Group
|
||||
address string
|
||||
concurrentRequests int
|
||||
}
|
||||
|
||||
// NewServer returns an initialized Server pointer.
|
||||
func NewServer(store store.BlobStore, requestQueueSize int, address string) *Server {
|
||||
return &Server{
|
||||
store: store,
|
||||
grp: stop.New(),
|
||||
concurrentRequests: requestQueueSize,
|
||||
address: address,
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown gracefully shuts down the peer server.
|
||||
func (s *Server) Shutdown() {
|
||||
log.Debug("shutting down http3 peer server")
|
||||
s.grp.StopAndWait()
|
||||
log.Debug("http3 peer server stopped")
|
||||
}
|
||||
|
||||
func (s *Server) logError(e error) {
|
||||
if e == nil {
|
||||
return
|
||||
}
|
||||
shouldLog := metrics.TrackError(metrics.DirectionDownload, e)
|
||||
if shouldLog {
|
||||
log.Errorln(errors.FullTrace(e))
|
||||
}
|
||||
}
|
||||
|
||||
type availabilityResponse struct {
|
||||
LbrycrdAddress string `json:"lbrycrd_address"`
|
||||
IsAvailable bool `json:"is_available"`
|
||||
}
|
||||
|
||||
// Start starts the server listener to handle connections.
|
||||
func (s *Server) Start() error {
|
||||
log.Println("HTTP3 peer listening on " + s.address)
|
||||
window500M := 500 * 1 << 20
|
||||
|
||||
quicConf := &quic.Config{
|
||||
MaxStreamReceiveWindow: uint64(window500M),
|
||||
MaxConnectionReceiveWindow: uint64(window500M),
|
||||
EnableDatagrams: true,
|
||||
HandshakeIdleTimeout: 4 * time.Second,
|
||||
MaxIdleTimeout: 20 * time.Second,
|
||||
}
|
||||
r := mux.NewRouter()
|
||||
r.HandleFunc("/get/{hash}", func(w http.ResponseWriter, r *http.Request) {
|
||||
waiter := &sync.WaitGroup{}
|
||||
waiter.Add(1)
|
||||
enqueue(&blobRequest{request: r, reply: w, finished: waiter})
|
||||
waiter.Wait()
|
||||
})
|
||||
r.HandleFunc("/has/{hash}", func(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
requestedBlob := vars["hash"]
|
||||
blobExists, err := s.store.Has(requestedBlob)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
s.logError(err)
|
||||
return
|
||||
}
|
||||
if !blobExists {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
// LbrycrdAddress to be used when paying for data. Not implemented yet.
|
||||
const LbrycrdAddress = "bJxKvpD96kaJLriqVajZ7SaQTsWWyrGQct"
|
||||
resp, err := json.Marshal(availabilityResponse{
|
||||
LbrycrdAddress: LbrycrdAddress,
|
||||
IsAvailable: blobExists,
|
||||
})
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
s.logError(err)
|
||||
return
|
||||
}
|
||||
_, err = w.Write(resp)
|
||||
if err != nil {
|
||||
s.logError(err)
|
||||
}
|
||||
})
|
||||
server := http3.Server{
|
||||
Addr: s.address,
|
||||
Handler: r,
|
||||
TLSConfig: generateTLSConfig(),
|
||||
QUICConfig: quicConf,
|
||||
}
|
||||
go InitWorkers(s, s.concurrentRequests)
|
||||
go s.listenForShutdown(&server)
|
||||
s.grp.Add(1)
|
||||
go func() {
|
||||
s.listenAndServe(&server)
|
||||
s.grp.Done()
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Setup a bare-bones TLS config for the server
|
||||
func generateTLSConfig() *tls.Config {
|
||||
key, err := rsa.GenerateKey(rand.Reader, 1024)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
template := x509.Certificate{SerialNumber: big.NewInt(1)}
|
||||
certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})
|
||||
certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
|
||||
|
||||
tlsCert, err := tls.X509KeyPair(certPEM, keyPEM)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return &tls.Config{
|
||||
Certificates: []tls.Certificate{tlsCert},
|
||||
NextProtos: []string{"http3-reflector-server"},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) listenAndServe(server *http3.Server) {
|
||||
err := server.ListenAndServe()
|
||||
if err != nil && !strings.Contains(err.Error(), "Server closed") {
|
||||
log.Errorln(errors.FullTrace(err))
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) listenForShutdown(listener *http3.Server) {
|
||||
<-s.grp.Ch()
|
||||
err := listener.Close()
|
||||
if err != nil {
|
||||
log.Error("error closing listener for peer server - ", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) HandleGetBlob(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
requestedBlob := vars["hash"]
|
||||
traceParam := r.URL.Query().Get("trace")
|
||||
var err error
|
||||
wantsTrace := false
|
||||
if traceParam != "" {
|
||||
wantsTrace, err = strconv.ParseBool(traceParam)
|
||||
if err != nil {
|
||||
wantsTrace = false
|
||||
}
|
||||
}
|
||||
if reflector.IsProtected(requestedBlob) {
|
||||
http.Error(w, "requested blob is protected", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
blob, trace, err := s.store.Get(requestedBlob)
|
||||
|
||||
if wantsTrace {
|
||||
var serialized string
|
||||
serialized, err = trace.Serialize()
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
w.Header().Add("Via", serialized)
|
||||
log.Debug(trace.String())
|
||||
}
|
||||
if err != nil {
|
||||
if errors.Is(err, store.ErrBlobNotFound) {
|
||||
http.Error(w, err.Error(), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
fmt.Printf("%s: %s", requestedBlob, errors.FullTrace(err))
|
||||
s.logError(err)
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
_, err = w.Write(blob)
|
||||
if err != nil {
|
||||
s.logError(err)
|
||||
}
|
||||
metrics.MtrOutBytesUdp.Add(float64(len(blob)))
|
||||
metrics.BlobDownloadCount.Inc()
|
||||
metrics.Http3DownloadCount.Inc()
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package http3
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"sync"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
)
|
||||
|
||||
type blobRequest struct {
|
||||
request *http.Request
|
||||
reply http.ResponseWriter
|
||||
finished *sync.WaitGroup
|
||||
}
|
||||
|
||||
var getReqCh = make(chan *blobRequest, 20000)
|
||||
|
||||
func InitWorkers(server *Server, workers int) {
|
||||
stopper := stop.New(server.grp)
|
||||
for i := 0; i < workers; i++ {
|
||||
metrics.RoutinesQueue.WithLabelValues("http3", "worker").Inc()
|
||||
go func(worker int) {
|
||||
defer metrics.RoutinesQueue.WithLabelValues("http3", "worker").Dec()
|
||||
for {
|
||||
select {
|
||||
case <-stopper.Ch():
|
||||
case r := <-getReqCh:
|
||||
metrics.Http3BlobReqQueue.Dec()
|
||||
process(server, r)
|
||||
}
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
}
|
||||
|
||||
func enqueue(b *blobRequest) {
|
||||
metrics.Http3BlobReqQueue.Inc()
|
||||
getReqCh <- b
|
||||
}
|
||||
|
||||
func process(server *Server, r *blobRequest) {
|
||||
server.HandleGetBlob(r.reply, r.request)
|
||||
r.finished.Done()
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
package peer
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// Client is an instance of a client connected to a server.
|
||||
type Client struct {
|
||||
conn net.Conn
|
||||
buf *bufio.Reader
|
||||
Timeout time.Duration
|
||||
connected bool
|
||||
}
|
||||
|
||||
// Connect connects to a specific clients and errors if it cannot be contacted.
|
||||
func (c *Client) Connect(address string) error {
|
||||
var err error
|
||||
if c.Timeout == 0 {
|
||||
c.Timeout = 5 * time.Second
|
||||
}
|
||||
c.conn, err = net.Dial("tcp4", address)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.connected = true
|
||||
c.buf = bufio.NewReader(c.conn)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close closes the connection with the client.
|
||||
func (c *Client) Close() error {
|
||||
c.connected = false
|
||||
return c.conn.Close()
|
||||
}
|
||||
|
||||
// GetStream gets a stream
|
||||
func (c *Client) GetStream(sdHash string, blobCache store.BlobStore) (stream.Stream, error) {
|
||||
if !c.connected {
|
||||
return nil, errors.Err("not connected")
|
||||
}
|
||||
|
||||
var sd stream.SDBlob
|
||||
|
||||
b, trace, err := c.GetBlob(sdHash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
log.Debug(trace.String())
|
||||
|
||||
err = sd.FromBlob(b)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s := make(stream.Stream, len(sd.BlobInfos)+1-1) // +1 for sd blob, -1 for last null blob
|
||||
s[0] = b
|
||||
|
||||
for i := 0; i < len(sd.BlobInfos)-1; i++ {
|
||||
s[i+1], trace, err = c.GetBlob(hex.EncodeToString(sd.BlobInfos[i].BlobHash))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
log.Debug(trace.String())
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// HasBlob checks if the blob is available
|
||||
func (c *Client) HasBlob(hash string) (bool, error) {
|
||||
if !c.connected {
|
||||
return false, errors.Err("not connected")
|
||||
}
|
||||
|
||||
sendRequest, err := json.Marshal(availabilityRequest{
|
||||
RequestedBlobs: []string{hash},
|
||||
})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
err = c.write(sendRequest)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
var resp availabilityResponse
|
||||
err = c.read(&resp)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
for _, h := range resp.AvailableBlobs {
|
||||
if h == hash {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// GetBlob gets a blob
|
||||
func (c *Client) GetBlob(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
start := time.Now()
|
||||
if !c.connected {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), "tcp"), errors.Err("not connected")
|
||||
}
|
||||
|
||||
sendRequest, err := json.Marshal(blobRequest{
|
||||
RequestedBlob: hash,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), "tcp"), err
|
||||
}
|
||||
|
||||
err = c.write(sendRequest)
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), "tcp"), err
|
||||
}
|
||||
|
||||
var resp blobResponse
|
||||
err = c.read(&resp)
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), "tcp"), err
|
||||
}
|
||||
|
||||
trace := shared.NewBlobTrace(time.Since(start), "tcp")
|
||||
if resp.RequestTrace != nil {
|
||||
trace = *resp.RequestTrace
|
||||
}
|
||||
if resp.IncomingBlob.Error != "" {
|
||||
return nil, trace, errors.Prefix(hash[:8], resp.IncomingBlob.Error)
|
||||
}
|
||||
if resp.IncomingBlob.BlobHash != hash {
|
||||
return nil, trace.Stack(time.Since(start), "tcp"), errors.Prefix(hash[:8], "blob hash in response does not match requested hash")
|
||||
}
|
||||
if resp.IncomingBlob.Length <= 0 {
|
||||
return nil, trace, errors.Prefix(hash[:8], "length reported as <= 0")
|
||||
}
|
||||
|
||||
log.Debugf("receiving blob %s from %s", hash[:8], c.conn.RemoteAddr())
|
||||
|
||||
blob, err := c.readRawBlob(resp.IncomingBlob.Length)
|
||||
if err != nil {
|
||||
return nil, (*resp.RequestTrace).Stack(time.Since(start), "tcp"), err
|
||||
}
|
||||
metrics.MtrInBytesTcp.Add(float64(len(blob)))
|
||||
return blob, trace.Stack(time.Since(start), "tcp"), nil
|
||||
}
|
||||
|
||||
func (c *Client) read(v interface{}) error {
|
||||
err := c.conn.SetReadDeadline(time.Now().Add(c.Timeout))
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
m, err := readNextMessage(c.buf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Debugf("read %d bytes from %s", len(m), c.conn.RemoteAddr())
|
||||
|
||||
err = json.Unmarshal(m, v)
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
func (c *Client) readRawBlob(blobSize int) ([]byte, error) {
|
||||
err := c.conn.SetReadDeadline(time.Now().Add(c.Timeout))
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
blob := make([]byte, blobSize)
|
||||
n, err := io.ReadFull(c.buf, blob)
|
||||
log.Debugf("read %d bytes from %s", n, c.conn.RemoteAddr())
|
||||
return blob, errors.Err(err)
|
||||
}
|
||||
|
||||
func (c *Client) write(b []byte) error {
|
||||
err := c.conn.SetWriteDeadline(time.Now().Add(c.Timeout))
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
log.Debugf("writing %d bytes to %s", len(b), c.conn.RemoteAddr())
|
||||
|
||||
n, err := c.conn.Write(b)
|
||||
if err == nil && n != len(b) {
|
||||
err = io.ErrShortWrite
|
||||
}
|
||||
return errors.Err(err)
|
||||
}
|
||||
@@ -0,0 +1,422 @@
|
||||
package peer
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
ee "errors"
|
||||
"io"
|
||||
"net"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const (
|
||||
// DefaultPort is the port the peer server listens on if not passed in.
|
||||
DefaultPort = 3333
|
||||
// LbrycrdAddress to be used when paying for data. Not implemented yet.
|
||||
LbrycrdAddress = "bJxKvpD96kaJLriqVajZ7SaQTsWWyrGQct"
|
||||
)
|
||||
|
||||
// Server is an instance of a peer server that houses the listener and store.
|
||||
type Server struct {
|
||||
store store.BlobStore
|
||||
grp *stop.Group
|
||||
address string
|
||||
closed bool
|
||||
}
|
||||
|
||||
// NewServer returns an initialized Server pointer.
|
||||
func NewServer(store store.BlobStore, address string) *Server {
|
||||
return &Server{
|
||||
store: store,
|
||||
grp: stop.New(),
|
||||
address: address,
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown gracefully shuts down the peer server.
|
||||
func (s *Server) Shutdown() {
|
||||
log.Debug("shutting down peer server")
|
||||
s.grp.StopAndWait()
|
||||
log.Debug("peer server stopped")
|
||||
}
|
||||
|
||||
// Start starts the server listener to handle connections.
|
||||
func (s *Server) Start() error {
|
||||
log.Println("peer listening on " + s.address)
|
||||
l, err := net.Listen("tcp4", s.address)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
go s.listenForShutdown(l)
|
||||
s.grp.Add(1)
|
||||
go func() {
|
||||
s.listenAndServe(l)
|
||||
s.grp.Done()
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) listenForShutdown(listener net.Listener) {
|
||||
<-s.grp.Ch()
|
||||
s.closed = true
|
||||
err := listener.Close()
|
||||
if err != nil {
|
||||
log.Error("error closing listener for peer server - ", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) listenAndServe(listener net.Listener) {
|
||||
for {
|
||||
conn, err := listener.Accept()
|
||||
if err != nil {
|
||||
if s.closed {
|
||||
return
|
||||
}
|
||||
log.Error(errors.Prefix("accepting conn", err))
|
||||
} else {
|
||||
s.grp.Add(1)
|
||||
metrics.RoutinesQueue.WithLabelValues("peer", "server-handleconn").Inc()
|
||||
go func() {
|
||||
defer metrics.RoutinesQueue.WithLabelValues("peer", "server-handleconn").Dec()
|
||||
s.handleConnection(conn)
|
||||
s.grp.Done()
|
||||
}()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) handleConnection(conn net.Conn) {
|
||||
defer func() {
|
||||
if err := conn.Close(); err != nil {
|
||||
log.Error(errors.Prefix("closing peer conn", err))
|
||||
}
|
||||
}()
|
||||
|
||||
timeoutDuration := 1 * time.Minute
|
||||
buf := bufio.NewReader(conn)
|
||||
|
||||
for {
|
||||
var request []byte
|
||||
var response []byte
|
||||
|
||||
err := conn.SetReadDeadline(time.Now().Add(timeoutDuration))
|
||||
if err != nil {
|
||||
log.Error(errors.FullTrace(err))
|
||||
}
|
||||
|
||||
request, err = readNextMessage(buf)
|
||||
if err != nil {
|
||||
if err != io.EOF {
|
||||
s.logError(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
err = conn.SetReadDeadline(time.Time{})
|
||||
if err != nil {
|
||||
log.Error(errors.FullTrace(err))
|
||||
}
|
||||
|
||||
response, err = s.handleCompositeRequest(request)
|
||||
if err != nil {
|
||||
log.Error(errors.FullTrace(err))
|
||||
return
|
||||
}
|
||||
|
||||
err = conn.SetWriteDeadline(time.Now().Add(timeoutDuration))
|
||||
if err != nil {
|
||||
log.Error(errors.FullTrace(err))
|
||||
}
|
||||
|
||||
n, err := conn.Write(response)
|
||||
if err != nil {
|
||||
if !strings.Contains(err.Error(), "connection reset by peer") { // means the other side closed the connection using TCP reset
|
||||
s.logError(err)
|
||||
}
|
||||
return
|
||||
} else if n != len(response) {
|
||||
log.Errorln(io.ErrShortWrite)
|
||||
return
|
||||
}
|
||||
|
||||
err = conn.SetWriteDeadline(time.Time{})
|
||||
if err != nil {
|
||||
log.Error(errors.FullTrace(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) handleAvailabilityRequest(data []byte) ([]byte, error) {
|
||||
var request availabilityRequest
|
||||
err := json.Unmarshal(data, &request)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
availableBlobs := []string{}
|
||||
for _, blobHash := range request.RequestedBlobs {
|
||||
exists, err := s.store.Has(blobHash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if exists {
|
||||
availableBlobs = append(availableBlobs, blobHash)
|
||||
}
|
||||
}
|
||||
|
||||
return json.Marshal(availabilityResponse{LbrycrdAddress: LbrycrdAddress, AvailableBlobs: availableBlobs})
|
||||
}
|
||||
|
||||
//func (s *Server) handlePaymentRateNegotiation(data []byte) ([]byte, error) {
|
||||
// var request paymentRateRequest
|
||||
// err := json.Unmarshal(data, &request)
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
//
|
||||
// offerReply := paymentRateAccepted
|
||||
// if request.BlobDataPaymentRate < 0 {
|
||||
// offerReply = paymentRateTooLow
|
||||
// }
|
||||
//
|
||||
// return json.Marshal(paymentRateResponse{BlobDataPaymentRate: offerReply})
|
||||
//}
|
||||
//
|
||||
//func (s *Server) handleBlobRequest(data []byte) ([]byte, error) {
|
||||
// var request blobRequest
|
||||
// err := json.Unmarshal(data, &request)
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
//
|
||||
// log.Debugln("Sending blob " + request.RequestedBlob[:8])
|
||||
//
|
||||
// blob, err := s.store.Get(request.RequestedBlob)
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
//
|
||||
// response, err := json.Marshal(blobResponse{IncomingBlob: incomingBlob{
|
||||
// BlobHash: reflector.BlobHash(blob),
|
||||
// Length: len(blob),
|
||||
// }})
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
//
|
||||
// return append(response, blob...), nil
|
||||
//}
|
||||
|
||||
func (s *Server) handleCompositeRequest(data []byte) ([]byte, error) {
|
||||
var request compositeRequest
|
||||
err := json.Unmarshal(data, &request)
|
||||
if err != nil {
|
||||
var je *json.SyntaxError
|
||||
if ee.As(err, &je) {
|
||||
return nil, errors.Err("invalid json request: offset %d in data %s", je.Offset, hex.EncodeToString(data))
|
||||
}
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
response := compositeResponse{
|
||||
LbrycrdAddress: LbrycrdAddress,
|
||||
AvailableBlobs: []string{},
|
||||
}
|
||||
|
||||
if len(request.RequestedBlobs) > 0 {
|
||||
for _, blobHash := range request.RequestedBlobs {
|
||||
if reflector.IsProtected(blobHash) {
|
||||
return nil, errors.Err("requested blob is protected")
|
||||
}
|
||||
var exists bool
|
||||
exists, err = s.store.Has(blobHash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if exists {
|
||||
response.AvailableBlobs = append(response.AvailableBlobs, blobHash)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if request.BlobDataPaymentRate != nil {
|
||||
response.BlobDataPaymentRate = paymentRateAccepted
|
||||
if *request.BlobDataPaymentRate < 0 {
|
||||
response.BlobDataPaymentRate = paymentRateTooLow
|
||||
}
|
||||
}
|
||||
|
||||
var blob []byte
|
||||
var trace shared.BlobTrace
|
||||
if request.RequestedBlob != "" {
|
||||
if len(request.RequestedBlob) != stream.BlobHashHexLength {
|
||||
return nil, errors.Err("Invalid blob hash length")
|
||||
}
|
||||
|
||||
log.Debugln("Sending blob " + request.RequestedBlob[:8])
|
||||
|
||||
blob, trace, err = s.store.Get(request.RequestedBlob)
|
||||
log.Debug(trace.String())
|
||||
if errors.Is(err, store.ErrBlobNotFound) {
|
||||
response.IncomingBlob = &incomingBlob{
|
||||
Error: err.Error(),
|
||||
}
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
response.IncomingBlob = &incomingBlob{
|
||||
BlobHash: request.RequestedBlob,
|
||||
Length: len(blob),
|
||||
}
|
||||
metrics.MtrOutBytesTcp.Add(float64(len(blob)))
|
||||
metrics.BlobDownloadCount.Inc()
|
||||
metrics.PeerDownloadCount.Inc()
|
||||
}
|
||||
}
|
||||
|
||||
respData, err := json.Marshal(response)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return append(respData, blob...), nil
|
||||
}
|
||||
|
||||
func (s *Server) logError(e error) {
|
||||
if e == nil {
|
||||
return
|
||||
}
|
||||
shouldLog := metrics.TrackError(metrics.DirectionDownload, e)
|
||||
if shouldLog {
|
||||
log.Errorln(errors.FullTrace(e))
|
||||
}
|
||||
}
|
||||
|
||||
func readNextMessage(buf *bufio.Reader) ([]byte, error) {
|
||||
firstByte, err := buf.ReadByte()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if firstByte != '{' {
|
||||
// every request starts with '{'. Checking here disconnects earlier, so we don't wait until timeout
|
||||
return nil, errInvalidData
|
||||
}
|
||||
msg := []byte("{")
|
||||
eof := false
|
||||
|
||||
for {
|
||||
chunk, err := buf.ReadBytes('}')
|
||||
if err != nil {
|
||||
if err != io.EOF {
|
||||
//log.Errorln("readBytes error:", err) // logged by caller
|
||||
return msg, err
|
||||
}
|
||||
eof = true
|
||||
}
|
||||
|
||||
//log.Debugln("got", len(chunk), "bytes.")
|
||||
//spew.Dump(chunk)
|
||||
|
||||
if len(chunk) > 0 {
|
||||
msg = append(msg, chunk...)
|
||||
|
||||
if len(msg) > maxRequestSize {
|
||||
return msg, errRequestTooLarge
|
||||
} else if len(msg) > 0 && msg[0] != '{' {
|
||||
return msg, errInvalidData
|
||||
}
|
||||
|
||||
// yes, this is how the peer protocol knows when the request finishes
|
||||
if reflector.IsValidJSON(msg) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if eof {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
//log.Debugln("total size:", len(request))
|
||||
//if len(request) > 0 {
|
||||
// spew.Dump(request)
|
||||
//}
|
||||
|
||||
if len(msg) == 0 && eof {
|
||||
return nil, io.EOF
|
||||
}
|
||||
|
||||
return msg, nil
|
||||
}
|
||||
|
||||
const (
|
||||
maxRequestSize = 64 * (2 ^ 10) // 64kb
|
||||
paymentRateAccepted = "RATE_ACCEPTED"
|
||||
paymentRateTooLow = "RATE_TOO_LOW"
|
||||
//ToDo: paymentRateUnset is not used but exists in the protocol.
|
||||
//paymentRateUnset = "RATE_UNSET"
|
||||
)
|
||||
|
||||
var errRequestTooLarge = errors.Base("request is too large")
|
||||
var errInvalidData = errors.Base("Invalid data")
|
||||
|
||||
type availabilityRequest struct {
|
||||
RequestedBlobs []string `json:"requested_blobs"`
|
||||
LbrycrdAddress bool `json:"lbrycrd_address"`
|
||||
}
|
||||
|
||||
type availabilityResponse struct {
|
||||
LbrycrdAddress string `json:"lbrycrd_address"`
|
||||
AvailableBlobs []string `json:"available_blobs"`
|
||||
}
|
||||
|
||||
type paymentRateRequest struct {
|
||||
BlobDataPaymentRate float64 `json:"blob_data_payment_rate"`
|
||||
}
|
||||
|
||||
type paymentRateResponse struct {
|
||||
BlobDataPaymentRate string `json:"blob_data_payment_rate"`
|
||||
}
|
||||
|
||||
type blobRequest struct {
|
||||
RequestedBlob string `json:"requested_blob"`
|
||||
}
|
||||
|
||||
type incomingBlob struct {
|
||||
Error string `json:"error,omitempty"`
|
||||
BlobHash string `json:"blob_hash"`
|
||||
Length int `json:"length"`
|
||||
}
|
||||
type blobResponse struct {
|
||||
RequestTrace *shared.BlobTrace
|
||||
IncomingBlob incomingBlob `json:"incoming_blob"`
|
||||
}
|
||||
|
||||
type compositeRequest struct {
|
||||
BlobDataPaymentRate *float64 `json:"blob_data_payment_rate"`
|
||||
RequestedBlob string `json:"requested_blob"`
|
||||
RequestedBlobs []string `json:"requested_blobs"`
|
||||
LbrycrdAddress bool `json:"lbrycrd_address"`
|
||||
}
|
||||
|
||||
type compositeResponse struct {
|
||||
IncomingBlob *incomingBlob `json:"incoming_blob,omitempty"`
|
||||
LbrycrdAddress string `json:"lbrycrd_address,omitempty"`
|
||||
BlobDataPaymentRate string `json:"blob_data_payment_rate,omitempty"`
|
||||
AvailableBlobs []string `json:"available_blobs"`
|
||||
}
|
||||
@@ -2,7 +2,10 @@ package peer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"net"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
)
|
||||
@@ -34,7 +37,9 @@ var availabilityRequests = []pair{
|
||||
}
|
||||
|
||||
func getServer(t *testing.T, withBlobs bool) *Server {
|
||||
st := store.MemoryBlobStore{}
|
||||
st := store.NewMemStore(store.MemParams{
|
||||
Name: "test",
|
||||
})
|
||||
if withBlobs {
|
||||
for k, v := range blobs {
|
||||
err := st.Put(k, v)
|
||||
@@ -43,7 +48,7 @@ func getServer(t *testing.T, withBlobs bool) *Server {
|
||||
}
|
||||
}
|
||||
}
|
||||
return NewServer(&st)
|
||||
return NewServer(st, "127.0.0.1:50505")
|
||||
}
|
||||
|
||||
func TestAvailabilityRequest_NoBlobs(t *testing.T) {
|
||||
@@ -75,3 +80,62 @@ func TestAvailabilityRequest_WithBlobs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestFromConnection(t *testing.T) {
|
||||
s := getServer(t, true)
|
||||
err := s.Start()
|
||||
defer s.Shutdown()
|
||||
if err != nil {
|
||||
t.Error("error starting server", err)
|
||||
}
|
||||
|
||||
for _, p := range availabilityRequests {
|
||||
conn, err := net.Dial("tcp", "127.0.0.1:50505")
|
||||
if err != nil {
|
||||
t.Error("error opening connection", err)
|
||||
}
|
||||
defer func() { _ = conn.Close() }()
|
||||
|
||||
response := make([]byte, 8192)
|
||||
_, err = conn.Write(p.request)
|
||||
if err != nil {
|
||||
t.Error("error writing", err)
|
||||
}
|
||||
_, err = conn.Read(response)
|
||||
if err != nil {
|
||||
t.Error("error reading", err)
|
||||
}
|
||||
if !bytes.Equal(response[:len(p.response)], p.response) {
|
||||
t.Errorf("Response did not match expected response.\nExpected: %s\nGot: %s", string(p.response), string(response))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvalidData(t *testing.T) {
|
||||
s := getServer(t, true)
|
||||
err := s.Start()
|
||||
defer s.Shutdown()
|
||||
if err != nil {
|
||||
t.Error("error starting server", err)
|
||||
}
|
||||
conn, err := net.Dial("tcp", "127.0.0.1:50505")
|
||||
if err != nil {
|
||||
t.Error("error opening connection", err)
|
||||
}
|
||||
defer func() { _ = conn.Close() }()
|
||||
|
||||
response := make([]byte, 8192)
|
||||
_, err = conn.Write([]byte("hello dear server, I would like blobs. Please"))
|
||||
if err != nil {
|
||||
t.Error("error writing", err)
|
||||
}
|
||||
err = conn.SetReadDeadline(time.Now().Add(5 * time.Second))
|
||||
if err != nil {
|
||||
t.Error("error setting read deadline", err)
|
||||
}
|
||||
_, err = conn.Read(response)
|
||||
if err != io.EOF {
|
||||
t.Error("error reading", err)
|
||||
}
|
||||
println(response)
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package server
|
||||
|
||||
// BlobServer defines the common interface for all blob server implementations
|
||||
type BlobServer interface {
|
||||
Start() error
|
||||
Shutdown()
|
||||
}
|
||||
|
||||
type BlobServerConfig struct {
|
||||
Address string `mapstructure:"address"`
|
||||
EdgeToken string `mapstructure:"edge_token"`
|
||||
Port int `mapstructure:"port"`
|
||||
MaxConcurrentRequests int `mapstructure:"max_concurrent_requests"`
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
package shared
|
||||
|
||||
import "github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
|
||||
// ErrNotImplemented is a standard error when a store that implements the store interface does not implement a method
|
||||
var ErrNotImplemented = errors.Base("this store does not implement this method")
|
||||
@@ -0,0 +1,82 @@
|
||||
package shared
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
)
|
||||
|
||||
type BlobStack struct {
|
||||
OriginName string `json:"origin_name"`
|
||||
HostName string `json:"host_name"`
|
||||
Timing time.Duration `json:"timing"`
|
||||
}
|
||||
type BlobTrace struct {
|
||||
Stacks []BlobStack `json:"stacks"`
|
||||
}
|
||||
|
||||
var hostName *string
|
||||
|
||||
func getHostName() string {
|
||||
if hostName == nil {
|
||||
hn, err := os.Hostname()
|
||||
if err != nil {
|
||||
hn = "unknown"
|
||||
}
|
||||
hostName = &hn
|
||||
}
|
||||
return *hostName
|
||||
}
|
||||
func (b *BlobTrace) Stack(timing time.Duration, originName string) BlobTrace {
|
||||
b.Stacks = append(b.Stacks, BlobStack{
|
||||
Timing: timing,
|
||||
OriginName: originName,
|
||||
HostName: getHostName(),
|
||||
})
|
||||
return *b
|
||||
}
|
||||
func (b *BlobTrace) Merge(otherTrance BlobTrace) BlobTrace {
|
||||
b.Stacks = append(b.Stacks, otherTrance.Stacks...)
|
||||
return *b
|
||||
}
|
||||
func NewBlobTrace(timing time.Duration, originName string) BlobTrace {
|
||||
b := BlobTrace{}
|
||||
b.Stacks = append(b.Stacks, BlobStack{
|
||||
Timing: timing,
|
||||
OriginName: originName,
|
||||
HostName: getHostName(),
|
||||
})
|
||||
return b
|
||||
}
|
||||
|
||||
func (b BlobTrace) String() string {
|
||||
var fullTrace string
|
||||
for i, stack := range b.Stacks {
|
||||
delta := time.Duration(0)
|
||||
if i > 0 {
|
||||
delta = stack.Timing - b.Stacks[i-1].Timing
|
||||
}
|
||||
fullTrace += fmt.Sprintf("[%d](%s) origin: %s - timing: %s - delta: %s\n", i, stack.HostName, stack.OriginName, stack.Timing.String(), delta.String())
|
||||
}
|
||||
return fullTrace
|
||||
}
|
||||
|
||||
func (b BlobTrace) Serialize() (string, error) {
|
||||
t, err := json.Marshal(b)
|
||||
if err != nil {
|
||||
return "", errors.Err(err)
|
||||
}
|
||||
return string(t), nil
|
||||
}
|
||||
|
||||
func Deserialize(serializedData string) (*BlobTrace, error) {
|
||||
var trace BlobTrace
|
||||
err := json.Unmarshal([]byte(serializedData), &trace)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
return &trace, nil
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package shared
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestBlobTrace_Serialize(t *testing.T) {
|
||||
hostName = util.PtrToString("test_machine")
|
||||
stack := NewBlobTrace(10*time.Second, "test")
|
||||
stack.Stack(20*time.Second, "test2")
|
||||
stack.Stack(30*time.Second, "test3")
|
||||
serialized, err := stack.Serialize()
|
||||
assert.NoError(t, err)
|
||||
|
||||
deserialized, err := Deserialize(serialized)
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, deserialized.Stacks, 3)
|
||||
assert.Equal(t, 10*time.Second, deserialized.Stacks[0].Timing)
|
||||
assert.Equal(t, "test", deserialized.Stacks[0].OriginName)
|
||||
assert.Equal(t, "test_machine", deserialized.Stacks[0].HostName)
|
||||
assert.Equal(t, 20*time.Second, deserialized.Stacks[1].Timing)
|
||||
assert.Equal(t, "test2", deserialized.Stacks[1].OriginName)
|
||||
assert.Equal(t, 30*time.Second, deserialized.Stacks[2].Timing)
|
||||
assert.Equal(t, "test3", deserialized.Stacks[2].OriginName)
|
||||
}
|
||||
|
||||
func TestBlobTrace_Deserialize(t *testing.T) {
|
||||
hostName = util.PtrToString("test_machine")
|
||||
serialized := "{\"stacks\":[{\"timing\":10000000000,\"origin_name\":\"test\"},{\"timing\":20000000000,\"origin_name\":\"test2\"},{\"timing\":30000000000,\"origin_name\":\"test3\"}]}"
|
||||
stack, err := Deserialize(serialized)
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, stack.Stacks, 3)
|
||||
assert.Equal(t, stack.Stacks[0].Timing, 10*time.Second)
|
||||
assert.Equal(t, stack.Stacks[1].Timing, 20*time.Second)
|
||||
assert.Equal(t, stack.Stacks[2].Timing, 30*time.Second)
|
||||
assert.Equal(t, stack.Stacks[0].OriginName, "test")
|
||||
assert.Equal(t, stack.Stacks[1].OriginName, "test2")
|
||||
assert.Equal(t, stack.Stacks[2].OriginName, "test3")
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
//go:build linux
|
||||
// +build linux
|
||||
|
||||
package store
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
func timespecToTime(ts syscall.Timespec) time.Time {
|
||||
return time.Unix(ts.Sec, ts.Nsec)
|
||||
}
|
||||
|
||||
func atime(fi os.FileInfo) time.Time {
|
||||
return timespecToTime(fi.Sys().(*syscall.Stat_t).Atim)
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
//go:build !linux
|
||||
// +build !linux
|
||||
|
||||
package store
|
||||
|
||||
import (
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
func atime(fi os.FileInfo) time.Time {
|
||||
return fi.ModTime()
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
// CachingStore combines two stores, typically a local and a remote store, to improve performance.
|
||||
// Accessed blobs are stored in and retrieved from the cache. If they are not in the cache, they
|
||||
// are retrieved from the origin and cached. Puts are cached and also forwarded to the origin.
|
||||
type CachingStore struct {
|
||||
origin, cache BlobStore
|
||||
name string
|
||||
}
|
||||
|
||||
type CachingParams struct {
|
||||
Origin BlobStore `mapstructure:"origin"`
|
||||
Cache BlobStore `mapstructure:"cache"`
|
||||
Name string `mapstructure:"name"`
|
||||
}
|
||||
|
||||
type CachingConfig struct {
|
||||
Origin *viper.Viper
|
||||
Cache *viper.Viper
|
||||
Name string `mapstructure:"name"`
|
||||
}
|
||||
|
||||
// NewCachingStore makes a new caching disk store and returns a pointer to it.
|
||||
func NewCachingStore(params CachingParams) *CachingStore {
|
||||
return &CachingStore{
|
||||
name: params.Name,
|
||||
origin: WithSingleFlight(params.Name, params.Origin),
|
||||
cache: WithSingleFlight(params.Name, params.Cache),
|
||||
}
|
||||
}
|
||||
|
||||
const nameCaching = "caching"
|
||||
|
||||
func CachingStoreFactory(config *viper.Viper) (BlobStore, error) {
|
||||
var cfg CachingConfig
|
||||
err := config.Unmarshal(&cfg)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
cfg.Cache = config.Sub("cache")
|
||||
cfg.Origin = config.Sub("origin")
|
||||
if cfg.Cache == nil || cfg.Origin == nil {
|
||||
return nil, errors.Err("cache and origin missing")
|
||||
}
|
||||
|
||||
originStoreType := strings.Split(cfg.Origin.AllKeys()[0], ".")[0]
|
||||
originStoreConfig := cfg.Origin.Sub(originStoreType)
|
||||
factory, ok := Factories[originStoreType]
|
||||
if !ok {
|
||||
return nil, errors.Err("unknown store type %s", originStoreType)
|
||||
}
|
||||
originStore, err := factory(originStoreConfig)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
cacheStoreType := strings.Split(cfg.Cache.AllKeys()[0], ".")[0]
|
||||
cacheStoreConfig := cfg.Cache.Sub(cacheStoreType)
|
||||
factory, ok = Factories[cacheStoreType]
|
||||
if !ok {
|
||||
return nil, errors.Err("unknown store type %s", cacheStoreType)
|
||||
}
|
||||
cacheStore, err := factory(cacheStoreConfig)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
return NewCachingStore(CachingParams{
|
||||
Name: cfg.Name,
|
||||
Origin: originStore,
|
||||
Cache: cacheStore,
|
||||
}), nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
RegisterStore(nameCaching, CachingStoreFactory)
|
||||
}
|
||||
|
||||
// Name is the cache type name
|
||||
func (c *CachingStore) Name() string { return nameCaching }
|
||||
|
||||
// Has checks the cache and then the origin for a hash. It returns true if either store has it.
|
||||
func (c *CachingStore) Has(hash string) (bool, error) {
|
||||
has, err := c.cache.Has(hash)
|
||||
if has || err != nil {
|
||||
return has, err
|
||||
}
|
||||
return c.origin.Has(hash)
|
||||
}
|
||||
|
||||
// Get tries to get the blob from the cache first, falling back to the origin. If the blob comes
|
||||
// from the origin, it is also stored in the cache.
|
||||
func (c *CachingStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
start := time.Now()
|
||||
blob, trace, err := c.cache.Get(hash)
|
||||
if err == nil || !errors.Is(err, ErrBlobNotFound) {
|
||||
metrics.CacheHitCount.With(metrics.CacheLabels(c.cache.Name(), c.name)).Inc()
|
||||
rate := float64(len(blob)) / 1024 / 1024 / time.Since(start).Seconds()
|
||||
metrics.CacheRetrievalSpeed.With(map[string]string{
|
||||
metrics.LabelCacheType: c.cache.Name(),
|
||||
metrics.LabelComponent: c.name,
|
||||
metrics.LabelSource: "cache",
|
||||
}).Set(rate)
|
||||
return blob, trace.Stack(time.Since(start), c.Name()), err
|
||||
}
|
||||
|
||||
metrics.CacheMissCount.With(metrics.CacheLabels(c.cache.Name(), c.name)).Inc()
|
||||
|
||||
blob, trace, err = c.origin.Get(hash)
|
||||
if err != nil {
|
||||
return nil, trace.Stack(time.Since(start), c.Name()), err
|
||||
}
|
||||
// do not do this async unless you're prepared to deal with mayhem
|
||||
err = c.cache.Put(hash, blob)
|
||||
if err != nil {
|
||||
log.Errorf("error saving blob to underlying cache: %s", errors.FullTrace(err))
|
||||
}
|
||||
return blob, trace.Stack(time.Since(start), c.Name()), nil
|
||||
}
|
||||
|
||||
// Put stores the blob in the origin and the cache
|
||||
func (c *CachingStore) Put(hash string, blob stream.Blob) error {
|
||||
err := c.origin.Put(hash, blob)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return c.cache.Put(hash, blob)
|
||||
}
|
||||
|
||||
// PutSD stores the sd blob in the origin and the cache
|
||||
func (c *CachingStore) PutSD(hash string, blob stream.Blob) error {
|
||||
err := c.origin.PutSD(hash, blob)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return c.cache.PutSD(hash, blob)
|
||||
}
|
||||
|
||||
// Delete deletes the blob from the origin and the cache
|
||||
func (c *CachingStore) Delete(hash string) error {
|
||||
err := c.origin.Delete(hash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return c.cache.Delete(hash)
|
||||
}
|
||||
|
||||
// Shutdown shuts down the store gracefully
|
||||
func (c *CachingStore) Shutdown() {
|
||||
c.origin.Shutdown()
|
||||
c.cache.Shutdown()
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func TestCachingStore_Put(t *testing.T) {
|
||||
origin := NewMemStore(MemParams{Name: "test"})
|
||||
cache := NewMemStore(MemParams{Name: "test"})
|
||||
s := NewCachingStore(CachingParams{Name: "test", Origin: origin, Cache: cache})
|
||||
|
||||
b := []byte("this is a blob of stuff")
|
||||
hash := "hash"
|
||||
|
||||
err := s.Put(hash, b)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
has, err := origin.Has(hash)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !has {
|
||||
t.Errorf("failed to store blob in origin")
|
||||
}
|
||||
|
||||
has, err = cache.Has(hash)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !has {
|
||||
t.Errorf("failed to store blob in cache")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCachingStore_CacheMiss(t *testing.T) {
|
||||
origin := NewMemStore(MemParams{Name: "test"})
|
||||
cache := NewMemStore(MemParams{Name: "test"})
|
||||
s := NewCachingStore(CachingParams{Name: "test", Origin: origin, Cache: cache})
|
||||
|
||||
b := []byte("this is a blob of stuff")
|
||||
hash := "hash"
|
||||
err := origin.Put(hash, b)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
res, stack, err := s.Get(hash)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(b, res) {
|
||||
t.Errorf("expected Get() to return %s, got %s", string(b), string(res))
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond) //storing to cache is done async so let's give it some time
|
||||
|
||||
has, err := cache.Has(hash)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !has {
|
||||
t.Errorf("Get() did not copy blob to cache")
|
||||
}
|
||||
t.Logf("stack: %s", stack.String())
|
||||
|
||||
res, stack, err = cache.Get(hash)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(b, res) {
|
||||
t.Errorf("expected cached Get() to return %s, got %s", string(b), string(res))
|
||||
}
|
||||
t.Logf("stack: %s", stack.String())
|
||||
}
|
||||
|
||||
func TestCachingStore_ThunderingHerd(t *testing.T) {
|
||||
storeDelay := 100 * time.Millisecond
|
||||
origin := NewSlowBlobStore(storeDelay)
|
||||
cache := NewMemStore(MemParams{Name: "test"})
|
||||
s := NewCachingStore(CachingParams{Name: "test", Origin: origin, Cache: cache})
|
||||
|
||||
b := []byte("this is a blob of stuff")
|
||||
hash := "hash"
|
||||
err := origin.Put(hash, b)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
wg := &sync.WaitGroup{}
|
||||
|
||||
getNoErr := func() {
|
||||
res, _, err := s.Get(hash)
|
||||
if err != nil {
|
||||
logrus.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(b, res) {
|
||||
t.Errorf("expected Get() to return %s, got %s", string(b), string(res))
|
||||
}
|
||||
wg.Done()
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
|
||||
wg.Add(4)
|
||||
go func() {
|
||||
go getNoErr()
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
go getNoErr()
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
go getNoErr()
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
go getNoErr()
|
||||
}()
|
||||
|
||||
wg.Wait()
|
||||
duration := time.Since(start)
|
||||
|
||||
// only the first getNoErr() should hit the origin. the rest should wait for the first request to return
|
||||
// once the first returns, the others should return immediately
|
||||
// therefore, if the delay much longer than 100ms, it means subsequent requests also went to the origin
|
||||
expectedMaxDelay := storeDelay + 5*time.Millisecond // a bit of extra time to let requests finish
|
||||
if duration > expectedMaxDelay {
|
||||
t.Errorf("Expected delay of at most %s, got %s", expectedMaxDelay, duration)
|
||||
}
|
||||
}
|
||||
|
||||
// SlowBlobStore adds a delay to each request
|
||||
type SlowBlobStore struct {
|
||||
mem *MemStore
|
||||
delay time.Duration
|
||||
}
|
||||
|
||||
func NewSlowBlobStore(delay time.Duration) *SlowBlobStore {
|
||||
return &SlowBlobStore{
|
||||
mem: NewMemStore(MemParams{Name: "test"}),
|
||||
delay: delay,
|
||||
}
|
||||
}
|
||||
func (s *SlowBlobStore) Name() string {
|
||||
return "slow"
|
||||
}
|
||||
|
||||
func (s *SlowBlobStore) Has(hash string) (bool, error) {
|
||||
time.Sleep(s.delay)
|
||||
return s.mem.Has(hash)
|
||||
}
|
||||
|
||||
func (s *SlowBlobStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
time.Sleep(s.delay)
|
||||
return s.mem.Get(hash)
|
||||
}
|
||||
|
||||
func (s *SlowBlobStore) Put(hash string, blob stream.Blob) error {
|
||||
time.Sleep(s.delay)
|
||||
return s.mem.Put(hash, blob)
|
||||
}
|
||||
|
||||
func (s *SlowBlobStore) PutSD(hash string, blob stream.Blob) error {
|
||||
time.Sleep(s.delay)
|
||||
return s.mem.PutSD(hash, blob)
|
||||
}
|
||||
|
||||
func (s *SlowBlobStore) Delete(hash string) error {
|
||||
time.Sleep(s.delay)
|
||||
return s.mem.Delete(hash)
|
||||
}
|
||||
|
||||
func (s *SlowBlobStore) Shutdown() {}
|
||||
+311
-22
@@ -2,35 +2,110 @@ package store
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/errors"
|
||||
"github.com/lbryio/reflector.go/db"
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
"github.com/c2h5oh/datasize"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
// DBBackedS3Store is an instance of an S3 Store that is backed by a DB for what is stored.
|
||||
type DBBackedS3Store struct {
|
||||
s3 *S3BlobStore
|
||||
db *db.SQL
|
||||
// DBBackedStore is a store that's backed by a DB. The DB contains data about what's in the store.
|
||||
type DBBackedStore struct {
|
||||
blobs BlobStore
|
||||
db *db.SQL
|
||||
blocked map[string]bool
|
||||
cleanerStop *stop.Group
|
||||
name string
|
||||
maxSize int
|
||||
blockedMu sync.RWMutex
|
||||
deleteOnMiss bool
|
||||
}
|
||||
|
||||
// NewDBBackedS3Store returns an initialized store pointer.
|
||||
func NewDBBackedS3Store(s3 *S3BlobStore, db *db.SQL) *DBBackedS3Store {
|
||||
return &DBBackedS3Store{s3: s3, db: db}
|
||||
type DBBackedParams struct {
|
||||
Store BlobStore `mapstructure:"store"`
|
||||
DB *db.SQL `mapstructure:"db"`
|
||||
MaxSize *int `mapstructure:"max_size"`
|
||||
Name string `mapstructure:"name"`
|
||||
DeleteOnMiss bool `mapstructure:"delete_on_miss"`
|
||||
}
|
||||
|
||||
// Has returns T/F or Error ( if the DB errors ) if store contains the blob.
|
||||
func (d *DBBackedS3Store) Has(hash string) (bool, error) {
|
||||
return d.db.HasBlob(hash)
|
||||
type DBBackedConfig struct {
|
||||
Store *viper.Viper
|
||||
Database string `mapstructure:"database"`
|
||||
User string `mapstructure:"user"`
|
||||
Password string `mapstructure:"password"`
|
||||
Host string `mapstructure:"host"`
|
||||
Name string `mapstructure:"name"`
|
||||
MaxSize string `mapstructure:"max_size"`
|
||||
Port int `mapstructure:"port"`
|
||||
AccessTracking int `mapstructure:"access_tracking"`
|
||||
DeleteOnMiss bool `mapstructure:"delete_on_miss"`
|
||||
SoftDeletes bool `mapstructure:"soft_deletes"`
|
||||
LogQueries bool `mapstructure:"log_queries"`
|
||||
HasCap bool `mapstructure:"has_cap"`
|
||||
}
|
||||
|
||||
// Get returns the byte slice of the blob or an error.
|
||||
func (d *DBBackedS3Store) Get(hash string) ([]byte, error) {
|
||||
return d.s3.Get(hash)
|
||||
// NewDBBackedStore returns an initialized store pointer.
|
||||
func NewDBBackedStore(params DBBackedParams) *DBBackedStore {
|
||||
store := &DBBackedStore{
|
||||
blobs: params.Store,
|
||||
db: params.DB,
|
||||
deleteOnMiss: params.DeleteOnMiss,
|
||||
cleanerStop: stop.New(),
|
||||
name: params.Name,
|
||||
}
|
||||
if params.MaxSize != nil {
|
||||
store.maxSize = *params.MaxSize
|
||||
go store.cleanOldestBlobs()
|
||||
}
|
||||
return store
|
||||
}
|
||||
|
||||
const nameDBBacked = "db_backed"
|
||||
|
||||
// Name is the cache type name
|
||||
func (d *DBBackedStore) Name() string { return nameDBBacked + "-" + d.name }
|
||||
|
||||
// Has returns true if the blob is in the store
|
||||
func (d *DBBackedStore) Has(hash string) (bool, error) {
|
||||
return d.db.HasBlob(hash, false)
|
||||
}
|
||||
|
||||
// Get gets the blob
|
||||
func (d *DBBackedStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
start := time.Now()
|
||||
has, err := d.db.HasBlob(hash, true)
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), err
|
||||
}
|
||||
if !has {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), ErrBlobNotFound
|
||||
}
|
||||
|
||||
b, stack, err := d.blobs.Get(hash)
|
||||
if d.deleteOnMiss && errors.Is(err, ErrBlobNotFound) {
|
||||
e2 := d.Delete(hash)
|
||||
if e2 != nil {
|
||||
log.Errorf("error while deleting blob from db: %s", errors.FullTrace(err))
|
||||
}
|
||||
}
|
||||
|
||||
return b, stack.Stack(time.Since(start), d.Name()), err
|
||||
}
|
||||
|
||||
// Put stores the blob in the S3 store and stores the blob information in the DB.
|
||||
func (d *DBBackedS3Store) Put(hash string, blob []byte) error {
|
||||
err := d.s3.Put(hash, blob)
|
||||
func (d *DBBackedStore) Put(hash string, blob stream.Blob) error {
|
||||
err := d.blobs.Put(hash, blob)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -40,17 +115,17 @@ func (d *DBBackedS3Store) Put(hash string, blob []byte) error {
|
||||
|
||||
// PutSD stores the SDBlob in the S3 store. It will return an error if the sd blob is missing the stream hash or if
|
||||
// there is an error storing the blob information in the DB.
|
||||
func (d *DBBackedS3Store) PutSD(hash string, blob []byte) error {
|
||||
func (d *DBBackedStore) PutSD(hash string, blob stream.Blob) error {
|
||||
var blobContents db.SdBlob
|
||||
err := json.Unmarshal(blob, &blobContents)
|
||||
if err != nil {
|
||||
return err
|
||||
return errors.Err(err)
|
||||
}
|
||||
if blobContents.StreamHash == "" {
|
||||
return errors.Err("sd blob is missing stream hash")
|
||||
}
|
||||
|
||||
err = d.s3.PutSD(hash, blob)
|
||||
err = d.blobs.PutSD(hash, blob)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -58,7 +133,221 @@ func (d *DBBackedS3Store) PutSD(hash string, blob []byte) error {
|
||||
return d.db.AddSDBlob(hash, len(blob), blobContents)
|
||||
}
|
||||
|
||||
// HasFullStream checks if the full stream has been uploaded (i.e. if we have the sd blob and all the content blobs)
|
||||
func (d *DBBackedS3Store) HasFullStream(sdHash string) (bool, error) {
|
||||
return d.db.HasFullStream(sdHash)
|
||||
func (d *DBBackedStore) Delete(hash string) error {
|
||||
err := d.blobs.Delete(hash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return d.db.Delete(hash)
|
||||
}
|
||||
|
||||
// Block deletes the blob and prevents it from being uploaded in the future
|
||||
func (d *DBBackedStore) Block(hash string) error {
|
||||
blocked, err := d.isBlocked(hash)
|
||||
if blocked || err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Debugf("blocking %s", hash)
|
||||
|
||||
err = d.db.Block(hash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return d.markBlocked(hash)
|
||||
}
|
||||
|
||||
// Wants returns false if the hash exists or is blocked, true otherwise
|
||||
func (d *DBBackedStore) Wants(hash string) (bool, error) {
|
||||
blocked, err := d.isBlocked(hash)
|
||||
if blocked || err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
has, err := d.Has(hash)
|
||||
return !has, err
|
||||
}
|
||||
|
||||
// MissingBlobsForKnownStream returns missing blobs for an existing stream
|
||||
// WARNING: if the stream does NOT exist, no blob hashes will be returned, which looks
|
||||
// like no blobs are missing
|
||||
func (d *DBBackedStore) MissingBlobsForKnownStream(sdHash string) ([]string, error) {
|
||||
return d.db.MissingBlobsForKnownStream(sdHash)
|
||||
}
|
||||
|
||||
func (d *DBBackedStore) markBlocked(hash string) error {
|
||||
err := d.initBlocked()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
d.blockedMu.Lock()
|
||||
defer d.blockedMu.Unlock()
|
||||
|
||||
d.blocked[hash] = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *DBBackedStore) isBlocked(hash string) (bool, error) {
|
||||
err := d.initBlocked()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
d.blockedMu.RLock()
|
||||
defer d.blockedMu.RUnlock()
|
||||
|
||||
return d.blocked[hash], nil
|
||||
}
|
||||
|
||||
func (d *DBBackedStore) initBlocked() error {
|
||||
// first check without blocking since this is the most likely scenario
|
||||
if d.blocked != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
d.blockedMu.Lock()
|
||||
defer d.blockedMu.Unlock()
|
||||
|
||||
// check again in case of race condition
|
||||
if d.blocked != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var err error
|
||||
d.blocked, err = d.db.GetBlocked()
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// cleanOldestBlobs periodically cleans up the oldest blobs if maxSize is set
|
||||
func (d *DBBackedStore) cleanOldestBlobs() {
|
||||
// Run on startup without waiting for 10 minutes
|
||||
err := d.doClean()
|
||||
if err != nil {
|
||||
log.Error(errors.FullTrace(err))
|
||||
}
|
||||
const cleanupInterval = 10 * time.Minute
|
||||
for {
|
||||
select {
|
||||
case <-d.cleanerStop.Ch():
|
||||
log.Infoln("stopping self cleanup")
|
||||
return
|
||||
case <-time.After(cleanupInterval):
|
||||
err := d.doClean()
|
||||
if err != nil {
|
||||
log.Error(errors.FullTrace(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// doClean removes the least recently accessed blobs if the store exceeds maxItems
|
||||
func (d *DBBackedStore) doClean() error {
|
||||
blobsCount, err := d.db.Count()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if blobsCount >= d.maxSize {
|
||||
itemsToDelete := blobsCount / 10
|
||||
blobs, err := d.db.LeastRecentlyAccessedHashes(itemsToDelete)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
blobsChan := make(chan string, len(blobs))
|
||||
wg := stop.New()
|
||||
go func() {
|
||||
for _, hash := range blobs {
|
||||
select {
|
||||
case <-d.cleanerStop.Ch():
|
||||
return
|
||||
default:
|
||||
}
|
||||
blobsChan <- hash
|
||||
}
|
||||
close(blobsChan)
|
||||
}()
|
||||
for i := 0; i < 3; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for h := range blobsChan {
|
||||
select {
|
||||
case <-d.cleanerStop.Ch():
|
||||
return
|
||||
default:
|
||||
}
|
||||
err = d.Delete(h)
|
||||
if err != nil {
|
||||
log.Errorf("error pruning %s: %s", h, errors.FullTrace(err))
|
||||
continue
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Shutdown shuts down the store gracefully
|
||||
func (d *DBBackedStore) Shutdown() {
|
||||
d.cleanerStop.Stop()
|
||||
d.blobs.Shutdown()
|
||||
}
|
||||
|
||||
func DBBackedStoreFactory(config *viper.Viper) (BlobStore, error) {
|
||||
var cfg DBBackedConfig
|
||||
err := config.Unmarshal(&cfg)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
cfg.Store = config.Sub("store")
|
||||
|
||||
storeType := strings.Split(cfg.Store.AllKeys()[0], ".")[0]
|
||||
storeConfig := cfg.Store.Sub(storeType)
|
||||
factory, ok := Factories[storeType]
|
||||
if !ok {
|
||||
return nil, errors.Err("unknown store type %s", storeType)
|
||||
}
|
||||
underlyingStore, err := factory(storeConfig)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
parsedDb := &db.SQL{
|
||||
TrackingLevel: db.AccessTrackingLevel(cfg.AccessTracking),
|
||||
SoftDelete: cfg.SoftDeletes,
|
||||
LogQueries: cfg.LogQueries || log.GetLevel() == log.DebugLevel,
|
||||
}
|
||||
|
||||
err = parsedDb.Connect(fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", cfg.User, cfg.Password, cfg.Host, cfg.Port, cfg.Database))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
params := DBBackedParams{
|
||||
Name: cfg.Name,
|
||||
Store: underlyingStore,
|
||||
DB: parsedDb,
|
||||
DeleteOnMiss: cfg.DeleteOnMiss,
|
||||
}
|
||||
if cfg.HasCap {
|
||||
var parsedSize datasize.ByteSize
|
||||
err = parsedSize.UnmarshalText([]byte(cfg.MaxSize))
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
maxSize := int(float64(parsedSize) / float64(stream.MaxBlobSize))
|
||||
params.MaxSize = &maxSize
|
||||
}
|
||||
|
||||
return NewDBBackedStore(params), nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
RegisterStore(nameDBBacked, DBBackedStoreFactory)
|
||||
}
|
||||
+165
@@ -0,0 +1,165 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
"github.com/lbryio/reflector.go/store/speedwalk"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
// DiskStore stores blobs on a local disk
|
||||
type DiskStore struct {
|
||||
blobDir string
|
||||
name string
|
||||
prefixLength int
|
||||
initialized bool
|
||||
}
|
||||
|
||||
type DiskParams struct {
|
||||
Name string `mapstructure:"name"`
|
||||
MountPoint string `mapstructure:"mount_point"`
|
||||
ShardingSize int `mapstructure:"sharding_size"`
|
||||
}
|
||||
|
||||
// NewDiskStore returns an initialized file disk store pointer.
|
||||
func NewDiskStore(params DiskParams) *DiskStore {
|
||||
return &DiskStore{
|
||||
blobDir: params.MountPoint,
|
||||
prefixLength: params.ShardingSize,
|
||||
name: params.Name,
|
||||
}
|
||||
}
|
||||
|
||||
const nameDisk = "disk"
|
||||
|
||||
func DiskStoreFactory(config *viper.Viper) (BlobStore, error) {
|
||||
var cfg DiskParams
|
||||
err := config.Unmarshal(&cfg)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
return NewDiskStore(cfg), nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
RegisterStore(nameDisk, DiskStoreFactory)
|
||||
}
|
||||
|
||||
// Name is the cache type name
|
||||
func (d *DiskStore) Name() string { return nameDisk + "-" + d.name }
|
||||
|
||||
// Has returns T/F or Error if it the blob stored already. It will error with any IO disk error.
|
||||
func (d *DiskStore) Has(hash string) (bool, error) {
|
||||
err := d.initOnce()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
_, err = os.Stat(d.path(hash))
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return false, nil
|
||||
}
|
||||
return false, errors.Err(err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// Get returns the blob or an error if the blob doesn't exist.
|
||||
func (d *DiskStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
start := time.Now()
|
||||
err := d.initOnce()
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), err
|
||||
}
|
||||
|
||||
blob, err := os.ReadFile(d.path(hash))
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(ErrBlobNotFound)
|
||||
}
|
||||
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(err)
|
||||
}
|
||||
return blob, shared.NewBlobTrace(time.Since(start), d.Name()), nil
|
||||
}
|
||||
|
||||
// PutSD stores the sd blob on the disk
|
||||
func (d *DiskStore) PutSD(hash string, blob stream.Blob) error {
|
||||
return d.Put(hash, blob)
|
||||
}
|
||||
|
||||
// Delete deletes the blob from the store
|
||||
func (d *DiskStore) Delete(hash string) error {
|
||||
err := d.initOnce()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
has, err := d.Has(hash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !has {
|
||||
return nil
|
||||
}
|
||||
|
||||
err = os.Remove(d.path(hash))
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
// list returns the hashes of blobs that already exist in the blobDir
|
||||
func (d *DiskStore) list() ([]string, error) {
|
||||
err := d.initOnce()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return speedwalk.AllFiles(d.blobDir, true)
|
||||
}
|
||||
|
||||
func (d *DiskStore) dir(hash string) string {
|
||||
if d.prefixLength <= 0 || len(hash) < d.prefixLength {
|
||||
return d.blobDir
|
||||
}
|
||||
return path.Join(d.blobDir, hash[:d.prefixLength])
|
||||
}
|
||||
func (d *DiskStore) tmpDir() string {
|
||||
return path.Join(d.blobDir, "tmp")
|
||||
}
|
||||
func (d *DiskStore) path(hash string) string {
|
||||
return path.Join(d.dir(hash), hash)
|
||||
}
|
||||
func (d *DiskStore) tmpPath(hash string) string {
|
||||
return path.Join(d.tmpDir(), hash)
|
||||
}
|
||||
func (d *DiskStore) ensureDirExists(dir string) error {
|
||||
return errors.Err(os.MkdirAll(dir, 0755))
|
||||
}
|
||||
|
||||
func (d *DiskStore) initOnce() error {
|
||||
if d.initialized {
|
||||
return nil
|
||||
}
|
||||
|
||||
err := d.ensureDirExists(d.blobDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = d.ensureDirExists(d.tmpDir())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
d.initialized = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// Shutdown shuts down the store gracefully
|
||||
func (d *DiskStore) Shutdown() {
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestDiskStore_Get(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "reflector_test_*")
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = os.RemoveAll(tmpDir) }()
|
||||
d := NewDiskStore(DiskParams{
|
||||
Name: "test",
|
||||
MountPoint: tmpDir,
|
||||
ShardingSize: 2,
|
||||
})
|
||||
|
||||
hash := "f428b8265d65dad7f8ffa52922bba836404cbd62f3ecfe10adba6b444f8f658938e54f5981ac4de39644d5b93d89a94b"
|
||||
data := []byte("oyuntyausntoyaunpdoyruoyduanrstjwfjyuwf")
|
||||
|
||||
expectedPath := path.Join(tmpDir, hash[:2], hash)
|
||||
err = os.MkdirAll(filepath.Dir(expectedPath), os.ModePerm)
|
||||
require.NoError(t, err)
|
||||
err = os.WriteFile(expectedPath, data, os.ModePerm)
|
||||
require.NoError(t, err)
|
||||
|
||||
blob, _, err := d.Get(hash)
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, data, blob)
|
||||
}
|
||||
|
||||
func TestDiskStore_GetNonexistentBlob(t *testing.T) {
|
||||
tmpDir, err := os.MkdirTemp("", "reflector_test_*")
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = os.RemoveAll(tmpDir) }()
|
||||
d := NewDiskStore(DiskParams{
|
||||
Name: "test",
|
||||
MountPoint: tmpDir,
|
||||
ShardingSize: 2,
|
||||
})
|
||||
|
||||
blob, _, err := d.Get("nonexistent")
|
||||
assert.Nil(t, blob)
|
||||
assert.True(t, errors.Is(err, ErrBlobNotFound))
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
//go:build darwin
|
||||
// +build darwin
|
||||
|
||||
package store
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
)
|
||||
|
||||
var openFileFlags = os.O_WRONLY | os.O_CREATE
|
||||
|
||||
// Put stores the blob on disk
|
||||
func (d *DiskStore) Put(hash string, blob stream.Blob) error {
|
||||
err := d.initOnce()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = d.ensureDirExists(d.dir(hash))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Open file with O_DIRECT
|
||||
f, err := os.OpenFile(d.tmpPath(hash), openFileFlags, 0644)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
_, err = io.Copy(f, bytes.NewReader(blob))
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
err = os.Rename(d.tmpPath(hash), d.path(hash))
|
||||
return errors.Err(err)
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
//go:build linux
|
||||
// +build linux
|
||||
|
||||
package store
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
"syscall"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
"github.com/brk0v/directio"
|
||||
)
|
||||
|
||||
var openFileFlags = os.O_WRONLY | os.O_CREATE | syscall.O_DIRECT
|
||||
|
||||
// Put stores the blob on disk
|
||||
func (d *DiskStore) Put(hash string, blob stream.Blob) error {
|
||||
err := d.initOnce()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = d.ensureDirExists(d.dir(hash))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Open file with O_DIRECT
|
||||
f, err := os.OpenFile(d.tmpPath(hash), openFileFlags, 0644)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer func() { _ = f.Close() }()
|
||||
dio, err := directio.New(f)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer func() { _ = dio.Flush() }()
|
||||
_, err = io.Copy(dio, bytes.NewReader(blob))
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
err = os.Rename(d.tmpPath(hash), d.path(hash))
|
||||
return errors.Err(err)
|
||||
}
|
||||
Loaded 100 of 124 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user