66 Commits
Author SHA1 Message Date
Niko Storni 0359d723c5 Add fallback DiskStore.Put for non-linux/darwin platforms
Go / build (push) Waiting to run
2026-01-03 04:47:29 +01:00
Niko Storni 0cfc336be5 Fix flaky BlobTrace serialization test 2026-01-03 04:36:48 +01:00
Niko Storni 05acdcb9e0 update go
rename cmd back to reflector
remove travis
update deps
2026-01-03 04:25:04 +01:00
Niko Storni 9e2cccc51f Migrate to AWS SDK v2 2026-01-03 04:08:40 +01:00
Niko Storni 1b79371f8b major linting
code cleanup
readme update
vulnerability fix
2025-10-11 04:30:30 +02:00
Niko Storni cbcc611196 deterministic imports 2025-08-29 16:27:14 +02:00
Niko 68c0fe3b97 Merge pull request #68 from lbryio/modular_reflector
Modular blobcache stores (huge changes)
2025-08-22 02:22:03 +02:00
Niko Storni 8bfe7e101d fix tests
fix build
2025-08-22 02:09:55 +02:00
Niko Storni 7b3cca45b4 update readme
remove dead code
2025-08-22 01:56:39 +02:00
Niko Storni ad710dfeea fix livelock
adjust API endpoint
2025-08-22 01:34:56 +02:00
Niko Storni ebc52245f3 fix viper for real 2025-05-14 01:17:13 +02:00
Niko Storni b098aa2dc8 stupid viper... 2025-05-14 00:58:37 +02:00
Niko Storni ddd23f9a36 fix config loading bug 2025-05-14 00:25:48 +02:00
Niko Storni 4928e430c0 fix bugs 2025-05-13 22:52:40 +02:00
Niko Storni d21b0867dd remove unused cmds
change conf param
fix upload cmd
2025-05-13 22:30:40 +02:00
Niko Storni 32a85a815e adjust s3 configuration
hardcode multiwriter for now
2025-05-13 18:01:28 +02:00
Niko Storni 9cdac902bd add multiwriter store 2025-05-13 14:47:19 +02:00
Niko Storni 8ec01bf1a7 fix tests 2025-05-13 14:37:51 +02:00
Niko Storni ba8d1ba67b add s3 sharding 2025-05-13 14:08:33 +02:00
Niko Storni edecf4f988 delegate cascading for blocker and missing blobs checker 2025-05-13 13:55:54 +02:00
Niko Storni 77836f9c83 prepare stores for reflector upload
create a new reflector cmd that will replace old reflector cmd
2025-05-13 05:54:46 +02:00
Niko Storni 9716e41d63 replace hub servers with valid ones
change ITTT metrics
fix tracing for http store
remove unused launch params
2025-04-25 20:04:50 +02:00
Niko Storni 7260388255 make servers modular
fix bugs
2025-04-25 17:39:52 +02:00
Niko Storni 071a7907c1 Refactors store implementations for config loading
Updates store implementations to load configurations from YAML files.

This change introduces a configuration loader that reads store parameters from a YAML file, allowing for more flexible and manageable store configurations.
2025-04-24 05:11:52 +02:00
Niko Storni 185900582d add support for sharding in http stores 2025-04-19 04:21:46 +02:00
Niko Storni 03445476ec rename stores to better reflect what they do
move cleanup code in db store
fix bugs
refactor code
update go
2025-04-19 03:49:54 +02:00
Niko Storni e5a9590cb6 make s3 endpoint configurable
update deps
2024-12-16 22:35:50 +01:00
Niko Storni de736b068d remove outdated linter
remove redundant imports
remove deprecated and implicit rand seed init
adjust test
2024-04-09 20:00:46 +02:00
Niko Storni 14fbd2ae27 update dependencies 2024-04-09 19:17:32 +02:00
Niko Storni 4d81a43a8f update dependencies 2023-10-11 21:01:37 +02:00
Niko Storni b3f0d63b4d fix bug 2023-10-11 20:52:13 +02:00
Niko Storni c880f0b80f fix bug rendering sf useless 2023-10-11 20:45:31 +02:00
Niko Storni 085490e92b upgrade go version
update dependencies
2023-07-07 15:34:22 +02:00
Niko 456fe53e01 Merge pull request #67 from lbryio/upgrades
upgrade dependencies & introduce github ci
2023-03-09 13:12:59 -05:00
Niko Storni 778fc17adf update readme 2023-03-09 18:59:21 +01:00
Niko Storni e93c097fd9 upgrade dependencies
replace deprecated function calls
refactor build process
2023-03-09 18:41:41 +01:00
Niko 0dfda70c70 Merge pull request #63 from lbryio/declare_err
Add undeclared errors from peer.go to metrics
2022-10-17 19:01:56 +02:00
Victor Shyba 6c082993cf metrics: declare request is too large 2022-10-14 15:21:06 -03:00
Victor Shyba 08ed3c9f13 metrics: declare invalid blob hash len 2022-10-14 15:18:46 -03:00
Victor Shyba 7f75602841 metrics: declare protected blob 2022-10-14 15:16:54 -03:00
Victor Shyba 4d168ddefc metrics: declare invalid json request 2022-10-14 15:15:32 -03:00
Niko b7abb77ea1 Merge pull request #62 from lbryio/declare_err
declare errInvalidPeerData on metrics
2022-10-14 19:45:40 +02:00
Victor Shyba 8c2a46752c declare errInvalidPeerData on metrics 2022-10-14 14:42:22 -03:00
Niko Storni 0c8da4abe5 Merge remote-tracking branch 'origin/peer_proto_tests' 2022-10-14 18:30:08 +02:00
Victor Shyba 87680d806c check for valid data earlier. reply composite as tests are expecting 2022-10-11 22:58:41 -03:00
Victor Shyba e1c59b9b63 test through the connection 2022-10-11 22:57:52 -03:00
Niko Storni c79634706b ignore server closed errors the right way 2022-09-23 14:46:55 +02:00
Niko Storni f5d30b1a6e update quic-go 2022-07-30 20:10:28 +02:00
Niko Storni 0177dd4ce0 fix query param 2022-07-29 05:40:50 +02:00
Niko Storni 5693529216 protect protected content 2022-07-29 04:59:15 +02:00
Niko a1c2e92ca3 Merge pull request #57 from andybeletsky/fix-store-macos
Do not use direction package on macos
2022-05-12 16:01:00 +02:00
Andrey Beletsky 9c0554ef05 Do not use direction package on macos 2022-05-08 15:39:34 +07:00
Niko Storni 4e80f91a57 fix memory leak 2022-05-04 19:14:20 +02:00
Niko Storni c211f83ba7 update dependencies 2022-05-02 23:32:36 +02:00
Niko Storni 29d1ccf68c add singleflight to web requests 2022-05-02 23:07:22 +02:00
Niko Storni 2f7d67794f remove debug leftover 2022-02-10 00:57:13 +01:00
Niko Storni 4d8e7739d7 Merge branch 'fix-ci' 2021-12-14 22:36:01 +01:00
Niko Storni 6fc0ceea2a adjust travis vars
fix build script


fix more scripts


adapt script to scale
2021-12-14 22:35:44 +01:00
Niko Storni ae0c7dd2bb upgrade quic-go for go1.17 2021-12-14 20:49:50 +01:00
Niko Storni 4af5c2f4c6 make slack channel configurable 2021-10-30 00:21:58 +02:00
Alex Grin def0a97f49 Update readme.md 2021-09-28 10:15:24 -04:00
Niko Storni 6dde793745 do not delete blobs for blocked content 2021-09-21 18:00:55 +02:00
Niko Storni 654cc44935 Merge branch 'hash_twice' 2021-09-21 16:21:25 +02:00
Victor Shyba 90d6d29452 remove all hashing on the download path 2021-09-21 16:19:49 +02:00
Niko Storni b2272fef3a delete overflowing blobs from underlying cache 2021-08-18 19:36:47 +02:00
Niko Storni 86f3e62aa8 fix a panic error
update gin-go
2021-08-05 17:47:20 +02:00
88 changed files with 3535 additions and 2557 deletions

No files matched your search

+37
View File
@@ -0,0 +1,37 @@
name: Go
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v4
with:
go-version: 1.25.x
- name: Build linux
run: make linux
- name: Build macos
run: make macos
- name: Test
run: make test
- name: Lint
run: make lint
- name: retrieve all tags
run: git fetch --prune --unshallow --tags
- name: Print changes since last version
run: git log $(git describe --tags --abbrev=0)..HEAD --no-merges --oneline
+62
View File
@@ -0,0 +1,62 @@
name: release
on:
push:
tags:
- "*.*.*"
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v4
with:
go-version: 1.25.x
- name: Build linux
run: make linux
- name: Build macos
run: make macos
- name: Test
run: make test
- name: Lint
run: make lint
- name: Zip macos
run: zip -r reflector_darwin_amd64.zip ./dist/darwin_amd64
- name: Zip linux
run: zip -r reflector_linux_amd64.zip ./dist/linux_amd64
- name: retrieve all tags
run: git fetch --prune --unshallow --tags
- name: Generate Changelog
run: git log $(git describe --tags --abbrev=0 @^)..@ --no-merges --oneline > ${{ github.workspace }}-CHANGELOG.txt
- name: upload to github releases
uses: softprops/action-gh-release@v1
with:
files: |
./reflector_linux_amd64.zip
./reflector_darwin_amd64.zip
body_path: ${{ github.workspace }}-CHANGELOG.txt
# - name: Login to DockerHub
# uses: docker/login-action@v2
# with:
# username: ${{ secrets.DOCKERHUB_USERNAME }}
# password: ${{ secrets.DOCKERHUB_TOKEN }}
# - name: Generate docker image
# run: make image
# - name: Docker push
# run: make publish_image
+9
View File
@@ -1,3 +1,12 @@
/vendor
/config.json*
/dist
/bin
blobcache.yaml
reflector.yaml
upload.yaml
remotereflector.yaml
+57
View File
@@ -0,0 +1,57 @@
run:
timeout: 5m
tests: false
modules-download-mode: readonly
linters-settings:
gci:
# Checks that no inline Comments are present.
# Default: false
no-inline-comments: false
# Checks that no prefix Comments(comment lines above an import) are present.
# Default: false
no-prefix-comments: false
# Skip generated files
skip-generated: true
# Use custom order
custom-order: true
sections:
- standard
- prefix(github.com/lbryio/reflector.go)
- prefix(github.com/OdyseeTeam,github.com/lbryio)
- default
gofmt:
simplify: true
govet:
enable-all: true
misspell:
locale: US
revive:
rules:
- name: exported
disabled: true
staticcheck:
checks: ["all"]
linters:
enable:
- errcheck
- gosimple
- govet
- ineffassign
- gci
- gofmt
- goimports
- misspell
- revive
- unconvert
- whitespace
- typecheck
disable:
- unused
- unparam
- staticcheck
-83
View File
@@ -1,83 +0,0 @@
os: linux
dist: trusty
language: go
env:
- GO111MODULE=on
go:
- 1.15.x
cache:
directories:
- $HOME/.cache/go-build
- $HOME/gopath/pkg/mod
notifications:
email: false
# Skip the install step. Don't `go get` dependencies. Only build with the code in vendor/
install: true
# Anything in before_script that returns a nonzero exit code will
# flunk the build and immediately stop. It's sorta like having
# set -e enabled in bash.
before_script:
# All the .go files, excluding vendor/ and model (auto generated)
- GO_FILES=$(find . -iname '*.go' ! -iname '*_test.go' -type f | grep -v /vendor/ ) #i wish we were this crazy :p
- go get golang.org/x/tools/cmd/goimports # Used in build script for generated files
# - go get github.com/golang/lint/golint # Linter
# - go get honnef.co/go/tools/cmd/megacheck # Badass static analyzer/linter
- go get github.com/jgautheron/gocyclo # Check against high complexity
- go get github.com/mdempsky/unconvert # Identifies unnecessary type conversions
- go get github.com/kisielk/errcheck # Checks for unhandled errors
- go get github.com/opennota/check/cmd/varcheck # Checks for unused vars
- go get github.com/opennota/check/cmd/structcheck # Checks for unused fields in structs
# script always run to completion (set +e). All of these code checks are must haves
# in a modern Go project.
script:
# Fail if a .go file hasn't been formatted with gofmt
- test -z $(gofmt -s -l $GO_FILES)
# Run unit tests
- make test
# Checks for unused vars and fields on structs
- varcheck ./...
- structcheck ./...
# go vet is the official Go static analyzer
- go vet ./...
# forbid code with huge functions
#- gocyclo -ignore "_test.go" -avg -over 19 $GO_FILES
# checks for unhandled errors
# - errcheck ./... # COMMENTED OUT UNTIL https://github.com/kisielk/errcheck/issues/155 is fixed
# "go vet on steroids" + linter - ignore autogen code
# - megacheck -simple.exit-non-zero=true ./... # DISABLED UNTIL https://github.com/dominikh/go-tools/issues/328 is fixed
# check for unnecessary conversions - ignore autogen code
# - unconvert ./... # SEEMS TO BE BROKEN WITH GO MODULES
# one last linter - ignore autogen code
#- golint -set_exit_status $(go list ./... | grep -v /vendor/ )
# Finally, build the binary
- make
deploy:
- provider: s3
local_dir: ./bin
skip_cleanup: true
on:
repo: lbryio/reflector.go
access_key_id:
secure: "ABmoSTxTee1GubJmmi+MlyBcnqRT0ywEOtTL4hrH7T+Vj5UPcNBhjqGRiMXzO3MdrVWom19OnqMvsvAdehrB4uQJVlX/+zB26gTpEiiU5dWJgerNAYmfxDGNz/p9pyYWLvVY/cymSlJ4HkggJVg+dMTBTOP+gTNgQUhHL7pAd/Z5HMPmPW/rlNlQgDdczoY44OdiniuHny4Lr7mo8dcT4dcyJysPpHLCPGWp3kXcffIEbgO02aQZ14BbX9oU9xm1BacYnr4t8gRCU+uK0mt7TREHUzZ8bQBRXUymmpeeEQshXdifdwtk6sdRGIZiTi0NmQ7kYter6oh6623TH6rpNy/lkxKi823FbSPC1XZGH7CwcbnSIEwY++7e9PT0EA14VuDUF7+iXC/gpRkVQwUoMcoC9rHBjkXW+SiwHWoUeDKXrfVO9QGEJTi5bDRNwklwu2zkYWeRIPzfDaVayUBYwjunjT1eST997ygmJwJTJ6DeOWLx+WFlmwBJfQpBwbrlMrceiNjreMwGB6ffXEBzuM6S5inXGIChllGDpNyJeFVcH9zPpU0JsIpWoRjwLNFOiEhFsnzK5rvA2SKmlOOnSlgOCBQnFi2SmLDM790KoY8qjan2s984chQz2qsKnhYuVpJhG6LKgrI1QwE0KHr80spVFfewq11AU4Q+lfA9aAQ="
secret_access_key:
secure: "AxYRTy/GnjeTJKQdeJ/AEeAd+yXs783bFDKdyKNswtsHlU8sWPQgNcvTLpVqnSQMpiwkGDGi/70rvR5C+AT3SIWNw13RYrgBRpduQU0J+B2JS+3dN2DIePu25uvs++Wo22OfS8I+UjZ1mWY1SSHI2spPXvDCq5tb+Ih8nlYflEyAtxU9Oq2R3Kus2tkIlRnL25sP/2fY7RvuJFYIV63z8ZIJRzB5WzOeERqnXq2zfwos+hycAqyo/VevJnWAYTEDsvBuSODOpZF+QfKtIQ2rYSoqy8Lq1M6UOZimnC3Ulea4euBVf2ssBCnI7csGNG5UzkTiwrPDi2xIP8nM01rHW1yHJ7tQsJaghnUsfw2t6ui4ZofvbbOFTN/YCloHITifEi8Tc1/17isi3y+kX5yQ/Nk5UNry0Wbt91CP+nkL/ZmA5grkBXDL7VJMmB60TnO3ap24CtwBQartN3LoWs7h+4ov+LqbCt6IqpJVWQWlwJeb2MFPFByALtBpsqAyL1SxXlGNpPa94CuXxfQ6Bv436PtefA5FlTzR8uMmqsjWciv06bVnSvVlFEVovN2Fkplrjt7AASJ/8KJs4THDg4k61nfd8roAHx6ewQzl4wCWKCikQ0MuFd2mVHwdrbnCH1mIHuPRyvWMMIAK0ooc1/rmKiJlpgumjxoFYNE10MXtt+I="
bucket: "lbry-reflector-binary"
# - provider: releases
# file: ./bin/prism-bin
# skip_cleanup: true
# on:
# repo: lbryio/reflector.go
# tags: true
# api_key:
# secure: epAlhp3SUr8hhISarJ22n6tRw2TEa4s4oNFIvJUb5HGECVp1SYN7ao0ln5NoNLmfJS60pi911i/kMhhi21/uhZ0kCYlEhhIE2pc1zsiAxK9L9ENCssJ205HfVbe5grhwskLGzgjhU9OznO8WtmyOPWXr0it8M8RCTjx6rEC0A33Id3WMYyhP938Sj9CxEYeH4KS8wFvBXkgBVtrgaYwRTCIROFddHFXOb9jyNhqQ1RbfKtllsVtQhVk5WMlomheBNSS4vr6WMS4X4+2okFqnLtiSn1wrn5I/94UQbnrI1juVnQj0K+j32EyQbAOt4T2cLW3GtG0jhaYKyNMT9ycDCdVACPSDELlHWjeyoes9bnhUFftm6kDbQxwA1UsTF1yG8tMKXxBSmYyoT7qDloi6pBifZMrFXL61uTs6yhVB9LS/2oqg4sc0Ne87bRcn4OxsBeVCe3kbBHDTR/NTyF2gNPtRvgMAWULxTVcUm9VYdO0IWvAig5g4Row0DnFzEquD6CzezbRWD9WyZyV/AFyYHeeQ2PO7jTw0/3M7aDX33Fuhh34lehzmrC03cfgD/wZW+spxozIcQCYdiJqVw+u+/NvbNr0kkFzE9zW26JEmUFTyDvKxvnza1Kwtww3EgH6zaOL8r4yVbb54rePRvLw7pl93zlfJnEB2MCPqJOY5ZpU=
+1 -1
View File
@@ -3,7 +3,7 @@ EXPOSE 8080
RUN mkdir /app
WORKDIR /app
COPY bin/prism-bin ./prism
COPY dist/linux_amd64/prism-bin ./prism
RUN chmod +x prism
ENTRYPOINT [ "/app/prism" ]
+30 -18
View File
@@ -1,25 +1,37 @@
version := $(shell git describe --dirty --always --long --abbrev=7)
commit := $(shell git rev-parse --short HEAD)
commit_long := $(shell git rev-parse HEAD)
branch := $(shell git rev-parse --abbrev-ref HEAD)
curTime := $(shell date +%s)
BINARY=prism-bin
DIR = $(shell cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)
BIN_DIR = ${DIR}/bin
IMPORT_PATH = github.com/lbryio/reflector.go
LDFLAGS="-X ${IMPORT_PATH}/meta.version=$(version) -X ${IMPORT_PATH}/meta.commit=$(commit) -X ${IMPORT_PATH}/meta.commitLong=$(commit_long) -X ${IMPORT_PATH}/meta.branch=$(branch) -X '${IMPORT_PATH}/meta.Time=$(curTime)'"
DIR = $(shell cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)
BIN_DIR = $(DIR)/dist
VERSION = $(shell git --git-dir=${DIR}/.git describe --dirty --always --long --abbrev=7)
LDFLAGS = -ldflags "-X ${IMPORT_PATH}/meta.Version=${VERSION} -X ${IMPORT_PATH}/meta.Time=$(shell date +%s)"
.PHONY: build clean test lint
.DEFAULT_GOAL: build
build:
mkdir -p ${BIN_DIR} && CGO_ENABLED=0 go build ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/${BINARY} main.go
clean:
if [ -f ${BIN_DIR}/${BINARY} ]; then rm ${BIN_DIR}/${BINARY}; fi
.DEFAULT_GOAL := linux
.PHONY: test
test:
go test ./... -v -cover
go test -cover -v ./...
.PHONY: lint
lint:
go get github.com/alecthomas/gometalinter && gometalinter --install && gometalinter ./...
./scripts/lint.sh
.PHONY: imports
imports:
gci write --skip-generated --custom-order -s standard -s "prefix(github.com/lbryio/reflector.go)" -s "prefix(github.com/OdyseeTeam,github.com/lbryio)" -s default .
.PHONY: linux
linux:
GOARCH=amd64 CGO_ENABLED=0 GOOS=linux go build -ldflags ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/linux_amd64/${BINARY}
.PHONY: macos
macos:
GOARCH=amd64 GOOS=darwin go build -ldflags ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/darwin_amd64/${BINARY}
.PHONY: image
image:
docker buildx build -t lbry/reflector:$(version) -t lbry/reflector:latest --platform linux/amd64 .
+6 -7
View File
@@ -1,7 +1,7 @@
package cluster
import (
"io/ioutil"
"io"
baselog "log"
"sort"
"time"
@@ -23,13 +23,12 @@ const (
type Cluster struct {
OnMembershipChange func(n, total int)
s *serf.Serf
eventCh chan serf.Event
stop *stop.Group
name string
port int
seedAddr string
s *serf.Serf
eventCh chan serf.Event
stop *stop.Group
port int
}
// New returns a new Cluster instance that is not connected.
@@ -52,7 +51,7 @@ func (c *Cluster) Connect() error {
conf.MemberlistConfig.AdvertisePort = c.port
conf.NodeName = c.name
nullLogger := baselog.New(ioutil.Discard, "", 0)
nullLogger := baselog.New(io.Discard, "", 0)
conf.Logger = nullLogger
c.eventCh = make(chan serf.Event)
+55
View File
@@ -0,0 +1,55 @@
package cmd
import (
"os"
"os/signal"
"strconv"
"syscall"
"github.com/lbryio/reflector.go/config"
"github.com/lbryio/reflector.go/internal/metrics"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
func init() {
var cmd = &cobra.Command{
Use: "blobcache",
Short: "Run blobcache server",
Run: blobcacheCmd,
}
cmd.Flags().IntVar(&metricsPort, "metrics-port", 2112, "The port reflector will use for prometheus metrics")
cmd.Flags().BoolVar(&disableBlocklist, "disable-blocklist", false, "Disable blocklist watching/updating")
rootCmd.AddCommand(cmd)
}
func blobcacheCmd(cmd *cobra.Command, args []string) {
store, err := config.LoadStores(conf, "blobcache")
if err != nil {
log.Fatal(err)
}
defer store.Shutdown()
servers, err := config.LoadServers(store, conf, "blobcache")
if err != nil {
log.Fatal(err)
}
for _, s := range servers {
err = s.Start()
if err != nil {
log.Fatal(err)
}
defer s.Shutdown()
}
metricsServer := metrics.NewServer(":"+strconv.Itoa(metricsPort), "/metrics")
metricsServer.Start()
defer metricsServer.Shutdown()
interruptChan := make(chan os.Signal, 1)
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
<-interruptChan
}
+2 -2
View File
@@ -6,10 +6,10 @@ import (
"strconv"
"syscall"
"github.com/lbryio/lbry.go/v2/extras/crypto"
"github.com/lbryio/reflector.go/cluster"
"github.com/lbryio/lbry.go/v2/extras/crypto"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
+13 -6
View File
@@ -5,7 +5,6 @@ import (
"os"
"time"
"github.com/lbryio/reflector.go/server/peer"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/stream"
@@ -28,11 +27,19 @@ func getStreamCmd(cmd *cobra.Command, args []string) {
addr := args[0]
sdHash := args[1]
s := store.NewCachingStore(
"getstream",
peer.NewStore(peer.StoreOpts{Address: addr}),
store.NewDiskStore("/tmp/lbry_downloaded_blobs", 2),
)
s := store.NewCachingStore(store.CachingParams{
Name: "getstream",
Cache: store.NewPeerStore(store.PeerParams{
Name: "getstream",
Address: addr,
Timeout: 30 * time.Second,
}),
Origin: store.NewDiskStore(store.DiskParams{
Name: "getstream",
MountPoint: "/tmp/lbry_downloaded_blobs",
ShardingSize: 2,
}),
})
wd, err := os.Getwd()
if err != nil {
+1 -2
View File
@@ -3,7 +3,6 @@ package cmd
import (
"crypto/sha512"
"encoding/hex"
"io/ioutil"
"os"
"path"
"runtime"
@@ -72,7 +71,7 @@ func consume(worker int, tasks <-chan string, done chan<- bool, totalTasks int,
log.Infof("[T%d] %d/%d blobs processed so far. ETA: %s", worker, processedSoFar, totalTasks, remainingTime.String())
}
blobPath := path.Join(diskStorePath, b[:2], b)
blob, err := ioutil.ReadFile(blobPath)
blob, err := os.ReadFile(blobPath)
if err != nil {
if os.IsNotExist(err) {
continue
-55
View File
@@ -1,55 +0,0 @@
package cmd
import (
"os"
"os/signal"
"strconv"
"syscall"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/server/peer"
"github.com/lbryio/reflector.go/store"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
var peerNoDB bool
func init() {
var cmd = &cobra.Command{
Use: "peer",
Short: "Run peer server",
Run: peerCmd,
}
cmd.Flags().BoolVar(&peerNoDB, "nodb", false, "Don't connect to a db and don't use a db-backed blob store")
rootCmd.AddCommand(cmd)
}
func peerCmd(cmd *cobra.Command, args []string) {
var err error
s3 := store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
peerServer := peer.NewServer(s3)
if !peerNoDB {
db := &db.SQL{
LogQueries: log.GetLevel() == log.DebugLevel,
}
err = db.Connect(globalConfig.DBConn)
checkErr(err)
combo := store.NewDBBackedStore(s3, db, false)
peerServer = peer.NewServer(combo)
}
err = peerServer.Start(":" + strconv.Itoa(peer.DefaultPort))
if err != nil {
log.Fatal(err)
}
interruptChan := make(chan os.Signal, 1)
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
<-interruptChan
peerServer.Shutdown()
}
+3 -3
View File
@@ -32,9 +32,9 @@ func populateDbCmd(cmd *cobra.Command, args []string) {
log.Fatal("store-path must be defined")
}
localDb := &db.SQL{
SoftDelete: true,
TrackAccess: db.TrackAccessBlobs,
LogQueries: log.GetLevel() == log.DebugLevel,
SoftDelete: true,
TrackingLevel: db.TrackAccessBlobs,
LogQueries: log.GetLevel() == log.DebugLevel,
}
err := localDb.Connect("reflector:reflector@tcp(localhost:3306)/reflector")
if err != nil {
+21 -324
View File
@@ -4,66 +4,25 @@ import (
"os"
"os/signal"
"strconv"
"strings"
"syscall"
"time"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/config"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/meta"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/server/http"
http32 "github.com/lbryio/reflector.go/server/http3"
"github.com/lbryio/reflector.go/server/peer"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/c2h5oh/datasize"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
var (
//port configuration
tcpPeerPort int
http3PeerPort int
httpPeerPort int
receiverPort int
metricsPort int
receiverPort int
metricsPort int
//flags configuration
disableUploads bool
disableBlocklist bool
useDB bool
//upstream configuration
upstreamReflector string
upstreamProtocol string
//downstream configuration
requestQueueSize int
//upstream edge configuration (to "cold" storage)
originEndpoint string
originEndpointFallback string
//cache configuration
diskCache string
secondaryDiskCache string
memCache int
)
var cacheManagers = []string{"localdb", "lfu", "arc", "lru", "simple"}
var cacheMangerToGcache = map[string]store.EvictionStrategy{
"lfu": store.LFU,
"arc": store.ARC,
"lru": store.LRU,
"simple": store.SIMPLE,
}
func init() {
var cmd = &cobra.Command{
@@ -72,308 +31,46 @@ func init() {
Run: reflectorCmd,
}
cmd.Flags().IntVar(&tcpPeerPort, "tcp-peer-port", 5567, "The port reflector will distribute content from for the TCP (LBRY) protocol")
cmd.Flags().IntVar(&http3PeerPort, "http3-peer-port", 5568, "The port reflector will distribute content from over HTTP3 protocol")
cmd.Flags().IntVar(&httpPeerPort, "http-peer-port", 5569, "The port reflector will distribute content from over HTTP protocol")
cmd.Flags().IntVar(&receiverPort, "receiver-port", 5566, "The port reflector will receive content from")
cmd.Flags().IntVar(&metricsPort, "metrics-port", 2112, "The port reflector will use for prometheus metrics")
cmd.Flags().BoolVar(&disableUploads, "disable-uploads", false, "Disable uploads to this reflector server")
cmd.Flags().IntVar(&receiverPort, "receiver-port", 5566, "The port reflector will receive content from")
cmd.Flags().BoolVar(&disableBlocklist, "disable-blocklist", false, "Disable blocklist watching/updating")
cmd.Flags().BoolVar(&useDB, "use-db", true, "Whether to connect to the reflector db or not")
cmd.Flags().StringVar(&upstreamReflector, "upstream-reflector", "", "host:port of a reflector server where blobs are fetched from")
cmd.Flags().StringVar(&upstreamProtocol, "upstream-protocol", "http", "protocol used to fetch blobs from another upstream reflector server (tcp/http3/http)")
cmd.Flags().IntVar(&requestQueueSize, "request-queue-size", 200, "How many concurrent requests from downstream should be handled at once (the rest will wait)")
cmd.Flags().StringVar(&originEndpoint, "origin-endpoint", "", "HTTP edge endpoint for standard HTTP retrieval")
cmd.Flags().StringVar(&originEndpointFallback, "origin-endpoint-fallback", "", "HTTP edge endpoint for standard HTTP retrieval if first origin fails")
cmd.Flags().StringVar(&diskCache, "disk-cache", "100GB:/tmp/downloaded_blobs:localdb", "Where to cache blobs on the file system. format is 'sizeGB:CACHE_PATH:cachemanager' (cachemanagers: localdb/lfu/arc/lru)")
cmd.Flags().StringVar(&secondaryDiskCache, "optional-disk-cache", "", "Optional secondary file system cache for blobs. format is 'sizeGB:CACHE_PATH:cachemanager' (cachemanagers: localdb/lfu/arc/lru) (this would get hit before the one specified in disk-cache)")
cmd.Flags().IntVar(&memCache, "mem-cache", 0, "enable in-memory cache with a max size of this many blobs")
rootCmd.AddCommand(cmd)
}
func reflectorCmd(cmd *cobra.Command, args []string) {
log.Printf("reflector %s", meta.VersionString())
store, err := config.LoadStores(conf, "reflector")
if err != nil {
log.Fatal(err)
}
defer store.Shutdown()
// the blocklist logic requires the db backed store to be the outer-most store
underlyingStore := initStores()
underlyingStoreWithCaches, cleanerStopper := initCaches(underlyingStore)
if !disableUploads {
reflectorServer := reflector.NewServer(underlyingStore, underlyingStoreWithCaches)
reflectorServer.Timeout = 3 * time.Minute
reflectorServer.EnableBlocklist = !disableBlocklist
err := reflectorServer.Start(":" + strconv.Itoa(receiverPort))
servers, err := config.LoadServers(store, conf, "reflector")
if err != nil {
log.Fatal(err)
}
for _, s := range servers {
err = s.Start()
if err != nil {
log.Fatal(err)
}
defer reflectorServer.Shutdown()
defer s.Shutdown()
}
peerServer := peer.NewServer(underlyingStoreWithCaches)
err := peerServer.Start(":" + strconv.Itoa(tcpPeerPort))
reflectorServer := reflector.NewIngestionServer(store)
reflectorServer.Timeout = 3 * time.Minute
reflectorServer.EnableBlocklist = !disableBlocklist
err = reflectorServer.Start(":" + strconv.Itoa(receiverPort))
if err != nil {
log.Fatal(err)
}
defer peerServer.Shutdown()
http3PeerServer := http32.NewServer(underlyingStoreWithCaches, requestQueueSize)
err = http3PeerServer.Start(":" + strconv.Itoa(http3PeerPort))
if err != nil {
log.Fatal(err)
}
defer http3PeerServer.Shutdown()
httpServer := http.NewServer(underlyingStoreWithCaches, requestQueueSize)
err = httpServer.Start(":" + strconv.Itoa(httpPeerPort))
if err != nil {
log.Fatal(err)
}
defer httpServer.Shutdown()
defer reflectorServer.Shutdown()
metricsServer := metrics.NewServer(":"+strconv.Itoa(metricsPort), "/metrics")
metricsServer.Start()
defer metricsServer.Shutdown()
defer underlyingStoreWithCaches.Shutdown()
defer underlyingStore.Shutdown() //do we actually need this? Oo
interruptChan := make(chan os.Signal, 1)
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
<-interruptChan
// deferred shutdowns happen now
cleanerStopper.StopAndWait()
}
func initUpstreamStore() store.BlobStore {
var s store.BlobStore
if upstreamReflector == "" {
return nil
}
switch upstreamProtocol {
case "tcp":
s = peer.NewStore(peer.StoreOpts{
Address: upstreamReflector,
Timeout: 30 * time.Second,
})
case "http3":
s = http32.NewStore(http32.StoreOpts{
Address: upstreamReflector,
Timeout: 30 * time.Second,
})
case "http":
s = store.NewHttpStore(upstreamReflector)
default:
log.Fatalf("protocol is not recognized: %s", upstreamProtocol)
}
return s
}
func initEdgeStore() store.BlobStore {
var s3Store *store.S3Store
var s store.BlobStore
if conf != "none" {
s3Store = store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
}
if originEndpointFallback != "" && originEndpoint != "" {
ittt := store.NewITTTStore(store.NewCloudFrontROStore(originEndpoint), store.NewCloudFrontROStore(originEndpointFallback))
if s3Store != nil {
s = store.NewCloudFrontRWStore(ittt, s3Store)
} else {
s = ittt
}
} else if s3Store != nil {
s = s3Store
} else {
log.Fatalf("this configuration does not include a valid upstream source")
}
return s
}
func initDBStore(s store.BlobStore) store.BlobStore {
if useDB {
dbInst := &db.SQL{
TrackAccess: db.TrackAccessStreams,
LogQueries: log.GetLevel() == log.DebugLevel,
}
err := dbInst.Connect(globalConfig.DBConn)
if err != nil {
log.Fatal(err)
}
s = store.NewDBBackedStore(s, dbInst, false)
}
return s
}
func initStores() store.BlobStore {
s := initUpstreamStore()
if s == nil {
s = initEdgeStore()
}
s = initDBStore(s)
return s
}
// initCaches returns a store wrapped with caches and a stop group to execute a clean shutdown
func initCaches(s store.BlobStore) (store.BlobStore, *stop.Group) {
stopper := stop.New()
diskStore := initDiskStore(s, diskCache, stopper)
finalStore := initDiskStore(diskStore, secondaryDiskCache, stopper)
stop.New()
if memCache > 0 {
finalStore = store.NewCachingStore(
"reflector",
finalStore,
store.NewGcacheStore("mem", store.NewMemStore(), memCache, store.LRU),
)
}
return finalStore, stopper
}
func initDiskStore(upstreamStore store.BlobStore, diskParams string, stopper *stop.Group) store.BlobStore {
diskCacheMaxSize, diskCachePath, cacheManager := diskCacheParams(diskParams)
//we are tracking blobs in memory with a 1 byte long boolean, which means that for each 2MB (a blob) we need 1Byte
// so if the underlying cache holds 10MB, 10MB/2MB=5Bytes which is also the exact count of objects to restore on startup
realCacheSize := float64(diskCacheMaxSize) / float64(stream.MaxBlobSize)
if diskCacheMaxSize == 0 {
return upstreamStore
}
err := os.MkdirAll(diskCachePath, os.ModePerm)
if err != nil {
log.Fatal(err)
}
diskStore := store.NewDiskStore(diskCachePath, 2)
var unwrappedStore store.BlobStore
cleanerStopper := stop.New(stopper)
if cacheManager == "localdb" {
localDb := &db.SQL{
SoftDelete: true,
TrackAccess: db.TrackAccessBlobs,
LogQueries: log.GetLevel() == log.DebugLevel,
}
err = localDb.Connect("reflector:reflector@tcp(localhost:3306)/reflector")
if err != nil {
log.Fatal(err)
}
unwrappedStore = store.NewDBBackedStore(diskStore, localDb, true)
go cleanOldestBlobs(int(realCacheSize), localDb, unwrappedStore, cleanerStopper)
} else {
unwrappedStore = store.NewGcacheStore("nvme", store.NewDiskStore(diskCachePath, 2), int(realCacheSize), cacheMangerToGcache[cacheManager])
}
wrapped := store.NewCachingStore(
"reflector",
upstreamStore,
unwrappedStore,
)
return wrapped
}
func diskCacheParams(diskParams string) (int, string, string) {
if diskParams == "" {
return 0, "", ""
}
parts := strings.Split(diskParams, ":")
if len(parts) != 3 {
log.Fatalf("%s does is formatted incorrectly. Expected format: 'sizeGB:CACHE_PATH:cachemanager' for example: '100GB:/tmp/downloaded_blobs:localdb'", diskParams)
}
diskCacheSize := parts[0]
path := parts[1]
cacheManager := parts[2]
if len(path) == 0 || path[0] != '/' {
log.Fatalf("disk cache paths must start with '/'")
}
if !util.InSlice(cacheManager, cacheManagers) {
log.Fatalf("specified cache manager '%s' is not supported. Use one of the following: %v", cacheManager, cacheManagers)
}
var maxSize datasize.ByteSize
err := maxSize.UnmarshalText([]byte(diskCacheSize))
if err != nil {
log.Fatal(err)
}
if maxSize <= 0 {
log.Fatal("disk cache size must be more than 0")
}
return int(maxSize), path, cacheManager
}
func cleanOldestBlobs(maxItems int, db *db.SQL, store store.BlobStore, stopper *stop.Group) {
// this is so that it runs on startup without having to wait for 10 minutes
err := doClean(maxItems, db, store, stopper)
if err != nil {
log.Error(errors.FullTrace(err))
}
const cleanupInterval = 10 * time.Minute
for {
select {
case <-stopper.Ch():
log.Infoln("stopping self cleanup")
return
case <-time.After(cleanupInterval):
err := doClean(maxItems, db, store, stopper)
if err != nil {
log.Error(errors.FullTrace(err))
}
}
}
}
func doClean(maxItems int, db *db.SQL, store store.BlobStore, stopper *stop.Group) error {
blobsCount, err := db.Count()
if err != nil {
return err
}
if blobsCount >= maxItems {
itemsToDelete := blobsCount / 10
blobs, err := db.LeastRecentlyAccessedHashes(itemsToDelete)
if err != nil {
return err
}
blobsChan := make(chan string, len(blobs))
wg := &stop.Group{}
go func() {
for _, hash := range blobs {
select {
case <-stopper.Ch():
return
default:
}
blobsChan <- hash
}
close(blobsChan)
}()
for i := 0; i < 3; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for h := range blobsChan {
select {
case <-stopper.Ch():
return
default:
}
err = store.Delete(h)
if err != nil {
log.Errorf("error pruning %s: %s", h, errors.FullTrace(err))
continue
}
}
}()
}
wg.Wait()
}
return nil
}
+48 -48
View File
@@ -1,20 +1,15 @@
package cmd
import (
"encoding/json"
"io/ioutil"
"os"
"strings"
"github.com/lbryio/reflector.go/updater"
"github.com/lbryio/lbry.go/v2/dht"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/johntdyer/slackrus"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
// Config is the base configuration for Prism when no sub commands are used.
@@ -23,8 +18,10 @@ type Config struct {
AwsSecret string `json:"aws_secret"`
BucketRegion string `json:"bucket_region"`
BucketName string `json:"bucket_name"`
S3Endpoint string `json:"s3_endpoint"`
DBConn string `json:"db_conn"`
SlackHookURL string `json:"slack_hook_url"`
SlackChannel string `json:"slack_channel"`
UpdateBinURL string `json:"update_bin_url"`
UpdateCmd string `json:"update_cmd"`
}
@@ -38,7 +35,8 @@ const (
)
var conf string
var globalConfig Config
//var globalConfig Config
var rootCmd = &cobra.Command{
Use: "prism",
@@ -51,7 +49,7 @@ var rootCmd = &cobra.Command{
func init() {
rootCmd.PersistentFlags().StringSliceVarP(&verbose, "verbose", "v", []string{}, "Verbose logging for specific components")
rootCmd.PersistentFlags().StringVar(&conf, "conf", "config.json", "Path to config. Use 'none' to disable")
rootCmd.PersistentFlags().StringVar(&conf, "conf-dir", "./", "Path to config directory")
}
// Execute adds all child commands to the root command and sets flags appropriately.
@@ -86,38 +84,40 @@ func preRun(cmd *cobra.Command, args []string) {
}
}
var err error
//var err error
if conf == "" {
logrus.Errorln("--conf flag required")
os.Exit(1)
} else if conf != "none" {
globalConfig, err = loadConfig(conf)
if err != nil {
logrus.Error(err)
os.Exit(1)
}
}
//else if conf != "none" {
// globalConfig, err = loadConfig(conf)
// if err != nil {
// logrus.Error(err)
// os.Exit(1)
// }
//}
viper.AddConfigPath(conf)
if globalConfig.SlackHookURL != "" {
hook := &slackrus.SlackrusHook{
HookURL: globalConfig.SlackHookURL,
AcceptedLevels: slackrus.LevelThreshold(logrus.InfoLevel),
Channel: "#reflector-logs",
//IconEmoji: ":ghost:",
//Username: "reflector.go",
}
//logrus.SetFormatter(&logrus.JSONFormatter{})
logrus.AddHook(hook)
debugLogger.AddHook(hook)
}
//if globalConfig.SlackHookURL != "" {
// hook := &slackrus.SlackrusHook{
// HookURL: globalConfig.SlackHookURL,
// AcceptedLevels: slackrus.LevelThreshold(logrus.InfoLevel),
// Channel: globalConfig.SlackChannel,
// //IconEmoji: ":ghost:",
// //Username: "reflector.go",
// }
// //logrus.SetFormatter(&logrus.JSONFormatter{})
// logrus.AddHook(hook)
// debugLogger.AddHook(hook)
//}
if globalConfig.UpdateBinURL != "" {
if globalConfig.UpdateCmd == "" {
logrus.Warnln("update_cmd is empty in conf file")
}
logrus.Println("starting update checker")
go updater.Run(globalConfig.UpdateBinURL, globalConfig.UpdateCmd)
}
//if globalConfig.UpdateBinURL != "" {
// if globalConfig.UpdateCmd == "" {
// logrus.Warnln("update_cmd is empty in conf file")
// }
// logrus.Println("starting update checker")
// go updater.Run(globalConfig.UpdateBinURL, globalConfig.UpdateCmd)
//}
}
func checkErr(err error) {
@@ -138,20 +138,20 @@ func argFuncs(funcs ...cobra.PositionalArgs) cobra.PositionalArgs {
}
}
func loadConfig(path string) (Config, error) {
var c Config
raw, err := ioutil.ReadFile(path)
if err != nil {
if os.IsNotExist(err) {
return c, errors.Err("config file not found")
}
return c, errors.Err(err)
}
err = json.Unmarshal(raw, &c)
return c, errors.Err(err)
}
//func loadConfig(path string) (Config, error) {
// var c Config
//
// raw, err := os.ReadFile(path)
// if err != nil {
// if os.IsNotExist(err) {
// return c, errors.Err("config file not found")
// }
// return c, errors.Err(err)
// }
//
// err = json.Unmarshal(raw, &c)
// return c, errors.Err(err)
//}
func mustGetFlagString(cmd *cobra.Command, name string) string {
v, err := cmd.Flags().GetString(name)
+9 -9
View File
@@ -5,7 +5,6 @@ import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"os/signal"
"path"
@@ -39,17 +38,16 @@ func sendCmd(cmd *cobra.Command, args []string) {
reflectorAddress := args[0]
err := hackyReflector.Connect(reflectorAddress)
checkErr(err)
defer hackyReflector.Close()
defer func() { _ = hackyReflector.Close() }()
filePath := args[1]
file, err := os.Open(filePath)
checkErr(err)
defer file.Close()
defer func() { _ = file.Close() }()
sdCachePath := ""
sdCacheDir := mustGetFlagString(cmd, "sd-cache")
if sdCacheDir != "" {
if _, err := os.Stat(sdCacheDir); os.IsNotExist(err) {
if _, statErr := os.Stat(sdCacheDir); os.IsNotExist(statErr) {
err = os.MkdirAll(sdCacheDir, 0777)
checkErr(err)
}
@@ -59,8 +57,9 @@ func sendCmd(cmd *cobra.Command, args []string) {
var enc *stream.Encoder
if sdCachePath != "" {
if _, err := os.Stat(sdCachePath); !os.IsNotExist(err) {
sdBlob, err := ioutil.ReadFile(sdCachePath)
if _, statErr := os.Stat(sdCachePath); !os.IsNotExist(statErr) {
var sdBlob []byte
sdBlob, err = os.ReadFile(sdCachePath)
checkErr(err)
cachedSDBlob := &stream.SDBlob{}
err = cachedSDBlob.FromBlob(sdBlob)
@@ -89,7 +88,8 @@ func sendCmd(cmd *cobra.Command, args []string) {
break
}
b, err := enc.Next()
var b stream.Blob
b, err = enc.Next()
if errors.Is(err, io.EOF) {
break
}
@@ -110,7 +110,7 @@ func sendCmd(cmd *cobra.Command, args []string) {
sd := enc.SDBlob()
//sd.StreamName = filepath.Base(filePath)
//sd.SuggestedFileName = filepath.Base(filePath)
err = ioutil.WriteFile(sdCachePath, sd.ToBlob(), 0666)
err = os.WriteFile(sdCachePath, sd.ToBlob(), 0666)
if err != nil {
fmt.Printf("error saving sd blob: %v\n", err)
fmt.Println(sd.ToJson())
+1 -1
View File
@@ -51,7 +51,7 @@ func sendBlobCmd(cmd *cobra.Command, args []string) {
file, err := os.Open(path)
checkErr(err)
defer file.Close()
defer func() { _ = file.Close() }()
s, err := stream.New(file)
checkErr(err)
-102
View File
@@ -1,102 +0,0 @@
package cmd
import (
"os"
"os/signal"
"strconv"
"strings"
"syscall"
"github.com/lbryio/reflector.go/cluster"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/prism"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/server/peer"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/dht"
"github.com/lbryio/lbry.go/v2/dht/bits"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
const (
startNewCluster = "new"
)
var (
startClusterPort int
startPeerPort int
startReflectorPort int
startDhtPort int
startDhtSeeds []string
startHashRange string
)
func init() {
var cmd = &cobra.Command{
Use: `start [cluster-address|"new"]`,
Short: "Runs full prism application with cluster, dht, peer server, and reflector server.",
Run: startCmd,
Args: cobra.ExactArgs(1),
}
cmd.PersistentFlags().IntVar(&startClusterPort, "cluster-port", cluster.DefaultPort, "Port that cluster listens on")
cmd.PersistentFlags().IntVar(&startPeerPort, "peer-port", peer.DefaultPort, "Port to start peer protocol on")
cmd.PersistentFlags().IntVar(&startReflectorPort, "reflector-port", reflector.DefaultPort, "Port to start reflector protocol on")
cmd.PersistentFlags().IntVar(&startDhtPort, "dht-port", dht.DefaultPort, "Port that dht will listen on")
cmd.PersistentFlags().StringSliceVar(&startDhtSeeds, "dht-seeds", []string{}, "Comma-separated list of dht seed nodes (addr:port,addr:port,...)")
cmd.PersistentFlags().StringVar(&startHashRange, "hash-range", "", "Limit on range of hashes to announce (start-end)")
rootCmd.AddCommand(cmd)
}
func startCmd(cmd *cobra.Command, args []string) {
db := &db.SQL{
LogQueries: log.GetLevel() == log.DebugLevel,
}
err := db.Connect(globalConfig.DBConn)
checkErr(err)
s3 := store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
comboStore := store.NewDBBackedStore(s3, db, false)
conf := prism.DefaultConf()
// TODO: args we need:
// minNodes - minimum number of nodes before announcing starts. otherwise first node will try to announce all the blobs in the db
// or maybe we should do maxHashesPerNode?
// in either case, this should not kill the cluster, but should only limit announces (and notify when some hashes are being left unannounced)
if args[0] != startNewCluster {
conf.ClusterSeedAddr = args[0]
}
conf.DB = db
conf.Blobs = comboStore
conf.DhtAddress = "0.0.0.0:" + strconv.Itoa(startDhtPort)
conf.DhtSeedNodes = startDhtSeeds
conf.ClusterPort = startClusterPort
conf.PeerPort = startPeerPort
conf.ReflectorPort = startReflectorPort
if startHashRange != "" {
hashRange := strings.Split(startHashRange, "-")
if len(hashRange) != 2 {
log.Fatal("invalid hash range")
}
r := bits.Range{Start: bits.FromShortHexP(hashRange[0]), End: bits.FromShortHexP(hashRange[1])}
conf.HashRange = &r
}
p := prism.New(conf)
err = p.Start()
if err != nil {
log.Fatal(err)
}
interruptChan := make(chan os.Signal, 1)
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
<-interruptChan
p.Shutdown()
}
+4 -4
View File
@@ -29,9 +29,9 @@ func init() {
func testCmd(cmd *cobra.Command, args []string) {
log.Printf("reflector %s", meta.VersionString())
memStore := store.NewMemStore()
memStore := store.NewMemStore(store.MemParams{Name: "test"})
reflectorServer := reflector.NewServer(memStore, memStore)
reflectorServer := reflector.NewIngestionServer(memStore)
reflectorServer.Timeout = 3 * time.Minute
err := reflectorServer.Start(":" + strconv.Itoa(reflector.DefaultPort))
@@ -39,8 +39,8 @@ func testCmd(cmd *cobra.Command, args []string) {
log.Fatal(err)
}
peerServer := peer.NewServer(memStore)
err = peerServer.Start(":" + strconv.Itoa(reflector.DefaultPort+1))
peerServer := peer.NewServer(memStore, fmt.Sprintf(":%d", reflector.DefaultPort+1))
err = peerServer.Start()
if err != nil {
log.Fatal(err)
}
+10 -10
View File
@@ -5,9 +5,8 @@ import (
"os/signal"
"syscall"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/config"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/store"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
@@ -31,17 +30,18 @@ func init() {
}
func uploadCmd(cmd *cobra.Command, args []string) {
db := &db.SQL{
LogQueries: log.GetLevel() == log.DebugLevel,
store, err := config.LoadStores(conf, "upload")
if err != nil {
log.Fatal(err)
}
err := db.Connect(globalConfig.DBConn)
checkErr(err)
defer store.Shutdown()
st := store.NewDBBackedStore(
store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName),
db, false)
databaseConn, err := config.LoadDatabase(conf, "upload")
if err != nil {
log.Fatal(err)
}
uploader := reflector.NewUploader(db, st, uploadWorkers, uploadSkipExistsCheck, uploadDeleteBlobsAfterUpload)
uploader := reflector.NewUploader(databaseConn, store, uploadWorkers, uploadSkipExistsCheck, uploadDeleteBlobsAfterUpload)
interruptChan := make(chan os.Signal, 1)
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
+1 -1
View File
@@ -18,5 +18,5 @@ func init() {
}
func versionCmd(cmd *cobra.Command, args []string) {
fmt.Println(meta.VersionString())
fmt.Println(meta.FullName())
}
+113
View File
@@ -0,0 +1,113 @@
package config
import (
"fmt"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/server"
"github.com/lbryio/reflector.go/server/http"
"github.com/lbryio/reflector.go/server/http3"
"github.com/lbryio/reflector.go/server/peer"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/extras/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
)
func LoadStores(path, file string) (store.BlobStore, error) {
v := viper.New()
v.SetConfigType("yaml")
v.AddConfigPath(path)
v.SetConfigName(file)
err := v.ReadInConfig()
if err != nil {
return nil, errors.Err(err)
}
storeViper := v.Sub("store")
for storeType := range storeViper.AllSettings() {
factory, exists := store.Factories[storeType]
if !exists {
return nil, errors.Err("unknown store type: %s", storeType)
}
storeConfig := storeViper.Sub(storeType)
s, err := factory(storeConfig)
if err != nil {
return nil, errors.Err(err)
}
//we only expect 1 store as the root, so let's return it
return s, nil
}
return nil, nil
}
func LoadServers(store store.BlobStore, path, file string) ([]server.BlobServer, error) {
v := viper.New()
v.SetConfigType("yaml")
v.AddConfigPath(path)
v.SetConfigName(file)
err := v.ReadInConfig()
if err != nil {
return nil, errors.Err(err)
}
servers := make([]server.BlobServer, 0)
serversViper := v.Sub("servers")
for serverType := range serversViper.AllSettings() {
var cfg server.BlobServerConfig
err := serversViper.Sub(serverType).Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
switch serverType {
case "http":
servers = append(servers, http.NewServer(store, cfg.MaxConcurrentRequests, cfg.EdgeToken, fmt.Sprintf("%s:%d", cfg.Address, cfg.Port)))
case "http3":
servers = append(servers, http3.NewServer(store, cfg.MaxConcurrentRequests, fmt.Sprintf("%s:%d", cfg.Address, cfg.Port)))
case "peer":
servers = append(servers, peer.NewServer(store, fmt.Sprintf("%s:%d", cfg.Address, cfg.Port)))
default:
return nil, errors.Err("unknown server type: %s", serverType)
}
}
return servers, nil
}
func LoadDatabase(path, file string) (*db.SQL, error) {
v := viper.New()
v.SetConfigType("yaml")
v.AddConfigPath(path)
v.SetConfigName(file)
err := v.ReadInConfig()
if err != nil {
return nil, errors.Err(err)
}
dbConfig := v.Sub("database")
if dbConfig == nil {
return nil, errors.Err("db config not found")
}
user := dbConfig.GetString("user")
password := dbConfig.GetString("password")
host := dbConfig.GetString("host")
port := dbConfig.GetInt("port")
database := dbConfig.GetString("database")
logQueries := dbConfig.GetBool("log_queries")
accessTracking := dbConfig.GetInt("access_tracking")
if user == "" || password == "" || host == "" || port == 0 || database == "" {
return nil, errors.Err("db config is missing required fields")
}
dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", user, password, host, port, database)
dbInstance := &db.SQL{
TrackingLevel: db.AccessTrackingLevel(accessTracking),
SoftDelete: true,
LogQueries: logQueries || log.GetLevel() == log.DebugLevel,
}
err = dbInstance.Connect(dsn)
return dbInstance, err
}
+31 -32
View File
@@ -15,33 +15,32 @@ import (
"github.com/lbryio/lbry.go/v2/stream"
"github.com/go-sql-driver/mysql"
_ "github.com/go-sql-driver/mysql" // blank import for db driver ensures its imported even if its not used
log "github.com/sirupsen/logrus"
"github.com/volatiletech/null"
"github.com/volatiletech/null/v8"
"go.uber.org/atomic"
)
// SdBlob is a special blob that contains information on the rest of the blobs in the stream
type SdBlob struct {
StreamName string `json:"stream_name"`
Blobs []struct {
Length int `json:"length"`
BlobNum int `json:"blob_num"`
BlobHash string `json:"blob_hash,omitempty"`
IV string `json:"iv"`
} `json:"blobs"`
StreamName string `json:"stream_name"`
StreamType string `json:"stream_type"`
Key string `json:"key"`
SuggestedFileName string `json:"suggested_file_name"`
StreamHash string `json:"stream_hash"`
Blobs []struct {
BlobHash string `json:"blob_hash,omitempty"`
IV string `json:"iv"`
Length int `json:"length"`
BlobNum int `json:"blob_num"`
} `json:"blobs"`
}
type trackAccess int
type AccessTrackingLevel int
const (
TrackAccessNone trackAccess = iota // Don't track accesses
TrackAccessStreams // Track accesses at the stream level
TrackAccessBlobs // Track accesses at the blob level
TrackAccessNone AccessTrackingLevel = iota // Don't track accesses
TrackAccessStreams // Track accesses at the stream level
TrackAccessBlobs // Track accesses at the blob level
)
// SQL implements the DB interface
@@ -49,7 +48,7 @@ type SQL struct {
conn *sql.DB
// Track the approx last time a blob or stream was accessed
TrackAccess trackAccess
TrackingLevel AccessTrackingLevel
// Instead of deleting a blob, marked it as not stored in the db
SoftDelete bool
@@ -58,7 +57,7 @@ type SQL struct {
LogQueries bool
}
func (s SQL) logQuery(query string, args ...interface{}) {
func (s *SQL) logQuery(query string, args ...interface{}) {
if !s.LogQueries {
return
}
@@ -98,7 +97,7 @@ func (s *SQL) AddBlob(hash string, length int, isStored bool) error {
return err
}
//AddBlobs adds blobs to the database.
// AddBlobs adds blobs to the database.
func (s *SQL) AddBlobs(hash []string) error {
if s.conn == nil {
return errors.Err("not connected")
@@ -180,7 +179,7 @@ func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error)
q string
args []interface{}
)
if s.TrackAccess == TrackAccessBlobs {
if s.TrackingLevel == TrackAccessBlobs {
args = []interface{}{hash, isStored, length, time.Now()}
q = "INSERT INTO blob_ (hash, is_stored, length, last_accessed_at) VALUES (" + qt.Qs(len(args)) + ") ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored)), last_accessed_at = VALUES(last_accessed_at)"
} else {
@@ -202,7 +201,7 @@ func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error)
return 0, errors.Err("blob ID is 0 even after INSERTing and SELECTing")
}
if s.TrackAccess == TrackAccessBlobs {
if s.TrackingLevel == TrackAccessBlobs {
err := s.touchBlobs([]uint64{uint64(blobID)})
if err != nil {
return 0, errors.Err(err)
@@ -219,7 +218,7 @@ func (s *SQL) insertStream(hash string, sdBlobID int64) (int64, error) {
args []interface{}
)
if s.TrackAccess == TrackAccessStreams {
if s.TrackingLevel == TrackAccessStreams {
args = []interface{}{hash, sdBlobID, time.Now()}
q = "INSERT IGNORE INTO stream (hash, sd_blob_id, last_accessed_at) VALUES (" + qt.Qs(len(args)) + ")"
} else {
@@ -241,7 +240,7 @@ func (s *SQL) insertStream(hash string, sdBlobID int64) (int64, error) {
return 0, errors.Err("stream ID is 0 even after INSERTing and SELECTing")
}
if s.TrackAccess == TrackAccessStreams {
if s.TrackingLevel == TrackAccessStreams {
err := s.touchStreams([]uint64{uint64(streamID)})
if err != nil {
return 0, errors.Err(err)
@@ -265,9 +264,9 @@ func (s *SQL) HasBlobs(hashes []string, touch bool) (map[string]bool, error) {
exists, idsNeedingTouch, err := s.hasBlobs(hashes)
if touch {
if s.TrackAccess == TrackAccessBlobs {
if s.TrackingLevel == TrackAccessBlobs {
_ = s.touchBlobs(idsNeedingTouch)
} else if s.TrackAccess == TrackAccessStreams {
} else if s.TrackingLevel == TrackAccessStreams {
_ = s.touchStreams(idsNeedingTouch)
}
}
@@ -339,11 +338,11 @@ func (s *SQL) hasBlobs(hashes []string) (map[string]bool, []uint64, error) {
batch := hashes[doneIndex:sliceEnd]
var query string
if s.TrackAccess == TrackAccessBlobs {
if s.TrackingLevel == TrackAccessBlobs {
query = `SELECT b.hash, b.id, NULL, b.last_accessed_at
FROM blob_ b
WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)`
} else if s.TrackAccess == TrackAccessStreams {
} else if s.TrackingLevel == TrackAccessStreams {
query = `SELECT b.hash, b.id, s.id, s.last_accessed_at
FROM blob_ b
LEFT JOIN stream_blob sb ON b.id = sb.blob_id
@@ -372,15 +371,15 @@ WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)`
defer closeRows(rows)
for rows.Next() {
err := rows.Scan(&hash, &blobID, &streamID, &lastAccessedAt)
err = rows.Scan(&hash, &blobID, &streamID, &lastAccessedAt)
if err != nil {
return errors.Err(err)
}
exists[hash] = true
if !lastAccessedAt.Valid || lastAccessedAt.Time.Before(touchDeadline) {
if s.TrackAccess == TrackAccessBlobs {
if s.TrackingLevel == TrackAccessBlobs {
needsTouch = append(needsTouch, blobID)
} else if s.TrackAccess == TrackAccessStreams && !streamID.IsZero() {
} else if s.TrackingLevel == TrackAccessStreams && !streamID.IsZero() {
needsTouch = append(needsTouch, streamID.Uint64)
}
}
@@ -419,13 +418,13 @@ func (s *SQL) Delete(hash string) error {
return errors.Err(err)
}
//LeastRecentlyAccessedHashes gets the least recently accessed blobs
// LeastRecentlyAccessedHashes gets the least recently accessed blobs
func (s *SQL) LeastRecentlyAccessedHashes(maxBlobs int) ([]string, error) {
if s.conn == nil {
return nil, errors.Err("not connected")
}
if s.TrackAccess != TrackAccessBlobs {
if s.TrackingLevel != TrackAccessBlobs {
return nil, errors.Err("blob access tracking is disabled")
}
@@ -490,7 +489,7 @@ func (s *SQL) GetBlocked() (map[string]bool, error) {
var hash string
for rows.Next() {
err := rows.Scan(&hash)
err = rows.Scan(&hash)
if err != nil {
return nil, errors.Err(err)
}
@@ -534,7 +533,7 @@ func (s *SQL) MissingBlobsForKnownStream(sdHash string) ([]string, error) {
var hash string
for rows.Next() {
err := rows.Scan(&hash)
err = rows.Scan(&hash)
if err != nil {
return nil, errors.Err(err)
}
@@ -637,7 +636,7 @@ func (s *SQL) GetStoredHashesInRange(ctx context.Context, start, end bits.Bitmap
var hash string
ScanLoop:
for rows.Next() {
err := rows.Scan(&hash)
err = rows.Scan(&hash)
if err != nil {
ech <- err
return
+129 -41
View File
@@ -1,51 +1,139 @@
module github.com/lbryio/reflector.go
go 1.25
replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19
require (
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
github.com/aws/aws-sdk-go v1.27.0
github.com/aws/aws-sdk-go-v2 v1.41.0
github.com/aws/aws-sdk-go-v2/config v1.32.6
github.com/aws/aws-sdk-go-v2/credentials v1.19.6
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.18
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0
github.com/bluele/gcache v0.0.2
github.com/brk0v/directio v0.0.0-20190225130936-69406e757cf7
github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3
github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d
github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2
github.com/davecgh/go-spew v1.1.1
github.com/brk0v/directio v0.0.0-20241105172640-ae9d82eb8fee
github.com/btcsuite/btcd v0.24.2
github.com/btcsuite/btcutil v1.0.2
github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
github.com/ekyoung/gin-nice-recovery v0.0.0-20160510022553-1654dca486db
github.com/gin-gonic/gin v1.7.1
github.com/go-sql-driver/mysql v1.6.0
github.com/gogo/protobuf v1.2.1
github.com/golang/protobuf v1.5.2
github.com/google/go-cmp v0.5.6 // indirect
github.com/google/gops v0.3.18
github.com/gorilla/mux v1.8.0
github.com/hashicorp/go-msgpack v0.5.5 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/hashicorp/serf v0.9.5
github.com/gin-gonic/gin v1.11.0
github.com/go-sql-driver/mysql v1.9.3
github.com/gogo/protobuf v1.3.2
github.com/golang/protobuf v1.5.4
github.com/google/gops v0.3.28
github.com/gorilla/mux v1.8.1
github.com/hashicorp/serf v0.10.2
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf
github.com/johntdyer/slackrus v0.0.0-20210521205746-42486fb4c48c
github.com/karrick/godirwalk v1.16.1
github.com/lbryio/chainquery v1.9.0
github.com/lbryio/lbry.go/v2 v2.7.2-0.20210416195322-6516df1418e3
github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386
github.com/lucas-clemente/quic-go v0.20.1
github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6
github.com/prometheus/client_golang v1.10.0
github.com/sergi/go-diff v1.2.0 // indirect
github.com/sirupsen/logrus v1.8.1
github.com/spf13/afero v1.4.1 // indirect
github.com/spf13/cast v1.3.0
github.com/spf13/cobra v1.1.3
github.com/spf13/viper v1.7.1 // indirect
github.com/stretchr/testify v1.7.0
github.com/volatiletech/null v8.0.0+incompatible
go.uber.org/atomic v1.7.0
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
golang.org/x/text v0.3.6 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/protobuf v1.27.1 // indirect
github.com/karrick/godirwalk v1.17.0
github.com/lbryio/chainquery v1.9.1-0.20240927170248-48c092515dea
github.com/lbryio/lbry.go/v2 v2.7.2-0.20230307181431-a01aa6dc0629
github.com/lbryio/types v0.0.0-20220224142228-73610f6654a6
github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5
github.com/prometheus/client_golang v1.23.2
github.com/quic-go/quic-go v0.58.0
github.com/sirupsen/logrus v1.9.3
github.com/spf13/cast v1.10.0
github.com/spf13/cobra v1.10.2
github.com/spf13/viper v1.21.0
github.com/stretchr/testify v1.11.1
github.com/volatiletech/null/v8 v8.1.2
go.uber.org/atomic v1.11.0
golang.org/x/sync v0.19.0
)
go 1.16
require (
filippo.io/edwards25519 v1.1.0 // indirect
github.com/armon/go-metrics v0.4.1 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 // indirect
github.com/aws/smithy-go v1.24.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/btcsuite/btclog v1.0.0 // indirect
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd // indirect
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 // indirect
github.com/bytedance/sonic v1.14.0 // indirect
github.com/bytedance/sonic/loader v0.3.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/cloudwego/base64x v0.1.6 // indirect
github.com/friendsofgo/errors v0.9.2 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.8 // indirect
github.com/gin-contrib/sse v1.1.0 // indirect
github.com/go-errors/errors v1.5.1 // indirect
github.com/go-ini/ini v1.67.0 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.27.0 // indirect
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
github.com/goccy/go-json v0.10.5 // indirect
github.com/goccy/go-yaml v1.18.0 // indirect
github.com/gofrs/uuid v4.4.0+incompatible // indirect
github.com/google/btree v1.1.2 // indirect
github.com/gorilla/rpc v1.2.0 // indirect
github.com/gorilla/websocket v1.5.3 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
github.com/hashicorp/go-metrics v0.5.4 // indirect
github.com/hashicorp/go-msgpack/v2 v2.1.2 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-sockaddr v1.0.5 // indirect
github.com/hashicorp/golang-lru v1.0.2 // indirect
github.com/hashicorp/memberlist v0.5.2 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/miekg/dns v1.1.56 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.66.1 // indirect
github.com/prometheus/procfs v0.16.1 // indirect
github.com/quic-go/qpack v0.6.0 // indirect
github.com/sagikazarmark/locafero v0.11.0 // indirect
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
github.com/shopspring/decimal v1.4.0 // indirect
github.com/slack-go/slack v0.16.0 // indirect
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
github.com/spf13/afero v1.15.0 // indirect
github.com/spf13/pflag v1.0.10 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.3.0 // indirect
github.com/volatiletech/inflect v0.0.1 // indirect
github.com/volatiletech/randomize v0.0.1 // indirect
github.com/volatiletech/strmangle v0.0.8 // indirect
go.yaml.in/yaml/v2 v2.4.2 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/arch v0.20.0 // indirect
golang.org/x/crypto v0.41.0 // indirect
golang.org/x/mod v0.27.0 // indirect
golang.org/x/net v0.43.0 // indirect
golang.org/x/sys v0.35.0 // indirect
golang.org/x/text v0.28.0 // indirect
golang.org/x/time v0.8.0 // indirect
golang.org/x/tools v0.36.0 // indirect
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
google.golang.org/protobuf v1.36.9 // indirect
gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
+292 -734
View File
File diff suppressed because it is too large. Load diff
+30 -10
View File
@@ -69,6 +69,7 @@ const (
DirectionDownload = "download" // from reflector
LabelCacheType = "cache_type"
LabelOrigin = "origin"
LabelComponent = "component"
LabelSource = "source"
@@ -86,8 +87,13 @@ const (
errUnexpectedEOFStr = "unexpected_eof_str"
errJSONSyntax = "json_syntax"
errBlobTooBig = "blob_too_big"
errInvalidPeerJSON = "invalid_peer_json"
errInvalidPeerData = "invalid_peer_data"
errRequestTooLarge = "request_too_large"
errDeadlineExceeded = "deadline_exceeded"
errHashMismatch = "hash_mismatch"
errProtectedBlob = "protected_blob"
errInvalidBlobHash = "invalid_blob_hash"
errZeroByteBlob = "zero_byte_blob"
errInvalidCharacter = "invalid_character"
errBlobNotFound = "blob_not_found"
@@ -130,18 +136,12 @@ var (
Name: "hit_total",
Help: "Total number of blobs retrieved from the cache storage",
}, []string{LabelCacheType, LabelComponent})
ThisHitCount = promauto.NewCounter(prometheus.CounterOpts{
ItttHitCount = promauto.NewCounterVec(prometheus.CounterOpts{
Namespace: ns,
Subsystem: subsystemITTT,
Name: "this_hit_total",
Help: "Total number of blobs retrieved from the this storage",
})
ThatHitCount = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Subsystem: subsystemITTT,
Name: "that_hit_total",
Help: "Total number of blobs retrieved from the that storage",
})
Name: "hits_total",
Help: "Total number of blobs retrieved from the this/that storage",
}, []string{LabelOrigin})
CacheMissCount = promauto.NewCounterVec(prometheus.CounterOpts{
Namespace: ns,
Subsystem: subsystemCache,
@@ -224,6 +224,11 @@ var (
Name: "s3_out_bytes",
Help: "Total number of outgoing bytes (to S3)",
})
MtrInBytesUpstream = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "upstream_in_bytes",
Help: "Total number of incoming bytes (from Upstream)",
})
MtrInBytesS3 = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "s3_in_bytes",
@@ -252,6 +257,11 @@ func CacheLabels(name, component string) prometheus.Labels {
LabelComponent: component,
}
}
func ItttLabels(orig string) prometheus.Labels {
return prometheus.Labels{
LabelOrigin: orig,
}
}
func TrackError(direction string, e error) (shouldLog bool) { // shouldLog is a hack, but whatever
if e == nil {
@@ -296,10 +306,20 @@ func TrackError(direction string, e error) (shouldLog bool) { // shouldLog is a
} else if strings.Contains(err.Error(), "blob must be at most") {
//log.Warnln("blob must be at most X bytes is not the same as ErrBlobTooBig")
errType = errBlobTooBig
} else if strings.Contains(err.Error(), "invalid json request") {
errType = errInvalidPeerJSON
} else if strings.Contains(err.Error(), "Invalid data") {
errType = errInvalidPeerData
} else if strings.Contains(err.Error(), "request is too large") {
errType = errRequestTooLarge
} else if strings.Contains(err.Error(), "Invalid blob hash length") {
errType = errInvalidBlobHash
} else if strings.Contains(err.Error(), "hash of received blob data does not match hash from send request") {
errType = errHashMismatch
} else if strings.Contains(err.Error(), "blob not found") {
errType = errBlobNotFound
} else if strings.Contains(err.Error(), "requested blob is protected") {
errType = errProtectedBlob
} else if strings.Contains(err.Error(), "0-byte blob received") {
errType = errZeroByteBlob
} else if strings.Contains(err.Error(), "PROTOCOL_VIOLATION: tried to retire connection") {
+10 -11
View File
@@ -8,24 +8,23 @@ import (
qt "github.com/lbryio/lbry.go/v2/extras/query"
"github.com/go-sql-driver/mysql"
_ "github.com/go-sql-driver/mysql" // blank import for db driver ensures its imported even if its not used
log "github.com/sirupsen/logrus"
"github.com/volatiletech/null"
"github.com/volatiletech/null/v8"
)
// SdBlob is a special blob that contains information on the rest of the blobs in the stream
type SdBlob struct {
StreamName string `json:"stream_name"`
Blobs []struct {
Length int `json:"length"`
BlobNum int `json:"blob_num"`
BlobHash string `json:"blob_hash,omitempty"`
IV string `json:"iv"`
} `json:"blobs"`
StreamName string `json:"stream_name"`
StreamType string `json:"stream_type"`
Key string `json:"key"`
SuggestedFileName string `json:"suggested_file_name"`
StreamHash string `json:"stream_hash"`
Blobs []struct {
BlobHash string `json:"blob_hash,omitempty"`
IV string `json:"iv"`
Length int `json:"length"`
BlobNum int `json:"blob_num"`
} `json:"blobs"`
}
// SQL implements the DB interface
@@ -111,7 +110,7 @@ func (s *SQL) HasBlob(hash string) (bool, error) {
// HasBlobs checks if the database contains the set of blobs and returns a bool map.
func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
exists, streamsNeedingTouch, err := s.hasBlobs(hashes)
s.touch(streamsNeedingTouch)
_ = s.touch(streamsNeedingTouch)
return exists, err
}
@@ -182,7 +181,7 @@ WHERE is_stored = ? and hash IN (` + qt.Qs(len(batch)) + `)`
defer closeRows(rows)
for rows.Next() {
err := rows.Scan(&hash, &blobID, &lastAccessedAt)
err = rows.Scan(&hash, &blobID, &lastAccessedAt)
if err != nil {
return errors.Err(err)
}
-4
View File
@@ -1,9 +1,6 @@
package main
import (
"math/rand"
"time"
"github.com/lbryio/reflector.go/cmd"
"github.com/google/gops/agent"
@@ -14,6 +11,5 @@ func main() {
if err := agent.Listen(agent.Options{}); err != nil {
log.Fatal(err)
}
rand.Seed(time.Now().UnixNano())
cmd.Execute()
}
+30 -8
View File
@@ -6,9 +6,36 @@ import (
"time"
)
var Version = ""
var Time = ""
var BuildTime time.Time
var (
name = "prism-bin"
version = "unknown"
commit = "unknown"
commitLong = "unknown"
branch = "unknown"
Time = "unknown"
BuildTime time.Time
)
// Name returns main application name
func Name() string {
return name
}
// Version returns current application version
func Version() string {
return version
}
// FullName returns current app version, commit and build time
func FullName() string {
return fmt.Sprintf(
`Name: %v
Version: %v
branch: %v
commit: %v
commit long: %v
build date: %v`, Name(), Version(), branch, commit, commitLong, BuildTime.String())
}
func init() {
if Time != "" {
@@ -20,11 +47,6 @@ func init() {
}
func VersionString() string {
version := Version
if version == "" {
version = "<unset>"
}
var buildTime string
if BuildTime.IsZero() {
buildTime = "<now>"
+13 -16
View File
@@ -2,6 +2,7 @@ package prism
import (
"context"
"fmt"
"strconv"
"sync"
@@ -20,20 +21,16 @@ import (
)
type Config struct {
PeerPort int
ReflectorPort int
DhtAddress string
DhtSeedNodes []string
ClusterPort int
ClusterSeedAddr string
// limit the range of hashes to announce. useful for testing
HashRange *bits.Range
DB *db.SQL
Blobs store.BlobStore
// limit the range of hashes to announce. useful for testing
HashRange *bits.Range
DB *db.SQL
DhtAddress string
ClusterSeedAddr string
DhtSeedNodes []string
PeerPort int
ReflectorPort int
ClusterPort int
}
// DefaultConf returns a default config
@@ -78,8 +75,8 @@ func New(conf *Config) *Prism {
db: conf.DB,
dht: d,
cluster: c,
peer: peer.NewServer(conf.Blobs),
reflector: reflector.NewServer(conf.Blobs, conf.Blobs),
peer: peer.NewServer(conf.Blobs, fmt.Sprintf(":%d", conf.PeerPort)),
reflector: reflector.NewIngestionServer(conf.Blobs),
grp: stop.New(),
}
@@ -107,7 +104,7 @@ func (p *Prism) Start() error {
return errors.Err("blobs required in conf")
}
err = p.peer.Start(":" + strconv.Itoa(p.conf.PeerPort))
err = p.peer.Start()
if err != nil {
return err
}
+2 -3
View File
@@ -115,7 +115,7 @@ func Publish(client *lbrycrd.Client, path, name, address string, details Details
return signedTx, txid, nil
}
//TODO: lots of assumptions. hardcoded values need to be passed in or calculated
// TODO: lots of assumptions. hardcoded values need to be passed in or calculated
func baseTx(client *lbrycrd.Client, amount float64, changeAddress btcutil.Address) (*wire.MsgTx, error) {
txFee := 0.0002 // TODO: estimate this better?
@@ -222,8 +222,7 @@ func makeStream(path string) (stream.Stream, *pb.Stream, error) {
if err != nil {
return nil, nil, errors.Err(err)
}
defer file.Close()
defer func() { _ = file.Close() }()
enc := stream.NewEncoder(file)
s, err := enc.Stream()
+11 -11
View File
@@ -13,21 +13,21 @@ func LoadWallet(r io.Reader) (WalletFile, error) {
type WalletFile struct {
Name string `json:"name"`
Version int `json:"version"`
Preferences WalletPrefs `json:"preferences"`
Accounts []Account `json:"accounts"`
Preferences WalletPrefs `json:"preferences"`
Version int `json:"version"`
}
type Account struct {
AddressGenerator AddressGenerator `json:"address_generator"`
Certificates map[string]string `json:"certificates"`
Encrypted bool `json:"encrypted"`
Ledger string `json:"ledger"`
ModifiedOn float64 `json:"modified_on"`
Name string `json:"name"`
PrivateKey string `json:"private_key"`
PublicKey string `json:"public_key"`
Seed string `json:"seed"`
AddressGenerator AddressGenerator `json:"address_generator"`
ModifiedOn float64 `json:"modified_on"`
Encrypted bool `json:"encrypted"`
}
type AddressGenerator struct {
@@ -43,17 +43,17 @@ type AddressGenParams struct {
type WalletPrefs struct {
Shared struct {
Ts float64 `json:"ts"`
Value struct {
Type string `json:"type"`
Value struct {
AppWelcomeVersion int `json:"app_welcome_version"`
Type string `json:"type"`
Version string `json:"version"`
Value struct {
Blocked []interface{} `json:"blocked"`
Sharing3P bool `json:"sharing_3P"`
Subscriptions []string `json:"subscriptions"`
Tags []string `json:"tags"`
AppWelcomeVersion int `json:"app_welcome_version"`
Sharing3P bool `json:"sharing_3P"`
} `json:"value"`
Version string `json:"version"`
} `json:"value"`
Ts float64 `json:"ts"`
} `json:"shared"`
}
+168 -103
View File
@@ -1,124 +1,189 @@
# Reflector
Reflector is a central piece of software that providers LBRY with the following features:
- Blobs reflection: when something is published, we capture the data and store it on our servers for quicker retrieval
- Blobs distribution: when a piece of content is requested and the LBRY network doesn't have it, reflector will retrieve it from its storage and distribute it
- Blobs caching: reflectors can be chained together in multiple regions or servers to form a chain of cached content. We call those "blobcaches". They are layered so that content distribution is favorable in all the regions we deploy it to
Production-ready blob reflection, distribution, and caching for Odysee.
There are a few other features embedded in reflector.go including publishing streams from Go, downloading or upload blobs, resolving content and more unfinished tools.
This repository provides the components used in production:
- Reflector ingestion server (command name: `reflector`)
- Blob cache/edge server (`blobcache`)
- Uploader to object storage (`upload`)
This code includes a Go implementations of the LBRY peer protocol, reflector protocol, and DHT.
Other commands exist in the tree for historical/legacy reasons and are not supported.
## Installation
## How it works (at a glance)
- Ingestion (reflector): accepts uploaded blobs, persists them to object storage (e.g., S3/Wasabi) and tracks state in MySQL.
- Distribution: serves blobs over HTTP/HTTP3/Peer. Blobcaches can be deployed in front of the origin to reduce latency and egress.
- Caching (blobcache): layered disk caches backed by HTTP(S) origins (e.g., S3 endpoints), with optional local DB metadata for capacity/eviction.
- Install mysql 8 (5.7 might work too)
- add a reflector user and database with password `reflector` with localhost access only
- Create the tables as described [here](https://github.com/lbryio/reflector.go/blob/ittt/db/db.go#L735) (the link might not update as the code does so just look for the schema in that file)
All services are started by the `prism` binary and are configured via YAML files loaded from a configuration directory.
#### We do not support running reflector.go as a blob receiver, however if you want to run it as a private blobcache you may compile it yourself and run it as following:
## Supported commands
The following are the only supported commands for production use:
- Reflector ingestion: `prism reflector`
- Flags: `--receiver-port` (default 5566), `--metrics-port` (default 2112), `--disable-blocklist`
- Loads `reflector.yaml` from the config directory.
- Blob cache: `prism blobcache`
- Flags: `--metrics-port` (default 2112), `--disable-blocklist`
- Loads `blobcache.yaml` from the config directory.
- Uploader: `prism upload PATH`
- Flags: `--workers`, `--skipExistsCheck`, `--deleteBlobsAfterUpload`
- Loads `upload.yaml` from the config directory.
Global flag for all commands:
- `--conf-dir` (default `./`): directory containing YAML config files.
## Configuration
Configuration is per-command. The loader reads `<command>.yaml` from `--conf-dir`.
Common sections:
- `servers`: enables HTTP/HTTP3/Peer servers. Keys: `http`, `http3`, `peer`. Each accepts:
- `port` (int)
- `max_concurrent_requests` (int, http/http3)
- `edge_token` (string, http)
- `address` (string, optional; bind address, omit for all interfaces)
- `store`: defines the storage topology using composable stores. Frequently used:
- `proxied-s3`: production pattern with a `writer` (DB-backed -> S3/multiwriter) and a `reader` (caching -> disk + HTTP origins).
- `caching`: layered cache with a `cache` (often `db_backed` -> `disk`) and an `origin` chain (`http`, `http3`, or `ittt` fan-in).
- `s3`, `disk`, `multiwriter`, `db_backed`, `http`, `http3`, `peer`, `upstream` are also available building blocks.
### Minimal examples
Reflector – conf-dir contains `reflector.yaml`:
```yaml
servers:
http:
port: 5569
max_concurrent_requests: 200
http3:
port: 5568
max_concurrent_requests: 200
peer:
port: 5567
store:
proxied-s3:
name: s3_read_proxy
writer:
db_backed:
user: reflector
password: reflector
database: reflector
host: localhost
port: 3306
access_tracking: 1
soft_deletes: true
store:
s3:
name: primary
aws_id: YOUR_KEY
aws_secret: YOUR_SECRET
region: us-east-1
bucket: blobs-bucket
endpoint: https://s3.yourendpoint.tv
reader:
caching:
cache:
disk:
name: local_cache
mount_point: /mnt/reflector/cache
sharding_size: 2
origin:
http:
endpoint: https://s3.yourendpoint.tv/blobs-bucket/
sharding_size: 4
```
Blobcache – conf-dir contains `blobcache.yaml`:
```yaml
servers:
http:
port: 5569
max_concurrent_requests: 200
http3:
port: 5568
max_concurrent_requests: 200
peer:
port: 5567
store:
caching:
cache:
db_backed:
user: reflector
password: reflector
database: reflector
host: localhost
port: 3306
has_cap: true
max_size: 500GB
store:
disk:
name: blobcache
mount_point: /mnt/blobcache/cache
sharding_size: 2
origin:
http:
endpoint: https://s3.yourendpoint.tv/blobs-bucket/
sharding_size: 4
```
Uploader – conf-dir contains `upload.yaml` (points to the same writer/backend as reflector):
```yaml
database:
user: reflector
password: reflector
database: reflector
host: localhost
port: 3306
store:
proxied-s3:
writer:
db_backed:
user: reflector
password: reflector
database: reflector
host: localhost
port: 3306
store:
s3:
aws_id: YOUR_KEY
aws_secret: YOUR_SECRET
region: us-east-1
bucket: blobs-bucket
endpoint: https://s3.yourendpoint.tv
```
## Quick start
1) Build
- Requires Go 1.25+
- `make` (binaries in `dist/<platform>/prism-bin`)
2) Run a local blobcache
```bash
./prism-bin reflector \
--conf="none" \
--disable-uploads=true \
--use-db=false \
--upstream-reflector="reflector.lbry.com" \
--upstream-protocol="http" \
--request-queue-size=200 \
--disk-cache="2GB:/path/to/your/storage/:localdb" \
./dist/linux_amd64/prism-bin --conf-dir=./ blobcache
```
Place your `blobcache.yaml` in the `--conf-dir` directory.
Create a systemd script if you want to run it automatically on startup or as a service.
## Usage
Usage as reflector/blobcache:
3) Run reflector ingestion
```bash
Run reflector server
Usage:
prism reflector [flags]
Flags:
--disable-blocklist Disable blocklist watching/updating
--disable-uploads Disable uploads to this reflector server
--disk-cache string Where to cache blobs on the file system. format is 'sizeGB:CACHE_PATH:cachemanager' (cachemanagers: localdb/lfuda/lru) (default "100GB:/tmp/downloaded_blobs:localdb")
-h, --help help for reflector
--http-peer-port int The port reflector will distribute content from over HTTP protocol (default 5569)
--http3-peer-port int The port reflector will distribute content from over HTTP3 protocol (default 5568)
--mem-cache int enable in-memory cache with a max size of this many blobs
--metrics-port int The port reflector will use for prometheus metrics (default 2112)
--optional-disk-cache string Optional secondary file system cache for blobs. format is 'sizeGB:CACHE_PATH:cachemanager' (cachemanagers: localdb/lfuda/lru) (this would get hit before the one specified in disk-cache)
--origin-endpoint string HTTP edge endpoint for standard HTTP retrieval
--origin-endpoint-fallback string HTTP edge endpoint for standard HTTP retrieval if first origin fails
--receiver-port int The port reflector will receive content from (default 5566)
--request-queue-size int How many concurrent requests from downstream should be handled at once (the rest will wait) (default 200)
--tcp-peer-port int The port reflector will distribute content from for the TCP (LBRY) protocol (default 5567)
--upstream-protocol string protocol used to fetch blobs from another upstream reflector server (tcp/http3/http) (default "http")
--upstream-reflector string host:port of a reflector server where blobs are fetched from
--use-db Whether to connect to the reflector db or not (default true)
Global Flags:
--conf string Path to config. Use 'none' to disable (default "config.json")
-v, --verbose strings Verbose logging for specific components
./dist/linux_amd64/prism-bin --conf-dir=./ reflector --receiver-port=5566 --metrics-port=2112
```
Other uses:
4) Upload blobs
```bash
Prism is a single entry point application with multiple sub modules which can be leveraged individually or together
Usage:
prism [command]
Available Commands:
check-integrity check blobs integrity for a given path
cluster Start(join) to or Start a new cluster
decode Decode a claim value
dht Run dht node
getstream Get a stream from a reflector server
help Help about any command
peer Run peer server
populate-db populate local database with blobs from a disk storage
publish Publish a file
reflector Run reflector server
resolve Resolve a URL
send Send a file to a reflector
sendblob Send a random blob to a reflector server
start Runs full prism application with cluster, dht, peer server, and reflector server.
test Test things
upload Upload blobs to S3
version Print the version
Flags:
--conf string Path to config. Use 'none' to disable (default "config.json")
-h, --help help for prism
-v, --verbose strings Verbose logging for specific components
```
## Running from Source
This project requires [Go v1.16](https://golang.org/doc/install).
On Ubuntu you can install it with `sudo snap install go --classic`
```
git clone git@github.com:lbryio/reflector.go.git
cd reflector.go
make
./bin/prism-bin
./dist/linux_amd64/prism-bin --conf-dir=./ upload /path/to/blobs \
--workers=4 --skipExistsCheck
```
## Contributing
coming soon
## License
This project is MIT licensed.
## Notes
- Only reflector, blobcache, and upload are supported. All other commands are legacy and may be removed in the future.
- Metrics are exposed on the configured `--metrics-port` at `/metrics` (Prometheus format).
- MySQL is required when using DB-backed stores (e.g., ingestion writer, capacity-aware caches).
## Security
If you discover a security issue, please email security@lbry.com. Our PGP key is available at https://lbry.com/faq/pgp-key.
We take security seriously. Please contact security@lbry.com regarding any security issues.
Our PGP key is [here](https://keybase.io/lbry/key.asc) if you need it.
## License
MIT License. See LICENSE.
## Contact
The primary contact for this project is [@Nikooo777](https://github.com/Nikooo777) (niko-at-lbry.com)
The primary contact for this project is [@Nikooo777](https://github.com/Nikooo777)
+11 -11
View File
@@ -22,10 +22,10 @@ const blocklistURL = "https://api.lbry.com/file/list_blocked"
func (s *Server) enableBlocklist(b store.Blocklister) {
walletServers := []string{
"spv25.lbry.com:50001",
"spv26.lbry.com:50001",
"spv19.lbry.com:50001",
"spv14.lbry.com:50001",
"a-hub1.odysee.com:50001",
"b-hub1.odysee.com:50001",
"c-hub1.odysee.com:50001",
"s-hub1.odysee.com:50001",
}
updateBlocklist(b, walletServers, s.grp.Ch())
@@ -69,18 +69,18 @@ func blockedSdHashes(walletServers []string, stopper stop.Chan) (map[string]valO
return nil, errors.Err(err)
}
defer func() {
err := resp.Body.Close()
if err != nil {
log.Errorln(errors.Err(err))
closeErr := resp.Body.Close()
if closeErr != nil {
log.Errorln(errors.Err(closeErr))
}
}()
var r struct {
Success bool `json:"success"`
Error string `json:"error"`
Data struct {
Error string `json:"error"`
Data struct {
Outpoints []string `json:"outpoints"`
} `json:"data"`
Success bool `json:"success"`
}
if err = json.NewDecoder(resp.Body).Decode(&r); err != nil {
@@ -95,8 +95,8 @@ func blockedSdHashes(walletServers []string, stopper stop.Chan) (map[string]valO
}
type valOrErr struct {
Value string
Err error
Value string
}
// sdHashesForOutpoints queries wallet server for the sd hashes in a given outpoints
+85
View File
@@ -0,0 +1,85 @@
package reflector
import (
"encoding/json"
"net/http"
"time"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/bluele/gcache"
"golang.org/x/sync/singleflight"
)
const protectedListURL = "https://direct.api.odysee.com/file/list_protected"
type ProtectedContent struct {
SDHash string `json:"sd_hash"`
ClaimID string `json:"claim_id"`
}
var protectedCache = gcache.New(10).Expiration(2 * time.Minute).Build()
func GetProtectedContent() (interface{}, error) {
cachedVal, err := protectedCache.Get("protected")
if err == nil && cachedVal != nil {
return cachedVal.(map[string]bool), nil
}
method := "GET"
var r struct {
Error string `json:"error"`
Data []ProtectedContent `json:"data"`
Success bool `json:"success"`
}
// Bound the request to avoid hanging the entire request path.
// Without a timeout, a slow or unreachable endpoint can block
// singleflight callers indefinitely and stall HTTP handlers.
client := &http.Client{Timeout: 5 * time.Second}
req, err := http.NewRequest(method, protectedListURL, nil)
if err != nil {
return nil, errors.Err(err)
}
res, err := client.Do(req)
if err != nil {
return nil, errors.Err(err)
}
defer func() { _ = res.Body.Close() }()
if res.StatusCode != http.StatusOK {
return nil, errors.Err("unexpected status code %d", res.StatusCode)
}
if err = json.NewDecoder(res.Body).Decode(&r); err != nil {
return nil, errors.Err(err)
}
if !r.Success {
return nil, errors.Prefix("file/list_protected API call", r.Error)
}
protectedMap := make(map[string]bool, len(r.Data))
for _, pc := range r.Data {
protectedMap[pc.SDHash] = true
}
err = protectedCache.Set("protected", protectedMap)
if err != nil {
return protectedMap, errors.Err(err)
}
return protectedMap, nil
}
var sf = singleflight.Group{}
func IsProtected(sdHash string) bool {
val, err, _ := sf.Do("protected", GetProtectedContent)
if err != nil {
return false
}
cachedMap, ok := val.(map[string]bool)
if !ok {
return false
}
return cachedMap[sdHash]
}
+22 -28
View File
@@ -6,7 +6,6 @@ import (
"encoding/hex"
"encoding/json"
"io"
"io/ioutil"
"net"
"time"
@@ -36,22 +35,21 @@ var ErrBlobTooBig = errors.Base("blob must be at most %d bytes", maxBlobSize)
// Server is and instance of the reflector server. It houses the blob store and listener.
type Server struct {
store store.BlobStore
grp *stop.Group
Timeout time.Duration // timeout to read or write next message
EnableBlocklist bool // if true, blocklist checking and blob deletion will be enabled
underlyingStore store.BlobStore
outerStore store.BlobStore
grp *stop.Group
//underlyingStore store.BlobStore
//outerStore store.BlobStore
}
// NewServer returns an initialized reflector server pointer.
func NewServer(underlying store.BlobStore, outer store.BlobStore) *Server {
func NewIngestionServer(store store.BlobStore) *Server {
return &Server{
Timeout: DefaultTimeout,
underlyingStore: underlying,
outerStore: outer,
grp: stop.New(),
Timeout: DefaultTimeout,
store: store,
grp: stop.New(),
}
}
@@ -62,7 +60,7 @@ func (s *Server) Shutdown() {
log.Println("reflector server stopped")
}
//Start starts the server to handle connections.
// Start starts the server to handle connections.
func (s *Server) Start(address string) error {
l, err := net.Listen(network, address)
if err != nil {
@@ -90,7 +88,7 @@ func (s *Server) Start(address string) error {
}()
if s.EnableBlocklist {
if b, ok := s.underlyingStore.(store.Blocklister); ok {
if b, ok := s.store.(store.Blocklister); ok {
s.grp.Add(1)
metrics.RoutinesQueue.WithLabelValues("reflector", "enableblocklist").Inc()
go func() {
@@ -153,7 +151,7 @@ func (s *Server) handleConn(conn net.Conn) {
if errors.Is(err, io.EOF) || s.quitting() {
return
}
err := s.doError(conn, err)
err = s.doError(conn, err)
if err != nil {
log.Error(errors.Prefix("sending handshake error", err))
}
@@ -166,7 +164,7 @@ func (s *Server) handleConn(conn net.Conn) {
if errors.Is(err, io.EOF) || s.quitting() {
return
}
err := s.doError(conn, err)
err = s.doError(conn, err)
if err != nil {
log.Error(errors.Prefix("sending blob receive error", err))
}
@@ -201,13 +199,14 @@ func (s *Server) receiveBlob(conn net.Conn) error {
}
var wantsBlob bool
if bl, ok := s.underlyingStore.(store.Blocklister); ok {
if bl, ok := s.store.(store.Blocklister); ok {
wantsBlob, err = bl.Wants(blobHash)
if err != nil {
return err
}
} else {
blobExists, err := s.underlyingStore.Has(blobHash)
var blobExists bool
blobExists, err = s.store.Has(blobHash)
if err != nil {
return err
}
@@ -217,14 +216,14 @@ func (s *Server) receiveBlob(conn net.Conn) error {
var neededBlobs []string
if isSdBlob && !wantsBlob {
if nbc, ok := s.underlyingStore.(neededBlobChecker); ok {
if nbc, ok := s.store.(store.NeededBlobChecker); ok {
neededBlobs, err = nbc.MissingBlobsForKnownStream(blobHash)
if err != nil {
return err
}
} else {
// if we can't check for blobs in a stream, we have to say that the sd blob is
// missing. if we say we have the sd blob, they wont try to send any content blobs
// missing. if we say we have the sd blob, they won't try to send any content blobs
wantsBlob = true
}
}
@@ -260,9 +259,9 @@ func (s *Server) receiveBlob(conn net.Conn) error {
log.Debugln("Got blob " + blobHash[:8])
if isSdBlob {
err = s.outerStore.PutSD(blobHash, blob)
err = s.store.PutSD(blobHash, blob)
} else {
err = s.outerStore.Put(blobHash, blob)
err = s.store.Put(blobHash, blob)
}
if err != nil {
return err
@@ -367,7 +366,7 @@ func (s *Server) read(conn net.Conn, v interface{}) error {
dec := json.NewDecoder(conn)
err = dec.Decode(v)
if err != nil {
data, _ := ioutil.ReadAll(dec.Buffered())
data, _ := io.ReadAll(dec.Buffered())
if len(data) > 0 {
return errors.Err("%s. Data: %s", err.Error(), hex.EncodeToString(data))
}
@@ -430,8 +429,8 @@ type handshakeRequestResponse struct {
type sendBlobRequest struct {
BlobHash string `json:"blob_hash,omitempty"`
BlobSize int `json:"blob_size,omitempty"`
SdBlobHash string `json:"sd_blob_hash,omitempty"`
BlobSize int `json:"blob_size,omitempty"`
SdBlobSize int `json:"sd_blob_size,omitempty"`
}
@@ -440,8 +439,8 @@ type sendBlobResponse struct {
}
type sendSdBlobResponse struct {
SendSdBlob bool `json:"send_sd_blob"`
NeededBlobs []string `json:"needed_blobs,omitempty"`
SendSdBlob bool `json:"send_sd_blob"`
}
type blobTransferResponse struct {
@@ -451,8 +450,3 @@ type blobTransferResponse struct {
type sdBlobTransferResponse struct {
ReceivedSdBlob bool `json:"received_sd_blob"`
}
// neededBlobChecker can check which blobs from a known stream are not uploaded yet
type neededBlobChecker interface {
MissingBlobsForKnownStream(string) ([]string, error)
}
+5 -5
View File
@@ -23,7 +23,7 @@ func startServerOnRandomPort(t *testing.T) (*Server, int) {
t.Fatal(err)
}
srv := NewServer(store.NewMemStore(), store.NewMemStore())
srv := NewIngestionServer(store.NewMemStore(store.MemParams{Name: "test"}))
err = srv.Start("127.0.0.1:" + strconv.Itoa(port))
if err != nil {
t.Fatal(err)
@@ -120,7 +120,7 @@ func TestServer_Timeout(t *testing.T) {
t.Fatal(err)
}
srv := NewServer(store.NewMemStore(), store.NewMemStore())
srv := NewIngestionServer(store.NewMemStore(store.MemParams{Name: "test"}))
srv.Timeout = testTimeout
err = srv.Start("127.0.0.1:" + strconv.Itoa(port))
if err != nil {
@@ -182,8 +182,8 @@ func TestServer_PartialUpload(t *testing.T) {
missing[i] = bits.Rand().String()
}
st := store.BlobStore(&mockPartialStore{MemStore: store.NewMemStore(), missing: missing})
if _, ok := st.(neededBlobChecker); !ok {
st := store.BlobStore(&mockPartialStore{MemStore: store.NewMemStore(store.MemParams{Name: "test"}), missing: missing})
if _, ok := st.(store.NeededBlobChecker); !ok {
t.Fatal("mock does not implement the relevant interface")
}
err = st.Put(sdHash, randBlob(10))
@@ -191,7 +191,7 @@ func TestServer_PartialUpload(t *testing.T) {
t.Fatal(err)
}
srv := NewServer(st, st)
srv := NewIngestionServer(st)
err = srv.Start("127.0.0.1:" + strconv.Itoa(port))
if err != nil {
t.Fatal(err)
+8 -10
View File
@@ -1,7 +1,6 @@
package reflector
import (
"io/ioutil"
"os"
"path"
"sync"
@@ -31,17 +30,16 @@ type Summary struct {
type Uploader struct {
db *db.SQL
store *store.DBBackedStore // could just be store.BlobStore interface
store store.BlobStore
stopper *stop.Group
countChan chan increment
count Summary
workers int
skipExistsCheck bool
deleteBlobsAfterUpload bool
stopper *stop.Group
countChan chan increment
count Summary
}
func NewUploader(db *db.SQL, store *store.DBBackedStore, workers int, skipExistsCheck, deleteBlobsAfterUpload bool) *Uploader {
func NewUploader(db *db.SQL, store store.BlobStore, workers int, skipExistsCheck, deleteBlobsAfterUpload bool) *Uploader {
return &Uploader{
db: db,
store: store,
@@ -133,7 +131,7 @@ Upload:
return nil
}
// worker reads paths from a channel, uploads them, and optionally deletes them
// worker reads paths from a channel, uploads them, and optionally deletes them
func (u *Uploader) worker(pathChan chan string) {
for {
select {
@@ -165,7 +163,7 @@ func (u *Uploader) uploadBlob(filepath string) (err error) {
}
}()
blob, err := ioutil.ReadFile(filepath)
blob, err := os.ReadFile(filepath)
if err != nil {
return errors.Err(err)
}
@@ -177,7 +175,7 @@ func (u *Uploader) uploadBlob(filepath string) (err error) {
if IsValidJSON(blob) {
log.Debugf("uploading SD blob %s", hash)
err := u.store.PutSD(hash, blob)
err = u.store.PutSD(hash, blob)
if err != nil {
return errors.Prefix("uploading SD blob "+hash, err)
}
+16
View File
@@ -0,0 +1,16 @@
#!/bin/bash
set -euo pipefail
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
cd "$SCRIPT_DIR/.."
if ! command -v golangci-lint &> /dev/null; then
echo "Installing golangci-lint..."
go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
else
echo "Ensuring golangci-lint is up to date..."
go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
fi
echo "Running golangci-lint..."
golangci-lint run --config .golangci.yml ./...
+15
View File
@@ -2,16 +2,19 @@ package http
import (
"net/http"
"strconv"
"sync"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/gin-gonic/gin"
log "github.com/sirupsen/logrus"
)
func (s *Server) getBlob(c *gin.Context) {
@@ -22,9 +25,20 @@ func (s *Server) getBlob(c *gin.Context) {
}
func (s *Server) HandleGetBlob(c *gin.Context) {
defer func() {
if r := recover(); r != nil {
log.Errorf("Recovered from panic: %v", r)
}
}()
start := time.Now()
hash := c.Query("hash")
edgeToken := c.Query("edge_token")
if reflector.IsProtected(hash) && edgeToken != s.edgeToken {
_ = c.Error(errors.Err("requested blob is protected"))
c.String(http.StatusForbidden, "requested blob is protected")
return
}
if s.missesCache.Has(hash) {
serialized, err := shared.NewBlobTrace(time.Since(start), "http").Serialize()
c.Header("Via", serialized)
@@ -66,6 +80,7 @@ func (s *Server) HandleGetBlob(c *gin.Context) {
metrics.HttpDownloadCount.Inc()
c.Header("Via", serialized)
c.Header("Content-Disposition", "filename="+hash)
c.Header("Content-Length", strconv.Itoa(len(blob)))
c.Data(http.StatusOK, "application/octet-stream", blob)
}
+13 -8
View File
@@ -19,17 +19,21 @@ import (
type Server struct {
store store.BlobStore
grp *stop.Group
concurrentRequests int
missesCache gcache.Cache
edgeToken string
address string
concurrentRequests int
}
// NewServer returns an initialized Server pointer.
func NewServer(store store.BlobStore, requestQueueSize int) *Server {
func NewServer(store store.BlobStore, requestQueueSize int, edgeToken string, address string) *Server {
return &Server{
store: store,
grp: stop.New(),
concurrentRequests: requestQueueSize,
missesCache: gcache.New(2000).Expiration(5 * time.Minute).ARC().Build(),
edgeToken: edgeToken,
address: address,
}
}
@@ -41,15 +45,16 @@ func (s *Server) Shutdown() {
}
// Start starts the server listener to handle connections.
func (s *Server) Start(address string) error {
func (s *Server) Start() error {
gin.SetMode(gin.ReleaseMode)
router := gin.Default()
router.GET("/blob", s.getBlob)
router.HEAD("/blob", s.hasBlob)
router := gin.New()
router.Use(gin.Logger())
// Install nice.Recovery, passing the handler to call after recovery
router.Use(nice.Recovery(s.recoveryHandler))
router.GET("/blob", s.getBlob)
router.HEAD("/blob", s.hasBlob)
srv := &http.Server{
Addr: address,
Addr: s.address,
Handler: router,
}
go s.listenForShutdown(srv)
@@ -59,7 +64,7 @@ func (s *Server) Start(address string) error {
s.grp.Add(1)
go func() {
defer s.grp.Done()
log.Println("HTTP server listening on " + address)
log.Println("HTTP server listening on " + s.address)
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
log.Fatalf("listen: %s\n", err)
}
+7 -6
View File
@@ -16,16 +16,16 @@ import (
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/lucas-clemente/quic-go/http3"
"github.com/quic-go/quic-go/http3"
log "github.com/sirupsen/logrus"
)
// Client is an instance of a client connected to a server.
type Client struct {
Timeout time.Duration
conn *http.Client
roundTripper *http3.RoundTripper
roundTripper *http3.Transport
ServerAddr string
Timeout time.Duration
}
// Close closes the connection with the client.
@@ -69,7 +69,7 @@ func (c *Client) HasBlob(hash string) (bool, error) {
if err != nil {
return false, errors.Err(err)
}
defer resp.Body.Close()
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode == http.StatusOK {
return true, nil
}
@@ -86,7 +86,7 @@ func (c *Client) GetBlob(hash string) (stream.Blob, shared.BlobTrace, error) {
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err(err)
}
defer resp.Body.Close()
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode == http.StatusNotFound {
fmt.Printf("%s blob not found %d\n", hash, resp.StatusCode)
@@ -100,7 +100,8 @@ func (c *Client) GetBlob(hash string) (stream.Blob, shared.BlobTrace, error) {
serialized := resp.Header.Get("Via")
trace := shared.NewBlobTrace(time.Since(start), "http3")
if serialized != "" {
parsedTrace, err := shared.Deserialize(serialized)
var parsedTrace *shared.BlobTrace
parsedTrace, err = shared.Deserialize(serialized)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "http3"), err
}
+20 -14
View File
@@ -11,18 +11,20 @@ import (
"math/big"
"net/http"
"strconv"
"strings"
"sync"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/gorilla/mux"
"github.com/lucas-clemente/quic-go"
"github.com/lucas-clemente/quic-go/http3"
"github.com/quic-go/quic-go"
"github.com/quic-go/quic-go/http3"
log "github.com/sirupsen/logrus"
)
@@ -30,15 +32,17 @@ import (
type Server struct {
store store.BlobStore
grp *stop.Group
address string
concurrentRequests int
}
// NewServer returns an initialized Server pointer.
func NewServer(store store.BlobStore, requestQueueSize int) *Server {
func NewServer(store store.BlobStore, requestQueueSize int, address string) *Server {
return &Server{
store: store,
grp: stop.New(),
concurrentRequests: requestQueueSize,
address: address,
}
}
@@ -65,8 +69,8 @@ type availabilityResponse struct {
}
// Start starts the server listener to handle connections.
func (s *Server) Start(address string) error {
log.Println("HTTP3 peer listening on " + address)
func (s *Server) Start() error {
log.Println("HTTP3 peer listening on " + s.address)
window500M := 500 * 1 << 20
quicConf := &quic.Config{
@@ -112,12 +116,10 @@ func (s *Server) Start(address string) error {
}
})
server := http3.Server{
Server: &http.Server{
Handler: r,
Addr: address,
TLSConfig: generateTLSConfig(),
},
QuicConfig: quicConf,
Addr: s.address,
Handler: r,
TLSConfig: generateTLSConfig(),
QUICConfig: quicConf,
}
go InitWorkers(s, s.concurrentRequests)
go s.listenForShutdown(&server)
@@ -156,7 +158,7 @@ func generateTLSConfig() *tls.Config {
func (s *Server) listenAndServe(server *http3.Server) {
err := server.ListenAndServe()
if err != nil && err.Error() != "server closed" {
if err != nil && !strings.Contains(err.Error(), "Server closed") {
log.Errorln(errors.FullTrace(err))
}
}
@@ -181,11 +183,15 @@ func (s *Server) HandleGetBlob(w http.ResponseWriter, r *http.Request) {
wantsTrace = false
}
}
if reflector.IsProtected(requestedBlob) {
http.Error(w, "requested blob is protected", http.StatusForbidden)
return
}
blob, trace, err := s.store.Get(requestedBlob)
if wantsTrace {
serialized, err := trace.Serialize()
var serialized string
serialized, err = trace.Serialize()
if err != nil {
http.Error(w, err.Error(), http.StatusNotFound)
return
-117
View File
@@ -1,117 +0,0 @@
package http3
import (
"crypto/tls"
"crypto/x509"
"net/http"
"strings"
"sync"
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/lucas-clemente/quic-go"
"github.com/lucas-clemente/quic-go/http3"
)
// Store is a blob store that gets blobs from a peer.
// It satisfies the store.BlobStore interface but cannot put or delete blobs.
type Store struct {
opts StoreOpts
NotFoundCache *sync.Map
}
// StoreOpts allows to set options for a new Store.
type StoreOpts struct {
Address string
Timeout time.Duration
}
// NewStore makes a new peer store.
func NewStore(opts StoreOpts) *Store {
return &Store{opts: opts, NotFoundCache: &sync.Map{}}
}
func (p *Store) getClient() (*Client, error) {
var qconf quic.Config
window500M := 500 * 1 << 20
qconf.MaxStreamReceiveWindow = uint64(window500M)
qconf.MaxConnectionReceiveWindow = uint64(window500M)
qconf.EnableDatagrams = true
qconf.HandshakeIdleTimeout = 4 * time.Second
qconf.MaxIdleTimeout = 20 * time.Second
pool, err := x509.SystemCertPool()
if err != nil {
return nil, err
}
roundTripper := &http3.RoundTripper{
TLSClientConfig: &tls.Config{
RootCAs: pool,
InsecureSkipVerify: true,
},
QuicConfig: &qconf,
}
connection := &http.Client{
Transport: roundTripper,
}
c := &Client{
conn: connection,
roundTripper: roundTripper,
ServerAddr: p.opts.Address,
}
return c, errors.Prefix("connection error", err)
}
func (p *Store) Name() string { return "http3" }
// Has asks the peer if they have a hash
func (p *Store) Has(hash string) (bool, error) {
c, err := p.getClient()
if err != nil {
return false, err
}
defer c.Close()
return c.HasBlob(hash)
}
// Get downloads the blob from the peer
func (p *Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
if lastChecked, ok := p.NotFoundCache.Load(hash); ok {
if lastChecked.(time.Time).After(time.Now().Add(-5 * time.Minute)) {
return nil, shared.NewBlobTrace(time.Since(start), p.Name()+"-notfoundcache"), store.ErrBlobNotFound
}
}
c, err := p.getClient()
if err != nil && strings.Contains(err.Error(), "blob not found") {
p.NotFoundCache.Store(hash, time.Now())
}
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), p.Name()), err
}
defer c.Close()
return c.GetBlob(hash)
}
// Put is not supported
func (p *Store) Put(hash string, blob stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// PutSD is not supported
func (p *Store) PutSD(hash string, blob stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// Delete is not supported
func (p *Store) Delete(hash string) error {
return errors.Err(shared.ErrNotImplemented)
}
// Shutdown is not supported
func (p *Store) Shutdown() {
}
+1 -1
View File
@@ -20,9 +20,9 @@ import (
// Client is an instance of a client connected to a server.
type Client struct {
Timeout time.Duration
conn net.Conn
buf *bufio.Reader
Timeout time.Duration
connected bool
}
+64 -46
View File
@@ -31,19 +31,18 @@ const (
// Server is an instance of a peer server that houses the listener and store.
type Server struct {
store store.BlobStore
disk *store.DiskStore
closed bool
grp *stop.Group
store store.BlobStore
grp *stop.Group
address string
closed bool
}
// NewServer returns an initialized Server pointer.
func NewServer(blobstore store.BlobStore) *Server {
func NewServer(store store.BlobStore, address string) *Server {
return &Server{
store: blobstore,
disk: store.NewDiskStore("/external/blobs", 2),
grp: stop.New(),
store: store,
grp: stop.New(),
address: address,
}
}
@@ -55,9 +54,9 @@ func (s *Server) Shutdown() {
}
// Start starts the server listener to handle connections.
func (s *Server) Start(address string) error {
log.Println("peer listening on " + address)
l, err := net.Listen("tcp4", address)
func (s *Server) Start() error {
log.Println("peer listening on " + s.address)
l, err := net.Listen("tcp4", s.address)
if err != nil {
return err
}
@@ -133,7 +132,7 @@ func (s *Server) handleConnection(conn net.Conn) {
log.Error(errors.FullTrace(err))
}
response, blobHash, err := s.handleCompositeRequest(request)
response, err = s.handleCompositeRequest(request)
if err != nil {
log.Error(errors.FullTrace(err))
return
@@ -145,7 +144,6 @@ func (s *Server) handleConnection(conn net.Conn) {
}
n, err := conn.Write(response)
s.disk.Sendfile(blobHash, conn)
if err != nil {
if !strings.Contains(err.Error(), "connection reset by peer") { // means the other side closed the connection using TCP reset
s.logError(err)
@@ -224,70 +222,79 @@ func (s *Server) handleAvailabilityRequest(data []byte) ([]byte, error) {
// return append(response, blob...), nil
//}
func (s *Server) handleCompositeRequest(data []byte) ([]byte, string, error) {
func (s *Server) handleCompositeRequest(data []byte) ([]byte, error) {
var request compositeRequest
err := json.Unmarshal(data, &request)
if err != nil {
var je *json.SyntaxError
if ee.As(err, &je) {
return nil, "", errors.Err("invalid json at offset %d in data %s", je.Offset, hex.EncodeToString(data))
return nil, errors.Err("invalid json request: offset %d in data %s", je.Offset, hex.EncodeToString(data))
}
return nil, "", errors.Err(err)
return nil, errors.Err(err)
}
response := compositeResponse{
LbrycrdAddress: LbrycrdAddress,
AvailableBlobs: []string{},
}
if len(request.RequestedBlobs) > 0 {
var availableBlobs []string
for _, blobHash := range request.RequestedBlobs {
exists, err := s.store.Has(blobHash)
if reflector.IsProtected(blobHash) {
return nil, errors.Err("requested blob is protected")
}
var exists bool
exists, err = s.store.Has(blobHash)
if err != nil {
return nil, "", err
return nil, err
}
if exists {
availableBlobs = append(availableBlobs, blobHash)
response.AvailableBlobs = append(response.AvailableBlobs, blobHash)
}
}
response.AvailableBlobs = availableBlobs
}
response.BlobDataPaymentRate = paymentRateAccepted
if request.BlobDataPaymentRate < 0 {
response.BlobDataPaymentRate = paymentRateTooLow
if request.BlobDataPaymentRate != nil {
response.BlobDataPaymentRate = paymentRateAccepted
if *request.BlobDataPaymentRate < 0 {
response.BlobDataPaymentRate = paymentRateTooLow
}
}
var blob []byte
var trace shared.BlobTrace
if request.RequestedBlob != "" {
if len(request.RequestedBlob) != stream.BlobHashHexLength {
return nil, "", errors.Err("Invalid blob hash length")
return nil, errors.Err("Invalid blob hash length")
}
log.Debugln("Sending blob " + request.RequestedBlob[:8])
size, err := s.disk.Size(request.RequestedBlob)
if err != nil {
response.IncomingBlob = incomingBlob{
blob, trace, err = s.store.Get(request.RequestedBlob)
log.Debug(trace.String())
if errors.Is(err, store.ErrBlobNotFound) {
response.IncomingBlob = &incomingBlob{
Error: err.Error(),
}
} else if err != nil {
return nil, err
} else {
response.IncomingBlob = incomingBlob{
response.IncomingBlob = &incomingBlob{
BlobHash: request.RequestedBlob,
Length: int(size),
Length: len(blob),
}
metrics.MtrOutBytesTcp.Add(float64(len(blob)))
metrics.BlobDownloadCount.Inc()
metrics.PeerDownloadCount.Inc()
}
metrics.MtrOutBytesTcp.Add(float64(len(blob)))
metrics.BlobDownloadCount.Inc()
metrics.PeerDownloadCount.Inc()
}
respData, err := json.Marshal(response)
if err != nil {
return nil, "", err
return nil, err
}
return respData, request.RequestedBlob, nil
return append(respData, blob...), nil
}
func (s *Server) logError(e error) {
@@ -301,7 +308,15 @@ func (s *Server) logError(e error) {
}
func readNextMessage(buf *bufio.Reader) ([]byte, error) {
msg := make([]byte, 0)
firstByte, err := buf.ReadByte()
if err != nil {
return nil, err
}
if firstByte != '{' {
// every request starts with '{'. Checking here disconnects earlier, so we don't wait until timeout
return nil, errInvalidData
}
msg := []byte("{")
eof := false
for {
@@ -322,6 +337,8 @@ func readNextMessage(buf *bufio.Reader) ([]byte, error) {
if len(msg) > maxRequestSize {
return msg, errRequestTooLarge
} else if len(msg) > 0 && msg[0] != '{' {
return msg, errInvalidData
}
// yes, this is how the peer protocol knows when the request finishes
@@ -356,10 +373,11 @@ const (
)
var errRequestTooLarge = errors.Base("request is too large")
var errInvalidData = errors.Base("Invalid data")
type availabilityRequest struct {
LbrycrdAddress bool `json:"lbrycrd_address"`
RequestedBlobs []string `json:"requested_blobs"`
LbrycrdAddress bool `json:"lbrycrd_address"`
}
type availabilityResponse struct {
@@ -385,20 +403,20 @@ type incomingBlob struct {
Length int `json:"length"`
}
type blobResponse struct {
IncomingBlob incomingBlob `json:"incoming_blob"`
RequestTrace *shared.BlobTrace
IncomingBlob incomingBlob `json:"incoming_blob"`
}
type compositeRequest struct {
LbrycrdAddress bool `json:"lbrycrd_address"`
RequestedBlobs []string `json:"requested_blobs"`
BlobDataPaymentRate float64 `json:"blob_data_payment_rate"`
BlobDataPaymentRate *float64 `json:"blob_data_payment_rate"`
RequestedBlob string `json:"requested_blob"`
RequestedBlobs []string `json:"requested_blobs"`
LbrycrdAddress bool `json:"lbrycrd_address"`
}
type compositeResponse struct {
LbrycrdAddress string `json:"lbrycrd_address,omitempty"`
AvailableBlobs []string `json:"available_blobs,omitempty"`
BlobDataPaymentRate string `json:"blob_data_payment_rate,omitempty"`
IncomingBlob incomingBlob `json:"incoming_blob,omitempty"`
IncomingBlob *incomingBlob `json:"incoming_blob,omitempty"`
LbrycrdAddress string `json:"lbrycrd_address,omitempty"`
BlobDataPaymentRate string `json:"blob_data_payment_rate,omitempty"`
AvailableBlobs []string `json:"available_blobs"`
}
+66 -2
View File
@@ -2,7 +2,10 @@ package peer
import (
"bytes"
"io"
"net"
"testing"
"time"
"github.com/lbryio/reflector.go/store"
)
@@ -34,7 +37,9 @@ var availabilityRequests = []pair{
}
func getServer(t *testing.T, withBlobs bool) *Server {
st := store.NewMemStore()
st := store.NewMemStore(store.MemParams{
Name: "test",
})
if withBlobs {
for k, v := range blobs {
err := st.Put(k, v)
@@ -43,7 +48,7 @@ func getServer(t *testing.T, withBlobs bool) *Server {
}
}
}
return NewServer(st)
return NewServer(st, "127.0.0.1:50505")
}
func TestAvailabilityRequest_NoBlobs(t *testing.T) {
@@ -75,3 +80,62 @@ func TestAvailabilityRequest_WithBlobs(t *testing.T) {
}
}
}
func TestRequestFromConnection(t *testing.T) {
s := getServer(t, true)
err := s.Start()
defer s.Shutdown()
if err != nil {
t.Error("error starting server", err)
}
for _, p := range availabilityRequests {
conn, err := net.Dial("tcp", "127.0.0.1:50505")
if err != nil {
t.Error("error opening connection", err)
}
defer func() { _ = conn.Close() }()
response := make([]byte, 8192)
_, err = conn.Write(p.request)
if err != nil {
t.Error("error writing", err)
}
_, err = conn.Read(response)
if err != nil {
t.Error("error reading", err)
}
if !bytes.Equal(response[:len(p.response)], p.response) {
t.Errorf("Response did not match expected response.\nExpected: %s\nGot: %s", string(p.response), string(response))
}
}
}
func TestInvalidData(t *testing.T) {
s := getServer(t, true)
err := s.Start()
defer s.Shutdown()
if err != nil {
t.Error("error starting server", err)
}
conn, err := net.Dial("tcp", "127.0.0.1:50505")
if err != nil {
t.Error("error opening connection", err)
}
defer func() { _ = conn.Close() }()
response := make([]byte, 8192)
_, err = conn.Write([]byte("hello dear server, I would like blobs. Please"))
if err != nil {
t.Error("error writing", err)
}
err = conn.SetReadDeadline(time.Now().Add(5 * time.Second))
if err != nil {
t.Error("error setting read deadline", err)
}
_, err = conn.Read(response)
if err != io.EOF {
t.Error("error reading", err)
}
println(response)
}
-82
View File
@@ -1,82 +0,0 @@
package peer
import (
"strings"
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
)
// Store is a blob store that gets blobs from a peer.
// It satisfies the store.BlobStore interface but cannot put or delete blobs.
type Store struct {
opts StoreOpts
}
// StoreOpts allows to set options for a new Store.
type StoreOpts struct {
Address string
Timeout time.Duration
}
// NewStore makes a new peer store.
func NewStore(opts StoreOpts) *Store {
return &Store{opts: opts}
}
func (p *Store) getClient() (*Client, error) {
c := &Client{Timeout: p.opts.Timeout}
err := c.Connect(p.opts.Address)
return c, errors.Prefix("connection error", err)
}
func (p *Store) Name() string { return "peer" }
// Has asks the peer if they have a hash
func (p *Store) Has(hash string) (bool, error) {
c, err := p.getClient()
if err != nil {
return false, err
}
defer c.Close()
return c.HasBlob(hash)
}
// Get downloads the blob from the peer
func (p *Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
c, err := p.getClient()
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), p.Name()), err
}
defer c.Close()
blob, trace, err := c.GetBlob(hash)
if err != nil && strings.Contains(err.Error(), "blob not found") {
return nil, trace, store.ErrBlobNotFound
}
return blob, trace, err
}
// Put is not supported
func (p *Store) Put(hash string, blob stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// PutSD is not supported
func (p *Store) PutSD(hash string, blob stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// Delete is not supported
func (p *Store) Delete(hash string) error {
return errors.Err(shared.ErrNotImplemented)
}
// Shutdown is not supported
func (p *Store) Shutdown() {
}
+14
View File
@@ -0,0 +1,14 @@
package server
// BlobServer defines the common interface for all blob server implementations
type BlobServer interface {
Start() error
Shutdown()
}
type BlobServerConfig struct {
Address string `mapstructure:"address"`
EdgeToken string `mapstructure:"edge_token"`
Port int `mapstructure:"port"`
MaxConcurrentRequests int `mapstructure:"max_concurrent_requests"`
}
+1 -1
View File
@@ -2,5 +2,5 @@ package shared
import "github.com/lbryio/lbry.go/v2/extras/errors"
//ErrNotImplemented is a standard error when a store that implements the store interface does not implement a method
// ErrNotImplemented is a standard error when a store that implements the store interface does not implement a method
var ErrNotImplemented = errors.Base("this store does not implement this method")
+1 -1
View File
@@ -10,9 +10,9 @@ import (
)
type BlobStack struct {
Timing time.Duration `json:"timing"`
OriginName string `json:"origin_name"`
HostName string `json:"host_name"`
Timing time.Duration `json:"timing"`
}
type BlobTrace struct {
Stacks []BlobStack `json:"stacks"`
+11 -3
View File
@@ -16,9 +16,17 @@ func TestBlobTrace_Serialize(t *testing.T) {
stack.Stack(30*time.Second, "test3")
serialized, err := stack.Serialize()
assert.NoError(t, err)
t.Log(serialized)
expected := "{\"stacks\":[{\"timing\":10000000000,\"origin_name\":\"test\",\"host_name\":\"test_machine\"},{\"timing\":20000000000,\"origin_name\":\"test2\",\"host_name\":\"test_machine\"},{\"timing\":30000000000,\"origin_name\":\"test3\",\"host_name\":\"test_machine\"}]}"
assert.Equal(t, expected, serialized)
deserialized, err := Deserialize(serialized)
assert.NoError(t, err)
assert.Len(t, deserialized.Stacks, 3)
assert.Equal(t, 10*time.Second, deserialized.Stacks[0].Timing)
assert.Equal(t, "test", deserialized.Stacks[0].OriginName)
assert.Equal(t, "test_machine", deserialized.Stacks[0].HostName)
assert.Equal(t, 20*time.Second, deserialized.Stacks[1].Timing)
assert.Equal(t, "test2", deserialized.Stacks[1].OriginName)
assert.Equal(t, 30*time.Second, deserialized.Stacks[2].Timing)
assert.Equal(t, "test3", deserialized.Stacks[2].OriginName)
}
func TestBlobTrace_Deserialize(t *testing.T) {
+2 -1
View File
@@ -1,3 +1,4 @@
//go:build linux
// +build linux
package store
@@ -9,7 +10,7 @@ import (
)
func timespecToTime(ts syscall.Timespec) time.Time {
return time.Unix(int64(ts.Sec), int64(ts.Nsec))
return time.Unix(ts.Sec, ts.Nsec)
}
func atime(fi os.FileInfo) time.Time {
+1
View File
@@ -1,3 +1,4 @@
//go:build !linux
// +build !linux
package store
+67 -8
View File
@@ -1,6 +1,7 @@
package store
import (
"strings"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
@@ -10,6 +11,7 @@ import (
"github.com/lbryio/lbry.go/v2/stream"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
)
// CachingStore combines two stores, typically a local and a remote store, to improve performance.
@@ -17,20 +19,77 @@ import (
// are retrieved from the origin and cached. Puts are cached and also forwarded to the origin.
type CachingStore struct {
origin, cache BlobStore
component string
name string
}
type CachingParams struct {
Origin BlobStore `mapstructure:"origin"`
Cache BlobStore `mapstructure:"cache"`
Name string `mapstructure:"name"`
}
type CachingConfig struct {
Origin *viper.Viper
Cache *viper.Viper
Name string `mapstructure:"name"`
}
// NewCachingStore makes a new caching disk store and returns a pointer to it.
func NewCachingStore(component string, origin, cache BlobStore) *CachingStore {
func NewCachingStore(params CachingParams) *CachingStore {
return &CachingStore{
component: component,
origin: WithSingleFlight(component, origin),
cache: WithSingleFlight(component, cache),
name: params.Name,
origin: WithSingleFlight(params.Name, params.Origin),
cache: WithSingleFlight(params.Name, params.Cache),
}
}
const nameCaching = "caching"
func CachingStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg CachingConfig
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
cfg.Cache = config.Sub("cache")
cfg.Origin = config.Sub("origin")
if cfg.Cache == nil || cfg.Origin == nil {
return nil, errors.Err("cache and origin missing")
}
originStoreType := strings.Split(cfg.Origin.AllKeys()[0], ".")[0]
originStoreConfig := cfg.Origin.Sub(originStoreType)
factory, ok := Factories[originStoreType]
if !ok {
return nil, errors.Err("unknown store type %s", originStoreType)
}
originStore, err := factory(originStoreConfig)
if err != nil {
return nil, errors.Err(err)
}
cacheStoreType := strings.Split(cfg.Cache.AllKeys()[0], ".")[0]
cacheStoreConfig := cfg.Cache.Sub(cacheStoreType)
factory, ok = Factories[cacheStoreType]
if !ok {
return nil, errors.Err("unknown store type %s", cacheStoreType)
}
cacheStore, err := factory(cacheStoreConfig)
if err != nil {
return nil, errors.Err(err)
}
return NewCachingStore(CachingParams{
Name: cfg.Name,
Origin: originStore,
Cache: cacheStore,
}), nil
}
func init() {
RegisterStore(nameCaching, CachingStoreFactory)
}
// Name is the cache type name
func (c *CachingStore) Name() string { return nameCaching }
@@ -49,17 +108,17 @@ func (c *CachingStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
blob, trace, err := c.cache.Get(hash)
if err == nil || !errors.Is(err, ErrBlobNotFound) {
metrics.CacheHitCount.With(metrics.CacheLabels(c.cache.Name(), c.component)).Inc()
metrics.CacheHitCount.With(metrics.CacheLabels(c.cache.Name(), c.name)).Inc()
rate := float64(len(blob)) / 1024 / 1024 / time.Since(start).Seconds()
metrics.CacheRetrievalSpeed.With(map[string]string{
metrics.LabelCacheType: c.cache.Name(),
metrics.LabelComponent: c.component,
metrics.LabelComponent: c.name,
metrics.LabelSource: "cache",
}).Set(rate)
return blob, trace.Stack(time.Since(start), c.Name()), err
}
metrics.CacheMissCount.With(metrics.CacheLabels(c.cache.Name(), c.component)).Inc()
metrics.CacheMissCount.With(metrics.CacheLabels(c.cache.Name(), c.name)).Inc()
blob, trace, err = c.origin.Get(hash)
if err != nil {
+13 -13
View File
@@ -9,12 +9,14 @@ import (
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/sirupsen/logrus"
)
func TestCachingStore_Put(t *testing.T) {
origin := NewMemStore()
cache := NewMemStore()
s := NewCachingStore("test", origin, cache)
origin := NewMemStore(MemParams{Name: "test"})
cache := NewMemStore(MemParams{Name: "test"})
s := NewCachingStore(CachingParams{Name: "test", Origin: origin, Cache: cache})
b := []byte("this is a blob of stuff")
hash := "hash"
@@ -42,9 +44,9 @@ func TestCachingStore_Put(t *testing.T) {
}
func TestCachingStore_CacheMiss(t *testing.T) {
origin := NewMemStore()
cache := NewMemStore()
s := NewCachingStore("test", origin, cache)
origin := NewMemStore(MemParams{Name: "test"})
cache := NewMemStore(MemParams{Name: "test"})
s := NewCachingStore(CachingParams{Name: "test", Origin: origin, Cache: cache})
b := []byte("this is a blob of stuff")
hash := "hash"
@@ -84,8 +86,8 @@ func TestCachingStore_CacheMiss(t *testing.T) {
func TestCachingStore_ThunderingHerd(t *testing.T) {
storeDelay := 100 * time.Millisecond
origin := NewSlowBlobStore(storeDelay)
cache := NewMemStore()
s := NewCachingStore("test", origin, cache)
cache := NewMemStore(MemParams{Name: "test"})
s := NewCachingStore(CachingParams{Name: "test", Origin: origin, Cache: cache})
b := []byte("this is a blob of stuff")
hash := "hash"
@@ -99,7 +101,7 @@ func TestCachingStore_ThunderingHerd(t *testing.T) {
getNoErr := func() {
res, _, err := s.Get(hash)
if err != nil {
t.Fatal(err)
logrus.Fatal(err)
}
if !bytes.Equal(b, res) {
t.Errorf("expected Get() to return %s, got %s", string(b), string(res))
@@ -140,7 +142,7 @@ type SlowBlobStore struct {
func NewSlowBlobStore(delay time.Duration) *SlowBlobStore {
return &SlowBlobStore{
mem: NewMemStore(),
mem: NewMemStore(MemParams{Name: "test"}),
delay: delay,
}
}
@@ -173,6 +175,4 @@ func (s *SlowBlobStore) Delete(hash string) error {
return s.mem.Delete(hash)
}
func (s *SlowBlobStore) Shutdown() {
return
}
func (s *SlowBlobStore) Shutdown() {}
-110
View File
@@ -1,110 +0,0 @@
package store
import (
"io"
"io/ioutil"
"net/http"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/meta"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
log "github.com/sirupsen/logrus"
)
// CloudFrontROStore reads from cloudfront. All writes panic.
type CloudFrontROStore struct {
endpoint string // cloudflare endpoint
}
// NewCloudFrontROStore returns an initialized CloudFrontROStore store pointer.
func NewCloudFrontROStore(endpoint string) *CloudFrontROStore {
return &CloudFrontROStore{endpoint: endpoint}
}
const nameCloudFrontRO = "cloudfront_ro"
// Name is the cache type name
func (c *CloudFrontROStore) Name() string { return nameCloudFrontRO }
// Has checks if the hash is in the store.
func (c *CloudFrontROStore) Has(hash string) (bool, error) {
status, body, err := c.cfRequest(http.MethodHead, hash)
if err != nil {
return false, err
}
defer body.Close()
switch status {
case http.StatusNotFound, http.StatusForbidden:
return false, nil
case http.StatusOK:
return true, nil
default:
return false, errors.Err("unexpected status %d", status)
}
}
// Get gets the blob from Cloudfront.
func (c *CloudFrontROStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
log.Debugf("Getting %s from S3", hash[:8])
start := time.Now()
defer func(t time.Time) {
log.Debugf("Getting %s from S3 took %s", hash[:8], time.Since(t).String())
}(start)
status, body, err := c.cfRequest(http.MethodGet, hash)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), c.Name()), err
}
defer body.Close()
switch status {
case http.StatusNotFound, http.StatusForbidden:
return nil, shared.NewBlobTrace(time.Since(start), c.Name()), errors.Err(ErrBlobNotFound)
case http.StatusOK:
b, err := ioutil.ReadAll(body)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), c.Name()), errors.Err(err)
}
metrics.MtrInBytesS3.Add(float64(len(b)))
return b, shared.NewBlobTrace(time.Since(start), c.Name()), nil
default:
return nil, shared.NewBlobTrace(time.Since(start), c.Name()), errors.Err("unexpected status %d", status)
}
}
func (c *CloudFrontROStore) cfRequest(method, hash string) (int, io.ReadCloser, error) {
url := c.endpoint + hash
req, err := http.NewRequest(method, url, nil)
if err != nil {
return 0, nil, errors.Err(err)
}
req.Header.Add("User-Agent", "reflector.go/"+meta.Version)
res, err := http.DefaultClient.Do(req)
if err != nil {
return 0, nil, errors.Err(err)
}
return res.StatusCode, res.Body, nil
}
func (c *CloudFrontROStore) Put(_ string, _ stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
func (c *CloudFrontROStore) PutSD(_ string, _ stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
func (c *CloudFrontROStore) Delete(_ string) error {
return errors.Err(shared.ErrNotImplemented)
}
// Shutdown shuts down the store gracefully
func (c *CloudFrontROStore) Shutdown() {
}
-62
View File
@@ -1,62 +0,0 @@
package store
import (
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/stream"
)
// CloudFrontRWStore combines a Cloudfront and an S3 store. Reads go to Cloudfront/Wasabi, writes go to S3.
type CloudFrontRWStore struct {
cf *ITTTStore
s3 *S3Store
}
// NewCloudFrontRWStore returns an initialized CloudFrontRWStore store pointer.
// NOTE: It panics if either argument is nil.
func NewCloudFrontRWStore(cf *ITTTStore, s3 *S3Store) *CloudFrontRWStore {
if cf == nil || s3 == nil {
panic("both stores must be set")
}
return &CloudFrontRWStore{cf: cf, s3: s3}
}
const nameCloudFrontRW = "cloudfront_rw"
// Name is the cache type name
func (c *CloudFrontRWStore) Name() string { return nameCloudFrontRW }
// Has checks if the hash is in the store.
func (c *CloudFrontRWStore) Has(hash string) (bool, error) {
return c.cf.Has(hash)
}
// Get gets the blob from Cloudfront.
func (c *CloudFrontRWStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
blob, trace, err := c.cf.Get(hash)
return blob, trace.Stack(time.Since(start), c.Name()), err
}
// Put stores the blob on S3
func (c *CloudFrontRWStore) Put(hash string, blob stream.Blob) error {
return c.s3.Put(hash, blob)
}
// PutSD stores the sd blob on S3
func (c *CloudFrontRWStore) PutSD(hash string, blob stream.Blob) error {
return c.s3.PutSD(hash, blob)
}
// Delete deletes the blob from S3
func (c *CloudFrontRWStore) Delete(hash string) error {
return c.s3.Delete(hash)
}
// Shutdown shuts down the store gracefully
func (c *CloudFrontRWStore) Shutdown() {
c.s3.Shutdown()
c.cf.Shutdown()
}
+177 -25
View File
@@ -2,6 +2,8 @@ package store
import (
"encoding/json"
"fmt"
"strings"
"sync"
"time"
@@ -9,29 +11,70 @@ import (
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/c2h5oh/datasize"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
)
// DBBackedStore is a store that's backed by a DB. The DB contains data about what's in the store.
type DBBackedStore struct {
blobs BlobStore
db *db.SQL
blockedMu sync.RWMutex
blocked map[string]bool
cleanerStop *stop.Group
name string
maxSize int
blockedMu sync.RWMutex
deleteOnMiss bool
}
// NewDBBackedStore returns an initialized store pointer.
func NewDBBackedStore(blobs BlobStore, db *db.SQL, deleteOnMiss bool) *DBBackedStore {
return &DBBackedStore{blobs: blobs, db: db, deleteOnMiss: deleteOnMiss}
type DBBackedParams struct {
Store BlobStore `mapstructure:"store"`
DB *db.SQL `mapstructure:"db"`
MaxSize *int `mapstructure:"max_size"`
Name string `mapstructure:"name"`
DeleteOnMiss bool `mapstructure:"delete_on_miss"`
}
const nameDBBacked = "db-backed"
type DBBackedConfig struct {
Store *viper.Viper
Database string `mapstructure:"database"`
User string `mapstructure:"user"`
Password string `mapstructure:"password"`
Host string `mapstructure:"host"`
Name string `mapstructure:"name"`
MaxSize string `mapstructure:"max_size"`
Port int `mapstructure:"port"`
AccessTracking int `mapstructure:"access_tracking"`
DeleteOnMiss bool `mapstructure:"delete_on_miss"`
SoftDeletes bool `mapstructure:"soft_deletes"`
LogQueries bool `mapstructure:"log_queries"`
HasCap bool `mapstructure:"has_cap"`
}
// NewDBBackedStore returns an initialized store pointer.
func NewDBBackedStore(params DBBackedParams) *DBBackedStore {
store := &DBBackedStore{
blobs: params.Store,
db: params.DB,
deleteOnMiss: params.DeleteOnMiss,
cleanerStop: stop.New(),
name: params.Name,
}
if params.MaxSize != nil {
store.maxSize = *params.MaxSize
go store.cleanOldestBlobs()
}
return store
}
const nameDBBacked = "db_backed"
// Name is the cache type name
func (d *DBBackedStore) Name() string { return nameDBBacked }
func (d *DBBackedStore) Name() string { return nameDBBacked + "-" + d.name }
// Has returns true if the blob is in the store
func (d *DBBackedStore) Has(hash string) (bool, error) {
@@ -101,34 +144,18 @@ func (d *DBBackedStore) Delete(hash string) error {
// Block deletes the blob and prevents it from being uploaded in the future
func (d *DBBackedStore) Block(hash string) error {
if blocked, err := d.isBlocked(hash); blocked || err != nil {
blocked, err := d.isBlocked(hash)
if blocked || err != nil {
return err
}
log.Debugf("blocking %s", hash)
err := d.db.Block(hash)
err = d.db.Block(hash)
if err != nil {
return err
}
has, err := d.db.HasBlob(hash, false)
if err != nil {
return err
}
if has {
err = d.blobs.Delete(hash)
if err != nil {
return err
}
err = d.db.Delete(hash)
if err != nil {
return err
}
}
return d.markBlocked(hash)
}
@@ -195,7 +222,132 @@ func (d *DBBackedStore) initBlocked() error {
return err
}
// cleanOldestBlobs periodically cleans up the oldest blobs if maxSize is set
func (d *DBBackedStore) cleanOldestBlobs() {
// Run on startup without waiting for 10 minutes
err := d.doClean()
if err != nil {
log.Error(errors.FullTrace(err))
}
const cleanupInterval = 10 * time.Minute
for {
select {
case <-d.cleanerStop.Ch():
log.Infoln("stopping self cleanup")
return
case <-time.After(cleanupInterval):
err := d.doClean()
if err != nil {
log.Error(errors.FullTrace(err))
}
}
}
}
// doClean removes the least recently accessed blobs if the store exceeds maxItems
func (d *DBBackedStore) doClean() error {
blobsCount, err := d.db.Count()
if err != nil {
return err
}
if blobsCount >= d.maxSize {
itemsToDelete := blobsCount / 10
blobs, err := d.db.LeastRecentlyAccessedHashes(itemsToDelete)
if err != nil {
return err
}
blobsChan := make(chan string, len(blobs))
wg := stop.New()
go func() {
for _, hash := range blobs {
select {
case <-d.cleanerStop.Ch():
return
default:
}
blobsChan <- hash
}
close(blobsChan)
}()
for i := 0; i < 3; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for h := range blobsChan {
select {
case <-d.cleanerStop.Ch():
return
default:
}
err = d.Delete(h)
if err != nil {
log.Errorf("error pruning %s: %s", h, errors.FullTrace(err))
continue
}
}
}()
}
wg.Wait()
}
return nil
}
// Shutdown shuts down the store gracefully
func (d *DBBackedStore) Shutdown() {
d.cleanerStop.Stop()
d.blobs.Shutdown()
}
func DBBackedStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg DBBackedConfig
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
cfg.Store = config.Sub("store")
storeType := strings.Split(cfg.Store.AllKeys()[0], ".")[0]
storeConfig := cfg.Store.Sub(storeType)
factory, ok := Factories[storeType]
if !ok {
return nil, errors.Err("unknown store type %s", storeType)
}
underlyingStore, err := factory(storeConfig)
if err != nil {
return nil, errors.Err(err)
}
parsedDb := &db.SQL{
TrackingLevel: db.AccessTrackingLevel(cfg.AccessTracking),
SoftDelete: cfg.SoftDeletes,
LogQueries: cfg.LogQueries || log.GetLevel() == log.DebugLevel,
}
err = parsedDb.Connect(fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", cfg.User, cfg.Password, cfg.Host, cfg.Port, cfg.Database))
if err != nil {
return nil, err
}
params := DBBackedParams{
Name: cfg.Name,
Store: underlyingStore,
DB: parsedDb,
DeleteOnMiss: cfg.DeleteOnMiss,
}
if cfg.HasCap {
var parsedSize datasize.ByteSize
err = parsedSize.UnmarshalText([]byte(cfg.MaxSize))
if err != nil {
return nil, errors.Err(err)
}
maxSize := int(float64(parsedSize) / float64(stream.MaxBlobSize))
params.MaxSize = &maxSize
}
return NewDBBackedStore(params), nil
}
func init() {
RegisterStore(nameDBBacked, DBBackedStoreFactory)
}
+31 -100
View File
@@ -1,13 +1,6 @@
package store
import (
"bytes"
"crypto/sha512"
"encoding/hex"
"fmt"
"io"
"io/ioutil"
"net"
"os"
"path"
"time"
@@ -18,38 +11,49 @@ import (
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/brk0v/directio"
log "github.com/sirupsen/logrus"
"go.uber.org/atomic"
"github.com/spf13/viper"
)
// DiskStore stores blobs on a local disk
type DiskStore struct {
// the location of blobs on disk
blobDir string
// store files in subdirectories based on the first N chars in the filename. 0 = don't create subdirectories.
blobDir string
name string
prefixLength int
// true if initOnce ran, false otherwise
initialized bool
concurrentChecks atomic.Int32
initialized bool
}
const maxConcurrentChecks = 30
type DiskParams struct {
Name string `mapstructure:"name"`
MountPoint string `mapstructure:"mount_point"`
ShardingSize int `mapstructure:"sharding_size"`
}
// NewDiskStore returns an initialized file disk store pointer.
func NewDiskStore(dir string, prefixLength int) *DiskStore {
func NewDiskStore(params DiskParams) *DiskStore {
return &DiskStore{
blobDir: dir,
prefixLength: prefixLength,
blobDir: params.MountPoint,
prefixLength: params.ShardingSize,
name: params.Name,
}
}
const nameDisk = "disk"
func DiskStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg DiskParams
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
return NewDiskStore(cfg), nil
}
func init() {
RegisterStore(nameDisk, DiskStoreFactory)
}
// Name is the cache type name
func (d *DiskStore) Name() string { return nameDisk }
func (d *DiskStore) Name() string { return nameDisk + "-" + d.name }
// Has returns T/F or Error if it the blob stored already. It will error with any IO disk error.
func (d *DiskStore) Has(hash string) (bool, error) {
@@ -68,14 +72,6 @@ func (d *DiskStore) Has(hash string) (bool, error) {
return true, nil
}
func (d *DiskStore) Size(hash string) (int64, error) {
stat, err := os.Stat(d.path(hash))
if err != nil {
return 0, err
}
return stat.Size(), nil
}
// Get returns the blob or an error if the blob doesn't exist.
func (d *DiskStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
@@ -84,69 +80,16 @@ func (d *DiskStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), err
}
blob, err := ioutil.ReadFile(d.path(hash))
blob, err := os.ReadFile(d.path(hash))
if err != nil {
if os.IsNotExist(err) {
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(ErrBlobNotFound)
}
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(err)
}
// this is a rather poor yet effective way of throttling how many blobs can be checked concurrently
// poor because there is a possible race condition between the check and the actual +1
if d.concurrentChecks.Load() < maxConcurrentChecks {
d.concurrentChecks.Add(1)
defer d.concurrentChecks.Sub(1)
hashBytes := sha512.Sum384(blob)
readHash := hex.EncodeToString(hashBytes[:])
if hash != readHash {
message := fmt.Sprintf("[%s] found a broken blob while reading from disk. Actual hash: %s", hash, readHash)
log.Errorf("%s", message)
err := d.Delete(hash)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), err
}
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(message)
}
}
return blob, shared.NewBlobTrace(time.Since(start), d.Name()), nil
}
// Put stores the blob on disk
func (d *DiskStore) Put(hash string, blob stream.Blob) error {
err := d.initOnce()
if err != nil {
return err
}
err = d.ensureDirExists(d.dir(hash))
if err != nil {
return err
}
// Open file with O_DIRECT
f, err := os.OpenFile(d.tmpPath(hash), openFileFlags, 0644)
if err != nil {
return errors.Err(err)
}
defer f.Close()
// Use directio writer
dio, err := directio.New(f)
if err != nil {
return errors.Err(err)
}
defer dio.Flush()
// Write the body to file
_, err = io.Copy(dio, bytes.NewReader(blob))
if err != nil {
return errors.Err(err)
}
err = os.Rename(d.tmpPath(hash), d.path(hash))
return errors.Err(err)
}
// PutSD stores the sd blob on the disk
func (d *DiskStore) PutSD(hash string, blob stream.Blob) error {
return d.Put(hash, blob)
@@ -187,14 +130,14 @@ func (d *DiskStore) dir(hash string) string {
}
return path.Join(d.blobDir, hash[:d.prefixLength])
}
func (d *DiskStore) tmpDir(hash string) string {
func (d *DiskStore) tmpDir() string {
return path.Join(d.blobDir, "tmp")
}
func (d *DiskStore) path(hash string) string {
return path.Join(d.dir(hash), hash)
}
func (d *DiskStore) tmpPath(hash string) string {
return path.Join(d.tmpDir(hash), hash)
return path.Join(d.tmpDir(), hash)
}
func (d *DiskStore) ensureDirExists(dir string) error {
return errors.Err(os.MkdirAll(dir, 0755))
@@ -209,7 +152,7 @@ func (d *DiskStore) initOnce() error {
if err != nil {
return err
}
err = d.ensureDirExists(path.Join(d.blobDir, "tmp"))
err = d.ensureDirExists(d.tmpDir())
if err != nil {
return err
}
@@ -220,15 +163,3 @@ func (d *DiskStore) initOnce() error {
// Shutdown shuts down the store gracefully
func (d *DiskStore) Shutdown() {
}
func (d *DiskStore) Sendfile(hash string, conn net.Conn) error {
f, err := os.Open(d.path(hash))
if err != nil {
return err
}
_, err = io.Copy(conn, f)
if err != nil {
return err
}
return nil
}
+15 -8
View File
@@ -1,7 +1,6 @@
package store
import (
"io/ioutil"
"os"
"path"
"path/filepath"
@@ -14,10 +13,14 @@ import (
)
func TestDiskStore_Get(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "reflector_test_*")
tmpDir, err := os.MkdirTemp("", "reflector_test_*")
require.NoError(t, err)
defer os.RemoveAll(tmpDir)
d := NewDiskStore(tmpDir, 2)
defer func() { _ = os.RemoveAll(tmpDir) }()
d := NewDiskStore(DiskParams{
Name: "test",
MountPoint: tmpDir,
ShardingSize: 2,
})
hash := "f428b8265d65dad7f8ffa52922bba836404cbd62f3ecfe10adba6b444f8f658938e54f5981ac4de39644d5b93d89a94b"
data := []byte("oyuntyausntoyaunpdoyruoyduanrstjwfjyuwf")
@@ -25,7 +28,7 @@ func TestDiskStore_Get(t *testing.T) {
expectedPath := path.Join(tmpDir, hash[:2], hash)
err = os.MkdirAll(filepath.Dir(expectedPath), os.ModePerm)
require.NoError(t, err)
err = ioutil.WriteFile(expectedPath, data, os.ModePerm)
err = os.WriteFile(expectedPath, data, os.ModePerm)
require.NoError(t, err)
blob, _, err := d.Get(hash)
@@ -34,10 +37,14 @@ func TestDiskStore_Get(t *testing.T) {
}
func TestDiskStore_GetNonexistentBlob(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "reflector_test_*")
tmpDir, err := os.MkdirTemp("", "reflector_test_*")
require.NoError(t, err)
defer os.RemoveAll(tmpDir)
d := NewDiskStore(tmpDir, 2)
defer func() { _ = os.RemoveAll(tmpDir) }()
d := NewDiskStore(DiskParams{
Name: "test",
MountPoint: tmpDir,
ShardingSize: 2,
})
blob, _, err := d.Get("nonexistent")
assert.Nil(t, blob)
+42
View File
@@ -0,0 +1,42 @@
//go:build darwin
// +build darwin
package store
import (
"bytes"
"io"
"os"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
)
var openFileFlags = os.O_WRONLY | os.O_CREATE
// Put stores the blob on disk
func (d *DiskStore) Put(hash string, blob stream.Blob) error {
err := d.initOnce()
if err != nil {
return err
}
err = d.ensureDirExists(d.dir(hash))
if err != nil {
return err
}
// Open file with O_DIRECT
f, err := os.OpenFile(d.tmpPath(hash), openFileFlags, 0644)
if err != nil {
return errors.Err(err)
}
defer f.Close()
_, err = io.Copy(f, bytes.NewReader(blob))
if err != nil {
return errors.Err(err)
}
err = os.Rename(d.tmpPath(hash), d.path(hash))
return errors.Err(err)
}
+49
View File
@@ -0,0 +1,49 @@
//go:build linux
// +build linux
package store
import (
"bytes"
"io"
"os"
"syscall"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/brk0v/directio"
)
var openFileFlags = os.O_WRONLY | os.O_CREATE | syscall.O_DIRECT
// Put stores the blob on disk
func (d *DiskStore) Put(hash string, blob stream.Blob) error {
err := d.initOnce()
if err != nil {
return err
}
err = d.ensureDirExists(d.dir(hash))
if err != nil {
return err
}
// Open file with O_DIRECT
f, err := os.OpenFile(d.tmpPath(hash), openFileFlags, 0644)
if err != nil {
return errors.Err(err)
}
defer func() { _ = f.Close() }()
dio, err := directio.New(f)
if err != nil {
return errors.Err(err)
}
defer func() { _ = dio.Flush() }()
_, err = io.Copy(dio, bytes.NewReader(blob))
if err != nil {
return errors.Err(err)
}
err = os.Rename(d.tmpPath(hash), d.path(hash))
return errors.Err(err)
}
+31
View File
@@ -0,0 +1,31 @@
//go:build !linux && !darwin
// +build !linux,!darwin
package store
import (
"os"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
)
// Put stores the blob on disk
func (d *DiskStore) Put(hash string, blob stream.Blob) error {
err := d.initOnce()
if err != nil {
return err
}
err = d.ensureDirExists(d.dir(hash))
if err != nil {
return err
}
err = os.WriteFile(d.tmpPath(hash), blob, 0644)
if err != nil {
return errors.Err(err)
}
err = os.Rename(d.tmpPath(hash), d.path(hash))
return errors.Err(err)
}
-9
View File
@@ -1,9 +0,0 @@
// +build darwin
package store
import (
"os"
)
var openFileFlags = os.O_WRONLY | os.O_CREATE
-10
View File
@@ -1,10 +0,0 @@
// +build linux
package store
import (
"os"
"syscall"
)
var openFileFlags = os.O_WRONLY | os.O_CREATE | syscall.O_DIRECT
+73 -21
View File
@@ -1,6 +1,7 @@
package store
import (
"strings"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
@@ -11,15 +12,16 @@ import (
"github.com/bluele/gcache"
"github.com/sirupsen/logrus"
"github.com/spf13/viper"
)
// GcacheStore adds a max cache size and Greedy-Dual-Size-Frequency cache eviction strategy to a BlobStore
type GcacheStore struct {
// underlying store
store BlobStore
// cache implementation
cache gcache.Cache
underlyingStore BlobStore
cache gcache.Cache
name string
}
type EvictionStrategy int
const (
@@ -33,16 +35,30 @@ const (
SIMPLE
)
type GcacheParams struct {
Store BlobStore `mapstructure:"store"`
Name string `mapstructure:"name"`
MaxSize int `mapstructure:"max_size"`
Strategy EvictionStrategy `mapstructure:"strategy"`
}
type GcacheConfig struct {
Store *viper.Viper
Name string `mapstructure:"name"`
MaxSize int `mapstructure:"max_size"`
Strategy EvictionStrategy `mapstructure:"strategy"`
}
// NewGcacheStore initialize a new LRUStore
func NewGcacheStore(component string, store BlobStore, maxSize int, strategy EvictionStrategy) *GcacheStore {
cacheBuilder := gcache.New(maxSize)
func NewGcacheStore(params GcacheParams) *GcacheStore {
cacheBuilder := gcache.New(params.MaxSize)
var cache gcache.Cache
evictFunc := func(key interface{}, value interface{}) {
logrus.Infof("evicting %s", key)
metrics.CacheLRUEvictCount.With(metrics.CacheLabels(store.Name(), component)).Inc()
_ = store.Delete(key.(string)) // TODO: log this error. may happen if underlying entry is gone but cache entry still there
metrics.CacheLRUEvictCount.With(metrics.CacheLabels(params.Store.Name(), params.Name)).Inc()
_ = params.Store.Delete(key.(string)) // TODO: log this error. may happen if underlying entry is gone but cache entry still there
}
switch strategy {
switch params.Strategy {
case LFU:
cache = cacheBuilder.LFU().EvictedFunc(evictFunc).Build()
case ARC:
@@ -51,15 +67,15 @@ func NewGcacheStore(component string, store BlobStore, maxSize int, strategy Evi
cache = cacheBuilder.LRU().EvictedFunc(evictFunc).Build()
case SIMPLE:
cache = cacheBuilder.Simple().EvictedFunc(evictFunc).Build()
}
l := &GcacheStore{
store: store,
cache: cache,
underlyingStore: params.Store,
cache: cache,
name: params.Name,
}
go func() {
if lstr, ok := store.(lister); ok {
err := l.loadExisting(lstr, int(maxSize))
if lstr, ok := params.Store.(lister); ok {
err := l.loadExisting(lstr, params.MaxSize)
if err != nil {
panic(err) // TODO: what should happen here? panic? return nil? just keep going?
}
@@ -71,8 +87,40 @@ func NewGcacheStore(component string, store BlobStore, maxSize int, strategy Evi
const nameGcache = "gcache"
func GcacheStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg GcacheConfig
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
cfg.Store = config.Sub("store")
storeType := strings.Split(cfg.Store.AllKeys()[0], ".")[0]
storeConfig := cfg.Store.Sub(storeType)
factory, ok := Factories[storeType]
if !ok {
return nil, errors.Err("unknown store type %s", storeType)
}
underlyingStore, err := factory(storeConfig)
if err != nil {
return nil, errors.Err(err)
}
return NewGcacheStore(GcacheParams{
Name: cfg.Name,
Store: underlyingStore,
MaxSize: cfg.MaxSize,
Strategy: cfg.Strategy,
}), nil
}
func init() {
RegisterStore(nameGcache, GcacheStoreFactory)
}
// Name is the cache type name
func (l *GcacheStore) Name() string { return nameGcache }
func (l *GcacheStore) Name() string { return nameGcache + "-" + l.name }
// Has returns whether the blob is in the store, without updating the recent-ness.
func (l *GcacheStore) Has(hash string) (bool, error) {
@@ -86,7 +134,7 @@ func (l *GcacheStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), l.Name()), errors.Err(ErrBlobNotFound)
}
blob, stack, err := l.store.Get(hash)
blob, stack, err := l.underlyingStore.Get(hash)
if errors.Is(err, ErrBlobNotFound) {
// Blob disappeared from underlying store
l.cache.Remove(hash)
@@ -99,7 +147,7 @@ func (l *GcacheStore) Put(hash string, blob stream.Blob) error {
_ = l.cache.Set(hash, true)
has, _ := l.Has(hash)
if has {
err := l.store.Put(hash, blob)
err := l.underlyingStore.Put(hash, blob)
if err != nil {
return err
}
@@ -112,7 +160,7 @@ func (l *GcacheStore) PutSD(hash string, blob stream.Blob) error {
_ = l.cache.Set(hash, true)
has, _ := l.Has(hash)
if has {
err := l.store.PutSD(hash, blob)
err := l.underlyingStore.PutSD(hash, blob)
if err != nil {
return err
}
@@ -122,7 +170,7 @@ func (l *GcacheStore) PutSD(hash string, blob stream.Blob) error {
// Delete deletes the blob from the store
func (l *GcacheStore) Delete(hash string) error {
err := l.store.Delete(hash)
err := l.underlyingStore.Delete(hash)
if err != nil {
return err
}
@@ -144,11 +192,15 @@ func (l *GcacheStore) loadExisting(store lister, maxItems int) error {
logrus.Infof("read %d files from underlying store", len(existing))
added := 0
for _, h := range existing {
for i, h := range existing {
_ = l.cache.Set(h, true)
added++
if maxItems > 0 && added >= maxItems { // underlying cache is bigger than the cache
break
err := l.Delete(h)
logrus.Infof("deleted overflowing blob: %s (%d/%d)", h, i, len(existing))
if err != nil {
logrus.Warnf("error while deleting a blob that's overflowing the cache: %s", err.Error())
}
}
}
return nil
+11 -8
View File
@@ -2,7 +2,6 @@ package store
import (
"fmt"
"io/ioutil"
"os"
"reflect"
"testing"
@@ -17,8 +16,8 @@ import (
const cacheMaxSize = 3
func getTestGcacheStore() (*GcacheStore, *MemStore) {
m := NewMemStore()
return NewGcacheStore("test", m, cacheMaxSize, LFU), m
m := NewMemStore(MemParams{Name: "test"})
return NewGcacheStore(GcacheParams{Name: "test", Store: m, MaxSize: cacheMaxSize, Strategy: LFU}), m
}
func TestGcacheStore_Eviction(t *testing.T) {
@@ -88,10 +87,14 @@ func TestGcacheStore_UnderlyingBlobMissing(t *testing.T) {
}
func TestGcacheStore_loadExisting(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "reflector_test_*")
tmpDir, err := os.MkdirTemp("", "reflector_test_*")
require.NoError(t, err)
defer os.RemoveAll(tmpDir)
d := NewDiskStore(tmpDir, 2)
defer func() { _ = os.RemoveAll(tmpDir) }()
d := NewDiskStore(DiskParams{
Name: "test",
MountPoint: tmpDir,
ShardingSize: 2,
})
hash := "hash"
b := []byte("this is a blob of stuff")
@@ -103,8 +106,8 @@ func TestGcacheStore_loadExisting(t *testing.T) {
require.Equal(t, 1, len(existing), "blob should exist in cache")
assert.Equal(t, hash, existing[0])
lfu := NewGcacheStore("test", d, 3, LFU) // lru should load existing blobs when it's created
time.Sleep(100 * time.Millisecond) // async load so let's wait...
lfu := NewGcacheStore(GcacheParams{Name: "test", Store: d, MaxSize: 3, Strategy: LFU}) // lru should load existing blobs when it's created
time.Sleep(100 * time.Millisecond) // async load so let's wait...
has, err := lfu.Has(hash)
require.NoError(t, err)
assert.True(t, has, "hash should be loaded from disk store but it's not")
+129 -101
View File
@@ -1,160 +1,188 @@
package store
import (
"bytes"
"io"
"io/ioutil"
"net"
"net/http"
"sync"
"path"
"strconv"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/meta"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
)
// HttpStore is a store that works on top of the HTTP protocol
// HttpStore reads from an HTTP endpoint that simply expects the hash to be appended to the endpoint
type HttpStore struct {
upstream string
httpClient *http.Client
httpClient *http.Client
endpoint string
name string
prefixLength int
}
func NewHttpStore(upstream string) *HttpStore {
type HttpParams struct {
Name string `mapstructure:"name"`
Endpoint string `mapstructure:"endpoint"`
ShardingSize int `mapstructure:"sharding_size"`
}
// NewHttpStore returns an initialized HttpStore store pointer.
func NewHttpStore(params HttpParams) *HttpStore {
return &HttpStore{
upstream: "http://" + upstream,
httpClient: getClient(),
endpoint: params.Endpoint,
httpClient: getClient(),
prefixLength: params.ShardingSize,
name: params.Name,
}
}
const nameHttp = "http"
func (n *HttpStore) Name() string { return nameHttp }
func (n *HttpStore) Has(hash string) (bool, error) {
url := n.upstream + "/blob?hash=" + hash
req, err := http.NewRequest("HEAD", url, nil)
func HttpStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg HttpParams
err := config.Unmarshal(&cfg)
if err != nil {
return false, errors.Err(err)
return nil, errors.Err(err)
}
return NewHttpStore(cfg), nil
}
res, err := n.httpClient.Do(req)
// Name is the cache type name
func (c *HttpStore) Name() string { return nameHttp + "-" + c.name }
// Has checks if the hash is in the store.
func (c *HttpStore) Has(hash string) (bool, error) {
status, body, err := c.cfRequest(http.MethodHead, hash)
if err != nil {
return false, errors.Err(err)
return false, err
}
defer res.Body.Close()
if res.StatusCode == http.StatusNotFound {
defer func() { _ = body.Close() }()
switch status {
case http.StatusNotFound, http.StatusForbidden:
return false, nil
}
if res.StatusCode == http.StatusNoContent {
case http.StatusOK:
return true, nil
default:
return false, errors.Err("unexpected status %d", status)
}
var body []byte
if res.Body != nil {
body, _ = ioutil.ReadAll(res.Body)
}
return false, errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
}
func (n *HttpStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
// Get downloads the blob using the http client
func (c *HttpStore) Get(hash string) (b stream.Blob, trace shared.BlobTrace, err error) {
log.Debugf("Getting %s from HTTP(s) source", hash[:8])
start := time.Now()
url := n.upstream + "/blob?hash=" + hash
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), n.Name()), errors.Err(err)
}
defer func(t time.Time) {
log.Debugf("Getting %s from HTTP(s) source took %s", hash[:8], time.Since(t).String())
trace = trace.Stack(time.Since(start), c.Name())
}(start)
res, err := n.httpClient.Do(req)
url := c.endpoint + c.shardedPath(hash)
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), n.Name()), errors.Err(err)
return nil, trace, errors.Err(err)
}
defer res.Body.Close()
tmp := getBuffer()
defer putBuffer(tmp)
serialized := res.Header.Get("Via")
trace := shared.NewBlobTrace(time.Since(start), n.Name())
if serialized != "" {
parsedTrace, err := shared.Deserialize(serialized)
req.Header.Add("User-Agent", "reflector.go/"+meta.Version())
res, err := c.httpClient.Do(req)
if err != nil {
return nil, trace, errors.Err(err)
}
defer func(Body io.ReadCloser) {
err := Body.Close()
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), n.Name()), err
log.Errorf("Error closing response body in HTTP-GET: %s", err.Error())
}
}(res.Body)
// Parse Via header if present
viaHeader := res.Header.Get("Via")
if viaHeader != "" {
parsedTrace, err := shared.Deserialize(viaHeader)
if err == nil {
trace = *parsedTrace
}
trace = *parsedTrace
}
if res.StatusCode == http.StatusNotFound {
return nil, trace.Stack(time.Since(start), n.Name()), ErrBlobNotFound
}
if res.StatusCode == http.StatusOK {
written, err := io.Copy(tmp, res.Body)
if err != nil {
return nil, trace.Stack(time.Since(start), n.Name()), errors.Err(err)
switch res.StatusCode {
case http.StatusNotFound:
return nil, trace, ErrBlobNotFound
case http.StatusOK:
contentLength := res.Header.Get("Content-Length")
if contentLength != "" {
size, err := strconv.Atoi(contentLength)
if err == nil && size > 0 && size <= stream.MaxBlobSize {
blob := make([]byte, size)
_, err = io.ReadFull(res.Body, blob)
if err == nil {
metrics.MtrInBytesHttp.Add(float64(size))
return blob, trace, nil
}
log.Warnf("Error reading body with known size: %s", err.Error())
}
}
blob := make([]byte, written)
copy(blob, tmp.Bytes())
buffer := getBuffer()
defer putBuffer(buffer)
if _, err := io.Copy(buffer, res.Body); err != nil {
return nil, trace, errors.Err(err)
}
blob := make([]byte, buffer.Len())
copy(blob, buffer.Bytes())
metrics.MtrInBytesHttp.Add(float64(len(blob)))
return blob, trace.Stack(time.Since(start), n.Name()), nil
return blob, trace, nil
default:
body, _ := io.ReadAll(res.Body)
log.Warnf("Got status code %d (%s)", res.StatusCode, string(body))
return nil, trace, errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
}
var body []byte
if res.Body != nil {
body, _ = ioutil.ReadAll(res.Body)
}
func (c *HttpStore) cfRequest(method, hash string) (int, io.ReadCloser, error) {
url := c.endpoint + c.shardedPath(hash)
req, err := http.NewRequest(method, url, nil)
if err != nil {
return 0, nil, errors.Err(err)
}
req.Header.Add("User-Agent", "reflector.go/"+meta.Version())
res, err := c.httpClient.Do(req)
if err != nil {
return 0, nil, errors.Err(err)
}
return nil, trace.Stack(time.Since(start), n.Name()), errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
return res.StatusCode, res.Body, nil
}
func (n *HttpStore) Put(string, stream.Blob) error {
return shared.ErrNotImplemented
}
func (n *HttpStore) PutSD(string, stream.Blob) error {
return shared.ErrNotImplemented
}
func (n *HttpStore) Delete(string) error {
return shared.ErrNotImplemented
}
func (n *HttpStore) Shutdown() {}
// buffer pool to reduce GC
// https://www.captaincodeman.com/2017/06/02/golang-buffer-pool-gotcha
var buffers = sync.Pool{
// New is called when a new instance is needed
New: func() interface{} {
buf := make([]byte, 0, stream.MaxBlobSize)
return bytes.NewBuffer(buf)
},
func (c *HttpStore) Put(_ string, _ stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// getBuffer fetches a buffer from the pool
func getBuffer() *bytes.Buffer {
return buffers.Get().(*bytes.Buffer)
func (c *HttpStore) PutSD(_ string, _ stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// putBuffer returns a buffer to the pool
func putBuffer(buf *bytes.Buffer) {
buf.Reset()
buffers.Put(buf)
func (c *HttpStore) Delete(_ string) error {
return errors.Err(shared.ErrNotImplemented)
}
// getClient gets an http client that's customized to be more performant when dealing with blobs of 2MB in size (most of our blobs)
func getClient() *http.Client {
// Customize the Transport to have larger connection pool
defaultTransport := &http.Transport{
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).DialContext,
ForceAttemptHTTP2: true,
MaxIdleConns: 100,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
DisableCompression: true,
MaxIdleConnsPerHost: 100,
ReadBufferSize: stream.MaxBlobSize + 1024*10, //add an extra few KBs to make sure it fits the extra information
// Shutdown shuts down the store gracefully
func (c *HttpStore) Shutdown() {
}
func (c *HttpStore) shardedPath(hash string) string {
if c.prefixLength <= 0 || len(hash) < c.prefixLength {
return hash
}
return path.Join(hash[:c.prefixLength], hash)
}
return &http.Client{Transport: defaultTransport}
func init() {
RegisterStore(nameHttp, HttpStoreFactory)
}
+136
View File
@@ -0,0 +1,136 @@
package store
import (
"strings"
"sync"
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
)
// Http3Store is a blob store that gets blobs from a peer over HTTP3.
// It satisfies the BlobStore interface but cannot put or delete blobs.
type Http3Store struct {
NotFoundCache *sync.Map
client *Http3Client
name string
address string
timeout time.Duration
clientMu sync.RWMutex
}
// Http3Params allows to set options for a new Http3Store.
type Http3Params struct {
Name string `mapstructure:"name"`
Address string `mapstructure:"address"`
Timeout time.Duration `mapstructure:"timeout"`
}
// NewHttp3Store makes a new HTTP3 store.
func NewHttp3Store(params Http3Params) *Http3Store {
return &Http3Store{
name: params.Name,
NotFoundCache: &sync.Map{},
address: params.Address,
timeout: params.Timeout,
}
}
const nameHttp3 = "http3"
func Http3StoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg Http3Params
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
return NewHttp3Store(cfg), nil
}
func init() {
RegisterStore(nameHttp3, Http3StoreFactory)
}
func (h *Http3Store) Name() string { return nameHttp3 + "-" + h.name }
func (h *Http3Store) getClient() (*Http3Client, error) {
h.clientMu.RLock()
if h.client != nil {
client := h.client
h.clientMu.RUnlock()
return client, nil
}
h.clientMu.RUnlock()
h.clientMu.Lock()
defer h.clientMu.Unlock()
// Check again in case another goroutine created the client
if h.client != nil {
return h.client, nil
}
client, err := NewHttp3Client(h.address)
if err != nil {
return nil, err
}
h.client = client
return client, nil
}
// Has asks the peer if they have a hash
func (h *Http3Store) Has(hash string) (bool, error) {
c, err := h.getClient()
if err != nil {
return false, err
}
return c.HasBlob(hash)
}
// Get downloads the blob from the peer
func (h *Http3Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
if lastChecked, ok := h.NotFoundCache.Load(hash); ok {
if lastChecked.(time.Time).After(time.Now().Add(-5 * time.Minute)) {
return nil, shared.NewBlobTrace(time.Since(start), h.Name()+"-notfoundcache"), ErrBlobNotFound
}
}
c, err := h.getClient()
if err != nil && strings.Contains(err.Error(), "blob not found") {
h.NotFoundCache.Store(hash, time.Now())
}
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), h.Name()), err
}
return c.GetBlob(hash)
}
// Put is not supported
func (h *Http3Store) Put(hash string, blob stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// PutSD is not supported
func (h *Http3Store) PutSD(hash string, blob stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// Delete is not supported
func (h *Http3Store) Delete(hash string) error {
return errors.Err(shared.ErrNotImplemented)
}
// Shutdown shuts down the store gracefully
func (h *Http3Store) Shutdown() {
h.clientMu.Lock()
defer h.clientMu.Unlock()
if h.client != nil {
_ = h.client.Close()
h.client = nil
}
}
+137
View File
@@ -0,0 +1,137 @@
package store
import (
"crypto/tls"
"crypto/x509"
"io"
"net/http"
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/quic-go/quic-go"
"github.com/quic-go/quic-go/http3"
)
// Http3Client is a client for HTTP3 blob store
type Http3Client struct {
conn *http.Client
roundTripper *http3.Transport
ServerAddr string
}
// NewHttp3Client creates a new HTTP3 client
func NewHttp3Client(address string) (*Http3Client, error) {
var qconf quic.Config
window500M := 500 * 1 << 20
qconf.MaxStreamReceiveWindow = uint64(window500M)
qconf.MaxConnectionReceiveWindow = uint64(window500M)
qconf.EnableDatagrams = true
qconf.HandshakeIdleTimeout = 4 * time.Second
qconf.MaxIdleTimeout = 20 * time.Second
pool, err := x509.SystemCertPool()
if err != nil {
return nil, err
}
roundTripper := &http3.Transport{
TLSClientConfig: &tls.Config{
RootCAs: pool,
InsecureSkipVerify: true,
},
QUICConfig: &qconf,
}
connection := &http.Client{
Transport: roundTripper,
}
return &Http3Client{
conn: connection,
roundTripper: roundTripper,
ServerAddr: address,
}, nil
}
// Close closes the client
func (c *Http3Client) Close() error {
return nil
}
// HasBlob checks if the peer has a blob
func (c *Http3Client) HasBlob(hash string) (bool, error) {
url := c.ServerAddr + "/blob?hash=" + hash
req, err := http.NewRequest("HEAD", url, nil)
if err != nil {
return false, errors.Err(err)
}
res, err := c.conn.Do(req)
if err != nil {
return false, errors.Err(err)
}
defer func() { _ = res.Body.Close() }()
if res.StatusCode == http.StatusNotFound {
return false, nil
}
if res.StatusCode == http.StatusNoContent {
return true, nil
}
var body []byte
if res.Body != nil {
body, _ = io.ReadAll(res.Body)
}
return false, errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
}
// GetBlob gets a blob from the peer
func (c *Http3Client) GetBlob(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
url := c.ServerAddr + "/blob?hash=" + hash
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err(err)
}
res, err := c.conn.Do(req)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err(err)
}
defer func() { _ = res.Body.Close() }()
viaHeader := res.Header.Get("Via")
var trace shared.BlobTrace
if viaHeader != "" {
parsedTrace, err := shared.Deserialize(viaHeader)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "http3"), err
}
trace = *parsedTrace
} else {
trace = shared.NewBlobTrace(0, "http3")
}
switch res.StatusCode {
case http.StatusNotFound:
return nil, trace.Stack(time.Since(start), "http3"), ErrBlobNotFound
case http.StatusOK:
buffer := getBuffer()
defer putBuffer(buffer)
if _, err := io.Copy(buffer, res.Body); err != nil {
return nil, trace.Stack(time.Since(start), "http3"), errors.Err(err)
}
blob := make([]byte, buffer.Len())
copy(blob, buffer.Bytes())
return blob, trace.Stack(time.Since(start), "http3"), nil
default:
body, _ := io.ReadAll(res.Body)
return nil, trace.Stack(time.Since(start), "http3"),
errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
}
}
+70 -7
View File
@@ -1,6 +1,7 @@
package store
import (
"strings"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
@@ -8,25 +9,84 @@ import (
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
)
// ITTTStore performs an operation on this storage, if this fails, it attempts to run it on that
type ITTTStore struct {
this, that BlobStore
name string
}
type ITTTParams struct {
This BlobStore `mapstructure:"this"`
That BlobStore `mapstructure:"that"`
Name string `mapstructure:"name"`
}
type ITTTConfig struct {
This *viper.Viper
That *viper.Viper
Name string `mapstructure:"name"`
}
// NewITTTStore returns a new instance of the IF THIS THAN THAT store
func NewITTTStore(this, that BlobStore) *ITTTStore {
func NewITTTStore(params ITTTParams) *ITTTStore {
return &ITTTStore{
this: this,
that: that,
this: params.This,
that: params.That,
name: params.Name,
}
}
const nameIttt = "ittt"
func ITTTStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg ITTTConfig
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
cfg.This = config.Sub("this")
cfg.That = config.Sub("that")
thisStoreType := strings.Split(cfg.This.AllKeys()[0], ".")[0]
thisStoreConfig := cfg.This.Sub(thisStoreType)
factory, ok := Factories[thisStoreType]
if !ok {
return nil, errors.Err("unknown store type %s", thisStoreType)
}
thisStore, err := factory(thisStoreConfig)
if err != nil {
return nil, errors.Err(err)
}
thatStoreType := strings.Split(cfg.That.AllKeys()[0], ".")[0]
thatStoreConfig := cfg.That.Sub(thatStoreType)
factory, ok = Factories[thatStoreType]
if !ok {
return nil, errors.Err("unknown store type %s", thatStoreType)
}
thatStore, err := factory(thatStoreConfig)
if err != nil {
return nil, errors.Err(err)
}
return NewITTTStore(ITTTParams{
Name: cfg.Name,
This: thisStore,
That: thatStore,
}), nil
}
func init() {
RegisterStore(nameIttt, ITTTStoreFactory)
}
// Name is the cache type name
func (c *ITTTStore) Name() string { return nameIttt }
func (c *ITTTStore) Name() string { return nameIttt + "-" + c.name }
// Has checks in this for a hash, if it fails it checks in that. It returns true if either store has it.
func (c *ITTTStore) Has(hash string) (bool, error) {
@@ -42,7 +102,7 @@ func (c *ITTTStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
blob, trace, err := c.this.Get(hash)
if err == nil {
metrics.ThisHitCount.Inc()
metrics.ItttHitCount.With(metrics.ItttLabels(c.this.Name())).Inc()
return blob, trace.Stack(time.Since(start), c.Name()), err
}
@@ -50,7 +110,7 @@ func (c *ITTTStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
if err != nil {
return nil, trace.Stack(time.Since(start), c.Name()), err
}
metrics.ThatHitCount.Inc()
metrics.ItttHitCount.With(metrics.ItttLabels(c.that.Name())).Inc()
return blob, trace.Stack(time.Since(start), c.Name()), nil
}
@@ -70,4 +130,7 @@ func (c *ITTTStore) Delete(hash string) error {
}
// Shutdown shuts down the store gracefully
func (c *ITTTStore) Shutdown() {}
func (c *ITTTStore) Shutdown() {
c.this.Shutdown()
c.that.Shutdown()
}
+23 -2
View File
@@ -8,25 +8,46 @@ import (
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
)
// MemStore is an in memory only blob store with no persistence.
type MemStore struct {
blobs map[string]stream.Blob
mu *sync.RWMutex
name string
}
func NewMemStore() *MemStore {
type MemParams struct {
Name string `mapstructure:"name"`
}
func NewMemStore(params MemParams) *MemStore {
return &MemStore{
blobs: make(map[string]stream.Blob),
mu: &sync.RWMutex{},
name: params.Name,
}
}
const nameMem = "mem"
func MemStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg MemParams
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
return NewMemStore(cfg), nil
}
func init() {
RegisterStore(nameMem, MemStoreFactory)
}
// Name is the cache type name
func (m *MemStore) Name() string { return nameMem }
func (m *MemStore) Name() string { return nameMem + "-" + m.name }
// Has returns T/F if the blob is currently stored. It will never error.
func (m *MemStore) Has(hash string) (bool, error) {
+2 -2
View File
@@ -8,7 +8,7 @@ import (
)
func TestMemStore_Put(t *testing.T) {
s := NewMemStore()
s := NewMemStore(MemParams{Name: "test"})
blob := []byte("abcdefg")
err := s.Put("abc", blob)
if err != nil {
@@ -17,7 +17,7 @@ func TestMemStore_Put(t *testing.T) {
}
func TestMemStore_Get(t *testing.T) {
s := NewMemStore()
s := NewMemStore(MemParams{Name: "test"})
hash := "abc"
blob := []byte("abcdefg")
err := s.Put(hash, blob)
+206
View File
@@ -0,0 +1,206 @@
package store
import (
"strings"
"sync"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
)
// MultiWriterStore writes to multiple destination stores
type MultiWriterStore struct {
name string
destinations []BlobStore
}
type MultiWriterParams struct {
Name string
Destinations []BlobStore
}
type MultiWriterConfig struct {
Name string `mapstructure:"name"`
One viper.Viper
Two viper.Viper
Three viper.Viper
}
// NewMultiWriterStore returns a new instance of the MultiWriter store
func NewMultiWriterStore(params MultiWriterParams) *MultiWriterStore {
return &MultiWriterStore{
name: params.Name,
destinations: params.Destinations,
}
}
const nameMultiWriter = "multiwriter"
func MultiWriterStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg MultiWriterConfig
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
var destinations []BlobStore
one := config.Sub("one")
two := config.Sub("two")
//three := config.Sub("three")
storeTypeOne := strings.Split(one.AllKeys()[0], ".")[0]
storeTypeTwo := strings.Split(two.AllKeys()[0], ".")[0]
//storeTypeThree := strings.Split(three.AllKeys()[0], ".")[0]
storeCfgOne := one.Sub(storeTypeOne)
storeCfgTwo := two.Sub(storeTypeTwo)
//storeCfgThree := config.Sub(storeTypeThree)
factoryOne, ok := Factories[storeTypeOne]
if !ok {
return nil, errors.Err("unknown store type %s", storeTypeOne)
}
factoryTwo, ok := Factories[storeTypeTwo]
if !ok {
return nil, errors.Err("unknown store type %s", storeTypeTwo)
}
//factoryThree, ok := Factories[storeTypeThree]
//if !ok {
// return nil, errors.Err("unknown store type %s", storeTypeThree)
//}
store1, err := factoryOne(storeCfgOne)
if err != nil {
return nil, errors.Err(err)
}
store2, err := factoryTwo(storeCfgTwo)
if err != nil {
return nil, errors.Err(err)
}
//store3, err := factoryThree(storeCfgThree)
//if err != nil {
// return nil, errors.Err(err)
//}
destinations = append(destinations, store1, store2)
return NewMultiWriterStore(MultiWriterParams{
Name: cfg.Name,
Destinations: destinations,
}), nil
}
func init() {
RegisterStore(nameMultiWriter, MultiWriterStoreFactory)
}
// Name returns the store name
func (m *MultiWriterStore) Name() string { return nameMultiWriter + "-" + m.name }
// Has is not supported by MultiWriter
func (m *MultiWriterStore) Has(hash string) (bool, error) {
return false, errors.Err(shared.ErrNotImplemented)
}
// Get is not supported by MultiWriter
func (m *MultiWriterStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
return nil, shared.BlobTrace{}, errors.Err(shared.ErrNotImplemented)
}
// Put writes the blob to all destination stores
func (m *MultiWriterStore) Put(hash string, blob stream.Blob) error {
var wg sync.WaitGroup
errChan := make(chan error, len(m.destinations))
for _, dest := range m.destinations {
wg.Add(1)
go func(d BlobStore) {
defer wg.Done()
if err := d.Put(hash, blob); err != nil {
errChan <- errors.Err("failed to write to %s: %v", d.Name(), err)
}
}(dest)
}
wg.Wait()
close(errChan)
var errs []string
for err := range errChan {
errs = append(errs, err.Error())
}
if len(errs) > 0 {
return errors.Err("failed to write to some destinations: %s", strings.Join(errs, "; "))
}
return nil
}
// PutSD writes the SD blob to all destination stores
func (m *MultiWriterStore) PutSD(hash string, blob stream.Blob) error {
var wg sync.WaitGroup
errChan := make(chan error, len(m.destinations))
for _, dest := range m.destinations {
wg.Add(1)
go func(d BlobStore) {
defer wg.Done()
if err := d.PutSD(hash, blob); err != nil {
errChan <- errors.Err("failed to write SD to %s: %v", d.Name(), err)
}
}(dest)
}
wg.Wait()
close(errChan)
var errs []string
for err := range errChan {
errs = append(errs, err.Error())
}
if len(errs) > 0 {
return errors.Err("failed to write SD to some destinations: %s", strings.Join(errs, "; "))
}
return nil
}
// Delete deletes the blob from all destination stores
func (m *MultiWriterStore) Delete(hash string) error {
var wg sync.WaitGroup
errChan := make(chan error, len(m.destinations))
for _, dest := range m.destinations {
wg.Add(1)
go func(d BlobStore) {
defer wg.Done()
if err := d.Delete(hash); err != nil {
errChan <- errors.Err("failed to delete from %s: %v", d.Name(), err)
}
}(dest)
}
wg.Wait()
close(errChan)
var errs []string
for err := range errChan {
errs = append(errs, err.Error())
}
if len(errs) > 0 {
return errors.Err("failed to delete from some destinations: %s", strings.Join(errs, "; "))
}
return nil
}
// Shutdown shuts down all destination stores gracefully
func (m *MultiWriterStore) Shutdown() {
for _, dest := range m.destinations {
dest.Shutdown()
}
}
+27 -3
View File
@@ -5,15 +5,39 @@ import (
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
)
// NoopStore is a store that does nothing
type NoopStore struct{}
type NoopStore struct {
name string
}
func NewNoopStore(name string) *NoopStore {
return &NoopStore{name: name}
}
const nameNoop = "noop"
func (n *NoopStore) Name() string { return nameNoop }
func NoopStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg struct {
Name string `mapstructure:"name"`
}
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
return NewNoopStore(cfg.Name), nil
}
func init() {
RegisterStore(nameNoop, NoopStoreFactory)
}
func (n *NoopStore) Name() string { return nameNoop + "-" + n.name }
func (n *NoopStore) Has(_ string) (bool, error) { return false, nil }
func (n *NoopStore) Get(_ string) (stream.Blob, shared.BlobTrace, error) {
return nil, shared.NewBlobTrace(time.Since(time.Now()), n.Name()), nil
@@ -21,4 +45,4 @@ func (n *NoopStore) Get(_ string) (stream.Blob, shared.BlobTrace, error) {
func (n *NoopStore) Put(_ string, _ stream.Blob) error { return nil }
func (n *NoopStore) PutSD(_ string, _ stream.Blob) error { return nil }
func (n *NoopStore) Delete(_ string) error { return nil }
func (n *NoopStore) Shutdown() { return }
func (n *NoopStore) Shutdown() {}
+100
View File
@@ -0,0 +1,100 @@
package store
import (
"strings"
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
)
// PeerStore is a blob store that gets blobs from a peer.
// It satisfies the BlobStore interface but cannot put or delete blobs.
type PeerStore struct {
name string
opts PeerParams
}
// PeerParams allows to set options for a new PeerStore.
type PeerParams struct {
Name string `mapstructure:"name"`
Address string `mapstructure:"address"`
Timeout time.Duration `mapstructure:"timeout"`
}
// NewPeerStore makes a new peer store.
func NewPeerStore(params PeerParams) *PeerStore {
return &PeerStore{opts: params, name: params.Name}
}
const namePeer = "peer"
func PeerStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg PeerParams
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
return NewPeerStore(cfg), nil
}
func init() {
RegisterStore(namePeer, PeerStoreFactory)
}
func (p *PeerStore) Name() string { return namePeer + "-" + p.name }
func (p *PeerStore) getClient() (*PeerClient, error) {
c := &PeerClient{Timeout: p.opts.Timeout}
err := c.Connect(p.opts.Address)
return c, errors.Prefix("connection error", err)
}
// Has asks the peer if they have a hash
func (p *PeerStore) Has(hash string) (bool, error) {
c, err := p.getClient()
if err != nil {
return false, err
}
defer func() { _ = c.Close() }()
return c.HasBlob(hash)
}
// Get downloads the blob from the peer
func (p *PeerStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
c, err := p.getClient()
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), p.Name()), err
}
defer func() { _ = c.Close() }()
blob, trace, err := c.GetBlob(hash)
if err != nil && strings.Contains(err.Error(), "blob not found") {
return nil, trace, ErrBlobNotFound
}
return blob, trace, err
}
// Put is not supported
func (p *PeerStore) Put(hash string, blob stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// PutSD is not supported
func (p *PeerStore) PutSD(hash string, blob stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// Delete is not supported
func (p *PeerStore) Delete(hash string) error {
return errors.Err(shared.ErrNotImplemented)
}
// Shutdown is not supported
func (p *PeerStore) Shutdown() {
}
+95
View File
@@ -0,0 +1,95 @@
package store
import (
"bufio"
"bytes"
"encoding/binary"
"io"
"net"
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/stream"
)
// PeerClient is a client for peer blob store
type PeerClient struct {
conn net.Conn
Timeout time.Duration
}
// Connect connects to a peer
func (c *PeerClient) Connect(address string) error {
var err error
c.conn, err = net.DialTimeout("tcp", address, c.Timeout)
return err
}
// Close closes the connection
func (c *PeerClient) Close() error {
if c.conn != nil {
return c.conn.Close()
}
return nil
}
// HasBlob checks if the peer has a blob
func (c *PeerClient) HasBlob(hash string) (bool, error) {
err := c.writeRequest("has", hash)
if err != nil {
return false, err
}
response, err := c.readResponse()
if err != nil {
return false, err
}
return response == "yes", nil
}
// GetBlob gets a blob from the peer
func (c *PeerClient) GetBlob(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
err := c.writeRequest("get", hash)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "peer"), err
}
response, err := c.readResponse()
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "peer"), err
}
if response == "no" {
return nil, shared.NewBlobTrace(time.Since(start), "peer"), ErrBlobNotFound
}
size, err := binary.ReadVarint(bufio.NewReader(bytes.NewReader([]byte(response))))
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "peer"), err
}
blob := make([]byte, size)
_, err = io.ReadFull(c.conn, blob)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "peer"), err
}
return blob, shared.NewBlobTrace(time.Since(start), "peer"), nil
}
func (c *PeerClient) writeRequest(cmd, hash string) error {
_, err := c.conn.Write([]byte(cmd + " " + hash + "\n"))
return err
}
func (c *PeerClient) readResponse() (string, error) {
reader := bufio.NewReader(c.conn)
response, err := reader.ReadString('\n')
if err != nil {
return "", err
}
return response[:len(response)-1], nil
}
+147
View File
@@ -0,0 +1,147 @@
package store
import (
"strings"
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
)
// ProxiedS3Store writes to an S3 store and reads from any BlobStore (usually an ITTTStore of HttpStore endpoints).
type ProxiedS3Store struct {
readerStore BlobStore
writerStore BlobStore
name string
}
func (c *ProxiedS3Store) MissingBlobsForKnownStream(s string) ([]string, error) {
if bc, ok := c.writerStore.(NeededBlobChecker); ok {
return bc.MissingBlobsForKnownStream(s)
}
return nil, errors.Err("writer does not implement neededBlobChecker")
}
func (c *ProxiedS3Store) Block(hash string) error {
if bl, ok := c.writerStore.(Blocklister); ok {
return bl.Block(hash)
}
return errors.Err("writer does not implement Blocklister")
}
func (c *ProxiedS3Store) Wants(hash string) (bool, error) {
if bl, ok := c.writerStore.(Blocklister); ok {
return bl.Wants(hash)
}
return true, errors.Err("writer does not implement Blocklister")
}
type ProxiedS3Params struct {
Reader BlobStore `mapstructure:"reader"`
Writer BlobStore `mapstructure:"writer"`
Name string `mapstructure:"name"`
}
type ProxiedS3Config struct {
Reader *viper.Viper
Writer *viper.Viper
Name string `mapstructure:"name"`
}
// NewProxiedS3Store returns an initialized ProxiedS3Store store pointer.
// NOTE: It panics if either argument is nil.
func NewProxiedS3Store(params ProxiedS3Params) *ProxiedS3Store {
if params.Reader == nil || params.Writer == nil {
panic("both stores must be set")
}
return &ProxiedS3Store{
readerStore: params.Reader,
writerStore: params.Writer,
name: params.Name,
}
}
const nameProxiedS3 = "proxied-s3"
// Name is the cache type name
func (c *ProxiedS3Store) Name() string { return nameProxiedS3 + "-" + c.name }
// Has checks if the hash is in the store.
func (c *ProxiedS3Store) Has(hash string) (bool, error) {
return c.writerStore.Has(hash)
}
// Get gets the blob from Cloudfront.
func (c *ProxiedS3Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
blob, trace, err := c.readerStore.Get(hash)
return blob, trace.Stack(time.Since(start), c.Name()), err
}
// Put stores the blob on S3
func (c *ProxiedS3Store) Put(hash string, blob stream.Blob) error {
return c.writerStore.Put(hash, blob)
}
// PutSD stores the sd blob on S3
func (c *ProxiedS3Store) PutSD(hash string, blob stream.Blob) error {
return c.writerStore.PutSD(hash, blob)
}
// Delete deletes the blob from S3
func (c *ProxiedS3Store) Delete(hash string) error {
return c.writerStore.Delete(hash)
}
// Shutdown shuts down the store gracefully
func (c *ProxiedS3Store) Shutdown() {
c.writerStore.Shutdown()
c.readerStore.Shutdown()
}
func ProxiedS3StoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg ProxiedS3Config
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
cfg.Reader = config.Sub("reader")
cfg.Writer = config.Sub("writer")
readerStoreType := strings.Split(cfg.Reader.AllKeys()[0], ".")[0]
readerStoreConfig := cfg.Reader.Sub(readerStoreType)
factory, ok := Factories[readerStoreType]
if !ok {
return nil, errors.Err("unknown store type %s", readerStoreType)
}
readerStore, err := factory(readerStoreConfig)
if err != nil {
return nil, errors.Err(err)
}
writerStoreType := strings.Split(cfg.Writer.AllKeys()[0], ".")[0]
writerStoreConfig := cfg.Writer.Sub(writerStoreType)
factory, ok = Factories[writerStoreType]
if !ok {
return nil, errors.Err("unknown store type %s", writerStoreType)
}
writerStore, err := factory(writerStoreConfig)
if err != nil {
return nil, errors.Err(err)
}
return NewProxiedS3Store(ProxiedS3Params{
Name: cfg.Name,
Reader: readerStore,
Writer: writerStore,
}), nil
}
func init() {
RegisterStore(nameProxiedS3, ProxiedS3StoreFactory)
}
+103 -55
View File
@@ -2,7 +2,9 @@ package store
import (
"bytes"
"net/http"
"context"
stderrors "errors"
"path"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
@@ -11,56 +13,88 @@ import (
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/aws/aws-sdk-go-v2/aws"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
)
// S3Store is an S3 store
type S3Store struct {
awsID string
awsSecret string
region string
bucket string
client *s3.Client
awsID string
awsSecret string
region string
bucket string
endpoint string
name string
prefixLength int
}
session *session.Session
type S3Params struct {
Name string `mapstructure:"name"`
AwsID string `mapstructure:"aws_id"`
AwsSecret string `mapstructure:"aws_secret"`
Region string `mapstructure:"region"`
Bucket string `mapstructure:"bucket"`
Endpoint string `mapstructure:"endpoint"`
ShardingSize int `mapstructure:"sharding_size"`
}
// NewS3Store returns an initialized S3 store pointer.
func NewS3Store(awsID, awsSecret, region, bucket string) *S3Store {
func NewS3Store(params S3Params) *S3Store {
return &S3Store{
awsID: awsID,
awsSecret: awsSecret,
region: region,
bucket: bucket,
awsID: params.AwsID,
awsSecret: params.AwsSecret,
region: params.Region,
bucket: params.Bucket,
endpoint: params.Endpoint,
name: params.Name,
prefixLength: params.ShardingSize,
}
}
const nameS3 = "s3"
// Name is the cache type name
func (s *S3Store) Name() string { return nameS3 }
func S3StoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg S3Params
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
return NewS3Store(cfg), nil
}
// Has returns T/F or Error ( from S3 ) if the store contains the blob.
func init() {
RegisterStore(nameS3, S3StoreFactory)
}
// Name is the cache type name
func (s *S3Store) Name() string { return nameS3 + "-" + s.name }
// Has returns T/F or Error (from S3) if the store contains the blob.
func (s *S3Store) Has(hash string) (bool, error) {
err := s.initOnce()
if err != nil {
return false, err
}
_, err = s3.New(s.session).HeadObject(&s3.HeadObjectInput{
ctx := context.Background()
_, err = s.client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(hash),
Key: aws.String(s.shardedPath(hash)),
})
if err != nil {
if reqFail, ok := err.(s3.RequestFailure); ok && reqFail.StatusCode() == http.StatusNotFound {
var notFound *types.NotFound
var noSuchKey *types.NoSuchKey
if stderrors.As(err, &notFound) || stderrors.As(err, &noSuchKey) {
return false, nil
}
return false, err
return false, errors.Err(err)
}
return true, nil
@@ -69,7 +103,6 @@ func (s *S3Store) Has(hash string) (bool, error) {
// Get returns the blob slice if present or errors on S3.
func (s *S3Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
//Todo-Need to handle error for blob doesn't exist for consistency.
err := s.initOnce()
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), s.Name()), err
@@ -77,24 +110,26 @@ func (s *S3Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
log.Debugf("Getting %s from S3", hash[:8])
defer func(t time.Time) {
log.Debugf("Getting %s from S3 took %s", hash[:8], time.Since(t).String())
log.Debugf("Getting %s from %s took %s", hash[:8], s.Name(), time.Since(t).String())
}(start)
buf := &aws.WriteAtBuffer{}
_, err = s3manager.NewDownloader(s.session).Download(buf, &s3.GetObjectInput{
ctx := context.Background()
buf := manager.NewWriteAtBuffer([]byte{})
_, err = manager.NewDownloader(s.client).Download(ctx, buf, &s3.GetObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(hash),
Key: aws.String(s.shardedPath(hash)),
})
if err != nil {
if aerr, ok := err.(awserr.Error); ok {
switch aerr.Code() {
case s3.ErrCodeNoSuchBucket:
return nil, shared.NewBlobTrace(time.Since(start), s.Name()), errors.Err("bucket %s does not exist", s.bucket)
case s3.ErrCodeNoSuchKey:
return nil, shared.NewBlobTrace(time.Since(start), s.Name()), errors.Err(ErrBlobNotFound)
}
var noSuchBucket *types.NoSuchBucket
var noSuchKey *types.NoSuchKey
var notFound *types.NotFound
switch {
case stderrors.As(err, &noSuchBucket):
return nil, shared.NewBlobTrace(time.Since(start), s.Name()), errors.Err("bucket %s does not exist", s.bucket)
case stderrors.As(err, &noSuchKey), stderrors.As(err, &notFound):
return nil, shared.NewBlobTrace(time.Since(start), s.Name()), errors.Err(ErrBlobNotFound)
}
return buf.Bytes(), shared.NewBlobTrace(time.Since(start), s.Name()), err
return nil, shared.NewBlobTrace(time.Since(start), s.Name()), errors.Err(err)
}
return buf.Bytes(), shared.NewBlobTrace(time.Since(start), s.Name()), nil
@@ -112,16 +147,18 @@ func (s *S3Store) Put(hash string, blob stream.Blob) error {
log.Debugf("Uploading %s took %s", hash[:8], time.Since(t).String())
}(time.Now())
_, err = s3manager.NewUploader(s.session).Upload(&s3manager.UploadInput{
ctx := context.Background()
_, err = manager.NewUploader(s.client).Upload(ctx, &s3.PutObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(hash),
Key: aws.String(s.shardedPath(hash)),
Body: bytes.NewBuffer(blob),
ACL: aws.String("public-read"),
//StorageClass: aws.String(s3.StorageClassIntelligentTiering),
ACL: types.ObjectCannedACLPublicRead,
})
if err != nil {
return errors.Err(err)
}
metrics.MtrOutBytesReflector.Add(float64(blob.Size()))
return err
return nil
}
// PutSD stores the sd blob on S3 or errors if S3 connection errors.
@@ -138,33 +175,44 @@ func (s *S3Store) Delete(hash string) error {
log.Debugf("Deleting %s from S3", hash[:8])
_, err = s3.New(s.session).DeleteObject(&s3.DeleteObjectInput{
ctx := context.Background()
_, err = s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(hash),
Key: aws.String(s.shardedPath(hash)),
})
return err
return errors.Err(err)
}
func (s *S3Store) initOnce() error {
if s.session != nil {
if s.client != nil {
return nil
}
sess, err := session.NewSession(&aws.Config{
Credentials: credentials.NewStaticCredentials(s.awsID, s.awsSecret, ""),
Region: aws.String(s.region),
Endpoint: aws.String("https://s3.wasabisys.com"),
})
ctx := context.Background()
cfg, err := awsconfig.LoadDefaultConfig(ctx,
awsconfig.WithRegion(s.region),
awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(s.awsID, s.awsSecret, "")),
)
if err != nil {
return err
return errors.Err(err)
}
s.session = sess
s.client = s3.NewFromConfig(cfg, func(o *s3.Options) {
o.BaseEndpoint = aws.String(s.endpoint)
o.UsePathStyle = true
o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired
})
return nil
}
func (s *S3Store) shardedPath(hash string) string {
if s.prefixLength <= 0 || len(hash) < s.prefixLength {
return hash
}
return path.Join(hash[:s.prefixLength], hash)
}
// Shutdown shuts down the store gracefully
func (s *S3Store) Shutdown() {
return
}
+35 -3
View File
@@ -1,6 +1,7 @@
package store
import (
"strings"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
@@ -9,6 +10,7 @@ import (
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
"golang.org/x/sync/singleflight"
)
@@ -22,9 +24,40 @@ func WithSingleFlight(component string, origin BlobStore) BlobStore {
type singleflightStore struct {
BlobStore
component string
sf *singleflight.Group
component string
}
type SingleFlightConfig struct {
Store *viper.Viper
Component string `mapstructure:"component"`
}
func SingleFlightStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg SingleFlightConfig
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
cfg.Store = config.Sub("store")
storeType := strings.Split(cfg.Store.AllKeys()[0], ".")[0]
storeConfig := cfg.Store.Sub(storeType)
factory, ok := Factories[storeType]
if !ok {
return nil, errors.Err("unknown store type %s", storeType)
}
underlyingStore, err := factory(storeConfig)
if err != nil {
return nil, errors.Err(err)
}
return WithSingleFlight(cfg.Component, underlyingStore), nil
}
func init() {
RegisterStore("singleflight", SingleFlightStoreFactory)
}
func (s *singleflightStore) Name() string {
@@ -124,5 +157,4 @@ func (s *singleflightStore) putter(hash string, blob stream.Blob) func() (interf
// Shutdown shuts down the store gracefully
func (s *singleflightStore) Shutdown() {
s.BlobStore.Shutdown()
return
}
+17 -3
View File
@@ -1,7 +1,8 @@
package speedwalk
import (
"io/ioutil"
"io/fs"
"os"
"path/filepath"
"runtime"
"sync"
@@ -17,7 +18,20 @@ import (
// AllFiles recursively lists every file in every subdirectory of a given directory
// If basename is true, return the basename of each file. Otherwise return the full path starting at startDir.
func AllFiles(startDir string, basename bool) ([]string, error) {
items, err := ioutil.ReadDir(startDir)
entries, err := os.ReadDir(startDir)
if err != nil {
return nil, err
}
items := make([]fs.FileInfo, 0, len(entries))
for _, entry := range entries {
var info fs.FileInfo
info, err = entry.Info()
if err != nil {
return nil, err
}
items = append(items, info)
}
if err != nil {
return nil, err
}
@@ -77,7 +91,7 @@ func AllFiles(startDir string, basename bool) ([]string, error) {
},
})
if err != nil {
logrus.Errorf(errors.FullTrace(err))
logrus.Error(errors.FullTrace(err))
}
}(item.Name())
}
+15 -1
View File
@@ -5,6 +5,8 @@ import (
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
)
// BlobStore is an interface for handling blob storage.
@@ -33,11 +35,23 @@ type Blocklister interface {
Wants(hash string) (bool, error)
}
// NeededBlobChecker can check which blobs from a known stream are not uploaded yet
type NeededBlobChecker interface {
MissingBlobsForKnownStream(string) ([]string, error)
}
// lister is a store that can list cached blobs. This is helpful when an overlay
// cache needs to track blob existence.
type lister interface {
list() ([]string, error)
}
//ErrBlobNotFound is a standard error when a blob is not found in the store.
// ErrBlobNotFound is a standard error when a blob is not found in the store.
var ErrBlobNotFound = errors.Base("blob not found")
var Factories = make(map[string]Factory)
func RegisterStore(name string, factory Factory) {
Factories[name] = factory
}
type Factory func(config *viper.Viper) (BlobStore, error)
+204
View File
@@ -0,0 +1,204 @@
package store
import (
"bytes"
"context"
"io"
"net"
"net/http"
"sync"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
)
// UpstreamStore is a store that works on top of the HTTP protocol
type UpstreamStore struct {
upstream string
httpClient *http.Client
edgeToken string
name string
}
type UpstreamParams struct {
Name string `mapstructure:"name"`
Upstream string `mapstructure:"upstream"`
EdgeToken string `mapstructure:"edge_token"`
}
func NewUpstreamStore(params UpstreamParams) *UpstreamStore {
return &UpstreamStore{
upstream: params.Upstream,
httpClient: getClient(),
edgeToken: params.EdgeToken,
name: params.Name,
}
}
const nameUpstream = "upstream"
func UpstreamStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg UpstreamParams
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
return NewUpstreamStore(cfg), nil
}
func init() {
RegisterStore(nameUpstream, UpstreamStoreFactory)
}
func (n *UpstreamStore) Name() string { return nameUpstream + "-" + n.name }
func (n *UpstreamStore) Has(hash string) (bool, error) {
url := n.upstream + "/blob?hash=" + hash
req, err := http.NewRequest("HEAD", url, nil)
if err != nil {
return false, errors.Err(err)
}
res, err := n.httpClient.Do(req)
if err != nil {
return false, errors.Err(err)
}
defer func() { _ = res.Body.Close() }()
if res.StatusCode == http.StatusNotFound {
return false, nil
}
if res.StatusCode == http.StatusNoContent {
return true, nil
}
var body []byte
if res.Body != nil {
body, _ = io.ReadAll(res.Body)
}
return false, errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
}
func (n *UpstreamStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
url := n.upstream + "/blob?hash=" + hash
if n.edgeToken != "" {
url += "&edge_token=" + n.edgeToken
}
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), n.Name()), errors.Err(err)
}
res, err := n.httpClient.Do(req)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), n.Name()), errors.Err(err)
}
defer func(Body io.ReadCloser) {
err := Body.Close()
if err != nil {
log.Errorf("Error closing response body in HTTP-GET: %s", err.Error())
}
}(res.Body)
viaHeader := res.Header.Get("Via")
var trace shared.BlobTrace
if viaHeader != "" {
parsedTrace, err := shared.Deserialize(viaHeader)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), n.Name()), err
}
trace = *parsedTrace
} else {
trace = shared.NewBlobTrace(0, n.Name())
}
switch res.StatusCode {
case http.StatusNotFound:
return nil, trace.Stack(time.Since(start), n.Name()), ErrBlobNotFound
case http.StatusOK:
buffer := getBuffer()
defer putBuffer(buffer)
if _, err := io.Copy(buffer, res.Body); err != nil {
return nil, trace.Stack(time.Since(start), n.Name()), errors.Err(err)
}
blob := make([]byte, buffer.Len())
copy(blob, buffer.Bytes())
metrics.MtrInBytesUpstream.Add(float64(len(blob)))
return blob, trace.Stack(time.Since(start), n.Name()), nil
default:
body, _ := io.ReadAll(res.Body)
log.Warnf("Got status code %d (%s)", res.StatusCode, string(body))
return nil, trace.Stack(time.Since(start), n.Name()),
errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
}
}
func (n *UpstreamStore) Put(string, stream.Blob) error {
return shared.ErrNotImplemented
}
func (n *UpstreamStore) PutSD(string, stream.Blob) error {
return shared.ErrNotImplemented
}
func (n *UpstreamStore) Delete(string) error {
return shared.ErrNotImplemented
}
func (n *UpstreamStore) Shutdown() {}
// buffer pool to reduce GC
// https://www.captaincodeman.com/2017/06/02/golang-buffer-pool-gotcha
var buffers = sync.Pool{
// New is called when a new instance is needed
New: func() interface{} {
buf := make([]byte, 0, stream.MaxBlobSize)
return bytes.NewBuffer(buf)
},
}
// getBuffer fetches a buffer from the pool
func getBuffer() *bytes.Buffer {
return buffers.Get().(*bytes.Buffer)
}
// putBuffer returns a buffer to the pool
func putBuffer(buf *bytes.Buffer) {
buf.Reset()
buffers.Put(buf)
}
func dialContext(ctx context.Context, network, address string) (net.Conn, error) {
dialer := &net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}
return dialer.DialContext(ctx, network, address)
}
// getClient gets an http client that's customized to be more performant when dealing with blobs of 2MB in size (most of our blobs)
func getClient() *http.Client {
// Customize the Transport to have larger connection pool
defaultTransport := &http.Transport{
DialContext: dialContext,
ForceAttemptHTTP2: true,
MaxIdleConns: 100,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
DisableCompression: true,
MaxIdleConnsPerHost: 100,
ReadBufferSize: stream.MaxBlobSize + 1024*10, //add an extra few KBs to make sure it fits the extra information
}
return &http.Client{Transport: defaultTransport}
}
+4 -4
View File
@@ -69,22 +69,22 @@ func (n *Node) Resolve(url string) (*types.Output, error) {
type GetClaimsInTxResp struct {
Jsonrpc string `json:"jsonrpc"`
ID int `json:"id"`
Result []struct {
Name string `json:"name"`
ClaimID string `json:"claim_id"`
Txid string `json:"txid"`
Value string `json:"value"`
Address string `json:"address"`
Supports []interface{} `json:"supports"` // TODO: finish me
Nout int `json:"nout"`
Amount int `json:"amount"`
Depth int `json:"depth"`
Height int `json:"height"`
Value string `json:"value"`
ClaimSequence int `json:"claim_sequence"`
Address string `json:"address"`
Supports []interface{} `json:"supports"` // TODO: finish me
EffectiveAmount int `json:"effective_amount"`
ValidAtHeight int `json:"valid_at_height"`
} `json:"result"`
ID int `json:"id"`
}
func (n *Node) GetClaimsInTx(txid string) (*GetClaimsInTxResp, error) {
+13 -16
View File
@@ -30,22 +30,19 @@ var (
)
type response struct {
data []byte
err error
data []byte
}
type Node struct {
transport *TCPTransport
nextId atomic.Uint32
grp *stop.Group
handlersMu *sync.RWMutex
handlers map[uint32]chan response
transport *TCPTransport
grp *stop.Group
handlersMu *sync.RWMutex
handlers map[uint32]chan response
pushHandlersMu *sync.RWMutex
pushHandlers map[string][]chan response
timeout time.Duration
nextId atomic.Uint32
timeout time.Duration
}
// NewNode creates a new node.
@@ -155,23 +152,23 @@ func (n *Node) listen() {
return
case bytes := <-n.transport.Responses():
msg := &struct {
Id uint32 `json:"id"`
Method string `json:"method"`
Error struct {
Code int `json:"code"`
Message string `json:"message"`
Code int `json:"code"`
} `json:"error"`
Id uint32 `json:"id"`
}{}
msg2 := &struct {
Id uint32 `json:"id"`
Method string `json:"method"`
Error struct {
Code int `json:"code"`
Message struct {
Code int `json:"code"`
Message string `json:"message"`
Code int `json:"code"`
} `json:"message"`
Code int `json:"code"`
} `json:"error"`
Id uint32 `json:"id"`
}{}
r := response{}
@@ -231,9 +228,9 @@ func (n *Node) listen() {
// request makes a request to the server and unmarshals the response into v.
func (n *Node) request(method string, params []string, v interface{}) error {
msg := struct {
Id uint32 `json:"id"`
Method string `json:"method"`
Params []string `json:"params"`
Id uint32 `json:"id"`
}{
Id: n.nextId.Load(),
Method: method,
+1 -1
View File
@@ -124,7 +124,7 @@ func (t *TCPTransport) test() error {
return errors.Err(err)
}
if response.Error.Message != "" {
return fmt.Errorf(response.Error.Message)
return fmt.Errorf("%s", response.Error.Message)
}
return nil
}