52 Commits
Author SHA1 Message Date
Alex Grintsvayg 2a1557845d wrap blob insertion in tx. fixes lbryio/lbry-sdk#3296
The problem is that inserting an sd blob with ~5k
blobs takes longer than 30 seconds. So the client
times out and retries the request. At that point,
reflector is not done inserting so it replies with
a smaller number of blobs than it should. The client
uploads that many blobs and marks the stream as
reflected. The remaining blobs never get uploaded.

Doing the insert inside a transaction should be
faster than doing 10k (2 per blob) inserts
independently.
2021-05-24 10:40:31 -04:00
Niko Storni 9670bc14f8 fix unsafe dereference 2021-05-21 21:06:59 +02:00
Niko Storni df881e16b5 add metrics 2021-05-21 19:09:02 +02:00
Niko Storni bc889001bb update lbry.go dep 2021-05-21 18:05:37 +02:00
Niko Storni 1ec2184833 upgrade singleflight
http store fix
2021-05-21 17:58:33 +02:00
Niko Storni 5cc1e84adb remove locks causing deadlocks 2021-05-21 05:53:13 +02:00
Niko Storni a7086a00f3 add http server/client 2021-05-21 05:49:02 +02:00
Mark Beamer Jr 76ece1e117 Add queue to prevent writing too many files at once. 2021-05-20 20:43:01 -04:00
Mark Beamer Jr c1caf1938c Add queue to prevent writing too many files at once. 2021-05-20 20:41:47 -04:00
Mark Beamer Jr 213d21b021 Add locks to disk store. 2021-05-20 19:59:50 -04:00
Niko Storni 006b04f6e9 add a lot of extra heavy debugging 2021-05-21 01:48:46 +02:00
Mark Beamer Jr 45130499cd Add single flight for cache not just origin 2021-05-20 19:05:48 -04:00
Mark Beamer Jr 4ecce75e23 add metric calls for other packages 2021-05-20 18:12:30 -04:00
Mark Beamer Jr 0152300d8d add guage metrics for go routines in reflector package 2021-05-20 18:01:13 -04:00
Mark Beamer Jr eafc62f2a6 add gops to reflector server 2021-05-20 17:21:35 -04:00
Niko Storni 50c077a9cb request queue size param 2021-05-20 23:17:18 +02:00
Niko Storni 070938e12a increase window size 2021-05-06 22:53:18 +02:00
Niko Storni c4084eeb68 improve disk cleanup
add index to is_stored
fix test
replace LRU cache
2021-04-29 03:41:18 +02:00
Niko Storni 4392c97242 fix mess with lbry.go 2021-04-13 00:52:56 +02:00
Niko Storni ec3aae33ba add if this than that store
switch to wasabi for uploads
2021-04-12 23:05:50 +02:00
Niko Storni dc95351cf3 add integrity check cmd
throttle live integrity checks
bug fixes
2021-04-07 04:46:18 +02:00
Mark Beamer Jr 25a7fac4f0 use wait group not stopper 2021-04-06 14:28:29 -04:00
Mark Beamer Jr b97595311f Wait for request to be handled before returning 2021-04-06 14:21:05 -04:00
Mark Beamer Jr bd13836897 Add request queue for blob cache 2021-04-06 14:00:36 -04:00
Niko Storni 38b44218f2 check blobs when reading them 2021-04-05 23:34:45 +02:00
Niko Storni 90c36fbe24 upgrade quic-go
add cache for blobs not found
2021-03-31 04:53:27 +02:00
Niko Storni 3a441aed3a fix issues caused by beamer's renaming 2021-03-29 19:56:18 +02:00
Niko Storni ebb62d0a24 run go mod tidy 2021-03-29 19:44:27 +02:00
Niko Storni 8cb7389619 make it simpler 2021-02-23 15:23:46 +01:00
Niko Storni 7b49dd115b remove panics 2021-02-23 15:08:32 +01:00
Niko Storni 6291e33ee1 add tracing to blobs 2021-01-14 20:38:04 +01:00
Niko Storni 3e475e537b optimize batch insertions
reduce touch time to every 6 hours
2021-01-07 01:28:34 +01:00
Alex Grintsvayg c4504631bc avoid heavy interpolateparams call 2021-01-06 10:43:35 -05:00
Alex Grintsvayg cc504e6c44 fix long query 2021-01-05 12:16:44 -05:00
Alex Grintsvayg 49714c02a6 only touch blobs when you get them 2021-01-05 11:36:33 -05:00
Niko Storni b33651ae26 save uploaded blobs and work around the blocklist issue 2021-01-05 05:09:55 +01:00
Niko Storni 0d5004a83b add cmd to populate db
fix store init
try fixing unreasonable db bottleneck
2020-12-30 04:24:11 +01:00
Niko Storni 04f6859c74 Merge branch 'grin' into litedb 2020-12-24 23:13:31 +01:00
Alex Grintsvayg 3a1d9d3304 something like this 2020-12-23 17:08:13 -05:00
Niko Storni 03304312e8 add PoC for litedb to avoid all the overhead 2020-12-23 06:04:42 +01:00
Niko Storni 869030fc58 address some review comments 2020-12-22 21:19:48 +01:00
Niko Storni def551cc89 add option to run with RO-CF only as upstream
increase idle timeout to avoid errors downstream
add option to delete blobs from DB if storage doesn't have it (for future local tracking)
2020-12-22 20:53:48 +01:00
Niko Storni 74b76a11e4 upgrade quic 2020-12-17 23:49:37 +01:00
Niko Storni 2c0df2ca8a update lfuda library 2020-11-27 16:20:50 -05:00
Niko Storni 9fc96ac01b only store the blobs in the underlying storage if LFUDA accepted them 2020-11-27 16:20:50 -05:00
Niko Storni ff9b61b034 fix cache size mess 2020-11-27 16:20:49 -05:00
Niko Storni 7b80b2d4d2 fix buffer cache running out of space 2020-11-27 16:20:49 -05:00
Niko Storni d45abdbdb0 use LFUDA store
swap size to bytes
2020-11-27 16:19:46 -05:00
Niko Storni bc54601dde add LFUDA store
update quic
fix tests
2020-11-27 16:19:46 -05:00
Alex Grintsvayg bb41a84bb7 rename cahces 2020-11-27 16:19:45 -05:00
Niko Storni a574fecf4e add buffer cache for nvme drive 2020-11-27 16:19:45 -05:00
Niko Storni 9146c8b084 update quic
don't wait for a blob to be written to disk before sending it downstream
don't wait for the disk store to be walked before starting everything up
2020-11-27 16:19:45 -05:00
95 changed files with 3026 additions and 4430 deletions

No files matched your search

-37
View File
@@ -1,37 +0,0 @@
name: Go
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v4
with:
go-version: 1.25.x
- name: Build linux
run: make linux
- name: Build macos
run: make macos
- name: Test
run: make test
- name: Lint
run: make lint
- name: retrieve all tags
run: git fetch --prune --unshallow --tags
- name: Print changes since last version
run: git log $(git describe --tags --abbrev=0)..HEAD --no-merges --oneline
-62
View File
@@ -1,62 +0,0 @@
name: release
on:
push:
tags:
- "*.*.*"
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v4
with:
go-version: 1.25.x
- name: Build linux
run: make linux
- name: Build macos
run: make macos
- name: Test
run: make test
- name: Lint
run: make lint
- name: Zip macos
run: zip -r reflector_darwin_amd64.zip ./dist/darwin_amd64
- name: Zip linux
run: zip -r reflector_linux_amd64.zip ./dist/linux_amd64
- name: retrieve all tags
run: git fetch --prune --unshallow --tags
- name: Generate Changelog
run: git log $(git describe --tags --abbrev=0 @^)..@ --no-merges --oneline > ${{ github.workspace }}-CHANGELOG.txt
- name: upload to github releases
uses: softprops/action-gh-release@v1
with:
files: |
./reflector_linux_amd64.zip
./reflector_darwin_amd64.zip
body_path: ${{ github.workspace }}-CHANGELOG.txt
# - name: Login to DockerHub
# uses: docker/login-action@v2
# with:
# username: ${{ secrets.DOCKERHUB_USERNAME }}
# password: ${{ secrets.DOCKERHUB_TOKEN }}
# - name: Generate docker image
# run: make image
# - name: Docker push
# run: make publish_image
-9
View File
@@ -1,12 +1,3 @@
/vendor
/config.json*
/dist
/bin
blobcache.yaml
reflector.yaml
upload.yaml
remotereflector.yaml
-57
View File
@@ -1,57 +0,0 @@
run:
timeout: 5m
tests: false
modules-download-mode: readonly
linters-settings:
gci:
# Checks that no inline Comments are present.
# Default: false
no-inline-comments: false
# Checks that no prefix Comments(comment lines above an import) are present.
# Default: false
no-prefix-comments: false
# Skip generated files
skip-generated: true
# Use custom order
custom-order: true
sections:
- standard
- prefix(github.com/lbryio/reflector.go)
- prefix(github.com/OdyseeTeam,github.com/lbryio)
- default
gofmt:
simplify: true
govet:
enable-all: true
misspell:
locale: US
revive:
rules:
- name: exported
disabled: true
staticcheck:
checks: ["all"]
linters:
enable:
- errcheck
- gosimple
- govet
- ineffassign
- gci
- gofmt
- goimports
- misspell
- revive
- unconvert
- whitespace
- typecheck
disable:
- unused
- unparam
- staticcheck
+83
View File
@@ -0,0 +1,83 @@
os: linux
dist: trusty
language: go
env:
- GO111MODULE=on
go:
- 1.15.x
cache:
directories:
- $HOME/.cache/go-build
- $HOME/gopath/pkg/mod
notifications:
email: false
# Skip the install step. Don't `go get` dependencies. Only build with the code in vendor/
install: true
# Anything in before_script that returns a nonzero exit code will
# flunk the build and immediately stop. It's sorta like having
# set -e enabled in bash.
before_script:
# All the .go files, excluding vendor/ and model (auto generated)
- GO_FILES=$(find . -iname '*.go' ! -iname '*_test.go' -type f | grep -v /vendor/ ) #i wish we were this crazy :p
- go get golang.org/x/tools/cmd/goimports # Used in build script for generated files
# - go get github.com/golang/lint/golint # Linter
# - go get honnef.co/go/tools/cmd/megacheck # Badass static analyzer/linter
- go get github.com/jgautheron/gocyclo # Check against high complexity
- go get github.com/mdempsky/unconvert # Identifies unnecessary type conversions
- go get github.com/kisielk/errcheck # Checks for unhandled errors
- go get github.com/opennota/check/cmd/varcheck # Checks for unused vars
- go get github.com/opennota/check/cmd/structcheck # Checks for unused fields in structs
# script always run to completion (set +e). All of these code checks are must haves
# in a modern Go project.
script:
# Fail if a .go file hasn't been formatted with gofmt
- test -z $(gofmt -s -l $GO_FILES)
# Run unit tests
- make test
# Checks for unused vars and fields on structs
- varcheck ./...
- structcheck ./...
# go vet is the official Go static analyzer
- go vet ./...
# forbid code with huge functions
#- gocyclo -ignore "_test.go" -avg -over 19 $GO_FILES
# checks for unhandled errors
# - errcheck ./... # COMMENTED OUT UNTIL https://github.com/kisielk/errcheck/issues/155 is fixed
# "go vet on steroids" + linter - ignore autogen code
# - megacheck -simple.exit-non-zero=true ./... # DISABLED UNTIL https://github.com/dominikh/go-tools/issues/328 is fixed
# check for unnecessary conversions - ignore autogen code
# - unconvert ./... # SEEMS TO BE BROKEN WITH GO MODULES
# one last linter - ignore autogen code
#- golint -set_exit_status $(go list ./... | grep -v /vendor/ )
# Finally, build the binary
- make
deploy:
- provider: s3
local_dir: ./bin
skip_cleanup: true
on:
repo: lbryio/reflector.go
access_key_id:
secure: "ABmoSTxTee1GubJmmi+MlyBcnqRT0ywEOtTL4hrH7T+Vj5UPcNBhjqGRiMXzO3MdrVWom19OnqMvsvAdehrB4uQJVlX/+zB26gTpEiiU5dWJgerNAYmfxDGNz/p9pyYWLvVY/cymSlJ4HkggJVg+dMTBTOP+gTNgQUhHL7pAd/Z5HMPmPW/rlNlQgDdczoY44OdiniuHny4Lr7mo8dcT4dcyJysPpHLCPGWp3kXcffIEbgO02aQZ14BbX9oU9xm1BacYnr4t8gRCU+uK0mt7TREHUzZ8bQBRXUymmpeeEQshXdifdwtk6sdRGIZiTi0NmQ7kYter6oh6623TH6rpNy/lkxKi823FbSPC1XZGH7CwcbnSIEwY++7e9PT0EA14VuDUF7+iXC/gpRkVQwUoMcoC9rHBjkXW+SiwHWoUeDKXrfVO9QGEJTi5bDRNwklwu2zkYWeRIPzfDaVayUBYwjunjT1eST997ygmJwJTJ6DeOWLx+WFlmwBJfQpBwbrlMrceiNjreMwGB6ffXEBzuM6S5inXGIChllGDpNyJeFVcH9zPpU0JsIpWoRjwLNFOiEhFsnzK5rvA2SKmlOOnSlgOCBQnFi2SmLDM790KoY8qjan2s984chQz2qsKnhYuVpJhG6LKgrI1QwE0KHr80spVFfewq11AU4Q+lfA9aAQ="
secret_access_key:
secure: "AxYRTy/GnjeTJKQdeJ/AEeAd+yXs783bFDKdyKNswtsHlU8sWPQgNcvTLpVqnSQMpiwkGDGi/70rvR5C+AT3SIWNw13RYrgBRpduQU0J+B2JS+3dN2DIePu25uvs++Wo22OfS8I+UjZ1mWY1SSHI2spPXvDCq5tb+Ih8nlYflEyAtxU9Oq2R3Kus2tkIlRnL25sP/2fY7RvuJFYIV63z8ZIJRzB5WzOeERqnXq2zfwos+hycAqyo/VevJnWAYTEDsvBuSODOpZF+QfKtIQ2rYSoqy8Lq1M6UOZimnC3Ulea4euBVf2ssBCnI7csGNG5UzkTiwrPDi2xIP8nM01rHW1yHJ7tQsJaghnUsfw2t6ui4ZofvbbOFTN/YCloHITifEi8Tc1/17isi3y+kX5yQ/Nk5UNry0Wbt91CP+nkL/ZmA5grkBXDL7VJMmB60TnO3ap24CtwBQartN3LoWs7h+4ov+LqbCt6IqpJVWQWlwJeb2MFPFByALtBpsqAyL1SxXlGNpPa94CuXxfQ6Bv436PtefA5FlTzR8uMmqsjWciv06bVnSvVlFEVovN2Fkplrjt7AASJ/8KJs4THDg4k61nfd8roAHx6ewQzl4wCWKCikQ0MuFd2mVHwdrbnCH1mIHuPRyvWMMIAK0ooc1/rmKiJlpgumjxoFYNE10MXtt+I="
bucket: "lbry-reflector-binary"
# - provider: releases
# file: ./bin/prism-bin
# skip_cleanup: true
# on:
# repo: lbryio/reflector.go
# tags: true
# api_key:
# secure: epAlhp3SUr8hhISarJ22n6tRw2TEa4s4oNFIvJUb5HGECVp1SYN7ao0ln5NoNLmfJS60pi911i/kMhhi21/uhZ0kCYlEhhIE2pc1zsiAxK9L9ENCssJ205HfVbe5grhwskLGzgjhU9OznO8WtmyOPWXr0it8M8RCTjx6rEC0A33Id3WMYyhP938Sj9CxEYeH4KS8wFvBXkgBVtrgaYwRTCIROFddHFXOb9jyNhqQ1RbfKtllsVtQhVk5WMlomheBNSS4vr6WMS4X4+2okFqnLtiSn1wrn5I/94UQbnrI1juVnQj0K+j32EyQbAOt4T2cLW3GtG0jhaYKyNMT9ycDCdVACPSDELlHWjeyoes9bnhUFftm6kDbQxwA1UsTF1yG8tMKXxBSmYyoT7qDloi6pBifZMrFXL61uTs6yhVB9LS/2oqg4sc0Ne87bRcn4OxsBeVCe3kbBHDTR/NTyF2gNPtRvgMAWULxTVcUm9VYdO0IWvAig5g4Row0DnFzEquD6CzezbRWD9WyZyV/AFyYHeeQ2PO7jTw0/3M7aDX33Fuhh34lehzmrC03cfgD/wZW+spxozIcQCYdiJqVw+u+/NvbNr0kkFzE9zW26JEmUFTyDvKxvnza1Kwtww3EgH6zaOL8r4yVbb54rePRvLw7pl93zlfJnEB2MCPqJOY5ZpU=
+1 -1
View File
@@ -3,7 +3,7 @@ EXPOSE 8080
RUN mkdir /app
WORKDIR /app
COPY dist/linux_amd64/prism-bin ./prism
COPY bin/prism-bin ./prism
RUN chmod +x prism
ENTRYPOINT [ "/app/prism" ]
+18 -30
View File
@@ -1,37 +1,25 @@
version := $(shell git describe --dirty --always --long --abbrev=7)
commit := $(shell git rev-parse --short HEAD)
commit_long := $(shell git rev-parse HEAD)
branch := $(shell git rev-parse --abbrev-ref HEAD)
curTime := $(shell date +%s)
BINARY=prism-bin
IMPORT_PATH = github.com/lbryio/reflector.go
LDFLAGS="-X ${IMPORT_PATH}/meta.version=$(version) -X ${IMPORT_PATH}/meta.commit=$(commit) -X ${IMPORT_PATH}/meta.commitLong=$(commit_long) -X ${IMPORT_PATH}/meta.branch=$(branch) -X '${IMPORT_PATH}/meta.Time=$(curTime)'"
DIR = $(shell cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)
BIN_DIR = $(DIR)/dist
BIN_DIR = ${DIR}/bin
IMPORT_PATH = github.com/lbryio/reflector.go
.DEFAULT_GOAL := linux
VERSION = $(shell git --git-dir=${DIR}/.git describe --dirty --always --long --abbrev=7)
LDFLAGS = -ldflags "-X ${IMPORT_PATH}/meta.Version=${VERSION} -X ${IMPORT_PATH}/meta.Time=$(shell date +%s)"
.PHONY: build clean test lint
.DEFAULT_GOAL: build
build:
mkdir -p ${BIN_DIR} && CGO_ENABLED=0 go build ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/${BINARY} main.go
clean:
if [ -f ${BIN_DIR}/${BINARY} ]; then rm ${BIN_DIR}/${BINARY}; fi
.PHONY: test
test:
go test -cover -v ./...
go test ./... -v -cover
.PHONY: lint
lint:
./scripts/lint.sh
.PHONY: imports
imports:
gci write --skip-generated --custom-order -s standard -s "prefix(github.com/lbryio/reflector.go)" -s "prefix(github.com/OdyseeTeam,github.com/lbryio)" -s default .
.PHONY: linux
linux:
GOARCH=amd64 CGO_ENABLED=0 GOOS=linux go build -ldflags ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/linux_amd64/${BINARY}
.PHONY: macos
macos:
GOARCH=amd64 GOOS=darwin go build -ldflags ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/darwin_amd64/${BINARY}
.PHONY: image
image:
docker buildx build -t lbry/reflector:$(version) -t lbry/reflector:latest --platform linux/amd64 .
go get github.com/alecthomas/gometalinter && gometalinter --install && gometalinter ./...
+7 -6
View File
@@ -1,7 +1,7 @@
package cluster
import (
"io"
"io/ioutil"
baselog "log"
"sort"
"time"
@@ -23,12 +23,13 @@ const (
type Cluster struct {
OnMembershipChange func(n, total int)
s *serf.Serf
eventCh chan serf.Event
stop *stop.Group
name string
seedAddr string
port int
seedAddr string
s *serf.Serf
eventCh chan serf.Event
stop *stop.Group
}
// New returns a new Cluster instance that is not connected.
@@ -51,7 +52,7 @@ func (c *Cluster) Connect() error {
conf.MemberlistConfig.AdvertisePort = c.port
conf.NodeName = c.name
nullLogger := baselog.New(io.Discard, "", 0)
nullLogger := baselog.New(ioutil.Discard, "", 0)
conf.Logger = nullLogger
c.eventCh = make(chan serf.Event)
-55
View File
@@ -1,55 +0,0 @@
package cmd
import (
"os"
"os/signal"
"strconv"
"syscall"
"github.com/lbryio/reflector.go/config"
"github.com/lbryio/reflector.go/internal/metrics"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
func init() {
var cmd = &cobra.Command{
Use: "blobcache",
Short: "Run blobcache server",
Run: blobcacheCmd,
}
cmd.Flags().IntVar(&metricsPort, "metrics-port", 2112, "The port reflector will use for prometheus metrics")
cmd.Flags().BoolVar(&disableBlocklist, "disable-blocklist", false, "Disable blocklist watching/updating")
rootCmd.AddCommand(cmd)
}
func blobcacheCmd(cmd *cobra.Command, args []string) {
store, err := config.LoadStores(conf, "blobcache")
if err != nil {
log.Fatal(err)
}
defer store.Shutdown()
servers, err := config.LoadServers(store, conf, "blobcache")
if err != nil {
log.Fatal(err)
}
for _, s := range servers {
err = s.Start()
if err != nil {
log.Fatal(err)
}
defer s.Shutdown()
}
metricsServer := metrics.NewServer(":"+strconv.Itoa(metricsPort), "/metrics")
metricsServer.Start()
defer metricsServer.Shutdown()
interruptChan := make(chan os.Signal, 1)
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
<-interruptChan
}
+1 -2
View File
@@ -6,9 +6,8 @@ import (
"strconv"
"syscall"
"github.com/lbryio/reflector.go/cluster"
"github.com/lbryio/lbry.go/v2/extras/crypto"
"github.com/lbryio/reflector.go/cluster"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
+1 -1
View File
@@ -7,7 +7,7 @@ import (
"github.com/lbryio/lbry.go/v2/schema/stake"
"github.com/davecgh/go-spew/spew"
"github.com/gogo/protobuf/jsonpb"
"github.com/golang/protobuf/jsonpb"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
+6 -13
View File
@@ -5,6 +5,7 @@ import (
"os"
"time"
"github.com/lbryio/reflector.go/peer"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/stream"
@@ -27,19 +28,11 @@ func getStreamCmd(cmd *cobra.Command, args []string) {
addr := args[0]
sdHash := args[1]
s := store.NewCachingStore(store.CachingParams{
Name: "getstream",
Cache: store.NewPeerStore(store.PeerParams{
Name: "getstream",
Address: addr,
Timeout: 30 * time.Second,
}),
Origin: store.NewDiskStore(store.DiskParams{
Name: "getstream",
MountPoint: "/tmp/lbry_downloaded_blobs",
ShardingSize: 2,
}),
})
s := store.NewCachingStore(
"getstream",
peer.NewStore(peer.StoreOpts{Address: addr}),
store.NewDiskStore("/tmp/lbry_downloaded_blobs", 2),
)
wd, err := os.Getwd()
if err != nil {
+8 -8
View File
@@ -3,17 +3,17 @@ package cmd
import (
"crypto/sha512"
"encoding/hex"
"io/ioutil"
"os"
"path"
"runtime"
"sync/atomic"
"time"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/reflector.go/meta"
"github.com/lbryio/reflector.go/store/speedwalk"
"github.com/lbryio/lbry.go/v2/extras/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
@@ -39,7 +39,7 @@ func integrityCheckCmd(cmd *cobra.Command, args []string) {
blobs, err := speedwalk.AllFiles(diskStorePath, true)
if err != nil {
log.Fatalf("error while reading blobs from disk %s", errors.FullTrace(err))
log.Errorf("error while reading blobs from disk %s", errors.FullTrace(err))
}
tasks := make(chan string, len(blobs))
done := make(chan bool)
@@ -63,15 +63,15 @@ func consume(worker int, tasks <-chan string, done chan<- bool, totalTasks int,
start := time.Now()
for b := range tasks {
processedSoFar := atomic.AddInt32(processed, 1)
checked := atomic.AddInt32(processed, 1)
if worker == 0 {
remaining := int32(totalTasks) - processedSoFar
timePerBlob := time.Since(start).Microseconds() / int64(processedSoFar)
remaining := int32(totalTasks) - checked
timePerBlob := time.Since(start).Microseconds() / int64(checked)
remainingTime := time.Duration(int64(remaining)*timePerBlob) * time.Microsecond
log.Infof("[T%d] %d/%d blobs processed so far. ETA: %s", worker, processedSoFar, totalTasks, remainingTime.String())
log.Infof("[T%d] %d/%d blobs checked. ETA: %s", worker, checked, totalTasks, remainingTime.String())
}
blobPath := path.Join(diskStorePath, b[:2], b)
blob, err := os.ReadFile(blobPath)
blob, err := ioutil.ReadFile(blobPath)
if err != nil {
if os.IsNotExist(err) {
continue
+55
View File
@@ -0,0 +1,55 @@
package cmd
import (
"os"
"os/signal"
"strconv"
"syscall"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/peer"
"github.com/lbryio/reflector.go/store"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
var peerNoDB bool
func init() {
var cmd = &cobra.Command{
Use: "peer",
Short: "Run peer server",
Run: peerCmd,
}
cmd.Flags().BoolVar(&peerNoDB, "nodb", false, "Don't connect to a db and don't use a db-backed blob store")
rootCmd.AddCommand(cmd)
}
func peerCmd(cmd *cobra.Command, args []string) {
var err error
s3 := store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
peerServer := peer.NewServer(s3)
if !peerNoDB {
db := &db.SQL{
LogQueries: log.GetLevel() == log.DebugLevel,
}
err = db.Connect(globalConfig.DBConn)
checkErr(err)
combo := store.NewDBBackedStore(s3, db, false)
peerServer = peer.NewServer(combo)
}
err = peerServer.Start(":" + strconv.Itoa(peer.DefaultPort))
if err != nil {
log.Fatal(err)
}
interruptChan := make(chan os.Signal, 1)
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
<-interruptChan
peerServer.Shutdown()
}
+4 -8
View File
@@ -1,12 +1,11 @@
package cmd
import (
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/meta"
"github.com/lbryio/reflector.go/store/speedwalk"
"github.com/lbryio/lbry.go/v2/extras/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
@@ -32,18 +31,15 @@ func populateDbCmd(cmd *cobra.Command, args []string) {
log.Fatal("store-path must be defined")
}
localDb := &db.SQL{
SoftDelete: true,
TrackingLevel: db.TrackAccessBlobs,
LogQueries: log.GetLevel() == log.DebugLevel,
SoftDelete: true,
TrackAccess: db.TrackAccessBlobs,
LogQueries: log.GetLevel() == log.DebugLevel,
}
err := localDb.Connect("reflector:reflector@tcp(localhost:3306)/reflector")
if err != nil {
log.Fatal(err)
}
blobs, err := speedwalk.AllFiles(diskStorePath, true)
if err != nil {
log.Fatal(err)
}
err = localDb.AddBlobs(blobs)
if err != nil {
log.Errorf("error while storing to db: %s", errors.FullTrace(err))
+288 -27
View File
@@ -4,24 +4,46 @@ import (
"os"
"os/signal"
"strconv"
"strings"
"syscall"
"time"
"github.com/lbryio/reflector.go/config"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/meta"
"github.com/lbryio/reflector.go/peer"
"github.com/lbryio/reflector.go/peer/http3"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/server/http"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/c2h5oh/datasize"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
var (
//port configuration
receiverPort int
metricsPort int
//flags configuration
disableBlocklist bool
tcpPeerPort int
http3PeerPort int
httpPort int
receiverPort int
metricsPort int
disableUploads bool
disableBlocklist bool
proxyAddress string
proxyPort string
proxyProtocol string
useDB bool
cloudFrontEndpoint string
WasabiEndpoint string
reflectorCmdDiskCache string
bufferReflectorCmdDiskCache string
reflectorCmdMemCache int
requestQueueSize int
)
func init() {
@@ -30,47 +52,286 @@ func init() {
Short: "Run reflector server",
Run: reflectorCmd,
}
cmd.Flags().IntVar(&metricsPort, "metrics-port", 2112, "The port reflector will use for prometheus metrics")
cmd.Flags().StringVar(&proxyAddress, "proxy-address", "", "address of another reflector server where blobs are fetched from")
cmd.Flags().StringVar(&proxyPort, "proxy-port", "5567", "port of another reflector server where blobs are fetched from")
cmd.Flags().StringVar(&proxyProtocol, "proxy-protocol", "http3", "protocol used to fetch blobs from another reflector server (tcp/http3)")
cmd.Flags().StringVar(&cloudFrontEndpoint, "cloudfront-endpoint", "", "CloudFront edge endpoint for standard HTTP retrieval")
cmd.Flags().StringVar(&WasabiEndpoint, "wasabi-endpoint", "", "Wasabi edge endpoint for standard HTTP retrieval")
cmd.Flags().IntVar(&tcpPeerPort, "tcp-peer-port", 5567, "The port reflector will distribute content from")
cmd.Flags().IntVar(&http3PeerPort, "http3-peer-port", 5568, "The port reflector will distribute content from over HTTP3 protocol")
cmd.Flags().IntVar(&httpPort, "http-port", 5569, "The port reflector will distribute content from over HTTP protocol")
cmd.Flags().IntVar(&receiverPort, "receiver-port", 5566, "The port reflector will receive content from")
cmd.Flags().IntVar(&metricsPort, "metrics-port", 2112, "The port reflector will use for metrics")
cmd.Flags().IntVar(&requestQueueSize, "request-queue-size", 200, "How many concurrent requests should be submitted to upstream")
cmd.Flags().BoolVar(&disableUploads, "disable-uploads", false, "Disable uploads to this reflector server")
cmd.Flags().BoolVar(&disableBlocklist, "disable-blocklist", false, "Disable blocklist watching/updating")
cmd.Flags().BoolVar(&useDB, "use-db", true, "whether to connect to the reflector db or not")
cmd.Flags().StringVar(&reflectorCmdDiskCache, "disk-cache", "",
"enable disk cache, setting max size and path where to store blobs. format is 'sizeGB:CACHE_PATH'")
cmd.Flags().StringVar(&bufferReflectorCmdDiskCache, "buffer-disk-cache", "",
"enable buffer disk cache, setting max size and path where to store blobs. format is 'sizeGB:CACHE_PATH'")
cmd.Flags().IntVar(&reflectorCmdMemCache, "mem-cache", 0, "enable in-memory cache with a max size of this many blobs")
rootCmd.AddCommand(cmd)
}
func reflectorCmd(cmd *cobra.Command, args []string) {
store, err := config.LoadStores(conf, "reflector")
if err != nil {
log.Fatal(err)
}
defer store.Shutdown()
log.Printf("reflector %s", meta.VersionString())
cleanerStopper := stop.New()
servers, err := config.LoadServers(store, conf, "reflector")
if err != nil {
log.Fatal(err)
}
for _, s := range servers {
err = s.Start()
// the blocklist logic requires the db backed store to be the outer-most store
underlyingStore := setupStore()
outerStore := wrapWithCache(underlyingStore, cleanerStopper)
if !disableUploads {
reflectorServer := reflector.NewServer(underlyingStore, outerStore)
reflectorServer.Timeout = 3 * time.Minute
reflectorServer.EnableBlocklist = !disableBlocklist
err := reflectorServer.Start(":" + strconv.Itoa(receiverPort))
if err != nil {
log.Fatal(err)
}
defer s.Shutdown()
defer reflectorServer.Shutdown()
}
reflectorServer := reflector.NewIngestionServer(store)
reflectorServer.Timeout = 3 * time.Minute
reflectorServer.EnableBlocklist = !disableBlocklist
err = reflectorServer.Start(":" + strconv.Itoa(receiverPort))
peerServer := peer.NewServer(outerStore)
err := peerServer.Start(":" + strconv.Itoa(tcpPeerPort))
if err != nil {
log.Fatal(err)
}
defer reflectorServer.Shutdown()
defer peerServer.Shutdown()
http3PeerServer := http3.NewServer(outerStore, requestQueueSize)
err = http3PeerServer.Start(":" + strconv.Itoa(http3PeerPort))
if err != nil {
log.Fatal(err)
}
defer http3PeerServer.Shutdown()
httpServer := http.NewServer(outerStore)
err = httpServer.Start(":" + strconv.Itoa(httpPort))
if err != nil {
log.Fatal(err)
}
defer httpServer.Shutdown()
metricsServer := metrics.NewServer(":"+strconv.Itoa(metricsPort), "/metrics")
metricsServer.Start()
defer metricsServer.Shutdown()
defer outerStore.Shutdown()
defer underlyingStore.Shutdown()
interruptChan := make(chan os.Signal, 1)
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
<-interruptChan
// deferred shutdowns happen now
cleanerStopper.StopAndWait()
}
func setupStore() store.BlobStore {
var s store.BlobStore
if proxyAddress != "" {
switch proxyProtocol {
case "tcp":
s = peer.NewStore(peer.StoreOpts{
Address: proxyAddress + ":" + proxyPort,
Timeout: 30 * time.Second,
})
case "http3":
s = http3.NewStore(http3.StoreOpts{
Address: proxyAddress + ":" + proxyPort,
Timeout: 30 * time.Second,
})
case "http":
s = store.NewHttpStore(proxyAddress + ":" + proxyPort)
default:
log.Fatalf("protocol is not recognized: %s", proxyProtocol)
}
} else {
var s3Store *store.S3Store
if conf != "none" {
s3Store = store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
}
if cloudFrontEndpoint != "" && WasabiEndpoint != "" {
ittt := store.NewITTTStore(store.NewCloudFrontROStore(WasabiEndpoint), store.NewCloudFrontROStore(cloudFrontEndpoint))
if s3Store != nil {
s = store.NewCloudFrontRWStore(ittt, s3Store)
} else {
s = ittt
}
} else if s3Store != nil {
s = s3Store
} else {
log.Fatalf("this configuration does not include a valid upstream source")
}
}
if useDB {
dbInst := &db.SQL{
TrackAccess: db.TrackAccessStreams,
LogQueries: log.GetLevel() == log.DebugLevel,
}
err := dbInst.Connect(globalConfig.DBConn)
if err != nil {
log.Fatal(err)
}
s = store.NewDBBackedStore(s, dbInst, false)
}
return s
}
func wrapWithCache(s store.BlobStore, cleanerStopper *stop.Group) store.BlobStore {
wrapped := s
diskCacheMaxSize, diskCachePath := diskCacheParams(reflectorCmdDiskCache)
//we are tracking blobs in memory with a 1 byte long boolean, which means that for each 2MB (a blob) we need 1Byte
// so if the underlying cache holds 10MB, 10MB/2MB=5Bytes which is also the exact count of objects to restore on startup
realCacheSize := float64(diskCacheMaxSize) / float64(stream.MaxBlobSize)
if diskCacheMaxSize > 0 {
err := os.MkdirAll(diskCachePath, os.ModePerm)
if err != nil {
log.Fatal(err)
}
localDb := &db.SQL{
SoftDelete: true,
TrackAccess: db.TrackAccessBlobs,
LogQueries: log.GetLevel() == log.DebugLevel,
}
err = localDb.Connect("reflector:reflector@tcp(localhost:3306)/reflector")
if err != nil {
log.Fatal(err)
}
dbBackedDiskStore := store.NewDBBackedStore(store.NewDiskStore(diskCachePath, 2), localDb, true)
wrapped = store.NewCachingStore(
"reflector",
wrapped,
dbBackedDiskStore,
)
go cleanOldestBlobs(int(realCacheSize), localDb, dbBackedDiskStore, cleanerStopper)
}
diskCacheMaxSize, diskCachePath = diskCacheParams(bufferReflectorCmdDiskCache)
realCacheSize = float64(diskCacheMaxSize) / float64(stream.MaxBlobSize)
if diskCacheMaxSize > 0 {
err := os.MkdirAll(diskCachePath, os.ModePerm)
if err != nil {
log.Fatal(err)
}
wrapped = store.NewCachingStore(
"reflector",
wrapped,
store.NewLFUDAStore("nvme", store.NewDiskStore(diskCachePath, 2), realCacheSize),
)
}
if reflectorCmdMemCache > 0 {
wrapped = store.NewCachingStore(
"reflector",
wrapped,
store.NewLRUStore("mem", store.NewMemStore(), reflectorCmdMemCache),
)
}
return wrapped
}
func diskCacheParams(diskParams string) (int, string) {
if diskParams == "" {
return 0, ""
}
parts := strings.Split(diskParams, ":")
if len(parts) != 2 {
log.Fatalf("--disk-cache must be a number, followed by ':', followed by a string")
}
diskCacheSize := parts[0]
path := parts[1]
if len(path) == 0 || path[0] != '/' {
log.Fatalf("--disk-cache path must start with '/'")
}
var maxSize datasize.ByteSize
err := maxSize.UnmarshalText([]byte(diskCacheSize))
if err != nil {
log.Fatal(err)
}
if maxSize <= 0 {
log.Fatal("--disk-cache size must be more than 0")
}
return int(maxSize), path
}
func cleanOldestBlobs(maxItems int, db *db.SQL, store store.BlobStore, stopper *stop.Group) {
// this is so that it runs on startup without having to wait for 10 minutes
err := doClean(maxItems, db, store, stopper)
if err != nil {
log.Error(errors.FullTrace(err))
}
const cleanupInterval = 10 * time.Minute
for {
select {
case <-stopper.Ch():
log.Infoln("stopping self cleanup")
return
case <-time.After(cleanupInterval):
err := doClean(maxItems, db, store, stopper)
if err != nil {
log.Error(errors.FullTrace(err))
}
}
}
}
func doClean(maxItems int, db *db.SQL, store store.BlobStore, stopper *stop.Group) error {
blobsCount, err := db.Count()
if err != nil {
return err
}
if blobsCount >= maxItems {
itemsToDelete := blobsCount / 10
blobs, err := db.LeastRecentlyAccessedHashes(itemsToDelete)
if err != nil {
return err
}
blobsChan := make(chan string, len(blobs))
wg := &stop.Group{}
go func() {
for _, hash := range blobs {
select {
case <-stopper.Ch():
return
default:
}
blobsChan <- hash
}
close(blobsChan)
}()
for i := 0; i < 3; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for h := range blobsChan {
select {
case <-stopper.Ch():
return
default:
}
err = store.Delete(h)
if err != nil {
log.Errorf("error pruning %s: %s", h, errors.FullTrace(err))
continue
}
}
}()
}
wg.Wait()
}
return nil
}
+52 -53
View File
@@ -1,15 +1,19 @@
package cmd
import (
"encoding/json"
"io/ioutil"
"os"
"strings"
"github.com/lbryio/lbry.go/v2/dht"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/lbryio/reflector.go/updater"
"github.com/johntdyer/slackrus"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
// Config is the base configuration for Prism when no sub commands are used.
@@ -18,10 +22,8 @@ type Config struct {
AwsSecret string `json:"aws_secret"`
BucketRegion string `json:"bucket_region"`
BucketName string `json:"bucket_name"`
S3Endpoint string `json:"s3_endpoint"`
DBConn string `json:"db_conn"`
SlackHookURL string `json:"slack_hook_url"`
SlackChannel string `json:"slack_channel"`
UpdateBinURL string `json:"update_bin_url"`
UpdateCmd string `json:"update_cmd"`
}
@@ -35,8 +37,7 @@ const (
)
var conf string
//var globalConfig Config
var globalConfig Config
var rootCmd = &cobra.Command{
Use: "prism",
@@ -49,7 +50,7 @@ var rootCmd = &cobra.Command{
func init() {
rootCmd.PersistentFlags().StringSliceVarP(&verbose, "verbose", "v", []string{}, "Verbose logging for specific components")
rootCmd.PersistentFlags().StringVar(&conf, "conf-dir", "./", "Path to config directory")
rootCmd.PersistentFlags().StringVar(&conf, "conf", "config.json", "Path to config. Use 'none' to disable")
}
// Execute adds all child commands to the root command and sets flags appropriately.
@@ -84,40 +85,38 @@ func preRun(cmd *cobra.Command, args []string) {
}
}
//var err error
var err error
if conf == "" {
logrus.Errorln("--conf flag required")
os.Exit(1)
} else if conf != "none" {
globalConfig, err = loadConfig(conf)
if err != nil {
logrus.Error(err)
os.Exit(1)
}
}
//else if conf != "none" {
// globalConfig, err = loadConfig(conf)
// if err != nil {
// logrus.Error(err)
// os.Exit(1)
// }
//}
viper.AddConfigPath(conf)
//if globalConfig.SlackHookURL != "" {
// hook := &slackrus.SlackrusHook{
// HookURL: globalConfig.SlackHookURL,
// AcceptedLevels: slackrus.LevelThreshold(logrus.InfoLevel),
// Channel: globalConfig.SlackChannel,
// //IconEmoji: ":ghost:",
// //Username: "reflector.go",
// }
// //logrus.SetFormatter(&logrus.JSONFormatter{})
// logrus.AddHook(hook)
// debugLogger.AddHook(hook)
//}
if globalConfig.SlackHookURL != "" {
hook := &slackrus.SlackrusHook{
HookURL: globalConfig.SlackHookURL,
AcceptedLevels: slackrus.LevelThreshold(logrus.InfoLevel),
Channel: "#reflector-logs",
//IconEmoji: ":ghost:",
//Username: "reflector.go",
}
//logrus.SetFormatter(&logrus.JSONFormatter{})
logrus.AddHook(hook)
debugLogger.AddHook(hook)
}
//if globalConfig.UpdateBinURL != "" {
// if globalConfig.UpdateCmd == "" {
// logrus.Warnln("update_cmd is empty in conf file")
// }
// logrus.Println("starting update checker")
// go updater.Run(globalConfig.UpdateBinURL, globalConfig.UpdateCmd)
//}
if globalConfig.UpdateBinURL != "" {
if globalConfig.UpdateCmd == "" {
logrus.Warnln("update_cmd is empty in conf file")
}
logrus.Println("starting update checker")
go updater.Run(globalConfig.UpdateBinURL, globalConfig.UpdateCmd)
}
}
func checkErr(err error) {
@@ -138,20 +137,20 @@ func argFuncs(funcs ...cobra.PositionalArgs) cobra.PositionalArgs {
}
}
//func loadConfig(path string) (Config, error) {
// var c Config
//
// raw, err := os.ReadFile(path)
// if err != nil {
// if os.IsNotExist(err) {
// return c, errors.Err("config file not found")
// }
// return c, errors.Err(err)
// }
//
// err = json.Unmarshal(raw, &c)
// return c, errors.Err(err)
//}
func loadConfig(path string) (Config, error) {
var c Config
raw, err := ioutil.ReadFile(path)
if err != nil {
if os.IsNotExist(err) {
return c, errors.Err("config file not found")
}
return c, errors.Err(err)
}
err = json.Unmarshal(raw, &c)
return c, errors.Err(err)
}
func mustGetFlagString(cmd *cobra.Command, name string) string {
v, err := cmd.Flags().GetString(name)
@@ -165,8 +164,8 @@ func mustGetFlagInt64(cmd *cobra.Command, name string) int64 {
return v
}
//func mustGetFlagBool(cmd *cobra.Command, name string) bool {
// v, err := cmd.Flags().GetBool(name)
// checkErr(err)
// return v
//}
func mustGetFlagBool(cmd *cobra.Command, name string) bool {
v, err := cmd.Flags().GetBool(name)
checkErr(err)
return v
}
+9 -9
View File
@@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"os/signal"
"path"
@@ -38,16 +39,17 @@ func sendCmd(cmd *cobra.Command, args []string) {
reflectorAddress := args[0]
err := hackyReflector.Connect(reflectorAddress)
checkErr(err)
defer func() { _ = hackyReflector.Close() }()
defer hackyReflector.Close()
filePath := args[1]
file, err := os.Open(filePath)
checkErr(err)
defer func() { _ = file.Close() }()
defer file.Close()
sdCachePath := ""
sdCacheDir := mustGetFlagString(cmd, "sd-cache")
if sdCacheDir != "" {
if _, statErr := os.Stat(sdCacheDir); os.IsNotExist(statErr) {
if _, err := os.Stat(sdCacheDir); os.IsNotExist(err) {
err = os.MkdirAll(sdCacheDir, 0777)
checkErr(err)
}
@@ -57,9 +59,8 @@ func sendCmd(cmd *cobra.Command, args []string) {
var enc *stream.Encoder
if sdCachePath != "" {
if _, statErr := os.Stat(sdCachePath); !os.IsNotExist(statErr) {
var sdBlob []byte
sdBlob, err = os.ReadFile(sdCachePath)
if _, err := os.Stat(sdCachePath); !os.IsNotExist(err) {
sdBlob, err := ioutil.ReadFile(sdCachePath)
checkErr(err)
cachedSDBlob := &stream.SDBlob{}
err = cachedSDBlob.FromBlob(sdBlob)
@@ -88,8 +89,7 @@ func sendCmd(cmd *cobra.Command, args []string) {
break
}
var b stream.Blob
b, err = enc.Next()
b, err := enc.Next()
if errors.Is(err, io.EOF) {
break
}
@@ -110,7 +110,7 @@ func sendCmd(cmd *cobra.Command, args []string) {
sd := enc.SDBlob()
//sd.StreamName = filepath.Base(filePath)
//sd.SuggestedFileName = filepath.Base(filePath)
err = os.WriteFile(sdCachePath, sd.ToBlob(), 0666)
err = ioutil.WriteFile(sdCachePath, sd.ToBlob(), 0666)
if err != nil {
fmt.Printf("error saving sd blob: %v\n", err)
fmt.Println(sd.ToJson())
+1 -1
View File
@@ -51,7 +51,7 @@ func sendBlobCmd(cmd *cobra.Command, args []string) {
file, err := os.Open(path)
checkErr(err)
defer func() { _ = file.Close() }()
defer file.Close()
s, err := stream.New(file)
checkErr(err)
+101
View File
@@ -0,0 +1,101 @@
package cmd
import (
"os"
"os/signal"
"strconv"
"strings"
"syscall"
"github.com/lbryio/lbry.go/v2/dht"
"github.com/lbryio/lbry.go/v2/dht/bits"
"github.com/lbryio/reflector.go/cluster"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/peer"
"github.com/lbryio/reflector.go/prism"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/store"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
const (
startNewCluster = "new"
)
var (
startClusterPort int
startPeerPort int
startReflectorPort int
startDhtPort int
startDhtSeeds []string
startHashRange string
)
func init() {
var cmd = &cobra.Command{
Use: `start [cluster-address|"new"]`,
Short: "Runs full prism application with cluster, dht, peer server, and reflector server.",
Run: startCmd,
Args: cobra.ExactArgs(1),
}
cmd.PersistentFlags().IntVar(&startClusterPort, "cluster-port", cluster.DefaultPort, "Port that cluster listens on")
cmd.PersistentFlags().IntVar(&startPeerPort, "peer-port", peer.DefaultPort, "Port to start peer protocol on")
cmd.PersistentFlags().IntVar(&startReflectorPort, "reflector-port", reflector.DefaultPort, "Port to start reflector protocol on")
cmd.PersistentFlags().IntVar(&startDhtPort, "dht-port", dht.DefaultPort, "Port that dht will listen on")
cmd.PersistentFlags().StringSliceVar(&startDhtSeeds, "dht-seeds", []string{}, "Comma-separated list of dht seed nodes (addr:port,addr:port,...)")
cmd.PersistentFlags().StringVar(&startHashRange, "hash-range", "", "Limit on range of hashes to announce (start-end)")
rootCmd.AddCommand(cmd)
}
func startCmd(cmd *cobra.Command, args []string) {
db := &db.SQL{
LogQueries: log.GetLevel() == log.DebugLevel,
}
err := db.Connect(globalConfig.DBConn)
checkErr(err)
s3 := store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
comboStore := store.NewDBBackedStore(s3, db, false)
conf := prism.DefaultConf()
// TODO: args we need:
// minNodes - minimum number of nodes before announcing starts. otherwise first node will try to announce all the blobs in the db
// or maybe we should do maxHashesPerNode?
// in either case, this should not kill the cluster, but should only limit announces (and notify when some hashes are being left unannounced)
if args[0] != startNewCluster {
conf.ClusterSeedAddr = args[0]
}
conf.DB = db
conf.Blobs = comboStore
conf.DhtAddress = "0.0.0.0:" + strconv.Itoa(startDhtPort)
conf.DhtSeedNodes = startDhtSeeds
conf.ClusterPort = startClusterPort
conf.PeerPort = startPeerPort
conf.ReflectorPort = startReflectorPort
if startHashRange != "" {
hashRange := strings.Split(startHashRange, "-")
if len(hashRange) != 2 {
log.Fatal("invalid hash range")
}
r := bits.Range{Start: bits.FromShortHexP(hashRange[0]), End: bits.FromShortHexP(hashRange[1])}
conf.HashRange = &r
}
p := prism.New(conf)
err = p.Start()
if err != nil {
log.Fatal(err)
}
interruptChan := make(chan os.Signal, 1)
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
<-interruptChan
p.Shutdown()
}
+5 -5
View File
@@ -9,8 +9,8 @@ import (
"time"
"github.com/lbryio/reflector.go/meta"
"github.com/lbryio/reflector.go/peer"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/server/peer"
"github.com/lbryio/reflector.go/store"
log "github.com/sirupsen/logrus"
@@ -29,9 +29,9 @@ func init() {
func testCmd(cmd *cobra.Command, args []string) {
log.Printf("reflector %s", meta.VersionString())
memStore := store.NewMemStore(store.MemParams{Name: "test"})
memStore := store.NewMemStore()
reflectorServer := reflector.NewIngestionServer(memStore)
reflectorServer := reflector.NewServer(memStore, memStore)
reflectorServer.Timeout = 3 * time.Minute
err := reflectorServer.Start(":" + strconv.Itoa(reflector.DefaultPort))
@@ -39,8 +39,8 @@ func testCmd(cmd *cobra.Command, args []string) {
log.Fatal(err)
}
peerServer := peer.NewServer(memStore, fmt.Sprintf(":%d", reflector.DefaultPort+1))
err = peerServer.Start()
peerServer := peer.NewServer(memStore)
err = peerServer.Start(":" + strconv.Itoa(reflector.DefaultPort+1))
if err != nil {
log.Fatal(err)
}
+11 -11
View File
@@ -5,10 +5,11 @@ import (
"os/signal"
"syscall"
"github.com/lbryio/reflector.go/config"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/store"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
@@ -30,18 +31,17 @@ func init() {
}
func uploadCmd(cmd *cobra.Command, args []string) {
store, err := config.LoadStores(conf, "upload")
if err != nil {
log.Fatal(err)
db := &db.SQL{
LogQueries: log.GetLevel() == log.DebugLevel,
}
defer store.Shutdown()
err := db.Connect(globalConfig.DBConn)
checkErr(err)
databaseConn, err := config.LoadDatabase(conf, "upload")
if err != nil {
log.Fatal(err)
}
st := store.NewDBBackedStore(
store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName),
db, false)
uploader := reflector.NewUploader(databaseConn, store, uploadWorkers, uploadSkipExistsCheck, uploadDeleteBlobsAfterUpload)
uploader := reflector.NewUploader(db, st, uploadWorkers, uploadSkipExistsCheck, uploadDeleteBlobsAfterUpload)
interruptChan := make(chan os.Signal, 1)
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
+1 -2
View File
@@ -4,7 +4,6 @@ import (
"fmt"
"github.com/lbryio/reflector.go/meta"
"github.com/spf13/cobra"
)
@@ -18,5 +17,5 @@ func init() {
}
func versionCmd(cmd *cobra.Command, args []string) {
fmt.Println(meta.FullName())
fmt.Println(meta.VersionString())
}
-113
View File
@@ -1,113 +0,0 @@
package config
import (
"fmt"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/server"
"github.com/lbryio/reflector.go/server/http"
"github.com/lbryio/reflector.go/server/http3"
"github.com/lbryio/reflector.go/server/peer"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/extras/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
)
func LoadStores(path, file string) (store.BlobStore, error) {
v := viper.New()
v.SetConfigType("yaml")
v.AddConfigPath(path)
v.SetConfigName(file)
err := v.ReadInConfig()
if err != nil {
return nil, errors.Err(err)
}
storeViper := v.Sub("store")
for storeType := range storeViper.AllSettings() {
factory, exists := store.Factories[storeType]
if !exists {
return nil, errors.Err("unknown store type: %s", storeType)
}
storeConfig := storeViper.Sub(storeType)
s, err := factory(storeConfig)
if err != nil {
return nil, errors.Err(err)
}
//we only expect 1 store as the root, so let's return it
return s, nil
}
return nil, nil
}
func LoadServers(store store.BlobStore, path, file string) ([]server.BlobServer, error) {
v := viper.New()
v.SetConfigType("yaml")
v.AddConfigPath(path)
v.SetConfigName(file)
err := v.ReadInConfig()
if err != nil {
return nil, errors.Err(err)
}
servers := make([]server.BlobServer, 0)
serversViper := v.Sub("servers")
for serverType := range serversViper.AllSettings() {
var cfg server.BlobServerConfig
err := serversViper.Sub(serverType).Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
switch serverType {
case "http":
servers = append(servers, http.NewServer(store, cfg.MaxConcurrentRequests, cfg.EdgeToken, fmt.Sprintf("%s:%d", cfg.Address, cfg.Port)))
case "http3":
servers = append(servers, http3.NewServer(store, cfg.MaxConcurrentRequests, fmt.Sprintf("%s:%d", cfg.Address, cfg.Port)))
case "peer":
servers = append(servers, peer.NewServer(store, fmt.Sprintf("%s:%d", cfg.Address, cfg.Port)))
default:
return nil, errors.Err("unknown server type: %s", serverType)
}
}
return servers, nil
}
func LoadDatabase(path, file string) (*db.SQL, error) {
v := viper.New()
v.SetConfigType("yaml")
v.AddConfigPath(path)
v.SetConfigName(file)
err := v.ReadInConfig()
if err != nil {
return nil, errors.Err(err)
}
dbConfig := v.Sub("database")
if dbConfig == nil {
return nil, errors.Err("db config not found")
}
user := dbConfig.GetString("user")
password := dbConfig.GetString("password")
host := dbConfig.GetString("host")
port := dbConfig.GetInt("port")
database := dbConfig.GetString("database")
logQueries := dbConfig.GetBool("log_queries")
accessTracking := dbConfig.GetInt("access_tracking")
if user == "" || password == "" || host == "" || port == 0 || database == "" {
return nil, errors.Err("db config is missing required fields")
}
dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", user, password, host, port, database)
dbInstance := &db.SQL{
TrackingLevel: db.AccessTrackingLevel(accessTracking),
SoftDelete: true,
LogQueries: logQueries || log.GetLevel() == log.DebugLevel,
}
err = dbInstance.Connect(dsn)
return dbInstance, err
}
+176 -96
View File
@@ -15,32 +15,36 @@ import (
"github.com/lbryio/lbry.go/v2/stream"
"github.com/go-sql-driver/mysql"
_ "github.com/go-sql-driver/mysql" // blank import for db driver ensures its imported even if its not used
log "github.com/sirupsen/logrus"
"github.com/volatiletech/null/v8"
"github.com/volatiletech/null"
"go.uber.org/atomic"
)
// SdBlob is a special blob that contains information on the rest of the blobs in the stream
type SdBlob struct {
StreamName string `json:"stream_name"`
StreamName string `json:"stream_name"`
Blobs []struct {
Length int `json:"length"`
BlobNum int `json:"blob_num"`
BlobHash string `json:"blob_hash,omitempty"`
IV string `json:"iv"`
} `json:"blobs"`
StreamType string `json:"stream_type"`
Key string `json:"key"`
SuggestedFileName string `json:"suggested_file_name"`
StreamHash string `json:"stream_hash"`
Blobs []struct {
BlobHash string `json:"blob_hash,omitempty"`
IV string `json:"iv"`
Length int `json:"length"`
BlobNum int `json:"blob_num"`
} `json:"blobs"`
}
type AccessTrackingLevel int
type trackAccess int
const (
TrackAccessNone AccessTrackingLevel = iota // Don't track accesses
TrackAccessStreams // Track accesses at the stream level
TrackAccessBlobs // Track accesses at the blob level
// Don't track accesses
TrackAccessNone trackAccess = iota
// Track accesses at the stream level
TrackAccessStreams
// Track accesses at the blob level
TrackAccessBlobs
)
// SQL implements the DB interface
@@ -48,7 +52,7 @@ type SQL struct {
conn *sql.DB
// Track the approx last time a blob or stream was accessed
TrackingLevel AccessTrackingLevel
TrackAccess trackAccess
// Instead of deleting a blob, marked it as not stored in the db
SoftDelete bool
@@ -57,7 +61,7 @@ type SQL struct {
LogQueries bool
}
func (s *SQL) logQuery(query string, args ...interface{}) {
func (s SQL) logQuery(query string, args ...interface{}) {
if !s.LogQueries {
return
}
@@ -93,23 +97,22 @@ func (s *SQL) AddBlob(hash string, length int, isStored bool) error {
return errors.Err("not connected")
}
_, err := s.insertBlob(hash, length, isStored)
_, err := s.insertBlob(s.conn, hash, length, isStored)
return err
}
// AddBlobs adds blobs to the database.
// AddBlob adds a blob to the database.
func (s *SQL) AddBlobs(hash []string) error {
if s.conn == nil {
return errors.Err("not connected")
}
// Split the slice into batches of 20 items.
batch := 10000
totalBlobs := int64(len(hash))
work := make(chan []string, 1000)
stopper := stop.New()
var totalInserted atomic.Int64
start := time.Now()
go func() {
for i := 0; i < len(hash); i += batch {
j := i + batch
@@ -121,7 +124,6 @@ func (s *SQL) AddBlobs(hash []string) error {
log.Infof("done loading %d hashes in the work queue", len(hash))
close(work)
}()
for i := 0; i < runtime.NumCPU(); i++ {
stopper.Add(1)
go func(worker int) {
@@ -143,7 +145,6 @@ func (s *SQL) AddBlobs(hash []string) error {
}
}(i)
}
stopper.Wait()
return nil
}
@@ -162,7 +163,7 @@ func (s *SQL) insertBlobs(hashes []string) error {
//args = append(args, hash, true, stream.MaxBlobSize, dayAgo)
}
q = strings.TrimSuffix(q, ",")
_, err := s.exec(q)
_, err := s.exec(s.conn, q)
if err != nil {
return err
}
@@ -170,7 +171,7 @@ func (s *SQL) insertBlobs(hashes []string) error {
return nil
}
func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error) {
func (s *SQL) insertBlob(ex Executor, hash string, length int, isStored bool) (int64, error) {
if length <= 0 {
return 0, errors.Err("length must be positive")
}
@@ -179,7 +180,7 @@ func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error)
q string
args []interface{}
)
if s.TrackingLevel == TrackAccessBlobs {
if s.TrackAccess == TrackAccessBlobs {
args = []interface{}{hash, isStored, length, time.Now()}
q = "INSERT INTO blob_ (hash, is_stored, length, last_accessed_at) VALUES (" + qt.Qs(len(args)) + ") ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored)), last_accessed_at = VALUES(last_accessed_at)"
} else {
@@ -187,13 +188,13 @@ func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error)
q = "INSERT INTO blob_ (hash, is_stored, length) VALUES (" + qt.Qs(len(args)) + ") ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored))"
}
blobID, err := s.exec(q, args...)
blobID, err := s.exec(ex, q, args...)
if err != nil {
return 0, err
}
if blobID == 0 {
err = s.conn.QueryRow("SELECT id FROM blob_ WHERE hash = ?", hash).Scan(&blobID)
err = ex.QueryRow("SELECT id FROM blob_ WHERE hash = ?", hash).Scan(&blobID)
if err != nil {
return 0, errors.Err(err)
}
@@ -201,8 +202,8 @@ func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error)
return 0, errors.Err("blob ID is 0 even after INSERTing and SELECTing")
}
if s.TrackingLevel == TrackAccessBlobs {
err := s.touchBlobs([]uint64{uint64(blobID)})
if s.TrackAccess == TrackAccessBlobs {
err := s.touchBlobs(ex, []uint64{uint64(blobID)})
if err != nil {
return 0, errors.Err(err)
}
@@ -218,7 +219,7 @@ func (s *SQL) insertStream(hash string, sdBlobID int64) (int64, error) {
args []interface{}
)
if s.TrackingLevel == TrackAccessStreams {
if s.TrackAccess == TrackAccessStreams {
args = []interface{}{hash, sdBlobID, time.Now()}
q = "INSERT IGNORE INTO stream (hash, sd_blob_id, last_accessed_at) VALUES (" + qt.Qs(len(args)) + ")"
} else {
@@ -226,7 +227,7 @@ func (s *SQL) insertStream(hash string, sdBlobID int64) (int64, error) {
q = "INSERT IGNORE INTO stream (hash, sd_blob_id) VALUES (" + qt.Qs(len(args)) + ")"
}
streamID, err := s.exec(q, args...)
streamID, err := s.exec(s.conn, q, args...)
if err != nil {
return 0, errors.Err(err)
}
@@ -240,7 +241,7 @@ func (s *SQL) insertStream(hash string, sdBlobID int64) (int64, error) {
return 0, errors.Err("stream ID is 0 even after INSERTing and SELECTing")
}
if s.TrackingLevel == TrackAccessStreams {
if s.TrackAccess == TrackAccessStreams {
err := s.touchStreams([]uint64{uint64(streamID)})
if err != nil {
return 0, errors.Err(err)
@@ -264,9 +265,9 @@ func (s *SQL) HasBlobs(hashes []string, touch bool) (map[string]bool, error) {
exists, idsNeedingTouch, err := s.hasBlobs(hashes)
if touch {
if s.TrackingLevel == TrackAccessBlobs {
_ = s.touchBlobs(idsNeedingTouch)
} else if s.TrackingLevel == TrackAccessStreams {
if s.TrackAccess == TrackAccessBlobs {
_ = s.touchBlobs(s.conn, idsNeedingTouch)
} else if s.TrackAccess == TrackAccessStreams {
_ = s.touchStreams(idsNeedingTouch)
}
}
@@ -274,7 +275,7 @@ func (s *SQL) HasBlobs(hashes []string, touch bool) (map[string]bool, error) {
return exists, err
}
func (s *SQL) touchBlobs(blobIDs []uint64) error {
func (s *SQL) touchBlobs(ex Executor, blobIDs []uint64) error {
if len(blobIDs) == 0 {
return nil
}
@@ -287,7 +288,7 @@ func (s *SQL) touchBlobs(blobIDs []uint64) error {
}
startTime := time.Now()
_, err := s.exec(query, args...)
_, err := s.exec(ex, query, args...)
log.Debugf("touched %d blobs and took %s", len(blobIDs), time.Since(startTime))
return errors.Err(err)
}
@@ -305,7 +306,7 @@ func (s *SQL) touchStreams(streamIDs []uint64) error {
}
startTime := time.Now()
_, err := s.exec(query, args...)
_, err := s.exec(s.conn, query, args...)
log.Debugf("touched %d streams and took %s", len(streamIDs), time.Since(startTime))
return errors.Err(err)
}
@@ -338,11 +339,11 @@ func (s *SQL) hasBlobs(hashes []string) (map[string]bool, []uint64, error) {
batch := hashes[doneIndex:sliceEnd]
var query string
if s.TrackingLevel == TrackAccessBlobs {
if s.TrackAccess == TrackAccessBlobs {
query = `SELECT b.hash, b.id, NULL, b.last_accessed_at
FROM blob_ b
WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)`
} else if s.TrackingLevel == TrackAccessStreams {
} else if s.TrackAccess == TrackAccessStreams {
query = `SELECT b.hash, b.id, s.id, s.last_accessed_at
FROM blob_ b
LEFT JOIN stream_blob sb ON b.id = sb.blob_id
@@ -371,15 +372,15 @@ WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)`
defer closeRows(rows)
for rows.Next() {
err = rows.Scan(&hash, &blobID, &streamID, &lastAccessedAt)
err := rows.Scan(&hash, &blobID, &streamID, &lastAccessedAt)
if err != nil {
return errors.Err(err)
}
exists[hash] = true
if !lastAccessedAt.Valid || lastAccessedAt.Time.Before(touchDeadline) {
if s.TrackingLevel == TrackAccessBlobs {
if s.TrackAccess == TrackAccessBlobs {
needsTouch = append(needsTouch, blobID)
} else if s.TrackingLevel == TrackAccessStreams && !streamID.IsZero() {
} else if s.TrackAccess == TrackAccessStreams && !streamID.IsZero() {
needsTouch = append(needsTouch, streamID.Uint64)
}
}
@@ -405,26 +406,26 @@ WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)`
// NOTE: If SoftDelete is enabled, streams will never be deleted
func (s *SQL) Delete(hash string) error {
if s.SoftDelete {
_, err := s.exec("UPDATE blob_ SET is_stored = 0 WHERE hash = ?", hash)
_, err := s.exec(s.conn, "UPDATE blob_ SET is_stored = 0 WHERE hash = ?", hash)
return errors.Err(err)
}
_, err := s.exec("DELETE FROM stream WHERE sd_blob_id = (SELECT id FROM blob_ WHERE hash = ?)", hash)
_, err := s.exec(s.conn, "DELETE FROM stream WHERE sd_blob_id = (SELECT id FROM blob_ WHERE hash = ?)", hash)
if err != nil {
return errors.Err(err)
}
_, err = s.exec("DELETE FROM blob_ WHERE hash = ?", hash)
_, err = s.exec(s.conn, "DELETE FROM blob_ WHERE hash = ?", hash)
return errors.Err(err)
}
// LeastRecentlyAccessedHashes gets the least recently accessed blobs
// GetHashRange gets the smallest and biggest hashes in the db
func (s *SQL) LeastRecentlyAccessedHashes(maxBlobs int) ([]string, error) {
if s.conn == nil {
return nil, errors.Err("not connected")
}
if s.TrackingLevel != TrackAccessBlobs {
if s.TrackAccess != TrackAccessBlobs {
return nil, errors.Err("blob access tracking is disabled")
}
@@ -450,6 +451,40 @@ func (s *SQL) LeastRecentlyAccessedHashes(maxBlobs int) ([]string, error) {
return blobs, nil
}
// AllHashes writes all hashes from the db into the channel.
// It does not close the channel when it finishes.
//func (s *SQL) AllHashes(ch chan<- string) error {
// if s.conn == nil {
// return errors.Err("not connected")
// }
//
// query := "SELECT hash from blob_"
// if s.SoftDelete {
// query += " where is_stored = 1"
// }
// s.logQuery(query)
//
// rows, err := s.conn.Query(query)
// if err != nil {
// return errors.Err(err)
// }
// defer closeRows(rows)
//
// for rows.Next() {
// var hash string
// err := rows.Scan(&hash)
// if err != nil {
// return errors.Err(err)
// }
// ch <- hash
// // TODO: this needs testing
// // TODO: need a way to cancel this early (e.g. in case of shutdown)
// }
//
// close(ch)
// return nil
//}
func (s *SQL) Count() (int, error) {
if s.conn == nil {
return 0, errors.Err("not connected")
@@ -489,7 +524,7 @@ func (s *SQL) GetBlocked() (map[string]bool, error) {
var hash string
for rows.Next() {
err = rows.Scan(&hash)
err := rows.Scan(&hash)
if err != nil {
return nil, errors.Err(err)
}
@@ -533,7 +568,7 @@ func (s *SQL) MissingBlobsForKnownStream(sdHash string) ([]string, error) {
var hash string
for rows.Next() {
err = rows.Scan(&hash)
err := rows.Scan(&hash)
if err != nil {
return nil, errors.Err(err)
}
@@ -555,7 +590,7 @@ func (s *SQL) AddSDBlob(sdHash string, sdBlobLength int, sdBlob SdBlob) error {
return errors.Err("not connected")
}
sdBlobID, err := s.insertBlob(sdHash, sdBlobLength, true)
sdBlobID, err := s.insertBlob(s.conn, sdHash, sdBlobLength, true)
if err != nil {
return err
}
@@ -565,28 +600,30 @@ func (s *SQL) AddSDBlob(sdHash string, sdBlobLength int, sdBlob SdBlob) error {
return err
}
// insert content blobs and connect them to stream
for _, contentBlob := range sdBlob.Blobs {
if contentBlob.BlobHash == "" {
// null terminator blob
continue
}
return withTx(s.conn, func(tx Transactor) error {
// insert content blobs and connect them to stream
for _, contentBlob := range sdBlob.Blobs {
if contentBlob.BlobHash == "" {
// null terminator blob
continue
}
blobID, err := s.insertBlob(contentBlob.BlobHash, contentBlob.Length, false)
if err != nil {
return err
}
blobID, err := s.insertBlob(tx, contentBlob.BlobHash, contentBlob.Length, false)
if err != nil {
return err
}
args := []interface{}{streamID, blobID, contentBlob.BlobNum}
_, err = s.exec(
"INSERT IGNORE INTO stream_blob (stream_id, blob_id, num) VALUES ("+qt.Qs(len(args))+")",
args...,
)
if err != nil {
return errors.Err(err)
args := []interface{}{streamID, blobID, contentBlob.BlobNum}
_, err = s.exec(tx,
"INSERT IGNORE INTO stream_blob (stream_id, blob_id, num) VALUES ("+qt.Qs(len(args))+")",
args...,
)
if err != nil {
return errors.Err(err)
}
}
}
return nil
return nil
})
}
// GetHashRange gets the smallest and biggest hashes in the db
@@ -636,7 +673,7 @@ func (s *SQL) GetStoredHashesInRange(ctx context.Context, start, end bits.Bitmap
var hash string
ScanLoop:
for rows.Next() {
err = rows.Scan(&hash)
err := rows.Scan(&hash)
if err != nil {
ech <- err
return
@@ -659,39 +696,38 @@ func (s *SQL) GetStoredHashesInRange(ctx context.Context, start, end bits.Bitmap
}
// txFunc is a function that can be wrapped in a transaction
type txFunc func(tx *sql.Tx) error
type txFunc func(tx Transactor) error
// withTx wraps a function in an sql transaction. the transaction is committed if there's no error, or rolled back if there is one.
// if dbOrTx is an sql.DB, a new transaction is started
// withTx wraps a function in an sql transaction. the transaction is committed if there's
// no error, or rolled back if there is one. if dbOrTx is not a Transactor (e.g. if it's
// an *sql.DB), withTx attempts to start a new transaction to use.
func withTx(dbOrTx interface{}, f txFunc) (err error) {
var tx *sql.Tx
var tx Transactor
var ok bool
switch t := dbOrTx.(type) {
case *sql.Tx:
tx = t
case *sql.DB:
tx, err = t.Begin()
tx, ok = dbOrTx.(Transactor)
if !ok {
tx, err = Begin(dbOrTx)
if err != nil {
return err
}
defer func() {
if p := recover(); p != nil {
if rollBackError := tx.Rollback(); rollBackError != nil {
log.Error("failed to rollback tx on panic - ", rollBackError)
}
panic(p)
} else if err != nil {
if rollBackError := tx.Rollback(); rollBackError != nil {
log.Error("failed to rollback tx on panic - ", rollBackError)
}
} else {
err = errors.Err(tx.Commit())
}
}()
default:
return errors.Err("db or tx required")
}
defer func() {
if p := recover(); p != nil {
if rollBackError := tx.Rollback(); rollBackError != nil {
log.Error("failed to rollback tx on panic: ", rollBackError)
}
err = errors.Prefix("panic", p)
} else if err != nil {
if rollBackError := tx.Rollback(); rollBackError != nil {
log.Error("failed to rollback tx: ", rollBackError)
}
} else {
err = errors.Err(tx.Commit())
}
}()
return f(tx)
}
@@ -704,12 +740,12 @@ func closeRows(rows *sql.Rows) {
}
}
func (s *SQL) exec(query string, args ...interface{}) (int64, error) {
func (s *SQL) exec(ex Executor, query string, args ...interface{}) (int64, error) {
s.logQuery(query, args...)
attempt, maxAttempts := 0, 3
Retry:
attempt++
result, err := s.conn.Exec(query, args...)
result, err := ex.Exec(query, args...)
if isLockTimeoutError(err) {
if attempt <= maxAttempts {
//Error 1205: Lock wait timeout exceeded; try restarting transaction
@@ -778,3 +814,47 @@ CREATE TABLE blocked (
);
*/
//func (d *LiteDBBackedStore) selfClean() {
// d.stopper.Add(1)
// defer d.stopper.Done()
// lastCleanup := time.Now()
// const cleanupInterval = 10 * time.Second
// for {
// select {
// case <-d.stopper.Ch():
// log.Infoln("stopping self cleanup")
// return
// default:
// time.Sleep(1 * time.Second)
// }
// if time.Since(lastCleanup) < cleanupInterval {
// continue
//
// blobsCount, err := d.db.BlobsCount()
// if err != nil {
// log.Errorf(errors.FullTrace(err))
// }
// if blobsCount >= d.maxItems {
// itemsToDelete := blobsCount / 100 * 10
// blobs, err := d.db.GetLRUBlobs(itemsToDelete)
// if err != nil {
// log.Errorf(errors.FullTrace(err))
// }
// for _, hash := range blobs {
// select {
// case <-d.stopper.Ch():
// return
// default:
//
// }
// err = d.Delete(hash)
// if err != nil {
// log.Errorf(errors.FullTrace(err))
// }
// metrics.CacheLRUEvictCount.With(metrics.CacheLabels(d.Name(), d.component)).Inc()
// }
// }
// lastCleanup = time.Now()
// }
//}
+45
View File
@@ -0,0 +1,45 @@
package db
import (
"database/sql"
"github.com/lbryio/lbry.go/v2/extras/errors"
)
// Executor can perform SQL queries.
type Executor interface {
Exec(query string, args ...interface{}) (sql.Result, error)
Query(query string, args ...interface{}) (*sql.Rows, error)
QueryRow(query string, args ...interface{}) *sql.Row
}
// Transactor can commit and rollback, on top of being able to execute queries.
type Transactor interface {
Commit() error
Rollback() error
Executor
}
// Begin begins a transaction
func Begin(db interface{}) (Transactor, error) {
type beginner interface {
Begin() (Transactor, error)
}
creator, ok := db.(beginner)
if ok {
return creator.Begin()
}
type sqlBeginner interface {
Begin() (*sql.Tx, error)
}
creator2, ok := db.(sqlBeginner)
if ok {
return creator2.Begin()
}
return nil, errors.Err("database does not support transactions")
}
+42 -132
View File
@@ -1,139 +1,49 @@
module github.com/lbryio/reflector.go
go 1.25
replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19
require (
github.com/aws/aws-sdk-go-v2 v1.41.0
github.com/aws/aws-sdk-go-v2/config v1.32.6
github.com/aws/aws-sdk-go-v2/credentials v1.19.6
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.18
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0
github.com/bluele/gcache v0.0.2
github.com/brk0v/directio v0.0.0-20241105172640-ae9d82eb8fee
github.com/btcsuite/btcd v0.24.2
github.com/btcsuite/btcutil v1.0.2
github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
github.com/ekyoung/gin-nice-recovery v0.0.0-20160510022553-1654dca486db
github.com/gin-gonic/gin v1.11.0
github.com/go-sql-driver/mysql v1.9.3
github.com/gogo/protobuf v1.3.2
github.com/golang/protobuf v1.5.4
github.com/google/gops v0.3.28
github.com/gorilla/mux v1.8.1
github.com/hashicorp/serf v0.10.2
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf
github.com/karrick/godirwalk v1.17.0
github.com/lbryio/chainquery v1.9.1-0.20240927170248-48c092515dea
github.com/lbryio/lbry.go/v2 v2.7.2-0.20230307181431-a01aa6dc0629
github.com/lbryio/types v0.0.0-20220224142228-73610f6654a6
github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5
github.com/prometheus/client_golang v1.23.2
github.com/quic-go/quic-go v0.58.0
github.com/sirupsen/logrus v1.9.3
github.com/spf13/cast v1.10.0
github.com/spf13/cobra v1.10.2
github.com/spf13/viper v1.21.0
github.com/stretchr/testify v1.11.1
github.com/volatiletech/null/v8 v8.1.2
go.uber.org/atomic v1.11.0
golang.org/x/sync v0.19.0
)
//replace github.com/lbryio/lbry.go/v2 => ../lbry.go
require (
filippo.io/edwards25519 v1.1.0 // indirect
github.com/armon/go-metrics v0.4.1 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 // indirect
github.com/aws/smithy-go v1.24.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/btcsuite/btclog v1.0.0 // indirect
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd // indirect
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 // indirect
github.com/bytedance/sonic v1.14.0 // indirect
github.com/bytedance/sonic/loader v0.3.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/cloudwego/base64x v0.1.6 // indirect
github.com/friendsofgo/errors v0.9.2 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.8 // indirect
github.com/gin-contrib/sse v1.1.0 // indirect
github.com/go-errors/errors v1.5.1 // indirect
github.com/go-ini/ini v1.67.0 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.27.0 // indirect
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
github.com/goccy/go-json v0.10.5 // indirect
github.com/goccy/go-yaml v1.18.0 // indirect
github.com/gofrs/uuid v4.4.0+incompatible // indirect
github.com/google/btree v1.1.2 // indirect
github.com/gorilla/rpc v1.2.0 // indirect
github.com/gorilla/websocket v1.5.3 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
github.com/hashicorp/go-metrics v0.5.4 // indirect
github.com/hashicorp/go-msgpack/v2 v2.1.2 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-sockaddr v1.0.5 // indirect
github.com/hashicorp/golang-lru v1.0.2 // indirect
github.com/hashicorp/memberlist v0.5.2 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/miekg/dns v1.1.56 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.66.1 // indirect
github.com/prometheus/procfs v0.16.1 // indirect
github.com/quic-go/qpack v0.6.0 // indirect
github.com/sagikazarmark/locafero v0.11.0 // indirect
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
github.com/shopspring/decimal v1.4.0 // indirect
github.com/slack-go/slack v0.16.0 // indirect
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
github.com/spf13/afero v1.15.0 // indirect
github.com/spf13/pflag v1.0.10 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.3.0 // indirect
github.com/volatiletech/inflect v0.0.1 // indirect
github.com/volatiletech/randomize v0.0.1 // indirect
github.com/volatiletech/strmangle v0.0.8 // indirect
go.yaml.in/yaml/v2 v2.4.2 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/arch v0.20.0 // indirect
golang.org/x/crypto v0.41.0 // indirect
golang.org/x/mod v0.27.0 // indirect
golang.org/x/net v0.43.0 // indirect
golang.org/x/sys v0.35.0 // indirect
golang.org/x/text v0.28.0 // indirect
golang.org/x/time v0.8.0 // indirect
golang.org/x/tools v0.36.0 // indirect
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
google.golang.org/protobuf v1.36.9 // indirect
gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
github.com/aws/aws-sdk-go v1.16.11
github.com/bluele/gcache v0.0.2
github.com/bparli/lfuda-go v0.3.1
github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3
github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d
github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2
github.com/davecgh/go-spew v1.1.1
github.com/gin-gonic/gin v1.7.1
github.com/go-sql-driver/mysql v1.4.1
github.com/golang/protobuf v1.4.2
github.com/google/gops v0.3.18
github.com/gorilla/mux v1.7.4
github.com/hashicorp/go-msgpack v0.5.5 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/hashicorp/memberlist v0.1.4 // indirect
github.com/hashicorp/serf v0.8.2
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf
github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07
github.com/karrick/godirwalk v1.16.1
github.com/lbryio/chainquery v1.9.0
github.com/lbryio/lbry.go v1.1.2 // indirect
github.com/lbryio/lbry.go/v2 v2.7.2-0.20210416195322-6516df1418e3
github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386
github.com/lucas-clemente/quic-go v0.20.1
github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6
github.com/prometheus/client_golang v0.9.3
github.com/sirupsen/logrus v1.4.2
github.com/spf13/afero v1.4.1 // indirect
github.com/spf13/cast v1.3.0
github.com/spf13/cobra v0.0.3
github.com/spf13/viper v1.7.1 // indirect
github.com/stretchr/testify v1.7.0
github.com/volatiletech/null v8.0.0+incompatible
go.uber.org/atomic v1.5.1
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4 // indirect
google.golang.org/appengine v1.6.2 // indirect
)
go 1.15
+572 -332
View File
File diff suppressed because it is too large. Load diff
+11 -36
View File
@@ -69,7 +69,6 @@ const (
DirectionDownload = "download" // from reflector
LabelCacheType = "cache_type"
LabelOrigin = "origin"
LabelComponent = "component"
LabelSource = "source"
@@ -87,13 +86,8 @@ const (
errUnexpectedEOFStr = "unexpected_eof_str"
errJSONSyntax = "json_syntax"
errBlobTooBig = "blob_too_big"
errInvalidPeerJSON = "invalid_peer_json"
errInvalidPeerData = "invalid_peer_data"
errRequestTooLarge = "request_too_large"
errDeadlineExceeded = "deadline_exceeded"
errHashMismatch = "hash_mismatch"
errProtectedBlob = "protected_blob"
errInvalidBlobHash = "invalid_blob_hash"
errZeroByteBlob = "zero_byte_blob"
errInvalidCharacter = "invalid_character"
errBlobNotFound = "blob_not_found"
@@ -136,12 +130,18 @@ var (
Name: "hit_total",
Help: "Total number of blobs retrieved from the cache storage",
}, []string{LabelCacheType, LabelComponent})
ItttHitCount = promauto.NewCounterVec(prometheus.CounterOpts{
ThisHitCount = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Subsystem: subsystemITTT,
Name: "hits_total",
Help: "Total number of blobs retrieved from the this/that storage",
}, []string{LabelOrigin})
Name: "this_hit_total",
Help: "Total number of blobs retrieved from the this storage",
})
ThatHitCount = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Subsystem: subsystemITTT,
Name: "that_hit_total",
Help: "Total number of blobs retrieved from the that storage",
})
CacheMissCount = promauto.NewCounterVec(prometheus.CounterOpts{
Namespace: ns,
Subsystem: subsystemCache,
@@ -154,7 +154,7 @@ var (
Name: "origin_requests_total",
Help: "How many Get requests are in flight from the cache to the origin",
}, []string{LabelCacheType, LabelComponent})
//during thundering-herd situations, the metric below should be a lot smaller than the metric above
// during thundering-herd situations, the metric below should be a lot smaller than the metric above
CacheWaitingRequestsCount = promauto.NewGaugeVec(prometheus.GaugeOpts{
Namespace: ns,
Subsystem: subsystemCache,
@@ -224,11 +224,6 @@ var (
Name: "s3_out_bytes",
Help: "Total number of outgoing bytes (to S3)",
})
MtrInBytesUpstream = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "upstream_in_bytes",
Help: "Total number of incoming bytes (from Upstream)",
})
MtrInBytesS3 = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "s3_in_bytes",
@@ -239,11 +234,6 @@ var (
Name: "http3_blob_request_queue_size",
Help: "Blob requests of https queue size",
})
HttpBlobReqQueue = promauto.NewGauge(prometheus.GaugeOpts{
Namespace: ns,
Name: "http_blob_request_queue_size",
Help: "Blob requests queue size of the HTTP protocol",
})
RoutinesQueue = promauto.NewGaugeVec(prometheus.GaugeOpts{
Namespace: ns,
Name: "routines",
@@ -257,11 +247,6 @@ func CacheLabels(name, component string) prometheus.Labels {
LabelComponent: component,
}
}
func ItttLabels(orig string) prometheus.Labels {
return prometheus.Labels{
LabelOrigin: orig,
}
}
func TrackError(direction string, e error) (shouldLog bool) { // shouldLog is a hack, but whatever
if e == nil {
@@ -306,20 +291,10 @@ func TrackError(direction string, e error) (shouldLog bool) { // shouldLog is a
} else if strings.Contains(err.Error(), "blob must be at most") {
//log.Warnln("blob must be at most X bytes is not the same as ErrBlobTooBig")
errType = errBlobTooBig
} else if strings.Contains(err.Error(), "invalid json request") {
errType = errInvalidPeerJSON
} else if strings.Contains(err.Error(), "Invalid data") {
errType = errInvalidPeerData
} else if strings.Contains(err.Error(), "request is too large") {
errType = errRequestTooLarge
} else if strings.Contains(err.Error(), "Invalid blob hash length") {
errType = errInvalidBlobHash
} else if strings.Contains(err.Error(), "hash of received blob data does not match hash from send request") {
errType = errHashMismatch
} else if strings.Contains(err.Error(), "blob not found") {
errType = errBlobNotFound
} else if strings.Contains(err.Error(), "requested blob is protected") {
errType = errProtectedBlob
} else if strings.Contains(err.Error(), "0-byte blob received") {
errType = errZeroByteBlob
} else if strings.Contains(err.Error(), "PROTOCOL_VIOLATION: tried to retire connection") {
-347
View File
@@ -1,347 +0,0 @@
package lite_db
import (
"database/sql"
"time"
"github.com/lbryio/lbry.go/v2/extras/errors"
qt "github.com/lbryio/lbry.go/v2/extras/query"
"github.com/go-sql-driver/mysql"
log "github.com/sirupsen/logrus"
"github.com/volatiletech/null/v8"
)
// SdBlob is a special blob that contains information on the rest of the blobs in the stream
type SdBlob struct {
StreamName string `json:"stream_name"`
StreamType string `json:"stream_type"`
Key string `json:"key"`
SuggestedFileName string `json:"suggested_file_name"`
StreamHash string `json:"stream_hash"`
Blobs []struct {
BlobHash string `json:"blob_hash,omitempty"`
IV string `json:"iv"`
Length int `json:"length"`
BlobNum int `json:"blob_num"`
} `json:"blobs"`
}
// SQL implements the DB interface
type SQL struct {
conn *sql.DB
TrackAccessTime bool
}
func logQuery(query string, args ...interface{}) {
s, err := qt.InterpolateParams(query, args...)
if err != nil {
log.Errorln(err)
} else {
log.Debugln(s)
}
}
// Connect will create a connection to the database
func (s *SQL) Connect(dsn string) error {
var err error
// interpolateParams is necessary. otherwise uploading a stream with thousands of blobs
// will hit MySQL's max_prepared_stmt_count limit because the prepared statements are all
// opened inside a transaction. closing them manually doesn't seem to help
dsn += "?parseTime=1&collation=utf8mb4_unicode_ci&interpolateParams=1"
s.conn, err = sql.Open("mysql", dsn)
if err != nil {
return errors.Err(err)
}
s.conn.SetMaxIdleConns(12)
return errors.Err(s.conn.Ping())
}
// AddBlob adds a blob to the database.
func (s *SQL) AddBlob(hash string, length int) error {
if s.conn == nil {
return errors.Err("not connected")
}
_, err := s.insertBlob(hash, length)
return err
}
func (s *SQL) insertBlob(hash string, length int) (int64, error) {
if length <= 0 {
return 0, errors.Err("length must be positive")
}
const isStored = true
now := time.Now()
args := []interface{}{hash, isStored, length, now}
blobID, err := s.exec(
"INSERT INTO blob_ (hash, is_stored, length, last_accessed_at) VALUES ("+qt.Qs(len(args))+") ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored)), last_accessed_at=VALUES(last_accessed_at)",
args...,
)
if err != nil {
return 0, err
}
if blobID == 0 {
err = s.conn.QueryRow("SELECT id FROM blob_ WHERE hash = ?", hash).Scan(&blobID)
if err != nil {
return 0, errors.Err(err)
}
if blobID == 0 {
return 0, errors.Err("blob ID is 0 even after INSERTing and SELECTing")
}
}
return blobID, nil
}
// HasBlob checks if the database contains the blob information.
func (s *SQL) HasBlob(hash string) (bool, error) {
exists, err := s.HasBlobs([]string{hash})
if err != nil {
return false, err
}
return exists[hash], nil
}
// HasBlobs checks if the database contains the set of blobs and returns a bool map.
func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
exists, streamsNeedingTouch, err := s.hasBlobs(hashes)
_ = s.touch(streamsNeedingTouch)
return exists, err
}
func (s *SQL) touch(blobIDs []uint64) error {
if len(blobIDs) == 0 {
return nil
}
query := "UPDATE blob_ SET last_accessed_at = ? WHERE id IN (" + qt.Qs(len(blobIDs)) + ")"
args := make([]interface{}, len(blobIDs)+1)
args[0] = time.Now()
for i := range blobIDs {
args[i+1] = blobIDs[i]
}
startTime := time.Now()
_, err := s.exec(query, args...)
log.Debugf("blobs access query touched %d blobs and took %s", len(blobIDs), time.Since(startTime))
return errors.Err(err)
}
func (s *SQL) hasBlobs(hashes []string) (map[string]bool, []uint64, error) {
if s.conn == nil {
return nil, nil, errors.Err("not connected")
}
var (
hash string
blobID uint64
lastAccessedAt null.Time
)
var needsTouch []uint64
exists := make(map[string]bool)
touchDeadline := time.Now().AddDate(0, 0, -1) // touch blob if last accessed before this time
maxBatchSize := 10000
doneIndex := 0
for len(hashes) > doneIndex {
sliceEnd := doneIndex + maxBatchSize
if sliceEnd > len(hashes) {
sliceEnd = len(hashes)
}
log.Debugf("getting hashes[%d:%d] of %d", doneIndex, sliceEnd, len(hashes))
batch := hashes[doneIndex:sliceEnd]
// TODO: this query doesn't work for SD blobs, which are not in the stream_blob table
query := `SELECT hash, id, last_accessed_at
FROM blob_
WHERE is_stored = ? and hash IN (` + qt.Qs(len(batch)) + `)`
args := make([]interface{}, len(batch)+1)
args[0] = true
for i := range batch {
args[i+1] = batch[i]
}
logQuery(query, args...)
err := func() error {
startTime := time.Now()
rows, err := s.conn.Query(query, args...)
log.Debugf("hashes query took %s", time.Since(startTime))
if err != nil {
return errors.Err(err)
}
defer closeRows(rows)
for rows.Next() {
err = rows.Scan(&hash, &blobID, &lastAccessedAt)
if err != nil {
return errors.Err(err)
}
exists[hash] = true
if s.TrackAccessTime && (!lastAccessedAt.Valid || lastAccessedAt.Time.Before(touchDeadline)) {
needsTouch = append(needsTouch, blobID)
}
}
err = rows.Err()
if err != nil {
return errors.Err(err)
}
doneIndex += len(batch)
return nil
}()
if err != nil {
return nil, nil, err
}
}
return exists, needsTouch, nil
}
// Delete will remove the blob from the db
func (s *SQL) Delete(hash string) error {
_, err := s.exec("UPDATE blob_ set is_stored = ? WHERE hash = ?", 0, hash)
return errors.Err(err)
}
// AddSDBlob insert the SD blob and all the content blobs. The content blobs are marked as "not stored",
// but they are tracked so reflector knows what it is missing.
func (s *SQL) AddSDBlob(sdHash string, sdBlobLength int) error {
if s.conn == nil {
return errors.Err("not connected")
}
_, err := s.insertBlob(sdHash, sdBlobLength)
return err
}
// GetHashRange gets the smallest and biggest hashes in the db
func (s *SQL) GetLRUBlobs(maxBlobs int) ([]string, error) {
if s.conn == nil {
return nil, errors.Err("not connected")
}
query := "SELECT hash from blob_ where is_stored = ? order by last_accessed_at limit ?"
const isStored = true
logQuery(query, isStored, maxBlobs)
rows, err := s.conn.Query(query, isStored, maxBlobs)
if err != nil {
return nil, errors.Err(err)
}
defer closeRows(rows)
blobs := make([]string, 0, maxBlobs)
for rows.Next() {
var hash string
err := rows.Scan(&hash)
if err != nil {
return nil, errors.Err(err)
}
blobs = append(blobs, hash)
}
return blobs, nil
}
func (s *SQL) AllBlobs() ([]string, error) {
if s.conn == nil {
return nil, errors.Err("not connected")
}
query := "SELECT hash from blob_ where is_stored = ?" //TODO: maybe sorting them makes more sense?
const isStored = true
logQuery(query, isStored)
rows, err := s.conn.Query(query, isStored)
if err != nil {
return nil, errors.Err(err)
}
defer closeRows(rows)
totalBlobs, err := s.BlobsCount()
if err != nil {
return nil, err
}
blobs := make([]string, 0, totalBlobs)
for rows.Next() {
var hash string
err := rows.Scan(&hash)
if err != nil {
return nil, errors.Err(err)
}
blobs = append(blobs, hash)
}
return blobs, nil
}
func (s *SQL) BlobsCount() (int, error) {
if s.conn == nil {
return 0, errors.Err("not connected")
}
query := "SELECT count(id) from blob_ where is_stored = ?" //TODO: maybe sorting them makes more sense?
const isStored = true
logQuery(query, isStored)
var count int
err := s.conn.QueryRow(query, isStored).Scan(&count)
return count, errors.Err(err)
}
func closeRows(rows *sql.Rows) {
if rows != nil {
err := rows.Close()
if err != nil {
log.Error("error closing rows: ", err)
}
}
}
func (s *SQL) exec(query string, args ...interface{}) (int64, error) {
logQuery(query, args...)
attempt, maxAttempts := 0, 3
Retry:
attempt++
result, err := s.conn.Exec(query, args...)
if isLockTimeoutError(err) {
if attempt <= maxAttempts {
//Error 1205: Lock wait timeout exceeded; try restarting transaction
goto Retry
}
err = errors.Prefix("Lock timeout for query "+query, err)
}
if err != nil {
return 0, errors.Err(err)
}
lastID, err := result.LastInsertId()
return lastID, errors.Err(err)
}
func isLockTimeoutError(err error) bool {
e, ok := err.(*mysql.MySQLError)
return ok && e != nil && e.Number == 1205
}
/* SQL schema
in prod make sure you use latin1 or utf8 charset, NOT utf8mb4. that's a waste of space.
CREATE TABLE `blob_` (
`id` bigint unsigned NOT NULL AUTO_INCREMENT,
`hash` char(96) NOT NULL,
`is_stored` tinyint(1) NOT NULL DEFAULT '0',
`length` bigint unsigned DEFAULT NULL,
`last_accessed_at` datetime DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
UNIQUE KEY `id` (`id`),
UNIQUE KEY `blob_hash_idx` (`hash`),
KEY `blob_last_accessed_idx` (`last_accessed_at`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1
*/
+4
View File
@@ -1,6 +1,9 @@
package main
import (
"math/rand"
"time"
"github.com/lbryio/reflector.go/cmd"
"github.com/google/gops/agent"
@@ -11,5 +14,6 @@ func main() {
if err := agent.Listen(agent.Options{}); err != nil {
log.Fatal(err)
}
rand.Seed(time.Now().UnixNano())
cmd.Execute()
}
+8 -30
View File
@@ -6,36 +6,9 @@ import (
"time"
)
var (
name = "prism-bin"
version = "unknown"
commit = "unknown"
commitLong = "unknown"
branch = "unknown"
Time = "unknown"
BuildTime time.Time
)
// Name returns main application name
func Name() string {
return name
}
// Version returns current application version
func Version() string {
return version
}
// FullName returns current app version, commit and build time
func FullName() string {
return fmt.Sprintf(
`Name: %v
Version: %v
branch: %v
commit: %v
commit long: %v
build date: %v`, Name(), Version(), branch, commit, commitLong, BuildTime.String())
}
var Version = ""
var Time = ""
var BuildTime time.Time
func init() {
if Time != "" {
@@ -47,6 +20,11 @@ func init() {
}
func VersionString() string {
version := Version
if version == "" {
version = "<unset>"
}
var buildTime string
if BuildTime.IsZero() {
buildTime = "<now>"
+4 -1
View File
@@ -18,11 +18,14 @@ import (
log "github.com/sirupsen/logrus"
)
// ErrBlobExists is a default error for when a blob already exists on the reflector server.
var ErrBlobExists = errors.Base("blob exists on server")
// Client is an instance of a client connected to a server.
type Client struct {
Timeout time.Duration
conn net.Conn
buf *bufio.Reader
Timeout time.Duration
connected bool
}
@@ -15,17 +15,16 @@ import (
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/quic-go/quic-go/http3"
"github.com/lucas-clemente/quic-go/http3"
log "github.com/sirupsen/logrus"
)
// Client is an instance of a client connected to a server.
type Client struct {
conn *http.Client
roundTripper *http3.Transport
ServerAddr string
Timeout time.Duration
conn *http.Client
roundTripper *http3.RoundTripper
ServerAddr string
}
// Close closes the connection with the client.
@@ -69,7 +68,7 @@ func (c *Client) HasBlob(hash string) (bool, error) {
if err != nil {
return false, errors.Err(err)
}
defer func() { _ = resp.Body.Close() }()
defer resp.Body.Close()
if resp.StatusCode == http.StatusOK {
return true, nil
}
@@ -86,7 +85,7 @@ func (c *Client) GetBlob(hash string) (stream.Blob, shared.BlobTrace, error) {
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err(err)
}
defer func() { _ = resp.Body.Close() }()
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound {
fmt.Printf("%s blob not found %d\n", hash, resp.StatusCode)
@@ -100,8 +99,7 @@ func (c *Client) GetBlob(hash string) (stream.Blob, shared.BlobTrace, error) {
serialized := resp.Header.Get("Via")
trace := shared.NewBlobTrace(time.Since(start), "http3")
if serialized != "" {
var parsedTrace *shared.BlobTrace
parsedTrace, err = shared.Deserialize(serialized)
parsedTrace, err := shared.Deserialize(serialized)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "http3"), err
}
+19 -27
View File
@@ -11,38 +11,32 @@ import (
"math/big"
"net/http"
"strconv"
"strings"
"sync"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/gorilla/mux"
"github.com/quic-go/quic-go"
"github.com/quic-go/quic-go/http3"
"github.com/lucas-clemente/quic-go"
"github.com/lucas-clemente/quic-go/http3"
log "github.com/sirupsen/logrus"
)
// Server is an instance of a peer server that houses the listener and store.
type Server struct {
store store.BlobStore
grp *stop.Group
address string
concurrentRequests int
store store.BlobStore
grp *stop.Group
}
// NewServer returns an initialized Server pointer.
func NewServer(store store.BlobStore, requestQueueSize int, address string) *Server {
func NewServer(store store.BlobStore, requestQueueSize int) *Server {
return &Server{
store: store,
grp: stop.New(),
concurrentRequests: requestQueueSize,
address: address,
store: store,
grp: stop.New(),
}
}
@@ -69,8 +63,8 @@ type availabilityResponse struct {
}
// Start starts the server listener to handle connections.
func (s *Server) Start() error {
log.Println("HTTP3 peer listening on " + s.address)
func (s *Server) Start(address string) error {
log.Println("HTTP3 peer listening on " + address)
window500M := 500 * 1 << 20
quicConf := &quic.Config{
@@ -116,12 +110,14 @@ func (s *Server) Start() error {
}
})
server := http3.Server{
Addr: s.address,
Handler: r,
TLSConfig: generateTLSConfig(),
QUICConfig: quicConf,
Server: &http.Server{
Handler: r,
Addr: address,
TLSConfig: generateTLSConfig(),
},
QuicConfig: quicConf,
}
go InitWorkers(s, s.concurrentRequests)
go InitWorkers(s, 200)
go s.listenForShutdown(&server)
s.grp.Add(1)
go func() {
@@ -158,7 +154,7 @@ func generateTLSConfig() *tls.Config {
func (s *Server) listenAndServe(server *http3.Server) {
err := server.ListenAndServe()
if err != nil && !strings.Contains(err.Error(), "Server closed") {
if err != nil && err.Error() != "server closed" {
log.Errorln(errors.FullTrace(err))
}
}
@@ -183,15 +179,11 @@ func (s *Server) HandleGetBlob(w http.ResponseWriter, r *http.Request) {
wantsTrace = false
}
}
if reflector.IsProtected(requestedBlob) {
http.Error(w, "requested blob is protected", http.StatusForbidden)
return
}
blob, trace, err := s.store.Get(requestedBlob)
if wantsTrace {
var serialized string
serialized, err = trace.Serialize()
serialized, err := trace.Serialize()
if err != nil {
http.Error(w, err.Error(), http.StatusNotFound)
return
+116
View File
@@ -0,0 +1,116 @@
package http3
import (
"crypto/tls"
"crypto/x509"
"net/http"
"strings"
"sync"
"time"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/reflector.go/store"
"github.com/lucas-clemente/quic-go"
"github.com/lucas-clemente/quic-go/http3"
)
// Store is a blob store that gets blobs from a peer.
// It satisfies the store.BlobStore interface but cannot put or delete blobs.
type Store struct {
opts StoreOpts
NotFoundCache *sync.Map
}
// StoreOpts allows to set options for a new Store.
type StoreOpts struct {
Address string
Timeout time.Duration
}
// NewStore makes a new peer store.
func NewStore(opts StoreOpts) *Store {
return &Store{opts: opts, NotFoundCache: &sync.Map{}}
}
func (p *Store) getClient() (*Client, error) {
var qconf quic.Config
window500M := 500 * 1 << 20
qconf.MaxStreamReceiveWindow = uint64(window500M)
qconf.MaxConnectionReceiveWindow = uint64(window500M)
qconf.EnableDatagrams = true
qconf.HandshakeIdleTimeout = 4 * time.Second
qconf.MaxIdleTimeout = 20 * time.Second
pool, err := x509.SystemCertPool()
if err != nil {
return nil, err
}
roundTripper := &http3.RoundTripper{
TLSClientConfig: &tls.Config{
RootCAs: pool,
InsecureSkipVerify: true,
},
QuicConfig: &qconf,
}
connection := &http.Client{
Transport: roundTripper,
}
c := &Client{
conn: connection,
roundTripper: roundTripper,
ServerAddr: p.opts.Address,
}
return c, errors.Prefix("connection error", err)
}
func (p *Store) Name() string { return "http3" }
// Has asks the peer if they have a hash
func (p *Store) Has(hash string) (bool, error) {
c, err := p.getClient()
if err != nil {
return false, err
}
defer c.Close()
return c.HasBlob(hash)
}
// Get downloads the blob from the peer
func (p *Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
if lastChecked, ok := p.NotFoundCache.Load(hash); ok {
if lastChecked.(time.Time).After(time.Now().Add(-5 * time.Minute)) {
return nil, shared.NewBlobTrace(time.Since(start), p.Name()+"-notfoundcache"), store.ErrBlobNotFound
}
}
c, err := p.getClient()
if err != nil && strings.Contains(err.Error(), "blob not found") {
p.NotFoundCache.Store(hash, time.Now())
}
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), p.Name()), err
}
defer c.Close()
return c.GetBlob(hash)
}
// Put is not supported
func (p *Store) Put(hash string, blob stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// PutSD is not supported
func (p *Store) PutSD(hash string, blob stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// Delete is not supported
func (p *Store) Delete(hash string) error {
return errors.Err(shared.ErrNotImplemented)
}
// Delete is not supported
func (p *Store) Shutdown() {
return
}
@@ -33,6 +33,7 @@ func InitWorkers(server *Server, workers int) {
}
}(i)
}
return
}
func enqueue(b *blobRequest) {
+31 -48
View File
@@ -31,18 +31,17 @@ const (
// Server is an instance of a peer server that houses the listener and store.
type Server struct {
store store.BlobStore
grp *stop.Group
address string
closed bool
store store.BlobStore
closed bool
grp *stop.Group
}
// NewServer returns an initialized Server pointer.
func NewServer(store store.BlobStore, address string) *Server {
func NewServer(store store.BlobStore) *Server {
return &Server{
store: store,
grp: stop.New(),
address: address,
store: store,
grp: stop.New(),
}
}
@@ -54,9 +53,9 @@ func (s *Server) Shutdown() {
}
// Start starts the server listener to handle connections.
func (s *Server) Start() error {
log.Println("peer listening on " + s.address)
l, err := net.Listen("tcp4", s.address)
func (s *Server) Start(address string) error {
log.Println("peer listening on " + address)
l, err := net.Listen("tcp4", address)
if err != nil {
return err
}
@@ -228,37 +227,32 @@ func (s *Server) handleCompositeRequest(data []byte) ([]byte, error) {
if err != nil {
var je *json.SyntaxError
if ee.As(err, &je) {
return nil, errors.Err("invalid json request: offset %d in data %s", je.Offset, hex.EncodeToString(data))
return nil, errors.Err("invalid json at offset %d in data %s", je.Offset, hex.EncodeToString(data))
}
return nil, errors.Err(err)
}
response := compositeResponse{
LbrycrdAddress: LbrycrdAddress,
AvailableBlobs: []string{},
}
if len(request.RequestedBlobs) > 0 {
var availableBlobs []string
for _, blobHash := range request.RequestedBlobs {
if reflector.IsProtected(blobHash) {
return nil, errors.Err("requested blob is protected")
}
var exists bool
exists, err = s.store.Has(blobHash)
exists, err := s.store.Has(blobHash)
if err != nil {
return nil, err
}
if exists {
response.AvailableBlobs = append(response.AvailableBlobs, blobHash)
availableBlobs = append(availableBlobs, blobHash)
}
}
response.AvailableBlobs = availableBlobs
}
if request.BlobDataPaymentRate != nil {
response.BlobDataPaymentRate = paymentRateAccepted
if *request.BlobDataPaymentRate < 0 {
response.BlobDataPaymentRate = paymentRateTooLow
}
response.BlobDataPaymentRate = paymentRateAccepted
if request.BlobDataPaymentRate < 0 {
response.BlobDataPaymentRate = paymentRateTooLow
}
var blob []byte
@@ -273,14 +267,14 @@ func (s *Server) handleCompositeRequest(data []byte) ([]byte, error) {
blob, trace, err = s.store.Get(request.RequestedBlob)
log.Debug(trace.String())
if errors.Is(err, store.ErrBlobNotFound) {
response.IncomingBlob = &incomingBlob{
response.IncomingBlob = incomingBlob{
Error: err.Error(),
}
} else if err != nil {
return nil, err
} else {
response.IncomingBlob = &incomingBlob{
BlobHash: request.RequestedBlob,
response.IncomingBlob = incomingBlob{
BlobHash: reflector.BlobHash(blob),
Length: len(blob),
}
metrics.MtrOutBytesTcp.Add(float64(len(blob)))
@@ -308,15 +302,7 @@ func (s *Server) logError(e error) {
}
func readNextMessage(buf *bufio.Reader) ([]byte, error) {
firstByte, err := buf.ReadByte()
if err != nil {
return nil, err
}
if firstByte != '{' {
// every request starts with '{'. Checking here disconnects earlier, so we don't wait until timeout
return nil, errInvalidData
}
msg := []byte("{")
msg := make([]byte, 0)
eof := false
for {
@@ -337,8 +323,6 @@ func readNextMessage(buf *bufio.Reader) ([]byte, error) {
if len(msg) > maxRequestSize {
return msg, errRequestTooLarge
} else if len(msg) > 0 && msg[0] != '{' {
return msg, errInvalidData
}
// yes, this is how the peer protocol knows when the request finishes
@@ -373,11 +357,10 @@ const (
)
var errRequestTooLarge = errors.Base("request is too large")
var errInvalidData = errors.Base("Invalid data")
type availabilityRequest struct {
RequestedBlobs []string `json:"requested_blobs"`
LbrycrdAddress bool `json:"lbrycrd_address"`
RequestedBlobs []string `json:"requested_blobs"`
}
type availabilityResponse struct {
@@ -403,20 +386,20 @@ type incomingBlob struct {
Length int `json:"length"`
}
type blobResponse struct {
RequestTrace *shared.BlobTrace
IncomingBlob incomingBlob `json:"incoming_blob"`
RequestTrace *shared.BlobTrace
}
type compositeRequest struct {
BlobDataPaymentRate *float64 `json:"blob_data_payment_rate"`
RequestedBlob string `json:"requested_blob"`
RequestedBlobs []string `json:"requested_blobs"`
LbrycrdAddress bool `json:"lbrycrd_address"`
RequestedBlobs []string `json:"requested_blobs"`
BlobDataPaymentRate float64 `json:"blob_data_payment_rate"`
RequestedBlob string `json:"requested_blob"`
}
type compositeResponse struct {
IncomingBlob *incomingBlob `json:"incoming_blob,omitempty"`
LbrycrdAddress string `json:"lbrycrd_address,omitempty"`
BlobDataPaymentRate string `json:"blob_data_payment_rate,omitempty"`
AvailableBlobs []string `json:"available_blobs"`
LbrycrdAddress string `json:"lbrycrd_address,omitempty"`
AvailableBlobs []string `json:"available_blobs,omitempty"`
BlobDataPaymentRate string `json:"blob_data_payment_rate,omitempty"`
IncomingBlob incomingBlob `json:"incoming_blob,omitempty"`
}
@@ -2,10 +2,7 @@ package peer
import (
"bytes"
"io"
"net"
"testing"
"time"
"github.com/lbryio/reflector.go/store"
)
@@ -37,9 +34,7 @@ var availabilityRequests = []pair{
}
func getServer(t *testing.T, withBlobs bool) *Server {
st := store.NewMemStore(store.MemParams{
Name: "test",
})
st := store.NewMemStore()
if withBlobs {
for k, v := range blobs {
err := st.Put(k, v)
@@ -48,7 +43,7 @@ func getServer(t *testing.T, withBlobs bool) *Server {
}
}
}
return NewServer(st, "127.0.0.1:50505")
return NewServer(st)
}
func TestAvailabilityRequest_NoBlobs(t *testing.T) {
@@ -80,62 +75,3 @@ func TestAvailabilityRequest_WithBlobs(t *testing.T) {
}
}
}
func TestRequestFromConnection(t *testing.T) {
s := getServer(t, true)
err := s.Start()
defer s.Shutdown()
if err != nil {
t.Error("error starting server", err)
}
for _, p := range availabilityRequests {
conn, err := net.Dial("tcp", "127.0.0.1:50505")
if err != nil {
t.Error("error opening connection", err)
}
defer func() { _ = conn.Close() }()
response := make([]byte, 8192)
_, err = conn.Write(p.request)
if err != nil {
t.Error("error writing", err)
}
_, err = conn.Read(response)
if err != nil {
t.Error("error reading", err)
}
if !bytes.Equal(response[:len(p.response)], p.response) {
t.Errorf("Response did not match expected response.\nExpected: %s\nGot: %s", string(p.response), string(response))
}
}
}
func TestInvalidData(t *testing.T) {
s := getServer(t, true)
err := s.Start()
defer s.Shutdown()
if err != nil {
t.Error("error starting server", err)
}
conn, err := net.Dial("tcp", "127.0.0.1:50505")
if err != nil {
t.Error("error opening connection", err)
}
defer func() { _ = conn.Close() }()
response := make([]byte, 8192)
_, err = conn.Write([]byte("hello dear server, I would like blobs. Please"))
if err != nil {
t.Error("error writing", err)
}
err = conn.SetReadDeadline(time.Now().Add(5 * time.Second))
if err != nil {
t.Error("error setting read deadline", err)
}
_, err = conn.Read(response)
if err != io.EOF {
t.Error("error reading", err)
}
println(response)
}
+82
View File
@@ -0,0 +1,82 @@
package peer
import (
"strings"
"time"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/reflector.go/store"
)
// Store is a blob store that gets blobs from a peer.
// It satisfies the store.BlobStore interface but cannot put or delete blobs.
type Store struct {
opts StoreOpts
}
// StoreOpts allows to set options for a new Store.
type StoreOpts struct {
Address string
Timeout time.Duration
}
// NewStore makes a new peer store.
func NewStore(opts StoreOpts) *Store {
return &Store{opts: opts}
}
func (p *Store) getClient() (*Client, error) {
c := &Client{Timeout: p.opts.Timeout}
err := c.Connect(p.opts.Address)
return c, errors.Prefix("connection error", err)
}
func (p *Store) Name() string { return "peer" }
// Has asks the peer if they have a hash
func (p *Store) Has(hash string) (bool, error) {
c, err := p.getClient()
if err != nil {
return false, err
}
defer c.Close()
return c.HasBlob(hash)
}
// Get downloads the blob from the peer
func (p *Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
c, err := p.getClient()
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), p.Name()), err
}
defer c.Close()
blob, trace, err := c.GetBlob(hash)
if err != nil && strings.Contains(err.Error(), "blob not found") {
return nil, trace, store.ErrBlobNotFound
}
return blob, trace, err
}
// Put is not supported
func (p *Store) Put(hash string, blob stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// PutSD is not supported
func (p *Store) PutSD(hash string, blob stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// Delete is not supported
func (p *Store) Delete(hash string) error {
return errors.Err(shared.ErrNotImplemented)
}
// Delete is not supported
func (p *Store) Shutdown() {
return
}
+22 -19
View File
@@ -2,18 +2,17 @@ package prism
import (
"context"
"fmt"
"strconv"
"sync"
"github.com/lbryio/reflector.go/cluster"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/server/peer"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/dht"
"github.com/lbryio/lbry.go/v2/dht/bits"
"github.com/lbryio/reflector.go/cluster"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/peer"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
@@ -21,16 +20,20 @@ import (
)
type Config struct {
Blobs store.BlobStore
// limit the range of hashes to announce. useful for testing
HashRange *bits.Range
DB *db.SQL
DhtAddress string
ClusterSeedAddr string
DhtSeedNodes []string
PeerPort int
ReflectorPort int
PeerPort int
ReflectorPort int
DhtAddress string
DhtSeedNodes []string
ClusterPort int
ClusterSeedAddr string
// limit the range of hashes to announce. useful for testing
HashRange *bits.Range
DB *db.SQL
Blobs store.BlobStore
}
// DefaultConf returns a default config
@@ -75,8 +78,8 @@ func New(conf *Config) *Prism {
db: conf.DB,
dht: d,
cluster: c,
peer: peer.NewServer(conf.Blobs, fmt.Sprintf(":%d", conf.PeerPort)),
reflector: reflector.NewIngestionServer(conf.Blobs),
peer: peer.NewServer(conf.Blobs),
reflector: reflector.NewServer(conf.Blobs, conf.Blobs),
grp: stop.New(),
}
@@ -104,7 +107,7 @@ func (p *Prism) Start() error {
return errors.Err("blobs required in conf")
}
err = p.peer.Start()
err = p.peer.Start(":" + strconv.Itoa(p.conf.PeerPort))
if err != nil {
return err
}
+1 -2
View File
@@ -4,9 +4,8 @@ import (
"math/big"
"testing"
"github.com/lbryio/lbry.go/v2/dht/bits"
"github.com/davecgh/go-spew/spew"
"github.com/lbryio/lbry.go/v2/dht/bits"
)
func TestAnnounceRange(t *testing.T) {
+20 -18
View File
@@ -22,23 +22,24 @@ import (
"github.com/golang/protobuf/proto"
)
/* TODO:
import cert from wallet
get all utxos from chainquery
create transaction
sign it with the channel
track state of utxos across publishes from this channel so that we can just do one query to get utxos
prioritize only confirmed utxos
var TODO = `
import cert from wallet
get all utxos from chainquery
create transaction
sign it with the channel
track state of utxos across publishes from this channel so that we can just do one query to get utxos
prioritize only confirmed utxos
Handling all the issues we handle currently with lbrynet:
"Couldn't find private key for id",
"You already have a stream claim published under the name",
"Cannot publish using channel",
"txn-mempool-conflict",
"too-long-mempool-chain",
"Missing inputs",
"Not enough funds to cover this transaction",
*/
Handling all the issues we handle currently with lbrynet:
"Couldn't find private key for id",
"You already have a stream claim published under the name",
"Cannot publish using channel",
"txn-mempool-conflict",
"too-long-mempool-chain",
"Missing inputs",
"Not enough funds to cover this transaction",
}
`
type Details struct {
Title string
@@ -115,7 +116,7 @@ func Publish(client *lbrycrd.Client, path, name, address string, details Details
return signedTx, txid, nil
}
// TODO: lots of assumptions. hardcoded values need to be passed in or calculated
//TODO: lots of assumptions. hardcoded values need to be passed in or calculated
func baseTx(client *lbrycrd.Client, amount float64, changeAddress btcutil.Address) (*wire.MsgTx, error) {
txFee := 0.0002 // TODO: estimate this better?
@@ -222,7 +223,8 @@ func makeStream(path string) (stream.Stream, *pb.Stream, error) {
if err != nil {
return nil, nil, errors.Err(err)
}
defer func() { _ = file.Close() }()
defer file.Close()
enc := stream.NewEncoder(file)
s, err := enc.Stream()
+11 -11
View File
@@ -13,21 +13,21 @@ func LoadWallet(r io.Reader) (WalletFile, error) {
type WalletFile struct {
Name string `json:"name"`
Accounts []Account `json:"accounts"`
Preferences WalletPrefs `json:"preferences"`
Version int `json:"version"`
Preferences WalletPrefs `json:"preferences"`
Accounts []Account `json:"accounts"`
}
type Account struct {
AddressGenerator AddressGenerator `json:"address_generator"`
Certificates map[string]string `json:"certificates"`
Encrypted bool `json:"encrypted"`
Ledger string `json:"ledger"`
ModifiedOn float64 `json:"modified_on"`
Name string `json:"name"`
PrivateKey string `json:"private_key"`
PublicKey string `json:"public_key"`
Seed string `json:"seed"`
AddressGenerator AddressGenerator `json:"address_generator"`
ModifiedOn float64 `json:"modified_on"`
Encrypted bool `json:"encrypted"`
}
type AddressGenerator struct {
@@ -43,17 +43,17 @@ type AddressGenParams struct {
type WalletPrefs struct {
Shared struct {
Ts float64 `json:"ts"`
Value struct {
Type string `json:"type"`
Version string `json:"version"`
Value struct {
Type string `json:"type"`
Value struct {
AppWelcomeVersion int `json:"app_welcome_version"`
Blocked []interface{} `json:"blocked"`
Sharing3P bool `json:"sharing_3P"`
Subscriptions []string `json:"subscriptions"`
Tags []string `json:"tags"`
AppWelcomeVersion int `json:"app_welcome_version"`
Sharing3P bool `json:"sharing_3P"`
} `json:"value"`
Version string `json:"version"`
} `json:"value"`
Ts float64 `json:"ts"`
} `json:"shared"`
}
+24 -173
View File
@@ -1,189 +1,40 @@
# Reflector
Production-ready blob reflection, distribution, and caching for Odysee.
A reflector cluster to accept LBRY content for hosting en masse, rehost the content, and make money on data fees (TODO).
This code includes Go implementations of the LBRY peer protocol, reflector protocol, and DHT.
This repository provides the components used in production:
- Reflector ingestion server (command name: `reflector`)
- Blob cache/edge server (`blobcache`)
- Uploader to object storage (`upload`)
## Installation
Other commands exist in the tree for historical/legacy reasons and are not supported.
coming soon
## How it works (at a glance)
- Ingestion (reflector): accepts uploaded blobs, persists them to object storage (e.g., S3/Wasabi) and tracks state in MySQL.
- Distribution: serves blobs over HTTP/HTTP3/Peer. Blobcaches can be deployed in front of the origin to reduce latency and egress.
- Caching (blobcache): layered disk caches backed by HTTP(S) origins (e.g., S3 endpoints), with optional local DB metadata for capacity/eviction.
## Usage
All services are started by the `prism` binary and are configured via YAML files loaded from a configuration directory.
coming soon
## Supported commands
The following are the only supported commands for production use:
## Running from Source
- Reflector ingestion: `prism reflector`
- Flags: `--receiver-port` (default 5566), `--metrics-port` (default 2112), `--disable-blocklist`
- Loads `reflector.yaml` from the config directory.
This project requires [Go v1.11](https://golang.org/doc/install) or higher because it uses Go modules.
- Blob cache: `prism blobcache`
- Flags: `--metrics-port` (default 2112), `--disable-blocklist`
- Loads `blobcache.yaml` from the config directory.
- Uploader: `prism upload PATH`
- Flags: `--workers`, `--skipExistsCheck`, `--deleteBlobsAfterUpload`
- Loads `upload.yaml` from the config directory.
Global flag for all commands:
- `--conf-dir` (default `./`): directory containing YAML config files.
## Configuration
Configuration is per-command. The loader reads `<command>.yaml` from `--conf-dir`.
Common sections:
- `servers`: enables HTTP/HTTP3/Peer servers. Keys: `http`, `http3`, `peer`. Each accepts:
- `port` (int)
- `max_concurrent_requests` (int, http/http3)
- `edge_token` (string, http)
- `address` (string, optional; bind address, omit for all interfaces)
- `store`: defines the storage topology using composable stores. Frequently used:
- `proxied-s3`: production pattern with a `writer` (DB-backed -> S3/multiwriter) and a `reader` (caching -> disk + HTTP origins).
- `caching`: layered cache with a `cache` (often `db_backed` -> `disk`) and an `origin` chain (`http`, `http3`, or `ittt` fan-in).
- `s3`, `disk`, `multiwriter`, `db_backed`, `http`, `http3`, `peer`, `upstream` are also available building blocks.
### Minimal examples
Reflector – conf-dir contains `reflector.yaml`:
```yaml
servers:
http:
port: 5569
max_concurrent_requests: 200
http3:
port: 5568
max_concurrent_requests: 200
peer:
port: 5567
store:
proxied-s3:
name: s3_read_proxy
writer:
db_backed:
user: reflector
password: reflector
database: reflector
host: localhost
port: 3306
access_tracking: 1
soft_deletes: true
store:
s3:
name: primary
aws_id: YOUR_KEY
aws_secret: YOUR_SECRET
region: us-east-1
bucket: blobs-bucket
endpoint: https://s3.yourendpoint.tv
reader:
caching:
cache:
disk:
name: local_cache
mount_point: /mnt/reflector/cache
sharding_size: 2
origin:
http:
endpoint: https://s3.yourendpoint.tv/blobs-bucket/
sharding_size: 4
```
git clone git@github.com:lbryio/reflector.go.git
cd reflector.go
make
./bin/prism-bin
```
Blobcache – conf-dir contains `blobcache.yaml`:
```yaml
servers:
http:
port: 5569
max_concurrent_requests: 200
http3:
port: 5568
max_concurrent_requests: 200
peer:
port: 5567
store:
caching:
cache:
db_backed:
user: reflector
password: reflector
database: reflector
host: localhost
port: 3306
has_cap: true
max_size: 500GB
store:
disk:
name: blobcache
mount_point: /mnt/blobcache/cache
sharding_size: 2
origin:
http:
endpoint: https://s3.yourendpoint.tv/blobs-bucket/
sharding_size: 4
```
## Contributing
Uploader – conf-dir contains `upload.yaml` (points to the same writer/backend as reflector):
```yaml
database:
user: reflector
password: reflector
database: reflector
host: localhost
port: 3306
store:
proxied-s3:
writer:
db_backed:
user: reflector
password: reflector
database: reflector
host: localhost
port: 3306
store:
s3:
aws_id: YOUR_KEY
aws_secret: YOUR_SECRET
region: us-east-1
bucket: blobs-bucket
endpoint: https://s3.yourendpoint.tv
```
## Quick start
1) Build
- Requires Go 1.25+
- `make` (binaries in `dist/<platform>/prism-bin`)
2) Run a local blobcache
```bash
./dist/linux_amd64/prism-bin --conf-dir=./ blobcache
```
Place your `blobcache.yaml` in the `--conf-dir` directory.
3) Run reflector ingestion
```bash
./dist/linux_amd64/prism-bin --conf-dir=./ reflector --receiver-port=5566 --metrics-port=2112
```
4) Upload blobs
```bash
./dist/linux_amd64/prism-bin --conf-dir=./ upload /path/to/blobs \
--workers=4 --skipExistsCheck
```
## Notes
- Only reflector, blobcache, and upload are supported. All other commands are legacy and may be removed in the future.
- Metrics are exposed on the configured `--metrics-port` at `/metrics` (Prometheus format).
- MySQL is required when using DB-backed stores (e.g., ingestion writer, capacity-aware caches).
## Security
If you discover a security issue, please email security@lbry.com. Our PGP key is available at https://lbry.com/faq/pgp-key.
coming soon
## License
MIT License. See LICENSE.
This project is MIT licensed.
## Security
We take security seriously. Please contact security@lbry.com regarding any security issues.
Our PGP key is [here](https://keybase.io/lbry/key.asc) if you need it.
## Contact
The primary contact for this project is [@Nikooo777](https://github.com/Nikooo777)
The primary contact for this project is [@lyoshenka](https://github.com/lyoshenka) (grin@lbry.com)
+12 -11
View File
@@ -9,6 +9,7 @@ import (
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/reflector.go/wallet"
@@ -22,10 +23,10 @@ const blocklistURL = "https://api.lbry.com/file/list_blocked"
func (s *Server) enableBlocklist(b store.Blocklister) {
walletServers := []string{
"a-hub1.odysee.com:50001",
"b-hub1.odysee.com:50001",
"c-hub1.odysee.com:50001",
"s-hub1.odysee.com:50001",
"spv25.lbry.com:50001",
"spv26.lbry.com:50001",
"spv19.lbry.com:50001",
"spv14.lbry.com:50001",
}
updateBlocklist(b, walletServers, s.grp.Ch())
@@ -69,18 +70,18 @@ func blockedSdHashes(walletServers []string, stopper stop.Chan) (map[string]valO
return nil, errors.Err(err)
}
defer func() {
closeErr := resp.Body.Close()
if closeErr != nil {
log.Errorln(errors.Err(closeErr))
err := resp.Body.Close()
if err != nil {
log.Errorln(errors.Err(err))
}
}()
var r struct {
Error string `json:"error"`
Data struct {
Success bool `json:"success"`
Error string `json:"error"`
Data struct {
Outpoints []string `json:"outpoints"`
} `json:"data"`
Success bool `json:"success"`
}
if err = json.NewDecoder(resp.Body).Decode(&r); err != nil {
@@ -95,8 +96,8 @@ func blockedSdHashes(walletServers []string, stopper stop.Chan) (map[string]valO
}
type valOrErr struct {
Err error
Value string
Err error
}
// sdHashesForOutpoints queries wallet server for the sd hashes in a given outpoints
-85
View File
@@ -1,85 +0,0 @@
package reflector
import (
"encoding/json"
"net/http"
"time"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/bluele/gcache"
"golang.org/x/sync/singleflight"
)
const protectedListURL = "https://direct.api.odysee.com/file/list_protected"
type ProtectedContent struct {
SDHash string `json:"sd_hash"`
ClaimID string `json:"claim_id"`
}
var protectedCache = gcache.New(10).Expiration(2 * time.Minute).Build()
func GetProtectedContent() (interface{}, error) {
cachedVal, err := protectedCache.Get("protected")
if err == nil && cachedVal != nil {
return cachedVal.(map[string]bool), nil
}
method := "GET"
var r struct {
Error string `json:"error"`
Data []ProtectedContent `json:"data"`
Success bool `json:"success"`
}
// Bound the request to avoid hanging the entire request path.
// Without a timeout, a slow or unreachable endpoint can block
// singleflight callers indefinitely and stall HTTP handlers.
client := &http.Client{Timeout: 5 * time.Second}
req, err := http.NewRequest(method, protectedListURL, nil)
if err != nil {
return nil, errors.Err(err)
}
res, err := client.Do(req)
if err != nil {
return nil, errors.Err(err)
}
defer func() { _ = res.Body.Close() }()
if res.StatusCode != http.StatusOK {
return nil, errors.Err("unexpected status code %d", res.StatusCode)
}
if err = json.NewDecoder(res.Body).Decode(&r); err != nil {
return nil, errors.Err(err)
}
if !r.Success {
return nil, errors.Prefix("file/list_protected API call", r.Error)
}
protectedMap := make(map[string]bool, len(r.Data))
for _, pc := range r.Data {
protectedMap[pc.SDHash] = true
}
err = protectedCache.Set("protected", protectedMap)
if err != nil {
return protectedMap, errors.Err(err)
}
return protectedMap, nil
}
var sf = singleflight.Group{}
func IsProtected(sdHash string) bool {
val, err, _ := sf.Do("protected", GetProtectedContent)
if err != nil {
return false
}
cachedMap, ok := val.(map[string]bool)
if !ok {
return false
}
return cachedMap[sdHash]
}
+33 -23
View File
@@ -6,9 +6,11 @@ import (
"encoding/hex"
"encoding/json"
"io"
"io/ioutil"
"net"
"time"
"github.com/google/gops/agent"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/store"
@@ -35,21 +37,22 @@ var ErrBlobTooBig = errors.Base("blob must be at most %d bytes", maxBlobSize)
// Server is and instance of the reflector server. It houses the blob store and listener.
type Server struct {
store store.BlobStore
grp *stop.Group
Timeout time.Duration // timeout to read or write next message
EnableBlocklist bool // if true, blocklist checking and blob deletion will be enabled
//underlyingStore store.BlobStore
//outerStore store.BlobStore
underlyingStore store.BlobStore
outerStore store.BlobStore
grp *stop.Group
}
func NewIngestionServer(store store.BlobStore) *Server {
// NewServer returns an initialized reflector server pointer.
func NewServer(underlying store.BlobStore, outer store.BlobStore) *Server {
return &Server{
Timeout: DefaultTimeout,
store: store,
grp: stop.New(),
Timeout: DefaultTimeout,
underlyingStore: underlying,
outerStore: outer,
grp: stop.New(),
}
}
@@ -60,13 +63,16 @@ func (s *Server) Shutdown() {
log.Println("reflector server stopped")
}
// Start starts the server to handle connections.
//Start starts the server to handle connections.
func (s *Server) Start(address string) error {
l, err := net.Listen(network, address)
if err != nil {
return errors.Err(err)
}
log.Println("reflector listening on " + address)
if err := agent.Listen(agent.Options{}); err != nil {
log.Fatal(err)
}
s.grp.Add(1)
metrics.RoutinesQueue.WithLabelValues("reflector", "listener").Inc()
go func() {
@@ -88,7 +94,7 @@ func (s *Server) Start(address string) error {
}()
if s.EnableBlocklist {
if b, ok := s.store.(store.Blocklister); ok {
if b, ok := s.underlyingStore.(store.Blocklister); ok {
s.grp.Add(1)
metrics.RoutinesQueue.WithLabelValues("reflector", "enableblocklist").Inc()
go func() {
@@ -117,7 +123,7 @@ func (s *Server) listenAndServe(listener net.Listener) {
s.grp.Add(1)
metrics.RoutinesQueue.WithLabelValues("reflector", "server-listenandserve").Inc()
go func() {
defer metrics.RoutinesQueue.WithLabelValues("reflector", "server-listenandserve").Dec()
defer metrics.RoutinesQueue.WithLabelValues("reflector", "server-listenandserve").Inc()
s.handleConn(conn)
s.grp.Done()
}()
@@ -151,7 +157,7 @@ func (s *Server) handleConn(conn net.Conn) {
if errors.Is(err, io.EOF) || s.quitting() {
return
}
err = s.doError(conn, err)
err := s.doError(conn, err)
if err != nil {
log.Error(errors.Prefix("sending handshake error", err))
}
@@ -164,7 +170,7 @@ func (s *Server) handleConn(conn net.Conn) {
if errors.Is(err, io.EOF) || s.quitting() {
return
}
err = s.doError(conn, err)
err := s.doError(conn, err)
if err != nil {
log.Error(errors.Prefix("sending blob receive error", err))
}
@@ -199,14 +205,13 @@ func (s *Server) receiveBlob(conn net.Conn) error {
}
var wantsBlob bool
if bl, ok := s.store.(store.Blocklister); ok {
if bl, ok := s.underlyingStore.(store.Blocklister); ok {
wantsBlob, err = bl.Wants(blobHash)
if err != nil {
return err
}
} else {
var blobExists bool
blobExists, err = s.store.Has(blobHash)
blobExists, err := s.underlyingStore.Has(blobHash)
if err != nil {
return err
}
@@ -216,14 +221,14 @@ func (s *Server) receiveBlob(conn net.Conn) error {
var neededBlobs []string
if isSdBlob && !wantsBlob {
if nbc, ok := s.store.(store.NeededBlobChecker); ok {
if nbc, ok := s.underlyingStore.(neededBlobChecker); ok {
neededBlobs, err = nbc.MissingBlobsForKnownStream(blobHash)
if err != nil {
return err
}
} else {
// if we can't check for blobs in a stream, we have to say that the sd blob is
// missing. if we say we have the sd blob, they won't try to send any content blobs
// missing. if we say we have the sd blob, they wont try to send any content blobs
wantsBlob = true
}
}
@@ -259,9 +264,9 @@ func (s *Server) receiveBlob(conn net.Conn) error {
log.Debugln("Got blob " + blobHash[:8])
if isSdBlob {
err = s.store.PutSD(blobHash, blob)
err = s.outerStore.PutSD(blobHash, blob)
} else {
err = s.store.Put(blobHash, blob)
err = s.outerStore.Put(blobHash, blob)
}
if err != nil {
return err
@@ -366,7 +371,7 @@ func (s *Server) read(conn net.Conn, v interface{}) error {
dec := json.NewDecoder(conn)
err = dec.Decode(v)
if err != nil {
data, _ := io.ReadAll(dec.Buffered())
data, _ := ioutil.ReadAll(dec.Buffered())
if len(data) > 0 {
return errors.Err("%s. Data: %s", err.Error(), hex.EncodeToString(data))
}
@@ -429,8 +434,8 @@ type handshakeRequestResponse struct {
type sendBlobRequest struct {
BlobHash string `json:"blob_hash,omitempty"`
SdBlobHash string `json:"sd_blob_hash,omitempty"`
BlobSize int `json:"blob_size,omitempty"`
SdBlobHash string `json:"sd_blob_hash,omitempty"`
SdBlobSize int `json:"sd_blob_size,omitempty"`
}
@@ -439,8 +444,8 @@ type sendBlobResponse struct {
}
type sendSdBlobResponse struct {
NeededBlobs []string `json:"needed_blobs,omitempty"`
SendSdBlob bool `json:"send_sd_blob"`
NeededBlobs []string `json:"needed_blobs,omitempty"`
}
type blobTransferResponse struct {
@@ -450,3 +455,8 @@ type blobTransferResponse struct {
type sdBlobTransferResponse struct {
ReceivedSdBlob bool `json:"received_sd_blob"`
}
// neededBlobChecker can check which blobs from a known stream are not uploaded yet
type neededBlobChecker interface {
MissingBlobsForKnownStream(string) ([]string, error)
}
+6 -7
View File
@@ -9,9 +9,8 @@ import (
"testing"
"time"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/dht/bits"
"github.com/lbryio/reflector.go/store"
"github.com/davecgh/go-spew/spew"
"github.com/phayes/freeport"
@@ -23,7 +22,7 @@ func startServerOnRandomPort(t *testing.T) (*Server, int) {
t.Fatal(err)
}
srv := NewIngestionServer(store.NewMemStore(store.MemParams{Name: "test"}))
srv := NewServer(store.NewMemStore(), store.NewMemStore())
err = srv.Start("127.0.0.1:" + strconv.Itoa(port))
if err != nil {
t.Fatal(err)
@@ -120,7 +119,7 @@ func TestServer_Timeout(t *testing.T) {
t.Fatal(err)
}
srv := NewIngestionServer(store.NewMemStore(store.MemParams{Name: "test"}))
srv := NewServer(store.NewMemStore(), store.NewMemStore())
srv.Timeout = testTimeout
err = srv.Start("127.0.0.1:" + strconv.Itoa(port))
if err != nil {
@@ -182,8 +181,8 @@ func TestServer_PartialUpload(t *testing.T) {
missing[i] = bits.Rand().String()
}
st := store.BlobStore(&mockPartialStore{MemStore: store.NewMemStore(store.MemParams{Name: "test"}), missing: missing})
if _, ok := st.(store.NeededBlobChecker); !ok {
st := store.BlobStore(&mockPartialStore{MemStore: store.NewMemStore(), missing: missing})
if _, ok := st.(neededBlobChecker); !ok {
t.Fatal("mock does not implement the relevant interface")
}
err = st.Put(sdHash, randBlob(10))
@@ -191,7 +190,7 @@ func TestServer_PartialUpload(t *testing.T) {
t.Fatal(err)
}
srv := NewIngestionServer(st)
srv := NewServer(st, st)
err = srv.Start("127.0.0.1:" + strconv.Itoa(port))
if err != nil {
t.Fatal(err)
+12 -9
View File
@@ -1,13 +1,15 @@
package reflector
import (
"io/ioutil"
"os"
"path"
"sync"
"time"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/extras/errors"
@@ -30,16 +32,17 @@ type Summary struct {
type Uploader struct {
db *db.SQL
store store.BlobStore
stopper *stop.Group
countChan chan increment
count Summary
store *store.DBBackedStore // could just be store.BlobStore interface
workers int
skipExistsCheck bool
deleteBlobsAfterUpload bool
stopper *stop.Group
countChan chan increment
count Summary
}
func NewUploader(db *db.SQL, store store.BlobStore, workers int, skipExistsCheck, deleteBlobsAfterUpload bool) *Uploader {
func NewUploader(db *db.SQL, store *store.DBBackedStore, workers int, skipExistsCheck, deleteBlobsAfterUpload bool) *Uploader {
return &Uploader{
db: db,
store: store,
@@ -131,7 +134,7 @@ Upload:
return nil
}
// worker reads paths from a channel, uploads them, and optionally deletes them
// worker reads paths from a channel, uploads them, and optionally deletes them
func (u *Uploader) worker(pathChan chan string) {
for {
select {
@@ -163,7 +166,7 @@ func (u *Uploader) uploadBlob(filepath string) (err error) {
}
}()
blob, err := os.ReadFile(filepath)
blob, err := ioutil.ReadFile(filepath)
if err != nil {
return errors.Err(err)
}
@@ -175,7 +178,7 @@ func (u *Uploader) uploadBlob(filepath string) (err error) {
if IsValidJSON(blob) {
log.Debugf("uploading SD blob %s", hash)
err = u.store.PutSD(hash, blob)
err := u.store.PutSD(hash, blob)
if err != nil {
return errors.Prefix("uploading SD blob "+hash, err)
}
-16
View File
@@ -1,16 +0,0 @@
#!/bin/bash
set -euo pipefail
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
cd "$SCRIPT_DIR/.."
if ! command -v golangci-lint &> /dev/null; then
echo "Installing golangci-lint..."
go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
else
echo "Ensuring golangci-lint is up to date..."
go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
fi
echo "Running golangci-lint..."
golangci-lint run --config .golangci.yml ./...
+10 -58
View File
@@ -2,77 +2,38 @@ package http
import (
"net/http"
"strconv"
"sync"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/gin-gonic/gin"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/store"
log "github.com/sirupsen/logrus"
)
func (s *Server) getBlob(c *gin.Context) {
waiter := &sync.WaitGroup{}
waiter.Add(1)
enqueue(&blobRequest{c: c, finished: waiter})
waiter.Wait()
}
func (s *Server) HandleGetBlob(c *gin.Context) {
defer func() {
if r := recover(); r != nil {
log.Errorf("Recovered from panic: %v", r)
}
}()
start := time.Now()
hash := c.Query("hash")
edgeToken := c.Query("edge_token")
if reflector.IsProtected(hash) && edgeToken != s.edgeToken {
_ = c.Error(errors.Err("requested blob is protected"))
c.String(http.StatusForbidden, "requested blob is protected")
return
}
if s.missesCache.Has(hash) {
serialized, err := shared.NewBlobTrace(time.Since(start), "http").Serialize()
c.Header("Via", serialized)
if err != nil {
_ = c.Error(errors.Err(err))
c.String(http.StatusInternalServerError, err.Error())
return
}
c.AbortWithStatus(http.StatusNotFound)
return
}
blob, trace, err := s.store.Get(hash)
if err != nil {
serialized, serializeErr := trace.Serialize()
if serializeErr != nil {
_ = c.Error(errors.Prefix(serializeErr.Error(), err))
c.String(http.StatusInternalServerError, errors.Prefix(serializeErr.Error(), err).Error())
_ = c.AbortWithError(http.StatusInternalServerError, errors.Prefix(serializeErr.Error(), err))
return
}
c.Header("Via", serialized)
if errors.Is(err, store.ErrBlobNotFound) {
_ = s.missesCache.Set(hash, true)
log.Errorf("wtf: %s", err.Error())
c.AbortWithStatus(http.StatusNotFound)
return
}
_ = c.Error(err)
c.String(http.StatusInternalServerError, err.Error())
_ = c.AbortWithError(http.StatusInternalServerError, err)
return
}
serialized, err := trace.Serialize()
if err != nil {
_ = c.Error(err)
c.String(http.StatusInternalServerError, err.Error())
_ = c.AbortWithError(http.StatusInternalServerError, err)
return
}
metrics.MtrOutBytesHttp.Add(float64(len(blob)))
@@ -80,7 +41,6 @@ func (s *Server) HandleGetBlob(c *gin.Context) {
metrics.HttpDownloadCount.Inc()
c.Header("Via", serialized)
c.Header("Content-Disposition", "filename="+hash)
c.Header("Content-Length", strconv.Itoa(len(blob)))
c.Data(http.StatusOK, "application/octet-stream", blob)
}
@@ -88,8 +48,7 @@ func (s *Server) hasBlob(c *gin.Context) {
hash := c.Query("hash")
has, err := s.store.Has(hash)
if err != nil {
_ = c.Error(err)
c.String(http.StatusInternalServerError, err.Error())
_ = c.AbortWithError(http.StatusInternalServerError, err)
return
}
if has {
@@ -98,10 +57,3 @@ func (s *Server) hasBlob(c *gin.Context) {
}
c.Status(http.StatusNotFound)
}
func (s *Server) recoveryHandler(c *gin.Context, err interface{}) {
c.JSON(500, gin.H{
"title": "Error",
"err": err,
})
}
+11 -27
View File
@@ -5,35 +5,23 @@ import (
"net/http"
"time"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/bluele/gcache"
nice "github.com/ekyoung/gin-nice-recovery"
"github.com/gin-gonic/gin"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/reflector.go/store"
log "github.com/sirupsen/logrus"
)
// Server is an instance of a peer server that houses the listener and store.
type Server struct {
store store.BlobStore
grp *stop.Group
missesCache gcache.Cache
edgeToken string
address string
concurrentRequests int
store store.BlobStore
grp *stop.Group
}
// NewServer returns an initialized Server pointer.
func NewServer(store store.BlobStore, requestQueueSize int, edgeToken string, address string) *Server {
func NewServer(store store.BlobStore) *Server {
return &Server{
store: store,
grp: stop.New(),
concurrentRequests: requestQueueSize,
missesCache: gcache.New(2000).Expiration(5 * time.Minute).ARC().Build(),
edgeToken: edgeToken,
address: address,
store: store,
grp: stop.New(),
}
}
@@ -45,26 +33,22 @@ func (s *Server) Shutdown() {
}
// Start starts the server listener to handle connections.
func (s *Server) Start() error {
func (s *Server) Start(address string) error {
gin.SetMode(gin.ReleaseMode)
router := gin.New()
router.Use(gin.Logger())
// Install nice.Recovery, passing the handler to call after recovery
router.Use(nice.Recovery(s.recoveryHandler))
router := gin.Default()
router.GET("/blob", s.getBlob)
router.HEAD("/blob", s.hasBlob)
srv := &http.Server{
Addr: s.address,
Addr: address,
Handler: router,
}
go s.listenForShutdown(srv)
go InitWorkers(s, s.concurrentRequests)
// Initializing the server in a goroutine so that
// it won't block the graceful shutdown handling below
s.grp.Add(1)
go func() {
defer s.grp.Done()
log.Println("HTTP server listening on " + s.address)
log.Println("HTTP server listening on " + address)
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
log.Fatalf("listen: %s\n", err)
}
-46
View File
@@ -1,46 +0,0 @@
package http
import (
"sync"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/gin-gonic/gin"
)
type blobRequest struct {
c *gin.Context
finished *sync.WaitGroup
}
var getReqCh = make(chan *blobRequest, 20000)
func InitWorkers(server *Server, workers int) {
stopper := stop.New(server.grp)
for i := 0; i < workers; i++ {
metrics.RoutinesQueue.WithLabelValues("http", "worker").Inc()
go func(worker int) {
defer metrics.RoutinesQueue.WithLabelValues("http", "worker").Dec()
for {
select {
case <-stopper.Ch():
case r := <-getReqCh:
process(server, r)
metrics.HttpBlobReqQueue.Dec()
}
}
}(i)
}
}
func enqueue(b *blobRequest) {
metrics.HttpBlobReqQueue.Inc()
getReqCh <- b
}
func process(server *Server, r *blobRequest) {
server.HandleGetBlob(r.c)
r.finished.Done()
}
-14
View File
@@ -1,14 +0,0 @@
package server
// BlobServer defines the common interface for all blob server implementations
type BlobServer interface {
Start() error
Shutdown()
}
type BlobServerConfig struct {
Address string `mapstructure:"address"`
EdgeToken string `mapstructure:"edge_token"`
Port int `mapstructure:"port"`
MaxConcurrentRequests int `mapstructure:"max_concurrent_requests"`
}
+1 -1
View File
@@ -2,5 +2,5 @@ package shared
import "github.com/lbryio/lbry.go/v2/extras/errors"
// ErrNotImplemented is a standard error when a store that implements the store interface does not implement a method
//ErrNotImplemented is a standard error when a store that implements the store interface does not implement a method
var ErrNotImplemented = errors.Base("this store does not implement this method")
+1 -1
View File
@@ -10,9 +10,9 @@ import (
)
type BlobStack struct {
Timing time.Duration `json:"timing"`
OriginName string `json:"origin_name"`
HostName string `json:"host_name"`
Timing time.Duration `json:"timing"`
}
type BlobTrace struct {
Stacks []BlobStack `json:"stacks"`
+17 -17
View File
@@ -1,39 +1,39 @@
package shared
import (
"os"
"testing"
"time"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestBlobTrace_Serialize(t *testing.T) {
hostName = util.PtrToString("test_machine")
hostName, err := os.Hostname()
require.NoError(t, err)
stack := NewBlobTrace(10*time.Second, "test")
stack.Stack(20*time.Second, "test2")
stack.Stack(30*time.Second, "test3")
serialized, err := stack.Serialize()
assert.NoError(t, err)
require.NoError(t, err)
deserialized, err := Deserialize(serialized)
assert.NoError(t, err)
assert.Len(t, deserialized.Stacks, 3)
assert.Equal(t, 10*time.Second, deserialized.Stacks[0].Timing)
assert.Equal(t, "test", deserialized.Stacks[0].OriginName)
assert.Equal(t, "test_machine", deserialized.Stacks[0].HostName)
assert.Equal(t, 20*time.Second, deserialized.Stacks[1].Timing)
assert.Equal(t, "test2", deserialized.Stacks[1].OriginName)
assert.Equal(t, 30*time.Second, deserialized.Stacks[2].Timing)
assert.Equal(t, "test3", deserialized.Stacks[2].OriginName)
expected := `{"stacks":[{"timing":10000000000,"origin_name":"test","host_name":"` +
hostName +
`"},{"timing":20000000000,"origin_name":"test2","host_name":"` +
hostName +
`"},{"timing":30000000000,"origin_name":"test3","host_name":"` +
hostName +
`"}]}`
assert.Equal(t, expected, serialized)
}
func TestBlobTrace_Deserialize(t *testing.T) {
hostName = util.PtrToString("test_machine")
serialized := "{\"stacks\":[{\"timing\":10000000000,\"origin_name\":\"test\"},{\"timing\":20000000000,\"origin_name\":\"test2\"},{\"timing\":30000000000,\"origin_name\":\"test3\"}]}"
serialized := `{"stacks":[{"timing":10000000000,"origin_name":"test"},{"timing":20000000000,"origin_name":"test2"},{"timing":30000000000,"origin_name":"test3"}]}`
stack, err := Deserialize(serialized)
assert.NoError(t, err)
require.NoError(t, err)
assert.Len(t, stack.Stacks, 3)
assert.Equal(t, stack.Stacks[0].Timing, 10*time.Second)
assert.Equal(t, stack.Stacks[1].Timing, 20*time.Second)
+1 -2
View File
@@ -1,4 +1,3 @@
//go:build linux
// +build linux
package store
@@ -10,7 +9,7 @@ import (
)
func timespecToTime(ts syscall.Timespec) time.Time {
return time.Unix(ts.Sec, ts.Nsec)
return time.Unix(int64(ts.Sec), int64(ts.Nsec))
}
func atime(fi os.FileInfo) time.Time {
-1
View File
@@ -1,4 +1,3 @@
//go:build !linux
// +build !linux
package store
+22 -76
View File
@@ -1,17 +1,14 @@
package store
import (
"strings"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/lbryio/reflector.go/shared"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
"github.com/lbryio/reflector.go/internal/metrics"
)
// CachingStore combines two stores, typically a local and a remote store, to improve performance.
@@ -19,77 +16,20 @@ import (
// are retrieved from the origin and cached. Puts are cached and also forwarded to the origin.
type CachingStore struct {
origin, cache BlobStore
name string
}
type CachingParams struct {
Origin BlobStore `mapstructure:"origin"`
Cache BlobStore `mapstructure:"cache"`
Name string `mapstructure:"name"`
}
type CachingConfig struct {
Origin *viper.Viper
Cache *viper.Viper
Name string `mapstructure:"name"`
component string
}
// NewCachingStore makes a new caching disk store and returns a pointer to it.
func NewCachingStore(params CachingParams) *CachingStore {
func NewCachingStore(component string, origin, cache BlobStore) *CachingStore {
return &CachingStore{
name: params.Name,
origin: WithSingleFlight(params.Name, params.Origin),
cache: WithSingleFlight(params.Name, params.Cache),
component: component,
origin: WithSingleFlight(component, origin),
cache: WithSingleFlight(component, cache),
}
}
const nameCaching = "caching"
func CachingStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg CachingConfig
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
cfg.Cache = config.Sub("cache")
cfg.Origin = config.Sub("origin")
if cfg.Cache == nil || cfg.Origin == nil {
return nil, errors.Err("cache and origin missing")
}
originStoreType := strings.Split(cfg.Origin.AllKeys()[0], ".")[0]
originStoreConfig := cfg.Origin.Sub(originStoreType)
factory, ok := Factories[originStoreType]
if !ok {
return nil, errors.Err("unknown store type %s", originStoreType)
}
originStore, err := factory(originStoreConfig)
if err != nil {
return nil, errors.Err(err)
}
cacheStoreType := strings.Split(cfg.Cache.AllKeys()[0], ".")[0]
cacheStoreConfig := cfg.Cache.Sub(cacheStoreType)
factory, ok = Factories[cacheStoreType]
if !ok {
return nil, errors.Err("unknown store type %s", cacheStoreType)
}
cacheStore, err := factory(cacheStoreConfig)
if err != nil {
return nil, errors.Err(err)
}
return NewCachingStore(CachingParams{
Name: cfg.Name,
Origin: originStore,
Cache: cacheStore,
}), nil
}
func init() {
RegisterStore(nameCaching, CachingStoreFactory)
}
// Name is the cache type name
func (c *CachingStore) Name() string { return nameCaching }
@@ -108,27 +48,32 @@ func (c *CachingStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
blob, trace, err := c.cache.Get(hash)
if err == nil || !errors.Is(err, ErrBlobNotFound) {
metrics.CacheHitCount.With(metrics.CacheLabels(c.cache.Name(), c.name)).Inc()
metrics.CacheHitCount.With(metrics.CacheLabels(c.cache.Name(), c.component)).Inc()
rate := float64(len(blob)) / 1024 / 1024 / time.Since(start).Seconds()
metrics.CacheRetrievalSpeed.With(map[string]string{
metrics.LabelCacheType: c.cache.Name(),
metrics.LabelComponent: c.name,
metrics.LabelComponent: c.component,
metrics.LabelSource: "cache",
}).Set(rate)
return blob, trace.Stack(time.Since(start), c.Name()), err
}
metrics.CacheMissCount.With(metrics.CacheLabels(c.cache.Name(), c.name)).Inc()
metrics.CacheMissCount.With(metrics.CacheLabels(c.cache.Name(), c.component)).Inc()
blob, trace, err = c.origin.Get(hash)
if err != nil {
return nil, trace.Stack(time.Since(start), c.Name()), err
}
// do not do this async unless you're prepared to deal with mayhem
err = c.cache.Put(hash, blob)
if err != nil {
log.Errorf("error saving blob to underlying cache: %s", errors.FullTrace(err))
}
// there is no need to wait for the blob to be stored before we return it
// TODO: however this should be refactored to limit the amount of routines that the process can spawn to avoid a possible DoS
metrics.RoutinesQueue.WithLabelValues("store", "cache-put").Inc()
go func() {
defer metrics.RoutinesQueue.WithLabelValues("store", "cache-put").Dec()
err = c.cache.Put(hash, blob)
if err != nil {
log.Errorf("error saving blob to underlying cache: %s", errors.FullTrace(err))
}
}()
return blob, trace.Stack(time.Since(start), c.Name()), nil
}
@@ -163,4 +108,5 @@ func (c *CachingStore) Delete(hash string) error {
func (c *CachingStore) Shutdown() {
c.origin.Shutdown()
c.cache.Shutdown()
return
}
+14 -15
View File
@@ -6,17 +6,14 @@ import (
"testing"
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/sirupsen/logrus"
"github.com/lbryio/reflector.go/shared"
)
func TestCachingStore_Put(t *testing.T) {
origin := NewMemStore(MemParams{Name: "test"})
cache := NewMemStore(MemParams{Name: "test"})
s := NewCachingStore(CachingParams{Name: "test", Origin: origin, Cache: cache})
origin := NewMemStore()
cache := NewMemStore()
s := NewCachingStore("test", origin, cache)
b := []byte("this is a blob of stuff")
hash := "hash"
@@ -44,9 +41,9 @@ func TestCachingStore_Put(t *testing.T) {
}
func TestCachingStore_CacheMiss(t *testing.T) {
origin := NewMemStore(MemParams{Name: "test"})
cache := NewMemStore(MemParams{Name: "test"})
s := NewCachingStore(CachingParams{Name: "test", Origin: origin, Cache: cache})
origin := NewMemStore()
cache := NewMemStore()
s := NewCachingStore("test", origin, cache)
b := []byte("this is a blob of stuff")
hash := "hash"
@@ -86,8 +83,8 @@ func TestCachingStore_CacheMiss(t *testing.T) {
func TestCachingStore_ThunderingHerd(t *testing.T) {
storeDelay := 100 * time.Millisecond
origin := NewSlowBlobStore(storeDelay)
cache := NewMemStore(MemParams{Name: "test"})
s := NewCachingStore(CachingParams{Name: "test", Origin: origin, Cache: cache})
cache := NewMemStore()
s := NewCachingStore("test", origin, cache)
b := []byte("this is a blob of stuff")
hash := "hash"
@@ -101,7 +98,7 @@ func TestCachingStore_ThunderingHerd(t *testing.T) {
getNoErr := func() {
res, _, err := s.Get(hash)
if err != nil {
logrus.Fatal(err)
t.Fatal(err)
}
if !bytes.Equal(b, res) {
t.Errorf("expected Get() to return %s, got %s", string(b), string(res))
@@ -142,7 +139,7 @@ type SlowBlobStore struct {
func NewSlowBlobStore(delay time.Duration) *SlowBlobStore {
return &SlowBlobStore{
mem: NewMemStore(MemParams{Name: "test"}),
mem: NewMemStore(),
delay: delay,
}
}
@@ -175,4 +172,6 @@ func (s *SlowBlobStore) Delete(hash string) error {
return s.mem.Delete(hash)
}
func (s *SlowBlobStore) Shutdown() {}
func (s *SlowBlobStore) Shutdown() {
return
}
+110
View File
@@ -0,0 +1,110 @@
package store
import (
"io"
"io/ioutil"
"net/http"
"time"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/meta"
"github.com/lbryio/reflector.go/shared"
log "github.com/sirupsen/logrus"
)
// CloudFrontROStore reads from cloudfront. All writes panic.
type CloudFrontROStore struct {
endpoint string // cloudflare endpoint
}
// NewCloudFrontROStore returns an initialized CloudFrontROStore store pointer.
func NewCloudFrontROStore(endpoint string) *CloudFrontROStore {
return &CloudFrontROStore{endpoint: endpoint}
}
const nameCloudFrontRO = "cloudfront_ro"
// Name is the cache type name
func (c *CloudFrontROStore) Name() string { return nameCloudFrontRO }
// Has checks if the hash is in the store.
func (c *CloudFrontROStore) Has(hash string) (bool, error) {
status, body, err := c.cfRequest(http.MethodHead, hash)
if err != nil {
return false, err
}
defer body.Close()
switch status {
case http.StatusNotFound, http.StatusForbidden:
return false, nil
case http.StatusOK:
return true, nil
default:
return false, errors.Err("unexpected status %d", status)
}
}
// Get gets the blob from Cloudfront.
func (c *CloudFrontROStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
log.Debugf("Getting %s from S3", hash[:8])
start := time.Now()
defer func(t time.Time) {
log.Debugf("Getting %s from S3 took %s", hash[:8], time.Since(t).String())
}(start)
status, body, err := c.cfRequest(http.MethodGet, hash)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), c.Name()), err
}
defer body.Close()
switch status {
case http.StatusNotFound, http.StatusForbidden:
return nil, shared.NewBlobTrace(time.Since(start), c.Name()), errors.Err(ErrBlobNotFound)
case http.StatusOK:
b, err := ioutil.ReadAll(body)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), c.Name()), errors.Err(err)
}
metrics.MtrInBytesS3.Add(float64(len(b)))
return b, shared.NewBlobTrace(time.Since(start), c.Name()), nil
default:
return nil, shared.NewBlobTrace(time.Since(start), c.Name()), errors.Err("unexpected status %d", status)
}
}
func (c *CloudFrontROStore) cfRequest(method, hash string) (int, io.ReadCloser, error) {
url := c.endpoint + hash
req, err := http.NewRequest(method, url, nil)
if err != nil {
return 0, nil, errors.Err(err)
}
req.Header.Add("User-Agent", "reflector.go/"+meta.Version)
res, err := http.DefaultClient.Do(req)
if err != nil {
return 0, nil, errors.Err(err)
}
return res.StatusCode, res.Body, nil
}
func (c *CloudFrontROStore) Put(_ string, _ stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
func (c *CloudFrontROStore) PutSD(_ string, _ stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
func (c *CloudFrontROStore) Delete(_ string) error {
return errors.Err(shared.ErrNotImplemented)
}
// Shutdown shuts down the store gracefully
func (c *CloudFrontROStore) Shutdown() {
return
}
+62
View File
@@ -0,0 +1,62 @@
package store
import (
"time"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/lbryio/reflector.go/shared"
)
// CloudFrontRWStore combines a Cloudfront and an S3 store. Reads go to Cloudfront/Wasabi, writes go to S3.
type CloudFrontRWStore struct {
cf *ITTTStore
s3 *S3Store
}
// NewCloudFrontRWStore returns an initialized CloudFrontRWStore store pointer.
// NOTE: It panics if either argument is nil.
func NewCloudFrontRWStore(cf *ITTTStore, s3 *S3Store) *CloudFrontRWStore {
if cf == nil || s3 == nil {
panic("both stores must be set")
}
return &CloudFrontRWStore{cf: cf, s3: s3}
}
const nameCloudFrontRW = "cloudfront_rw"
// Name is the cache type name
func (c *CloudFrontRWStore) Name() string { return nameCloudFrontRW }
// Has checks if the hash is in the store.
func (c *CloudFrontRWStore) Has(hash string) (bool, error) {
return c.cf.Has(hash)
}
// Get gets the blob from Cloudfront.
func (c *CloudFrontRWStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
blob, trace, err := c.cf.Get(hash)
return blob, trace.Stack(time.Since(start), c.Name()), err
}
// Put stores the blob on S3
func (c *CloudFrontRWStore) Put(hash string, blob stream.Blob) error {
return c.s3.Put(hash, blob)
}
// PutSD stores the sd blob on S3
func (c *CloudFrontRWStore) PutSD(hash string, blob stream.Blob) error {
return c.s3.PutSD(hash, blob)
}
// Delete deletes the blob from S3
func (c *CloudFrontRWStore) Delete(hash string) error {
return c.s3.Delete(hash)
}
// Shutdown shuts down the store gracefully
func (c *CloudFrontRWStore) Shutdown() {
c.s3.Shutdown()
c.cf.Shutdown()
return
}
+27 -179
View File
@@ -2,79 +2,35 @@ package store
import (
"encoding/json"
"fmt"
"strings"
"sync"
"time"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/c2h5oh/datasize"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
)
// DBBackedStore is a store that's backed by a DB. The DB contains data about what's in the store.
type DBBackedStore struct {
blobs BlobStore
db *db.SQL
blocked map[string]bool
cleanerStop *stop.Group
name string
maxSize int
blockedMu sync.RWMutex
blocked map[string]bool
deleteOnMiss bool
}
type DBBackedParams struct {
Store BlobStore `mapstructure:"store"`
DB *db.SQL `mapstructure:"db"`
MaxSize *int `mapstructure:"max_size"`
Name string `mapstructure:"name"`
DeleteOnMiss bool `mapstructure:"delete_on_miss"`
}
type DBBackedConfig struct {
Store *viper.Viper
Database string `mapstructure:"database"`
User string `mapstructure:"user"`
Password string `mapstructure:"password"`
Host string `mapstructure:"host"`
Name string `mapstructure:"name"`
MaxSize string `mapstructure:"max_size"`
Port int `mapstructure:"port"`
AccessTracking int `mapstructure:"access_tracking"`
DeleteOnMiss bool `mapstructure:"delete_on_miss"`
SoftDeletes bool `mapstructure:"soft_deletes"`
LogQueries bool `mapstructure:"log_queries"`
HasCap bool `mapstructure:"has_cap"`
}
// NewDBBackedStore returns an initialized store pointer.
func NewDBBackedStore(params DBBackedParams) *DBBackedStore {
store := &DBBackedStore{
blobs: params.Store,
db: params.DB,
deleteOnMiss: params.DeleteOnMiss,
cleanerStop: stop.New(),
name: params.Name,
}
if params.MaxSize != nil {
store.maxSize = *params.MaxSize
go store.cleanOldestBlobs()
}
return store
func NewDBBackedStore(blobs BlobStore, db *db.SQL, deleteOnMiss bool) *DBBackedStore {
return &DBBackedStore{blobs: blobs, db: db, deleteOnMiss: deleteOnMiss}
}
const nameDBBacked = "db_backed"
const nameDBBacked = "db-backed"
// Name is the cache type name
func (d *DBBackedStore) Name() string { return nameDBBacked + "-" + d.name }
func (d *DBBackedStore) Name() string { return nameDBBacked }
// Has returns true if the blob is in the store
func (d *DBBackedStore) Has(hash string) (bool, error) {
@@ -144,18 +100,34 @@ func (d *DBBackedStore) Delete(hash string) error {
// Block deletes the blob and prevents it from being uploaded in the future
func (d *DBBackedStore) Block(hash string) error {
blocked, err := d.isBlocked(hash)
if blocked || err != nil {
if blocked, err := d.isBlocked(hash); blocked || err != nil {
return err
}
log.Debugf("blocking %s", hash)
err = d.db.Block(hash)
err := d.db.Block(hash)
if err != nil {
return err
}
has, err := d.db.HasBlob(hash, false)
if err != nil {
return err
}
if has {
err = d.blobs.Delete(hash)
if err != nil {
return err
}
err = d.db.Delete(hash)
if err != nil {
return err
}
}
return d.markBlocked(hash)
}
@@ -222,132 +194,8 @@ func (d *DBBackedStore) initBlocked() error {
return err
}
// cleanOldestBlobs periodically cleans up the oldest blobs if maxSize is set
func (d *DBBackedStore) cleanOldestBlobs() {
// Run on startup without waiting for 10 minutes
err := d.doClean()
if err != nil {
log.Error(errors.FullTrace(err))
}
const cleanupInterval = 10 * time.Minute
for {
select {
case <-d.cleanerStop.Ch():
log.Infoln("stopping self cleanup")
return
case <-time.After(cleanupInterval):
err := d.doClean()
if err != nil {
log.Error(errors.FullTrace(err))
}
}
}
}
// doClean removes the least recently accessed blobs if the store exceeds maxItems
func (d *DBBackedStore) doClean() error {
blobsCount, err := d.db.Count()
if err != nil {
return err
}
if blobsCount >= d.maxSize {
itemsToDelete := blobsCount / 10
blobs, err := d.db.LeastRecentlyAccessedHashes(itemsToDelete)
if err != nil {
return err
}
blobsChan := make(chan string, len(blobs))
wg := stop.New()
go func() {
for _, hash := range blobs {
select {
case <-d.cleanerStop.Ch():
return
default:
}
blobsChan <- hash
}
close(blobsChan)
}()
for i := 0; i < 3; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for h := range blobsChan {
select {
case <-d.cleanerStop.Ch():
return
default:
}
err = d.Delete(h)
if err != nil {
log.Errorf("error pruning %s: %s", h, errors.FullTrace(err))
continue
}
}
}()
}
wg.Wait()
}
return nil
}
// Shutdown shuts down the store gracefully
func (d *DBBackedStore) Shutdown() {
d.cleanerStop.Stop()
d.blobs.Shutdown()
}
func DBBackedStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg DBBackedConfig
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
cfg.Store = config.Sub("store")
storeType := strings.Split(cfg.Store.AllKeys()[0], ".")[0]
storeConfig := cfg.Store.Sub(storeType)
factory, ok := Factories[storeType]
if !ok {
return nil, errors.Err("unknown store type %s", storeType)
}
underlyingStore, err := factory(storeConfig)
if err != nil {
return nil, errors.Err(err)
}
parsedDb := &db.SQL{
TrackingLevel: db.AccessTrackingLevel(cfg.AccessTracking),
SoftDelete: cfg.SoftDeletes,
LogQueries: cfg.LogQueries || log.GetLevel() == log.DebugLevel,
}
err = parsedDb.Connect(fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", cfg.User, cfg.Password, cfg.Host, cfg.Port, cfg.Database))
if err != nil {
return nil, err
}
params := DBBackedParams{
Name: cfg.Name,
Store: underlyingStore,
DB: parsedDb,
DeleteOnMiss: cfg.DeleteOnMiss,
}
if cfg.HasCap {
var parsedSize datasize.ByteSize
err = parsedSize.UnmarshalText([]byte(cfg.MaxSize))
if err != nil {
return nil, errors.Err(err)
}
maxSize := int(float64(parsedSize) / float64(stream.MaxBlobSize))
params.MaxSize = &maxSize
}
return NewDBBackedStore(params), nil
}
func init() {
RegisterStore(nameDBBacked, DBBackedStoreFactory)
return
}
+94 -42
View File
@@ -1,59 +1,69 @@
package store
import (
"crypto/sha512"
"encoding/hex"
"fmt"
"io/ioutil"
"os"
"path"
"runtime"
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/reflector.go/store/speedwalk"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/reflector.go/store/speedwalk"
log "github.com/sirupsen/logrus"
"go.uber.org/atomic"
)
func init() {
writeCh = make(chan writeRequest)
for i := 0; i < runtime.NumCPU(); i++ {
go func() {
for {
select {
case r := <-writeCh:
err := ioutil.WriteFile(r.filename, r.data, r.perm)
if err != nil {
log.Errorf("could not write file %s to disk, failed with error: %s", r.filename, err.Error())
}
}
}
}()
}
}
var writeCh chan writeRequest
// DiskStore stores blobs on a local disk
type DiskStore struct {
blobDir string
name string
// the location of blobs on disk
blobDir string
// store files in subdirectories based on the first N chars in the filename. 0 = don't create subdirectories.
prefixLength int
initialized bool
// true if initOnce ran, false otherwise
initialized bool
concurrentChecks atomic.Int32
}
type DiskParams struct {
Name string `mapstructure:"name"`
MountPoint string `mapstructure:"mount_point"`
ShardingSize int `mapstructure:"sharding_size"`
}
const maxConcurrentChecks = 3
// NewDiskStore returns an initialized file disk store pointer.
func NewDiskStore(params DiskParams) *DiskStore {
func NewDiskStore(dir string, prefixLength int) *DiskStore {
return &DiskStore{
blobDir: params.MountPoint,
prefixLength: params.ShardingSize,
name: params.Name,
blobDir: dir,
prefixLength: prefixLength,
}
}
const nameDisk = "disk"
func DiskStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg DiskParams
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
return NewDiskStore(cfg), nil
}
func init() {
RegisterStore(nameDisk, DiskStoreFactory)
}
// Name is the cache type name
func (d *DiskStore) Name() string { return nameDisk + "-" + d.name }
func (d *DiskStore) Name() string { return nameDisk }
// Has returns T/F or Error if it the blob stored already. It will error with any IO disk error.
func (d *DiskStore) Has(hash string) (bool, error) {
@@ -80,16 +90,50 @@ func (d *DiskStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), err
}
blob, err := os.ReadFile(d.path(hash))
blob, err := ioutil.ReadFile(d.path(hash))
if err != nil {
if os.IsNotExist(err) {
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(ErrBlobNotFound)
}
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(err)
}
// this is a rather poor yet effective way of throttling how many blobs can be checked concurrently
// poor because there is a possible race condition between the check and the actual +1
if d.concurrentChecks.Load() < maxConcurrentChecks {
d.concurrentChecks.Add(1)
defer d.concurrentChecks.Sub(1)
hashBytes := sha512.Sum384(blob)
readHash := hex.EncodeToString(hashBytes[:])
if hash != readHash {
message := fmt.Sprintf("[%s] found a broken blob while reading from disk. Actual hash: %s", hash, readHash)
log.Errorf("%s", message)
err := d.Delete(hash)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), err
}
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(message)
}
}
return blob, shared.NewBlobTrace(time.Since(start), d.Name()), nil
}
// Put stores the blob on disk
func (d *DiskStore) Put(hash string, blob stream.Blob) error {
err := d.initOnce()
if err != nil {
return err
}
err = d.ensureDirExists(d.dir(hash))
if err != nil {
return err
}
writeFile(d.path(hash), blob, 0644)
return errors.Err(err)
}
// PutSD stores the sd blob on the disk
func (d *DiskStore) PutSD(hash string, blob stream.Blob) error {
return d.Put(hash, blob)
@@ -130,15 +174,11 @@ func (d *DiskStore) dir(hash string) string {
}
return path.Join(d.blobDir, hash[:d.prefixLength])
}
func (d *DiskStore) tmpDir() string {
return path.Join(d.blobDir, "tmp")
}
func (d *DiskStore) path(hash string) string {
return path.Join(d.dir(hash), hash)
}
func (d *DiskStore) tmpPath(hash string) string {
return path.Join(d.tmpDir(), hash)
}
func (d *DiskStore) ensureDirExists(dir string) error {
return errors.Err(os.MkdirAll(dir, 0755))
}
@@ -152,14 +192,26 @@ func (d *DiskStore) initOnce() error {
if err != nil {
return err
}
err = d.ensureDirExists(d.tmpDir())
if err != nil {
return err
}
d.initialized = true
return nil
}
type writeRequest struct {
filename string
data []byte
perm os.FileMode
}
// Shutdown shuts down the store gracefully
func (d *DiskStore) Shutdown() {
return
}
func writeFile(filename string, data []byte, perm os.FileMode) {
writeCh <- writeRequest{
filename: filename,
data: data,
perm: perm,
}
}
+8 -15
View File
@@ -1,6 +1,7 @@
package store
import (
"io/ioutil"
"os"
"path"
"path/filepath"
@@ -13,14 +14,10 @@ import (
)
func TestDiskStore_Get(t *testing.T) {
tmpDir, err := os.MkdirTemp("", "reflector_test_*")
tmpDir, err := ioutil.TempDir("", "reflector_test_*")
require.NoError(t, err)
defer func() { _ = os.RemoveAll(tmpDir) }()
d := NewDiskStore(DiskParams{
Name: "test",
MountPoint: tmpDir,
ShardingSize: 2,
})
defer os.RemoveAll(tmpDir)
d := NewDiskStore(tmpDir, 2)
hash := "f428b8265d65dad7f8ffa52922bba836404cbd62f3ecfe10adba6b444f8f658938e54f5981ac4de39644d5b93d89a94b"
data := []byte("oyuntyausntoyaunpdoyruoyduanrstjwfjyuwf")
@@ -28,7 +25,7 @@ func TestDiskStore_Get(t *testing.T) {
expectedPath := path.Join(tmpDir, hash[:2], hash)
err = os.MkdirAll(filepath.Dir(expectedPath), os.ModePerm)
require.NoError(t, err)
err = os.WriteFile(expectedPath, data, os.ModePerm)
err = ioutil.WriteFile(expectedPath, data, os.ModePerm)
require.NoError(t, err)
blob, _, err := d.Get(hash)
@@ -37,14 +34,10 @@ func TestDiskStore_Get(t *testing.T) {
}
func TestDiskStore_GetNonexistentBlob(t *testing.T) {
tmpDir, err := os.MkdirTemp("", "reflector_test_*")
tmpDir, err := ioutil.TempDir("", "reflector_test_*")
require.NoError(t, err)
defer func() { _ = os.RemoveAll(tmpDir) }()
d := NewDiskStore(DiskParams{
Name: "test",
MountPoint: tmpDir,
ShardingSize: 2,
})
defer os.RemoveAll(tmpDir)
d := NewDiskStore(tmpDir, 2)
blob, _, err := d.Get("nonexistent")
assert.Nil(t, blob)
-42
View File
@@ -1,42 +0,0 @@
//go:build darwin
// +build darwin
package store
import (
"bytes"
"io"
"os"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
)
var openFileFlags = os.O_WRONLY | os.O_CREATE
// Put stores the blob on disk
func (d *DiskStore) Put(hash string, blob stream.Blob) error {
err := d.initOnce()
if err != nil {
return err
}
err = d.ensureDirExists(d.dir(hash))
if err != nil {
return err
}
// Open file with O_DIRECT
f, err := os.OpenFile(d.tmpPath(hash), openFileFlags, 0644)
if err != nil {
return errors.Err(err)
}
defer f.Close()
_, err = io.Copy(f, bytes.NewReader(blob))
if err != nil {
return errors.Err(err)
}
err = os.Rename(d.tmpPath(hash), d.path(hash))
return errors.Err(err)
}
-49
View File
@@ -1,49 +0,0 @@
//go:build linux
// +build linux
package store
import (
"bytes"
"io"
"os"
"syscall"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/brk0v/directio"
)
var openFileFlags = os.O_WRONLY | os.O_CREATE | syscall.O_DIRECT
// Put stores the blob on disk
func (d *DiskStore) Put(hash string, blob stream.Blob) error {
err := d.initOnce()
if err != nil {
return err
}
err = d.ensureDirExists(d.dir(hash))
if err != nil {
return err
}
// Open file with O_DIRECT
f, err := os.OpenFile(d.tmpPath(hash), openFileFlags, 0644)
if err != nil {
return errors.Err(err)
}
defer func() { _ = f.Close() }()
dio, err := directio.New(f)
if err != nil {
return errors.Err(err)
}
defer func() { _ = dio.Flush() }()
_, err = io.Copy(dio, bytes.NewReader(blob))
if err != nil {
return errors.Err(err)
}
err = os.Rename(d.tmpPath(hash), d.path(hash))
return errors.Err(err)
}
-31
View File
@@ -1,31 +0,0 @@
//go:build !linux && !darwin
// +build !linux,!darwin
package store
import (
"os"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
)
// Put stores the blob on disk
func (d *DiskStore) Put(hash string, blob stream.Blob) error {
err := d.initOnce()
if err != nil {
return err
}
err = d.ensureDirExists(d.dir(hash))
if err != nil {
return err
}
err = os.WriteFile(d.tmpPath(hash), blob, 0644)
if err != nil {
return errors.Err(err)
}
err = os.Rename(d.tmpPath(hash), d.path(hash))
return errors.Err(err)
}
-211
View File
@@ -1,211 +0,0 @@
package store
import (
"strings"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/bluele/gcache"
"github.com/sirupsen/logrus"
"github.com/spf13/viper"
)
// GcacheStore adds a max cache size and Greedy-Dual-Size-Frequency cache eviction strategy to a BlobStore
type GcacheStore struct {
underlyingStore BlobStore
cache gcache.Cache
name string
}
type EvictionStrategy int
const (
//LFU Discards the least frequently used items first.
LFU EvictionStrategy = iota
//ARC Constantly balances between LRU and LFU, to improve the combined result.
ARC
//LRU Discards the least recently used items first.
LRU
//SIMPLE has no clear priority for evict cache. It depends on key-value map order.
SIMPLE
)
type GcacheParams struct {
Store BlobStore `mapstructure:"store"`
Name string `mapstructure:"name"`
MaxSize int `mapstructure:"max_size"`
Strategy EvictionStrategy `mapstructure:"strategy"`
}
type GcacheConfig struct {
Store *viper.Viper
Name string `mapstructure:"name"`
MaxSize int `mapstructure:"max_size"`
Strategy EvictionStrategy `mapstructure:"strategy"`
}
// NewGcacheStore initialize a new LRUStore
func NewGcacheStore(params GcacheParams) *GcacheStore {
cacheBuilder := gcache.New(params.MaxSize)
var cache gcache.Cache
evictFunc := func(key interface{}, value interface{}) {
logrus.Infof("evicting %s", key)
metrics.CacheLRUEvictCount.With(metrics.CacheLabels(params.Store.Name(), params.Name)).Inc()
_ = params.Store.Delete(key.(string)) // TODO: log this error. may happen if underlying entry is gone but cache entry still there
}
switch params.Strategy {
case LFU:
cache = cacheBuilder.LFU().EvictedFunc(evictFunc).Build()
case ARC:
cache = cacheBuilder.ARC().EvictedFunc(evictFunc).Build()
case LRU:
cache = cacheBuilder.LRU().EvictedFunc(evictFunc).Build()
case SIMPLE:
cache = cacheBuilder.Simple().EvictedFunc(evictFunc).Build()
}
l := &GcacheStore{
underlyingStore: params.Store,
cache: cache,
name: params.Name,
}
go func() {
if lstr, ok := params.Store.(lister); ok {
err := l.loadExisting(lstr, params.MaxSize)
if err != nil {
panic(err) // TODO: what should happen here? panic? return nil? just keep going?
}
}
}()
return l
}
const nameGcache = "gcache"
func GcacheStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg GcacheConfig
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
cfg.Store = config.Sub("store")
storeType := strings.Split(cfg.Store.AllKeys()[0], ".")[0]
storeConfig := cfg.Store.Sub(storeType)
factory, ok := Factories[storeType]
if !ok {
return nil, errors.Err("unknown store type %s", storeType)
}
underlyingStore, err := factory(storeConfig)
if err != nil {
return nil, errors.Err(err)
}
return NewGcacheStore(GcacheParams{
Name: cfg.Name,
Store: underlyingStore,
MaxSize: cfg.MaxSize,
Strategy: cfg.Strategy,
}), nil
}
func init() {
RegisterStore(nameGcache, GcacheStoreFactory)
}
// Name is the cache type name
func (l *GcacheStore) Name() string { return nameGcache + "-" + l.name }
// Has returns whether the blob is in the store, without updating the recent-ness.
func (l *GcacheStore) Has(hash string) (bool, error) {
return l.cache.Has(hash), nil
}
// Get returns the blob or an error if the blob doesn't exist.
func (l *GcacheStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
_, err := l.cache.Get(hash)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), l.Name()), errors.Err(ErrBlobNotFound)
}
blob, stack, err := l.underlyingStore.Get(hash)
if errors.Is(err, ErrBlobNotFound) {
// Blob disappeared from underlying store
l.cache.Remove(hash)
}
return blob, stack.Stack(time.Since(start), l.Name()), err
}
// Put stores the blob. Following LFUDA rules it's not guaranteed that a SET will store the value!!!
func (l *GcacheStore) Put(hash string, blob stream.Blob) error {
_ = l.cache.Set(hash, true)
has, _ := l.Has(hash)
if has {
err := l.underlyingStore.Put(hash, blob)
if err != nil {
return err
}
}
return nil
}
// PutSD stores the sd blob. Following LFUDA rules it's not guaranteed that a SET will store the value!!!
func (l *GcacheStore) PutSD(hash string, blob stream.Blob) error {
_ = l.cache.Set(hash, true)
has, _ := l.Has(hash)
if has {
err := l.underlyingStore.PutSD(hash, blob)
if err != nil {
return err
}
}
return nil
}
// Delete deletes the blob from the store
func (l *GcacheStore) Delete(hash string) error {
err := l.underlyingStore.Delete(hash)
if err != nil {
return err
}
// This must come after store.Delete()
// Remove triggers onEvict function, which also tries to delete blob from store
// We need to delete it manually first so any errors can be propagated up
l.cache.Remove(hash)
return nil
}
// loadExisting imports existing blobs from the underlying store into the LRU cache
func (l *GcacheStore) loadExisting(store lister, maxItems int) error {
logrus.Infof("loading at most %d items", maxItems)
existing, err := store.list()
if err != nil {
return err
}
logrus.Infof("read %d files from underlying store", len(existing))
added := 0
for i, h := range existing {
_ = l.cache.Set(h, true)
added++
if maxItems > 0 && added >= maxItems { // underlying cache is bigger than the cache
err := l.Delete(h)
logrus.Infof("deleted overflowing blob: %s (%d/%d)", h, i, len(existing))
if err != nil {
logrus.Warnf("error while deleting a blob that's overflowing the cache: %s", err.Error())
}
}
}
return nil
}
// Shutdown shuts down the store gracefully
func (l *GcacheStore) Shutdown() {
}
-114
View File
@@ -1,114 +0,0 @@
package store
import (
"fmt"
"os"
"reflect"
"testing"
"time"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
const cacheMaxSize = 3
func getTestGcacheStore() (*GcacheStore, *MemStore) {
m := NewMemStore(MemParams{Name: "test"})
return NewGcacheStore(GcacheParams{Name: "test", Store: m, MaxSize: cacheMaxSize, Strategy: LFU}), m
}
func TestGcacheStore_Eviction(t *testing.T) {
lfu, mem := getTestGcacheStore()
b := []byte("x")
for i := 0; i < 3; i++ {
err := lfu.Put(fmt.Sprintf("%d", i), b)
require.NoError(t, err)
for j := 0; j < 3-i; j++ {
_, _, err = lfu.Get(fmt.Sprintf("%d", i))
require.NoError(t, err)
}
}
for k, v := range map[string]bool{
"0": true,
"1": true,
"2": true,
} {
has, err := lfu.Has(k)
assert.NoError(t, err)
assert.Equal(t, v, has)
}
err := lfu.Put("3", b)
require.NoError(t, err)
for k, v := range map[string]bool{
"0": true,
"1": true,
"2": false,
"3": true,
} {
has, err := lfu.Has(k)
assert.NoError(t, err)
assert.Equal(t, v, has)
}
assert.Equal(t, cacheMaxSize, len(mem.Debug()))
err = lfu.Delete("0")
assert.NoError(t, err)
err = lfu.Delete("1")
assert.NoError(t, err)
err = lfu.Delete("3")
assert.NoError(t, err)
assert.Equal(t, 0, len(mem.Debug()))
}
func TestGcacheStore_UnderlyingBlobMissing(t *testing.T) {
lfu, mem := getTestGcacheStore()
hash := "hash"
b := []byte("this is a blob of stuff")
err := lfu.Put(hash, b)
require.NoError(t, err)
err = mem.Delete(hash)
require.NoError(t, err)
// hash still exists in lru
assert.True(t, lfu.cache.Has(hash))
blob, _, err := lfu.Get(hash)
assert.Nil(t, blob)
assert.True(t, errors.Is(err, ErrBlobNotFound), "expected (%s) %s, got (%s) %s",
reflect.TypeOf(ErrBlobNotFound).String(), ErrBlobNotFound.Error(),
reflect.TypeOf(err).String(), err.Error())
// lru.Get() removes hash if underlying store doesn't have it
assert.False(t, lfu.cache.Has(hash))
}
func TestGcacheStore_loadExisting(t *testing.T) {
tmpDir, err := os.MkdirTemp("", "reflector_test_*")
require.NoError(t, err)
defer func() { _ = os.RemoveAll(tmpDir) }()
d := NewDiskStore(DiskParams{
Name: "test",
MountPoint: tmpDir,
ShardingSize: 2,
})
hash := "hash"
b := []byte("this is a blob of stuff")
err = d.Put(hash, b)
require.NoError(t, err)
existing, err := d.list()
require.NoError(t, err)
require.Equal(t, 1, len(existing), "blob should exist in cache")
assert.Equal(t, hash, existing[0])
lfu := NewGcacheStore(GcacheParams{Name: "test", Store: d, MaxSize: 3, Strategy: LFU}) // lru should load existing blobs when it's created
time.Sleep(100 * time.Millisecond) // async load so let's wait...
has, err := lfu.Has(hash)
require.NoError(t, err)
assert.True(t, has, "hash should be loaded from disk store but it's not")
}
+101 -129
View File
@@ -1,188 +1,160 @@
package store
import (
"bytes"
"io"
"io/ioutil"
"net"
"net/http"
"path"
"strconv"
"sync"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/meta"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
)
// HttpStore reads from an HTTP endpoint that simply expects the hash to be appended to the endpoint
// NoopStore is a store that does nothing
type HttpStore struct {
httpClient *http.Client
endpoint string
name string
prefixLength int
upstream string
httpClient *http.Client
}
type HttpParams struct {
Name string `mapstructure:"name"`
Endpoint string `mapstructure:"endpoint"`
ShardingSize int `mapstructure:"sharding_size"`
}
// NewHttpStore returns an initialized HttpStore store pointer.
func NewHttpStore(params HttpParams) *HttpStore {
func NewHttpStore(upstream string) *HttpStore {
return &HttpStore{
endpoint: params.Endpoint,
httpClient: getClient(),
prefixLength: params.ShardingSize,
name: params.Name,
upstream: "http://" + upstream,
httpClient: getClient(),
}
}
const nameHttp = "http"
func HttpStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg HttpParams
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
return NewHttpStore(cfg), nil
}
func (n *HttpStore) Name() string { return nameNoop }
func (n *HttpStore) Has(hash string) (bool, error) {
url := n.upstream + "/blob?hash=" + hash
// Name is the cache type name
func (c *HttpStore) Name() string { return nameHttp + "-" + c.name }
// Has checks if the hash is in the store.
func (c *HttpStore) Has(hash string) (bool, error) {
status, body, err := c.cfRequest(http.MethodHead, hash)
req, err := http.NewRequest("HEAD", url, nil)
if err != nil {
return false, err
return false, errors.Err(err)
}
defer func() { _ = body.Close() }()
switch status {
case http.StatusNotFound, http.StatusForbidden:
res, err := n.httpClient.Do(req)
if err != nil {
return false, errors.Err(err)
}
defer res.Body.Close()
if res.StatusCode == http.StatusNotFound {
return false, nil
case http.StatusOK:
}
if res.StatusCode == http.StatusNoContent {
return true, nil
default:
return false, errors.Err("unexpected status %d", status)
}
var body []byte
if res.Body != nil {
body, _ = ioutil.ReadAll(res.Body)
}
return false, errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
}
// Get downloads the blob using the http client
func (c *HttpStore) Get(hash string) (b stream.Blob, trace shared.BlobTrace, err error) {
log.Debugf("Getting %s from HTTP(s) source", hash[:8])
func (n *HttpStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
url := n.upstream + "/blob?hash=" + hash
defer func(t time.Time) {
log.Debugf("Getting %s from HTTP(s) source took %s", hash[:8], time.Since(t).String())
trace = trace.Stack(time.Since(start), c.Name())
}(start)
url := c.endpoint + c.shardedPath(hash)
req, err := http.NewRequest(http.MethodGet, url, nil)
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, trace, errors.Err(err)
return nil, shared.NewBlobTrace(time.Since(start), n.Name()), errors.Err(err)
}
req.Header.Add("User-Agent", "reflector.go/"+meta.Version())
res, err := c.httpClient.Do(req)
res, err := n.httpClient.Do(req)
if err != nil {
return nil, trace, errors.Err(err)
return nil, shared.NewBlobTrace(time.Since(start), n.Name()), errors.Err(err)
}
defer func(Body io.ReadCloser) {
err := Body.Close()
defer res.Body.Close()
tmp := getBuffer()
defer putBuffer(tmp)
serialized := res.Header.Get("Via")
trace := shared.NewBlobTrace(time.Since(start), n.Name())
if serialized != "" {
parsedTrace, err := shared.Deserialize(serialized)
if err != nil {
log.Errorf("Error closing response body in HTTP-GET: %s", err.Error())
}
}(res.Body)
// Parse Via header if present
viaHeader := res.Header.Get("Via")
if viaHeader != "" {
parsedTrace, err := shared.Deserialize(viaHeader)
if err == nil {
trace = *parsedTrace
return nil, shared.NewBlobTrace(time.Since(start), n.Name()), err
}
trace = *parsedTrace
}
switch res.StatusCode {
case http.StatusNotFound:
return nil, trace, ErrBlobNotFound
case http.StatusOK:
contentLength := res.Header.Get("Content-Length")
if contentLength != "" {
size, err := strconv.Atoi(contentLength)
if err == nil && size > 0 && size <= stream.MaxBlobSize {
blob := make([]byte, size)
_, err = io.ReadFull(res.Body, blob)
if err == nil {
metrics.MtrInBytesHttp.Add(float64(size))
return blob, trace, nil
}
log.Warnf("Error reading body with known size: %s", err.Error())
}
if res.StatusCode == http.StatusNotFound {
return nil, trace.Stack(time.Since(start), n.Name()), ErrBlobNotFound
}
if res.StatusCode == http.StatusOK {
written, err := io.Copy(tmp, res.Body)
if err != nil {
return nil, trace.Stack(time.Since(start), n.Name()), errors.Err(err)
}
buffer := getBuffer()
defer putBuffer(buffer)
if _, err := io.Copy(buffer, res.Body); err != nil {
return nil, trace, errors.Err(err)
}
blob := make([]byte, buffer.Len())
copy(blob, buffer.Bytes())
blob := make([]byte, written)
copy(blob, tmp.Bytes())
metrics.MtrInBytesHttp.Add(float64(len(blob)))
return blob, trace, nil
default:
body, _ := io.ReadAll(res.Body)
log.Warnf("Got status code %d (%s)", res.StatusCode, string(body))
return nil, trace, errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
return blob, trace.Stack(time.Since(start), n.Name()), nil
}
}
func (c *HttpStore) cfRequest(method, hash string) (int, io.ReadCloser, error) {
url := c.endpoint + c.shardedPath(hash)
req, err := http.NewRequest(method, url, nil)
if err != nil {
return 0, nil, errors.Err(err)
}
req.Header.Add("User-Agent", "reflector.go/"+meta.Version())
res, err := c.httpClient.Do(req)
if err != nil {
return 0, nil, errors.Err(err)
var body []byte
if res.Body != nil {
body, _ = ioutil.ReadAll(res.Body)
}
return res.StatusCode, res.Body, nil
return nil, trace.Stack(time.Since(start), n.Name()), errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
}
func (c *HttpStore) Put(_ string, _ stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
func (n *HttpStore) Put(string, stream.Blob) error {
return shared.ErrNotImplemented
}
func (n *HttpStore) PutSD(string, stream.Blob) error {
return shared.ErrNotImplemented
}
func (n *HttpStore) Delete(string) error {
return shared.ErrNotImplemented
}
func (n *HttpStore) Shutdown() { return }
// buffer pool to reduce GC
// https://www.captaincodeman.com/2017/06/02/golang-buffer-pool-gotcha
var buffers = sync.Pool{
// New is called when a new instance is needed
New: func() interface{} {
buf := make([]byte, 0, stream.MaxBlobSize)
return bytes.NewBuffer(buf)
},
}
func (c *HttpStore) PutSD(_ string, _ stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
// getBuffer fetches a buffer from the pool
func getBuffer() *bytes.Buffer {
return buffers.Get().(*bytes.Buffer)
}
func (c *HttpStore) Delete(_ string) error {
return errors.Err(shared.ErrNotImplemented)
// putBuffer returns a buffer to the pool
func putBuffer(buf *bytes.Buffer) {
buf.Reset()
buffers.Put(buf)
}
// Shutdown shuts down the store gracefully
func (c *HttpStore) Shutdown() {
}
func (c *HttpStore) shardedPath(hash string) string {
if c.prefixLength <= 0 || len(hash) < c.prefixLength {
return hash
// getClient gets an http client that's customized to be more performant when dealing with blobs of 2MB in size (most of our blobs)
func getClient() *http.Client {
// Customize the Transport to have larger connection pool
defaultTransport := &http.Transport{
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).DialContext,
ForceAttemptHTTP2: true,
MaxIdleConns: 100,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
DisableCompression: true,
MaxIdleConnsPerHost: 100,
ReadBufferSize: stream.MaxBlobSize + 1024*10, //add an extra few KBs to make sure it fits the extra information
}
return path.Join(hash[:c.prefixLength], hash)
}
func init() {
RegisterStore(nameHttp, HttpStoreFactory)
return &http.Client{Transport: defaultTransport}
}
-136
View File
@@ -1,136 +0,0 @@
package store
import (
"strings"
"sync"
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
)
// Http3Store is a blob store that gets blobs from a peer over HTTP3.
// It satisfies the BlobStore interface but cannot put or delete blobs.
type Http3Store struct {
NotFoundCache *sync.Map
client *Http3Client
name string
address string
timeout time.Duration
clientMu sync.RWMutex
}
// Http3Params allows to set options for a new Http3Store.
type Http3Params struct {
Name string `mapstructure:"name"`
Address string `mapstructure:"address"`
Timeout time.Duration `mapstructure:"timeout"`
}
// NewHttp3Store makes a new HTTP3 store.
func NewHttp3Store(params Http3Params) *Http3Store {
return &Http3Store{
name: params.Name,
NotFoundCache: &sync.Map{},
address: params.Address,
timeout: params.Timeout,
}
}
const nameHttp3 = "http3"
func Http3StoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg Http3Params
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
return NewHttp3Store(cfg), nil
}
func init() {
RegisterStore(nameHttp3, Http3StoreFactory)
}
func (h *Http3Store) Name() string { return nameHttp3 + "-" + h.name }
func (h *Http3Store) getClient() (*Http3Client, error) {
h.clientMu.RLock()
if h.client != nil {
client := h.client
h.clientMu.RUnlock()
return client, nil
}
h.clientMu.RUnlock()
h.clientMu.Lock()
defer h.clientMu.Unlock()
// Check again in case another goroutine created the client
if h.client != nil {
return h.client, nil
}
client, err := NewHttp3Client(h.address)
if err != nil {
return nil, err
}
h.client = client
return client, nil
}
// Has asks the peer if they have a hash
func (h *Http3Store) Has(hash string) (bool, error) {
c, err := h.getClient()
if err != nil {
return false, err
}
return c.HasBlob(hash)
}
// Get downloads the blob from the peer
func (h *Http3Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
if lastChecked, ok := h.NotFoundCache.Load(hash); ok {
if lastChecked.(time.Time).After(time.Now().Add(-5 * time.Minute)) {
return nil, shared.NewBlobTrace(time.Since(start), h.Name()+"-notfoundcache"), ErrBlobNotFound
}
}
c, err := h.getClient()
if err != nil && strings.Contains(err.Error(), "blob not found") {
h.NotFoundCache.Store(hash, time.Now())
}
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), h.Name()), err
}
return c.GetBlob(hash)
}
// Put is not supported
func (h *Http3Store) Put(hash string, blob stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// PutSD is not supported
func (h *Http3Store) PutSD(hash string, blob stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// Delete is not supported
func (h *Http3Store) Delete(hash string) error {
return errors.Err(shared.ErrNotImplemented)
}
// Shutdown shuts down the store gracefully
func (h *Http3Store) Shutdown() {
h.clientMu.Lock()
defer h.clientMu.Unlock()
if h.client != nil {
_ = h.client.Close()
h.client = nil
}
}
-137
View File
@@ -1,137 +0,0 @@
package store
import (
"crypto/tls"
"crypto/x509"
"io"
"net/http"
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/quic-go/quic-go"
"github.com/quic-go/quic-go/http3"
)
// Http3Client is a client for HTTP3 blob store
type Http3Client struct {
conn *http.Client
roundTripper *http3.Transport
ServerAddr string
}
// NewHttp3Client creates a new HTTP3 client
func NewHttp3Client(address string) (*Http3Client, error) {
var qconf quic.Config
window500M := 500 * 1 << 20
qconf.MaxStreamReceiveWindow = uint64(window500M)
qconf.MaxConnectionReceiveWindow = uint64(window500M)
qconf.EnableDatagrams = true
qconf.HandshakeIdleTimeout = 4 * time.Second
qconf.MaxIdleTimeout = 20 * time.Second
pool, err := x509.SystemCertPool()
if err != nil {
return nil, err
}
roundTripper := &http3.Transport{
TLSClientConfig: &tls.Config{
RootCAs: pool,
InsecureSkipVerify: true,
},
QUICConfig: &qconf,
}
connection := &http.Client{
Transport: roundTripper,
}
return &Http3Client{
conn: connection,
roundTripper: roundTripper,
ServerAddr: address,
}, nil
}
// Close closes the client
func (c *Http3Client) Close() error {
return nil
}
// HasBlob checks if the peer has a blob
func (c *Http3Client) HasBlob(hash string) (bool, error) {
url := c.ServerAddr + "/blob?hash=" + hash
req, err := http.NewRequest("HEAD", url, nil)
if err != nil {
return false, errors.Err(err)
}
res, err := c.conn.Do(req)
if err != nil {
return false, errors.Err(err)
}
defer func() { _ = res.Body.Close() }()
if res.StatusCode == http.StatusNotFound {
return false, nil
}
if res.StatusCode == http.StatusNoContent {
return true, nil
}
var body []byte
if res.Body != nil {
body, _ = io.ReadAll(res.Body)
}
return false, errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
}
// GetBlob gets a blob from the peer
func (c *Http3Client) GetBlob(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
url := c.ServerAddr + "/blob?hash=" + hash
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err(err)
}
res, err := c.conn.Do(req)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err(err)
}
defer func() { _ = res.Body.Close() }()
viaHeader := res.Header.Get("Via")
var trace shared.BlobTrace
if viaHeader != "" {
parsedTrace, err := shared.Deserialize(viaHeader)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "http3"), err
}
trace = *parsedTrace
} else {
trace = shared.NewBlobTrace(0, "http3")
}
switch res.StatusCode {
case http.StatusNotFound:
return nil, trace.Stack(time.Since(start), "http3"), ErrBlobNotFound
case http.StatusOK:
buffer := getBuffer()
defer putBuffer(buffer)
if _, err := io.Copy(buffer, res.Body); err != nil {
return nil, trace.Stack(time.Since(start), "http3"), errors.Err(err)
}
blob := make([]byte, buffer.Len())
copy(blob, buffer.Bytes())
return blob, trace.Stack(time.Since(start), "http3"), nil
default:
body, _ := io.ReadAll(res.Body)
return nil, trace.Stack(time.Since(start), "http3"),
errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
}
}
+12 -73
View File
@@ -1,94 +1,34 @@
package store
import (
"strings"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/shared"
)
// ITTTStore performs an operation on this storage, if this fails, it attempts to run it on that
// ITTT store performs an operation on this storage, if this fails, it attempts to run it on that
type ITTTStore struct {
this, that BlobStore
name string
}
type ITTTParams struct {
This BlobStore `mapstructure:"this"`
That BlobStore `mapstructure:"that"`
Name string `mapstructure:"name"`
}
type ITTTConfig struct {
This *viper.Viper
That *viper.Viper
Name string `mapstructure:"name"`
}
// NewITTTStore returns a new instance of the IF THIS THAN THAT store
func NewITTTStore(params ITTTParams) *ITTTStore {
// NewCachingStore makes a new caching disk store and returns a pointer to it.
func NewITTTStore(this, that BlobStore) *ITTTStore {
return &ITTTStore{
this: params.This,
that: params.That,
name: params.Name,
this: this,
that: that,
}
}
const nameIttt = "ittt"
func ITTTStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg ITTTConfig
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
cfg.This = config.Sub("this")
cfg.That = config.Sub("that")
thisStoreType := strings.Split(cfg.This.AllKeys()[0], ".")[0]
thisStoreConfig := cfg.This.Sub(thisStoreType)
factory, ok := Factories[thisStoreType]
if !ok {
return nil, errors.Err("unknown store type %s", thisStoreType)
}
thisStore, err := factory(thisStoreConfig)
if err != nil {
return nil, errors.Err(err)
}
thatStoreType := strings.Split(cfg.That.AllKeys()[0], ".")[0]
thatStoreConfig := cfg.That.Sub(thatStoreType)
factory, ok = Factories[thatStoreType]
if !ok {
return nil, errors.Err("unknown store type %s", thatStoreType)
}
thatStore, err := factory(thatStoreConfig)
if err != nil {
return nil, errors.Err(err)
}
return NewITTTStore(ITTTParams{
Name: cfg.Name,
This: thisStore,
That: thatStore,
}), nil
}
func init() {
RegisterStore(nameIttt, ITTTStoreFactory)
}
// Name is the cache type name
func (c *ITTTStore) Name() string { return nameIttt + "-" + c.name }
func (c *ITTTStore) Name() string { return nameIttt }
// Has checks in this for a hash, if it fails it checks in that. It returns true if either store has it.
// Has checks the cache and then the origin for a hash. It returns true if either store has it.
func (c *ITTTStore) Has(hash string) (bool, error) {
has, err := c.this.Has(hash)
if err != nil || !has {
@@ -102,7 +42,7 @@ func (c *ITTTStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
blob, trace, err := c.this.Get(hash)
if err == nil {
metrics.ItttHitCount.With(metrics.ItttLabels(c.this.Name())).Inc()
metrics.ThisHitCount.Inc()
return blob, trace.Stack(time.Since(start), c.Name()), err
}
@@ -110,7 +50,7 @@ func (c *ITTTStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
if err != nil {
return nil, trace.Stack(time.Since(start), c.Name()), err
}
metrics.ItttHitCount.With(metrics.ItttLabels(c.that.Name())).Inc()
metrics.ThatHitCount.Inc()
return blob, trace.Stack(time.Since(start), c.Name()), nil
}
@@ -131,6 +71,5 @@ func (c *ITTTStore) Delete(hash string) error {
// Shutdown shuts down the store gracefully
func (c *ITTTStore) Shutdown() {
c.this.Shutdown()
c.that.Shutdown()
return
}
+132
View File
@@ -0,0 +1,132 @@
package store
import (
"time"
"github.com/bparli/lfuda-go"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/shared"
"github.com/sirupsen/logrus"
)
// LRUStore adds a max cache size and LRU eviction to a BlobStore
type LFUDAStore struct {
// underlying store
store BlobStore
// lfuda implementation
lfuda *lfuda.Cache
}
// NewLRUStore initialize a new LRUStore
func NewLFUDAStore(component string, store BlobStore, maxSize float64) *LFUDAStore {
lfuda := lfuda.NewGDSFWithEvict(maxSize, func(key interface{}, value interface{}) {
metrics.CacheLRUEvictCount.With(metrics.CacheLabels(store.Name(), component)).Inc()
_ = store.Delete(key.(string)) // TODO: log this error. may happen if underlying entry is gone but cache entry still there
})
l := &LFUDAStore{
store: store,
lfuda: lfuda,
}
go func() {
if lstr, ok := store.(lister); ok {
err := l.loadExisting(lstr, int(maxSize))
if err != nil {
panic(err) // TODO: what should happen here? panic? return nil? just keep going?
}
}
}()
return l
}
const nameLFUDA = "lfuda"
// Name is the cache type name
func (l *LFUDAStore) Name() string { return nameLFUDA }
// Has returns whether the blob is in the store, without updating the recent-ness.
func (l *LFUDAStore) Has(hash string) (bool, error) {
return l.lfuda.Contains(hash), nil
}
// Get returns the blob or an error if the blob doesn't exist.
func (l *LFUDAStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
_, has := l.lfuda.Get(hash)
if !has {
return nil, shared.NewBlobTrace(time.Since(start), l.Name()), errors.Err(ErrBlobNotFound)
}
blob, stack, err := l.store.Get(hash)
if errors.Is(err, ErrBlobNotFound) {
// Blob disappeared from underlying store
l.lfuda.Remove(hash)
}
return blob, stack.Stack(time.Since(start), l.Name()), err
}
// Put stores the blob. Following LFUDA rules it's not guaranteed that a SET will store the value!!!
func (l *LFUDAStore) Put(hash string, blob stream.Blob) error {
l.lfuda.Set(hash, true)
has, _ := l.Has(hash)
if has {
err := l.store.Put(hash, blob)
if err != nil {
return err
}
}
return nil
}
// PutSD stores the sd blob. Following LFUDA rules it's not guaranteed that a SET will store the value!!!
func (l *LFUDAStore) PutSD(hash string, blob stream.Blob) error {
l.lfuda.Set(hash, true)
has, _ := l.Has(hash)
if has {
err := l.store.PutSD(hash, blob)
if err != nil {
return err
}
}
return nil
}
// Delete deletes the blob from the store
func (l *LFUDAStore) Delete(hash string) error {
err := l.store.Delete(hash)
if err != nil {
return err
}
// This must come after store.Delete()
// Remove triggers onEvict function, which also tries to delete blob from store
// We need to delete it manually first so any errors can be propagated up
l.lfuda.Remove(hash)
return nil
}
// loadExisting imports existing blobs from the underlying store into the LRU cache
func (l *LFUDAStore) loadExisting(store lister, maxItems int) error {
logrus.Infof("loading at most %d items", maxItems)
existing, err := store.list()
if err != nil {
return err
}
logrus.Infof("read %d files from underlying store", len(existing))
added := 0
for _, h := range existing {
l.lfuda.Set(h, true)
added++
if maxItems > 0 && added >= maxItems { // underlying cache is bigger than the cache
break
}
}
return nil
}
// Shutdown shuts down the store gracefully
func (l *LFUDAStore) Shutdown() {
return
}
+136
View File
@@ -0,0 +1,136 @@
package store
import (
"io/ioutil"
"os"
"reflect"
"testing"
"time"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
const cacheMaxSize = 3
func getTestLFUDAStore() (*LFUDAStore, *MemStore) {
m := NewMemStore()
return NewLFUDAStore("test", m, cacheMaxSize), m
}
func TestFUDAStore_Eviction(t *testing.T) {
lfuda, mem := getTestLFUDAStore()
b := []byte("x")
err := lfuda.Put("one", b)
require.NoError(t, err)
err = lfuda.Put("two", b)
require.NoError(t, err)
err = lfuda.Put("three", b)
require.NoError(t, err)
err = lfuda.Put("four", b)
require.NoError(t, err)
err = lfuda.Put("five", b)
require.NoError(t, err)
err = lfuda.Put("five", b)
require.NoError(t, err)
err = lfuda.Put("four", b)
require.NoError(t, err)
err = lfuda.Put("two", b)
require.NoError(t, err)
_, _, err = lfuda.Get("five")
require.NoError(t, err)
_, _, err = lfuda.Get("four")
require.NoError(t, err)
_, _, err = lfuda.Get("two")
require.NoError(t, err)
assert.Equal(t, cacheMaxBlobs, len(mem.Debug()))
for k, v := range map[string]bool{
"one": false,
"two": true,
"three": false,
"four": true,
"five": true,
"six": false,
} {
has, err := lfuda.Has(k)
assert.NoError(t, err)
assert.Equal(t, v, has)
}
lfuda.Get("two") // touch so it stays in cache
lfuda.Get("five") // touch so it stays in cache
lfuda.Put("six", b)
assert.Equal(t, cacheMaxBlobs, len(mem.Debug()))
for k, v := range map[string]bool{
"one": false,
"two": true,
"three": false,
"four": false,
"five": true,
"six": true,
} {
has, err := lfuda.Has(k)
assert.NoError(t, err)
assert.Equal(t, v, has)
}
err = lfuda.Delete("six")
assert.NoError(t, err)
err = lfuda.Delete("five")
assert.NoError(t, err)
err = lfuda.Delete("two")
assert.NoError(t, err)
assert.Equal(t, 0, len(mem.Debug()))
}
func TestFUDAStore_UnderlyingBlobMissing(t *testing.T) {
lfuda, mem := getTestLFUDAStore()
hash := "hash"
b := []byte("this is a blob of stuff")
err := lfuda.Put(hash, b)
require.NoError(t, err)
err = mem.Delete(hash)
require.NoError(t, err)
// hash still exists in lru
assert.True(t, lfuda.lfuda.Contains(hash))
blob, _, err := lfuda.Get(hash)
assert.Nil(t, blob)
assert.True(t, errors.Is(err, ErrBlobNotFound), "expected (%s) %s, got (%s) %s",
reflect.TypeOf(ErrBlobNotFound).String(), ErrBlobNotFound.Error(),
reflect.TypeOf(err).String(), err.Error())
// lru.Get() removes hash if underlying store doesn't have it
assert.False(t, lfuda.lfuda.Contains(hash))
}
func TestFUDAStore_loadExisting(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "reflector_test_*")
require.NoError(t, err)
defer os.RemoveAll(tmpDir)
d := NewDiskStore(tmpDir, 2)
hash := "hash"
b := []byte("this is a blob of stuff")
err = d.Put(hash, b)
require.NoError(t, err)
existing, err := d.list()
require.NoError(t, err)
require.Equal(t, 1, len(existing), "blob should exist in cache")
assert.Equal(t, hash, existing[0])
lfuda := NewLFUDAStore("test", d, 3) // lru should load existing blobs when it's created
time.Sleep(100 * time.Millisecond) // async load so let's wait...
has, err := lfuda.Has(hash)
require.NoError(t, err)
assert.True(t, has, "hash should be loaded from disk store but it's not")
}
+129
View File
@@ -0,0 +1,129 @@
package store
import (
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/bluele/gcache"
"github.com/sirupsen/logrus"
)
// LRUStore adds a max cache size and LRU eviction to a BlobStore
type LRUStore struct {
// underlying store
store BlobStore
// lru implementation
lru gcache.Cache
}
// NewLRUStore initialize a new LRUStore
func NewLRUStore(component string, store BlobStore, maxItems int) *LRUStore {
l := &LRUStore{
store: store,
}
l.lru = gcache.New(maxItems).ARC().EvictedFunc(func(key, value interface{}) {
metrics.CacheLRUEvictCount.With(metrics.CacheLabels(l.Name(), component)).Inc()
_ = store.Delete(key.(string))
}).Build()
go func() {
if lstr, ok := store.(lister); ok {
err := l.loadExisting(lstr, maxItems)
if err != nil {
panic(err) // TODO: what should happen here? panic? return nil? just keep going?
}
}
}()
return l
}
// Name is the cache type name
func (l *LRUStore) Name() string {
return "lru_" + l.store.Name()
}
// Has returns whether the blob is in the store, without updating the recent-ness.
func (l *LRUStore) Has(hash string) (bool, error) {
return l.lru.Has(hash), nil
}
// Get returns the blob or an error if the blob doesn't exist.
func (l *LRUStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
_, err := l.lru.Get(hash)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), l.Name()), errors.Err(ErrBlobNotFound)
}
blob, stack, err := l.store.Get(hash)
if errors.Is(err, ErrBlobNotFound) {
// Blob disappeared from underlying store
l.lru.Remove(hash)
}
return blob, stack.Stack(time.Since(start), l.Name()), err
}
// Put stores the blob
func (l *LRUStore) Put(hash string, blob stream.Blob) error {
err := l.store.Put(hash, blob)
if err != nil {
return err
}
l.lru.Set(hash, true)
return nil
}
// PutSD stores the sd blob
func (l *LRUStore) PutSD(hash string, blob stream.Blob) error {
err := l.store.PutSD(hash, blob)
if err != nil {
return err
}
_ = l.lru.Set(hash, true)
return nil
}
// Delete deletes the blob from the store
func (l *LRUStore) Delete(hash string) error {
err := l.store.Delete(hash)
if err != nil {
return err
}
// This must come after store.Delete()
// Remove triggers onEvict function, which also tries to delete blob from store
// We need to delete it manually first so any errors can be propagated up
l.lru.Remove(hash)
return nil
}
// loadExisting imports existing blobs from the underlying store into the LRU cache
func (l *LRUStore) loadExisting(store lister, maxItems int) error {
logrus.Infof("loading at most %d items", maxItems)
existing, err := store.list()
if err != nil {
return err
}
logrus.Infof("read %d files from disk", len(existing))
added := 0
for _, h := range existing {
l.lru.Set(h, true)
added++
if maxItems > 0 && added >= maxItems { // underlying cache is bigger than LRU cache
break
}
}
return nil
}
// Shutdown shuts down the store gracefully
func (l *LRUStore) Shutdown() {
return
}
+123
View File
@@ -0,0 +1,123 @@
package store
import (
"io/ioutil"
"os"
"reflect"
"testing"
"time"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
const cacheMaxBlobs = 3
func getTestLRUStore() (*LRUStore, *MemStore) {
m := NewMemStore()
return NewLRUStore("test", m, 3), m
}
func TestLRUStore_Eviction(t *testing.T) {
lru, mem := getTestLRUStore()
b := []byte("x")
err := lru.Put("one", b)
require.NoError(t, err)
err = lru.Put("two", b)
require.NoError(t, err)
err = lru.Put("three", b)
require.NoError(t, err)
err = lru.Put("four", b)
require.NoError(t, err)
err = lru.Put("five", b)
require.NoError(t, err)
assert.Equal(t, cacheMaxBlobs, len(mem.Debug()))
for k, v := range map[string]bool{
"one": false,
"two": false,
"three": true,
"four": true,
"five": true,
"six": false,
} {
has, err := lru.Has(k)
assert.NoError(t, err)
assert.Equal(t, v, has)
}
lru.Get("three") // touch so it stays in cache
lru.Put("six", b)
assert.Equal(t, cacheMaxBlobs, len(mem.Debug()))
for k, v := range map[string]bool{
"one": false,
"two": false,
"three": true,
"four": false,
"five": true,
"six": true,
} {
has, err := lru.Has(k)
assert.NoError(t, err)
assert.Equal(t, v, has)
}
err = lru.Delete("three")
assert.NoError(t, err)
err = lru.Delete("five")
assert.NoError(t, err)
err = lru.Delete("six")
assert.NoError(t, err)
assert.Equal(t, 0, len(mem.Debug()))
}
func TestLRUStore_UnderlyingBlobMissing(t *testing.T) {
lru, mem := getTestLRUStore()
hash := "hash"
b := []byte("this is a blob of stuff")
err := lru.Put(hash, b)
require.NoError(t, err)
err = mem.Delete(hash)
require.NoError(t, err)
// hash still exists in lru
assert.True(t, lru.lru.Has(hash))
blob, _, err := lru.Get(hash)
assert.Nil(t, blob)
assert.True(t, errors.Is(err, ErrBlobNotFound), "expected (%s) %s, got (%s) %s",
reflect.TypeOf(ErrBlobNotFound).String(), ErrBlobNotFound.Error(),
reflect.TypeOf(err).String(), err.Error())
// lru.Get() removes hash if underlying store doesn't have it
assert.False(t, lru.lru.Has(hash))
}
func TestLRUStore_loadExisting(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "reflector_test_*")
require.NoError(t, err)
defer os.RemoveAll(tmpDir)
d := NewDiskStore(tmpDir, 2)
hash := "hash"
b := []byte("this is a blob of stuff")
err = d.Put(hash, b)
require.NoError(t, err)
existing, err := d.list()
require.NoError(t, err)
require.Equal(t, 1, len(existing), "blob should exist in cache")
assert.Equal(t, hash, existing[0])
lru := NewLRUStore("test", d, 3) // lru should load existing blobs when it's created
time.Sleep(100 * time.Millisecond) // async load so let's wait...
has, err := lru.Has(hash)
require.NoError(t, err)
assert.True(t, has, "hash should be loaded from disk store but it's not")
}
+6 -26
View File
@@ -4,50 +4,28 @@ import (
"sync"
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
"github.com/lbryio/reflector.go/shared"
)
// MemStore is an in memory only blob store with no persistence.
type MemStore struct {
blobs map[string]stream.Blob
mu *sync.RWMutex
name string
}
type MemParams struct {
Name string `mapstructure:"name"`
}
func NewMemStore(params MemParams) *MemStore {
func NewMemStore() *MemStore {
return &MemStore{
blobs: make(map[string]stream.Blob),
mu: &sync.RWMutex{},
name: params.Name,
}
}
const nameMem = "mem"
func MemStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg MemParams
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
return NewMemStore(cfg), nil
}
func init() {
RegisterStore(nameMem, MemStoreFactory)
}
// Name is the cache type name
func (m *MemStore) Name() string { return nameMem + "-" + m.name }
func (m *MemStore) Name() string { return nameMem }
// Has returns T/F if the blob is currently stored. It will never error.
func (m *MemStore) Has(hash string) (bool, error) {
@@ -98,4 +76,6 @@ func (m *MemStore) Debug() map[string]stream.Blob {
}
// Shutdown shuts down the store gracefully
func (m *MemStore) Shutdown() {}
func (m *MemStore) Shutdown() {
return
}
+2 -2
View File
@@ -8,7 +8,7 @@ import (
)
func TestMemStore_Put(t *testing.T) {
s := NewMemStore(MemParams{Name: "test"})
s := NewMemStore()
blob := []byte("abcdefg")
err := s.Put("abc", blob)
if err != nil {
@@ -17,7 +17,7 @@ func TestMemStore_Put(t *testing.T) {
}
func TestMemStore_Get(t *testing.T) {
s := NewMemStore(MemParams{Name: "test"})
s := NewMemStore()
hash := "abc"
blob := []byte("abcdefg")
err := s.Put(hash, blob)
-206
View File
@@ -1,206 +0,0 @@
package store
import (
"strings"
"sync"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
)
// MultiWriterStore writes to multiple destination stores
type MultiWriterStore struct {
name string
destinations []BlobStore
}
type MultiWriterParams struct {
Name string
Destinations []BlobStore
}
type MultiWriterConfig struct {
Name string `mapstructure:"name"`
One viper.Viper
Two viper.Viper
Three viper.Viper
}
// NewMultiWriterStore returns a new instance of the MultiWriter store
func NewMultiWriterStore(params MultiWriterParams) *MultiWriterStore {
return &MultiWriterStore{
name: params.Name,
destinations: params.Destinations,
}
}
const nameMultiWriter = "multiwriter"
func MultiWriterStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg MultiWriterConfig
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
var destinations []BlobStore
one := config.Sub("one")
two := config.Sub("two")
//three := config.Sub("three")
storeTypeOne := strings.Split(one.AllKeys()[0], ".")[0]
storeTypeTwo := strings.Split(two.AllKeys()[0], ".")[0]
//storeTypeThree := strings.Split(three.AllKeys()[0], ".")[0]
storeCfgOne := one.Sub(storeTypeOne)
storeCfgTwo := two.Sub(storeTypeTwo)
//storeCfgThree := config.Sub(storeTypeThree)
factoryOne, ok := Factories[storeTypeOne]
if !ok {
return nil, errors.Err("unknown store type %s", storeTypeOne)
}
factoryTwo, ok := Factories[storeTypeTwo]
if !ok {
return nil, errors.Err("unknown store type %s", storeTypeTwo)
}
//factoryThree, ok := Factories[storeTypeThree]
//if !ok {
// return nil, errors.Err("unknown store type %s", storeTypeThree)
//}
store1, err := factoryOne(storeCfgOne)
if err != nil {
return nil, errors.Err(err)
}
store2, err := factoryTwo(storeCfgTwo)
if err != nil {
return nil, errors.Err(err)
}
//store3, err := factoryThree(storeCfgThree)
//if err != nil {
// return nil, errors.Err(err)
//}
destinations = append(destinations, store1, store2)
return NewMultiWriterStore(MultiWriterParams{
Name: cfg.Name,
Destinations: destinations,
}), nil
}
func init() {
RegisterStore(nameMultiWriter, MultiWriterStoreFactory)
}
// Name returns the store name
func (m *MultiWriterStore) Name() string { return nameMultiWriter + "-" + m.name }
// Has is not supported by MultiWriter
func (m *MultiWriterStore) Has(hash string) (bool, error) {
return false, errors.Err(shared.ErrNotImplemented)
}
// Get is not supported by MultiWriter
func (m *MultiWriterStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
return nil, shared.BlobTrace{}, errors.Err(shared.ErrNotImplemented)
}
// Put writes the blob to all destination stores
func (m *MultiWriterStore) Put(hash string, blob stream.Blob) error {
var wg sync.WaitGroup
errChan := make(chan error, len(m.destinations))
for _, dest := range m.destinations {
wg.Add(1)
go func(d BlobStore) {
defer wg.Done()
if err := d.Put(hash, blob); err != nil {
errChan <- errors.Err("failed to write to %s: %v", d.Name(), err)
}
}(dest)
}
wg.Wait()
close(errChan)
var errs []string
for err := range errChan {
errs = append(errs, err.Error())
}
if len(errs) > 0 {
return errors.Err("failed to write to some destinations: %s", strings.Join(errs, "; "))
}
return nil
}
// PutSD writes the SD blob to all destination stores
func (m *MultiWriterStore) PutSD(hash string, blob stream.Blob) error {
var wg sync.WaitGroup
errChan := make(chan error, len(m.destinations))
for _, dest := range m.destinations {
wg.Add(1)
go func(d BlobStore) {
defer wg.Done()
if err := d.PutSD(hash, blob); err != nil {
errChan <- errors.Err("failed to write SD to %s: %v", d.Name(), err)
}
}(dest)
}
wg.Wait()
close(errChan)
var errs []string
for err := range errChan {
errs = append(errs, err.Error())
}
if len(errs) > 0 {
return errors.Err("failed to write SD to some destinations: %s", strings.Join(errs, "; "))
}
return nil
}
// Delete deletes the blob from all destination stores
func (m *MultiWriterStore) Delete(hash string) error {
var wg sync.WaitGroup
errChan := make(chan error, len(m.destinations))
for _, dest := range m.destinations {
wg.Add(1)
go func(d BlobStore) {
defer wg.Done()
if err := d.Delete(hash); err != nil {
errChan <- errors.Err("failed to delete from %s: %v", d.Name(), err)
}
}(dest)
}
wg.Wait()
close(errChan)
var errs []string
for err := range errChan {
errs = append(errs, err.Error())
}
if len(errs) > 0 {
return errors.Err("failed to delete from some destinations: %s", strings.Join(errs, "; "))
}
return nil
}
// Shutdown shuts down all destination stores gracefully
func (m *MultiWriterStore) Shutdown() {
for _, dest := range m.destinations {
dest.Shutdown()
}
}
+4 -29
View File
@@ -3,41 +3,16 @@ package store
import (
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
"github.com/lbryio/reflector.go/shared"
)
// NoopStore is a store that does nothing
type NoopStore struct {
name string
}
func NewNoopStore(name string) *NoopStore {
return &NoopStore{name: name}
}
type NoopStore struct{}
const nameNoop = "noop"
func NoopStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg struct {
Name string `mapstructure:"name"`
}
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
return NewNoopStore(cfg.Name), nil
}
func init() {
RegisterStore(nameNoop, NoopStoreFactory)
}
func (n *NoopStore) Name() string { return nameNoop + "-" + n.name }
func (n *NoopStore) Name() string { return nameNoop }
func (n *NoopStore) Has(_ string) (bool, error) { return false, nil }
func (n *NoopStore) Get(_ string) (stream.Blob, shared.BlobTrace, error) {
return nil, shared.NewBlobTrace(time.Since(time.Now()), n.Name()), nil
@@ -45,4 +20,4 @@ func (n *NoopStore) Get(_ string) (stream.Blob, shared.BlobTrace, error) {
func (n *NoopStore) Put(_ string, _ stream.Blob) error { return nil }
func (n *NoopStore) PutSD(_ string, _ stream.Blob) error { return nil }
func (n *NoopStore) Delete(_ string) error { return nil }
func (n *NoopStore) Shutdown() {}
func (n *NoopStore) Shutdown() { return }
-100
View File
@@ -1,100 +0,0 @@
package store
import (
"strings"
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
)
// PeerStore is a blob store that gets blobs from a peer.
// It satisfies the BlobStore interface but cannot put or delete blobs.
type PeerStore struct {
name string
opts PeerParams
}
// PeerParams allows to set options for a new PeerStore.
type PeerParams struct {
Name string `mapstructure:"name"`
Address string `mapstructure:"address"`
Timeout time.Duration `mapstructure:"timeout"`
}
// NewPeerStore makes a new peer store.
func NewPeerStore(params PeerParams) *PeerStore {
return &PeerStore{opts: params, name: params.Name}
}
const namePeer = "peer"
func PeerStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg PeerParams
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
return NewPeerStore(cfg), nil
}
func init() {
RegisterStore(namePeer, PeerStoreFactory)
}
func (p *PeerStore) Name() string { return namePeer + "-" + p.name }
func (p *PeerStore) getClient() (*PeerClient, error) {
c := &PeerClient{Timeout: p.opts.Timeout}
err := c.Connect(p.opts.Address)
return c, errors.Prefix("connection error", err)
}
// Has asks the peer if they have a hash
func (p *PeerStore) Has(hash string) (bool, error) {
c, err := p.getClient()
if err != nil {
return false, err
}
defer func() { _ = c.Close() }()
return c.HasBlob(hash)
}
// Get downloads the blob from the peer
func (p *PeerStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
c, err := p.getClient()
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), p.Name()), err
}
defer func() { _ = c.Close() }()
blob, trace, err := c.GetBlob(hash)
if err != nil && strings.Contains(err.Error(), "blob not found") {
return nil, trace, ErrBlobNotFound
}
return blob, trace, err
}
// Put is not supported
func (p *PeerStore) Put(hash string, blob stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// PutSD is not supported
func (p *PeerStore) PutSD(hash string, blob stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// Delete is not supported
func (p *PeerStore) Delete(hash string) error {
return errors.Err(shared.ErrNotImplemented)
}
// Shutdown is not supported
func (p *PeerStore) Shutdown() {
}
-95
View File
@@ -1,95 +0,0 @@
package store
import (
"bufio"
"bytes"
"encoding/binary"
"io"
"net"
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/stream"
)
// PeerClient is a client for peer blob store
type PeerClient struct {
conn net.Conn
Timeout time.Duration
}
// Connect connects to a peer
func (c *PeerClient) Connect(address string) error {
var err error
c.conn, err = net.DialTimeout("tcp", address, c.Timeout)
return err
}
// Close closes the connection
func (c *PeerClient) Close() error {
if c.conn != nil {
return c.conn.Close()
}
return nil
}
// HasBlob checks if the peer has a blob
func (c *PeerClient) HasBlob(hash string) (bool, error) {
err := c.writeRequest("has", hash)
if err != nil {
return false, err
}
response, err := c.readResponse()
if err != nil {
return false, err
}
return response == "yes", nil
}
// GetBlob gets a blob from the peer
func (c *PeerClient) GetBlob(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
err := c.writeRequest("get", hash)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "peer"), err
}
response, err := c.readResponse()
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "peer"), err
}
if response == "no" {
return nil, shared.NewBlobTrace(time.Since(start), "peer"), ErrBlobNotFound
}
size, err := binary.ReadVarint(bufio.NewReader(bytes.NewReader([]byte(response))))
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "peer"), err
}
blob := make([]byte, size)
_, err = io.ReadFull(c.conn, blob)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "peer"), err
}
return blob, shared.NewBlobTrace(time.Since(start), "peer"), nil
}
func (c *PeerClient) writeRequest(cmd, hash string) error {
_, err := c.conn.Write([]byte(cmd + " " + hash + "\n"))
return err
}
func (c *PeerClient) readResponse() (string, error) {
reader := bufio.NewReader(c.conn)
response, err := reader.ReadString('\n')
if err != nil {
return "", err
}
return response[:len(response)-1], nil
}
-147
View File
@@ -1,147 +0,0 @@
package store
import (
"strings"
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
)
// ProxiedS3Store writes to an S3 store and reads from any BlobStore (usually an ITTTStore of HttpStore endpoints).
type ProxiedS3Store struct {
readerStore BlobStore
writerStore BlobStore
name string
}
func (c *ProxiedS3Store) MissingBlobsForKnownStream(s string) ([]string, error) {
if bc, ok := c.writerStore.(NeededBlobChecker); ok {
return bc.MissingBlobsForKnownStream(s)
}
return nil, errors.Err("writer does not implement neededBlobChecker")
}
func (c *ProxiedS3Store) Block(hash string) error {
if bl, ok := c.writerStore.(Blocklister); ok {
return bl.Block(hash)
}
return errors.Err("writer does not implement Blocklister")
}
func (c *ProxiedS3Store) Wants(hash string) (bool, error) {
if bl, ok := c.writerStore.(Blocklister); ok {
return bl.Wants(hash)
}
return true, errors.Err("writer does not implement Blocklister")
}
type ProxiedS3Params struct {
Reader BlobStore `mapstructure:"reader"`
Writer BlobStore `mapstructure:"writer"`
Name string `mapstructure:"name"`
}
type ProxiedS3Config struct {
Reader *viper.Viper
Writer *viper.Viper
Name string `mapstructure:"name"`
}
// NewProxiedS3Store returns an initialized ProxiedS3Store store pointer.
// NOTE: It panics if either argument is nil.
func NewProxiedS3Store(params ProxiedS3Params) *ProxiedS3Store {
if params.Reader == nil || params.Writer == nil {
panic("both stores must be set")
}
return &ProxiedS3Store{
readerStore: params.Reader,
writerStore: params.Writer,
name: params.Name,
}
}
const nameProxiedS3 = "proxied-s3"
// Name is the cache type name
func (c *ProxiedS3Store) Name() string { return nameProxiedS3 + "-" + c.name }
// Has checks if the hash is in the store.
func (c *ProxiedS3Store) Has(hash string) (bool, error) {
return c.writerStore.Has(hash)
}
// Get gets the blob from Cloudfront.
func (c *ProxiedS3Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
blob, trace, err := c.readerStore.Get(hash)
return blob, trace.Stack(time.Since(start), c.Name()), err
}
// Put stores the blob on S3
func (c *ProxiedS3Store) Put(hash string, blob stream.Blob) error {
return c.writerStore.Put(hash, blob)
}
// PutSD stores the sd blob on S3
func (c *ProxiedS3Store) PutSD(hash string, blob stream.Blob) error {
return c.writerStore.PutSD(hash, blob)
}
// Delete deletes the blob from S3
func (c *ProxiedS3Store) Delete(hash string) error {
return c.writerStore.Delete(hash)
}
// Shutdown shuts down the store gracefully
func (c *ProxiedS3Store) Shutdown() {
c.writerStore.Shutdown()
c.readerStore.Shutdown()
}
func ProxiedS3StoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg ProxiedS3Config
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
cfg.Reader = config.Sub("reader")
cfg.Writer = config.Sub("writer")
readerStoreType := strings.Split(cfg.Reader.AllKeys()[0], ".")[0]
readerStoreConfig := cfg.Reader.Sub(readerStoreType)
factory, ok := Factories[readerStoreType]
if !ok {
return nil, errors.Err("unknown store type %s", readerStoreType)
}
readerStore, err := factory(readerStoreConfig)
if err != nil {
return nil, errors.Err(err)
}
writerStoreType := strings.Split(cfg.Writer.AllKeys()[0], ".")[0]
writerStoreConfig := cfg.Writer.Sub(writerStoreType)
factory, ok = Factories[writerStoreType]
if !ok {
return nil, errors.Err("unknown store type %s", writerStoreType)
}
writerStore, err := factory(writerStoreConfig)
if err != nil {
return nil, errors.Err(err)
}
return NewProxiedS3Store(ProxiedS3Params{
Name: cfg.Name,
Reader: readerStore,
Writer: writerStore,
}), nil
}
func init() {
RegisterStore(nameProxiedS3, ProxiedS3StoreFactory)
}
+58 -107
View File
@@ -2,99 +2,64 @@ package store
import (
"bytes"
"context"
stderrors "errors"
"path"
"net/http"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/shared"
"github.com/aws/aws-sdk-go-v2/aws"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
)
// S3Store is an S3 store
type S3Store struct {
client *s3.Client
awsID string
awsSecret string
region string
bucket string
endpoint string
name string
prefixLength int
}
awsID string
awsSecret string
region string
bucket string
type S3Params struct {
Name string `mapstructure:"name"`
AwsID string `mapstructure:"aws_id"`
AwsSecret string `mapstructure:"aws_secret"`
Region string `mapstructure:"region"`
Bucket string `mapstructure:"bucket"`
Endpoint string `mapstructure:"endpoint"`
ShardingSize int `mapstructure:"sharding_size"`
session *session.Session
}
// NewS3Store returns an initialized S3 store pointer.
func NewS3Store(params S3Params) *S3Store {
func NewS3Store(awsID, awsSecret, region, bucket string) *S3Store {
return &S3Store{
awsID: params.AwsID,
awsSecret: params.AwsSecret,
region: params.Region,
bucket: params.Bucket,
endpoint: params.Endpoint,
name: params.Name,
prefixLength: params.ShardingSize,
awsID: awsID,
awsSecret: awsSecret,
region: region,
bucket: bucket,
}
}
const nameS3 = "s3"
func S3StoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg S3Params
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
return NewS3Store(cfg), nil
}
func init() {
RegisterStore(nameS3, S3StoreFactory)
}
// Name is the cache type name
func (s *S3Store) Name() string { return nameS3 + "-" + s.name }
func (s *S3Store) Name() string { return nameS3 }
// Has returns T/F or Error (from S3) if the store contains the blob.
// Has returns T/F or Error ( from S3 ) if the store contains the blob.
func (s *S3Store) Has(hash string) (bool, error) {
err := s.initOnce()
if err != nil {
return false, err
}
ctx := context.Background()
_, err = s.client.HeadObject(ctx, &s3.HeadObjectInput{
_, err = s3.New(s.session).HeadObject(&s3.HeadObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(s.shardedPath(hash)),
Key: aws.String(hash),
})
if err != nil {
var notFound *types.NotFound
var noSuchKey *types.NoSuchKey
if stderrors.As(err, &notFound) || stderrors.As(err, &noSuchKey) {
if reqFail, ok := err.(s3.RequestFailure); ok && reqFail.StatusCode() == http.StatusNotFound {
return false, nil
}
return false, errors.Err(err)
return false, err
}
return true, nil
@@ -103,6 +68,7 @@ func (s *S3Store) Has(hash string) (bool, error) {
// Get returns the blob slice if present or errors on S3.
func (s *S3Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
//Todo-Need to handle error for blob doesn't exist for consistency.
err := s.initOnce()
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), s.Name()), err
@@ -110,26 +76,24 @@ func (s *S3Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
log.Debugf("Getting %s from S3", hash[:8])
defer func(t time.Time) {
log.Debugf("Getting %s from %s took %s", hash[:8], s.Name(), time.Since(t).String())
log.Debugf("Getting %s from S3 took %s", hash[:8], time.Since(t).String())
}(start)
ctx := context.Background()
buf := manager.NewWriteAtBuffer([]byte{})
_, err = manager.NewDownloader(s.client).Download(ctx, buf, &s3.GetObjectInput{
buf := &aws.WriteAtBuffer{}
_, err = s3manager.NewDownloader(s.session).Download(buf, &s3.GetObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(s.shardedPath(hash)),
Key: aws.String(hash),
})
if err != nil {
var noSuchBucket *types.NoSuchBucket
var noSuchKey *types.NoSuchKey
var notFound *types.NotFound
switch {
case stderrors.As(err, &noSuchBucket):
return nil, shared.NewBlobTrace(time.Since(start), s.Name()), errors.Err("bucket %s does not exist", s.bucket)
case stderrors.As(err, &noSuchKey), stderrors.As(err, &notFound):
return nil, shared.NewBlobTrace(time.Since(start), s.Name()), errors.Err(ErrBlobNotFound)
if aerr, ok := err.(awserr.Error); ok {
switch aerr.Code() {
case s3.ErrCodeNoSuchBucket:
return nil, shared.NewBlobTrace(time.Since(start), s.Name()), errors.Err("bucket %s does not exist", s.bucket)
case s3.ErrCodeNoSuchKey:
return nil, shared.NewBlobTrace(time.Since(start), s.Name()), errors.Err(ErrBlobNotFound)
}
}
return nil, shared.NewBlobTrace(time.Since(start), s.Name()), errors.Err(err)
return buf.Bytes(), shared.NewBlobTrace(time.Since(start), s.Name()), err
}
return buf.Bytes(), shared.NewBlobTrace(time.Since(start), s.Name()), nil
@@ -147,18 +111,16 @@ func (s *S3Store) Put(hash string, blob stream.Blob) error {
log.Debugf("Uploading %s took %s", hash[:8], time.Since(t).String())
}(time.Now())
ctx := context.Background()
_, err = manager.NewUploader(s.client).Upload(ctx, &s3.PutObjectInput{
_, err = s3manager.NewUploader(s.session).Upload(&s3manager.UploadInput{
Bucket: aws.String(s.bucket),
Key: aws.String(s.shardedPath(hash)),
Key: aws.String(hash),
Body: bytes.NewBuffer(blob),
ACL: types.ObjectCannedACLPublicRead,
ACL: aws.String("public-read"),
//StorageClass: aws.String(s3.StorageClassIntelligentTiering),
})
if err != nil {
return errors.Err(err)
}
metrics.MtrOutBytesReflector.Add(float64(blob.Size()))
return nil
return err
}
// PutSD stores the sd blob on S3 or errors if S3 connection errors.
@@ -175,44 +137,33 @@ func (s *S3Store) Delete(hash string) error {
log.Debugf("Deleting %s from S3", hash[:8])
ctx := context.Background()
_, err = s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
_, err = s3.New(s.session).DeleteObject(&s3.DeleteObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(s.shardedPath(hash)),
Key: aws.String(hash),
})
return errors.Err(err)
return err
}
func (s *S3Store) initOnce() error {
if s.client != nil {
if s.session != nil {
return nil
}
ctx := context.Background()
cfg, err := awsconfig.LoadDefaultConfig(ctx,
awsconfig.WithRegion(s.region),
awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(s.awsID, s.awsSecret, "")),
)
if err != nil {
return errors.Err(err)
}
s.client = s3.NewFromConfig(cfg, func(o *s3.Options) {
o.BaseEndpoint = aws.String(s.endpoint)
o.UsePathStyle = true
o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired
sess, err := session.NewSession(&aws.Config{
Credentials: credentials.NewStaticCredentials(s.awsID, s.awsSecret, ""),
Region: aws.String(s.region),
Endpoint: aws.String("https://s3.wasabisys.com"),
})
return nil
}
func (s *S3Store) shardedPath(hash string) string {
if s.prefixLength <= 0 || len(hash) < s.prefixLength {
return hash
if err != nil {
return err
}
return path.Join(hash[:s.prefixLength], hash)
s.session = sess
return nil
}
// Shutdown shuts down the store gracefully
func (s *S3Store) Shutdown() {
return
}
+4 -36
View File
@@ -1,16 +1,14 @@
package store
import (
"strings"
"time"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
"golang.org/x/sync/singleflight"
)
@@ -24,40 +22,9 @@ func WithSingleFlight(component string, origin BlobStore) BlobStore {
type singleflightStore struct {
BlobStore
sf *singleflight.Group
component string
}
type SingleFlightConfig struct {
Store *viper.Viper
Component string `mapstructure:"component"`
}
func SingleFlightStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg SingleFlightConfig
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
cfg.Store = config.Sub("store")
storeType := strings.Split(cfg.Store.AllKeys()[0], ".")[0]
storeConfig := cfg.Store.Sub(storeType)
factory, ok := Factories[storeType]
if !ok {
return nil, errors.Err("unknown store type %s", storeType)
}
underlyingStore, err := factory(storeConfig)
if err != nil {
return nil, errors.Err(err)
}
return WithSingleFlight(cfg.Component, underlyingStore), nil
}
func init() {
RegisterStore("singleflight", SingleFlightStoreFactory)
sf *singleflight.Group
}
func (s *singleflightStore) Name() string {
@@ -157,4 +124,5 @@ func (s *singleflightStore) putter(hash string, blob stream.Blob) func() (interf
// Shutdown shuts down the store gracefully
func (s *singleflightStore) Shutdown() {
s.BlobStore.Shutdown()
return
}
+3 -17
View File
@@ -1,8 +1,7 @@
package speedwalk
import (
"io/fs"
"os"
"io/ioutil"
"path/filepath"
"runtime"
"sync"
@@ -18,20 +17,7 @@ import (
// AllFiles recursively lists every file in every subdirectory of a given directory
// If basename is true, return the basename of each file. Otherwise return the full path starting at startDir.
func AllFiles(startDir string, basename bool) ([]string, error) {
entries, err := os.ReadDir(startDir)
if err != nil {
return nil, err
}
items := make([]fs.FileInfo, 0, len(entries))
for _, entry := range entries {
var info fs.FileInfo
info, err = entry.Info()
if err != nil {
return nil, err
}
items = append(items, info)
}
items, err := ioutil.ReadDir(startDir)
if err != nil {
return nil, err
}
@@ -91,7 +77,7 @@ func AllFiles(startDir string, basename bool) ([]string, error) {
},
})
if err != nil {
logrus.Error(errors.FullTrace(err))
logrus.Errorf(errors.FullTrace(err))
}
}(item.Name())
}
+4 -19
View File
@@ -1,25 +1,22 @@
package store
import (
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
"github.com/lbryio/reflector.go/shared"
)
// BlobStore is an interface for handling blob storage.
type BlobStore interface {
// Name of blob store (useful for metrics)
Name() string
// Has Does blob exist in the store.
// Does blob exist in the store.
Has(hash string) (bool, error)
// Get the blob from the store. Must return ErrBlobNotFound if blob is not in store.
Get(hash string) (stream.Blob, shared.BlobTrace, error)
// Put the blob into the store.
Put(hash string, blob stream.Blob) error
// PutSD an SD blob into the store.
// Put an SD blob into the store.
PutSD(hash string, blob stream.Blob) error
// Delete the blob from the store.
Delete(hash string) error
@@ -35,23 +32,11 @@ type Blocklister interface {
Wants(hash string) (bool, error)
}
// NeededBlobChecker can check which blobs from a known stream are not uploaded yet
type NeededBlobChecker interface {
MissingBlobsForKnownStream(string) ([]string, error)
}
// lister is a store that can list cached blobs. This is helpful when an overlay
// cache needs to track blob existence.
type lister interface {
list() ([]string, error)
}
// ErrBlobNotFound is a standard error when a blob is not found in the store.
//ErrBlobNotFound is a standard error when a blob is not found in the store.
var ErrBlobNotFound = errors.Base("blob not found")
var Factories = make(map[string]Factory)
func RegisterStore(name string, factory Factory) {
Factories[name] = factory
}
type Factory func(config *viper.Viper) (BlobStore, error)
-204
View File
@@ -1,204 +0,0 @@
package store
import (
"bytes"
"context"
"io"
"net"
"net/http"
"sync"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
)
// UpstreamStore is a store that works on top of the HTTP protocol
type UpstreamStore struct {
upstream string
httpClient *http.Client
edgeToken string
name string
}
type UpstreamParams struct {
Name string `mapstructure:"name"`
Upstream string `mapstructure:"upstream"`
EdgeToken string `mapstructure:"edge_token"`
}
func NewUpstreamStore(params UpstreamParams) *UpstreamStore {
return &UpstreamStore{
upstream: params.Upstream,
httpClient: getClient(),
edgeToken: params.EdgeToken,
name: params.Name,
}
}
const nameUpstream = "upstream"
func UpstreamStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg UpstreamParams
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
return NewUpstreamStore(cfg), nil
}
func init() {
RegisterStore(nameUpstream, UpstreamStoreFactory)
}
func (n *UpstreamStore) Name() string { return nameUpstream + "-" + n.name }
func (n *UpstreamStore) Has(hash string) (bool, error) {
url := n.upstream + "/blob?hash=" + hash
req, err := http.NewRequest("HEAD", url, nil)
if err != nil {
return false, errors.Err(err)
}
res, err := n.httpClient.Do(req)
if err != nil {
return false, errors.Err(err)
}
defer func() { _ = res.Body.Close() }()
if res.StatusCode == http.StatusNotFound {
return false, nil
}
if res.StatusCode == http.StatusNoContent {
return true, nil
}
var body []byte
if res.Body != nil {
body, _ = io.ReadAll(res.Body)
}
return false, errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
}
func (n *UpstreamStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
url := n.upstream + "/blob?hash=" + hash
if n.edgeToken != "" {
url += "&edge_token=" + n.edgeToken
}
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), n.Name()), errors.Err(err)
}
res, err := n.httpClient.Do(req)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), n.Name()), errors.Err(err)
}
defer func(Body io.ReadCloser) {
err := Body.Close()
if err != nil {
log.Errorf("Error closing response body in HTTP-GET: %s", err.Error())
}
}(res.Body)
viaHeader := res.Header.Get("Via")
var trace shared.BlobTrace
if viaHeader != "" {
parsedTrace, err := shared.Deserialize(viaHeader)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), n.Name()), err
}
trace = *parsedTrace
} else {
trace = shared.NewBlobTrace(0, n.Name())
}
switch res.StatusCode {
case http.StatusNotFound:
return nil, trace.Stack(time.Since(start), n.Name()), ErrBlobNotFound
case http.StatusOK:
buffer := getBuffer()
defer putBuffer(buffer)
if _, err := io.Copy(buffer, res.Body); err != nil {
return nil, trace.Stack(time.Since(start), n.Name()), errors.Err(err)
}
blob := make([]byte, buffer.Len())
copy(blob, buffer.Bytes())
metrics.MtrInBytesUpstream.Add(float64(len(blob)))
return blob, trace.Stack(time.Since(start), n.Name()), nil
default:
body, _ := io.ReadAll(res.Body)
log.Warnf("Got status code %d (%s)", res.StatusCode, string(body))
return nil, trace.Stack(time.Since(start), n.Name()),
errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
}
}
func (n *UpstreamStore) Put(string, stream.Blob) error {
return shared.ErrNotImplemented
}
func (n *UpstreamStore) PutSD(string, stream.Blob) error {
return shared.ErrNotImplemented
}
func (n *UpstreamStore) Delete(string) error {
return shared.ErrNotImplemented
}
func (n *UpstreamStore) Shutdown() {}
// buffer pool to reduce GC
// https://www.captaincodeman.com/2017/06/02/golang-buffer-pool-gotcha
var buffers = sync.Pool{
// New is called when a new instance is needed
New: func() interface{} {
buf := make([]byte, 0, stream.MaxBlobSize)
return bytes.NewBuffer(buf)
},
}
// getBuffer fetches a buffer from the pool
func getBuffer() *bytes.Buffer {
return buffers.Get().(*bytes.Buffer)
}
// putBuffer returns a buffer to the pool
func putBuffer(buf *bytes.Buffer) {
buf.Reset()
buffers.Put(buf)
}
func dialContext(ctx context.Context, network, address string) (net.Conn, error) {
dialer := &net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}
return dialer.DialContext(ctx, network, address)
}
// getClient gets an http client that's customized to be more performant when dealing with blobs of 2MB in size (most of our blobs)
func getClient() *http.Client {
// Customize the Transport to have larger connection pool
defaultTransport := &http.Transport{
DialContext: dialContext,
ForceAttemptHTTP2: true,
MaxIdleConns: 100,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
DisableCompression: true,
MaxIdleConnsPerHost: 100,
ReadBufferSize: stream.MaxBlobSize + 1024*10, //add an extra few KBs to make sure it fits the extra information
}
return &http.Client{Transport: defaultTransport}
}
+4 -4
View File
@@ -69,22 +69,22 @@ func (n *Node) Resolve(url string) (*types.Output, error) {
type GetClaimsInTxResp struct {
Jsonrpc string `json:"jsonrpc"`
ID int `json:"id"`
Result []struct {
Name string `json:"name"`
ClaimID string `json:"claim_id"`
Txid string `json:"txid"`
Value string `json:"value"`
Address string `json:"address"`
Supports []interface{} `json:"supports"` // TODO: finish me
Nout int `json:"nout"`
Amount int `json:"amount"`
Depth int `json:"depth"`
Height int `json:"height"`
Value string `json:"value"`
ClaimSequence int `json:"claim_sequence"`
Address string `json:"address"`
Supports []interface{} `json:"supports"` // TODO: finish me
EffectiveAmount int `json:"effective_amount"`
ValidAtHeight int `json:"valid_at_height"`
} `json:"result"`
ID int `json:"id"`
}
func (n *Node) GetClaimsInTx(txid string) (*GetClaimsInTxResp, error) {
+16 -13
View File
@@ -30,19 +30,22 @@ var (
)
type response struct {
err error
data []byte
err error
}
type Node struct {
transport *TCPTransport
grp *stop.Group
handlersMu *sync.RWMutex
handlers map[uint32]chan response
transport *TCPTransport
nextId atomic.Uint32
grp *stop.Group
handlersMu *sync.RWMutex
handlers map[uint32]chan response
pushHandlersMu *sync.RWMutex
pushHandlers map[string][]chan response
nextId atomic.Uint32
timeout time.Duration
timeout time.Duration
}
// NewNode creates a new node.
@@ -152,23 +155,23 @@ func (n *Node) listen() {
return
case bytes := <-n.transport.Responses():
msg := &struct {
Id uint32 `json:"id"`
Method string `json:"method"`
Error struct {
Message string `json:"message"`
Code int `json:"code"`
Message string `json:"message"`
} `json:"error"`
Id uint32 `json:"id"`
}{}
msg2 := &struct {
Id uint32 `json:"id"`
Method string `json:"method"`
Error struct {
Code int `json:"code"`
Message struct {
Message string `json:"message"`
Code int `json:"code"`
Message string `json:"message"`
} `json:"message"`
Code int `json:"code"`
} `json:"error"`
Id uint32 `json:"id"`
}{}
r := response{}
@@ -228,9 +231,9 @@ func (n *Node) listen() {
// request makes a request to the server and unmarshals the response into v.
func (n *Node) request(method string, params []string, v interface{}) error {
msg := struct {
Id uint32 `json:"id"`
Method string `json:"method"`
Params []string `json:"params"`
Id uint32 `json:"id"`
}{
Id: n.nextId.Load(),
Method: method,
+1 -1
View File
@@ -124,7 +124,7 @@ func (t *TCPTransport) test() error {
return errors.Err(err)
}
if response.Error.Message != "" {
return fmt.Errorf("%s", response.Error.Message)
return fmt.Errorf(response.Error.Message)
}
return nil
}