Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0359d723c5 | ||
|
|
0cfc336be5 | ||
|
|
05acdcb9e0 | ||
|
|
9e2cccc51f | ||
|
|
1b79371f8b | ||
|
|
cbcc611196 | ||
|
|
68c0fe3b97 | ||
|
|
8bfe7e101d | ||
|
|
7b3cca45b4 | ||
|
|
ad710dfeea | ||
|
|
ebc52245f3 | ||
|
|
b098aa2dc8 | ||
|
|
ddd23f9a36 | ||
|
|
4928e430c0 | ||
|
|
d21b0867dd | ||
|
|
32a85a815e | ||
|
|
9cdac902bd | ||
|
|
8ec01bf1a7 | ||
|
|
ba8d1ba67b | ||
|
|
edecf4f988 | ||
|
|
77836f9c83 | ||
|
|
9716e41d63 | ||
|
|
7260388255 | ||
|
|
071a7907c1 | ||
|
|
185900582d | ||
|
|
03445476ec | ||
|
|
e5a9590cb6 | ||
|
|
de736b068d | ||
|
|
14fbd2ae27 | ||
|
|
4d81a43a8f | ||
|
|
b3f0d63b4d | ||
|
|
c880f0b80f | ||
|
|
085490e92b | ||
|
|
456fe53e01 | ||
|
|
778fc17adf | ||
|
|
e93c097fd9 | ||
|
|
0dfda70c70 | ||
|
|
6c082993cf | ||
|
|
08ed3c9f13 | ||
|
|
7f75602841 | ||
|
|
4d168ddefc | ||
|
|
b7abb77ea1 | ||
|
|
8c2a46752c | ||
|
|
0c8da4abe5 | ||
|
|
87680d806c | ||
|
|
e1c59b9b63 | ||
|
|
c79634706b | ||
|
|
f5d30b1a6e | ||
|
|
0177dd4ce0 | ||
|
|
5693529216 | ||
|
|
a1c2e92ca3 | ||
|
|
9c0554ef05 | ||
|
|
4e80f91a57 | ||
|
|
c211f83ba7 | ||
|
|
29d1ccf68c | ||
|
|
2f7d67794f | ||
|
|
4d8e7739d7 | ||
|
|
6fc0ceea2a | ||
|
|
ae0c7dd2bb | ||
|
|
4af5c2f4c6 | ||
|
|
def0a97f49 | ||
|
|
6dde793745 | ||
|
|
654cc44935 | ||
|
|
90d6d29452 | ||
|
|
b2272fef3a | ||
|
|
86f3e62aa8 |
No files matched your search
@@ -0,0 +1,37 @@
|
||||
name: Go
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: 1.25.x
|
||||
|
||||
- name: Build linux
|
||||
run: make linux
|
||||
|
||||
- name: Build macos
|
||||
run: make macos
|
||||
|
||||
- name: Test
|
||||
run: make test
|
||||
|
||||
- name: Lint
|
||||
run: make lint
|
||||
|
||||
- name: retrieve all tags
|
||||
run: git fetch --prune --unshallow --tags
|
||||
|
||||
- name: Print changes since last version
|
||||
run: git log $(git describe --tags --abbrev=0)..HEAD --no-merges --oneline
|
||||
@@ -0,0 +1,62 @@
|
||||
name: release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "*.*.*"
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: 1.25.x
|
||||
|
||||
- name: Build linux
|
||||
run: make linux
|
||||
|
||||
- name: Build macos
|
||||
run: make macos
|
||||
|
||||
- name: Test
|
||||
run: make test
|
||||
|
||||
- name: Lint
|
||||
run: make lint
|
||||
|
||||
- name: Zip macos
|
||||
run: zip -r reflector_darwin_amd64.zip ./dist/darwin_amd64
|
||||
|
||||
- name: Zip linux
|
||||
run: zip -r reflector_linux_amd64.zip ./dist/linux_amd64
|
||||
|
||||
- name: retrieve all tags
|
||||
run: git fetch --prune --unshallow --tags
|
||||
|
||||
- name: Generate Changelog
|
||||
run: git log $(git describe --tags --abbrev=0 @^)..@ --no-merges --oneline > ${{ github.workspace }}-CHANGELOG.txt
|
||||
|
||||
- name: upload to github releases
|
||||
uses: softprops/action-gh-release@v1
|
||||
with:
|
||||
files: |
|
||||
./reflector_linux_amd64.zip
|
||||
./reflector_darwin_amd64.zip
|
||||
body_path: ${{ github.workspace }}-CHANGELOG.txt
|
||||
|
||||
# - name: Login to DockerHub
|
||||
# uses: docker/login-action@v2
|
||||
# with:
|
||||
# username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
# password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
# - name: Generate docker image
|
||||
# run: make image
|
||||
|
||||
# - name: Docker push
|
||||
# run: make publish_image
|
||||
@@ -1,3 +1,12 @@
|
||||
/vendor
|
||||
/config.json*
|
||||
/dist
|
||||
/bin
|
||||
|
||||
blobcache.yaml
|
||||
|
||||
reflector.yaml
|
||||
|
||||
upload.yaml
|
||||
|
||||
remotereflector.yaml
|
||||
@@ -0,0 +1,57 @@
|
||||
run:
|
||||
timeout: 5m
|
||||
tests: false
|
||||
modules-download-mode: readonly
|
||||
|
||||
linters-settings:
|
||||
gci:
|
||||
# Checks that no inline Comments are present.
|
||||
# Default: false
|
||||
no-inline-comments: false
|
||||
|
||||
# Checks that no prefix Comments(comment lines above an import) are present.
|
||||
# Default: false
|
||||
no-prefix-comments: false
|
||||
|
||||
# Skip generated files
|
||||
skip-generated: true
|
||||
|
||||
# Use custom order
|
||||
custom-order: true
|
||||
|
||||
sections:
|
||||
- standard
|
||||
- prefix(github.com/lbryio/reflector.go)
|
||||
- prefix(github.com/OdyseeTeam,github.com/lbryio)
|
||||
- default
|
||||
gofmt:
|
||||
simplify: true
|
||||
govet:
|
||||
enable-all: true
|
||||
misspell:
|
||||
locale: US
|
||||
revive:
|
||||
rules:
|
||||
- name: exported
|
||||
disabled: true
|
||||
staticcheck:
|
||||
checks: ["all"]
|
||||
|
||||
linters:
|
||||
enable:
|
||||
- errcheck
|
||||
- gosimple
|
||||
- govet
|
||||
- ineffassign
|
||||
- gci
|
||||
- gofmt
|
||||
- goimports
|
||||
- misspell
|
||||
- revive
|
||||
- unconvert
|
||||
- whitespace
|
||||
- typecheck
|
||||
disable:
|
||||
- unused
|
||||
- unparam
|
||||
- staticcheck
|
||||
-83
@@ -1,83 +0,0 @@
|
||||
os: linux
|
||||
dist: trusty
|
||||
language: go
|
||||
|
||||
env:
|
||||
- GO111MODULE=on
|
||||
|
||||
go:
|
||||
- 1.15.x
|
||||
|
||||
cache:
|
||||
directories:
|
||||
- $HOME/.cache/go-build
|
||||
- $HOME/gopath/pkg/mod
|
||||
|
||||
notifications:
|
||||
email: false
|
||||
|
||||
# Skip the install step. Don't `go get` dependencies. Only build with the code in vendor/
|
||||
install: true
|
||||
|
||||
# Anything in before_script that returns a nonzero exit code will
|
||||
# flunk the build and immediately stop. It's sorta like having
|
||||
# set -e enabled in bash.
|
||||
before_script:
|
||||
# All the .go files, excluding vendor/ and model (auto generated)
|
||||
- GO_FILES=$(find . -iname '*.go' ! -iname '*_test.go' -type f | grep -v /vendor/ ) #i wish we were this crazy :p
|
||||
- go get golang.org/x/tools/cmd/goimports # Used in build script for generated files
|
||||
# - go get github.com/golang/lint/golint # Linter
|
||||
# - go get honnef.co/go/tools/cmd/megacheck # Badass static analyzer/linter
|
||||
- go get github.com/jgautheron/gocyclo # Check against high complexity
|
||||
- go get github.com/mdempsky/unconvert # Identifies unnecessary type conversions
|
||||
- go get github.com/kisielk/errcheck # Checks for unhandled errors
|
||||
- go get github.com/opennota/check/cmd/varcheck # Checks for unused vars
|
||||
- go get github.com/opennota/check/cmd/structcheck # Checks for unused fields in structs
|
||||
|
||||
|
||||
|
||||
# script always run to completion (set +e). All of these code checks are must haves
|
||||
# in a modern Go project.
|
||||
script:
|
||||
# Fail if a .go file hasn't been formatted with gofmt
|
||||
- test -z $(gofmt -s -l $GO_FILES)
|
||||
# Run unit tests
|
||||
- make test
|
||||
# Checks for unused vars and fields on structs
|
||||
- varcheck ./...
|
||||
- structcheck ./...
|
||||
# go vet is the official Go static analyzer
|
||||
- go vet ./...
|
||||
# forbid code with huge functions
|
||||
#- gocyclo -ignore "_test.go" -avg -over 19 $GO_FILES
|
||||
# checks for unhandled errors
|
||||
# - errcheck ./... # COMMENTED OUT UNTIL https://github.com/kisielk/errcheck/issues/155 is fixed
|
||||
# "go vet on steroids" + linter - ignore autogen code
|
||||
# - megacheck -simple.exit-non-zero=true ./... # DISABLED UNTIL https://github.com/dominikh/go-tools/issues/328 is fixed
|
||||
# check for unnecessary conversions - ignore autogen code
|
||||
# - unconvert ./... # SEEMS TO BE BROKEN WITH GO MODULES
|
||||
# one last linter - ignore autogen code
|
||||
#- golint -set_exit_status $(go list ./... | grep -v /vendor/ )
|
||||
# Finally, build the binary
|
||||
- make
|
||||
|
||||
deploy:
|
||||
- provider: s3
|
||||
local_dir: ./bin
|
||||
skip_cleanup: true
|
||||
on:
|
||||
repo: lbryio/reflector.go
|
||||
access_key_id:
|
||||
secure: "ABmoSTxTee1GubJmmi+MlyBcnqRT0ywEOtTL4hrH7T+Vj5UPcNBhjqGRiMXzO3MdrVWom19OnqMvsvAdehrB4uQJVlX/+zB26gTpEiiU5dWJgerNAYmfxDGNz/p9pyYWLvVY/cymSlJ4HkggJVg+dMTBTOP+gTNgQUhHL7pAd/Z5HMPmPW/rlNlQgDdczoY44OdiniuHny4Lr7mo8dcT4dcyJysPpHLCPGWp3kXcffIEbgO02aQZ14BbX9oU9xm1BacYnr4t8gRCU+uK0mt7TREHUzZ8bQBRXUymmpeeEQshXdifdwtk6sdRGIZiTi0NmQ7kYter6oh6623TH6rpNy/lkxKi823FbSPC1XZGH7CwcbnSIEwY++7e9PT0EA14VuDUF7+iXC/gpRkVQwUoMcoC9rHBjkXW+SiwHWoUeDKXrfVO9QGEJTi5bDRNwklwu2zkYWeRIPzfDaVayUBYwjunjT1eST997ygmJwJTJ6DeOWLx+WFlmwBJfQpBwbrlMrceiNjreMwGB6ffXEBzuM6S5inXGIChllGDpNyJeFVcH9zPpU0JsIpWoRjwLNFOiEhFsnzK5rvA2SKmlOOnSlgOCBQnFi2SmLDM790KoY8qjan2s984chQz2qsKnhYuVpJhG6LKgrI1QwE0KHr80spVFfewq11AU4Q+lfA9aAQ="
|
||||
secret_access_key:
|
||||
secure: "AxYRTy/GnjeTJKQdeJ/AEeAd+yXs783bFDKdyKNswtsHlU8sWPQgNcvTLpVqnSQMpiwkGDGi/70rvR5C+AT3SIWNw13RYrgBRpduQU0J+B2JS+3dN2DIePu25uvs++Wo22OfS8I+UjZ1mWY1SSHI2spPXvDCq5tb+Ih8nlYflEyAtxU9Oq2R3Kus2tkIlRnL25sP/2fY7RvuJFYIV63z8ZIJRzB5WzOeERqnXq2zfwos+hycAqyo/VevJnWAYTEDsvBuSODOpZF+QfKtIQ2rYSoqy8Lq1M6UOZimnC3Ulea4euBVf2ssBCnI7csGNG5UzkTiwrPDi2xIP8nM01rHW1yHJ7tQsJaghnUsfw2t6ui4ZofvbbOFTN/YCloHITifEi8Tc1/17isi3y+kX5yQ/Nk5UNry0Wbt91CP+nkL/ZmA5grkBXDL7VJMmB60TnO3ap24CtwBQartN3LoWs7h+4ov+LqbCt6IqpJVWQWlwJeb2MFPFByALtBpsqAyL1SxXlGNpPa94CuXxfQ6Bv436PtefA5FlTzR8uMmqsjWciv06bVnSvVlFEVovN2Fkplrjt7AASJ/8KJs4THDg4k61nfd8roAHx6ewQzl4wCWKCikQ0MuFd2mVHwdrbnCH1mIHuPRyvWMMIAK0ooc1/rmKiJlpgumjxoFYNE10MXtt+I="
|
||||
bucket: "lbry-reflector-binary"
|
||||
|
||||
# - provider: releases
|
||||
# file: ./bin/prism-bin
|
||||
# skip_cleanup: true
|
||||
# on:
|
||||
# repo: lbryio/reflector.go
|
||||
# tags: true
|
||||
# api_key:
|
||||
# secure: epAlhp3SUr8hhISarJ22n6tRw2TEa4s4oNFIvJUb5HGECVp1SYN7ao0ln5NoNLmfJS60pi911i/kMhhi21/uhZ0kCYlEhhIE2pc1zsiAxK9L9ENCssJ205HfVbe5grhwskLGzgjhU9OznO8WtmyOPWXr0it8M8RCTjx6rEC0A33Id3WMYyhP938Sj9CxEYeH4KS8wFvBXkgBVtrgaYwRTCIROFddHFXOb9jyNhqQ1RbfKtllsVtQhVk5WMlomheBNSS4vr6WMS4X4+2okFqnLtiSn1wrn5I/94UQbnrI1juVnQj0K+j32EyQbAOt4T2cLW3GtG0jhaYKyNMT9ycDCdVACPSDELlHWjeyoes9bnhUFftm6kDbQxwA1UsTF1yG8tMKXxBSmYyoT7qDloi6pBifZMrFXL61uTs6yhVB9LS/2oqg4sc0Ne87bRcn4OxsBeVCe3kbBHDTR/NTyF2gNPtRvgMAWULxTVcUm9VYdO0IWvAig5g4Row0DnFzEquD6CzezbRWD9WyZyV/AFyYHeeQ2PO7jTw0/3M7aDX33Fuhh34lehzmrC03cfgD/wZW+spxozIcQCYdiJqVw+u+/NvbNr0kkFzE9zW26JEmUFTyDvKxvnza1Kwtww3EgH6zaOL8r4yVbb54rePRvLw7pl93zlfJnEB2MCPqJOY5ZpU=
|
||||
+1
-1
@@ -3,7 +3,7 @@ EXPOSE 8080
|
||||
|
||||
RUN mkdir /app
|
||||
WORKDIR /app
|
||||
COPY bin/prism-bin ./prism
|
||||
COPY dist/linux_amd64/prism-bin ./prism
|
||||
RUN chmod +x prism
|
||||
|
||||
ENTRYPOINT [ "/app/prism" ]
|
||||
@@ -1,25 +1,37 @@
|
||||
version := $(shell git describe --dirty --always --long --abbrev=7)
|
||||
commit := $(shell git rev-parse --short HEAD)
|
||||
commit_long := $(shell git rev-parse HEAD)
|
||||
branch := $(shell git rev-parse --abbrev-ref HEAD)
|
||||
curTime := $(shell date +%s)
|
||||
|
||||
BINARY=prism-bin
|
||||
|
||||
DIR = $(shell cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)
|
||||
BIN_DIR = ${DIR}/bin
|
||||
IMPORT_PATH = github.com/lbryio/reflector.go
|
||||
LDFLAGS="-X ${IMPORT_PATH}/meta.version=$(version) -X ${IMPORT_PATH}/meta.commit=$(commit) -X ${IMPORT_PATH}/meta.commitLong=$(commit_long) -X ${IMPORT_PATH}/meta.branch=$(branch) -X '${IMPORT_PATH}/meta.Time=$(curTime)'"
|
||||
DIR = $(shell cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)
|
||||
BIN_DIR = $(DIR)/dist
|
||||
|
||||
VERSION = $(shell git --git-dir=${DIR}/.git describe --dirty --always --long --abbrev=7)
|
||||
LDFLAGS = -ldflags "-X ${IMPORT_PATH}/meta.Version=${VERSION} -X ${IMPORT_PATH}/meta.Time=$(shell date +%s)"
|
||||
|
||||
|
||||
.PHONY: build clean test lint
|
||||
.DEFAULT_GOAL: build
|
||||
|
||||
|
||||
build:
|
||||
mkdir -p ${BIN_DIR} && CGO_ENABLED=0 go build ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/${BINARY} main.go
|
||||
|
||||
clean:
|
||||
if [ -f ${BIN_DIR}/${BINARY} ]; then rm ${BIN_DIR}/${BINARY}; fi
|
||||
.DEFAULT_GOAL := linux
|
||||
|
||||
.PHONY: test
|
||||
test:
|
||||
go test ./... -v -cover
|
||||
go test -cover -v ./...
|
||||
|
||||
.PHONY: lint
|
||||
lint:
|
||||
go get github.com/alecthomas/gometalinter && gometalinter --install && gometalinter ./...
|
||||
./scripts/lint.sh
|
||||
|
||||
.PHONY: imports
|
||||
imports:
|
||||
gci write --skip-generated --custom-order -s standard -s "prefix(github.com/lbryio/reflector.go)" -s "prefix(github.com/OdyseeTeam,github.com/lbryio)" -s default .
|
||||
|
||||
.PHONY: linux
|
||||
linux:
|
||||
GOARCH=amd64 CGO_ENABLED=0 GOOS=linux go build -ldflags ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/linux_amd64/${BINARY}
|
||||
|
||||
.PHONY: macos
|
||||
macos:
|
||||
GOARCH=amd64 GOOS=darwin go build -ldflags ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/darwin_amd64/${BINARY}
|
||||
|
||||
.PHONY: image
|
||||
image:
|
||||
docker buildx build -t lbry/reflector:$(version) -t lbry/reflector:latest --platform linux/amd64 .
|
||||
+6
-7
@@ -1,7 +1,7 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"io"
|
||||
baselog "log"
|
||||
"sort"
|
||||
"time"
|
||||
@@ -23,13 +23,12 @@ const (
|
||||
type Cluster struct {
|
||||
OnMembershipChange func(n, total int)
|
||||
|
||||
s *serf.Serf
|
||||
eventCh chan serf.Event
|
||||
stop *stop.Group
|
||||
name string
|
||||
port int
|
||||
seedAddr string
|
||||
|
||||
s *serf.Serf
|
||||
eventCh chan serf.Event
|
||||
stop *stop.Group
|
||||
port int
|
||||
}
|
||||
|
||||
// New returns a new Cluster instance that is not connected.
|
||||
@@ -52,7 +51,7 @@ func (c *Cluster) Connect() error {
|
||||
conf.MemberlistConfig.AdvertisePort = c.port
|
||||
conf.NodeName = c.name
|
||||
|
||||
nullLogger := baselog.New(ioutil.Discard, "", 0)
|
||||
nullLogger := baselog.New(io.Discard, "", 0)
|
||||
conf.Logger = nullLogger
|
||||
|
||||
c.eventCh = make(chan serf.Event)
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"syscall"
|
||||
|
||||
"github.com/lbryio/reflector.go/config"
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func init() {
|
||||
var cmd = &cobra.Command{
|
||||
Use: "blobcache",
|
||||
Short: "Run blobcache server",
|
||||
Run: blobcacheCmd,
|
||||
}
|
||||
|
||||
cmd.Flags().IntVar(&metricsPort, "metrics-port", 2112, "The port reflector will use for prometheus metrics")
|
||||
cmd.Flags().BoolVar(&disableBlocklist, "disable-blocklist", false, "Disable blocklist watching/updating")
|
||||
|
||||
rootCmd.AddCommand(cmd)
|
||||
}
|
||||
|
||||
func blobcacheCmd(cmd *cobra.Command, args []string) {
|
||||
store, err := config.LoadStores(conf, "blobcache")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer store.Shutdown()
|
||||
|
||||
servers, err := config.LoadServers(store, conf, "blobcache")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
for _, s := range servers {
|
||||
err = s.Start()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer s.Shutdown()
|
||||
}
|
||||
|
||||
metricsServer := metrics.NewServer(":"+strconv.Itoa(metricsPort), "/metrics")
|
||||
metricsServer.Start()
|
||||
defer metricsServer.Shutdown()
|
||||
|
||||
interruptChan := make(chan os.Signal, 1)
|
||||
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
|
||||
<-interruptChan
|
||||
}
|
||||
+2
-2
@@ -6,10 +6,10 @@ import (
|
||||
"strconv"
|
||||
"syscall"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/crypto"
|
||||
|
||||
"github.com/lbryio/reflector.go/cluster"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/crypto"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
+13
-6
@@ -5,7 +5,6 @@ import (
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/server/peer"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
@@ -28,11 +27,19 @@ func getStreamCmd(cmd *cobra.Command, args []string) {
|
||||
addr := args[0]
|
||||
sdHash := args[1]
|
||||
|
||||
s := store.NewCachingStore(
|
||||
"getstream",
|
||||
peer.NewStore(peer.StoreOpts{Address: addr}),
|
||||
store.NewDiskStore("/tmp/lbry_downloaded_blobs", 2),
|
||||
)
|
||||
s := store.NewCachingStore(store.CachingParams{
|
||||
Name: "getstream",
|
||||
Cache: store.NewPeerStore(store.PeerParams{
|
||||
Name: "getstream",
|
||||
Address: addr,
|
||||
Timeout: 30 * time.Second,
|
||||
}),
|
||||
Origin: store.NewDiskStore(store.DiskParams{
|
||||
Name: "getstream",
|
||||
MountPoint: "/tmp/lbry_downloaded_blobs",
|
||||
ShardingSize: 2,
|
||||
}),
|
||||
})
|
||||
|
||||
wd, err := os.Getwd()
|
||||
if err != nil {
|
||||
|
||||
+1
-2
@@ -3,7 +3,6 @@ package cmd
|
||||
import (
|
||||
"crypto/sha512"
|
||||
"encoding/hex"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path"
|
||||
"runtime"
|
||||
@@ -72,7 +71,7 @@ func consume(worker int, tasks <-chan string, done chan<- bool, totalTasks int,
|
||||
log.Infof("[T%d] %d/%d blobs processed so far. ETA: %s", worker, processedSoFar, totalTasks, remainingTime.String())
|
||||
}
|
||||
blobPath := path.Join(diskStorePath, b[:2], b)
|
||||
blob, err := ioutil.ReadFile(blobPath)
|
||||
blob, err := os.ReadFile(blobPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
continue
|
||||
|
||||
-55
@@ -1,55 +0,0 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"syscall"
|
||||
|
||||
"github.com/lbryio/reflector.go/db"
|
||||
"github.com/lbryio/reflector.go/server/peer"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var peerNoDB bool
|
||||
|
||||
func init() {
|
||||
var cmd = &cobra.Command{
|
||||
Use: "peer",
|
||||
Short: "Run peer server",
|
||||
Run: peerCmd,
|
||||
}
|
||||
cmd.Flags().BoolVar(&peerNoDB, "nodb", false, "Don't connect to a db and don't use a db-backed blob store")
|
||||
rootCmd.AddCommand(cmd)
|
||||
}
|
||||
|
||||
func peerCmd(cmd *cobra.Command, args []string) {
|
||||
var err error
|
||||
|
||||
s3 := store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
|
||||
peerServer := peer.NewServer(s3)
|
||||
|
||||
if !peerNoDB {
|
||||
db := &db.SQL{
|
||||
LogQueries: log.GetLevel() == log.DebugLevel,
|
||||
}
|
||||
err = db.Connect(globalConfig.DBConn)
|
||||
checkErr(err)
|
||||
|
||||
combo := store.NewDBBackedStore(s3, db, false)
|
||||
peerServer = peer.NewServer(combo)
|
||||
}
|
||||
|
||||
err = peerServer.Start(":" + strconv.Itoa(peer.DefaultPort))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
interruptChan := make(chan os.Signal, 1)
|
||||
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
|
||||
<-interruptChan
|
||||
peerServer.Shutdown()
|
||||
}
|
||||
+3
-3
@@ -32,9 +32,9 @@ func populateDbCmd(cmd *cobra.Command, args []string) {
|
||||
log.Fatal("store-path must be defined")
|
||||
}
|
||||
localDb := &db.SQL{
|
||||
SoftDelete: true,
|
||||
TrackAccess: db.TrackAccessBlobs,
|
||||
LogQueries: log.GetLevel() == log.DebugLevel,
|
||||
SoftDelete: true,
|
||||
TrackingLevel: db.TrackAccessBlobs,
|
||||
LogQueries: log.GetLevel() == log.DebugLevel,
|
||||
}
|
||||
err := localDb.Connect("reflector:reflector@tcp(localhost:3306)/reflector")
|
||||
if err != nil {
|
||||
|
||||
+21
-324
@@ -4,66 +4,25 @@ import (
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
"github.com/lbryio/reflector.go/db"
|
||||
"github.com/lbryio/reflector.go/config"
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
"github.com/lbryio/reflector.go/meta"
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
"github.com/lbryio/reflector.go/server/http"
|
||||
http32 "github.com/lbryio/reflector.go/server/http3"
|
||||
"github.com/lbryio/reflector.go/server/peer"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
"github.com/c2h5oh/datasize"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var (
|
||||
//port configuration
|
||||
tcpPeerPort int
|
||||
http3PeerPort int
|
||||
httpPeerPort int
|
||||
receiverPort int
|
||||
metricsPort int
|
||||
receiverPort int
|
||||
metricsPort int
|
||||
|
||||
//flags configuration
|
||||
disableUploads bool
|
||||
disableBlocklist bool
|
||||
useDB bool
|
||||
|
||||
//upstream configuration
|
||||
upstreamReflector string
|
||||
upstreamProtocol string
|
||||
|
||||
//downstream configuration
|
||||
requestQueueSize int
|
||||
|
||||
//upstream edge configuration (to "cold" storage)
|
||||
originEndpoint string
|
||||
originEndpointFallback string
|
||||
|
||||
//cache configuration
|
||||
diskCache string
|
||||
secondaryDiskCache string
|
||||
memCache int
|
||||
)
|
||||
var cacheManagers = []string{"localdb", "lfu", "arc", "lru", "simple"}
|
||||
|
||||
var cacheMangerToGcache = map[string]store.EvictionStrategy{
|
||||
"lfu": store.LFU,
|
||||
"arc": store.ARC,
|
||||
"lru": store.LRU,
|
||||
"simple": store.SIMPLE,
|
||||
}
|
||||
|
||||
func init() {
|
||||
var cmd = &cobra.Command{
|
||||
@@ -72,308 +31,46 @@ func init() {
|
||||
Run: reflectorCmd,
|
||||
}
|
||||
|
||||
cmd.Flags().IntVar(&tcpPeerPort, "tcp-peer-port", 5567, "The port reflector will distribute content from for the TCP (LBRY) protocol")
|
||||
cmd.Flags().IntVar(&http3PeerPort, "http3-peer-port", 5568, "The port reflector will distribute content from over HTTP3 protocol")
|
||||
cmd.Flags().IntVar(&httpPeerPort, "http-peer-port", 5569, "The port reflector will distribute content from over HTTP protocol")
|
||||
cmd.Flags().IntVar(&receiverPort, "receiver-port", 5566, "The port reflector will receive content from")
|
||||
cmd.Flags().IntVar(&metricsPort, "metrics-port", 2112, "The port reflector will use for prometheus metrics")
|
||||
|
||||
cmd.Flags().BoolVar(&disableUploads, "disable-uploads", false, "Disable uploads to this reflector server")
|
||||
cmd.Flags().IntVar(&receiverPort, "receiver-port", 5566, "The port reflector will receive content from")
|
||||
cmd.Flags().BoolVar(&disableBlocklist, "disable-blocklist", false, "Disable blocklist watching/updating")
|
||||
cmd.Flags().BoolVar(&useDB, "use-db", true, "Whether to connect to the reflector db or not")
|
||||
|
||||
cmd.Flags().StringVar(&upstreamReflector, "upstream-reflector", "", "host:port of a reflector server where blobs are fetched from")
|
||||
cmd.Flags().StringVar(&upstreamProtocol, "upstream-protocol", "http", "protocol used to fetch blobs from another upstream reflector server (tcp/http3/http)")
|
||||
|
||||
cmd.Flags().IntVar(&requestQueueSize, "request-queue-size", 200, "How many concurrent requests from downstream should be handled at once (the rest will wait)")
|
||||
|
||||
cmd.Flags().StringVar(&originEndpoint, "origin-endpoint", "", "HTTP edge endpoint for standard HTTP retrieval")
|
||||
cmd.Flags().StringVar(&originEndpointFallback, "origin-endpoint-fallback", "", "HTTP edge endpoint for standard HTTP retrieval if first origin fails")
|
||||
|
||||
cmd.Flags().StringVar(&diskCache, "disk-cache", "100GB:/tmp/downloaded_blobs:localdb", "Where to cache blobs on the file system. format is 'sizeGB:CACHE_PATH:cachemanager' (cachemanagers: localdb/lfu/arc/lru)")
|
||||
cmd.Flags().StringVar(&secondaryDiskCache, "optional-disk-cache", "", "Optional secondary file system cache for blobs. format is 'sizeGB:CACHE_PATH:cachemanager' (cachemanagers: localdb/lfu/arc/lru) (this would get hit before the one specified in disk-cache)")
|
||||
cmd.Flags().IntVar(&memCache, "mem-cache", 0, "enable in-memory cache with a max size of this many blobs")
|
||||
|
||||
rootCmd.AddCommand(cmd)
|
||||
}
|
||||
|
||||
func reflectorCmd(cmd *cobra.Command, args []string) {
|
||||
log.Printf("reflector %s", meta.VersionString())
|
||||
store, err := config.LoadStores(conf, "reflector")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer store.Shutdown()
|
||||
|
||||
// the blocklist logic requires the db backed store to be the outer-most store
|
||||
underlyingStore := initStores()
|
||||
underlyingStoreWithCaches, cleanerStopper := initCaches(underlyingStore)
|
||||
|
||||
if !disableUploads {
|
||||
reflectorServer := reflector.NewServer(underlyingStore, underlyingStoreWithCaches)
|
||||
reflectorServer.Timeout = 3 * time.Minute
|
||||
reflectorServer.EnableBlocklist = !disableBlocklist
|
||||
|
||||
err := reflectorServer.Start(":" + strconv.Itoa(receiverPort))
|
||||
servers, err := config.LoadServers(store, conf, "reflector")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
for _, s := range servers {
|
||||
err = s.Start()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer reflectorServer.Shutdown()
|
||||
defer s.Shutdown()
|
||||
}
|
||||
|
||||
peerServer := peer.NewServer(underlyingStoreWithCaches)
|
||||
err := peerServer.Start(":" + strconv.Itoa(tcpPeerPort))
|
||||
reflectorServer := reflector.NewIngestionServer(store)
|
||||
reflectorServer.Timeout = 3 * time.Minute
|
||||
reflectorServer.EnableBlocklist = !disableBlocklist
|
||||
err = reflectorServer.Start(":" + strconv.Itoa(receiverPort))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer peerServer.Shutdown()
|
||||
|
||||
http3PeerServer := http32.NewServer(underlyingStoreWithCaches, requestQueueSize)
|
||||
err = http3PeerServer.Start(":" + strconv.Itoa(http3PeerPort))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer http3PeerServer.Shutdown()
|
||||
|
||||
httpServer := http.NewServer(underlyingStoreWithCaches, requestQueueSize)
|
||||
err = httpServer.Start(":" + strconv.Itoa(httpPeerPort))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer httpServer.Shutdown()
|
||||
defer reflectorServer.Shutdown()
|
||||
|
||||
metricsServer := metrics.NewServer(":"+strconv.Itoa(metricsPort), "/metrics")
|
||||
metricsServer.Start()
|
||||
defer metricsServer.Shutdown()
|
||||
defer underlyingStoreWithCaches.Shutdown()
|
||||
defer underlyingStore.Shutdown() //do we actually need this? Oo
|
||||
|
||||
interruptChan := make(chan os.Signal, 1)
|
||||
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
|
||||
<-interruptChan
|
||||
// deferred shutdowns happen now
|
||||
cleanerStopper.StopAndWait()
|
||||
}
|
||||
|
||||
func initUpstreamStore() store.BlobStore {
|
||||
var s store.BlobStore
|
||||
if upstreamReflector == "" {
|
||||
return nil
|
||||
}
|
||||
switch upstreamProtocol {
|
||||
case "tcp":
|
||||
s = peer.NewStore(peer.StoreOpts{
|
||||
Address: upstreamReflector,
|
||||
Timeout: 30 * time.Second,
|
||||
})
|
||||
case "http3":
|
||||
s = http32.NewStore(http32.StoreOpts{
|
||||
Address: upstreamReflector,
|
||||
Timeout: 30 * time.Second,
|
||||
})
|
||||
case "http":
|
||||
s = store.NewHttpStore(upstreamReflector)
|
||||
default:
|
||||
log.Fatalf("protocol is not recognized: %s", upstreamProtocol)
|
||||
}
|
||||
return s
|
||||
}
|
||||
func initEdgeStore() store.BlobStore {
|
||||
var s3Store *store.S3Store
|
||||
var s store.BlobStore
|
||||
|
||||
if conf != "none" {
|
||||
s3Store = store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
|
||||
}
|
||||
if originEndpointFallback != "" && originEndpoint != "" {
|
||||
ittt := store.NewITTTStore(store.NewCloudFrontROStore(originEndpoint), store.NewCloudFrontROStore(originEndpointFallback))
|
||||
if s3Store != nil {
|
||||
s = store.NewCloudFrontRWStore(ittt, s3Store)
|
||||
} else {
|
||||
s = ittt
|
||||
}
|
||||
} else if s3Store != nil {
|
||||
s = s3Store
|
||||
} else {
|
||||
log.Fatalf("this configuration does not include a valid upstream source")
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func initDBStore(s store.BlobStore) store.BlobStore {
|
||||
if useDB {
|
||||
dbInst := &db.SQL{
|
||||
TrackAccess: db.TrackAccessStreams,
|
||||
LogQueries: log.GetLevel() == log.DebugLevel,
|
||||
}
|
||||
err := dbInst.Connect(globalConfig.DBConn)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
s = store.NewDBBackedStore(s, dbInst, false)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func initStores() store.BlobStore {
|
||||
s := initUpstreamStore()
|
||||
if s == nil {
|
||||
s = initEdgeStore()
|
||||
}
|
||||
s = initDBStore(s)
|
||||
return s
|
||||
}
|
||||
|
||||
// initCaches returns a store wrapped with caches and a stop group to execute a clean shutdown
|
||||
func initCaches(s store.BlobStore) (store.BlobStore, *stop.Group) {
|
||||
stopper := stop.New()
|
||||
diskStore := initDiskStore(s, diskCache, stopper)
|
||||
finalStore := initDiskStore(diskStore, secondaryDiskCache, stopper)
|
||||
stop.New()
|
||||
if memCache > 0 {
|
||||
finalStore = store.NewCachingStore(
|
||||
"reflector",
|
||||
finalStore,
|
||||
store.NewGcacheStore("mem", store.NewMemStore(), memCache, store.LRU),
|
||||
)
|
||||
}
|
||||
return finalStore, stopper
|
||||
}
|
||||
|
||||
func initDiskStore(upstreamStore store.BlobStore, diskParams string, stopper *stop.Group) store.BlobStore {
|
||||
diskCacheMaxSize, diskCachePath, cacheManager := diskCacheParams(diskParams)
|
||||
//we are tracking blobs in memory with a 1 byte long boolean, which means that for each 2MB (a blob) we need 1Byte
|
||||
// so if the underlying cache holds 10MB, 10MB/2MB=5Bytes which is also the exact count of objects to restore on startup
|
||||
realCacheSize := float64(diskCacheMaxSize) / float64(stream.MaxBlobSize)
|
||||
if diskCacheMaxSize == 0 {
|
||||
return upstreamStore
|
||||
}
|
||||
err := os.MkdirAll(diskCachePath, os.ModePerm)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
diskStore := store.NewDiskStore(diskCachePath, 2)
|
||||
var unwrappedStore store.BlobStore
|
||||
cleanerStopper := stop.New(stopper)
|
||||
|
||||
if cacheManager == "localdb" {
|
||||
localDb := &db.SQL{
|
||||
SoftDelete: true,
|
||||
TrackAccess: db.TrackAccessBlobs,
|
||||
LogQueries: log.GetLevel() == log.DebugLevel,
|
||||
}
|
||||
err = localDb.Connect("reflector:reflector@tcp(localhost:3306)/reflector")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
unwrappedStore = store.NewDBBackedStore(diskStore, localDb, true)
|
||||
go cleanOldestBlobs(int(realCacheSize), localDb, unwrappedStore, cleanerStopper)
|
||||
} else {
|
||||
unwrappedStore = store.NewGcacheStore("nvme", store.NewDiskStore(diskCachePath, 2), int(realCacheSize), cacheMangerToGcache[cacheManager])
|
||||
}
|
||||
|
||||
wrapped := store.NewCachingStore(
|
||||
"reflector",
|
||||
upstreamStore,
|
||||
unwrappedStore,
|
||||
)
|
||||
return wrapped
|
||||
}
|
||||
|
||||
func diskCacheParams(diskParams string) (int, string, string) {
|
||||
if diskParams == "" {
|
||||
return 0, "", ""
|
||||
}
|
||||
|
||||
parts := strings.Split(diskParams, ":")
|
||||
if len(parts) != 3 {
|
||||
log.Fatalf("%s does is formatted incorrectly. Expected format: 'sizeGB:CACHE_PATH:cachemanager' for example: '100GB:/tmp/downloaded_blobs:localdb'", diskParams)
|
||||
}
|
||||
|
||||
diskCacheSize := parts[0]
|
||||
path := parts[1]
|
||||
cacheManager := parts[2]
|
||||
|
||||
if len(path) == 0 || path[0] != '/' {
|
||||
log.Fatalf("disk cache paths must start with '/'")
|
||||
}
|
||||
|
||||
if !util.InSlice(cacheManager, cacheManagers) {
|
||||
log.Fatalf("specified cache manager '%s' is not supported. Use one of the following: %v", cacheManager, cacheManagers)
|
||||
}
|
||||
|
||||
var maxSize datasize.ByteSize
|
||||
err := maxSize.UnmarshalText([]byte(diskCacheSize))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if maxSize <= 0 {
|
||||
log.Fatal("disk cache size must be more than 0")
|
||||
}
|
||||
return int(maxSize), path, cacheManager
|
||||
}
|
||||
|
||||
func cleanOldestBlobs(maxItems int, db *db.SQL, store store.BlobStore, stopper *stop.Group) {
|
||||
// this is so that it runs on startup without having to wait for 10 minutes
|
||||
err := doClean(maxItems, db, store, stopper)
|
||||
if err != nil {
|
||||
log.Error(errors.FullTrace(err))
|
||||
}
|
||||
const cleanupInterval = 10 * time.Minute
|
||||
for {
|
||||
select {
|
||||
case <-stopper.Ch():
|
||||
log.Infoln("stopping self cleanup")
|
||||
return
|
||||
case <-time.After(cleanupInterval):
|
||||
err := doClean(maxItems, db, store, stopper)
|
||||
if err != nil {
|
||||
log.Error(errors.FullTrace(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func doClean(maxItems int, db *db.SQL, store store.BlobStore, stopper *stop.Group) error {
|
||||
blobsCount, err := db.Count()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if blobsCount >= maxItems {
|
||||
itemsToDelete := blobsCount / 10
|
||||
blobs, err := db.LeastRecentlyAccessedHashes(itemsToDelete)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
blobsChan := make(chan string, len(blobs))
|
||||
wg := &stop.Group{}
|
||||
go func() {
|
||||
for _, hash := range blobs {
|
||||
select {
|
||||
case <-stopper.Ch():
|
||||
return
|
||||
default:
|
||||
}
|
||||
blobsChan <- hash
|
||||
}
|
||||
close(blobsChan)
|
||||
}()
|
||||
for i := 0; i < 3; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for h := range blobsChan {
|
||||
select {
|
||||
case <-stopper.Ch():
|
||||
return
|
||||
default:
|
||||
}
|
||||
err = store.Delete(h)
|
||||
if err != nil {
|
||||
log.Errorf("error pruning %s: %s", h, errors.FullTrace(err))
|
||||
continue
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
+48
-48
@@ -1,20 +1,15 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/lbryio/reflector.go/updater"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/dht"
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/util"
|
||||
|
||||
"github.com/johntdyer/slackrus"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
// Config is the base configuration for Prism when no sub commands are used.
|
||||
@@ -23,8 +18,10 @@ type Config struct {
|
||||
AwsSecret string `json:"aws_secret"`
|
||||
BucketRegion string `json:"bucket_region"`
|
||||
BucketName string `json:"bucket_name"`
|
||||
S3Endpoint string `json:"s3_endpoint"`
|
||||
DBConn string `json:"db_conn"`
|
||||
SlackHookURL string `json:"slack_hook_url"`
|
||||
SlackChannel string `json:"slack_channel"`
|
||||
UpdateBinURL string `json:"update_bin_url"`
|
||||
UpdateCmd string `json:"update_cmd"`
|
||||
}
|
||||
@@ -38,7 +35,8 @@ const (
|
||||
)
|
||||
|
||||
var conf string
|
||||
var globalConfig Config
|
||||
|
||||
//var globalConfig Config
|
||||
|
||||
var rootCmd = &cobra.Command{
|
||||
Use: "prism",
|
||||
@@ -51,7 +49,7 @@ var rootCmd = &cobra.Command{
|
||||
|
||||
func init() {
|
||||
rootCmd.PersistentFlags().StringSliceVarP(&verbose, "verbose", "v", []string{}, "Verbose logging for specific components")
|
||||
rootCmd.PersistentFlags().StringVar(&conf, "conf", "config.json", "Path to config. Use 'none' to disable")
|
||||
rootCmd.PersistentFlags().StringVar(&conf, "conf-dir", "./", "Path to config directory")
|
||||
}
|
||||
|
||||
// Execute adds all child commands to the root command and sets flags appropriately.
|
||||
@@ -86,38 +84,40 @@ func preRun(cmd *cobra.Command, args []string) {
|
||||
}
|
||||
}
|
||||
|
||||
var err error
|
||||
//var err error
|
||||
if conf == "" {
|
||||
logrus.Errorln("--conf flag required")
|
||||
os.Exit(1)
|
||||
} else if conf != "none" {
|
||||
globalConfig, err = loadConfig(conf)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
//else if conf != "none" {
|
||||
// globalConfig, err = loadConfig(conf)
|
||||
// if err != nil {
|
||||
// logrus.Error(err)
|
||||
// os.Exit(1)
|
||||
// }
|
||||
//}
|
||||
viper.AddConfigPath(conf)
|
||||
|
||||
if globalConfig.SlackHookURL != "" {
|
||||
hook := &slackrus.SlackrusHook{
|
||||
HookURL: globalConfig.SlackHookURL,
|
||||
AcceptedLevels: slackrus.LevelThreshold(logrus.InfoLevel),
|
||||
Channel: "#reflector-logs",
|
||||
//IconEmoji: ":ghost:",
|
||||
//Username: "reflector.go",
|
||||
}
|
||||
//logrus.SetFormatter(&logrus.JSONFormatter{})
|
||||
logrus.AddHook(hook)
|
||||
debugLogger.AddHook(hook)
|
||||
}
|
||||
//if globalConfig.SlackHookURL != "" {
|
||||
// hook := &slackrus.SlackrusHook{
|
||||
// HookURL: globalConfig.SlackHookURL,
|
||||
// AcceptedLevels: slackrus.LevelThreshold(logrus.InfoLevel),
|
||||
// Channel: globalConfig.SlackChannel,
|
||||
// //IconEmoji: ":ghost:",
|
||||
// //Username: "reflector.go",
|
||||
// }
|
||||
// //logrus.SetFormatter(&logrus.JSONFormatter{})
|
||||
// logrus.AddHook(hook)
|
||||
// debugLogger.AddHook(hook)
|
||||
//}
|
||||
|
||||
if globalConfig.UpdateBinURL != "" {
|
||||
if globalConfig.UpdateCmd == "" {
|
||||
logrus.Warnln("update_cmd is empty in conf file")
|
||||
}
|
||||
logrus.Println("starting update checker")
|
||||
go updater.Run(globalConfig.UpdateBinURL, globalConfig.UpdateCmd)
|
||||
}
|
||||
//if globalConfig.UpdateBinURL != "" {
|
||||
// if globalConfig.UpdateCmd == "" {
|
||||
// logrus.Warnln("update_cmd is empty in conf file")
|
||||
// }
|
||||
// logrus.Println("starting update checker")
|
||||
// go updater.Run(globalConfig.UpdateBinURL, globalConfig.UpdateCmd)
|
||||
//}
|
||||
}
|
||||
|
||||
func checkErr(err error) {
|
||||
@@ -138,20 +138,20 @@ func argFuncs(funcs ...cobra.PositionalArgs) cobra.PositionalArgs {
|
||||
}
|
||||
}
|
||||
|
||||
func loadConfig(path string) (Config, error) {
|
||||
var c Config
|
||||
|
||||
raw, err := ioutil.ReadFile(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return c, errors.Err("config file not found")
|
||||
}
|
||||
return c, errors.Err(err)
|
||||
}
|
||||
|
||||
err = json.Unmarshal(raw, &c)
|
||||
return c, errors.Err(err)
|
||||
}
|
||||
//func loadConfig(path string) (Config, error) {
|
||||
// var c Config
|
||||
//
|
||||
// raw, err := os.ReadFile(path)
|
||||
// if err != nil {
|
||||
// if os.IsNotExist(err) {
|
||||
// return c, errors.Err("config file not found")
|
||||
// }
|
||||
// return c, errors.Err(err)
|
||||
// }
|
||||
//
|
||||
// err = json.Unmarshal(raw, &c)
|
||||
// return c, errors.Err(err)
|
||||
//}
|
||||
|
||||
func mustGetFlagString(cmd *cobra.Command, name string) string {
|
||||
v, err := cmd.Flags().GetString(name)
|
||||
|
||||
+9
-9
@@ -5,7 +5,6 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path"
|
||||
@@ -39,17 +38,16 @@ func sendCmd(cmd *cobra.Command, args []string) {
|
||||
reflectorAddress := args[0]
|
||||
err := hackyReflector.Connect(reflectorAddress)
|
||||
checkErr(err)
|
||||
defer hackyReflector.Close()
|
||||
defer func() { _ = hackyReflector.Close() }()
|
||||
|
||||
filePath := args[1]
|
||||
file, err := os.Open(filePath)
|
||||
checkErr(err)
|
||||
defer file.Close()
|
||||
|
||||
defer func() { _ = file.Close() }()
|
||||
sdCachePath := ""
|
||||
sdCacheDir := mustGetFlagString(cmd, "sd-cache")
|
||||
if sdCacheDir != "" {
|
||||
if _, err := os.Stat(sdCacheDir); os.IsNotExist(err) {
|
||||
if _, statErr := os.Stat(sdCacheDir); os.IsNotExist(statErr) {
|
||||
err = os.MkdirAll(sdCacheDir, 0777)
|
||||
checkErr(err)
|
||||
}
|
||||
@@ -59,8 +57,9 @@ func sendCmd(cmd *cobra.Command, args []string) {
|
||||
var enc *stream.Encoder
|
||||
|
||||
if sdCachePath != "" {
|
||||
if _, err := os.Stat(sdCachePath); !os.IsNotExist(err) {
|
||||
sdBlob, err := ioutil.ReadFile(sdCachePath)
|
||||
if _, statErr := os.Stat(sdCachePath); !os.IsNotExist(statErr) {
|
||||
var sdBlob []byte
|
||||
sdBlob, err = os.ReadFile(sdCachePath)
|
||||
checkErr(err)
|
||||
cachedSDBlob := &stream.SDBlob{}
|
||||
err = cachedSDBlob.FromBlob(sdBlob)
|
||||
@@ -89,7 +88,8 @@ func sendCmd(cmd *cobra.Command, args []string) {
|
||||
break
|
||||
}
|
||||
|
||||
b, err := enc.Next()
|
||||
var b stream.Blob
|
||||
b, err = enc.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
@@ -110,7 +110,7 @@ func sendCmd(cmd *cobra.Command, args []string) {
|
||||
sd := enc.SDBlob()
|
||||
//sd.StreamName = filepath.Base(filePath)
|
||||
//sd.SuggestedFileName = filepath.Base(filePath)
|
||||
err = ioutil.WriteFile(sdCachePath, sd.ToBlob(), 0666)
|
||||
err = os.WriteFile(sdCachePath, sd.ToBlob(), 0666)
|
||||
if err != nil {
|
||||
fmt.Printf("error saving sd blob: %v\n", err)
|
||||
fmt.Println(sd.ToJson())
|
||||
|
||||
+1
-1
@@ -51,7 +51,7 @@ func sendBlobCmd(cmd *cobra.Command, args []string) {
|
||||
|
||||
file, err := os.Open(path)
|
||||
checkErr(err)
|
||||
defer file.Close()
|
||||
defer func() { _ = file.Close() }()
|
||||
s, err := stream.New(file)
|
||||
checkErr(err)
|
||||
|
||||
|
||||
-102
@@ -1,102 +0,0 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"github.com/lbryio/reflector.go/cluster"
|
||||
"github.com/lbryio/reflector.go/db"
|
||||
"github.com/lbryio/reflector.go/prism"
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
"github.com/lbryio/reflector.go/server/peer"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/dht"
|
||||
"github.com/lbryio/lbry.go/v2/dht/bits"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
const (
|
||||
startNewCluster = "new"
|
||||
)
|
||||
|
||||
var (
|
||||
startClusterPort int
|
||||
startPeerPort int
|
||||
startReflectorPort int
|
||||
startDhtPort int
|
||||
startDhtSeeds []string
|
||||
startHashRange string
|
||||
)
|
||||
|
||||
func init() {
|
||||
var cmd = &cobra.Command{
|
||||
Use: `start [cluster-address|"new"]`,
|
||||
Short: "Runs full prism application with cluster, dht, peer server, and reflector server.",
|
||||
Run: startCmd,
|
||||
Args: cobra.ExactArgs(1),
|
||||
}
|
||||
cmd.PersistentFlags().IntVar(&startClusterPort, "cluster-port", cluster.DefaultPort, "Port that cluster listens on")
|
||||
cmd.PersistentFlags().IntVar(&startPeerPort, "peer-port", peer.DefaultPort, "Port to start peer protocol on")
|
||||
cmd.PersistentFlags().IntVar(&startReflectorPort, "reflector-port", reflector.DefaultPort, "Port to start reflector protocol on")
|
||||
cmd.PersistentFlags().IntVar(&startDhtPort, "dht-port", dht.DefaultPort, "Port that dht will listen on")
|
||||
cmd.PersistentFlags().StringSliceVar(&startDhtSeeds, "dht-seeds", []string{}, "Comma-separated list of dht seed nodes (addr:port,addr:port,...)")
|
||||
|
||||
cmd.PersistentFlags().StringVar(&startHashRange, "hash-range", "", "Limit on range of hashes to announce (start-end)")
|
||||
|
||||
rootCmd.AddCommand(cmd)
|
||||
}
|
||||
|
||||
func startCmd(cmd *cobra.Command, args []string) {
|
||||
db := &db.SQL{
|
||||
LogQueries: log.GetLevel() == log.DebugLevel,
|
||||
}
|
||||
err := db.Connect(globalConfig.DBConn)
|
||||
checkErr(err)
|
||||
s3 := store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
|
||||
comboStore := store.NewDBBackedStore(s3, db, false)
|
||||
|
||||
conf := prism.DefaultConf()
|
||||
|
||||
// TODO: args we need:
|
||||
// minNodes - minimum number of nodes before announcing starts. otherwise first node will try to announce all the blobs in the db
|
||||
// or maybe we should do maxHashesPerNode?
|
||||
// in either case, this should not kill the cluster, but should only limit announces (and notify when some hashes are being left unannounced)
|
||||
|
||||
if args[0] != startNewCluster {
|
||||
conf.ClusterSeedAddr = args[0]
|
||||
}
|
||||
|
||||
conf.DB = db
|
||||
conf.Blobs = comboStore
|
||||
conf.DhtAddress = "0.0.0.0:" + strconv.Itoa(startDhtPort)
|
||||
conf.DhtSeedNodes = startDhtSeeds
|
||||
conf.ClusterPort = startClusterPort
|
||||
conf.PeerPort = startPeerPort
|
||||
conf.ReflectorPort = startReflectorPort
|
||||
|
||||
if startHashRange != "" {
|
||||
hashRange := strings.Split(startHashRange, "-")
|
||||
if len(hashRange) != 2 {
|
||||
log.Fatal("invalid hash range")
|
||||
}
|
||||
r := bits.Range{Start: bits.FromShortHexP(hashRange[0]), End: bits.FromShortHexP(hashRange[1])}
|
||||
conf.HashRange = &r
|
||||
}
|
||||
|
||||
p := prism.New(conf)
|
||||
err = p.Start()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
interruptChan := make(chan os.Signal, 1)
|
||||
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
|
||||
<-interruptChan
|
||||
p.Shutdown()
|
||||
}
|
||||
+4
-4
@@ -29,9 +29,9 @@ func init() {
|
||||
func testCmd(cmd *cobra.Command, args []string) {
|
||||
log.Printf("reflector %s", meta.VersionString())
|
||||
|
||||
memStore := store.NewMemStore()
|
||||
memStore := store.NewMemStore(store.MemParams{Name: "test"})
|
||||
|
||||
reflectorServer := reflector.NewServer(memStore, memStore)
|
||||
reflectorServer := reflector.NewIngestionServer(memStore)
|
||||
reflectorServer.Timeout = 3 * time.Minute
|
||||
|
||||
err := reflectorServer.Start(":" + strconv.Itoa(reflector.DefaultPort))
|
||||
@@ -39,8 +39,8 @@ func testCmd(cmd *cobra.Command, args []string) {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
peerServer := peer.NewServer(memStore)
|
||||
err = peerServer.Start(":" + strconv.Itoa(reflector.DefaultPort+1))
|
||||
peerServer := peer.NewServer(memStore, fmt.Sprintf(":%d", reflector.DefaultPort+1))
|
||||
err = peerServer.Start()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
+10
-10
@@ -5,9 +5,8 @@ import (
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/lbryio/reflector.go/db"
|
||||
"github.com/lbryio/reflector.go/config"
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
@@ -31,17 +30,18 @@ func init() {
|
||||
}
|
||||
|
||||
func uploadCmd(cmd *cobra.Command, args []string) {
|
||||
db := &db.SQL{
|
||||
LogQueries: log.GetLevel() == log.DebugLevel,
|
||||
store, err := config.LoadStores(conf, "upload")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
err := db.Connect(globalConfig.DBConn)
|
||||
checkErr(err)
|
||||
defer store.Shutdown()
|
||||
|
||||
st := store.NewDBBackedStore(
|
||||
store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName),
|
||||
db, false)
|
||||
databaseConn, err := config.LoadDatabase(conf, "upload")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
uploader := reflector.NewUploader(db, st, uploadWorkers, uploadSkipExistsCheck, uploadDeleteBlobsAfterUpload)
|
||||
uploader := reflector.NewUploader(databaseConn, store, uploadWorkers, uploadSkipExistsCheck, uploadDeleteBlobsAfterUpload)
|
||||
|
||||
interruptChan := make(chan os.Signal, 1)
|
||||
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
|
||||
|
||||
+1
-1
@@ -18,5 +18,5 @@ func init() {
|
||||
}
|
||||
|
||||
func versionCmd(cmd *cobra.Command, args []string) {
|
||||
fmt.Println(meta.VersionString())
|
||||
fmt.Println(meta.FullName())
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/lbryio/reflector.go/db"
|
||||
"github.com/lbryio/reflector.go/server"
|
||||
"github.com/lbryio/reflector.go/server/http"
|
||||
"github.com/lbryio/reflector.go/server/http3"
|
||||
"github.com/lbryio/reflector.go/server/peer"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
func LoadStores(path, file string) (store.BlobStore, error) {
|
||||
v := viper.New()
|
||||
v.SetConfigType("yaml")
|
||||
v.AddConfigPath(path)
|
||||
v.SetConfigName(file)
|
||||
err := v.ReadInConfig()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
storeViper := v.Sub("store")
|
||||
for storeType := range storeViper.AllSettings() {
|
||||
factory, exists := store.Factories[storeType]
|
||||
if !exists {
|
||||
return nil, errors.Err("unknown store type: %s", storeType)
|
||||
}
|
||||
storeConfig := storeViper.Sub(storeType)
|
||||
s, err := factory(storeConfig)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
//we only expect 1 store as the root, so let's return it
|
||||
return s, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func LoadServers(store store.BlobStore, path, file string) ([]server.BlobServer, error) {
|
||||
v := viper.New()
|
||||
v.SetConfigType("yaml")
|
||||
v.AddConfigPath(path)
|
||||
v.SetConfigName(file)
|
||||
|
||||
err := v.ReadInConfig()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
servers := make([]server.BlobServer, 0)
|
||||
serversViper := v.Sub("servers")
|
||||
for serverType := range serversViper.AllSettings() {
|
||||
var cfg server.BlobServerConfig
|
||||
err := serversViper.Sub(serverType).Unmarshal(&cfg)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
switch serverType {
|
||||
case "http":
|
||||
servers = append(servers, http.NewServer(store, cfg.MaxConcurrentRequests, cfg.EdgeToken, fmt.Sprintf("%s:%d", cfg.Address, cfg.Port)))
|
||||
case "http3":
|
||||
servers = append(servers, http3.NewServer(store, cfg.MaxConcurrentRequests, fmt.Sprintf("%s:%d", cfg.Address, cfg.Port)))
|
||||
case "peer":
|
||||
servers = append(servers, peer.NewServer(store, fmt.Sprintf("%s:%d", cfg.Address, cfg.Port)))
|
||||
default:
|
||||
return nil, errors.Err("unknown server type: %s", serverType)
|
||||
}
|
||||
}
|
||||
return servers, nil
|
||||
}
|
||||
|
||||
func LoadDatabase(path, file string) (*db.SQL, error) {
|
||||
v := viper.New()
|
||||
v.SetConfigType("yaml")
|
||||
v.AddConfigPath(path)
|
||||
v.SetConfigName(file)
|
||||
|
||||
err := v.ReadInConfig()
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
dbConfig := v.Sub("database")
|
||||
if dbConfig == nil {
|
||||
return nil, errors.Err("db config not found")
|
||||
}
|
||||
user := dbConfig.GetString("user")
|
||||
password := dbConfig.GetString("password")
|
||||
host := dbConfig.GetString("host")
|
||||
port := dbConfig.GetInt("port")
|
||||
database := dbConfig.GetString("database")
|
||||
logQueries := dbConfig.GetBool("log_queries")
|
||||
accessTracking := dbConfig.GetInt("access_tracking")
|
||||
|
||||
if user == "" || password == "" || host == "" || port == 0 || database == "" {
|
||||
return nil, errors.Err("db config is missing required fields")
|
||||
}
|
||||
dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", user, password, host, port, database)
|
||||
dbInstance := &db.SQL{
|
||||
TrackingLevel: db.AccessTrackingLevel(accessTracking),
|
||||
SoftDelete: true,
|
||||
LogQueries: logQueries || log.GetLevel() == log.DebugLevel,
|
||||
}
|
||||
err = dbInstance.Connect(dsn)
|
||||
return dbInstance, err
|
||||
}
|
||||
@@ -15,33 +15,32 @@ import (
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
_ "github.com/go-sql-driver/mysql" // blank import for db driver ensures its imported even if its not used
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/volatiletech/null"
|
||||
"github.com/volatiletech/null/v8"
|
||||
"go.uber.org/atomic"
|
||||
)
|
||||
|
||||
// SdBlob is a special blob that contains information on the rest of the blobs in the stream
|
||||
type SdBlob struct {
|
||||
StreamName string `json:"stream_name"`
|
||||
Blobs []struct {
|
||||
Length int `json:"length"`
|
||||
BlobNum int `json:"blob_num"`
|
||||
BlobHash string `json:"blob_hash,omitempty"`
|
||||
IV string `json:"iv"`
|
||||
} `json:"blobs"`
|
||||
StreamName string `json:"stream_name"`
|
||||
StreamType string `json:"stream_type"`
|
||||
Key string `json:"key"`
|
||||
SuggestedFileName string `json:"suggested_file_name"`
|
||||
StreamHash string `json:"stream_hash"`
|
||||
Blobs []struct {
|
||||
BlobHash string `json:"blob_hash,omitempty"`
|
||||
IV string `json:"iv"`
|
||||
Length int `json:"length"`
|
||||
BlobNum int `json:"blob_num"`
|
||||
} `json:"blobs"`
|
||||
}
|
||||
|
||||
type trackAccess int
|
||||
type AccessTrackingLevel int
|
||||
|
||||
const (
|
||||
TrackAccessNone trackAccess = iota // Don't track accesses
|
||||
TrackAccessStreams // Track accesses at the stream level
|
||||
TrackAccessBlobs // Track accesses at the blob level
|
||||
TrackAccessNone AccessTrackingLevel = iota // Don't track accesses
|
||||
TrackAccessStreams // Track accesses at the stream level
|
||||
TrackAccessBlobs // Track accesses at the blob level
|
||||
)
|
||||
|
||||
// SQL implements the DB interface
|
||||
@@ -49,7 +48,7 @@ type SQL struct {
|
||||
conn *sql.DB
|
||||
|
||||
// Track the approx last time a blob or stream was accessed
|
||||
TrackAccess trackAccess
|
||||
TrackingLevel AccessTrackingLevel
|
||||
|
||||
// Instead of deleting a blob, marked it as not stored in the db
|
||||
SoftDelete bool
|
||||
@@ -58,7 +57,7 @@ type SQL struct {
|
||||
LogQueries bool
|
||||
}
|
||||
|
||||
func (s SQL) logQuery(query string, args ...interface{}) {
|
||||
func (s *SQL) logQuery(query string, args ...interface{}) {
|
||||
if !s.LogQueries {
|
||||
return
|
||||
}
|
||||
@@ -98,7 +97,7 @@ func (s *SQL) AddBlob(hash string, length int, isStored bool) error {
|
||||
return err
|
||||
}
|
||||
|
||||
//AddBlobs adds blobs to the database.
|
||||
// AddBlobs adds blobs to the database.
|
||||
func (s *SQL) AddBlobs(hash []string) error {
|
||||
if s.conn == nil {
|
||||
return errors.Err("not connected")
|
||||
@@ -180,7 +179,7 @@ func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error)
|
||||
q string
|
||||
args []interface{}
|
||||
)
|
||||
if s.TrackAccess == TrackAccessBlobs {
|
||||
if s.TrackingLevel == TrackAccessBlobs {
|
||||
args = []interface{}{hash, isStored, length, time.Now()}
|
||||
q = "INSERT INTO blob_ (hash, is_stored, length, last_accessed_at) VALUES (" + qt.Qs(len(args)) + ") ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored)), last_accessed_at = VALUES(last_accessed_at)"
|
||||
} else {
|
||||
@@ -202,7 +201,7 @@ func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error)
|
||||
return 0, errors.Err("blob ID is 0 even after INSERTing and SELECTing")
|
||||
}
|
||||
|
||||
if s.TrackAccess == TrackAccessBlobs {
|
||||
if s.TrackingLevel == TrackAccessBlobs {
|
||||
err := s.touchBlobs([]uint64{uint64(blobID)})
|
||||
if err != nil {
|
||||
return 0, errors.Err(err)
|
||||
@@ -219,7 +218,7 @@ func (s *SQL) insertStream(hash string, sdBlobID int64) (int64, error) {
|
||||
args []interface{}
|
||||
)
|
||||
|
||||
if s.TrackAccess == TrackAccessStreams {
|
||||
if s.TrackingLevel == TrackAccessStreams {
|
||||
args = []interface{}{hash, sdBlobID, time.Now()}
|
||||
q = "INSERT IGNORE INTO stream (hash, sd_blob_id, last_accessed_at) VALUES (" + qt.Qs(len(args)) + ")"
|
||||
} else {
|
||||
@@ -241,7 +240,7 @@ func (s *SQL) insertStream(hash string, sdBlobID int64) (int64, error) {
|
||||
return 0, errors.Err("stream ID is 0 even after INSERTing and SELECTing")
|
||||
}
|
||||
|
||||
if s.TrackAccess == TrackAccessStreams {
|
||||
if s.TrackingLevel == TrackAccessStreams {
|
||||
err := s.touchStreams([]uint64{uint64(streamID)})
|
||||
if err != nil {
|
||||
return 0, errors.Err(err)
|
||||
@@ -265,9 +264,9 @@ func (s *SQL) HasBlobs(hashes []string, touch bool) (map[string]bool, error) {
|
||||
exists, idsNeedingTouch, err := s.hasBlobs(hashes)
|
||||
|
||||
if touch {
|
||||
if s.TrackAccess == TrackAccessBlobs {
|
||||
if s.TrackingLevel == TrackAccessBlobs {
|
||||
_ = s.touchBlobs(idsNeedingTouch)
|
||||
} else if s.TrackAccess == TrackAccessStreams {
|
||||
} else if s.TrackingLevel == TrackAccessStreams {
|
||||
_ = s.touchStreams(idsNeedingTouch)
|
||||
}
|
||||
}
|
||||
@@ -339,11 +338,11 @@ func (s *SQL) hasBlobs(hashes []string) (map[string]bool, []uint64, error) {
|
||||
batch := hashes[doneIndex:sliceEnd]
|
||||
|
||||
var query string
|
||||
if s.TrackAccess == TrackAccessBlobs {
|
||||
if s.TrackingLevel == TrackAccessBlobs {
|
||||
query = `SELECT b.hash, b.id, NULL, b.last_accessed_at
|
||||
FROM blob_ b
|
||||
WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)`
|
||||
} else if s.TrackAccess == TrackAccessStreams {
|
||||
} else if s.TrackingLevel == TrackAccessStreams {
|
||||
query = `SELECT b.hash, b.id, s.id, s.last_accessed_at
|
||||
FROM blob_ b
|
||||
LEFT JOIN stream_blob sb ON b.id = sb.blob_id
|
||||
@@ -372,15 +371,15 @@ WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)`
|
||||
defer closeRows(rows)
|
||||
|
||||
for rows.Next() {
|
||||
err := rows.Scan(&hash, &blobID, &streamID, &lastAccessedAt)
|
||||
err = rows.Scan(&hash, &blobID, &streamID, &lastAccessedAt)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
exists[hash] = true
|
||||
if !lastAccessedAt.Valid || lastAccessedAt.Time.Before(touchDeadline) {
|
||||
if s.TrackAccess == TrackAccessBlobs {
|
||||
if s.TrackingLevel == TrackAccessBlobs {
|
||||
needsTouch = append(needsTouch, blobID)
|
||||
} else if s.TrackAccess == TrackAccessStreams && !streamID.IsZero() {
|
||||
} else if s.TrackingLevel == TrackAccessStreams && !streamID.IsZero() {
|
||||
needsTouch = append(needsTouch, streamID.Uint64)
|
||||
}
|
||||
}
|
||||
@@ -419,13 +418,13 @@ func (s *SQL) Delete(hash string) error {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
//LeastRecentlyAccessedHashes gets the least recently accessed blobs
|
||||
// LeastRecentlyAccessedHashes gets the least recently accessed blobs
|
||||
func (s *SQL) LeastRecentlyAccessedHashes(maxBlobs int) ([]string, error) {
|
||||
if s.conn == nil {
|
||||
return nil, errors.Err("not connected")
|
||||
}
|
||||
|
||||
if s.TrackAccess != TrackAccessBlobs {
|
||||
if s.TrackingLevel != TrackAccessBlobs {
|
||||
return nil, errors.Err("blob access tracking is disabled")
|
||||
}
|
||||
|
||||
@@ -490,7 +489,7 @@ func (s *SQL) GetBlocked() (map[string]bool, error) {
|
||||
|
||||
var hash string
|
||||
for rows.Next() {
|
||||
err := rows.Scan(&hash)
|
||||
err = rows.Scan(&hash)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
@@ -534,7 +533,7 @@ func (s *SQL) MissingBlobsForKnownStream(sdHash string) ([]string, error) {
|
||||
var hash string
|
||||
|
||||
for rows.Next() {
|
||||
err := rows.Scan(&hash)
|
||||
err = rows.Scan(&hash)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
@@ -637,7 +636,7 @@ func (s *SQL) GetStoredHashesInRange(ctx context.Context, start, end bits.Bitmap
|
||||
var hash string
|
||||
ScanLoop:
|
||||
for rows.Next() {
|
||||
err := rows.Scan(&hash)
|
||||
err = rows.Scan(&hash)
|
||||
if err != nil {
|
||||
ech <- err
|
||||
return
|
||||
|
||||
@@ -1,51 +1,139 @@
|
||||
module github.com/lbryio/reflector.go
|
||||
|
||||
go 1.25
|
||||
|
||||
replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19
|
||||
|
||||
require (
|
||||
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
|
||||
github.com/aws/aws-sdk-go v1.27.0
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.0
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.6
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.6
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.18
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0
|
||||
github.com/bluele/gcache v0.0.2
|
||||
github.com/brk0v/directio v0.0.0-20190225130936-69406e757cf7
|
||||
github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3
|
||||
github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d
|
||||
github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2
|
||||
github.com/davecgh/go-spew v1.1.1
|
||||
github.com/brk0v/directio v0.0.0-20241105172640-ae9d82eb8fee
|
||||
github.com/btcsuite/btcd v0.24.2
|
||||
github.com/btcsuite/btcutil v1.0.2
|
||||
github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
|
||||
github.com/ekyoung/gin-nice-recovery v0.0.0-20160510022553-1654dca486db
|
||||
github.com/gin-gonic/gin v1.7.1
|
||||
github.com/go-sql-driver/mysql v1.6.0
|
||||
github.com/gogo/protobuf v1.2.1
|
||||
github.com/golang/protobuf v1.5.2
|
||||
github.com/google/go-cmp v0.5.6 // indirect
|
||||
github.com/google/gops v0.3.18
|
||||
github.com/gorilla/mux v1.8.0
|
||||
github.com/hashicorp/go-msgpack v0.5.5 // indirect
|
||||
github.com/hashicorp/golang-lru v0.5.4 // indirect
|
||||
github.com/hashicorp/serf v0.9.5
|
||||
github.com/gin-gonic/gin v1.11.0
|
||||
github.com/go-sql-driver/mysql v1.9.3
|
||||
github.com/gogo/protobuf v1.3.2
|
||||
github.com/golang/protobuf v1.5.4
|
||||
github.com/google/gops v0.3.28
|
||||
github.com/gorilla/mux v1.8.1
|
||||
github.com/hashicorp/serf v0.10.2
|
||||
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf
|
||||
github.com/johntdyer/slackrus v0.0.0-20210521205746-42486fb4c48c
|
||||
github.com/karrick/godirwalk v1.16.1
|
||||
github.com/lbryio/chainquery v1.9.0
|
||||
github.com/lbryio/lbry.go/v2 v2.7.2-0.20210416195322-6516df1418e3
|
||||
github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386
|
||||
github.com/lucas-clemente/quic-go v0.20.1
|
||||
github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6
|
||||
github.com/prometheus/client_golang v1.10.0
|
||||
github.com/sergi/go-diff v1.2.0 // indirect
|
||||
github.com/sirupsen/logrus v1.8.1
|
||||
github.com/spf13/afero v1.4.1 // indirect
|
||||
github.com/spf13/cast v1.3.0
|
||||
github.com/spf13/cobra v1.1.3
|
||||
github.com/spf13/viper v1.7.1 // indirect
|
||||
github.com/stretchr/testify v1.7.0
|
||||
github.com/volatiletech/null v8.0.0+incompatible
|
||||
go.uber.org/atomic v1.7.0
|
||||
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 // indirect
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
|
||||
golang.org/x/text v0.3.6 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
|
||||
google.golang.org/protobuf v1.27.1 // indirect
|
||||
github.com/karrick/godirwalk v1.17.0
|
||||
github.com/lbryio/chainquery v1.9.1-0.20240927170248-48c092515dea
|
||||
github.com/lbryio/lbry.go/v2 v2.7.2-0.20230307181431-a01aa6dc0629
|
||||
github.com/lbryio/types v0.0.0-20220224142228-73610f6654a6
|
||||
github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5
|
||||
github.com/prometheus/client_golang v1.23.2
|
||||
github.com/quic-go/quic-go v0.58.0
|
||||
github.com/sirupsen/logrus v1.9.3
|
||||
github.com/spf13/cast v1.10.0
|
||||
github.com/spf13/cobra v1.10.2
|
||||
github.com/spf13/viper v1.21.0
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/volatiletech/null/v8 v8.1.2
|
||||
go.uber.org/atomic v1.11.0
|
||||
golang.org/x/sync v0.19.0
|
||||
)
|
||||
|
||||
go 1.16
|
||||
require (
|
||||
filippo.io/edwards25519 v1.1.0 // indirect
|
||||
github.com/armon/go-metrics v0.4.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 // indirect
|
||||
github.com/aws/smithy-go v1.24.0 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/btcsuite/btclog v1.0.0 // indirect
|
||||
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd // indirect
|
||||
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 // indirect
|
||||
github.com/bytedance/sonic v1.14.0 // indirect
|
||||
github.com/bytedance/sonic/loader v0.3.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/cloudwego/base64x v0.1.6 // indirect
|
||||
github.com/friendsofgo/errors v0.9.2 // indirect
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.8 // indirect
|
||||
github.com/gin-contrib/sse v1.1.0 // indirect
|
||||
github.com/go-errors/errors v1.5.1 // indirect
|
||||
github.com/go-ini/ini v1.67.0 // indirect
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||
github.com/go-playground/validator/v10 v10.27.0 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
||||
github.com/goccy/go-json v0.10.5 // indirect
|
||||
github.com/goccy/go-yaml v1.18.0 // indirect
|
||||
github.com/gofrs/uuid v4.4.0+incompatible // indirect
|
||||
github.com/google/btree v1.1.2 // indirect
|
||||
github.com/gorilla/rpc v1.2.0 // indirect
|
||||
github.com/gorilla/websocket v1.5.3 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
|
||||
github.com/hashicorp/go-metrics v0.5.4 // indirect
|
||||
github.com/hashicorp/go-msgpack/v2 v2.1.2 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/hashicorp/go-sockaddr v1.0.5 // indirect
|
||||
github.com/hashicorp/golang-lru v1.0.2 // indirect
|
||||
github.com/hashicorp/memberlist v0.5.2 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
|
||||
github.com/leodido/go-urn v1.4.0 // indirect
|
||||
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/miekg/dns v1.1.56 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/common v0.66.1 // indirect
|
||||
github.com/prometheus/procfs v0.16.1 // indirect
|
||||
github.com/quic-go/qpack v0.6.0 // indirect
|
||||
github.com/sagikazarmark/locafero v0.11.0 // indirect
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
|
||||
github.com/shopspring/decimal v1.4.0 // indirect
|
||||
github.com/slack-go/slack v0.16.0 // indirect
|
||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
|
||||
github.com/spf13/afero v1.15.0 // indirect
|
||||
github.com/spf13/pflag v1.0.10 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
||||
github.com/ugorji/go/codec v1.3.0 // indirect
|
||||
github.com/volatiletech/inflect v0.0.1 // indirect
|
||||
github.com/volatiletech/randomize v0.0.1 // indirect
|
||||
github.com/volatiletech/strmangle v0.0.8 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.2 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/arch v0.20.0 // indirect
|
||||
golang.org/x/crypto v0.41.0 // indirect
|
||||
golang.org/x/mod v0.27.0 // indirect
|
||||
golang.org/x/net v0.43.0 // indirect
|
||||
golang.org/x/sys v0.35.0 // indirect
|
||||
golang.org/x/text v0.28.0 // indirect
|
||||
golang.org/x/time v0.8.0 // indirect
|
||||
golang.org/x/tools v0.36.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
|
||||
google.golang.org/protobuf v1.36.9 // indirect
|
||||
gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
+30
-10
@@ -69,6 +69,7 @@ const (
|
||||
DirectionDownload = "download" // from reflector
|
||||
|
||||
LabelCacheType = "cache_type"
|
||||
LabelOrigin = "origin"
|
||||
LabelComponent = "component"
|
||||
LabelSource = "source"
|
||||
|
||||
@@ -86,8 +87,13 @@ const (
|
||||
errUnexpectedEOFStr = "unexpected_eof_str"
|
||||
errJSONSyntax = "json_syntax"
|
||||
errBlobTooBig = "blob_too_big"
|
||||
errInvalidPeerJSON = "invalid_peer_json"
|
||||
errInvalidPeerData = "invalid_peer_data"
|
||||
errRequestTooLarge = "request_too_large"
|
||||
errDeadlineExceeded = "deadline_exceeded"
|
||||
errHashMismatch = "hash_mismatch"
|
||||
errProtectedBlob = "protected_blob"
|
||||
errInvalidBlobHash = "invalid_blob_hash"
|
||||
errZeroByteBlob = "zero_byte_blob"
|
||||
errInvalidCharacter = "invalid_character"
|
||||
errBlobNotFound = "blob_not_found"
|
||||
@@ -130,18 +136,12 @@ var (
|
||||
Name: "hit_total",
|
||||
Help: "Total number of blobs retrieved from the cache storage",
|
||||
}, []string{LabelCacheType, LabelComponent})
|
||||
ThisHitCount = promauto.NewCounter(prometheus.CounterOpts{
|
||||
ItttHitCount = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Subsystem: subsystemITTT,
|
||||
Name: "this_hit_total",
|
||||
Help: "Total number of blobs retrieved from the this storage",
|
||||
})
|
||||
ThatHitCount = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Subsystem: subsystemITTT,
|
||||
Name: "that_hit_total",
|
||||
Help: "Total number of blobs retrieved from the that storage",
|
||||
})
|
||||
Name: "hits_total",
|
||||
Help: "Total number of blobs retrieved from the this/that storage",
|
||||
}, []string{LabelOrigin})
|
||||
CacheMissCount = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Subsystem: subsystemCache,
|
||||
@@ -224,6 +224,11 @@ var (
|
||||
Name: "s3_out_bytes",
|
||||
Help: "Total number of outgoing bytes (to S3)",
|
||||
})
|
||||
MtrInBytesUpstream = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "upstream_in_bytes",
|
||||
Help: "Total number of incoming bytes (from Upstream)",
|
||||
})
|
||||
MtrInBytesS3 = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: ns,
|
||||
Name: "s3_in_bytes",
|
||||
@@ -252,6 +257,11 @@ func CacheLabels(name, component string) prometheus.Labels {
|
||||
LabelComponent: component,
|
||||
}
|
||||
}
|
||||
func ItttLabels(orig string) prometheus.Labels {
|
||||
return prometheus.Labels{
|
||||
LabelOrigin: orig,
|
||||
}
|
||||
}
|
||||
|
||||
func TrackError(direction string, e error) (shouldLog bool) { // shouldLog is a hack, but whatever
|
||||
if e == nil {
|
||||
@@ -296,10 +306,20 @@ func TrackError(direction string, e error) (shouldLog bool) { // shouldLog is a
|
||||
} else if strings.Contains(err.Error(), "blob must be at most") {
|
||||
//log.Warnln("blob must be at most X bytes is not the same as ErrBlobTooBig")
|
||||
errType = errBlobTooBig
|
||||
} else if strings.Contains(err.Error(), "invalid json request") {
|
||||
errType = errInvalidPeerJSON
|
||||
} else if strings.Contains(err.Error(), "Invalid data") {
|
||||
errType = errInvalidPeerData
|
||||
} else if strings.Contains(err.Error(), "request is too large") {
|
||||
errType = errRequestTooLarge
|
||||
} else if strings.Contains(err.Error(), "Invalid blob hash length") {
|
||||
errType = errInvalidBlobHash
|
||||
} else if strings.Contains(err.Error(), "hash of received blob data does not match hash from send request") {
|
||||
errType = errHashMismatch
|
||||
} else if strings.Contains(err.Error(), "blob not found") {
|
||||
errType = errBlobNotFound
|
||||
} else if strings.Contains(err.Error(), "requested blob is protected") {
|
||||
errType = errProtectedBlob
|
||||
} else if strings.Contains(err.Error(), "0-byte blob received") {
|
||||
errType = errZeroByteBlob
|
||||
} else if strings.Contains(err.Error(), "PROTOCOL_VIOLATION: tried to retire connection") {
|
||||
|
||||
+10
-11
@@ -8,24 +8,23 @@ import (
|
||||
qt "github.com/lbryio/lbry.go/v2/extras/query"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
_ "github.com/go-sql-driver/mysql" // blank import for db driver ensures its imported even if its not used
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/volatiletech/null"
|
||||
"github.com/volatiletech/null/v8"
|
||||
)
|
||||
|
||||
// SdBlob is a special blob that contains information on the rest of the blobs in the stream
|
||||
type SdBlob struct {
|
||||
StreamName string `json:"stream_name"`
|
||||
Blobs []struct {
|
||||
Length int `json:"length"`
|
||||
BlobNum int `json:"blob_num"`
|
||||
BlobHash string `json:"blob_hash,omitempty"`
|
||||
IV string `json:"iv"`
|
||||
} `json:"blobs"`
|
||||
StreamName string `json:"stream_name"`
|
||||
StreamType string `json:"stream_type"`
|
||||
Key string `json:"key"`
|
||||
SuggestedFileName string `json:"suggested_file_name"`
|
||||
StreamHash string `json:"stream_hash"`
|
||||
Blobs []struct {
|
||||
BlobHash string `json:"blob_hash,omitempty"`
|
||||
IV string `json:"iv"`
|
||||
Length int `json:"length"`
|
||||
BlobNum int `json:"blob_num"`
|
||||
} `json:"blobs"`
|
||||
}
|
||||
|
||||
// SQL implements the DB interface
|
||||
@@ -111,7 +110,7 @@ func (s *SQL) HasBlob(hash string) (bool, error) {
|
||||
// HasBlobs checks if the database contains the set of blobs and returns a bool map.
|
||||
func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
|
||||
exists, streamsNeedingTouch, err := s.hasBlobs(hashes)
|
||||
s.touch(streamsNeedingTouch)
|
||||
_ = s.touch(streamsNeedingTouch)
|
||||
return exists, err
|
||||
}
|
||||
|
||||
@@ -182,7 +181,7 @@ WHERE is_stored = ? and hash IN (` + qt.Qs(len(batch)) + `)`
|
||||
defer closeRows(rows)
|
||||
|
||||
for rows.Next() {
|
||||
err := rows.Scan(&hash, &blobID, &lastAccessedAt)
|
||||
err = rows.Scan(&hash, &blobID, &lastAccessedAt)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
@@ -1,9 +1,6 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"math/rand"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/cmd"
|
||||
|
||||
"github.com/google/gops/agent"
|
||||
@@ -14,6 +11,5 @@ func main() {
|
||||
if err := agent.Listen(agent.Options{}); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
cmd.Execute()
|
||||
}
|
||||
+30
-8
@@ -6,9 +6,36 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
var Version = ""
|
||||
var Time = ""
|
||||
var BuildTime time.Time
|
||||
var (
|
||||
name = "prism-bin"
|
||||
version = "unknown"
|
||||
commit = "unknown"
|
||||
commitLong = "unknown"
|
||||
branch = "unknown"
|
||||
Time = "unknown"
|
||||
BuildTime time.Time
|
||||
)
|
||||
|
||||
// Name returns main application name
|
||||
func Name() string {
|
||||
return name
|
||||
}
|
||||
|
||||
// Version returns current application version
|
||||
func Version() string {
|
||||
return version
|
||||
}
|
||||
|
||||
// FullName returns current app version, commit and build time
|
||||
func FullName() string {
|
||||
return fmt.Sprintf(
|
||||
`Name: %v
|
||||
Version: %v
|
||||
branch: %v
|
||||
commit: %v
|
||||
commit long: %v
|
||||
build date: %v`, Name(), Version(), branch, commit, commitLong, BuildTime.String())
|
||||
}
|
||||
|
||||
func init() {
|
||||
if Time != "" {
|
||||
@@ -20,11 +47,6 @@ func init() {
|
||||
}
|
||||
|
||||
func VersionString() string {
|
||||
version := Version
|
||||
if version == "" {
|
||||
version = "<unset>"
|
||||
}
|
||||
|
||||
var buildTime string
|
||||
if BuildTime.IsZero() {
|
||||
buildTime = "<now>"
|
||||
|
||||
+13
-16
@@ -2,6 +2,7 @@ package prism
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"sync"
|
||||
|
||||
@@ -20,20 +21,16 @@ import (
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
PeerPort int
|
||||
ReflectorPort int
|
||||
|
||||
DhtAddress string
|
||||
DhtSeedNodes []string
|
||||
|
||||
ClusterPort int
|
||||
ClusterSeedAddr string
|
||||
|
||||
// limit the range of hashes to announce. useful for testing
|
||||
HashRange *bits.Range
|
||||
|
||||
DB *db.SQL
|
||||
Blobs store.BlobStore
|
||||
// limit the range of hashes to announce. useful for testing
|
||||
HashRange *bits.Range
|
||||
DB *db.SQL
|
||||
DhtAddress string
|
||||
ClusterSeedAddr string
|
||||
DhtSeedNodes []string
|
||||
PeerPort int
|
||||
ReflectorPort int
|
||||
ClusterPort int
|
||||
}
|
||||
|
||||
// DefaultConf returns a default config
|
||||
@@ -78,8 +75,8 @@ func New(conf *Config) *Prism {
|
||||
db: conf.DB,
|
||||
dht: d,
|
||||
cluster: c,
|
||||
peer: peer.NewServer(conf.Blobs),
|
||||
reflector: reflector.NewServer(conf.Blobs, conf.Blobs),
|
||||
peer: peer.NewServer(conf.Blobs, fmt.Sprintf(":%d", conf.PeerPort)),
|
||||
reflector: reflector.NewIngestionServer(conf.Blobs),
|
||||
|
||||
grp: stop.New(),
|
||||
}
|
||||
@@ -107,7 +104,7 @@ func (p *Prism) Start() error {
|
||||
return errors.Err("blobs required in conf")
|
||||
}
|
||||
|
||||
err = p.peer.Start(":" + strconv.Itoa(p.conf.PeerPort))
|
||||
err = p.peer.Start()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
+2
-3
@@ -115,7 +115,7 @@ func Publish(client *lbrycrd.Client, path, name, address string, details Details
|
||||
return signedTx, txid, nil
|
||||
}
|
||||
|
||||
//TODO: lots of assumptions. hardcoded values need to be passed in or calculated
|
||||
// TODO: lots of assumptions. hardcoded values need to be passed in or calculated
|
||||
func baseTx(client *lbrycrd.Client, amount float64, changeAddress btcutil.Address) (*wire.MsgTx, error) {
|
||||
txFee := 0.0002 // TODO: estimate this better?
|
||||
|
||||
@@ -222,8 +222,7 @@ func makeStream(path string) (stream.Stream, *pb.Stream, error) {
|
||||
if err != nil {
|
||||
return nil, nil, errors.Err(err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
defer func() { _ = file.Close() }()
|
||||
enc := stream.NewEncoder(file)
|
||||
|
||||
s, err := enc.Stream()
|
||||
|
||||
+11
-11
@@ -13,21 +13,21 @@ func LoadWallet(r io.Reader) (WalletFile, error) {
|
||||
|
||||
type WalletFile struct {
|
||||
Name string `json:"name"`
|
||||
Version int `json:"version"`
|
||||
Preferences WalletPrefs `json:"preferences"`
|
||||
Accounts []Account `json:"accounts"`
|
||||
Preferences WalletPrefs `json:"preferences"`
|
||||
Version int `json:"version"`
|
||||
}
|
||||
|
||||
type Account struct {
|
||||
AddressGenerator AddressGenerator `json:"address_generator"`
|
||||
Certificates map[string]string `json:"certificates"`
|
||||
Encrypted bool `json:"encrypted"`
|
||||
Ledger string `json:"ledger"`
|
||||
ModifiedOn float64 `json:"modified_on"`
|
||||
Name string `json:"name"`
|
||||
PrivateKey string `json:"private_key"`
|
||||
PublicKey string `json:"public_key"`
|
||||
Seed string `json:"seed"`
|
||||
AddressGenerator AddressGenerator `json:"address_generator"`
|
||||
ModifiedOn float64 `json:"modified_on"`
|
||||
Encrypted bool `json:"encrypted"`
|
||||
}
|
||||
|
||||
type AddressGenerator struct {
|
||||
@@ -43,17 +43,17 @@ type AddressGenParams struct {
|
||||
|
||||
type WalletPrefs struct {
|
||||
Shared struct {
|
||||
Ts float64 `json:"ts"`
|
||||
Value struct {
|
||||
Type string `json:"type"`
|
||||
Value struct {
|
||||
AppWelcomeVersion int `json:"app_welcome_version"`
|
||||
Type string `json:"type"`
|
||||
Version string `json:"version"`
|
||||
Value struct {
|
||||
Blocked []interface{} `json:"blocked"`
|
||||
Sharing3P bool `json:"sharing_3P"`
|
||||
Subscriptions []string `json:"subscriptions"`
|
||||
Tags []string `json:"tags"`
|
||||
AppWelcomeVersion int `json:"app_welcome_version"`
|
||||
Sharing3P bool `json:"sharing_3P"`
|
||||
} `json:"value"`
|
||||
Version string `json:"version"`
|
||||
} `json:"value"`
|
||||
Ts float64 `json:"ts"`
|
||||
} `json:"shared"`
|
||||
}
|
||||
@@ -1,124 +1,189 @@
|
||||
# Reflector
|
||||
|
||||
Reflector is a central piece of software that providers LBRY with the following features:
|
||||
- Blobs reflection: when something is published, we capture the data and store it on our servers for quicker retrieval
|
||||
- Blobs distribution: when a piece of content is requested and the LBRY network doesn't have it, reflector will retrieve it from its storage and distribute it
|
||||
- Blobs caching: reflectors can be chained together in multiple regions or servers to form a chain of cached content. We call those "blobcaches". They are layered so that content distribution is favorable in all the regions we deploy it to
|
||||
Production-ready blob reflection, distribution, and caching for Odysee.
|
||||
|
||||
There are a few other features embedded in reflector.go including publishing streams from Go, downloading or upload blobs, resolving content and more unfinished tools.
|
||||
This repository provides the components used in production:
|
||||
- Reflector ingestion server (command name: `reflector`)
|
||||
- Blob cache/edge server (`blobcache`)
|
||||
- Uploader to object storage (`upload`)
|
||||
|
||||
This code includes a Go implementations of the LBRY peer protocol, reflector protocol, and DHT.
|
||||
Other commands exist in the tree for historical/legacy reasons and are not supported.
|
||||
|
||||
## Installation
|
||||
## How it works (at a glance)
|
||||
- Ingestion (reflector): accepts uploaded blobs, persists them to object storage (e.g., S3/Wasabi) and tracks state in MySQL.
|
||||
- Distribution: serves blobs over HTTP/HTTP3/Peer. Blobcaches can be deployed in front of the origin to reduce latency and egress.
|
||||
- Caching (blobcache): layered disk caches backed by HTTP(S) origins (e.g., S3 endpoints), with optional local DB metadata for capacity/eviction.
|
||||
|
||||
- Install mysql 8 (5.7 might work too)
|
||||
- add a reflector user and database with password `reflector` with localhost access only
|
||||
- Create the tables as described [here](https://github.com/lbryio/reflector.go/blob/ittt/db/db.go#L735) (the link might not update as the code does so just look for the schema in that file)
|
||||
All services are started by the `prism` binary and are configured via YAML files loaded from a configuration directory.
|
||||
|
||||
#### We do not support running reflector.go as a blob receiver, however if you want to run it as a private blobcache you may compile it yourself and run it as following:
|
||||
## Supported commands
|
||||
The following are the only supported commands for production use:
|
||||
|
||||
- Reflector ingestion: `prism reflector`
|
||||
- Flags: `--receiver-port` (default 5566), `--metrics-port` (default 2112), `--disable-blocklist`
|
||||
- Loads `reflector.yaml` from the config directory.
|
||||
|
||||
- Blob cache: `prism blobcache`
|
||||
- Flags: `--metrics-port` (default 2112), `--disable-blocklist`
|
||||
- Loads `blobcache.yaml` from the config directory.
|
||||
|
||||
- Uploader: `prism upload PATH`
|
||||
- Flags: `--workers`, `--skipExistsCheck`, `--deleteBlobsAfterUpload`
|
||||
- Loads `upload.yaml` from the config directory.
|
||||
|
||||
Global flag for all commands:
|
||||
- `--conf-dir` (default `./`): directory containing YAML config files.
|
||||
|
||||
## Configuration
|
||||
Configuration is per-command. The loader reads `<command>.yaml` from `--conf-dir`.
|
||||
|
||||
Common sections:
|
||||
- `servers`: enables HTTP/HTTP3/Peer servers. Keys: `http`, `http3`, `peer`. Each accepts:
|
||||
- `port` (int)
|
||||
- `max_concurrent_requests` (int, http/http3)
|
||||
- `edge_token` (string, http)
|
||||
- `address` (string, optional; bind address, omit for all interfaces)
|
||||
- `store`: defines the storage topology using composable stores. Frequently used:
|
||||
- `proxied-s3`: production pattern with a `writer` (DB-backed -> S3/multiwriter) and a `reader` (caching -> disk + HTTP origins).
|
||||
- `caching`: layered cache with a `cache` (often `db_backed` -> `disk`) and an `origin` chain (`http`, `http3`, or `ittt` fan-in).
|
||||
- `s3`, `disk`, `multiwriter`, `db_backed`, `http`, `http3`, `peer`, `upstream` are also available building blocks.
|
||||
|
||||
### Minimal examples
|
||||
Reflector – conf-dir contains `reflector.yaml`:
|
||||
```yaml
|
||||
servers:
|
||||
http:
|
||||
port: 5569
|
||||
max_concurrent_requests: 200
|
||||
http3:
|
||||
port: 5568
|
||||
max_concurrent_requests: 200
|
||||
peer:
|
||||
port: 5567
|
||||
store:
|
||||
proxied-s3:
|
||||
name: s3_read_proxy
|
||||
writer:
|
||||
db_backed:
|
||||
user: reflector
|
||||
password: reflector
|
||||
database: reflector
|
||||
host: localhost
|
||||
port: 3306
|
||||
access_tracking: 1
|
||||
soft_deletes: true
|
||||
store:
|
||||
s3:
|
||||
name: primary
|
||||
aws_id: YOUR_KEY
|
||||
aws_secret: YOUR_SECRET
|
||||
region: us-east-1
|
||||
bucket: blobs-bucket
|
||||
endpoint: https://s3.yourendpoint.tv
|
||||
reader:
|
||||
caching:
|
||||
cache:
|
||||
disk:
|
||||
name: local_cache
|
||||
mount_point: /mnt/reflector/cache
|
||||
sharding_size: 2
|
||||
origin:
|
||||
http:
|
||||
endpoint: https://s3.yourendpoint.tv/blobs-bucket/
|
||||
sharding_size: 4
|
||||
```
|
||||
|
||||
Blobcache – conf-dir contains `blobcache.yaml`:
|
||||
```yaml
|
||||
servers:
|
||||
http:
|
||||
port: 5569
|
||||
max_concurrent_requests: 200
|
||||
http3:
|
||||
port: 5568
|
||||
max_concurrent_requests: 200
|
||||
peer:
|
||||
port: 5567
|
||||
store:
|
||||
caching:
|
||||
cache:
|
||||
db_backed:
|
||||
user: reflector
|
||||
password: reflector
|
||||
database: reflector
|
||||
host: localhost
|
||||
port: 3306
|
||||
has_cap: true
|
||||
max_size: 500GB
|
||||
store:
|
||||
disk:
|
||||
name: blobcache
|
||||
mount_point: /mnt/blobcache/cache
|
||||
sharding_size: 2
|
||||
origin:
|
||||
http:
|
||||
endpoint: https://s3.yourendpoint.tv/blobs-bucket/
|
||||
sharding_size: 4
|
||||
```
|
||||
|
||||
Uploader – conf-dir contains `upload.yaml` (points to the same writer/backend as reflector):
|
||||
```yaml
|
||||
database:
|
||||
user: reflector
|
||||
password: reflector
|
||||
database: reflector
|
||||
host: localhost
|
||||
port: 3306
|
||||
store:
|
||||
proxied-s3:
|
||||
writer:
|
||||
db_backed:
|
||||
user: reflector
|
||||
password: reflector
|
||||
database: reflector
|
||||
host: localhost
|
||||
port: 3306
|
||||
store:
|
||||
s3:
|
||||
aws_id: YOUR_KEY
|
||||
aws_secret: YOUR_SECRET
|
||||
region: us-east-1
|
||||
bucket: blobs-bucket
|
||||
endpoint: https://s3.yourendpoint.tv
|
||||
```
|
||||
|
||||
## Quick start
|
||||
1) Build
|
||||
- Requires Go 1.25+
|
||||
- `make` (binaries in `dist/<platform>/prism-bin`)
|
||||
|
||||
2) Run a local blobcache
|
||||
```bash
|
||||
./prism-bin reflector \
|
||||
--conf="none" \
|
||||
--disable-uploads=true \
|
||||
--use-db=false \
|
||||
--upstream-reflector="reflector.lbry.com" \
|
||||
--upstream-protocol="http" \
|
||||
--request-queue-size=200 \
|
||||
--disk-cache="2GB:/path/to/your/storage/:localdb" \
|
||||
./dist/linux_amd64/prism-bin --conf-dir=./ blobcache
|
||||
```
|
||||
Place your `blobcache.yaml` in the `--conf-dir` directory.
|
||||
|
||||
Create a systemd script if you want to run it automatically on startup or as a service.
|
||||
|
||||
## Usage
|
||||
|
||||
Usage as reflector/blobcache:
|
||||
3) Run reflector ingestion
|
||||
```bash
|
||||
Run reflector server
|
||||
|
||||
Usage:
|
||||
prism reflector [flags]
|
||||
|
||||
Flags:
|
||||
--disable-blocklist Disable blocklist watching/updating
|
||||
--disable-uploads Disable uploads to this reflector server
|
||||
--disk-cache string Where to cache blobs on the file system. format is 'sizeGB:CACHE_PATH:cachemanager' (cachemanagers: localdb/lfuda/lru) (default "100GB:/tmp/downloaded_blobs:localdb")
|
||||
-h, --help help for reflector
|
||||
--http-peer-port int The port reflector will distribute content from over HTTP protocol (default 5569)
|
||||
--http3-peer-port int The port reflector will distribute content from over HTTP3 protocol (default 5568)
|
||||
--mem-cache int enable in-memory cache with a max size of this many blobs
|
||||
--metrics-port int The port reflector will use for prometheus metrics (default 2112)
|
||||
--optional-disk-cache string Optional secondary file system cache for blobs. format is 'sizeGB:CACHE_PATH:cachemanager' (cachemanagers: localdb/lfuda/lru) (this would get hit before the one specified in disk-cache)
|
||||
--origin-endpoint string HTTP edge endpoint for standard HTTP retrieval
|
||||
--origin-endpoint-fallback string HTTP edge endpoint for standard HTTP retrieval if first origin fails
|
||||
--receiver-port int The port reflector will receive content from (default 5566)
|
||||
--request-queue-size int How many concurrent requests from downstream should be handled at once (the rest will wait) (default 200)
|
||||
--tcp-peer-port int The port reflector will distribute content from for the TCP (LBRY) protocol (default 5567)
|
||||
--upstream-protocol string protocol used to fetch blobs from another upstream reflector server (tcp/http3/http) (default "http")
|
||||
--upstream-reflector string host:port of a reflector server where blobs are fetched from
|
||||
--use-db Whether to connect to the reflector db or not (default true)
|
||||
|
||||
Global Flags:
|
||||
--conf string Path to config. Use 'none' to disable (default "config.json")
|
||||
-v, --verbose strings Verbose logging for specific components
|
||||
./dist/linux_amd64/prism-bin --conf-dir=./ reflector --receiver-port=5566 --metrics-port=2112
|
||||
```
|
||||
|
||||
Other uses:
|
||||
|
||||
4) Upload blobs
|
||||
```bash
|
||||
Prism is a single entry point application with multiple sub modules which can be leveraged individually or together
|
||||
|
||||
Usage:
|
||||
prism [command]
|
||||
|
||||
Available Commands:
|
||||
check-integrity check blobs integrity for a given path
|
||||
cluster Start(join) to or Start a new cluster
|
||||
decode Decode a claim value
|
||||
dht Run dht node
|
||||
getstream Get a stream from a reflector server
|
||||
help Help about any command
|
||||
peer Run peer server
|
||||
populate-db populate local database with blobs from a disk storage
|
||||
publish Publish a file
|
||||
reflector Run reflector server
|
||||
resolve Resolve a URL
|
||||
send Send a file to a reflector
|
||||
sendblob Send a random blob to a reflector server
|
||||
start Runs full prism application with cluster, dht, peer server, and reflector server.
|
||||
test Test things
|
||||
upload Upload blobs to S3
|
||||
version Print the version
|
||||
|
||||
Flags:
|
||||
--conf string Path to config. Use 'none' to disable (default "config.json")
|
||||
-h, --help help for prism
|
||||
-v, --verbose strings Verbose logging for specific components
|
||||
```
|
||||
## Running from Source
|
||||
|
||||
This project requires [Go v1.16](https://golang.org/doc/install).
|
||||
|
||||
On Ubuntu you can install it with `sudo snap install go --classic`
|
||||
|
||||
```
|
||||
git clone git@github.com:lbryio/reflector.go.git
|
||||
cd reflector.go
|
||||
make
|
||||
./bin/prism-bin
|
||||
./dist/linux_amd64/prism-bin --conf-dir=./ upload /path/to/blobs \
|
||||
--workers=4 --skipExistsCheck
|
||||
```
|
||||
|
||||
## Contributing
|
||||
|
||||
coming soon
|
||||
|
||||
## License
|
||||
|
||||
This project is MIT licensed.
|
||||
## Notes
|
||||
- Only reflector, blobcache, and upload are supported. All other commands are legacy and may be removed in the future.
|
||||
- Metrics are exposed on the configured `--metrics-port` at `/metrics` (Prometheus format).
|
||||
- MySQL is required when using DB-backed stores (e.g., ingestion writer, capacity-aware caches).
|
||||
|
||||
## Security
|
||||
If you discover a security issue, please email security@lbry.com. Our PGP key is available at https://lbry.com/faq/pgp-key.
|
||||
|
||||
We take security seriously. Please contact security@lbry.com regarding any security issues.
|
||||
Our PGP key is [here](https://keybase.io/lbry/key.asc) if you need it.
|
||||
## License
|
||||
MIT License. See LICENSE.
|
||||
|
||||
## Contact
|
||||
The primary contact for this project is [@Nikooo777](https://github.com/Nikooo777) (niko-at-lbry.com)
|
||||
The primary contact for this project is [@Nikooo777](https://github.com/Nikooo777)
|
||||
+11
-11
@@ -22,10 +22,10 @@ const blocklistURL = "https://api.lbry.com/file/list_blocked"
|
||||
|
||||
func (s *Server) enableBlocklist(b store.Blocklister) {
|
||||
walletServers := []string{
|
||||
"spv25.lbry.com:50001",
|
||||
"spv26.lbry.com:50001",
|
||||
"spv19.lbry.com:50001",
|
||||
"spv14.lbry.com:50001",
|
||||
"a-hub1.odysee.com:50001",
|
||||
"b-hub1.odysee.com:50001",
|
||||
"c-hub1.odysee.com:50001",
|
||||
"s-hub1.odysee.com:50001",
|
||||
}
|
||||
|
||||
updateBlocklist(b, walletServers, s.grp.Ch())
|
||||
@@ -69,18 +69,18 @@ func blockedSdHashes(walletServers []string, stopper stop.Chan) (map[string]valO
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
defer func() {
|
||||
err := resp.Body.Close()
|
||||
if err != nil {
|
||||
log.Errorln(errors.Err(err))
|
||||
closeErr := resp.Body.Close()
|
||||
if closeErr != nil {
|
||||
log.Errorln(errors.Err(closeErr))
|
||||
}
|
||||
}()
|
||||
|
||||
var r struct {
|
||||
Success bool `json:"success"`
|
||||
Error string `json:"error"`
|
||||
Data struct {
|
||||
Error string `json:"error"`
|
||||
Data struct {
|
||||
Outpoints []string `json:"outpoints"`
|
||||
} `json:"data"`
|
||||
Success bool `json:"success"`
|
||||
}
|
||||
|
||||
if err = json.NewDecoder(resp.Body).Decode(&r); err != nil {
|
||||
@@ -95,8 +95,8 @@ func blockedSdHashes(walletServers []string, stopper stop.Chan) (map[string]valO
|
||||
}
|
||||
|
||||
type valOrErr struct {
|
||||
Value string
|
||||
Err error
|
||||
Value string
|
||||
}
|
||||
|
||||
// sdHashesForOutpoints queries wallet server for the sd hashes in a given outpoints
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
package reflector
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
|
||||
"github.com/bluele/gcache"
|
||||
"golang.org/x/sync/singleflight"
|
||||
)
|
||||
|
||||
const protectedListURL = "https://direct.api.odysee.com/file/list_protected"
|
||||
|
||||
type ProtectedContent struct {
|
||||
SDHash string `json:"sd_hash"`
|
||||
ClaimID string `json:"claim_id"`
|
||||
}
|
||||
|
||||
var protectedCache = gcache.New(10).Expiration(2 * time.Minute).Build()
|
||||
|
||||
func GetProtectedContent() (interface{}, error) {
|
||||
cachedVal, err := protectedCache.Get("protected")
|
||||
if err == nil && cachedVal != nil {
|
||||
return cachedVal.(map[string]bool), nil
|
||||
}
|
||||
|
||||
method := "GET"
|
||||
var r struct {
|
||||
Error string `json:"error"`
|
||||
Data []ProtectedContent `json:"data"`
|
||||
Success bool `json:"success"`
|
||||
}
|
||||
|
||||
// Bound the request to avoid hanging the entire request path.
|
||||
// Without a timeout, a slow or unreachable endpoint can block
|
||||
// singleflight callers indefinitely and stall HTTP handlers.
|
||||
client := &http.Client{Timeout: 5 * time.Second}
|
||||
req, err := http.NewRequest(method, protectedListURL, nil)
|
||||
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
defer func() { _ = res.Body.Close() }()
|
||||
if res.StatusCode != http.StatusOK {
|
||||
return nil, errors.Err("unexpected status code %d", res.StatusCode)
|
||||
}
|
||||
if err = json.NewDecoder(res.Body).Decode(&r); err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
if !r.Success {
|
||||
return nil, errors.Prefix("file/list_protected API call", r.Error)
|
||||
}
|
||||
|
||||
protectedMap := make(map[string]bool, len(r.Data))
|
||||
for _, pc := range r.Data {
|
||||
protectedMap[pc.SDHash] = true
|
||||
}
|
||||
err = protectedCache.Set("protected", protectedMap)
|
||||
if err != nil {
|
||||
return protectedMap, errors.Err(err)
|
||||
}
|
||||
return protectedMap, nil
|
||||
}
|
||||
|
||||
var sf = singleflight.Group{}
|
||||
|
||||
func IsProtected(sdHash string) bool {
|
||||
val, err, _ := sf.Do("protected", GetProtectedContent)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
cachedMap, ok := val.(map[string]bool)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
return cachedMap[sdHash]
|
||||
}
|
||||
+22
-28
@@ -6,7 +6,6 @@ import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"time"
|
||||
|
||||
@@ -36,22 +35,21 @@ var ErrBlobTooBig = errors.Base("blob must be at most %d bytes", maxBlobSize)
|
||||
|
||||
// Server is and instance of the reflector server. It houses the blob store and listener.
|
||||
type Server struct {
|
||||
store store.BlobStore
|
||||
grp *stop.Group
|
||||
Timeout time.Duration // timeout to read or write next message
|
||||
|
||||
EnableBlocklist bool // if true, blocklist checking and blob deletion will be enabled
|
||||
|
||||
underlyingStore store.BlobStore
|
||||
outerStore store.BlobStore
|
||||
grp *stop.Group
|
||||
//underlyingStore store.BlobStore
|
||||
//outerStore store.BlobStore
|
||||
}
|
||||
|
||||
// NewServer returns an initialized reflector server pointer.
|
||||
func NewServer(underlying store.BlobStore, outer store.BlobStore) *Server {
|
||||
func NewIngestionServer(store store.BlobStore) *Server {
|
||||
return &Server{
|
||||
Timeout: DefaultTimeout,
|
||||
underlyingStore: underlying,
|
||||
outerStore: outer,
|
||||
grp: stop.New(),
|
||||
Timeout: DefaultTimeout,
|
||||
store: store,
|
||||
grp: stop.New(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -62,7 +60,7 @@ func (s *Server) Shutdown() {
|
||||
log.Println("reflector server stopped")
|
||||
}
|
||||
|
||||
//Start starts the server to handle connections.
|
||||
// Start starts the server to handle connections.
|
||||
func (s *Server) Start(address string) error {
|
||||
l, err := net.Listen(network, address)
|
||||
if err != nil {
|
||||
@@ -90,7 +88,7 @@ func (s *Server) Start(address string) error {
|
||||
}()
|
||||
|
||||
if s.EnableBlocklist {
|
||||
if b, ok := s.underlyingStore.(store.Blocklister); ok {
|
||||
if b, ok := s.store.(store.Blocklister); ok {
|
||||
s.grp.Add(1)
|
||||
metrics.RoutinesQueue.WithLabelValues("reflector", "enableblocklist").Inc()
|
||||
go func() {
|
||||
@@ -153,7 +151,7 @@ func (s *Server) handleConn(conn net.Conn) {
|
||||
if errors.Is(err, io.EOF) || s.quitting() {
|
||||
return
|
||||
}
|
||||
err := s.doError(conn, err)
|
||||
err = s.doError(conn, err)
|
||||
if err != nil {
|
||||
log.Error(errors.Prefix("sending handshake error", err))
|
||||
}
|
||||
@@ -166,7 +164,7 @@ func (s *Server) handleConn(conn net.Conn) {
|
||||
if errors.Is(err, io.EOF) || s.quitting() {
|
||||
return
|
||||
}
|
||||
err := s.doError(conn, err)
|
||||
err = s.doError(conn, err)
|
||||
if err != nil {
|
||||
log.Error(errors.Prefix("sending blob receive error", err))
|
||||
}
|
||||
@@ -201,13 +199,14 @@ func (s *Server) receiveBlob(conn net.Conn) error {
|
||||
}
|
||||
|
||||
var wantsBlob bool
|
||||
if bl, ok := s.underlyingStore.(store.Blocklister); ok {
|
||||
if bl, ok := s.store.(store.Blocklister); ok {
|
||||
wantsBlob, err = bl.Wants(blobHash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
blobExists, err := s.underlyingStore.Has(blobHash)
|
||||
var blobExists bool
|
||||
blobExists, err = s.store.Has(blobHash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -217,14 +216,14 @@ func (s *Server) receiveBlob(conn net.Conn) error {
|
||||
var neededBlobs []string
|
||||
|
||||
if isSdBlob && !wantsBlob {
|
||||
if nbc, ok := s.underlyingStore.(neededBlobChecker); ok {
|
||||
if nbc, ok := s.store.(store.NeededBlobChecker); ok {
|
||||
neededBlobs, err = nbc.MissingBlobsForKnownStream(blobHash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
// if we can't check for blobs in a stream, we have to say that the sd blob is
|
||||
// missing. if we say we have the sd blob, they wont try to send any content blobs
|
||||
// missing. if we say we have the sd blob, they won't try to send any content blobs
|
||||
wantsBlob = true
|
||||
}
|
||||
}
|
||||
@@ -260,9 +259,9 @@ func (s *Server) receiveBlob(conn net.Conn) error {
|
||||
log.Debugln("Got blob " + blobHash[:8])
|
||||
|
||||
if isSdBlob {
|
||||
err = s.outerStore.PutSD(blobHash, blob)
|
||||
err = s.store.PutSD(blobHash, blob)
|
||||
} else {
|
||||
err = s.outerStore.Put(blobHash, blob)
|
||||
err = s.store.Put(blobHash, blob)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -367,7 +366,7 @@ func (s *Server) read(conn net.Conn, v interface{}) error {
|
||||
dec := json.NewDecoder(conn)
|
||||
err = dec.Decode(v)
|
||||
if err != nil {
|
||||
data, _ := ioutil.ReadAll(dec.Buffered())
|
||||
data, _ := io.ReadAll(dec.Buffered())
|
||||
if len(data) > 0 {
|
||||
return errors.Err("%s. Data: %s", err.Error(), hex.EncodeToString(data))
|
||||
}
|
||||
@@ -430,8 +429,8 @@ type handshakeRequestResponse struct {
|
||||
|
||||
type sendBlobRequest struct {
|
||||
BlobHash string `json:"blob_hash,omitempty"`
|
||||
BlobSize int `json:"blob_size,omitempty"`
|
||||
SdBlobHash string `json:"sd_blob_hash,omitempty"`
|
||||
BlobSize int `json:"blob_size,omitempty"`
|
||||
SdBlobSize int `json:"sd_blob_size,omitempty"`
|
||||
}
|
||||
|
||||
@@ -440,8 +439,8 @@ type sendBlobResponse struct {
|
||||
}
|
||||
|
||||
type sendSdBlobResponse struct {
|
||||
SendSdBlob bool `json:"send_sd_blob"`
|
||||
NeededBlobs []string `json:"needed_blobs,omitempty"`
|
||||
SendSdBlob bool `json:"send_sd_blob"`
|
||||
}
|
||||
|
||||
type blobTransferResponse struct {
|
||||
@@ -451,8 +450,3 @@ type blobTransferResponse struct {
|
||||
type sdBlobTransferResponse struct {
|
||||
ReceivedSdBlob bool `json:"received_sd_blob"`
|
||||
}
|
||||
|
||||
// neededBlobChecker can check which blobs from a known stream are not uploaded yet
|
||||
type neededBlobChecker interface {
|
||||
MissingBlobsForKnownStream(string) ([]string, error)
|
||||
}
|
||||
@@ -23,7 +23,7 @@ func startServerOnRandomPort(t *testing.T) (*Server, int) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
srv := NewServer(store.NewMemStore(), store.NewMemStore())
|
||||
srv := NewIngestionServer(store.NewMemStore(store.MemParams{Name: "test"}))
|
||||
err = srv.Start("127.0.0.1:" + strconv.Itoa(port))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -120,7 +120,7 @@ func TestServer_Timeout(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
srv := NewServer(store.NewMemStore(), store.NewMemStore())
|
||||
srv := NewIngestionServer(store.NewMemStore(store.MemParams{Name: "test"}))
|
||||
srv.Timeout = testTimeout
|
||||
err = srv.Start("127.0.0.1:" + strconv.Itoa(port))
|
||||
if err != nil {
|
||||
@@ -182,8 +182,8 @@ func TestServer_PartialUpload(t *testing.T) {
|
||||
missing[i] = bits.Rand().String()
|
||||
}
|
||||
|
||||
st := store.BlobStore(&mockPartialStore{MemStore: store.NewMemStore(), missing: missing})
|
||||
if _, ok := st.(neededBlobChecker); !ok {
|
||||
st := store.BlobStore(&mockPartialStore{MemStore: store.NewMemStore(store.MemParams{Name: "test"}), missing: missing})
|
||||
if _, ok := st.(store.NeededBlobChecker); !ok {
|
||||
t.Fatal("mock does not implement the relevant interface")
|
||||
}
|
||||
err = st.Put(sdHash, randBlob(10))
|
||||
@@ -191,7 +191,7 @@ func TestServer_PartialUpload(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
srv := NewServer(st, st)
|
||||
srv := NewIngestionServer(st)
|
||||
err = srv.Start("127.0.0.1:" + strconv.Itoa(port))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
+8
-10
@@ -1,7 +1,6 @@
|
||||
package reflector
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path"
|
||||
"sync"
|
||||
@@ -31,17 +30,16 @@ type Summary struct {
|
||||
|
||||
type Uploader struct {
|
||||
db *db.SQL
|
||||
store *store.DBBackedStore // could just be store.BlobStore interface
|
||||
store store.BlobStore
|
||||
stopper *stop.Group
|
||||
countChan chan increment
|
||||
count Summary
|
||||
workers int
|
||||
skipExistsCheck bool
|
||||
deleteBlobsAfterUpload bool
|
||||
stopper *stop.Group
|
||||
countChan chan increment
|
||||
|
||||
count Summary
|
||||
}
|
||||
|
||||
func NewUploader(db *db.SQL, store *store.DBBackedStore, workers int, skipExistsCheck, deleteBlobsAfterUpload bool) *Uploader {
|
||||
func NewUploader(db *db.SQL, store store.BlobStore, workers int, skipExistsCheck, deleteBlobsAfterUpload bool) *Uploader {
|
||||
return &Uploader{
|
||||
db: db,
|
||||
store: store,
|
||||
@@ -133,7 +131,7 @@ Upload:
|
||||
return nil
|
||||
}
|
||||
|
||||
// worker reads paths from a channel, uploads them, and optionally deletes them
|
||||
// worker reads paths from a channel, uploads them, and optionally deletes them
|
||||
func (u *Uploader) worker(pathChan chan string) {
|
||||
for {
|
||||
select {
|
||||
@@ -165,7 +163,7 @@ func (u *Uploader) uploadBlob(filepath string) (err error) {
|
||||
}
|
||||
}()
|
||||
|
||||
blob, err := ioutil.ReadFile(filepath)
|
||||
blob, err := os.ReadFile(filepath)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
@@ -177,7 +175,7 @@ func (u *Uploader) uploadBlob(filepath string) (err error) {
|
||||
|
||||
if IsValidJSON(blob) {
|
||||
log.Debugf("uploading SD blob %s", hash)
|
||||
err := u.store.PutSD(hash, blob)
|
||||
err = u.store.PutSD(hash, blob)
|
||||
if err != nil {
|
||||
return errors.Prefix("uploading SD blob "+hash, err)
|
||||
}
|
||||
|
||||
Executable
+16
@@ -0,0 +1,16 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
cd "$SCRIPT_DIR/.."
|
||||
|
||||
if ! command -v golangci-lint &> /dev/null; then
|
||||
echo "Installing golangci-lint..."
|
||||
go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
|
||||
else
|
||||
echo "Ensuring golangci-lint is up to date..."
|
||||
go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
|
||||
fi
|
||||
|
||||
echo "Running golangci-lint..."
|
||||
golangci-lint run --config .golangci.yml ./...
|
||||
@@ -2,16 +2,19 @@ package http
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func (s *Server) getBlob(c *gin.Context) {
|
||||
@@ -22,9 +25,20 @@ func (s *Server) getBlob(c *gin.Context) {
|
||||
}
|
||||
|
||||
func (s *Server) HandleGetBlob(c *gin.Context) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Errorf("Recovered from panic: %v", r)
|
||||
}
|
||||
}()
|
||||
start := time.Now()
|
||||
hash := c.Query("hash")
|
||||
edgeToken := c.Query("edge_token")
|
||||
|
||||
if reflector.IsProtected(hash) && edgeToken != s.edgeToken {
|
||||
_ = c.Error(errors.Err("requested blob is protected"))
|
||||
c.String(http.StatusForbidden, "requested blob is protected")
|
||||
return
|
||||
}
|
||||
if s.missesCache.Has(hash) {
|
||||
serialized, err := shared.NewBlobTrace(time.Since(start), "http").Serialize()
|
||||
c.Header("Via", serialized)
|
||||
@@ -66,6 +80,7 @@ func (s *Server) HandleGetBlob(c *gin.Context) {
|
||||
metrics.HttpDownloadCount.Inc()
|
||||
c.Header("Via", serialized)
|
||||
c.Header("Content-Disposition", "filename="+hash)
|
||||
c.Header("Content-Length", strconv.Itoa(len(blob)))
|
||||
c.Data(http.StatusOK, "application/octet-stream", blob)
|
||||
}
|
||||
|
||||
|
||||
+13
-8
@@ -19,17 +19,21 @@ import (
|
||||
type Server struct {
|
||||
store store.BlobStore
|
||||
grp *stop.Group
|
||||
concurrentRequests int
|
||||
missesCache gcache.Cache
|
||||
edgeToken string
|
||||
address string
|
||||
concurrentRequests int
|
||||
}
|
||||
|
||||
// NewServer returns an initialized Server pointer.
|
||||
func NewServer(store store.BlobStore, requestQueueSize int) *Server {
|
||||
func NewServer(store store.BlobStore, requestQueueSize int, edgeToken string, address string) *Server {
|
||||
return &Server{
|
||||
store: store,
|
||||
grp: stop.New(),
|
||||
concurrentRequests: requestQueueSize,
|
||||
missesCache: gcache.New(2000).Expiration(5 * time.Minute).ARC().Build(),
|
||||
edgeToken: edgeToken,
|
||||
address: address,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -41,15 +45,16 @@ func (s *Server) Shutdown() {
|
||||
}
|
||||
|
||||
// Start starts the server listener to handle connections.
|
||||
func (s *Server) Start(address string) error {
|
||||
func (s *Server) Start() error {
|
||||
gin.SetMode(gin.ReleaseMode)
|
||||
router := gin.Default()
|
||||
router.GET("/blob", s.getBlob)
|
||||
router.HEAD("/blob", s.hasBlob)
|
||||
router := gin.New()
|
||||
router.Use(gin.Logger())
|
||||
// Install nice.Recovery, passing the handler to call after recovery
|
||||
router.Use(nice.Recovery(s.recoveryHandler))
|
||||
router.GET("/blob", s.getBlob)
|
||||
router.HEAD("/blob", s.hasBlob)
|
||||
srv := &http.Server{
|
||||
Addr: address,
|
||||
Addr: s.address,
|
||||
Handler: router,
|
||||
}
|
||||
go s.listenForShutdown(srv)
|
||||
@@ -59,7 +64,7 @@ func (s *Server) Start(address string) error {
|
||||
s.grp.Add(1)
|
||||
go func() {
|
||||
defer s.grp.Done()
|
||||
log.Println("HTTP server listening on " + address)
|
||||
log.Println("HTTP server listening on " + s.address)
|
||||
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||
log.Fatalf("listen: %s\n", err)
|
||||
}
|
||||
|
||||
@@ -16,16 +16,16 @@ import (
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
"github.com/lucas-clemente/quic-go/http3"
|
||||
"github.com/quic-go/quic-go/http3"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// Client is an instance of a client connected to a server.
|
||||
type Client struct {
|
||||
Timeout time.Duration
|
||||
conn *http.Client
|
||||
roundTripper *http3.RoundTripper
|
||||
roundTripper *http3.Transport
|
||||
ServerAddr string
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
// Close closes the connection with the client.
|
||||
@@ -69,7 +69,7 @@ func (c *Client) HasBlob(hash string) (bool, error) {
|
||||
if err != nil {
|
||||
return false, errors.Err(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode == http.StatusOK {
|
||||
return true, nil
|
||||
}
|
||||
@@ -86,7 +86,7 @@ func (c *Client) GetBlob(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode == http.StatusNotFound {
|
||||
fmt.Printf("%s blob not found %d\n", hash, resp.StatusCode)
|
||||
@@ -100,7 +100,8 @@ func (c *Client) GetBlob(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
serialized := resp.Header.Get("Via")
|
||||
trace := shared.NewBlobTrace(time.Since(start), "http3")
|
||||
if serialized != "" {
|
||||
parsedTrace, err := shared.Deserialize(serialized)
|
||||
var parsedTrace *shared.BlobTrace
|
||||
parsedTrace, err = shared.Deserialize(serialized)
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), "http3"), err
|
||||
}
|
||||
|
||||
+20
-14
@@ -11,18 +11,20 @@ import (
|
||||
"math/big"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
"github.com/lbryio/reflector.go/reflector"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/lucas-clemente/quic-go"
|
||||
"github.com/lucas-clemente/quic-go/http3"
|
||||
"github.com/quic-go/quic-go"
|
||||
"github.com/quic-go/quic-go/http3"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
@@ -30,15 +32,17 @@ import (
|
||||
type Server struct {
|
||||
store store.BlobStore
|
||||
grp *stop.Group
|
||||
address string
|
||||
concurrentRequests int
|
||||
}
|
||||
|
||||
// NewServer returns an initialized Server pointer.
|
||||
func NewServer(store store.BlobStore, requestQueueSize int) *Server {
|
||||
func NewServer(store store.BlobStore, requestQueueSize int, address string) *Server {
|
||||
return &Server{
|
||||
store: store,
|
||||
grp: stop.New(),
|
||||
concurrentRequests: requestQueueSize,
|
||||
address: address,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,8 +69,8 @@ type availabilityResponse struct {
|
||||
}
|
||||
|
||||
// Start starts the server listener to handle connections.
|
||||
func (s *Server) Start(address string) error {
|
||||
log.Println("HTTP3 peer listening on " + address)
|
||||
func (s *Server) Start() error {
|
||||
log.Println("HTTP3 peer listening on " + s.address)
|
||||
window500M := 500 * 1 << 20
|
||||
|
||||
quicConf := &quic.Config{
|
||||
@@ -112,12 +116,10 @@ func (s *Server) Start(address string) error {
|
||||
}
|
||||
})
|
||||
server := http3.Server{
|
||||
Server: &http.Server{
|
||||
Handler: r,
|
||||
Addr: address,
|
||||
TLSConfig: generateTLSConfig(),
|
||||
},
|
||||
QuicConfig: quicConf,
|
||||
Addr: s.address,
|
||||
Handler: r,
|
||||
TLSConfig: generateTLSConfig(),
|
||||
QUICConfig: quicConf,
|
||||
}
|
||||
go InitWorkers(s, s.concurrentRequests)
|
||||
go s.listenForShutdown(&server)
|
||||
@@ -156,7 +158,7 @@ func generateTLSConfig() *tls.Config {
|
||||
|
||||
func (s *Server) listenAndServe(server *http3.Server) {
|
||||
err := server.ListenAndServe()
|
||||
if err != nil && err.Error() != "server closed" {
|
||||
if err != nil && !strings.Contains(err.Error(), "Server closed") {
|
||||
log.Errorln(errors.FullTrace(err))
|
||||
}
|
||||
}
|
||||
@@ -181,11 +183,15 @@ func (s *Server) HandleGetBlob(w http.ResponseWriter, r *http.Request) {
|
||||
wantsTrace = false
|
||||
}
|
||||
}
|
||||
|
||||
if reflector.IsProtected(requestedBlob) {
|
||||
http.Error(w, "requested blob is protected", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
blob, trace, err := s.store.Get(requestedBlob)
|
||||
|
||||
if wantsTrace {
|
||||
serialized, err := trace.Serialize()
|
||||
var serialized string
|
||||
serialized, err = trace.Serialize()
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusNotFound)
|
||||
return
|
||||
|
||||
@@ -1,117 +0,0 @@
|
||||
package http3
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
"github.com/lucas-clemente/quic-go"
|
||||
"github.com/lucas-clemente/quic-go/http3"
|
||||
)
|
||||
|
||||
// Store is a blob store that gets blobs from a peer.
|
||||
// It satisfies the store.BlobStore interface but cannot put or delete blobs.
|
||||
type Store struct {
|
||||
opts StoreOpts
|
||||
NotFoundCache *sync.Map
|
||||
}
|
||||
|
||||
// StoreOpts allows to set options for a new Store.
|
||||
type StoreOpts struct {
|
||||
Address string
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
// NewStore makes a new peer store.
|
||||
func NewStore(opts StoreOpts) *Store {
|
||||
return &Store{opts: opts, NotFoundCache: &sync.Map{}}
|
||||
}
|
||||
|
||||
func (p *Store) getClient() (*Client, error) {
|
||||
var qconf quic.Config
|
||||
window500M := 500 * 1 << 20
|
||||
qconf.MaxStreamReceiveWindow = uint64(window500M)
|
||||
qconf.MaxConnectionReceiveWindow = uint64(window500M)
|
||||
qconf.EnableDatagrams = true
|
||||
qconf.HandshakeIdleTimeout = 4 * time.Second
|
||||
qconf.MaxIdleTimeout = 20 * time.Second
|
||||
pool, err := x509.SystemCertPool()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
roundTripper := &http3.RoundTripper{
|
||||
TLSClientConfig: &tls.Config{
|
||||
RootCAs: pool,
|
||||
InsecureSkipVerify: true,
|
||||
},
|
||||
QuicConfig: &qconf,
|
||||
}
|
||||
connection := &http.Client{
|
||||
Transport: roundTripper,
|
||||
}
|
||||
c := &Client{
|
||||
conn: connection,
|
||||
roundTripper: roundTripper,
|
||||
ServerAddr: p.opts.Address,
|
||||
}
|
||||
return c, errors.Prefix("connection error", err)
|
||||
}
|
||||
|
||||
func (p *Store) Name() string { return "http3" }
|
||||
|
||||
// Has asks the peer if they have a hash
|
||||
func (p *Store) Has(hash string) (bool, error) {
|
||||
c, err := p.getClient()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer c.Close()
|
||||
return c.HasBlob(hash)
|
||||
}
|
||||
|
||||
// Get downloads the blob from the peer
|
||||
func (p *Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
start := time.Now()
|
||||
if lastChecked, ok := p.NotFoundCache.Load(hash); ok {
|
||||
if lastChecked.(time.Time).After(time.Now().Add(-5 * time.Minute)) {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), p.Name()+"-notfoundcache"), store.ErrBlobNotFound
|
||||
}
|
||||
}
|
||||
c, err := p.getClient()
|
||||
if err != nil && strings.Contains(err.Error(), "blob not found") {
|
||||
p.NotFoundCache.Store(hash, time.Now())
|
||||
}
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), p.Name()), err
|
||||
}
|
||||
defer c.Close()
|
||||
return c.GetBlob(hash)
|
||||
}
|
||||
|
||||
// Put is not supported
|
||||
func (p *Store) Put(hash string, blob stream.Blob) error {
|
||||
return errors.Err(shared.ErrNotImplemented)
|
||||
}
|
||||
|
||||
// PutSD is not supported
|
||||
func (p *Store) PutSD(hash string, blob stream.Blob) error {
|
||||
return errors.Err(shared.ErrNotImplemented)
|
||||
}
|
||||
|
||||
// Delete is not supported
|
||||
func (p *Store) Delete(hash string) error {
|
||||
return errors.Err(shared.ErrNotImplemented)
|
||||
}
|
||||
|
||||
// Shutdown is not supported
|
||||
func (p *Store) Shutdown() {
|
||||
}
|
||||
@@ -20,9 +20,9 @@ import (
|
||||
|
||||
// Client is an instance of a client connected to a server.
|
||||
type Client struct {
|
||||
Timeout time.Duration
|
||||
conn net.Conn
|
||||
buf *bufio.Reader
|
||||
Timeout time.Duration
|
||||
connected bool
|
||||
}
|
||||
|
||||
|
||||
+64
-46
@@ -31,19 +31,18 @@ const (
|
||||
|
||||
// Server is an instance of a peer server that houses the listener and store.
|
||||
type Server struct {
|
||||
store store.BlobStore
|
||||
disk *store.DiskStore
|
||||
closed bool
|
||||
|
||||
grp *stop.Group
|
||||
store store.BlobStore
|
||||
grp *stop.Group
|
||||
address string
|
||||
closed bool
|
||||
}
|
||||
|
||||
// NewServer returns an initialized Server pointer.
|
||||
func NewServer(blobstore store.BlobStore) *Server {
|
||||
func NewServer(store store.BlobStore, address string) *Server {
|
||||
return &Server{
|
||||
store: blobstore,
|
||||
disk: store.NewDiskStore("/external/blobs", 2),
|
||||
grp: stop.New(),
|
||||
store: store,
|
||||
grp: stop.New(),
|
||||
address: address,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,9 +54,9 @@ func (s *Server) Shutdown() {
|
||||
}
|
||||
|
||||
// Start starts the server listener to handle connections.
|
||||
func (s *Server) Start(address string) error {
|
||||
log.Println("peer listening on " + address)
|
||||
l, err := net.Listen("tcp4", address)
|
||||
func (s *Server) Start() error {
|
||||
log.Println("peer listening on " + s.address)
|
||||
l, err := net.Listen("tcp4", s.address)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -133,7 +132,7 @@ func (s *Server) handleConnection(conn net.Conn) {
|
||||
log.Error(errors.FullTrace(err))
|
||||
}
|
||||
|
||||
response, blobHash, err := s.handleCompositeRequest(request)
|
||||
response, err = s.handleCompositeRequest(request)
|
||||
if err != nil {
|
||||
log.Error(errors.FullTrace(err))
|
||||
return
|
||||
@@ -145,7 +144,6 @@ func (s *Server) handleConnection(conn net.Conn) {
|
||||
}
|
||||
|
||||
n, err := conn.Write(response)
|
||||
s.disk.Sendfile(blobHash, conn)
|
||||
if err != nil {
|
||||
if !strings.Contains(err.Error(), "connection reset by peer") { // means the other side closed the connection using TCP reset
|
||||
s.logError(err)
|
||||
@@ -224,70 +222,79 @@ func (s *Server) handleAvailabilityRequest(data []byte) ([]byte, error) {
|
||||
// return append(response, blob...), nil
|
||||
//}
|
||||
|
||||
func (s *Server) handleCompositeRequest(data []byte) ([]byte, string, error) {
|
||||
func (s *Server) handleCompositeRequest(data []byte) ([]byte, error) {
|
||||
var request compositeRequest
|
||||
err := json.Unmarshal(data, &request)
|
||||
if err != nil {
|
||||
var je *json.SyntaxError
|
||||
if ee.As(err, &je) {
|
||||
return nil, "", errors.Err("invalid json at offset %d in data %s", je.Offset, hex.EncodeToString(data))
|
||||
return nil, errors.Err("invalid json request: offset %d in data %s", je.Offset, hex.EncodeToString(data))
|
||||
}
|
||||
return nil, "", errors.Err(err)
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
response := compositeResponse{
|
||||
LbrycrdAddress: LbrycrdAddress,
|
||||
AvailableBlobs: []string{},
|
||||
}
|
||||
|
||||
if len(request.RequestedBlobs) > 0 {
|
||||
var availableBlobs []string
|
||||
for _, blobHash := range request.RequestedBlobs {
|
||||
exists, err := s.store.Has(blobHash)
|
||||
if reflector.IsProtected(blobHash) {
|
||||
return nil, errors.Err("requested blob is protected")
|
||||
}
|
||||
var exists bool
|
||||
exists, err = s.store.Has(blobHash)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
return nil, err
|
||||
}
|
||||
if exists {
|
||||
availableBlobs = append(availableBlobs, blobHash)
|
||||
response.AvailableBlobs = append(response.AvailableBlobs, blobHash)
|
||||
}
|
||||
}
|
||||
response.AvailableBlobs = availableBlobs
|
||||
}
|
||||
|
||||
response.BlobDataPaymentRate = paymentRateAccepted
|
||||
if request.BlobDataPaymentRate < 0 {
|
||||
response.BlobDataPaymentRate = paymentRateTooLow
|
||||
if request.BlobDataPaymentRate != nil {
|
||||
response.BlobDataPaymentRate = paymentRateAccepted
|
||||
if *request.BlobDataPaymentRate < 0 {
|
||||
response.BlobDataPaymentRate = paymentRateTooLow
|
||||
}
|
||||
}
|
||||
|
||||
var blob []byte
|
||||
var trace shared.BlobTrace
|
||||
if request.RequestedBlob != "" {
|
||||
if len(request.RequestedBlob) != stream.BlobHashHexLength {
|
||||
return nil, "", errors.Err("Invalid blob hash length")
|
||||
return nil, errors.Err("Invalid blob hash length")
|
||||
}
|
||||
|
||||
log.Debugln("Sending blob " + request.RequestedBlob[:8])
|
||||
|
||||
size, err := s.disk.Size(request.RequestedBlob)
|
||||
if err != nil {
|
||||
response.IncomingBlob = incomingBlob{
|
||||
blob, trace, err = s.store.Get(request.RequestedBlob)
|
||||
log.Debug(trace.String())
|
||||
if errors.Is(err, store.ErrBlobNotFound) {
|
||||
response.IncomingBlob = &incomingBlob{
|
||||
Error: err.Error(),
|
||||
}
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
response.IncomingBlob = incomingBlob{
|
||||
response.IncomingBlob = &incomingBlob{
|
||||
BlobHash: request.RequestedBlob,
|
||||
Length: int(size),
|
||||
Length: len(blob),
|
||||
}
|
||||
metrics.MtrOutBytesTcp.Add(float64(len(blob)))
|
||||
metrics.BlobDownloadCount.Inc()
|
||||
metrics.PeerDownloadCount.Inc()
|
||||
}
|
||||
metrics.MtrOutBytesTcp.Add(float64(len(blob)))
|
||||
metrics.BlobDownloadCount.Inc()
|
||||
metrics.PeerDownloadCount.Inc()
|
||||
}
|
||||
|
||||
respData, err := json.Marshal(response)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return respData, request.RequestedBlob, nil
|
||||
return append(respData, blob...), nil
|
||||
}
|
||||
|
||||
func (s *Server) logError(e error) {
|
||||
@@ -301,7 +308,15 @@ func (s *Server) logError(e error) {
|
||||
}
|
||||
|
||||
func readNextMessage(buf *bufio.Reader) ([]byte, error) {
|
||||
msg := make([]byte, 0)
|
||||
firstByte, err := buf.ReadByte()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if firstByte != '{' {
|
||||
// every request starts with '{'. Checking here disconnects earlier, so we don't wait until timeout
|
||||
return nil, errInvalidData
|
||||
}
|
||||
msg := []byte("{")
|
||||
eof := false
|
||||
|
||||
for {
|
||||
@@ -322,6 +337,8 @@ func readNextMessage(buf *bufio.Reader) ([]byte, error) {
|
||||
|
||||
if len(msg) > maxRequestSize {
|
||||
return msg, errRequestTooLarge
|
||||
} else if len(msg) > 0 && msg[0] != '{' {
|
||||
return msg, errInvalidData
|
||||
}
|
||||
|
||||
// yes, this is how the peer protocol knows when the request finishes
|
||||
@@ -356,10 +373,11 @@ const (
|
||||
)
|
||||
|
||||
var errRequestTooLarge = errors.Base("request is too large")
|
||||
var errInvalidData = errors.Base("Invalid data")
|
||||
|
||||
type availabilityRequest struct {
|
||||
LbrycrdAddress bool `json:"lbrycrd_address"`
|
||||
RequestedBlobs []string `json:"requested_blobs"`
|
||||
LbrycrdAddress bool `json:"lbrycrd_address"`
|
||||
}
|
||||
|
||||
type availabilityResponse struct {
|
||||
@@ -385,20 +403,20 @@ type incomingBlob struct {
|
||||
Length int `json:"length"`
|
||||
}
|
||||
type blobResponse struct {
|
||||
IncomingBlob incomingBlob `json:"incoming_blob"`
|
||||
RequestTrace *shared.BlobTrace
|
||||
IncomingBlob incomingBlob `json:"incoming_blob"`
|
||||
}
|
||||
|
||||
type compositeRequest struct {
|
||||
LbrycrdAddress bool `json:"lbrycrd_address"`
|
||||
RequestedBlobs []string `json:"requested_blobs"`
|
||||
BlobDataPaymentRate float64 `json:"blob_data_payment_rate"`
|
||||
BlobDataPaymentRate *float64 `json:"blob_data_payment_rate"`
|
||||
RequestedBlob string `json:"requested_blob"`
|
||||
RequestedBlobs []string `json:"requested_blobs"`
|
||||
LbrycrdAddress bool `json:"lbrycrd_address"`
|
||||
}
|
||||
|
||||
type compositeResponse struct {
|
||||
LbrycrdAddress string `json:"lbrycrd_address,omitempty"`
|
||||
AvailableBlobs []string `json:"available_blobs,omitempty"`
|
||||
BlobDataPaymentRate string `json:"blob_data_payment_rate,omitempty"`
|
||||
IncomingBlob incomingBlob `json:"incoming_blob,omitempty"`
|
||||
IncomingBlob *incomingBlob `json:"incoming_blob,omitempty"`
|
||||
LbrycrdAddress string `json:"lbrycrd_address,omitempty"`
|
||||
BlobDataPaymentRate string `json:"blob_data_payment_rate,omitempty"`
|
||||
AvailableBlobs []string `json:"available_blobs"`
|
||||
}
|
||||
@@ -2,7 +2,10 @@ package peer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"net"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
)
|
||||
@@ -34,7 +37,9 @@ var availabilityRequests = []pair{
|
||||
}
|
||||
|
||||
func getServer(t *testing.T, withBlobs bool) *Server {
|
||||
st := store.NewMemStore()
|
||||
st := store.NewMemStore(store.MemParams{
|
||||
Name: "test",
|
||||
})
|
||||
if withBlobs {
|
||||
for k, v := range blobs {
|
||||
err := st.Put(k, v)
|
||||
@@ -43,7 +48,7 @@ func getServer(t *testing.T, withBlobs bool) *Server {
|
||||
}
|
||||
}
|
||||
}
|
||||
return NewServer(st)
|
||||
return NewServer(st, "127.0.0.1:50505")
|
||||
}
|
||||
|
||||
func TestAvailabilityRequest_NoBlobs(t *testing.T) {
|
||||
@@ -75,3 +80,62 @@ func TestAvailabilityRequest_WithBlobs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestFromConnection(t *testing.T) {
|
||||
s := getServer(t, true)
|
||||
err := s.Start()
|
||||
defer s.Shutdown()
|
||||
if err != nil {
|
||||
t.Error("error starting server", err)
|
||||
}
|
||||
|
||||
for _, p := range availabilityRequests {
|
||||
conn, err := net.Dial("tcp", "127.0.0.1:50505")
|
||||
if err != nil {
|
||||
t.Error("error opening connection", err)
|
||||
}
|
||||
defer func() { _ = conn.Close() }()
|
||||
|
||||
response := make([]byte, 8192)
|
||||
_, err = conn.Write(p.request)
|
||||
if err != nil {
|
||||
t.Error("error writing", err)
|
||||
}
|
||||
_, err = conn.Read(response)
|
||||
if err != nil {
|
||||
t.Error("error reading", err)
|
||||
}
|
||||
if !bytes.Equal(response[:len(p.response)], p.response) {
|
||||
t.Errorf("Response did not match expected response.\nExpected: %s\nGot: %s", string(p.response), string(response))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvalidData(t *testing.T) {
|
||||
s := getServer(t, true)
|
||||
err := s.Start()
|
||||
defer s.Shutdown()
|
||||
if err != nil {
|
||||
t.Error("error starting server", err)
|
||||
}
|
||||
conn, err := net.Dial("tcp", "127.0.0.1:50505")
|
||||
if err != nil {
|
||||
t.Error("error opening connection", err)
|
||||
}
|
||||
defer func() { _ = conn.Close() }()
|
||||
|
||||
response := make([]byte, 8192)
|
||||
_, err = conn.Write([]byte("hello dear server, I would like blobs. Please"))
|
||||
if err != nil {
|
||||
t.Error("error writing", err)
|
||||
}
|
||||
err = conn.SetReadDeadline(time.Now().Add(5 * time.Second))
|
||||
if err != nil {
|
||||
t.Error("error setting read deadline", err)
|
||||
}
|
||||
_, err = conn.Read(response)
|
||||
if err != io.EOF {
|
||||
t.Error("error reading", err)
|
||||
}
|
||||
println(response)
|
||||
}
|
||||
@@ -1,82 +0,0 @@
|
||||
package peer
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
"github.com/lbryio/reflector.go/store"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
)
|
||||
|
||||
// Store is a blob store that gets blobs from a peer.
|
||||
// It satisfies the store.BlobStore interface but cannot put or delete blobs.
|
||||
type Store struct {
|
||||
opts StoreOpts
|
||||
}
|
||||
|
||||
// StoreOpts allows to set options for a new Store.
|
||||
type StoreOpts struct {
|
||||
Address string
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
// NewStore makes a new peer store.
|
||||
func NewStore(opts StoreOpts) *Store {
|
||||
return &Store{opts: opts}
|
||||
}
|
||||
|
||||
func (p *Store) getClient() (*Client, error) {
|
||||
c := &Client{Timeout: p.opts.Timeout}
|
||||
err := c.Connect(p.opts.Address)
|
||||
return c, errors.Prefix("connection error", err)
|
||||
}
|
||||
|
||||
func (p *Store) Name() string { return "peer" }
|
||||
|
||||
// Has asks the peer if they have a hash
|
||||
func (p *Store) Has(hash string) (bool, error) {
|
||||
c, err := p.getClient()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer c.Close()
|
||||
return c.HasBlob(hash)
|
||||
}
|
||||
|
||||
// Get downloads the blob from the peer
|
||||
func (p *Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
start := time.Now()
|
||||
c, err := p.getClient()
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), p.Name()), err
|
||||
}
|
||||
defer c.Close()
|
||||
blob, trace, err := c.GetBlob(hash)
|
||||
if err != nil && strings.Contains(err.Error(), "blob not found") {
|
||||
return nil, trace, store.ErrBlobNotFound
|
||||
}
|
||||
|
||||
return blob, trace, err
|
||||
}
|
||||
|
||||
// Put is not supported
|
||||
func (p *Store) Put(hash string, blob stream.Blob) error {
|
||||
return errors.Err(shared.ErrNotImplemented)
|
||||
}
|
||||
|
||||
// PutSD is not supported
|
||||
func (p *Store) PutSD(hash string, blob stream.Blob) error {
|
||||
return errors.Err(shared.ErrNotImplemented)
|
||||
}
|
||||
|
||||
// Delete is not supported
|
||||
func (p *Store) Delete(hash string) error {
|
||||
return errors.Err(shared.ErrNotImplemented)
|
||||
}
|
||||
|
||||
// Shutdown is not supported
|
||||
func (p *Store) Shutdown() {
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package server
|
||||
|
||||
// BlobServer defines the common interface for all blob server implementations
|
||||
type BlobServer interface {
|
||||
Start() error
|
||||
Shutdown()
|
||||
}
|
||||
|
||||
type BlobServerConfig struct {
|
||||
Address string `mapstructure:"address"`
|
||||
EdgeToken string `mapstructure:"edge_token"`
|
||||
Port int `mapstructure:"port"`
|
||||
MaxConcurrentRequests int `mapstructure:"max_concurrent_requests"`
|
||||
}
|
||||
+1
-1
@@ -2,5 +2,5 @@ package shared
|
||||
|
||||
import "github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
|
||||
//ErrNotImplemented is a standard error when a store that implements the store interface does not implement a method
|
||||
// ErrNotImplemented is a standard error when a store that implements the store interface does not implement a method
|
||||
var ErrNotImplemented = errors.Base("this store does not implement this method")
|
||||
+1
-1
@@ -10,9 +10,9 @@ import (
|
||||
)
|
||||
|
||||
type BlobStack struct {
|
||||
Timing time.Duration `json:"timing"`
|
||||
OriginName string `json:"origin_name"`
|
||||
HostName string `json:"host_name"`
|
||||
Timing time.Duration `json:"timing"`
|
||||
}
|
||||
type BlobTrace struct {
|
||||
Stacks []BlobStack `json:"stacks"`
|
||||
|
||||
+11
-3
@@ -16,9 +16,17 @@ func TestBlobTrace_Serialize(t *testing.T) {
|
||||
stack.Stack(30*time.Second, "test3")
|
||||
serialized, err := stack.Serialize()
|
||||
assert.NoError(t, err)
|
||||
t.Log(serialized)
|
||||
expected := "{\"stacks\":[{\"timing\":10000000000,\"origin_name\":\"test\",\"host_name\":\"test_machine\"},{\"timing\":20000000000,\"origin_name\":\"test2\",\"host_name\":\"test_machine\"},{\"timing\":30000000000,\"origin_name\":\"test3\",\"host_name\":\"test_machine\"}]}"
|
||||
assert.Equal(t, expected, serialized)
|
||||
|
||||
deserialized, err := Deserialize(serialized)
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, deserialized.Stacks, 3)
|
||||
assert.Equal(t, 10*time.Second, deserialized.Stacks[0].Timing)
|
||||
assert.Equal(t, "test", deserialized.Stacks[0].OriginName)
|
||||
assert.Equal(t, "test_machine", deserialized.Stacks[0].HostName)
|
||||
assert.Equal(t, 20*time.Second, deserialized.Stacks[1].Timing)
|
||||
assert.Equal(t, "test2", deserialized.Stacks[1].OriginName)
|
||||
assert.Equal(t, 30*time.Second, deserialized.Stacks[2].Timing)
|
||||
assert.Equal(t, "test3", deserialized.Stacks[2].OriginName)
|
||||
}
|
||||
|
||||
func TestBlobTrace_Deserialize(t *testing.T) {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
//go:build linux
|
||||
// +build linux
|
||||
|
||||
package store
|
||||
@@ -9,7 +10,7 @@ import (
|
||||
)
|
||||
|
||||
func timespecToTime(ts syscall.Timespec) time.Time {
|
||||
return time.Unix(int64(ts.Sec), int64(ts.Nsec))
|
||||
return time.Unix(ts.Sec, ts.Nsec)
|
||||
}
|
||||
|
||||
func atime(fi os.FileInfo) time.Time {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
//go:build !linux
|
||||
// +build !linux
|
||||
|
||||
package store
|
||||
|
||||
+67
-8
@@ -1,6 +1,7 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
@@ -10,6 +11,7 @@ import (
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
// CachingStore combines two stores, typically a local and a remote store, to improve performance.
|
||||
@@ -17,20 +19,77 @@ import (
|
||||
// are retrieved from the origin and cached. Puts are cached and also forwarded to the origin.
|
||||
type CachingStore struct {
|
||||
origin, cache BlobStore
|
||||
component string
|
||||
name string
|
||||
}
|
||||
|
||||
type CachingParams struct {
|
||||
Origin BlobStore `mapstructure:"origin"`
|
||||
Cache BlobStore `mapstructure:"cache"`
|
||||
Name string `mapstructure:"name"`
|
||||
}
|
||||
|
||||
type CachingConfig struct {
|
||||
Origin *viper.Viper
|
||||
Cache *viper.Viper
|
||||
Name string `mapstructure:"name"`
|
||||
}
|
||||
|
||||
// NewCachingStore makes a new caching disk store and returns a pointer to it.
|
||||
func NewCachingStore(component string, origin, cache BlobStore) *CachingStore {
|
||||
func NewCachingStore(params CachingParams) *CachingStore {
|
||||
return &CachingStore{
|
||||
component: component,
|
||||
origin: WithSingleFlight(component, origin),
|
||||
cache: WithSingleFlight(component, cache),
|
||||
name: params.Name,
|
||||
origin: WithSingleFlight(params.Name, params.Origin),
|
||||
cache: WithSingleFlight(params.Name, params.Cache),
|
||||
}
|
||||
}
|
||||
|
||||
const nameCaching = "caching"
|
||||
|
||||
func CachingStoreFactory(config *viper.Viper) (BlobStore, error) {
|
||||
var cfg CachingConfig
|
||||
err := config.Unmarshal(&cfg)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
cfg.Cache = config.Sub("cache")
|
||||
cfg.Origin = config.Sub("origin")
|
||||
if cfg.Cache == nil || cfg.Origin == nil {
|
||||
return nil, errors.Err("cache and origin missing")
|
||||
}
|
||||
|
||||
originStoreType := strings.Split(cfg.Origin.AllKeys()[0], ".")[0]
|
||||
originStoreConfig := cfg.Origin.Sub(originStoreType)
|
||||
factory, ok := Factories[originStoreType]
|
||||
if !ok {
|
||||
return nil, errors.Err("unknown store type %s", originStoreType)
|
||||
}
|
||||
originStore, err := factory(originStoreConfig)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
cacheStoreType := strings.Split(cfg.Cache.AllKeys()[0], ".")[0]
|
||||
cacheStoreConfig := cfg.Cache.Sub(cacheStoreType)
|
||||
factory, ok = Factories[cacheStoreType]
|
||||
if !ok {
|
||||
return nil, errors.Err("unknown store type %s", cacheStoreType)
|
||||
}
|
||||
cacheStore, err := factory(cacheStoreConfig)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
return NewCachingStore(CachingParams{
|
||||
Name: cfg.Name,
|
||||
Origin: originStore,
|
||||
Cache: cacheStore,
|
||||
}), nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
RegisterStore(nameCaching, CachingStoreFactory)
|
||||
}
|
||||
|
||||
// Name is the cache type name
|
||||
func (c *CachingStore) Name() string { return nameCaching }
|
||||
|
||||
@@ -49,17 +108,17 @@ func (c *CachingStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
start := time.Now()
|
||||
blob, trace, err := c.cache.Get(hash)
|
||||
if err == nil || !errors.Is(err, ErrBlobNotFound) {
|
||||
metrics.CacheHitCount.With(metrics.CacheLabels(c.cache.Name(), c.component)).Inc()
|
||||
metrics.CacheHitCount.With(metrics.CacheLabels(c.cache.Name(), c.name)).Inc()
|
||||
rate := float64(len(blob)) / 1024 / 1024 / time.Since(start).Seconds()
|
||||
metrics.CacheRetrievalSpeed.With(map[string]string{
|
||||
metrics.LabelCacheType: c.cache.Name(),
|
||||
metrics.LabelComponent: c.component,
|
||||
metrics.LabelComponent: c.name,
|
||||
metrics.LabelSource: "cache",
|
||||
}).Set(rate)
|
||||
return blob, trace.Stack(time.Since(start), c.Name()), err
|
||||
}
|
||||
|
||||
metrics.CacheMissCount.With(metrics.CacheLabels(c.cache.Name(), c.component)).Inc()
|
||||
metrics.CacheMissCount.With(metrics.CacheLabels(c.cache.Name(), c.name)).Inc()
|
||||
|
||||
blob, trace, err = c.origin.Get(hash)
|
||||
if err != nil {
|
||||
|
||||
+13
-13
@@ -9,12 +9,14 @@ import (
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func TestCachingStore_Put(t *testing.T) {
|
||||
origin := NewMemStore()
|
||||
cache := NewMemStore()
|
||||
s := NewCachingStore("test", origin, cache)
|
||||
origin := NewMemStore(MemParams{Name: "test"})
|
||||
cache := NewMemStore(MemParams{Name: "test"})
|
||||
s := NewCachingStore(CachingParams{Name: "test", Origin: origin, Cache: cache})
|
||||
|
||||
b := []byte("this is a blob of stuff")
|
||||
hash := "hash"
|
||||
@@ -42,9 +44,9 @@ func TestCachingStore_Put(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCachingStore_CacheMiss(t *testing.T) {
|
||||
origin := NewMemStore()
|
||||
cache := NewMemStore()
|
||||
s := NewCachingStore("test", origin, cache)
|
||||
origin := NewMemStore(MemParams{Name: "test"})
|
||||
cache := NewMemStore(MemParams{Name: "test"})
|
||||
s := NewCachingStore(CachingParams{Name: "test", Origin: origin, Cache: cache})
|
||||
|
||||
b := []byte("this is a blob of stuff")
|
||||
hash := "hash"
|
||||
@@ -84,8 +86,8 @@ func TestCachingStore_CacheMiss(t *testing.T) {
|
||||
func TestCachingStore_ThunderingHerd(t *testing.T) {
|
||||
storeDelay := 100 * time.Millisecond
|
||||
origin := NewSlowBlobStore(storeDelay)
|
||||
cache := NewMemStore()
|
||||
s := NewCachingStore("test", origin, cache)
|
||||
cache := NewMemStore(MemParams{Name: "test"})
|
||||
s := NewCachingStore(CachingParams{Name: "test", Origin: origin, Cache: cache})
|
||||
|
||||
b := []byte("this is a blob of stuff")
|
||||
hash := "hash"
|
||||
@@ -99,7 +101,7 @@ func TestCachingStore_ThunderingHerd(t *testing.T) {
|
||||
getNoErr := func() {
|
||||
res, _, err := s.Get(hash)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
logrus.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(b, res) {
|
||||
t.Errorf("expected Get() to return %s, got %s", string(b), string(res))
|
||||
@@ -140,7 +142,7 @@ type SlowBlobStore struct {
|
||||
|
||||
func NewSlowBlobStore(delay time.Duration) *SlowBlobStore {
|
||||
return &SlowBlobStore{
|
||||
mem: NewMemStore(),
|
||||
mem: NewMemStore(MemParams{Name: "test"}),
|
||||
delay: delay,
|
||||
}
|
||||
}
|
||||
@@ -173,6 +175,4 @@ func (s *SlowBlobStore) Delete(hash string) error {
|
||||
return s.mem.Delete(hash)
|
||||
}
|
||||
|
||||
func (s *SlowBlobStore) Shutdown() {
|
||||
return
|
||||
}
|
||||
func (s *SlowBlobStore) Shutdown() {}
|
||||
@@ -1,110 +0,0 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
"github.com/lbryio/reflector.go/meta"
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// CloudFrontROStore reads from cloudfront. All writes panic.
|
||||
type CloudFrontROStore struct {
|
||||
endpoint string // cloudflare endpoint
|
||||
}
|
||||
|
||||
// NewCloudFrontROStore returns an initialized CloudFrontROStore store pointer.
|
||||
func NewCloudFrontROStore(endpoint string) *CloudFrontROStore {
|
||||
return &CloudFrontROStore{endpoint: endpoint}
|
||||
}
|
||||
|
||||
const nameCloudFrontRO = "cloudfront_ro"
|
||||
|
||||
// Name is the cache type name
|
||||
func (c *CloudFrontROStore) Name() string { return nameCloudFrontRO }
|
||||
|
||||
// Has checks if the hash is in the store.
|
||||
func (c *CloudFrontROStore) Has(hash string) (bool, error) {
|
||||
status, body, err := c.cfRequest(http.MethodHead, hash)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer body.Close()
|
||||
|
||||
switch status {
|
||||
case http.StatusNotFound, http.StatusForbidden:
|
||||
return false, nil
|
||||
case http.StatusOK:
|
||||
return true, nil
|
||||
default:
|
||||
return false, errors.Err("unexpected status %d", status)
|
||||
}
|
||||
}
|
||||
|
||||
// Get gets the blob from Cloudfront.
|
||||
func (c *CloudFrontROStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
log.Debugf("Getting %s from S3", hash[:8])
|
||||
start := time.Now()
|
||||
defer func(t time.Time) {
|
||||
log.Debugf("Getting %s from S3 took %s", hash[:8], time.Since(t).String())
|
||||
}(start)
|
||||
|
||||
status, body, err := c.cfRequest(http.MethodGet, hash)
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), c.Name()), err
|
||||
}
|
||||
defer body.Close()
|
||||
switch status {
|
||||
case http.StatusNotFound, http.StatusForbidden:
|
||||
return nil, shared.NewBlobTrace(time.Since(start), c.Name()), errors.Err(ErrBlobNotFound)
|
||||
case http.StatusOK:
|
||||
b, err := ioutil.ReadAll(body)
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), c.Name()), errors.Err(err)
|
||||
}
|
||||
metrics.MtrInBytesS3.Add(float64(len(b)))
|
||||
return b, shared.NewBlobTrace(time.Since(start), c.Name()), nil
|
||||
default:
|
||||
return nil, shared.NewBlobTrace(time.Since(start), c.Name()), errors.Err("unexpected status %d", status)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *CloudFrontROStore) cfRequest(method, hash string) (int, io.ReadCloser, error) {
|
||||
url := c.endpoint + hash
|
||||
req, err := http.NewRequest(method, url, nil)
|
||||
if err != nil {
|
||||
return 0, nil, errors.Err(err)
|
||||
}
|
||||
req.Header.Add("User-Agent", "reflector.go/"+meta.Version)
|
||||
|
||||
res, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return 0, nil, errors.Err(err)
|
||||
}
|
||||
|
||||
return res.StatusCode, res.Body, nil
|
||||
}
|
||||
|
||||
func (c *CloudFrontROStore) Put(_ string, _ stream.Blob) error {
|
||||
return errors.Err(shared.ErrNotImplemented)
|
||||
}
|
||||
|
||||
func (c *CloudFrontROStore) PutSD(_ string, _ stream.Blob) error {
|
||||
return errors.Err(shared.ErrNotImplemented)
|
||||
}
|
||||
|
||||
func (c *CloudFrontROStore) Delete(_ string) error {
|
||||
return errors.Err(shared.ErrNotImplemented)
|
||||
}
|
||||
|
||||
// Shutdown shuts down the store gracefully
|
||||
func (c *CloudFrontROStore) Shutdown() {
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
)
|
||||
|
||||
// CloudFrontRWStore combines a Cloudfront and an S3 store. Reads go to Cloudfront/Wasabi, writes go to S3.
|
||||
type CloudFrontRWStore struct {
|
||||
cf *ITTTStore
|
||||
s3 *S3Store
|
||||
}
|
||||
|
||||
// NewCloudFrontRWStore returns an initialized CloudFrontRWStore store pointer.
|
||||
// NOTE: It panics if either argument is nil.
|
||||
func NewCloudFrontRWStore(cf *ITTTStore, s3 *S3Store) *CloudFrontRWStore {
|
||||
if cf == nil || s3 == nil {
|
||||
panic("both stores must be set")
|
||||
}
|
||||
return &CloudFrontRWStore{cf: cf, s3: s3}
|
||||
}
|
||||
|
||||
const nameCloudFrontRW = "cloudfront_rw"
|
||||
|
||||
// Name is the cache type name
|
||||
func (c *CloudFrontRWStore) Name() string { return nameCloudFrontRW }
|
||||
|
||||
// Has checks if the hash is in the store.
|
||||
func (c *CloudFrontRWStore) Has(hash string) (bool, error) {
|
||||
return c.cf.Has(hash)
|
||||
}
|
||||
|
||||
// Get gets the blob from Cloudfront.
|
||||
func (c *CloudFrontRWStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
start := time.Now()
|
||||
blob, trace, err := c.cf.Get(hash)
|
||||
return blob, trace.Stack(time.Since(start), c.Name()), err
|
||||
}
|
||||
|
||||
// Put stores the blob on S3
|
||||
func (c *CloudFrontRWStore) Put(hash string, blob stream.Blob) error {
|
||||
return c.s3.Put(hash, blob)
|
||||
}
|
||||
|
||||
// PutSD stores the sd blob on S3
|
||||
func (c *CloudFrontRWStore) PutSD(hash string, blob stream.Blob) error {
|
||||
return c.s3.PutSD(hash, blob)
|
||||
}
|
||||
|
||||
// Delete deletes the blob from S3
|
||||
func (c *CloudFrontRWStore) Delete(hash string) error {
|
||||
return c.s3.Delete(hash)
|
||||
}
|
||||
|
||||
// Shutdown shuts down the store gracefully
|
||||
func (c *CloudFrontRWStore) Shutdown() {
|
||||
c.s3.Shutdown()
|
||||
c.cf.Shutdown()
|
||||
}
|
||||
+177
-25
@@ -2,6 +2,8 @@ package store
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -9,29 +11,70 @@ import (
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/extras/stop"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
"github.com/c2h5oh/datasize"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
// DBBackedStore is a store that's backed by a DB. The DB contains data about what's in the store.
|
||||
type DBBackedStore struct {
|
||||
blobs BlobStore
|
||||
db *db.SQL
|
||||
blockedMu sync.RWMutex
|
||||
blocked map[string]bool
|
||||
cleanerStop *stop.Group
|
||||
name string
|
||||
maxSize int
|
||||
blockedMu sync.RWMutex
|
||||
deleteOnMiss bool
|
||||
}
|
||||
|
||||
// NewDBBackedStore returns an initialized store pointer.
|
||||
func NewDBBackedStore(blobs BlobStore, db *db.SQL, deleteOnMiss bool) *DBBackedStore {
|
||||
return &DBBackedStore{blobs: blobs, db: db, deleteOnMiss: deleteOnMiss}
|
||||
type DBBackedParams struct {
|
||||
Store BlobStore `mapstructure:"store"`
|
||||
DB *db.SQL `mapstructure:"db"`
|
||||
MaxSize *int `mapstructure:"max_size"`
|
||||
Name string `mapstructure:"name"`
|
||||
DeleteOnMiss bool `mapstructure:"delete_on_miss"`
|
||||
}
|
||||
|
||||
const nameDBBacked = "db-backed"
|
||||
type DBBackedConfig struct {
|
||||
Store *viper.Viper
|
||||
Database string `mapstructure:"database"`
|
||||
User string `mapstructure:"user"`
|
||||
Password string `mapstructure:"password"`
|
||||
Host string `mapstructure:"host"`
|
||||
Name string `mapstructure:"name"`
|
||||
MaxSize string `mapstructure:"max_size"`
|
||||
Port int `mapstructure:"port"`
|
||||
AccessTracking int `mapstructure:"access_tracking"`
|
||||
DeleteOnMiss bool `mapstructure:"delete_on_miss"`
|
||||
SoftDeletes bool `mapstructure:"soft_deletes"`
|
||||
LogQueries bool `mapstructure:"log_queries"`
|
||||
HasCap bool `mapstructure:"has_cap"`
|
||||
}
|
||||
|
||||
// NewDBBackedStore returns an initialized store pointer.
|
||||
func NewDBBackedStore(params DBBackedParams) *DBBackedStore {
|
||||
store := &DBBackedStore{
|
||||
blobs: params.Store,
|
||||
db: params.DB,
|
||||
deleteOnMiss: params.DeleteOnMiss,
|
||||
cleanerStop: stop.New(),
|
||||
name: params.Name,
|
||||
}
|
||||
if params.MaxSize != nil {
|
||||
store.maxSize = *params.MaxSize
|
||||
go store.cleanOldestBlobs()
|
||||
}
|
||||
return store
|
||||
}
|
||||
|
||||
const nameDBBacked = "db_backed"
|
||||
|
||||
// Name is the cache type name
|
||||
func (d *DBBackedStore) Name() string { return nameDBBacked }
|
||||
func (d *DBBackedStore) Name() string { return nameDBBacked + "-" + d.name }
|
||||
|
||||
// Has returns true if the blob is in the store
|
||||
func (d *DBBackedStore) Has(hash string) (bool, error) {
|
||||
@@ -101,34 +144,18 @@ func (d *DBBackedStore) Delete(hash string) error {
|
||||
|
||||
// Block deletes the blob and prevents it from being uploaded in the future
|
||||
func (d *DBBackedStore) Block(hash string) error {
|
||||
if blocked, err := d.isBlocked(hash); blocked || err != nil {
|
||||
blocked, err := d.isBlocked(hash)
|
||||
if blocked || err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Debugf("blocking %s", hash)
|
||||
|
||||
err := d.db.Block(hash)
|
||||
err = d.db.Block(hash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
has, err := d.db.HasBlob(hash, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if has {
|
||||
err = d.blobs.Delete(hash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = d.db.Delete(hash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return d.markBlocked(hash)
|
||||
}
|
||||
|
||||
@@ -195,7 +222,132 @@ func (d *DBBackedStore) initBlocked() error {
|
||||
return err
|
||||
}
|
||||
|
||||
// cleanOldestBlobs periodically cleans up the oldest blobs if maxSize is set
|
||||
func (d *DBBackedStore) cleanOldestBlobs() {
|
||||
// Run on startup without waiting for 10 minutes
|
||||
err := d.doClean()
|
||||
if err != nil {
|
||||
log.Error(errors.FullTrace(err))
|
||||
}
|
||||
const cleanupInterval = 10 * time.Minute
|
||||
for {
|
||||
select {
|
||||
case <-d.cleanerStop.Ch():
|
||||
log.Infoln("stopping self cleanup")
|
||||
return
|
||||
case <-time.After(cleanupInterval):
|
||||
err := d.doClean()
|
||||
if err != nil {
|
||||
log.Error(errors.FullTrace(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// doClean removes the least recently accessed blobs if the store exceeds maxItems
|
||||
func (d *DBBackedStore) doClean() error {
|
||||
blobsCount, err := d.db.Count()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if blobsCount >= d.maxSize {
|
||||
itemsToDelete := blobsCount / 10
|
||||
blobs, err := d.db.LeastRecentlyAccessedHashes(itemsToDelete)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
blobsChan := make(chan string, len(blobs))
|
||||
wg := stop.New()
|
||||
go func() {
|
||||
for _, hash := range blobs {
|
||||
select {
|
||||
case <-d.cleanerStop.Ch():
|
||||
return
|
||||
default:
|
||||
}
|
||||
blobsChan <- hash
|
||||
}
|
||||
close(blobsChan)
|
||||
}()
|
||||
for i := 0; i < 3; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for h := range blobsChan {
|
||||
select {
|
||||
case <-d.cleanerStop.Ch():
|
||||
return
|
||||
default:
|
||||
}
|
||||
err = d.Delete(h)
|
||||
if err != nil {
|
||||
log.Errorf("error pruning %s: %s", h, errors.FullTrace(err))
|
||||
continue
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Shutdown shuts down the store gracefully
|
||||
func (d *DBBackedStore) Shutdown() {
|
||||
d.cleanerStop.Stop()
|
||||
d.blobs.Shutdown()
|
||||
}
|
||||
|
||||
func DBBackedStoreFactory(config *viper.Viper) (BlobStore, error) {
|
||||
var cfg DBBackedConfig
|
||||
err := config.Unmarshal(&cfg)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
cfg.Store = config.Sub("store")
|
||||
|
||||
storeType := strings.Split(cfg.Store.AllKeys()[0], ".")[0]
|
||||
storeConfig := cfg.Store.Sub(storeType)
|
||||
factory, ok := Factories[storeType]
|
||||
if !ok {
|
||||
return nil, errors.Err("unknown store type %s", storeType)
|
||||
}
|
||||
underlyingStore, err := factory(storeConfig)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
parsedDb := &db.SQL{
|
||||
TrackingLevel: db.AccessTrackingLevel(cfg.AccessTracking),
|
||||
SoftDelete: cfg.SoftDeletes,
|
||||
LogQueries: cfg.LogQueries || log.GetLevel() == log.DebugLevel,
|
||||
}
|
||||
|
||||
err = parsedDb.Connect(fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", cfg.User, cfg.Password, cfg.Host, cfg.Port, cfg.Database))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
params := DBBackedParams{
|
||||
Name: cfg.Name,
|
||||
Store: underlyingStore,
|
||||
DB: parsedDb,
|
||||
DeleteOnMiss: cfg.DeleteOnMiss,
|
||||
}
|
||||
if cfg.HasCap {
|
||||
var parsedSize datasize.ByteSize
|
||||
err = parsedSize.UnmarshalText([]byte(cfg.MaxSize))
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
maxSize := int(float64(parsedSize) / float64(stream.MaxBlobSize))
|
||||
params.MaxSize = &maxSize
|
||||
}
|
||||
|
||||
return NewDBBackedStore(params), nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
RegisterStore(nameDBBacked, DBBackedStoreFactory)
|
||||
}
|
||||
+31
-100
@@ -1,13 +1,6 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha512"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"os"
|
||||
"path"
|
||||
"time"
|
||||
@@ -18,38 +11,49 @@ import (
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
"github.com/brk0v/directio"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"go.uber.org/atomic"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
// DiskStore stores blobs on a local disk
|
||||
type DiskStore struct {
|
||||
// the location of blobs on disk
|
||||
blobDir string
|
||||
// store files in subdirectories based on the first N chars in the filename. 0 = don't create subdirectories.
|
||||
blobDir string
|
||||
name string
|
||||
prefixLength int
|
||||
|
||||
// true if initOnce ran, false otherwise
|
||||
initialized bool
|
||||
|
||||
concurrentChecks atomic.Int32
|
||||
initialized bool
|
||||
}
|
||||
|
||||
const maxConcurrentChecks = 30
|
||||
type DiskParams struct {
|
||||
Name string `mapstructure:"name"`
|
||||
MountPoint string `mapstructure:"mount_point"`
|
||||
ShardingSize int `mapstructure:"sharding_size"`
|
||||
}
|
||||
|
||||
// NewDiskStore returns an initialized file disk store pointer.
|
||||
func NewDiskStore(dir string, prefixLength int) *DiskStore {
|
||||
func NewDiskStore(params DiskParams) *DiskStore {
|
||||
return &DiskStore{
|
||||
blobDir: dir,
|
||||
prefixLength: prefixLength,
|
||||
blobDir: params.MountPoint,
|
||||
prefixLength: params.ShardingSize,
|
||||
name: params.Name,
|
||||
}
|
||||
}
|
||||
|
||||
const nameDisk = "disk"
|
||||
|
||||
func DiskStoreFactory(config *viper.Viper) (BlobStore, error) {
|
||||
var cfg DiskParams
|
||||
err := config.Unmarshal(&cfg)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
return NewDiskStore(cfg), nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
RegisterStore(nameDisk, DiskStoreFactory)
|
||||
}
|
||||
|
||||
// Name is the cache type name
|
||||
func (d *DiskStore) Name() string { return nameDisk }
|
||||
func (d *DiskStore) Name() string { return nameDisk + "-" + d.name }
|
||||
|
||||
// Has returns T/F or Error if it the blob stored already. It will error with any IO disk error.
|
||||
func (d *DiskStore) Has(hash string) (bool, error) {
|
||||
@@ -68,14 +72,6 @@ func (d *DiskStore) Has(hash string) (bool, error) {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (d *DiskStore) Size(hash string) (int64, error) {
|
||||
stat, err := os.Stat(d.path(hash))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return stat.Size(), nil
|
||||
}
|
||||
|
||||
// Get returns the blob or an error if the blob doesn't exist.
|
||||
func (d *DiskStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
start := time.Now()
|
||||
@@ -84,69 +80,16 @@ func (d *DiskStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), err
|
||||
}
|
||||
|
||||
blob, err := ioutil.ReadFile(d.path(hash))
|
||||
blob, err := os.ReadFile(d.path(hash))
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(ErrBlobNotFound)
|
||||
}
|
||||
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(err)
|
||||
}
|
||||
|
||||
// this is a rather poor yet effective way of throttling how many blobs can be checked concurrently
|
||||
// poor because there is a possible race condition between the check and the actual +1
|
||||
if d.concurrentChecks.Load() < maxConcurrentChecks {
|
||||
d.concurrentChecks.Add(1)
|
||||
defer d.concurrentChecks.Sub(1)
|
||||
hashBytes := sha512.Sum384(blob)
|
||||
readHash := hex.EncodeToString(hashBytes[:])
|
||||
if hash != readHash {
|
||||
message := fmt.Sprintf("[%s] found a broken blob while reading from disk. Actual hash: %s", hash, readHash)
|
||||
log.Errorf("%s", message)
|
||||
err := d.Delete(hash)
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), err
|
||||
}
|
||||
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(message)
|
||||
}
|
||||
}
|
||||
|
||||
return blob, shared.NewBlobTrace(time.Since(start), d.Name()), nil
|
||||
}
|
||||
|
||||
// Put stores the blob on disk
|
||||
func (d *DiskStore) Put(hash string, blob stream.Blob) error {
|
||||
err := d.initOnce()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = d.ensureDirExists(d.dir(hash))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Open file with O_DIRECT
|
||||
f, err := os.OpenFile(d.tmpPath(hash), openFileFlags, 0644)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
// Use directio writer
|
||||
dio, err := directio.New(f)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer dio.Flush()
|
||||
// Write the body to file
|
||||
_, err = io.Copy(dio, bytes.NewReader(blob))
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
err = os.Rename(d.tmpPath(hash), d.path(hash))
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
// PutSD stores the sd blob on the disk
|
||||
func (d *DiskStore) PutSD(hash string, blob stream.Blob) error {
|
||||
return d.Put(hash, blob)
|
||||
@@ -187,14 +130,14 @@ func (d *DiskStore) dir(hash string) string {
|
||||
}
|
||||
return path.Join(d.blobDir, hash[:d.prefixLength])
|
||||
}
|
||||
func (d *DiskStore) tmpDir(hash string) string {
|
||||
func (d *DiskStore) tmpDir() string {
|
||||
return path.Join(d.blobDir, "tmp")
|
||||
}
|
||||
func (d *DiskStore) path(hash string) string {
|
||||
return path.Join(d.dir(hash), hash)
|
||||
}
|
||||
func (d *DiskStore) tmpPath(hash string) string {
|
||||
return path.Join(d.tmpDir(hash), hash)
|
||||
return path.Join(d.tmpDir(), hash)
|
||||
}
|
||||
func (d *DiskStore) ensureDirExists(dir string) error {
|
||||
return errors.Err(os.MkdirAll(dir, 0755))
|
||||
@@ -209,7 +152,7 @@ func (d *DiskStore) initOnce() error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = d.ensureDirExists(path.Join(d.blobDir, "tmp"))
|
||||
err = d.ensureDirExists(d.tmpDir())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -220,15 +163,3 @@ func (d *DiskStore) initOnce() error {
|
||||
// Shutdown shuts down the store gracefully
|
||||
func (d *DiskStore) Shutdown() {
|
||||
}
|
||||
|
||||
func (d *DiskStore) Sendfile(hash string, conn net.Conn) error {
|
||||
f, err := os.Open(d.path(hash))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = io.Copy(conn, f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
+15
-8
@@ -1,7 +1,6 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
@@ -14,10 +13,14 @@ import (
|
||||
)
|
||||
|
||||
func TestDiskStore_Get(t *testing.T) {
|
||||
tmpDir, err := ioutil.TempDir("", "reflector_test_*")
|
||||
tmpDir, err := os.MkdirTemp("", "reflector_test_*")
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(tmpDir)
|
||||
d := NewDiskStore(tmpDir, 2)
|
||||
defer func() { _ = os.RemoveAll(tmpDir) }()
|
||||
d := NewDiskStore(DiskParams{
|
||||
Name: "test",
|
||||
MountPoint: tmpDir,
|
||||
ShardingSize: 2,
|
||||
})
|
||||
|
||||
hash := "f428b8265d65dad7f8ffa52922bba836404cbd62f3ecfe10adba6b444f8f658938e54f5981ac4de39644d5b93d89a94b"
|
||||
data := []byte("oyuntyausntoyaunpdoyruoyduanrstjwfjyuwf")
|
||||
@@ -25,7 +28,7 @@ func TestDiskStore_Get(t *testing.T) {
|
||||
expectedPath := path.Join(tmpDir, hash[:2], hash)
|
||||
err = os.MkdirAll(filepath.Dir(expectedPath), os.ModePerm)
|
||||
require.NoError(t, err)
|
||||
err = ioutil.WriteFile(expectedPath, data, os.ModePerm)
|
||||
err = os.WriteFile(expectedPath, data, os.ModePerm)
|
||||
require.NoError(t, err)
|
||||
|
||||
blob, _, err := d.Get(hash)
|
||||
@@ -34,10 +37,14 @@ func TestDiskStore_Get(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestDiskStore_GetNonexistentBlob(t *testing.T) {
|
||||
tmpDir, err := ioutil.TempDir("", "reflector_test_*")
|
||||
tmpDir, err := os.MkdirTemp("", "reflector_test_*")
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(tmpDir)
|
||||
d := NewDiskStore(tmpDir, 2)
|
||||
defer func() { _ = os.RemoveAll(tmpDir) }()
|
||||
d := NewDiskStore(DiskParams{
|
||||
Name: "test",
|
||||
MountPoint: tmpDir,
|
||||
ShardingSize: 2,
|
||||
})
|
||||
|
||||
blob, _, err := d.Get("nonexistent")
|
||||
assert.Nil(t, blob)
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
//go:build darwin
|
||||
// +build darwin
|
||||
|
||||
package store
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
)
|
||||
|
||||
var openFileFlags = os.O_WRONLY | os.O_CREATE
|
||||
|
||||
// Put stores the blob on disk
|
||||
func (d *DiskStore) Put(hash string, blob stream.Blob) error {
|
||||
err := d.initOnce()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = d.ensureDirExists(d.dir(hash))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Open file with O_DIRECT
|
||||
f, err := os.OpenFile(d.tmpPath(hash), openFileFlags, 0644)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
_, err = io.Copy(f, bytes.NewReader(blob))
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
err = os.Rename(d.tmpPath(hash), d.path(hash))
|
||||
return errors.Err(err)
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
//go:build linux
|
||||
// +build linux
|
||||
|
||||
package store
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
"syscall"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
"github.com/brk0v/directio"
|
||||
)
|
||||
|
||||
var openFileFlags = os.O_WRONLY | os.O_CREATE | syscall.O_DIRECT
|
||||
|
||||
// Put stores the blob on disk
|
||||
func (d *DiskStore) Put(hash string, blob stream.Blob) error {
|
||||
err := d.initOnce()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = d.ensureDirExists(d.dir(hash))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Open file with O_DIRECT
|
||||
f, err := os.OpenFile(d.tmpPath(hash), openFileFlags, 0644)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer func() { _ = f.Close() }()
|
||||
dio, err := directio.New(f)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
defer func() { _ = dio.Flush() }()
|
||||
_, err = io.Copy(dio, bytes.NewReader(blob))
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
err = os.Rename(d.tmpPath(hash), d.path(hash))
|
||||
return errors.Err(err)
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
//go:build !linux && !darwin
|
||||
// +build !linux,!darwin
|
||||
|
||||
package store
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
)
|
||||
|
||||
// Put stores the blob on disk
|
||||
func (d *DiskStore) Put(hash string, blob stream.Blob) error {
|
||||
err := d.initOnce()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = d.ensureDirExists(d.dir(hash))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = os.WriteFile(d.tmpPath(hash), blob, 0644)
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
err = os.Rename(d.tmpPath(hash), d.path(hash))
|
||||
return errors.Err(err)
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
// +build darwin
|
||||
|
||||
package store
|
||||
|
||||
import (
|
||||
"os"
|
||||
)
|
||||
|
||||
var openFileFlags = os.O_WRONLY | os.O_CREATE
|
||||
@@ -1,10 +0,0 @@
|
||||
// +build linux
|
||||
|
||||
package store
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
var openFileFlags = os.O_WRONLY | os.O_CREATE | syscall.O_DIRECT
|
||||
+73
-21
@@ -1,6 +1,7 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
@@ -11,15 +12,16 @@ import (
|
||||
|
||||
"github.com/bluele/gcache"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
// GcacheStore adds a max cache size and Greedy-Dual-Size-Frequency cache eviction strategy to a BlobStore
|
||||
type GcacheStore struct {
|
||||
// underlying store
|
||||
store BlobStore
|
||||
// cache implementation
|
||||
cache gcache.Cache
|
||||
underlyingStore BlobStore
|
||||
cache gcache.Cache
|
||||
name string
|
||||
}
|
||||
|
||||
type EvictionStrategy int
|
||||
|
||||
const (
|
||||
@@ -33,16 +35,30 @@ const (
|
||||
SIMPLE
|
||||
)
|
||||
|
||||
type GcacheParams struct {
|
||||
Store BlobStore `mapstructure:"store"`
|
||||
Name string `mapstructure:"name"`
|
||||
MaxSize int `mapstructure:"max_size"`
|
||||
Strategy EvictionStrategy `mapstructure:"strategy"`
|
||||
}
|
||||
|
||||
type GcacheConfig struct {
|
||||
Store *viper.Viper
|
||||
Name string `mapstructure:"name"`
|
||||
MaxSize int `mapstructure:"max_size"`
|
||||
Strategy EvictionStrategy `mapstructure:"strategy"`
|
||||
}
|
||||
|
||||
// NewGcacheStore initialize a new LRUStore
|
||||
func NewGcacheStore(component string, store BlobStore, maxSize int, strategy EvictionStrategy) *GcacheStore {
|
||||
cacheBuilder := gcache.New(maxSize)
|
||||
func NewGcacheStore(params GcacheParams) *GcacheStore {
|
||||
cacheBuilder := gcache.New(params.MaxSize)
|
||||
var cache gcache.Cache
|
||||
evictFunc := func(key interface{}, value interface{}) {
|
||||
logrus.Infof("evicting %s", key)
|
||||
metrics.CacheLRUEvictCount.With(metrics.CacheLabels(store.Name(), component)).Inc()
|
||||
_ = store.Delete(key.(string)) // TODO: log this error. may happen if underlying entry is gone but cache entry still there
|
||||
metrics.CacheLRUEvictCount.With(metrics.CacheLabels(params.Store.Name(), params.Name)).Inc()
|
||||
_ = params.Store.Delete(key.(string)) // TODO: log this error. may happen if underlying entry is gone but cache entry still there
|
||||
}
|
||||
switch strategy {
|
||||
switch params.Strategy {
|
||||
case LFU:
|
||||
cache = cacheBuilder.LFU().EvictedFunc(evictFunc).Build()
|
||||
case ARC:
|
||||
@@ -51,15 +67,15 @@ func NewGcacheStore(component string, store BlobStore, maxSize int, strategy Evi
|
||||
cache = cacheBuilder.LRU().EvictedFunc(evictFunc).Build()
|
||||
case SIMPLE:
|
||||
cache = cacheBuilder.Simple().EvictedFunc(evictFunc).Build()
|
||||
|
||||
}
|
||||
l := &GcacheStore{
|
||||
store: store,
|
||||
cache: cache,
|
||||
underlyingStore: params.Store,
|
||||
cache: cache,
|
||||
name: params.Name,
|
||||
}
|
||||
go func() {
|
||||
if lstr, ok := store.(lister); ok {
|
||||
err := l.loadExisting(lstr, int(maxSize))
|
||||
if lstr, ok := params.Store.(lister); ok {
|
||||
err := l.loadExisting(lstr, params.MaxSize)
|
||||
if err != nil {
|
||||
panic(err) // TODO: what should happen here? panic? return nil? just keep going?
|
||||
}
|
||||
@@ -71,8 +87,40 @@ func NewGcacheStore(component string, store BlobStore, maxSize int, strategy Evi
|
||||
|
||||
const nameGcache = "gcache"
|
||||
|
||||
func GcacheStoreFactory(config *viper.Viper) (BlobStore, error) {
|
||||
var cfg GcacheConfig
|
||||
err := config.Unmarshal(&cfg)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
cfg.Store = config.Sub("store")
|
||||
|
||||
storeType := strings.Split(cfg.Store.AllKeys()[0], ".")[0]
|
||||
storeConfig := cfg.Store.Sub(storeType)
|
||||
factory, ok := Factories[storeType]
|
||||
if !ok {
|
||||
return nil, errors.Err("unknown store type %s", storeType)
|
||||
}
|
||||
underlyingStore, err := factory(storeConfig)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
return NewGcacheStore(GcacheParams{
|
||||
Name: cfg.Name,
|
||||
Store: underlyingStore,
|
||||
MaxSize: cfg.MaxSize,
|
||||
Strategy: cfg.Strategy,
|
||||
}), nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
RegisterStore(nameGcache, GcacheStoreFactory)
|
||||
}
|
||||
|
||||
// Name is the cache type name
|
||||
func (l *GcacheStore) Name() string { return nameGcache }
|
||||
func (l *GcacheStore) Name() string { return nameGcache + "-" + l.name }
|
||||
|
||||
// Has returns whether the blob is in the store, without updating the recent-ness.
|
||||
func (l *GcacheStore) Has(hash string) (bool, error) {
|
||||
@@ -86,7 +134,7 @@ func (l *GcacheStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), l.Name()), errors.Err(ErrBlobNotFound)
|
||||
}
|
||||
blob, stack, err := l.store.Get(hash)
|
||||
blob, stack, err := l.underlyingStore.Get(hash)
|
||||
if errors.Is(err, ErrBlobNotFound) {
|
||||
// Blob disappeared from underlying store
|
||||
l.cache.Remove(hash)
|
||||
@@ -99,7 +147,7 @@ func (l *GcacheStore) Put(hash string, blob stream.Blob) error {
|
||||
_ = l.cache.Set(hash, true)
|
||||
has, _ := l.Has(hash)
|
||||
if has {
|
||||
err := l.store.Put(hash, blob)
|
||||
err := l.underlyingStore.Put(hash, blob)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -112,7 +160,7 @@ func (l *GcacheStore) PutSD(hash string, blob stream.Blob) error {
|
||||
_ = l.cache.Set(hash, true)
|
||||
has, _ := l.Has(hash)
|
||||
if has {
|
||||
err := l.store.PutSD(hash, blob)
|
||||
err := l.underlyingStore.PutSD(hash, blob)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -122,7 +170,7 @@ func (l *GcacheStore) PutSD(hash string, blob stream.Blob) error {
|
||||
|
||||
// Delete deletes the blob from the store
|
||||
func (l *GcacheStore) Delete(hash string) error {
|
||||
err := l.store.Delete(hash)
|
||||
err := l.underlyingStore.Delete(hash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -144,11 +192,15 @@ func (l *GcacheStore) loadExisting(store lister, maxItems int) error {
|
||||
logrus.Infof("read %d files from underlying store", len(existing))
|
||||
|
||||
added := 0
|
||||
for _, h := range existing {
|
||||
for i, h := range existing {
|
||||
_ = l.cache.Set(h, true)
|
||||
added++
|
||||
if maxItems > 0 && added >= maxItems { // underlying cache is bigger than the cache
|
||||
break
|
||||
err := l.Delete(h)
|
||||
logrus.Infof("deleted overflowing blob: %s (%d/%d)", h, i, len(existing))
|
||||
if err != nil {
|
||||
logrus.Warnf("error while deleting a blob that's overflowing the cache: %s", err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
|
||||
+11
-8
@@ -2,7 +2,6 @@ package store
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"reflect"
|
||||
"testing"
|
||||
@@ -17,8 +16,8 @@ import (
|
||||
const cacheMaxSize = 3
|
||||
|
||||
func getTestGcacheStore() (*GcacheStore, *MemStore) {
|
||||
m := NewMemStore()
|
||||
return NewGcacheStore("test", m, cacheMaxSize, LFU), m
|
||||
m := NewMemStore(MemParams{Name: "test"})
|
||||
return NewGcacheStore(GcacheParams{Name: "test", Store: m, MaxSize: cacheMaxSize, Strategy: LFU}), m
|
||||
}
|
||||
|
||||
func TestGcacheStore_Eviction(t *testing.T) {
|
||||
@@ -88,10 +87,14 @@ func TestGcacheStore_UnderlyingBlobMissing(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestGcacheStore_loadExisting(t *testing.T) {
|
||||
tmpDir, err := ioutil.TempDir("", "reflector_test_*")
|
||||
tmpDir, err := os.MkdirTemp("", "reflector_test_*")
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(tmpDir)
|
||||
d := NewDiskStore(tmpDir, 2)
|
||||
defer func() { _ = os.RemoveAll(tmpDir) }()
|
||||
d := NewDiskStore(DiskParams{
|
||||
Name: "test",
|
||||
MountPoint: tmpDir,
|
||||
ShardingSize: 2,
|
||||
})
|
||||
|
||||
hash := "hash"
|
||||
b := []byte("this is a blob of stuff")
|
||||
@@ -103,8 +106,8 @@ func TestGcacheStore_loadExisting(t *testing.T) {
|
||||
require.Equal(t, 1, len(existing), "blob should exist in cache")
|
||||
assert.Equal(t, hash, existing[0])
|
||||
|
||||
lfu := NewGcacheStore("test", d, 3, LFU) // lru should load existing blobs when it's created
|
||||
time.Sleep(100 * time.Millisecond) // async load so let's wait...
|
||||
lfu := NewGcacheStore(GcacheParams{Name: "test", Store: d, MaxSize: 3, Strategy: LFU}) // lru should load existing blobs when it's created
|
||||
time.Sleep(100 * time.Millisecond) // async load so let's wait...
|
||||
has, err := lfu.Has(hash)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, has, "hash should be loaded from disk store but it's not")
|
||||
|
||||
+129
-101
@@ -1,160 +1,188 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"sync"
|
||||
"path"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
"github.com/lbryio/reflector.go/meta"
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
// HttpStore is a store that works on top of the HTTP protocol
|
||||
// HttpStore reads from an HTTP endpoint that simply expects the hash to be appended to the endpoint
|
||||
type HttpStore struct {
|
||||
upstream string
|
||||
httpClient *http.Client
|
||||
httpClient *http.Client
|
||||
endpoint string
|
||||
name string
|
||||
prefixLength int
|
||||
}
|
||||
|
||||
func NewHttpStore(upstream string) *HttpStore {
|
||||
type HttpParams struct {
|
||||
Name string `mapstructure:"name"`
|
||||
Endpoint string `mapstructure:"endpoint"`
|
||||
ShardingSize int `mapstructure:"sharding_size"`
|
||||
}
|
||||
|
||||
// NewHttpStore returns an initialized HttpStore store pointer.
|
||||
func NewHttpStore(params HttpParams) *HttpStore {
|
||||
return &HttpStore{
|
||||
upstream: "http://" + upstream,
|
||||
httpClient: getClient(),
|
||||
endpoint: params.Endpoint,
|
||||
httpClient: getClient(),
|
||||
prefixLength: params.ShardingSize,
|
||||
name: params.Name,
|
||||
}
|
||||
}
|
||||
|
||||
const nameHttp = "http"
|
||||
|
||||
func (n *HttpStore) Name() string { return nameHttp }
|
||||
func (n *HttpStore) Has(hash string) (bool, error) {
|
||||
url := n.upstream + "/blob?hash=" + hash
|
||||
|
||||
req, err := http.NewRequest("HEAD", url, nil)
|
||||
func HttpStoreFactory(config *viper.Viper) (BlobStore, error) {
|
||||
var cfg HttpParams
|
||||
err := config.Unmarshal(&cfg)
|
||||
if err != nil {
|
||||
return false, errors.Err(err)
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
return NewHttpStore(cfg), nil
|
||||
}
|
||||
|
||||
res, err := n.httpClient.Do(req)
|
||||
// Name is the cache type name
|
||||
func (c *HttpStore) Name() string { return nameHttp + "-" + c.name }
|
||||
|
||||
// Has checks if the hash is in the store.
|
||||
func (c *HttpStore) Has(hash string) (bool, error) {
|
||||
status, body, err := c.cfRequest(http.MethodHead, hash)
|
||||
if err != nil {
|
||||
return false, errors.Err(err)
|
||||
return false, err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode == http.StatusNotFound {
|
||||
defer func() { _ = body.Close() }()
|
||||
switch status {
|
||||
case http.StatusNotFound, http.StatusForbidden:
|
||||
return false, nil
|
||||
}
|
||||
if res.StatusCode == http.StatusNoContent {
|
||||
case http.StatusOK:
|
||||
return true, nil
|
||||
default:
|
||||
return false, errors.Err("unexpected status %d", status)
|
||||
}
|
||||
var body []byte
|
||||
if res.Body != nil {
|
||||
body, _ = ioutil.ReadAll(res.Body)
|
||||
}
|
||||
return false, errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
|
||||
}
|
||||
|
||||
func (n *HttpStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
// Get downloads the blob using the http client
|
||||
func (c *HttpStore) Get(hash string) (b stream.Blob, trace shared.BlobTrace, err error) {
|
||||
log.Debugf("Getting %s from HTTP(s) source", hash[:8])
|
||||
start := time.Now()
|
||||
url := n.upstream + "/blob?hash=" + hash
|
||||
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), n.Name()), errors.Err(err)
|
||||
}
|
||||
defer func(t time.Time) {
|
||||
log.Debugf("Getting %s from HTTP(s) source took %s", hash[:8], time.Since(t).String())
|
||||
trace = trace.Stack(time.Since(start), c.Name())
|
||||
}(start)
|
||||
|
||||
res, err := n.httpClient.Do(req)
|
||||
url := c.endpoint + c.shardedPath(hash)
|
||||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), n.Name()), errors.Err(err)
|
||||
return nil, trace, errors.Err(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
tmp := getBuffer()
|
||||
defer putBuffer(tmp)
|
||||
serialized := res.Header.Get("Via")
|
||||
trace := shared.NewBlobTrace(time.Since(start), n.Name())
|
||||
if serialized != "" {
|
||||
parsedTrace, err := shared.Deserialize(serialized)
|
||||
req.Header.Add("User-Agent", "reflector.go/"+meta.Version())
|
||||
|
||||
res, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, trace, errors.Err(err)
|
||||
}
|
||||
defer func(Body io.ReadCloser) {
|
||||
err := Body.Close()
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), n.Name()), err
|
||||
log.Errorf("Error closing response body in HTTP-GET: %s", err.Error())
|
||||
}
|
||||
}(res.Body)
|
||||
|
||||
// Parse Via header if present
|
||||
viaHeader := res.Header.Get("Via")
|
||||
if viaHeader != "" {
|
||||
parsedTrace, err := shared.Deserialize(viaHeader)
|
||||
if err == nil {
|
||||
trace = *parsedTrace
|
||||
}
|
||||
trace = *parsedTrace
|
||||
}
|
||||
|
||||
if res.StatusCode == http.StatusNotFound {
|
||||
return nil, trace.Stack(time.Since(start), n.Name()), ErrBlobNotFound
|
||||
}
|
||||
if res.StatusCode == http.StatusOK {
|
||||
written, err := io.Copy(tmp, res.Body)
|
||||
if err != nil {
|
||||
return nil, trace.Stack(time.Since(start), n.Name()), errors.Err(err)
|
||||
switch res.StatusCode {
|
||||
case http.StatusNotFound:
|
||||
return nil, trace, ErrBlobNotFound
|
||||
case http.StatusOK:
|
||||
contentLength := res.Header.Get("Content-Length")
|
||||
if contentLength != "" {
|
||||
size, err := strconv.Atoi(contentLength)
|
||||
if err == nil && size > 0 && size <= stream.MaxBlobSize {
|
||||
blob := make([]byte, size)
|
||||
_, err = io.ReadFull(res.Body, blob)
|
||||
if err == nil {
|
||||
metrics.MtrInBytesHttp.Add(float64(size))
|
||||
return blob, trace, nil
|
||||
}
|
||||
log.Warnf("Error reading body with known size: %s", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
blob := make([]byte, written)
|
||||
copy(blob, tmp.Bytes())
|
||||
buffer := getBuffer()
|
||||
defer putBuffer(buffer)
|
||||
if _, err := io.Copy(buffer, res.Body); err != nil {
|
||||
return nil, trace, errors.Err(err)
|
||||
}
|
||||
blob := make([]byte, buffer.Len())
|
||||
copy(blob, buffer.Bytes())
|
||||
metrics.MtrInBytesHttp.Add(float64(len(blob)))
|
||||
return blob, trace.Stack(time.Since(start), n.Name()), nil
|
||||
return blob, trace, nil
|
||||
default:
|
||||
body, _ := io.ReadAll(res.Body)
|
||||
log.Warnf("Got status code %d (%s)", res.StatusCode, string(body))
|
||||
return nil, trace, errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
|
||||
}
|
||||
var body []byte
|
||||
if res.Body != nil {
|
||||
body, _ = ioutil.ReadAll(res.Body)
|
||||
}
|
||||
|
||||
func (c *HttpStore) cfRequest(method, hash string) (int, io.ReadCloser, error) {
|
||||
url := c.endpoint + c.shardedPath(hash)
|
||||
req, err := http.NewRequest(method, url, nil)
|
||||
if err != nil {
|
||||
return 0, nil, errors.Err(err)
|
||||
}
|
||||
req.Header.Add("User-Agent", "reflector.go/"+meta.Version())
|
||||
|
||||
res, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return 0, nil, errors.Err(err)
|
||||
}
|
||||
|
||||
return nil, trace.Stack(time.Since(start), n.Name()), errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
|
||||
return res.StatusCode, res.Body, nil
|
||||
}
|
||||
|
||||
func (n *HttpStore) Put(string, stream.Blob) error {
|
||||
return shared.ErrNotImplemented
|
||||
}
|
||||
func (n *HttpStore) PutSD(string, stream.Blob) error {
|
||||
return shared.ErrNotImplemented
|
||||
}
|
||||
func (n *HttpStore) Delete(string) error {
|
||||
return shared.ErrNotImplemented
|
||||
}
|
||||
func (n *HttpStore) Shutdown() {}
|
||||
|
||||
// buffer pool to reduce GC
|
||||
// https://www.captaincodeman.com/2017/06/02/golang-buffer-pool-gotcha
|
||||
var buffers = sync.Pool{
|
||||
// New is called when a new instance is needed
|
||||
New: func() interface{} {
|
||||
buf := make([]byte, 0, stream.MaxBlobSize)
|
||||
return bytes.NewBuffer(buf)
|
||||
},
|
||||
func (c *HttpStore) Put(_ string, _ stream.Blob) error {
|
||||
return errors.Err(shared.ErrNotImplemented)
|
||||
}
|
||||
|
||||
// getBuffer fetches a buffer from the pool
|
||||
func getBuffer() *bytes.Buffer {
|
||||
return buffers.Get().(*bytes.Buffer)
|
||||
func (c *HttpStore) PutSD(_ string, _ stream.Blob) error {
|
||||
return errors.Err(shared.ErrNotImplemented)
|
||||
}
|
||||
|
||||
// putBuffer returns a buffer to the pool
|
||||
func putBuffer(buf *bytes.Buffer) {
|
||||
buf.Reset()
|
||||
buffers.Put(buf)
|
||||
func (c *HttpStore) Delete(_ string) error {
|
||||
return errors.Err(shared.ErrNotImplemented)
|
||||
}
|
||||
|
||||
// getClient gets an http client that's customized to be more performant when dealing with blobs of 2MB in size (most of our blobs)
|
||||
func getClient() *http.Client {
|
||||
// Customize the Transport to have larger connection pool
|
||||
defaultTransport := &http.Transport{
|
||||
DialContext: (&net.Dialer{
|
||||
Timeout: 30 * time.Second,
|
||||
KeepAlive: 30 * time.Second,
|
||||
}).DialContext,
|
||||
ForceAttemptHTTP2: true,
|
||||
MaxIdleConns: 100,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
TLSHandshakeTimeout: 10 * time.Second,
|
||||
ExpectContinueTimeout: 1 * time.Second,
|
||||
DisableCompression: true,
|
||||
MaxIdleConnsPerHost: 100,
|
||||
ReadBufferSize: stream.MaxBlobSize + 1024*10, //add an extra few KBs to make sure it fits the extra information
|
||||
// Shutdown shuts down the store gracefully
|
||||
func (c *HttpStore) Shutdown() {
|
||||
}
|
||||
|
||||
func (c *HttpStore) shardedPath(hash string) string {
|
||||
if c.prefixLength <= 0 || len(hash) < c.prefixLength {
|
||||
return hash
|
||||
}
|
||||
return path.Join(hash[:c.prefixLength], hash)
|
||||
}
|
||||
|
||||
return &http.Client{Transport: defaultTransport}
|
||||
func init() {
|
||||
RegisterStore(nameHttp, HttpStoreFactory)
|
||||
}
|
||||
+136
@@ -0,0 +1,136 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
// Http3Store is a blob store that gets blobs from a peer over HTTP3.
|
||||
// It satisfies the BlobStore interface but cannot put or delete blobs.
|
||||
type Http3Store struct {
|
||||
NotFoundCache *sync.Map
|
||||
client *Http3Client
|
||||
name string
|
||||
address string
|
||||
timeout time.Duration
|
||||
clientMu sync.RWMutex
|
||||
}
|
||||
|
||||
// Http3Params allows to set options for a new Http3Store.
|
||||
type Http3Params struct {
|
||||
Name string `mapstructure:"name"`
|
||||
Address string `mapstructure:"address"`
|
||||
Timeout time.Duration `mapstructure:"timeout"`
|
||||
}
|
||||
|
||||
// NewHttp3Store makes a new HTTP3 store.
|
||||
func NewHttp3Store(params Http3Params) *Http3Store {
|
||||
return &Http3Store{
|
||||
name: params.Name,
|
||||
NotFoundCache: &sync.Map{},
|
||||
address: params.Address,
|
||||
timeout: params.Timeout,
|
||||
}
|
||||
}
|
||||
|
||||
const nameHttp3 = "http3"
|
||||
|
||||
func Http3StoreFactory(config *viper.Viper) (BlobStore, error) {
|
||||
var cfg Http3Params
|
||||
err := config.Unmarshal(&cfg)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
return NewHttp3Store(cfg), nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
RegisterStore(nameHttp3, Http3StoreFactory)
|
||||
}
|
||||
|
||||
func (h *Http3Store) Name() string { return nameHttp3 + "-" + h.name }
|
||||
|
||||
func (h *Http3Store) getClient() (*Http3Client, error) {
|
||||
h.clientMu.RLock()
|
||||
if h.client != nil {
|
||||
client := h.client
|
||||
h.clientMu.RUnlock()
|
||||
return client, nil
|
||||
}
|
||||
h.clientMu.RUnlock()
|
||||
|
||||
h.clientMu.Lock()
|
||||
defer h.clientMu.Unlock()
|
||||
|
||||
// Check again in case another goroutine created the client
|
||||
if h.client != nil {
|
||||
return h.client, nil
|
||||
}
|
||||
|
||||
client, err := NewHttp3Client(h.address)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
h.client = client
|
||||
return client, nil
|
||||
}
|
||||
|
||||
// Has asks the peer if they have a hash
|
||||
func (h *Http3Store) Has(hash string) (bool, error) {
|
||||
c, err := h.getClient()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return c.HasBlob(hash)
|
||||
}
|
||||
|
||||
// Get downloads the blob from the peer
|
||||
func (h *Http3Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
start := time.Now()
|
||||
if lastChecked, ok := h.NotFoundCache.Load(hash); ok {
|
||||
if lastChecked.(time.Time).After(time.Now().Add(-5 * time.Minute)) {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), h.Name()+"-notfoundcache"), ErrBlobNotFound
|
||||
}
|
||||
}
|
||||
c, err := h.getClient()
|
||||
if err != nil && strings.Contains(err.Error(), "blob not found") {
|
||||
h.NotFoundCache.Store(hash, time.Now())
|
||||
}
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), h.Name()), err
|
||||
}
|
||||
return c.GetBlob(hash)
|
||||
}
|
||||
|
||||
// Put is not supported
|
||||
func (h *Http3Store) Put(hash string, blob stream.Blob) error {
|
||||
return errors.Err(shared.ErrNotImplemented)
|
||||
}
|
||||
|
||||
// PutSD is not supported
|
||||
func (h *Http3Store) PutSD(hash string, blob stream.Blob) error {
|
||||
return errors.Err(shared.ErrNotImplemented)
|
||||
}
|
||||
|
||||
// Delete is not supported
|
||||
func (h *Http3Store) Delete(hash string) error {
|
||||
return errors.Err(shared.ErrNotImplemented)
|
||||
}
|
||||
|
||||
// Shutdown shuts down the store gracefully
|
||||
func (h *Http3Store) Shutdown() {
|
||||
h.clientMu.Lock()
|
||||
defer h.clientMu.Unlock()
|
||||
if h.client != nil {
|
||||
_ = h.client.Close()
|
||||
h.client = nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
"github.com/quic-go/quic-go"
|
||||
"github.com/quic-go/quic-go/http3"
|
||||
)
|
||||
|
||||
// Http3Client is a client for HTTP3 blob store
|
||||
type Http3Client struct {
|
||||
conn *http.Client
|
||||
roundTripper *http3.Transport
|
||||
ServerAddr string
|
||||
}
|
||||
|
||||
// NewHttp3Client creates a new HTTP3 client
|
||||
func NewHttp3Client(address string) (*Http3Client, error) {
|
||||
var qconf quic.Config
|
||||
window500M := 500 * 1 << 20
|
||||
qconf.MaxStreamReceiveWindow = uint64(window500M)
|
||||
qconf.MaxConnectionReceiveWindow = uint64(window500M)
|
||||
qconf.EnableDatagrams = true
|
||||
qconf.HandshakeIdleTimeout = 4 * time.Second
|
||||
qconf.MaxIdleTimeout = 20 * time.Second
|
||||
pool, err := x509.SystemCertPool()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
roundTripper := &http3.Transport{
|
||||
TLSClientConfig: &tls.Config{
|
||||
RootCAs: pool,
|
||||
InsecureSkipVerify: true,
|
||||
},
|
||||
QUICConfig: &qconf,
|
||||
}
|
||||
connection := &http.Client{
|
||||
Transport: roundTripper,
|
||||
}
|
||||
return &Http3Client{
|
||||
conn: connection,
|
||||
roundTripper: roundTripper,
|
||||
ServerAddr: address,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Close closes the client
|
||||
func (c *Http3Client) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// HasBlob checks if the peer has a blob
|
||||
func (c *Http3Client) HasBlob(hash string) (bool, error) {
|
||||
url := c.ServerAddr + "/blob?hash=" + hash
|
||||
req, err := http.NewRequest("HEAD", url, nil)
|
||||
if err != nil {
|
||||
return false, errors.Err(err)
|
||||
}
|
||||
|
||||
res, err := c.conn.Do(req)
|
||||
if err != nil {
|
||||
return false, errors.Err(err)
|
||||
}
|
||||
defer func() { _ = res.Body.Close() }()
|
||||
if res.StatusCode == http.StatusNotFound {
|
||||
return false, nil
|
||||
}
|
||||
if res.StatusCode == http.StatusNoContent {
|
||||
return true, nil
|
||||
}
|
||||
var body []byte
|
||||
if res.Body != nil {
|
||||
body, _ = io.ReadAll(res.Body)
|
||||
}
|
||||
return false, errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
|
||||
}
|
||||
|
||||
// GetBlob gets a blob from the peer
|
||||
func (c *Http3Client) GetBlob(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
start := time.Now()
|
||||
url := c.ServerAddr + "/blob?hash=" + hash
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err(err)
|
||||
}
|
||||
|
||||
res, err := c.conn.Do(req)
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err(err)
|
||||
}
|
||||
defer func() { _ = res.Body.Close() }()
|
||||
|
||||
viaHeader := res.Header.Get("Via")
|
||||
var trace shared.BlobTrace
|
||||
if viaHeader != "" {
|
||||
parsedTrace, err := shared.Deserialize(viaHeader)
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), "http3"), err
|
||||
}
|
||||
trace = *parsedTrace
|
||||
} else {
|
||||
trace = shared.NewBlobTrace(0, "http3")
|
||||
}
|
||||
|
||||
switch res.StatusCode {
|
||||
case http.StatusNotFound:
|
||||
return nil, trace.Stack(time.Since(start), "http3"), ErrBlobNotFound
|
||||
|
||||
case http.StatusOK:
|
||||
buffer := getBuffer()
|
||||
defer putBuffer(buffer)
|
||||
|
||||
if _, err := io.Copy(buffer, res.Body); err != nil {
|
||||
return nil, trace.Stack(time.Since(start), "http3"), errors.Err(err)
|
||||
}
|
||||
|
||||
blob := make([]byte, buffer.Len())
|
||||
copy(blob, buffer.Bytes())
|
||||
|
||||
return blob, trace.Stack(time.Since(start), "http3"), nil
|
||||
|
||||
default:
|
||||
body, _ := io.ReadAll(res.Body)
|
||||
return nil, trace.Stack(time.Since(start), "http3"),
|
||||
errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
|
||||
}
|
||||
}
|
||||
+70
-7
@@ -1,6 +1,7 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
@@ -8,25 +9,84 @@ import (
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
// ITTTStore performs an operation on this storage, if this fails, it attempts to run it on that
|
||||
type ITTTStore struct {
|
||||
this, that BlobStore
|
||||
name string
|
||||
}
|
||||
|
||||
type ITTTParams struct {
|
||||
This BlobStore `mapstructure:"this"`
|
||||
That BlobStore `mapstructure:"that"`
|
||||
Name string `mapstructure:"name"`
|
||||
}
|
||||
|
||||
type ITTTConfig struct {
|
||||
This *viper.Viper
|
||||
That *viper.Viper
|
||||
Name string `mapstructure:"name"`
|
||||
}
|
||||
|
||||
// NewITTTStore returns a new instance of the IF THIS THAN THAT store
|
||||
func NewITTTStore(this, that BlobStore) *ITTTStore {
|
||||
func NewITTTStore(params ITTTParams) *ITTTStore {
|
||||
return &ITTTStore{
|
||||
this: this,
|
||||
that: that,
|
||||
this: params.This,
|
||||
that: params.That,
|
||||
name: params.Name,
|
||||
}
|
||||
}
|
||||
|
||||
const nameIttt = "ittt"
|
||||
|
||||
func ITTTStoreFactory(config *viper.Viper) (BlobStore, error) {
|
||||
var cfg ITTTConfig
|
||||
err := config.Unmarshal(&cfg)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
cfg.This = config.Sub("this")
|
||||
cfg.That = config.Sub("that")
|
||||
|
||||
thisStoreType := strings.Split(cfg.This.AllKeys()[0], ".")[0]
|
||||
thisStoreConfig := cfg.This.Sub(thisStoreType)
|
||||
factory, ok := Factories[thisStoreType]
|
||||
if !ok {
|
||||
return nil, errors.Err("unknown store type %s", thisStoreType)
|
||||
}
|
||||
thisStore, err := factory(thisStoreConfig)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
thatStoreType := strings.Split(cfg.That.AllKeys()[0], ".")[0]
|
||||
thatStoreConfig := cfg.That.Sub(thatStoreType)
|
||||
factory, ok = Factories[thatStoreType]
|
||||
if !ok {
|
||||
return nil, errors.Err("unknown store type %s", thatStoreType)
|
||||
}
|
||||
thatStore, err := factory(thatStoreConfig)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
return NewITTTStore(ITTTParams{
|
||||
Name: cfg.Name,
|
||||
This: thisStore,
|
||||
That: thatStore,
|
||||
}), nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
RegisterStore(nameIttt, ITTTStoreFactory)
|
||||
}
|
||||
|
||||
// Name is the cache type name
|
||||
func (c *ITTTStore) Name() string { return nameIttt }
|
||||
func (c *ITTTStore) Name() string { return nameIttt + "-" + c.name }
|
||||
|
||||
// Has checks in this for a hash, if it fails it checks in that. It returns true if either store has it.
|
||||
func (c *ITTTStore) Has(hash string) (bool, error) {
|
||||
@@ -42,7 +102,7 @@ func (c *ITTTStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
start := time.Now()
|
||||
blob, trace, err := c.this.Get(hash)
|
||||
if err == nil {
|
||||
metrics.ThisHitCount.Inc()
|
||||
metrics.ItttHitCount.With(metrics.ItttLabels(c.this.Name())).Inc()
|
||||
return blob, trace.Stack(time.Since(start), c.Name()), err
|
||||
}
|
||||
|
||||
@@ -50,7 +110,7 @@ func (c *ITTTStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
if err != nil {
|
||||
return nil, trace.Stack(time.Since(start), c.Name()), err
|
||||
}
|
||||
metrics.ThatHitCount.Inc()
|
||||
metrics.ItttHitCount.With(metrics.ItttLabels(c.that.Name())).Inc()
|
||||
return blob, trace.Stack(time.Since(start), c.Name()), nil
|
||||
}
|
||||
|
||||
@@ -70,4 +130,7 @@ func (c *ITTTStore) Delete(hash string) error {
|
||||
}
|
||||
|
||||
// Shutdown shuts down the store gracefully
|
||||
func (c *ITTTStore) Shutdown() {}
|
||||
func (c *ITTTStore) Shutdown() {
|
||||
c.this.Shutdown()
|
||||
c.that.Shutdown()
|
||||
}
|
||||
+23
-2
@@ -8,25 +8,46 @@ import (
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
// MemStore is an in memory only blob store with no persistence.
|
||||
type MemStore struct {
|
||||
blobs map[string]stream.Blob
|
||||
mu *sync.RWMutex
|
||||
name string
|
||||
}
|
||||
|
||||
func NewMemStore() *MemStore {
|
||||
type MemParams struct {
|
||||
Name string `mapstructure:"name"`
|
||||
}
|
||||
|
||||
func NewMemStore(params MemParams) *MemStore {
|
||||
return &MemStore{
|
||||
blobs: make(map[string]stream.Blob),
|
||||
mu: &sync.RWMutex{},
|
||||
name: params.Name,
|
||||
}
|
||||
}
|
||||
|
||||
const nameMem = "mem"
|
||||
|
||||
func MemStoreFactory(config *viper.Viper) (BlobStore, error) {
|
||||
var cfg MemParams
|
||||
err := config.Unmarshal(&cfg)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
return NewMemStore(cfg), nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
RegisterStore(nameMem, MemStoreFactory)
|
||||
}
|
||||
|
||||
// Name is the cache type name
|
||||
func (m *MemStore) Name() string { return nameMem }
|
||||
func (m *MemStore) Name() string { return nameMem + "-" + m.name }
|
||||
|
||||
// Has returns T/F if the blob is currently stored. It will never error.
|
||||
func (m *MemStore) Has(hash string) (bool, error) {
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
)
|
||||
|
||||
func TestMemStore_Put(t *testing.T) {
|
||||
s := NewMemStore()
|
||||
s := NewMemStore(MemParams{Name: "test"})
|
||||
blob := []byte("abcdefg")
|
||||
err := s.Put("abc", blob)
|
||||
if err != nil {
|
||||
@@ -17,7 +17,7 @@ func TestMemStore_Put(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestMemStore_Get(t *testing.T) {
|
||||
s := NewMemStore()
|
||||
s := NewMemStore(MemParams{Name: "test"})
|
||||
hash := "abc"
|
||||
blob := []byte("abcdefg")
|
||||
err := s.Put(hash, blob)
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
// MultiWriterStore writes to multiple destination stores
|
||||
type MultiWriterStore struct {
|
||||
name string
|
||||
destinations []BlobStore
|
||||
}
|
||||
|
||||
type MultiWriterParams struct {
|
||||
Name string
|
||||
Destinations []BlobStore
|
||||
}
|
||||
|
||||
type MultiWriterConfig struct {
|
||||
Name string `mapstructure:"name"`
|
||||
One viper.Viper
|
||||
Two viper.Viper
|
||||
Three viper.Viper
|
||||
}
|
||||
|
||||
// NewMultiWriterStore returns a new instance of the MultiWriter store
|
||||
func NewMultiWriterStore(params MultiWriterParams) *MultiWriterStore {
|
||||
return &MultiWriterStore{
|
||||
name: params.Name,
|
||||
destinations: params.Destinations,
|
||||
}
|
||||
}
|
||||
|
||||
const nameMultiWriter = "multiwriter"
|
||||
|
||||
func MultiWriterStoreFactory(config *viper.Viper) (BlobStore, error) {
|
||||
var cfg MultiWriterConfig
|
||||
err := config.Unmarshal(&cfg)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
var destinations []BlobStore
|
||||
|
||||
one := config.Sub("one")
|
||||
two := config.Sub("two")
|
||||
//three := config.Sub("three")
|
||||
|
||||
storeTypeOne := strings.Split(one.AllKeys()[0], ".")[0]
|
||||
storeTypeTwo := strings.Split(two.AllKeys()[0], ".")[0]
|
||||
//storeTypeThree := strings.Split(three.AllKeys()[0], ".")[0]
|
||||
|
||||
storeCfgOne := one.Sub(storeTypeOne)
|
||||
storeCfgTwo := two.Sub(storeTypeTwo)
|
||||
//storeCfgThree := config.Sub(storeTypeThree)
|
||||
|
||||
factoryOne, ok := Factories[storeTypeOne]
|
||||
if !ok {
|
||||
return nil, errors.Err("unknown store type %s", storeTypeOne)
|
||||
}
|
||||
factoryTwo, ok := Factories[storeTypeTwo]
|
||||
if !ok {
|
||||
return nil, errors.Err("unknown store type %s", storeTypeTwo)
|
||||
}
|
||||
//factoryThree, ok := Factories[storeTypeThree]
|
||||
//if !ok {
|
||||
// return nil, errors.Err("unknown store type %s", storeTypeThree)
|
||||
//}
|
||||
|
||||
store1, err := factoryOne(storeCfgOne)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
store2, err := factoryTwo(storeCfgTwo)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
//store3, err := factoryThree(storeCfgThree)
|
||||
//if err != nil {
|
||||
// return nil, errors.Err(err)
|
||||
//}
|
||||
destinations = append(destinations, store1, store2)
|
||||
|
||||
return NewMultiWriterStore(MultiWriterParams{
|
||||
Name: cfg.Name,
|
||||
Destinations: destinations,
|
||||
}), nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
RegisterStore(nameMultiWriter, MultiWriterStoreFactory)
|
||||
}
|
||||
|
||||
// Name returns the store name
|
||||
func (m *MultiWriterStore) Name() string { return nameMultiWriter + "-" + m.name }
|
||||
|
||||
// Has is not supported by MultiWriter
|
||||
func (m *MultiWriterStore) Has(hash string) (bool, error) {
|
||||
return false, errors.Err(shared.ErrNotImplemented)
|
||||
}
|
||||
|
||||
// Get is not supported by MultiWriter
|
||||
func (m *MultiWriterStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
return nil, shared.BlobTrace{}, errors.Err(shared.ErrNotImplemented)
|
||||
}
|
||||
|
||||
// Put writes the blob to all destination stores
|
||||
func (m *MultiWriterStore) Put(hash string, blob stream.Blob) error {
|
||||
var wg sync.WaitGroup
|
||||
errChan := make(chan error, len(m.destinations))
|
||||
|
||||
for _, dest := range m.destinations {
|
||||
wg.Add(1)
|
||||
go func(d BlobStore) {
|
||||
defer wg.Done()
|
||||
if err := d.Put(hash, blob); err != nil {
|
||||
errChan <- errors.Err("failed to write to %s: %v", d.Name(), err)
|
||||
}
|
||||
}(dest)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
close(errChan)
|
||||
|
||||
var errs []string
|
||||
for err := range errChan {
|
||||
errs = append(errs, err.Error())
|
||||
}
|
||||
|
||||
if len(errs) > 0 {
|
||||
return errors.Err("failed to write to some destinations: %s", strings.Join(errs, "; "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// PutSD writes the SD blob to all destination stores
|
||||
func (m *MultiWriterStore) PutSD(hash string, blob stream.Blob) error {
|
||||
var wg sync.WaitGroup
|
||||
errChan := make(chan error, len(m.destinations))
|
||||
|
||||
for _, dest := range m.destinations {
|
||||
wg.Add(1)
|
||||
go func(d BlobStore) {
|
||||
defer wg.Done()
|
||||
if err := d.PutSD(hash, blob); err != nil {
|
||||
errChan <- errors.Err("failed to write SD to %s: %v", d.Name(), err)
|
||||
}
|
||||
}(dest)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
close(errChan)
|
||||
|
||||
var errs []string
|
||||
for err := range errChan {
|
||||
errs = append(errs, err.Error())
|
||||
}
|
||||
|
||||
if len(errs) > 0 {
|
||||
return errors.Err("failed to write SD to some destinations: %s", strings.Join(errs, "; "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete deletes the blob from all destination stores
|
||||
func (m *MultiWriterStore) Delete(hash string) error {
|
||||
var wg sync.WaitGroup
|
||||
errChan := make(chan error, len(m.destinations))
|
||||
|
||||
for _, dest := range m.destinations {
|
||||
wg.Add(1)
|
||||
go func(d BlobStore) {
|
||||
defer wg.Done()
|
||||
if err := d.Delete(hash); err != nil {
|
||||
errChan <- errors.Err("failed to delete from %s: %v", d.Name(), err)
|
||||
}
|
||||
}(dest)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
close(errChan)
|
||||
|
||||
var errs []string
|
||||
for err := range errChan {
|
||||
errs = append(errs, err.Error())
|
||||
}
|
||||
|
||||
if len(errs) > 0 {
|
||||
return errors.Err("failed to delete from some destinations: %s", strings.Join(errs, "; "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Shutdown shuts down all destination stores gracefully
|
||||
func (m *MultiWriterStore) Shutdown() {
|
||||
for _, dest := range m.destinations {
|
||||
dest.Shutdown()
|
||||
}
|
||||
}
|
||||
+27
-3
@@ -5,15 +5,39 @@ import (
|
||||
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
// NoopStore is a store that does nothing
|
||||
type NoopStore struct{}
|
||||
type NoopStore struct {
|
||||
name string
|
||||
}
|
||||
|
||||
func NewNoopStore(name string) *NoopStore {
|
||||
return &NoopStore{name: name}
|
||||
}
|
||||
|
||||
const nameNoop = "noop"
|
||||
|
||||
func (n *NoopStore) Name() string { return nameNoop }
|
||||
func NoopStoreFactory(config *viper.Viper) (BlobStore, error) {
|
||||
var cfg struct {
|
||||
Name string `mapstructure:"name"`
|
||||
}
|
||||
err := config.Unmarshal(&cfg)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
return NewNoopStore(cfg.Name), nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
RegisterStore(nameNoop, NoopStoreFactory)
|
||||
}
|
||||
|
||||
func (n *NoopStore) Name() string { return nameNoop + "-" + n.name }
|
||||
func (n *NoopStore) Has(_ string) (bool, error) { return false, nil }
|
||||
func (n *NoopStore) Get(_ string) (stream.Blob, shared.BlobTrace, error) {
|
||||
return nil, shared.NewBlobTrace(time.Since(time.Now()), n.Name()), nil
|
||||
@@ -21,4 +45,4 @@ func (n *NoopStore) Get(_ string) (stream.Blob, shared.BlobTrace, error) {
|
||||
func (n *NoopStore) Put(_ string, _ stream.Blob) error { return nil }
|
||||
func (n *NoopStore) PutSD(_ string, _ stream.Blob) error { return nil }
|
||||
func (n *NoopStore) Delete(_ string) error { return nil }
|
||||
func (n *NoopStore) Shutdown() { return }
|
||||
func (n *NoopStore) Shutdown() {}
|
||||
+100
@@ -0,0 +1,100 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
// PeerStore is a blob store that gets blobs from a peer.
|
||||
// It satisfies the BlobStore interface but cannot put or delete blobs.
|
||||
type PeerStore struct {
|
||||
name string
|
||||
opts PeerParams
|
||||
}
|
||||
|
||||
// PeerParams allows to set options for a new PeerStore.
|
||||
type PeerParams struct {
|
||||
Name string `mapstructure:"name"`
|
||||
Address string `mapstructure:"address"`
|
||||
Timeout time.Duration `mapstructure:"timeout"`
|
||||
}
|
||||
|
||||
// NewPeerStore makes a new peer store.
|
||||
func NewPeerStore(params PeerParams) *PeerStore {
|
||||
return &PeerStore{opts: params, name: params.Name}
|
||||
}
|
||||
|
||||
const namePeer = "peer"
|
||||
|
||||
func PeerStoreFactory(config *viper.Viper) (BlobStore, error) {
|
||||
var cfg PeerParams
|
||||
err := config.Unmarshal(&cfg)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
return NewPeerStore(cfg), nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
RegisterStore(namePeer, PeerStoreFactory)
|
||||
}
|
||||
|
||||
func (p *PeerStore) Name() string { return namePeer + "-" + p.name }
|
||||
|
||||
func (p *PeerStore) getClient() (*PeerClient, error) {
|
||||
c := &PeerClient{Timeout: p.opts.Timeout}
|
||||
err := c.Connect(p.opts.Address)
|
||||
return c, errors.Prefix("connection error", err)
|
||||
}
|
||||
|
||||
// Has asks the peer if they have a hash
|
||||
func (p *PeerStore) Has(hash string) (bool, error) {
|
||||
c, err := p.getClient()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer func() { _ = c.Close() }()
|
||||
return c.HasBlob(hash)
|
||||
}
|
||||
|
||||
// Get downloads the blob from the peer
|
||||
func (p *PeerStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
start := time.Now()
|
||||
c, err := p.getClient()
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), p.Name()), err
|
||||
}
|
||||
defer func() { _ = c.Close() }()
|
||||
blob, trace, err := c.GetBlob(hash)
|
||||
if err != nil && strings.Contains(err.Error(), "blob not found") {
|
||||
return nil, trace, ErrBlobNotFound
|
||||
}
|
||||
|
||||
return blob, trace, err
|
||||
}
|
||||
|
||||
// Put is not supported
|
||||
func (p *PeerStore) Put(hash string, blob stream.Blob) error {
|
||||
return errors.Err(shared.ErrNotImplemented)
|
||||
}
|
||||
|
||||
// PutSD is not supported
|
||||
func (p *PeerStore) PutSD(hash string, blob stream.Blob) error {
|
||||
return errors.Err(shared.ErrNotImplemented)
|
||||
}
|
||||
|
||||
// Delete is not supported
|
||||
func (p *PeerStore) Delete(hash string) error {
|
||||
return errors.Err(shared.ErrNotImplemented)
|
||||
}
|
||||
|
||||
// Shutdown is not supported
|
||||
func (p *PeerStore) Shutdown() {
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"io"
|
||||
"net"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
)
|
||||
|
||||
// PeerClient is a client for peer blob store
|
||||
type PeerClient struct {
|
||||
conn net.Conn
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
// Connect connects to a peer
|
||||
func (c *PeerClient) Connect(address string) error {
|
||||
var err error
|
||||
c.conn, err = net.DialTimeout("tcp", address, c.Timeout)
|
||||
return err
|
||||
}
|
||||
|
||||
// Close closes the connection
|
||||
func (c *PeerClient) Close() error {
|
||||
if c.conn != nil {
|
||||
return c.conn.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// HasBlob checks if the peer has a blob
|
||||
func (c *PeerClient) HasBlob(hash string) (bool, error) {
|
||||
err := c.writeRequest("has", hash)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
response, err := c.readResponse()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
return response == "yes", nil
|
||||
}
|
||||
|
||||
// GetBlob gets a blob from the peer
|
||||
func (c *PeerClient) GetBlob(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
start := time.Now()
|
||||
err := c.writeRequest("get", hash)
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), "peer"), err
|
||||
}
|
||||
|
||||
response, err := c.readResponse()
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), "peer"), err
|
||||
}
|
||||
|
||||
if response == "no" {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), "peer"), ErrBlobNotFound
|
||||
}
|
||||
|
||||
size, err := binary.ReadVarint(bufio.NewReader(bytes.NewReader([]byte(response))))
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), "peer"), err
|
||||
}
|
||||
|
||||
blob := make([]byte, size)
|
||||
_, err = io.ReadFull(c.conn, blob)
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), "peer"), err
|
||||
}
|
||||
|
||||
return blob, shared.NewBlobTrace(time.Since(start), "peer"), nil
|
||||
}
|
||||
|
||||
func (c *PeerClient) writeRequest(cmd, hash string) error {
|
||||
_, err := c.conn.Write([]byte(cmd + " " + hash + "\n"))
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *PeerClient) readResponse() (string, error) {
|
||||
reader := bufio.NewReader(c.conn)
|
||||
response, err := reader.ReadString('\n')
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return response[:len(response)-1], nil
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
// ProxiedS3Store writes to an S3 store and reads from any BlobStore (usually an ITTTStore of HttpStore endpoints).
|
||||
type ProxiedS3Store struct {
|
||||
readerStore BlobStore
|
||||
writerStore BlobStore
|
||||
name string
|
||||
}
|
||||
|
||||
func (c *ProxiedS3Store) MissingBlobsForKnownStream(s string) ([]string, error) {
|
||||
if bc, ok := c.writerStore.(NeededBlobChecker); ok {
|
||||
return bc.MissingBlobsForKnownStream(s)
|
||||
}
|
||||
return nil, errors.Err("writer does not implement neededBlobChecker")
|
||||
}
|
||||
|
||||
func (c *ProxiedS3Store) Block(hash string) error {
|
||||
if bl, ok := c.writerStore.(Blocklister); ok {
|
||||
return bl.Block(hash)
|
||||
}
|
||||
return errors.Err("writer does not implement Blocklister")
|
||||
}
|
||||
|
||||
func (c *ProxiedS3Store) Wants(hash string) (bool, error) {
|
||||
if bl, ok := c.writerStore.(Blocklister); ok {
|
||||
return bl.Wants(hash)
|
||||
}
|
||||
return true, errors.Err("writer does not implement Blocklister")
|
||||
}
|
||||
|
||||
type ProxiedS3Params struct {
|
||||
Reader BlobStore `mapstructure:"reader"`
|
||||
Writer BlobStore `mapstructure:"writer"`
|
||||
Name string `mapstructure:"name"`
|
||||
}
|
||||
|
||||
type ProxiedS3Config struct {
|
||||
Reader *viper.Viper
|
||||
Writer *viper.Viper
|
||||
Name string `mapstructure:"name"`
|
||||
}
|
||||
|
||||
// NewProxiedS3Store returns an initialized ProxiedS3Store store pointer.
|
||||
// NOTE: It panics if either argument is nil.
|
||||
func NewProxiedS3Store(params ProxiedS3Params) *ProxiedS3Store {
|
||||
if params.Reader == nil || params.Writer == nil {
|
||||
panic("both stores must be set")
|
||||
}
|
||||
return &ProxiedS3Store{
|
||||
readerStore: params.Reader,
|
||||
writerStore: params.Writer,
|
||||
name: params.Name,
|
||||
}
|
||||
}
|
||||
|
||||
const nameProxiedS3 = "proxied-s3"
|
||||
|
||||
// Name is the cache type name
|
||||
func (c *ProxiedS3Store) Name() string { return nameProxiedS3 + "-" + c.name }
|
||||
|
||||
// Has checks if the hash is in the store.
|
||||
func (c *ProxiedS3Store) Has(hash string) (bool, error) {
|
||||
return c.writerStore.Has(hash)
|
||||
}
|
||||
|
||||
// Get gets the blob from Cloudfront.
|
||||
func (c *ProxiedS3Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
start := time.Now()
|
||||
blob, trace, err := c.readerStore.Get(hash)
|
||||
return blob, trace.Stack(time.Since(start), c.Name()), err
|
||||
}
|
||||
|
||||
// Put stores the blob on S3
|
||||
func (c *ProxiedS3Store) Put(hash string, blob stream.Blob) error {
|
||||
return c.writerStore.Put(hash, blob)
|
||||
}
|
||||
|
||||
// PutSD stores the sd blob on S3
|
||||
func (c *ProxiedS3Store) PutSD(hash string, blob stream.Blob) error {
|
||||
return c.writerStore.PutSD(hash, blob)
|
||||
}
|
||||
|
||||
// Delete deletes the blob from S3
|
||||
func (c *ProxiedS3Store) Delete(hash string) error {
|
||||
return c.writerStore.Delete(hash)
|
||||
}
|
||||
|
||||
// Shutdown shuts down the store gracefully
|
||||
func (c *ProxiedS3Store) Shutdown() {
|
||||
c.writerStore.Shutdown()
|
||||
c.readerStore.Shutdown()
|
||||
}
|
||||
|
||||
func ProxiedS3StoreFactory(config *viper.Viper) (BlobStore, error) {
|
||||
var cfg ProxiedS3Config
|
||||
err := config.Unmarshal(&cfg)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
cfg.Reader = config.Sub("reader")
|
||||
cfg.Writer = config.Sub("writer")
|
||||
|
||||
readerStoreType := strings.Split(cfg.Reader.AllKeys()[0], ".")[0]
|
||||
readerStoreConfig := cfg.Reader.Sub(readerStoreType)
|
||||
factory, ok := Factories[readerStoreType]
|
||||
if !ok {
|
||||
return nil, errors.Err("unknown store type %s", readerStoreType)
|
||||
}
|
||||
readerStore, err := factory(readerStoreConfig)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
writerStoreType := strings.Split(cfg.Writer.AllKeys()[0], ".")[0]
|
||||
writerStoreConfig := cfg.Writer.Sub(writerStoreType)
|
||||
factory, ok = Factories[writerStoreType]
|
||||
if !ok {
|
||||
return nil, errors.Err("unknown store type %s", writerStoreType)
|
||||
}
|
||||
writerStore, err := factory(writerStoreConfig)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
return NewProxiedS3Store(ProxiedS3Params{
|
||||
Name: cfg.Name,
|
||||
Reader: readerStore,
|
||||
Writer: writerStore,
|
||||
}), nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
RegisterStore(nameProxiedS3, ProxiedS3StoreFactory)
|
||||
}
|
||||
+103
-55
@@ -2,7 +2,9 @@ package store
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/http"
|
||||
"context"
|
||||
stderrors "errors"
|
||||
"path"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
@@ -11,56 +13,88 @@ import (
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
awsconfig "github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
// S3Store is an S3 store
|
||||
type S3Store struct {
|
||||
awsID string
|
||||
awsSecret string
|
||||
region string
|
||||
bucket string
|
||||
client *s3.Client
|
||||
awsID string
|
||||
awsSecret string
|
||||
region string
|
||||
bucket string
|
||||
endpoint string
|
||||
name string
|
||||
prefixLength int
|
||||
}
|
||||
|
||||
session *session.Session
|
||||
type S3Params struct {
|
||||
Name string `mapstructure:"name"`
|
||||
AwsID string `mapstructure:"aws_id"`
|
||||
AwsSecret string `mapstructure:"aws_secret"`
|
||||
Region string `mapstructure:"region"`
|
||||
Bucket string `mapstructure:"bucket"`
|
||||
Endpoint string `mapstructure:"endpoint"`
|
||||
ShardingSize int `mapstructure:"sharding_size"`
|
||||
}
|
||||
|
||||
// NewS3Store returns an initialized S3 store pointer.
|
||||
func NewS3Store(awsID, awsSecret, region, bucket string) *S3Store {
|
||||
func NewS3Store(params S3Params) *S3Store {
|
||||
return &S3Store{
|
||||
awsID: awsID,
|
||||
awsSecret: awsSecret,
|
||||
region: region,
|
||||
bucket: bucket,
|
||||
awsID: params.AwsID,
|
||||
awsSecret: params.AwsSecret,
|
||||
region: params.Region,
|
||||
bucket: params.Bucket,
|
||||
endpoint: params.Endpoint,
|
||||
name: params.Name,
|
||||
prefixLength: params.ShardingSize,
|
||||
}
|
||||
}
|
||||
|
||||
const nameS3 = "s3"
|
||||
|
||||
// Name is the cache type name
|
||||
func (s *S3Store) Name() string { return nameS3 }
|
||||
func S3StoreFactory(config *viper.Viper) (BlobStore, error) {
|
||||
var cfg S3Params
|
||||
err := config.Unmarshal(&cfg)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
return NewS3Store(cfg), nil
|
||||
}
|
||||
|
||||
// Has returns T/F or Error ( from S3 ) if the store contains the blob.
|
||||
func init() {
|
||||
RegisterStore(nameS3, S3StoreFactory)
|
||||
}
|
||||
|
||||
// Name is the cache type name
|
||||
func (s *S3Store) Name() string { return nameS3 + "-" + s.name }
|
||||
|
||||
// Has returns T/F or Error (from S3) if the store contains the blob.
|
||||
func (s *S3Store) Has(hash string) (bool, error) {
|
||||
err := s.initOnce()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
_, err = s3.New(s.session).HeadObject(&s3.HeadObjectInput{
|
||||
ctx := context.Background()
|
||||
_, err = s.client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(hash),
|
||||
Key: aws.String(s.shardedPath(hash)),
|
||||
})
|
||||
if err != nil {
|
||||
if reqFail, ok := err.(s3.RequestFailure); ok && reqFail.StatusCode() == http.StatusNotFound {
|
||||
var notFound *types.NotFound
|
||||
var noSuchKey *types.NoSuchKey
|
||||
if stderrors.As(err, ¬Found) || stderrors.As(err, &noSuchKey) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
return false, errors.Err(err)
|
||||
}
|
||||
|
||||
return true, nil
|
||||
@@ -69,7 +103,6 @@ func (s *S3Store) Has(hash string) (bool, error) {
|
||||
// Get returns the blob slice if present or errors on S3.
|
||||
func (s *S3Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
start := time.Now()
|
||||
//Todo-Need to handle error for blob doesn't exist for consistency.
|
||||
err := s.initOnce()
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), s.Name()), err
|
||||
@@ -77,24 +110,26 @@ func (s *S3Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
|
||||
log.Debugf("Getting %s from S3", hash[:8])
|
||||
defer func(t time.Time) {
|
||||
log.Debugf("Getting %s from S3 took %s", hash[:8], time.Since(t).String())
|
||||
log.Debugf("Getting %s from %s took %s", hash[:8], s.Name(), time.Since(t).String())
|
||||
}(start)
|
||||
|
||||
buf := &aws.WriteAtBuffer{}
|
||||
_, err = s3manager.NewDownloader(s.session).Download(buf, &s3.GetObjectInput{
|
||||
ctx := context.Background()
|
||||
buf := manager.NewWriteAtBuffer([]byte{})
|
||||
_, err = manager.NewDownloader(s.client).Download(ctx, buf, &s3.GetObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(hash),
|
||||
Key: aws.String(s.shardedPath(hash)),
|
||||
})
|
||||
if err != nil {
|
||||
if aerr, ok := err.(awserr.Error); ok {
|
||||
switch aerr.Code() {
|
||||
case s3.ErrCodeNoSuchBucket:
|
||||
return nil, shared.NewBlobTrace(time.Since(start), s.Name()), errors.Err("bucket %s does not exist", s.bucket)
|
||||
case s3.ErrCodeNoSuchKey:
|
||||
return nil, shared.NewBlobTrace(time.Since(start), s.Name()), errors.Err(ErrBlobNotFound)
|
||||
}
|
||||
var noSuchBucket *types.NoSuchBucket
|
||||
var noSuchKey *types.NoSuchKey
|
||||
var notFound *types.NotFound
|
||||
switch {
|
||||
case stderrors.As(err, &noSuchBucket):
|
||||
return nil, shared.NewBlobTrace(time.Since(start), s.Name()), errors.Err("bucket %s does not exist", s.bucket)
|
||||
case stderrors.As(err, &noSuchKey), stderrors.As(err, ¬Found):
|
||||
return nil, shared.NewBlobTrace(time.Since(start), s.Name()), errors.Err(ErrBlobNotFound)
|
||||
}
|
||||
return buf.Bytes(), shared.NewBlobTrace(time.Since(start), s.Name()), err
|
||||
return nil, shared.NewBlobTrace(time.Since(start), s.Name()), errors.Err(err)
|
||||
}
|
||||
|
||||
return buf.Bytes(), shared.NewBlobTrace(time.Since(start), s.Name()), nil
|
||||
@@ -112,16 +147,18 @@ func (s *S3Store) Put(hash string, blob stream.Blob) error {
|
||||
log.Debugf("Uploading %s took %s", hash[:8], time.Since(t).String())
|
||||
}(time.Now())
|
||||
|
||||
_, err = s3manager.NewUploader(s.session).Upload(&s3manager.UploadInput{
|
||||
ctx := context.Background()
|
||||
_, err = manager.NewUploader(s.client).Upload(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(hash),
|
||||
Key: aws.String(s.shardedPath(hash)),
|
||||
Body: bytes.NewBuffer(blob),
|
||||
ACL: aws.String("public-read"),
|
||||
//StorageClass: aws.String(s3.StorageClassIntelligentTiering),
|
||||
ACL: types.ObjectCannedACLPublicRead,
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Err(err)
|
||||
}
|
||||
metrics.MtrOutBytesReflector.Add(float64(blob.Size()))
|
||||
|
||||
return err
|
||||
return nil
|
||||
}
|
||||
|
||||
// PutSD stores the sd blob on S3 or errors if S3 connection errors.
|
||||
@@ -138,33 +175,44 @@ func (s *S3Store) Delete(hash string) error {
|
||||
|
||||
log.Debugf("Deleting %s from S3", hash[:8])
|
||||
|
||||
_, err = s3.New(s.session).DeleteObject(&s3.DeleteObjectInput{
|
||||
ctx := context.Background()
|
||||
_, err = s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(hash),
|
||||
Key: aws.String(s.shardedPath(hash)),
|
||||
})
|
||||
|
||||
return err
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
func (s *S3Store) initOnce() error {
|
||||
if s.session != nil {
|
||||
if s.client != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
sess, err := session.NewSession(&aws.Config{
|
||||
Credentials: credentials.NewStaticCredentials(s.awsID, s.awsSecret, ""),
|
||||
Region: aws.String(s.region),
|
||||
Endpoint: aws.String("https://s3.wasabisys.com"),
|
||||
})
|
||||
ctx := context.Background()
|
||||
cfg, err := awsconfig.LoadDefaultConfig(ctx,
|
||||
awsconfig.WithRegion(s.region),
|
||||
awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(s.awsID, s.awsSecret, "")),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
return errors.Err(err)
|
||||
}
|
||||
|
||||
s.session = sess
|
||||
s.client = s3.NewFromConfig(cfg, func(o *s3.Options) {
|
||||
o.BaseEndpoint = aws.String(s.endpoint)
|
||||
o.UsePathStyle = true
|
||||
o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *S3Store) shardedPath(hash string) string {
|
||||
if s.prefixLength <= 0 || len(hash) < s.prefixLength {
|
||||
return hash
|
||||
}
|
||||
return path.Join(hash[:s.prefixLength], hash)
|
||||
}
|
||||
|
||||
// Shutdown shuts down the store gracefully
|
||||
func (s *S3Store) Shutdown() {
|
||||
return
|
||||
}
|
||||
+35
-3
@@ -1,6 +1,7 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
@@ -9,6 +10,7 @@ import (
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
"golang.org/x/sync/singleflight"
|
||||
)
|
||||
|
||||
@@ -22,9 +24,40 @@ func WithSingleFlight(component string, origin BlobStore) BlobStore {
|
||||
|
||||
type singleflightStore struct {
|
||||
BlobStore
|
||||
|
||||
component string
|
||||
sf *singleflight.Group
|
||||
component string
|
||||
}
|
||||
|
||||
type SingleFlightConfig struct {
|
||||
Store *viper.Viper
|
||||
Component string `mapstructure:"component"`
|
||||
}
|
||||
|
||||
func SingleFlightStoreFactory(config *viper.Viper) (BlobStore, error) {
|
||||
var cfg SingleFlightConfig
|
||||
err := config.Unmarshal(&cfg)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
cfg.Store = config.Sub("store")
|
||||
|
||||
storeType := strings.Split(cfg.Store.AllKeys()[0], ".")[0]
|
||||
storeConfig := cfg.Store.Sub(storeType)
|
||||
factory, ok := Factories[storeType]
|
||||
if !ok {
|
||||
return nil, errors.Err("unknown store type %s", storeType)
|
||||
}
|
||||
underlyingStore, err := factory(storeConfig)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
|
||||
return WithSingleFlight(cfg.Component, underlyingStore), nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
RegisterStore("singleflight", SingleFlightStoreFactory)
|
||||
}
|
||||
|
||||
func (s *singleflightStore) Name() string {
|
||||
@@ -124,5 +157,4 @@ func (s *singleflightStore) putter(hash string, blob stream.Blob) func() (interf
|
||||
// Shutdown shuts down the store gracefully
|
||||
func (s *singleflightStore) Shutdown() {
|
||||
s.BlobStore.Shutdown()
|
||||
return
|
||||
}
|
||||
@@ -1,7 +1,8 @@
|
||||
package speedwalk
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sync"
|
||||
@@ -17,7 +18,20 @@ import (
|
||||
// AllFiles recursively lists every file in every subdirectory of a given directory
|
||||
// If basename is true, return the basename of each file. Otherwise return the full path starting at startDir.
|
||||
func AllFiles(startDir string, basename bool) ([]string, error) {
|
||||
items, err := ioutil.ReadDir(startDir)
|
||||
entries, err := os.ReadDir(startDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items := make([]fs.FileInfo, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
var info fs.FileInfo
|
||||
info, err = entry.Info()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, info)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -77,7 +91,7 @@ func AllFiles(startDir string, basename bool) ([]string, error) {
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
logrus.Errorf(errors.FullTrace(err))
|
||||
logrus.Error(errors.FullTrace(err))
|
||||
}
|
||||
}(item.Name())
|
||||
}
|
||||
|
||||
+15
-1
@@ -5,6 +5,8 @@ import (
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
// BlobStore is an interface for handling blob storage.
|
||||
@@ -33,11 +35,23 @@ type Blocklister interface {
|
||||
Wants(hash string) (bool, error)
|
||||
}
|
||||
|
||||
// NeededBlobChecker can check which blobs from a known stream are not uploaded yet
|
||||
type NeededBlobChecker interface {
|
||||
MissingBlobsForKnownStream(string) ([]string, error)
|
||||
}
|
||||
|
||||
// lister is a store that can list cached blobs. This is helpful when an overlay
|
||||
// cache needs to track blob existence.
|
||||
type lister interface {
|
||||
list() ([]string, error)
|
||||
}
|
||||
|
||||
//ErrBlobNotFound is a standard error when a blob is not found in the store.
|
||||
// ErrBlobNotFound is a standard error when a blob is not found in the store.
|
||||
var ErrBlobNotFound = errors.Base("blob not found")
|
||||
var Factories = make(map[string]Factory)
|
||||
|
||||
func RegisterStore(name string, factory Factory) {
|
||||
Factories[name] = factory
|
||||
}
|
||||
|
||||
type Factory func(config *viper.Viper) (BlobStore, error)
|
||||
@@ -0,0 +1,204 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lbryio/reflector.go/internal/metrics"
|
||||
"github.com/lbryio/reflector.go/shared"
|
||||
|
||||
"github.com/lbryio/lbry.go/v2/extras/errors"
|
||||
"github.com/lbryio/lbry.go/v2/stream"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
// UpstreamStore is a store that works on top of the HTTP protocol
|
||||
type UpstreamStore struct {
|
||||
upstream string
|
||||
httpClient *http.Client
|
||||
edgeToken string
|
||||
name string
|
||||
}
|
||||
|
||||
type UpstreamParams struct {
|
||||
Name string `mapstructure:"name"`
|
||||
Upstream string `mapstructure:"upstream"`
|
||||
EdgeToken string `mapstructure:"edge_token"`
|
||||
}
|
||||
|
||||
func NewUpstreamStore(params UpstreamParams) *UpstreamStore {
|
||||
return &UpstreamStore{
|
||||
upstream: params.Upstream,
|
||||
httpClient: getClient(),
|
||||
edgeToken: params.EdgeToken,
|
||||
name: params.Name,
|
||||
}
|
||||
}
|
||||
|
||||
const nameUpstream = "upstream"
|
||||
|
||||
func UpstreamStoreFactory(config *viper.Viper) (BlobStore, error) {
|
||||
var cfg UpstreamParams
|
||||
err := config.Unmarshal(&cfg)
|
||||
if err != nil {
|
||||
return nil, errors.Err(err)
|
||||
}
|
||||
return NewUpstreamStore(cfg), nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
RegisterStore(nameUpstream, UpstreamStoreFactory)
|
||||
}
|
||||
|
||||
func (n *UpstreamStore) Name() string { return nameUpstream + "-" + n.name }
|
||||
func (n *UpstreamStore) Has(hash string) (bool, error) {
|
||||
url := n.upstream + "/blob?hash=" + hash
|
||||
|
||||
req, err := http.NewRequest("HEAD", url, nil)
|
||||
if err != nil {
|
||||
return false, errors.Err(err)
|
||||
}
|
||||
|
||||
res, err := n.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return false, errors.Err(err)
|
||||
}
|
||||
defer func() { _ = res.Body.Close() }()
|
||||
if res.StatusCode == http.StatusNotFound {
|
||||
return false, nil
|
||||
}
|
||||
if res.StatusCode == http.StatusNoContent {
|
||||
return true, nil
|
||||
}
|
||||
var body []byte
|
||||
if res.Body != nil {
|
||||
body, _ = io.ReadAll(res.Body)
|
||||
}
|
||||
return false, errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
|
||||
}
|
||||
|
||||
func (n *UpstreamStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
|
||||
start := time.Now()
|
||||
url := n.upstream + "/blob?hash=" + hash
|
||||
if n.edgeToken != "" {
|
||||
url += "&edge_token=" + n.edgeToken
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), n.Name()), errors.Err(err)
|
||||
}
|
||||
|
||||
res, err := n.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), n.Name()), errors.Err(err)
|
||||
}
|
||||
defer func(Body io.ReadCloser) {
|
||||
err := Body.Close()
|
||||
if err != nil {
|
||||
log.Errorf("Error closing response body in HTTP-GET: %s", err.Error())
|
||||
}
|
||||
}(res.Body)
|
||||
|
||||
viaHeader := res.Header.Get("Via")
|
||||
var trace shared.BlobTrace
|
||||
if viaHeader != "" {
|
||||
parsedTrace, err := shared.Deserialize(viaHeader)
|
||||
if err != nil {
|
||||
return nil, shared.NewBlobTrace(time.Since(start), n.Name()), err
|
||||
}
|
||||
trace = *parsedTrace
|
||||
} else {
|
||||
trace = shared.NewBlobTrace(0, n.Name())
|
||||
}
|
||||
|
||||
switch res.StatusCode {
|
||||
case http.StatusNotFound:
|
||||
return nil, trace.Stack(time.Since(start), n.Name()), ErrBlobNotFound
|
||||
|
||||
case http.StatusOK:
|
||||
buffer := getBuffer()
|
||||
defer putBuffer(buffer)
|
||||
|
||||
if _, err := io.Copy(buffer, res.Body); err != nil {
|
||||
return nil, trace.Stack(time.Since(start), n.Name()), errors.Err(err)
|
||||
}
|
||||
|
||||
blob := make([]byte, buffer.Len())
|
||||
copy(blob, buffer.Bytes())
|
||||
|
||||
metrics.MtrInBytesUpstream.Add(float64(len(blob)))
|
||||
return blob, trace.Stack(time.Since(start), n.Name()), nil
|
||||
|
||||
default:
|
||||
body, _ := io.ReadAll(res.Body)
|
||||
log.Warnf("Got status code %d (%s)", res.StatusCode, string(body))
|
||||
return nil, trace.Stack(time.Since(start), n.Name()),
|
||||
errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
|
||||
}
|
||||
}
|
||||
|
||||
func (n *UpstreamStore) Put(string, stream.Blob) error {
|
||||
return shared.ErrNotImplemented
|
||||
}
|
||||
func (n *UpstreamStore) PutSD(string, stream.Blob) error {
|
||||
return shared.ErrNotImplemented
|
||||
}
|
||||
func (n *UpstreamStore) Delete(string) error {
|
||||
return shared.ErrNotImplemented
|
||||
}
|
||||
func (n *UpstreamStore) Shutdown() {}
|
||||
|
||||
// buffer pool to reduce GC
|
||||
// https://www.captaincodeman.com/2017/06/02/golang-buffer-pool-gotcha
|
||||
var buffers = sync.Pool{
|
||||
// New is called when a new instance is needed
|
||||
New: func() interface{} {
|
||||
buf := make([]byte, 0, stream.MaxBlobSize)
|
||||
return bytes.NewBuffer(buf)
|
||||
},
|
||||
}
|
||||
|
||||
// getBuffer fetches a buffer from the pool
|
||||
func getBuffer() *bytes.Buffer {
|
||||
return buffers.Get().(*bytes.Buffer)
|
||||
}
|
||||
|
||||
// putBuffer returns a buffer to the pool
|
||||
func putBuffer(buf *bytes.Buffer) {
|
||||
buf.Reset()
|
||||
buffers.Put(buf)
|
||||
}
|
||||
|
||||
func dialContext(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
dialer := &net.Dialer{
|
||||
Timeout: 30 * time.Second,
|
||||
KeepAlive: 30 * time.Second,
|
||||
}
|
||||
return dialer.DialContext(ctx, network, address)
|
||||
}
|
||||
|
||||
// getClient gets an http client that's customized to be more performant when dealing with blobs of 2MB in size (most of our blobs)
|
||||
func getClient() *http.Client {
|
||||
// Customize the Transport to have larger connection pool
|
||||
defaultTransport := &http.Transport{
|
||||
DialContext: dialContext,
|
||||
ForceAttemptHTTP2: true,
|
||||
MaxIdleConns: 100,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
TLSHandshakeTimeout: 10 * time.Second,
|
||||
ExpectContinueTimeout: 1 * time.Second,
|
||||
DisableCompression: true,
|
||||
MaxIdleConnsPerHost: 100,
|
||||
ReadBufferSize: stream.MaxBlobSize + 1024*10, //add an extra few KBs to make sure it fits the extra information
|
||||
}
|
||||
|
||||
return &http.Client{Transport: defaultTransport}
|
||||
}
|
||||
+4
-4
@@ -69,22 +69,22 @@ func (n *Node) Resolve(url string) (*types.Output, error) {
|
||||
|
||||
type GetClaimsInTxResp struct {
|
||||
Jsonrpc string `json:"jsonrpc"`
|
||||
ID int `json:"id"`
|
||||
Result []struct {
|
||||
Name string `json:"name"`
|
||||
ClaimID string `json:"claim_id"`
|
||||
Txid string `json:"txid"`
|
||||
Value string `json:"value"`
|
||||
Address string `json:"address"`
|
||||
Supports []interface{} `json:"supports"` // TODO: finish me
|
||||
Nout int `json:"nout"`
|
||||
Amount int `json:"amount"`
|
||||
Depth int `json:"depth"`
|
||||
Height int `json:"height"`
|
||||
Value string `json:"value"`
|
||||
ClaimSequence int `json:"claim_sequence"`
|
||||
Address string `json:"address"`
|
||||
Supports []interface{} `json:"supports"` // TODO: finish me
|
||||
EffectiveAmount int `json:"effective_amount"`
|
||||
ValidAtHeight int `json:"valid_at_height"`
|
||||
} `json:"result"`
|
||||
ID int `json:"id"`
|
||||
}
|
||||
|
||||
func (n *Node) GetClaimsInTx(txid string) (*GetClaimsInTxResp, error) {
|
||||
|
||||
+13
-16
@@ -30,22 +30,19 @@ var (
|
||||
)
|
||||
|
||||
type response struct {
|
||||
data []byte
|
||||
err error
|
||||
data []byte
|
||||
}
|
||||
|
||||
type Node struct {
|
||||
transport *TCPTransport
|
||||
nextId atomic.Uint32
|
||||
grp *stop.Group
|
||||
|
||||
handlersMu *sync.RWMutex
|
||||
handlers map[uint32]chan response
|
||||
|
||||
transport *TCPTransport
|
||||
grp *stop.Group
|
||||
handlersMu *sync.RWMutex
|
||||
handlers map[uint32]chan response
|
||||
pushHandlersMu *sync.RWMutex
|
||||
pushHandlers map[string][]chan response
|
||||
|
||||
timeout time.Duration
|
||||
nextId atomic.Uint32
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
// NewNode creates a new node.
|
||||
@@ -155,23 +152,23 @@ func (n *Node) listen() {
|
||||
return
|
||||
case bytes := <-n.transport.Responses():
|
||||
msg := &struct {
|
||||
Id uint32 `json:"id"`
|
||||
Method string `json:"method"`
|
||||
Error struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Code int `json:"code"`
|
||||
} `json:"error"`
|
||||
Id uint32 `json:"id"`
|
||||
}{}
|
||||
msg2 := &struct {
|
||||
Id uint32 `json:"id"`
|
||||
Method string `json:"method"`
|
||||
Error struct {
|
||||
Code int `json:"code"`
|
||||
Message struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Code int `json:"code"`
|
||||
} `json:"message"`
|
||||
Code int `json:"code"`
|
||||
} `json:"error"`
|
||||
Id uint32 `json:"id"`
|
||||
}{}
|
||||
r := response{}
|
||||
|
||||
@@ -231,9 +228,9 @@ func (n *Node) listen() {
|
||||
// request makes a request to the server and unmarshals the response into v.
|
||||
func (n *Node) request(method string, params []string, v interface{}) error {
|
||||
msg := struct {
|
||||
Id uint32 `json:"id"`
|
||||
Method string `json:"method"`
|
||||
Params []string `json:"params"`
|
||||
Id uint32 `json:"id"`
|
||||
}{
|
||||
Id: n.nextId.Load(),
|
||||
Method: method,
|
||||
|
||||
+1
-1
@@ -124,7 +124,7 @@ func (t *TCPTransport) test() error {
|
||||
return errors.Err(err)
|
||||
}
|
||||
if response.Error.Message != "" {
|
||||
return fmt.Errorf(response.Error.Message)
|
||||
return fmt.Errorf("%s", response.Error.Message)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user