1 Commits
Author SHA1 Message Date
Victor Shyba 71fb06fbb8 sendfile proof of concept 2021-08-02 03:23:54 -03:00
88 changed files with 2558 additions and 3536 deletions

No files matched your search

-37
View File
@@ -1,37 +0,0 @@
name: Go
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v4
with:
go-version: 1.25.x
- name: Build linux
run: make linux
- name: Build macos
run: make macos
- name: Test
run: make test
- name: Lint
run: make lint
- name: retrieve all tags
run: git fetch --prune --unshallow --tags
- name: Print changes since last version
run: git log $(git describe --tags --abbrev=0)..HEAD --no-merges --oneline
-62
View File
@@ -1,62 +0,0 @@
name: release
on:
push:
tags:
- "*.*.*"
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v4
with:
go-version: 1.25.x
- name: Build linux
run: make linux
- name: Build macos
run: make macos
- name: Test
run: make test
- name: Lint
run: make lint
- name: Zip macos
run: zip -r reflector_darwin_amd64.zip ./dist/darwin_amd64
- name: Zip linux
run: zip -r reflector_linux_amd64.zip ./dist/linux_amd64
- name: retrieve all tags
run: git fetch --prune --unshallow --tags
- name: Generate Changelog
run: git log $(git describe --tags --abbrev=0 @^)..@ --no-merges --oneline > ${{ github.workspace }}-CHANGELOG.txt
- name: upload to github releases
uses: softprops/action-gh-release@v1
with:
files: |
./reflector_linux_amd64.zip
./reflector_darwin_amd64.zip
body_path: ${{ github.workspace }}-CHANGELOG.txt
# - name: Login to DockerHub
# uses: docker/login-action@v2
# with:
# username: ${{ secrets.DOCKERHUB_USERNAME }}
# password: ${{ secrets.DOCKERHUB_TOKEN }}
# - name: Generate docker image
# run: make image
# - name: Docker push
# run: make publish_image
-9
View File
@@ -1,12 +1,3 @@
/vendor
/config.json*
/dist
/bin
blobcache.yaml
reflector.yaml
upload.yaml
remotereflector.yaml
-57
View File
@@ -1,57 +0,0 @@
run:
timeout: 5m
tests: false
modules-download-mode: readonly
linters-settings:
gci:
# Checks that no inline Comments are present.
# Default: false
no-inline-comments: false
# Checks that no prefix Comments(comment lines above an import) are present.
# Default: false
no-prefix-comments: false
# Skip generated files
skip-generated: true
# Use custom order
custom-order: true
sections:
- standard
- prefix(github.com/lbryio/reflector.go)
- prefix(github.com/OdyseeTeam,github.com/lbryio)
- default
gofmt:
simplify: true
govet:
enable-all: true
misspell:
locale: US
revive:
rules:
- name: exported
disabled: true
staticcheck:
checks: ["all"]
linters:
enable:
- errcheck
- gosimple
- govet
- ineffassign
- gci
- gofmt
- goimports
- misspell
- revive
- unconvert
- whitespace
- typecheck
disable:
- unused
- unparam
- staticcheck
+83
View File
@@ -0,0 +1,83 @@
os: linux
dist: trusty
language: go
env:
- GO111MODULE=on
go:
- 1.15.x
cache:
directories:
- $HOME/.cache/go-build
- $HOME/gopath/pkg/mod
notifications:
email: false
# Skip the install step. Don't `go get` dependencies. Only build with the code in vendor/
install: true
# Anything in before_script that returns a nonzero exit code will
# flunk the build and immediately stop. It's sorta like having
# set -e enabled in bash.
before_script:
# All the .go files, excluding vendor/ and model (auto generated)
- GO_FILES=$(find . -iname '*.go' ! -iname '*_test.go' -type f | grep -v /vendor/ ) #i wish we were this crazy :p
- go get golang.org/x/tools/cmd/goimports # Used in build script for generated files
# - go get github.com/golang/lint/golint # Linter
# - go get honnef.co/go/tools/cmd/megacheck # Badass static analyzer/linter
- go get github.com/jgautheron/gocyclo # Check against high complexity
- go get github.com/mdempsky/unconvert # Identifies unnecessary type conversions
- go get github.com/kisielk/errcheck # Checks for unhandled errors
- go get github.com/opennota/check/cmd/varcheck # Checks for unused vars
- go get github.com/opennota/check/cmd/structcheck # Checks for unused fields in structs
# script always run to completion (set +e). All of these code checks are must haves
# in a modern Go project.
script:
# Fail if a .go file hasn't been formatted with gofmt
- test -z $(gofmt -s -l $GO_FILES)
# Run unit tests
- make test
# Checks for unused vars and fields on structs
- varcheck ./...
- structcheck ./...
# go vet is the official Go static analyzer
- go vet ./...
# forbid code with huge functions
#- gocyclo -ignore "_test.go" -avg -over 19 $GO_FILES
# checks for unhandled errors
# - errcheck ./... # COMMENTED OUT UNTIL https://github.com/kisielk/errcheck/issues/155 is fixed
# "go vet on steroids" + linter - ignore autogen code
# - megacheck -simple.exit-non-zero=true ./... # DISABLED UNTIL https://github.com/dominikh/go-tools/issues/328 is fixed
# check for unnecessary conversions - ignore autogen code
# - unconvert ./... # SEEMS TO BE BROKEN WITH GO MODULES
# one last linter - ignore autogen code
#- golint -set_exit_status $(go list ./... | grep -v /vendor/ )
# Finally, build the binary
- make
deploy:
- provider: s3
local_dir: ./bin
skip_cleanup: true
on:
repo: lbryio/reflector.go
access_key_id:
secure: "ABmoSTxTee1GubJmmi+MlyBcnqRT0ywEOtTL4hrH7T+Vj5UPcNBhjqGRiMXzO3MdrVWom19OnqMvsvAdehrB4uQJVlX/+zB26gTpEiiU5dWJgerNAYmfxDGNz/p9pyYWLvVY/cymSlJ4HkggJVg+dMTBTOP+gTNgQUhHL7pAd/Z5HMPmPW/rlNlQgDdczoY44OdiniuHny4Lr7mo8dcT4dcyJysPpHLCPGWp3kXcffIEbgO02aQZ14BbX9oU9xm1BacYnr4t8gRCU+uK0mt7TREHUzZ8bQBRXUymmpeeEQshXdifdwtk6sdRGIZiTi0NmQ7kYter6oh6623TH6rpNy/lkxKi823FbSPC1XZGH7CwcbnSIEwY++7e9PT0EA14VuDUF7+iXC/gpRkVQwUoMcoC9rHBjkXW+SiwHWoUeDKXrfVO9QGEJTi5bDRNwklwu2zkYWeRIPzfDaVayUBYwjunjT1eST997ygmJwJTJ6DeOWLx+WFlmwBJfQpBwbrlMrceiNjreMwGB6ffXEBzuM6S5inXGIChllGDpNyJeFVcH9zPpU0JsIpWoRjwLNFOiEhFsnzK5rvA2SKmlOOnSlgOCBQnFi2SmLDM790KoY8qjan2s984chQz2qsKnhYuVpJhG6LKgrI1QwE0KHr80spVFfewq11AU4Q+lfA9aAQ="
secret_access_key:
secure: "AxYRTy/GnjeTJKQdeJ/AEeAd+yXs783bFDKdyKNswtsHlU8sWPQgNcvTLpVqnSQMpiwkGDGi/70rvR5C+AT3SIWNw13RYrgBRpduQU0J+B2JS+3dN2DIePu25uvs++Wo22OfS8I+UjZ1mWY1SSHI2spPXvDCq5tb+Ih8nlYflEyAtxU9Oq2R3Kus2tkIlRnL25sP/2fY7RvuJFYIV63z8ZIJRzB5WzOeERqnXq2zfwos+hycAqyo/VevJnWAYTEDsvBuSODOpZF+QfKtIQ2rYSoqy8Lq1M6UOZimnC3Ulea4euBVf2ssBCnI7csGNG5UzkTiwrPDi2xIP8nM01rHW1yHJ7tQsJaghnUsfw2t6ui4ZofvbbOFTN/YCloHITifEi8Tc1/17isi3y+kX5yQ/Nk5UNry0Wbt91CP+nkL/ZmA5grkBXDL7VJMmB60TnO3ap24CtwBQartN3LoWs7h+4ov+LqbCt6IqpJVWQWlwJeb2MFPFByALtBpsqAyL1SxXlGNpPa94CuXxfQ6Bv436PtefA5FlTzR8uMmqsjWciv06bVnSvVlFEVovN2Fkplrjt7AASJ/8KJs4THDg4k61nfd8roAHx6ewQzl4wCWKCikQ0MuFd2mVHwdrbnCH1mIHuPRyvWMMIAK0ooc1/rmKiJlpgumjxoFYNE10MXtt+I="
bucket: "lbry-reflector-binary"
# - provider: releases
# file: ./bin/prism-bin
# skip_cleanup: true
# on:
# repo: lbryio/reflector.go
# tags: true
# api_key:
# secure: epAlhp3SUr8hhISarJ22n6tRw2TEa4s4oNFIvJUb5HGECVp1SYN7ao0ln5NoNLmfJS60pi911i/kMhhi21/uhZ0kCYlEhhIE2pc1zsiAxK9L9ENCssJ205HfVbe5grhwskLGzgjhU9OznO8WtmyOPWXr0it8M8RCTjx6rEC0A33Id3WMYyhP938Sj9CxEYeH4KS8wFvBXkgBVtrgaYwRTCIROFddHFXOb9jyNhqQ1RbfKtllsVtQhVk5WMlomheBNSS4vr6WMS4X4+2okFqnLtiSn1wrn5I/94UQbnrI1juVnQj0K+j32EyQbAOt4T2cLW3GtG0jhaYKyNMT9ycDCdVACPSDELlHWjeyoes9bnhUFftm6kDbQxwA1UsTF1yG8tMKXxBSmYyoT7qDloi6pBifZMrFXL61uTs6yhVB9LS/2oqg4sc0Ne87bRcn4OxsBeVCe3kbBHDTR/NTyF2gNPtRvgMAWULxTVcUm9VYdO0IWvAig5g4Row0DnFzEquD6CzezbRWD9WyZyV/AFyYHeeQ2PO7jTw0/3M7aDX33Fuhh34lehzmrC03cfgD/wZW+spxozIcQCYdiJqVw+u+/NvbNr0kkFzE9zW26JEmUFTyDvKxvnza1Kwtww3EgH6zaOL8r4yVbb54rePRvLw7pl93zlfJnEB2MCPqJOY5ZpU=
+1 -1
View File
@@ -3,7 +3,7 @@ EXPOSE 8080
RUN mkdir /app
WORKDIR /app
COPY dist/linux_amd64/prism-bin ./prism
COPY bin/prism-bin ./prism
RUN chmod +x prism
ENTRYPOINT [ "/app/prism" ]
+18 -30
View File
@@ -1,37 +1,25 @@
version := $(shell git describe --dirty --always --long --abbrev=7)
commit := $(shell git rev-parse --short HEAD)
commit_long := $(shell git rev-parse HEAD)
branch := $(shell git rev-parse --abbrev-ref HEAD)
curTime := $(shell date +%s)
BINARY=prism-bin
IMPORT_PATH = github.com/lbryio/reflector.go
LDFLAGS="-X ${IMPORT_PATH}/meta.version=$(version) -X ${IMPORT_PATH}/meta.commit=$(commit) -X ${IMPORT_PATH}/meta.commitLong=$(commit_long) -X ${IMPORT_PATH}/meta.branch=$(branch) -X '${IMPORT_PATH}/meta.Time=$(curTime)'"
DIR = $(shell cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)
BIN_DIR = $(DIR)/dist
BIN_DIR = ${DIR}/bin
IMPORT_PATH = github.com/lbryio/reflector.go
.DEFAULT_GOAL := linux
VERSION = $(shell git --git-dir=${DIR}/.git describe --dirty --always --long --abbrev=7)
LDFLAGS = -ldflags "-X ${IMPORT_PATH}/meta.Version=${VERSION} -X ${IMPORT_PATH}/meta.Time=$(shell date +%s)"
.PHONY: build clean test lint
.DEFAULT_GOAL: build
build:
mkdir -p ${BIN_DIR} && CGO_ENABLED=0 go build ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/${BINARY} main.go
clean:
if [ -f ${BIN_DIR}/${BINARY} ]; then rm ${BIN_DIR}/${BINARY}; fi
.PHONY: test
test:
go test -cover -v ./...
go test ./... -v -cover
.PHONY: lint
lint:
./scripts/lint.sh
.PHONY: imports
imports:
gci write --skip-generated --custom-order -s standard -s "prefix(github.com/lbryio/reflector.go)" -s "prefix(github.com/OdyseeTeam,github.com/lbryio)" -s default .
.PHONY: linux
linux:
GOARCH=amd64 CGO_ENABLED=0 GOOS=linux go build -ldflags ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/linux_amd64/${BINARY}
.PHONY: macos
macos:
GOARCH=amd64 GOOS=darwin go build -ldflags ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/darwin_amd64/${BINARY}
.PHONY: image
image:
docker buildx build -t lbry/reflector:$(version) -t lbry/reflector:latest --platform linux/amd64 .
go get github.com/alecthomas/gometalinter && gometalinter --install && gometalinter ./...
+7 -6
View File
@@ -1,7 +1,7 @@
package cluster
import (
"io"
"io/ioutil"
baselog "log"
"sort"
"time"
@@ -23,12 +23,13 @@ const (
type Cluster struct {
OnMembershipChange func(n, total int)
s *serf.Serf
eventCh chan serf.Event
stop *stop.Group
name string
seedAddr string
port int
seedAddr string
s *serf.Serf
eventCh chan serf.Event
stop *stop.Group
}
// New returns a new Cluster instance that is not connected.
@@ -51,7 +52,7 @@ func (c *Cluster) Connect() error {
conf.MemberlistConfig.AdvertisePort = c.port
conf.NodeName = c.name
nullLogger := baselog.New(io.Discard, "", 0)
nullLogger := baselog.New(ioutil.Discard, "", 0)
conf.Logger = nullLogger
c.eventCh = make(chan serf.Event)
-55
View File
@@ -1,55 +0,0 @@
package cmd
import (
"os"
"os/signal"
"strconv"
"syscall"
"github.com/lbryio/reflector.go/config"
"github.com/lbryio/reflector.go/internal/metrics"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
func init() {
var cmd = &cobra.Command{
Use: "blobcache",
Short: "Run blobcache server",
Run: blobcacheCmd,
}
cmd.Flags().IntVar(&metricsPort, "metrics-port", 2112, "The port reflector will use for prometheus metrics")
cmd.Flags().BoolVar(&disableBlocklist, "disable-blocklist", false, "Disable blocklist watching/updating")
rootCmd.AddCommand(cmd)
}
func blobcacheCmd(cmd *cobra.Command, args []string) {
store, err := config.LoadStores(conf, "blobcache")
if err != nil {
log.Fatal(err)
}
defer store.Shutdown()
servers, err := config.LoadServers(store, conf, "blobcache")
if err != nil {
log.Fatal(err)
}
for _, s := range servers {
err = s.Start()
if err != nil {
log.Fatal(err)
}
defer s.Shutdown()
}
metricsServer := metrics.NewServer(":"+strconv.Itoa(metricsPort), "/metrics")
metricsServer.Start()
defer metricsServer.Shutdown()
interruptChan := make(chan os.Signal, 1)
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
<-interruptChan
}
+2 -2
View File
@@ -6,10 +6,10 @@ import (
"strconv"
"syscall"
"github.com/lbryio/reflector.go/cluster"
"github.com/lbryio/lbry.go/v2/extras/crypto"
"github.com/lbryio/reflector.go/cluster"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
+6 -13
View File
@@ -5,6 +5,7 @@ import (
"os"
"time"
"github.com/lbryio/reflector.go/server/peer"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/stream"
@@ -27,19 +28,11 @@ func getStreamCmd(cmd *cobra.Command, args []string) {
addr := args[0]
sdHash := args[1]
s := store.NewCachingStore(store.CachingParams{
Name: "getstream",
Cache: store.NewPeerStore(store.PeerParams{
Name: "getstream",
Address: addr,
Timeout: 30 * time.Second,
}),
Origin: store.NewDiskStore(store.DiskParams{
Name: "getstream",
MountPoint: "/tmp/lbry_downloaded_blobs",
ShardingSize: 2,
}),
})
s := store.NewCachingStore(
"getstream",
peer.NewStore(peer.StoreOpts{Address: addr}),
store.NewDiskStore("/tmp/lbry_downloaded_blobs", 2),
)
wd, err := os.Getwd()
if err != nil {
+2 -1
View File
@@ -3,6 +3,7 @@ package cmd
import (
"crypto/sha512"
"encoding/hex"
"io/ioutil"
"os"
"path"
"runtime"
@@ -71,7 +72,7 @@ func consume(worker int, tasks <-chan string, done chan<- bool, totalTasks int,
log.Infof("[T%d] %d/%d blobs processed so far. ETA: %s", worker, processedSoFar, totalTasks, remainingTime.String())
}
blobPath := path.Join(diskStorePath, b[:2], b)
blob, err := os.ReadFile(blobPath)
blob, err := ioutil.ReadFile(blobPath)
if err != nil {
if os.IsNotExist(err) {
continue
+55
View File
@@ -0,0 +1,55 @@
package cmd
import (
"os"
"os/signal"
"strconv"
"syscall"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/server/peer"
"github.com/lbryio/reflector.go/store"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
var peerNoDB bool
func init() {
var cmd = &cobra.Command{
Use: "peer",
Short: "Run peer server",
Run: peerCmd,
}
cmd.Flags().BoolVar(&peerNoDB, "nodb", false, "Don't connect to a db and don't use a db-backed blob store")
rootCmd.AddCommand(cmd)
}
func peerCmd(cmd *cobra.Command, args []string) {
var err error
s3 := store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
peerServer := peer.NewServer(s3)
if !peerNoDB {
db := &db.SQL{
LogQueries: log.GetLevel() == log.DebugLevel,
}
err = db.Connect(globalConfig.DBConn)
checkErr(err)
combo := store.NewDBBackedStore(s3, db, false)
peerServer = peer.NewServer(combo)
}
err = peerServer.Start(":" + strconv.Itoa(peer.DefaultPort))
if err != nil {
log.Fatal(err)
}
interruptChan := make(chan os.Signal, 1)
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
<-interruptChan
peerServer.Shutdown()
}
+3 -3
View File
@@ -32,9 +32,9 @@ func populateDbCmd(cmd *cobra.Command, args []string) {
log.Fatal("store-path must be defined")
}
localDb := &db.SQL{
SoftDelete: true,
TrackingLevel: db.TrackAccessBlobs,
LogQueries: log.GetLevel() == log.DebugLevel,
SoftDelete: true,
TrackAccess: db.TrackAccessBlobs,
LogQueries: log.GetLevel() == log.DebugLevel,
}
err := localDb.Connect("reflector:reflector@tcp(localhost:3306)/reflector")
if err != nil {
+324 -21
View File
@@ -4,25 +4,66 @@ import (
"os"
"os/signal"
"strconv"
"strings"
"syscall"
"time"
"github.com/lbryio/reflector.go/config"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/meta"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/server/http"
http32 "github.com/lbryio/reflector.go/server/http3"
"github.com/lbryio/reflector.go/server/peer"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/c2h5oh/datasize"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
var (
//port configuration
receiverPort int
metricsPort int
tcpPeerPort int
http3PeerPort int
httpPeerPort int
receiverPort int
metricsPort int
//flags configuration
disableUploads bool
disableBlocklist bool
useDB bool
//upstream configuration
upstreamReflector string
upstreamProtocol string
//downstream configuration
requestQueueSize int
//upstream edge configuration (to "cold" storage)
originEndpoint string
originEndpointFallback string
//cache configuration
diskCache string
secondaryDiskCache string
memCache int
)
var cacheManagers = []string{"localdb", "lfu", "arc", "lru", "simple"}
var cacheMangerToGcache = map[string]store.EvictionStrategy{
"lfu": store.LFU,
"arc": store.ARC,
"lru": store.LRU,
"simple": store.SIMPLE,
}
func init() {
var cmd = &cobra.Command{
@@ -31,46 +72,308 @@ func init() {
Run: reflectorCmd,
}
cmd.Flags().IntVar(&metricsPort, "metrics-port", 2112, "The port reflector will use for prometheus metrics")
cmd.Flags().IntVar(&tcpPeerPort, "tcp-peer-port", 5567, "The port reflector will distribute content from for the TCP (LBRY) protocol")
cmd.Flags().IntVar(&http3PeerPort, "http3-peer-port", 5568, "The port reflector will distribute content from over HTTP3 protocol")
cmd.Flags().IntVar(&httpPeerPort, "http-peer-port", 5569, "The port reflector will distribute content from over HTTP protocol")
cmd.Flags().IntVar(&receiverPort, "receiver-port", 5566, "The port reflector will receive content from")
cmd.Flags().IntVar(&metricsPort, "metrics-port", 2112, "The port reflector will use for prometheus metrics")
cmd.Flags().BoolVar(&disableUploads, "disable-uploads", false, "Disable uploads to this reflector server")
cmd.Flags().BoolVar(&disableBlocklist, "disable-blocklist", false, "Disable blocklist watching/updating")
cmd.Flags().BoolVar(&useDB, "use-db", true, "Whether to connect to the reflector db or not")
cmd.Flags().StringVar(&upstreamReflector, "upstream-reflector", "", "host:port of a reflector server where blobs are fetched from")
cmd.Flags().StringVar(&upstreamProtocol, "upstream-protocol", "http", "protocol used to fetch blobs from another upstream reflector server (tcp/http3/http)")
cmd.Flags().IntVar(&requestQueueSize, "request-queue-size", 200, "How many concurrent requests from downstream should be handled at once (the rest will wait)")
cmd.Flags().StringVar(&originEndpoint, "origin-endpoint", "", "HTTP edge endpoint for standard HTTP retrieval")
cmd.Flags().StringVar(&originEndpointFallback, "origin-endpoint-fallback", "", "HTTP edge endpoint for standard HTTP retrieval if first origin fails")
cmd.Flags().StringVar(&diskCache, "disk-cache", "100GB:/tmp/downloaded_blobs:localdb", "Where to cache blobs on the file system. format is 'sizeGB:CACHE_PATH:cachemanager' (cachemanagers: localdb/lfu/arc/lru)")
cmd.Flags().StringVar(&secondaryDiskCache, "optional-disk-cache", "", "Optional secondary file system cache for blobs. format is 'sizeGB:CACHE_PATH:cachemanager' (cachemanagers: localdb/lfu/arc/lru) (this would get hit before the one specified in disk-cache)")
cmd.Flags().IntVar(&memCache, "mem-cache", 0, "enable in-memory cache with a max size of this many blobs")
rootCmd.AddCommand(cmd)
}
func reflectorCmd(cmd *cobra.Command, args []string) {
store, err := config.LoadStores(conf, "reflector")
if err != nil {
log.Fatal(err)
}
defer store.Shutdown()
log.Printf("reflector %s", meta.VersionString())
servers, err := config.LoadServers(store, conf, "reflector")
if err != nil {
log.Fatal(err)
}
for _, s := range servers {
err = s.Start()
// the blocklist logic requires the db backed store to be the outer-most store
underlyingStore := initStores()
underlyingStoreWithCaches, cleanerStopper := initCaches(underlyingStore)
if !disableUploads {
reflectorServer := reflector.NewServer(underlyingStore, underlyingStoreWithCaches)
reflectorServer.Timeout = 3 * time.Minute
reflectorServer.EnableBlocklist = !disableBlocklist
err := reflectorServer.Start(":" + strconv.Itoa(receiverPort))
if err != nil {
log.Fatal(err)
}
defer s.Shutdown()
defer reflectorServer.Shutdown()
}
reflectorServer := reflector.NewIngestionServer(store)
reflectorServer.Timeout = 3 * time.Minute
reflectorServer.EnableBlocklist = !disableBlocklist
err = reflectorServer.Start(":" + strconv.Itoa(receiverPort))
peerServer := peer.NewServer(underlyingStoreWithCaches)
err := peerServer.Start(":" + strconv.Itoa(tcpPeerPort))
if err != nil {
log.Fatal(err)
}
defer reflectorServer.Shutdown()
defer peerServer.Shutdown()
http3PeerServer := http32.NewServer(underlyingStoreWithCaches, requestQueueSize)
err = http3PeerServer.Start(":" + strconv.Itoa(http3PeerPort))
if err != nil {
log.Fatal(err)
}
defer http3PeerServer.Shutdown()
httpServer := http.NewServer(underlyingStoreWithCaches, requestQueueSize)
err = httpServer.Start(":" + strconv.Itoa(httpPeerPort))
if err != nil {
log.Fatal(err)
}
defer httpServer.Shutdown()
metricsServer := metrics.NewServer(":"+strconv.Itoa(metricsPort), "/metrics")
metricsServer.Start()
defer metricsServer.Shutdown()
defer underlyingStoreWithCaches.Shutdown()
defer underlyingStore.Shutdown() //do we actually need this? Oo
interruptChan := make(chan os.Signal, 1)
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
<-interruptChan
// deferred shutdowns happen now
cleanerStopper.StopAndWait()
}
func initUpstreamStore() store.BlobStore {
var s store.BlobStore
if upstreamReflector == "" {
return nil
}
switch upstreamProtocol {
case "tcp":
s = peer.NewStore(peer.StoreOpts{
Address: upstreamReflector,
Timeout: 30 * time.Second,
})
case "http3":
s = http32.NewStore(http32.StoreOpts{
Address: upstreamReflector,
Timeout: 30 * time.Second,
})
case "http":
s = store.NewHttpStore(upstreamReflector)
default:
log.Fatalf("protocol is not recognized: %s", upstreamProtocol)
}
return s
}
func initEdgeStore() store.BlobStore {
var s3Store *store.S3Store
var s store.BlobStore
if conf != "none" {
s3Store = store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
}
if originEndpointFallback != "" && originEndpoint != "" {
ittt := store.NewITTTStore(store.NewCloudFrontROStore(originEndpoint), store.NewCloudFrontROStore(originEndpointFallback))
if s3Store != nil {
s = store.NewCloudFrontRWStore(ittt, s3Store)
} else {
s = ittt
}
} else if s3Store != nil {
s = s3Store
} else {
log.Fatalf("this configuration does not include a valid upstream source")
}
return s
}
func initDBStore(s store.BlobStore) store.BlobStore {
if useDB {
dbInst := &db.SQL{
TrackAccess: db.TrackAccessStreams,
LogQueries: log.GetLevel() == log.DebugLevel,
}
err := dbInst.Connect(globalConfig.DBConn)
if err != nil {
log.Fatal(err)
}
s = store.NewDBBackedStore(s, dbInst, false)
}
return s
}
func initStores() store.BlobStore {
s := initUpstreamStore()
if s == nil {
s = initEdgeStore()
}
s = initDBStore(s)
return s
}
// initCaches returns a store wrapped with caches and a stop group to execute a clean shutdown
func initCaches(s store.BlobStore) (store.BlobStore, *stop.Group) {
stopper := stop.New()
diskStore := initDiskStore(s, diskCache, stopper)
finalStore := initDiskStore(diskStore, secondaryDiskCache, stopper)
stop.New()
if memCache > 0 {
finalStore = store.NewCachingStore(
"reflector",
finalStore,
store.NewGcacheStore("mem", store.NewMemStore(), memCache, store.LRU),
)
}
return finalStore, stopper
}
func initDiskStore(upstreamStore store.BlobStore, diskParams string, stopper *stop.Group) store.BlobStore {
diskCacheMaxSize, diskCachePath, cacheManager := diskCacheParams(diskParams)
//we are tracking blobs in memory with a 1 byte long boolean, which means that for each 2MB (a blob) we need 1Byte
// so if the underlying cache holds 10MB, 10MB/2MB=5Bytes which is also the exact count of objects to restore on startup
realCacheSize := float64(diskCacheMaxSize) / float64(stream.MaxBlobSize)
if diskCacheMaxSize == 0 {
return upstreamStore
}
err := os.MkdirAll(diskCachePath, os.ModePerm)
if err != nil {
log.Fatal(err)
}
diskStore := store.NewDiskStore(diskCachePath, 2)
var unwrappedStore store.BlobStore
cleanerStopper := stop.New(stopper)
if cacheManager == "localdb" {
localDb := &db.SQL{
SoftDelete: true,
TrackAccess: db.TrackAccessBlobs,
LogQueries: log.GetLevel() == log.DebugLevel,
}
err = localDb.Connect("reflector:reflector@tcp(localhost:3306)/reflector")
if err != nil {
log.Fatal(err)
}
unwrappedStore = store.NewDBBackedStore(diskStore, localDb, true)
go cleanOldestBlobs(int(realCacheSize), localDb, unwrappedStore, cleanerStopper)
} else {
unwrappedStore = store.NewGcacheStore("nvme", store.NewDiskStore(diskCachePath, 2), int(realCacheSize), cacheMangerToGcache[cacheManager])
}
wrapped := store.NewCachingStore(
"reflector",
upstreamStore,
unwrappedStore,
)
return wrapped
}
func diskCacheParams(diskParams string) (int, string, string) {
if diskParams == "" {
return 0, "", ""
}
parts := strings.Split(diskParams, ":")
if len(parts) != 3 {
log.Fatalf("%s does is formatted incorrectly. Expected format: 'sizeGB:CACHE_PATH:cachemanager' for example: '100GB:/tmp/downloaded_blobs:localdb'", diskParams)
}
diskCacheSize := parts[0]
path := parts[1]
cacheManager := parts[2]
if len(path) == 0 || path[0] != '/' {
log.Fatalf("disk cache paths must start with '/'")
}
if !util.InSlice(cacheManager, cacheManagers) {
log.Fatalf("specified cache manager '%s' is not supported. Use one of the following: %v", cacheManager, cacheManagers)
}
var maxSize datasize.ByteSize
err := maxSize.UnmarshalText([]byte(diskCacheSize))
if err != nil {
log.Fatal(err)
}
if maxSize <= 0 {
log.Fatal("disk cache size must be more than 0")
}
return int(maxSize), path, cacheManager
}
func cleanOldestBlobs(maxItems int, db *db.SQL, store store.BlobStore, stopper *stop.Group) {
// this is so that it runs on startup without having to wait for 10 minutes
err := doClean(maxItems, db, store, stopper)
if err != nil {
log.Error(errors.FullTrace(err))
}
const cleanupInterval = 10 * time.Minute
for {
select {
case <-stopper.Ch():
log.Infoln("stopping self cleanup")
return
case <-time.After(cleanupInterval):
err := doClean(maxItems, db, store, stopper)
if err != nil {
log.Error(errors.FullTrace(err))
}
}
}
}
func doClean(maxItems int, db *db.SQL, store store.BlobStore, stopper *stop.Group) error {
blobsCount, err := db.Count()
if err != nil {
return err
}
if blobsCount >= maxItems {
itemsToDelete := blobsCount / 10
blobs, err := db.LeastRecentlyAccessedHashes(itemsToDelete)
if err != nil {
return err
}
blobsChan := make(chan string, len(blobs))
wg := &stop.Group{}
go func() {
for _, hash := range blobs {
select {
case <-stopper.Ch():
return
default:
}
blobsChan <- hash
}
close(blobsChan)
}()
for i := 0; i < 3; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for h := range blobsChan {
select {
case <-stopper.Ch():
return
default:
}
err = store.Delete(h)
if err != nil {
log.Errorf("error pruning %s: %s", h, errors.FullTrace(err))
continue
}
}
}()
}
wg.Wait()
}
return nil
}
+48 -48
View File
@@ -1,15 +1,20 @@
package cmd
import (
"encoding/json"
"io/ioutil"
"os"
"strings"
"github.com/lbryio/reflector.go/updater"
"github.com/lbryio/lbry.go/v2/dht"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/johntdyer/slackrus"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
// Config is the base configuration for Prism when no sub commands are used.
@@ -18,10 +23,8 @@ type Config struct {
AwsSecret string `json:"aws_secret"`
BucketRegion string `json:"bucket_region"`
BucketName string `json:"bucket_name"`
S3Endpoint string `json:"s3_endpoint"`
DBConn string `json:"db_conn"`
SlackHookURL string `json:"slack_hook_url"`
SlackChannel string `json:"slack_channel"`
UpdateBinURL string `json:"update_bin_url"`
UpdateCmd string `json:"update_cmd"`
}
@@ -35,8 +38,7 @@ const (
)
var conf string
//var globalConfig Config
var globalConfig Config
var rootCmd = &cobra.Command{
Use: "prism",
@@ -49,7 +51,7 @@ var rootCmd = &cobra.Command{
func init() {
rootCmd.PersistentFlags().StringSliceVarP(&verbose, "verbose", "v", []string{}, "Verbose logging for specific components")
rootCmd.PersistentFlags().StringVar(&conf, "conf-dir", "./", "Path to config directory")
rootCmd.PersistentFlags().StringVar(&conf, "conf", "config.json", "Path to config. Use 'none' to disable")
}
// Execute adds all child commands to the root command and sets flags appropriately.
@@ -84,40 +86,38 @@ func preRun(cmd *cobra.Command, args []string) {
}
}
//var err error
var err error
if conf == "" {
logrus.Errorln("--conf flag required")
os.Exit(1)
} else if conf != "none" {
globalConfig, err = loadConfig(conf)
if err != nil {
logrus.Error(err)
os.Exit(1)
}
}
//else if conf != "none" {
// globalConfig, err = loadConfig(conf)
// if err != nil {
// logrus.Error(err)
// os.Exit(1)
// }
//}
viper.AddConfigPath(conf)
//if globalConfig.SlackHookURL != "" {
// hook := &slackrus.SlackrusHook{
// HookURL: globalConfig.SlackHookURL,
// AcceptedLevels: slackrus.LevelThreshold(logrus.InfoLevel),
// Channel: globalConfig.SlackChannel,
// //IconEmoji: ":ghost:",
// //Username: "reflector.go",
// }
// //logrus.SetFormatter(&logrus.JSONFormatter{})
// logrus.AddHook(hook)
// debugLogger.AddHook(hook)
//}
if globalConfig.SlackHookURL != "" {
hook := &slackrus.SlackrusHook{
HookURL: globalConfig.SlackHookURL,
AcceptedLevels: slackrus.LevelThreshold(logrus.InfoLevel),
Channel: "#reflector-logs",
//IconEmoji: ":ghost:",
//Username: "reflector.go",
}
//logrus.SetFormatter(&logrus.JSONFormatter{})
logrus.AddHook(hook)
debugLogger.AddHook(hook)
}
//if globalConfig.UpdateBinURL != "" {
// if globalConfig.UpdateCmd == "" {
// logrus.Warnln("update_cmd is empty in conf file")
// }
// logrus.Println("starting update checker")
// go updater.Run(globalConfig.UpdateBinURL, globalConfig.UpdateCmd)
//}
if globalConfig.UpdateBinURL != "" {
if globalConfig.UpdateCmd == "" {
logrus.Warnln("update_cmd is empty in conf file")
}
logrus.Println("starting update checker")
go updater.Run(globalConfig.UpdateBinURL, globalConfig.UpdateCmd)
}
}
func checkErr(err error) {
@@ -138,20 +138,20 @@ func argFuncs(funcs ...cobra.PositionalArgs) cobra.PositionalArgs {
}
}
//func loadConfig(path string) (Config, error) {
// var c Config
//
// raw, err := os.ReadFile(path)
// if err != nil {
// if os.IsNotExist(err) {
// return c, errors.Err("config file not found")
// }
// return c, errors.Err(err)
// }
//
// err = json.Unmarshal(raw, &c)
// return c, errors.Err(err)
//}
func loadConfig(path string) (Config, error) {
var c Config
raw, err := ioutil.ReadFile(path)
if err != nil {
if os.IsNotExist(err) {
return c, errors.Err("config file not found")
}
return c, errors.Err(err)
}
err = json.Unmarshal(raw, &c)
return c, errors.Err(err)
}
func mustGetFlagString(cmd *cobra.Command, name string) string {
v, err := cmd.Flags().GetString(name)
+9 -9
View File
@@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"os/signal"
"path"
@@ -38,16 +39,17 @@ func sendCmd(cmd *cobra.Command, args []string) {
reflectorAddress := args[0]
err := hackyReflector.Connect(reflectorAddress)
checkErr(err)
defer func() { _ = hackyReflector.Close() }()
defer hackyReflector.Close()
filePath := args[1]
file, err := os.Open(filePath)
checkErr(err)
defer func() { _ = file.Close() }()
defer file.Close()
sdCachePath := ""
sdCacheDir := mustGetFlagString(cmd, "sd-cache")
if sdCacheDir != "" {
if _, statErr := os.Stat(sdCacheDir); os.IsNotExist(statErr) {
if _, err := os.Stat(sdCacheDir); os.IsNotExist(err) {
err = os.MkdirAll(sdCacheDir, 0777)
checkErr(err)
}
@@ -57,9 +59,8 @@ func sendCmd(cmd *cobra.Command, args []string) {
var enc *stream.Encoder
if sdCachePath != "" {
if _, statErr := os.Stat(sdCachePath); !os.IsNotExist(statErr) {
var sdBlob []byte
sdBlob, err = os.ReadFile(sdCachePath)
if _, err := os.Stat(sdCachePath); !os.IsNotExist(err) {
sdBlob, err := ioutil.ReadFile(sdCachePath)
checkErr(err)
cachedSDBlob := &stream.SDBlob{}
err = cachedSDBlob.FromBlob(sdBlob)
@@ -88,8 +89,7 @@ func sendCmd(cmd *cobra.Command, args []string) {
break
}
var b stream.Blob
b, err = enc.Next()
b, err := enc.Next()
if errors.Is(err, io.EOF) {
break
}
@@ -110,7 +110,7 @@ func sendCmd(cmd *cobra.Command, args []string) {
sd := enc.SDBlob()
//sd.StreamName = filepath.Base(filePath)
//sd.SuggestedFileName = filepath.Base(filePath)
err = os.WriteFile(sdCachePath, sd.ToBlob(), 0666)
err = ioutil.WriteFile(sdCachePath, sd.ToBlob(), 0666)
if err != nil {
fmt.Printf("error saving sd blob: %v\n", err)
fmt.Println(sd.ToJson())
+1 -1
View File
@@ -51,7 +51,7 @@ func sendBlobCmd(cmd *cobra.Command, args []string) {
file, err := os.Open(path)
checkErr(err)
defer func() { _ = file.Close() }()
defer file.Close()
s, err := stream.New(file)
checkErr(err)
+102
View File
@@ -0,0 +1,102 @@
package cmd
import (
"os"
"os/signal"
"strconv"
"strings"
"syscall"
"github.com/lbryio/reflector.go/cluster"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/prism"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/server/peer"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/dht"
"github.com/lbryio/lbry.go/v2/dht/bits"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
const (
startNewCluster = "new"
)
var (
startClusterPort int
startPeerPort int
startReflectorPort int
startDhtPort int
startDhtSeeds []string
startHashRange string
)
func init() {
var cmd = &cobra.Command{
Use: `start [cluster-address|"new"]`,
Short: "Runs full prism application with cluster, dht, peer server, and reflector server.",
Run: startCmd,
Args: cobra.ExactArgs(1),
}
cmd.PersistentFlags().IntVar(&startClusterPort, "cluster-port", cluster.DefaultPort, "Port that cluster listens on")
cmd.PersistentFlags().IntVar(&startPeerPort, "peer-port", peer.DefaultPort, "Port to start peer protocol on")
cmd.PersistentFlags().IntVar(&startReflectorPort, "reflector-port", reflector.DefaultPort, "Port to start reflector protocol on")
cmd.PersistentFlags().IntVar(&startDhtPort, "dht-port", dht.DefaultPort, "Port that dht will listen on")
cmd.PersistentFlags().StringSliceVar(&startDhtSeeds, "dht-seeds", []string{}, "Comma-separated list of dht seed nodes (addr:port,addr:port,...)")
cmd.PersistentFlags().StringVar(&startHashRange, "hash-range", "", "Limit on range of hashes to announce (start-end)")
rootCmd.AddCommand(cmd)
}
func startCmd(cmd *cobra.Command, args []string) {
db := &db.SQL{
LogQueries: log.GetLevel() == log.DebugLevel,
}
err := db.Connect(globalConfig.DBConn)
checkErr(err)
s3 := store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
comboStore := store.NewDBBackedStore(s3, db, false)
conf := prism.DefaultConf()
// TODO: args we need:
// minNodes - minimum number of nodes before announcing starts. otherwise first node will try to announce all the blobs in the db
// or maybe we should do maxHashesPerNode?
// in either case, this should not kill the cluster, but should only limit announces (and notify when some hashes are being left unannounced)
if args[0] != startNewCluster {
conf.ClusterSeedAddr = args[0]
}
conf.DB = db
conf.Blobs = comboStore
conf.DhtAddress = "0.0.0.0:" + strconv.Itoa(startDhtPort)
conf.DhtSeedNodes = startDhtSeeds
conf.ClusterPort = startClusterPort
conf.PeerPort = startPeerPort
conf.ReflectorPort = startReflectorPort
if startHashRange != "" {
hashRange := strings.Split(startHashRange, "-")
if len(hashRange) != 2 {
log.Fatal("invalid hash range")
}
r := bits.Range{Start: bits.FromShortHexP(hashRange[0]), End: bits.FromShortHexP(hashRange[1])}
conf.HashRange = &r
}
p := prism.New(conf)
err = p.Start()
if err != nil {
log.Fatal(err)
}
interruptChan := make(chan os.Signal, 1)
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
<-interruptChan
p.Shutdown()
}
+4 -4
View File
@@ -29,9 +29,9 @@ func init() {
func testCmd(cmd *cobra.Command, args []string) {
log.Printf("reflector %s", meta.VersionString())
memStore := store.NewMemStore(store.MemParams{Name: "test"})
memStore := store.NewMemStore()
reflectorServer := reflector.NewIngestionServer(memStore)
reflectorServer := reflector.NewServer(memStore, memStore)
reflectorServer.Timeout = 3 * time.Minute
err := reflectorServer.Start(":" + strconv.Itoa(reflector.DefaultPort))
@@ -39,8 +39,8 @@ func testCmd(cmd *cobra.Command, args []string) {
log.Fatal(err)
}
peerServer := peer.NewServer(memStore, fmt.Sprintf(":%d", reflector.DefaultPort+1))
err = peerServer.Start()
peerServer := peer.NewServer(memStore)
err = peerServer.Start(":" + strconv.Itoa(reflector.DefaultPort+1))
if err != nil {
log.Fatal(err)
}
+10 -10
View File
@@ -5,8 +5,9 @@ import (
"os/signal"
"syscall"
"github.com/lbryio/reflector.go/config"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/store"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
@@ -30,18 +31,17 @@ func init() {
}
func uploadCmd(cmd *cobra.Command, args []string) {
store, err := config.LoadStores(conf, "upload")
if err != nil {
log.Fatal(err)
db := &db.SQL{
LogQueries: log.GetLevel() == log.DebugLevel,
}
defer store.Shutdown()
err := db.Connect(globalConfig.DBConn)
checkErr(err)
databaseConn, err := config.LoadDatabase(conf, "upload")
if err != nil {
log.Fatal(err)
}
st := store.NewDBBackedStore(
store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName),
db, false)
uploader := reflector.NewUploader(databaseConn, store, uploadWorkers, uploadSkipExistsCheck, uploadDeleteBlobsAfterUpload)
uploader := reflector.NewUploader(db, st, uploadWorkers, uploadSkipExistsCheck, uploadDeleteBlobsAfterUpload)
interruptChan := make(chan os.Signal, 1)
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
+1 -1
View File
@@ -18,5 +18,5 @@ func init() {
}
func versionCmd(cmd *cobra.Command, args []string) {
fmt.Println(meta.FullName())
fmt.Println(meta.VersionString())
}
-113
View File
@@ -1,113 +0,0 @@
package config
import (
"fmt"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/server"
"github.com/lbryio/reflector.go/server/http"
"github.com/lbryio/reflector.go/server/http3"
"github.com/lbryio/reflector.go/server/peer"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/extras/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
)
func LoadStores(path, file string) (store.BlobStore, error) {
v := viper.New()
v.SetConfigType("yaml")
v.AddConfigPath(path)
v.SetConfigName(file)
err := v.ReadInConfig()
if err != nil {
return nil, errors.Err(err)
}
storeViper := v.Sub("store")
for storeType := range storeViper.AllSettings() {
factory, exists := store.Factories[storeType]
if !exists {
return nil, errors.Err("unknown store type: %s", storeType)
}
storeConfig := storeViper.Sub(storeType)
s, err := factory(storeConfig)
if err != nil {
return nil, errors.Err(err)
}
//we only expect 1 store as the root, so let's return it
return s, nil
}
return nil, nil
}
func LoadServers(store store.BlobStore, path, file string) ([]server.BlobServer, error) {
v := viper.New()
v.SetConfigType("yaml")
v.AddConfigPath(path)
v.SetConfigName(file)
err := v.ReadInConfig()
if err != nil {
return nil, errors.Err(err)
}
servers := make([]server.BlobServer, 0)
serversViper := v.Sub("servers")
for serverType := range serversViper.AllSettings() {
var cfg server.BlobServerConfig
err := serversViper.Sub(serverType).Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
switch serverType {
case "http":
servers = append(servers, http.NewServer(store, cfg.MaxConcurrentRequests, cfg.EdgeToken, fmt.Sprintf("%s:%d", cfg.Address, cfg.Port)))
case "http3":
servers = append(servers, http3.NewServer(store, cfg.MaxConcurrentRequests, fmt.Sprintf("%s:%d", cfg.Address, cfg.Port)))
case "peer":
servers = append(servers, peer.NewServer(store, fmt.Sprintf("%s:%d", cfg.Address, cfg.Port)))
default:
return nil, errors.Err("unknown server type: %s", serverType)
}
}
return servers, nil
}
func LoadDatabase(path, file string) (*db.SQL, error) {
v := viper.New()
v.SetConfigType("yaml")
v.AddConfigPath(path)
v.SetConfigName(file)
err := v.ReadInConfig()
if err != nil {
return nil, errors.Err(err)
}
dbConfig := v.Sub("database")
if dbConfig == nil {
return nil, errors.Err("db config not found")
}
user := dbConfig.GetString("user")
password := dbConfig.GetString("password")
host := dbConfig.GetString("host")
port := dbConfig.GetInt("port")
database := dbConfig.GetString("database")
logQueries := dbConfig.GetBool("log_queries")
accessTracking := dbConfig.GetInt("access_tracking")
if user == "" || password == "" || host == "" || port == 0 || database == "" {
return nil, errors.Err("db config is missing required fields")
}
dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", user, password, host, port, database)
dbInstance := &db.SQL{
TrackingLevel: db.AccessTrackingLevel(accessTracking),
SoftDelete: true,
LogQueries: logQueries || log.GetLevel() == log.DebugLevel,
}
err = dbInstance.Connect(dsn)
return dbInstance, err
}
+32 -31
View File
@@ -15,32 +15,33 @@ import (
"github.com/lbryio/lbry.go/v2/stream"
"github.com/go-sql-driver/mysql"
_ "github.com/go-sql-driver/mysql" // blank import for db driver ensures its imported even if its not used
log "github.com/sirupsen/logrus"
"github.com/volatiletech/null/v8"
"github.com/volatiletech/null"
"go.uber.org/atomic"
)
// SdBlob is a special blob that contains information on the rest of the blobs in the stream
type SdBlob struct {
StreamName string `json:"stream_name"`
StreamName string `json:"stream_name"`
Blobs []struct {
Length int `json:"length"`
BlobNum int `json:"blob_num"`
BlobHash string `json:"blob_hash,omitempty"`
IV string `json:"iv"`
} `json:"blobs"`
StreamType string `json:"stream_type"`
Key string `json:"key"`
SuggestedFileName string `json:"suggested_file_name"`
StreamHash string `json:"stream_hash"`
Blobs []struct {
BlobHash string `json:"blob_hash,omitempty"`
IV string `json:"iv"`
Length int `json:"length"`
BlobNum int `json:"blob_num"`
} `json:"blobs"`
}
type AccessTrackingLevel int
type trackAccess int
const (
TrackAccessNone AccessTrackingLevel = iota // Don't track accesses
TrackAccessStreams // Track accesses at the stream level
TrackAccessBlobs // Track accesses at the blob level
TrackAccessNone trackAccess = iota // Don't track accesses
TrackAccessStreams // Track accesses at the stream level
TrackAccessBlobs // Track accesses at the blob level
)
// SQL implements the DB interface
@@ -48,7 +49,7 @@ type SQL struct {
conn *sql.DB
// Track the approx last time a blob or stream was accessed
TrackingLevel AccessTrackingLevel
TrackAccess trackAccess
// Instead of deleting a blob, marked it as not stored in the db
SoftDelete bool
@@ -57,7 +58,7 @@ type SQL struct {
LogQueries bool
}
func (s *SQL) logQuery(query string, args ...interface{}) {
func (s SQL) logQuery(query string, args ...interface{}) {
if !s.LogQueries {
return
}
@@ -97,7 +98,7 @@ func (s *SQL) AddBlob(hash string, length int, isStored bool) error {
return err
}
// AddBlobs adds blobs to the database.
//AddBlobs adds blobs to the database.
func (s *SQL) AddBlobs(hash []string) error {
if s.conn == nil {
return errors.Err("not connected")
@@ -179,7 +180,7 @@ func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error)
q string
args []interface{}
)
if s.TrackingLevel == TrackAccessBlobs {
if s.TrackAccess == TrackAccessBlobs {
args = []interface{}{hash, isStored, length, time.Now()}
q = "INSERT INTO blob_ (hash, is_stored, length, last_accessed_at) VALUES (" + qt.Qs(len(args)) + ") ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored)), last_accessed_at = VALUES(last_accessed_at)"
} else {
@@ -201,7 +202,7 @@ func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error)
return 0, errors.Err("blob ID is 0 even after INSERTing and SELECTing")
}
if s.TrackingLevel == TrackAccessBlobs {
if s.TrackAccess == TrackAccessBlobs {
err := s.touchBlobs([]uint64{uint64(blobID)})
if err != nil {
return 0, errors.Err(err)
@@ -218,7 +219,7 @@ func (s *SQL) insertStream(hash string, sdBlobID int64) (int64, error) {
args []interface{}
)
if s.TrackingLevel == TrackAccessStreams {
if s.TrackAccess == TrackAccessStreams {
args = []interface{}{hash, sdBlobID, time.Now()}
q = "INSERT IGNORE INTO stream (hash, sd_blob_id, last_accessed_at) VALUES (" + qt.Qs(len(args)) + ")"
} else {
@@ -240,7 +241,7 @@ func (s *SQL) insertStream(hash string, sdBlobID int64) (int64, error) {
return 0, errors.Err("stream ID is 0 even after INSERTing and SELECTing")
}
if s.TrackingLevel == TrackAccessStreams {
if s.TrackAccess == TrackAccessStreams {
err := s.touchStreams([]uint64{uint64(streamID)})
if err != nil {
return 0, errors.Err(err)
@@ -264,9 +265,9 @@ func (s *SQL) HasBlobs(hashes []string, touch bool) (map[string]bool, error) {
exists, idsNeedingTouch, err := s.hasBlobs(hashes)
if touch {
if s.TrackingLevel == TrackAccessBlobs {
if s.TrackAccess == TrackAccessBlobs {
_ = s.touchBlobs(idsNeedingTouch)
} else if s.TrackingLevel == TrackAccessStreams {
} else if s.TrackAccess == TrackAccessStreams {
_ = s.touchStreams(idsNeedingTouch)
}
}
@@ -338,11 +339,11 @@ func (s *SQL) hasBlobs(hashes []string) (map[string]bool, []uint64, error) {
batch := hashes[doneIndex:sliceEnd]
var query string
if s.TrackingLevel == TrackAccessBlobs {
if s.TrackAccess == TrackAccessBlobs {
query = `SELECT b.hash, b.id, NULL, b.last_accessed_at
FROM blob_ b
WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)`
} else if s.TrackingLevel == TrackAccessStreams {
} else if s.TrackAccess == TrackAccessStreams {
query = `SELECT b.hash, b.id, s.id, s.last_accessed_at
FROM blob_ b
LEFT JOIN stream_blob sb ON b.id = sb.blob_id
@@ -371,15 +372,15 @@ WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)`
defer closeRows(rows)
for rows.Next() {
err = rows.Scan(&hash, &blobID, &streamID, &lastAccessedAt)
err := rows.Scan(&hash, &blobID, &streamID, &lastAccessedAt)
if err != nil {
return errors.Err(err)
}
exists[hash] = true
if !lastAccessedAt.Valid || lastAccessedAt.Time.Before(touchDeadline) {
if s.TrackingLevel == TrackAccessBlobs {
if s.TrackAccess == TrackAccessBlobs {
needsTouch = append(needsTouch, blobID)
} else if s.TrackingLevel == TrackAccessStreams && !streamID.IsZero() {
} else if s.TrackAccess == TrackAccessStreams && !streamID.IsZero() {
needsTouch = append(needsTouch, streamID.Uint64)
}
}
@@ -418,13 +419,13 @@ func (s *SQL) Delete(hash string) error {
return errors.Err(err)
}
// LeastRecentlyAccessedHashes gets the least recently accessed blobs
//LeastRecentlyAccessedHashes gets the least recently accessed blobs
func (s *SQL) LeastRecentlyAccessedHashes(maxBlobs int) ([]string, error) {
if s.conn == nil {
return nil, errors.Err("not connected")
}
if s.TrackingLevel != TrackAccessBlobs {
if s.TrackAccess != TrackAccessBlobs {
return nil, errors.Err("blob access tracking is disabled")
}
@@ -489,7 +490,7 @@ func (s *SQL) GetBlocked() (map[string]bool, error) {
var hash string
for rows.Next() {
err = rows.Scan(&hash)
err := rows.Scan(&hash)
if err != nil {
return nil, errors.Err(err)
}
@@ -533,7 +534,7 @@ func (s *SQL) MissingBlobsForKnownStream(sdHash string) ([]string, error) {
var hash string
for rows.Next() {
err = rows.Scan(&hash)
err := rows.Scan(&hash)
if err != nil {
return nil, errors.Err(err)
}
@@ -636,7 +637,7 @@ func (s *SQL) GetStoredHashesInRange(ctx context.Context, start, end bits.Bitmap
var hash string
ScanLoop:
for rows.Next() {
err = rows.Scan(&hash)
err := rows.Scan(&hash)
if err != nil {
ech <- err
return
+41 -129
View File
@@ -1,139 +1,51 @@
module github.com/lbryio/reflector.go
go 1.25
replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19
require (
github.com/aws/aws-sdk-go-v2 v1.41.0
github.com/aws/aws-sdk-go-v2/config v1.32.6
github.com/aws/aws-sdk-go-v2/credentials v1.19.6
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.18
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
github.com/aws/aws-sdk-go v1.27.0
github.com/bluele/gcache v0.0.2
github.com/brk0v/directio v0.0.0-20241105172640-ae9d82eb8fee
github.com/btcsuite/btcd v0.24.2
github.com/btcsuite/btcutil v1.0.2
github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
github.com/brk0v/directio v0.0.0-20190225130936-69406e757cf7
github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3
github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d
github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2
github.com/davecgh/go-spew v1.1.1
github.com/ekyoung/gin-nice-recovery v0.0.0-20160510022553-1654dca486db
github.com/gin-gonic/gin v1.11.0
github.com/go-sql-driver/mysql v1.9.3
github.com/gogo/protobuf v1.3.2
github.com/golang/protobuf v1.5.4
github.com/google/gops v0.3.28
github.com/gorilla/mux v1.8.1
github.com/hashicorp/serf v0.10.2
github.com/gin-gonic/gin v1.7.1
github.com/go-sql-driver/mysql v1.6.0
github.com/gogo/protobuf v1.2.1
github.com/golang/protobuf v1.5.2
github.com/google/go-cmp v0.5.6 // indirect
github.com/google/gops v0.3.18
github.com/gorilla/mux v1.8.0
github.com/hashicorp/go-msgpack v0.5.5 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/hashicorp/serf v0.9.5
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf
github.com/karrick/godirwalk v1.17.0
github.com/lbryio/chainquery v1.9.1-0.20240927170248-48c092515dea
github.com/lbryio/lbry.go/v2 v2.7.2-0.20230307181431-a01aa6dc0629
github.com/lbryio/types v0.0.0-20220224142228-73610f6654a6
github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5
github.com/prometheus/client_golang v1.23.2
github.com/quic-go/quic-go v0.58.0
github.com/sirupsen/logrus v1.9.3
github.com/spf13/cast v1.10.0
github.com/spf13/cobra v1.10.2
github.com/spf13/viper v1.21.0
github.com/stretchr/testify v1.11.1
github.com/volatiletech/null/v8 v8.1.2
go.uber.org/atomic v1.11.0
golang.org/x/sync v0.19.0
github.com/johntdyer/slackrus v0.0.0-20210521205746-42486fb4c48c
github.com/karrick/godirwalk v1.16.1
github.com/lbryio/chainquery v1.9.0
github.com/lbryio/lbry.go/v2 v2.7.2-0.20210416195322-6516df1418e3
github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386
github.com/lucas-clemente/quic-go v0.20.1
github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6
github.com/prometheus/client_golang v1.10.0
github.com/sergi/go-diff v1.2.0 // indirect
github.com/sirupsen/logrus v1.8.1
github.com/spf13/afero v1.4.1 // indirect
github.com/spf13/cast v1.3.0
github.com/spf13/cobra v1.1.3
github.com/spf13/viper v1.7.1 // indirect
github.com/stretchr/testify v1.7.0
github.com/volatiletech/null v8.0.0+incompatible
go.uber.org/atomic v1.7.0
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
golang.org/x/text v0.3.6 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/protobuf v1.27.1 // indirect
)
require (
filippo.io/edwards25519 v1.1.0 // indirect
github.com/armon/go-metrics v0.4.1 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 // indirect
github.com/aws/smithy-go v1.24.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/btcsuite/btclog v1.0.0 // indirect
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd // indirect
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 // indirect
github.com/bytedance/sonic v1.14.0 // indirect
github.com/bytedance/sonic/loader v0.3.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/cloudwego/base64x v0.1.6 // indirect
github.com/friendsofgo/errors v0.9.2 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.8 // indirect
github.com/gin-contrib/sse v1.1.0 // indirect
github.com/go-errors/errors v1.5.1 // indirect
github.com/go-ini/ini v1.67.0 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.27.0 // indirect
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
github.com/goccy/go-json v0.10.5 // indirect
github.com/goccy/go-yaml v1.18.0 // indirect
github.com/gofrs/uuid v4.4.0+incompatible // indirect
github.com/google/btree v1.1.2 // indirect
github.com/gorilla/rpc v1.2.0 // indirect
github.com/gorilla/websocket v1.5.3 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
github.com/hashicorp/go-metrics v0.5.4 // indirect
github.com/hashicorp/go-msgpack/v2 v2.1.2 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-sockaddr v1.0.5 // indirect
github.com/hashicorp/golang-lru v1.0.2 // indirect
github.com/hashicorp/memberlist v0.5.2 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/miekg/dns v1.1.56 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.66.1 // indirect
github.com/prometheus/procfs v0.16.1 // indirect
github.com/quic-go/qpack v0.6.0 // indirect
github.com/sagikazarmark/locafero v0.11.0 // indirect
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
github.com/shopspring/decimal v1.4.0 // indirect
github.com/slack-go/slack v0.16.0 // indirect
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
github.com/spf13/afero v1.15.0 // indirect
github.com/spf13/pflag v1.0.10 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.3.0 // indirect
github.com/volatiletech/inflect v0.0.1 // indirect
github.com/volatiletech/randomize v0.0.1 // indirect
github.com/volatiletech/strmangle v0.0.8 // indirect
go.yaml.in/yaml/v2 v2.4.2 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/arch v0.20.0 // indirect
golang.org/x/crypto v0.41.0 // indirect
golang.org/x/mod v0.27.0 // indirect
golang.org/x/net v0.43.0 // indirect
golang.org/x/sys v0.35.0 // indirect
golang.org/x/text v0.28.0 // indirect
golang.org/x/time v0.8.0 // indirect
golang.org/x/tools v0.36.0 // indirect
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
google.golang.org/protobuf v1.36.9 // indirect
gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
go 1.16
+734 -292
View File
File diff suppressed because it is too large. Load diff
+10 -30
View File
@@ -69,7 +69,6 @@ const (
DirectionDownload = "download" // from reflector
LabelCacheType = "cache_type"
LabelOrigin = "origin"
LabelComponent = "component"
LabelSource = "source"
@@ -87,13 +86,8 @@ const (
errUnexpectedEOFStr = "unexpected_eof_str"
errJSONSyntax = "json_syntax"
errBlobTooBig = "blob_too_big"
errInvalidPeerJSON = "invalid_peer_json"
errInvalidPeerData = "invalid_peer_data"
errRequestTooLarge = "request_too_large"
errDeadlineExceeded = "deadline_exceeded"
errHashMismatch = "hash_mismatch"
errProtectedBlob = "protected_blob"
errInvalidBlobHash = "invalid_blob_hash"
errZeroByteBlob = "zero_byte_blob"
errInvalidCharacter = "invalid_character"
errBlobNotFound = "blob_not_found"
@@ -136,12 +130,18 @@ var (
Name: "hit_total",
Help: "Total number of blobs retrieved from the cache storage",
}, []string{LabelCacheType, LabelComponent})
ItttHitCount = promauto.NewCounterVec(prometheus.CounterOpts{
ThisHitCount = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Subsystem: subsystemITTT,
Name: "hits_total",
Help: "Total number of blobs retrieved from the this/that storage",
}, []string{LabelOrigin})
Name: "this_hit_total",
Help: "Total number of blobs retrieved from the this storage",
})
ThatHitCount = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Subsystem: subsystemITTT,
Name: "that_hit_total",
Help: "Total number of blobs retrieved from the that storage",
})
CacheMissCount = promauto.NewCounterVec(prometheus.CounterOpts{
Namespace: ns,
Subsystem: subsystemCache,
@@ -224,11 +224,6 @@ var (
Name: "s3_out_bytes",
Help: "Total number of outgoing bytes (to S3)",
})
MtrInBytesUpstream = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "upstream_in_bytes",
Help: "Total number of incoming bytes (from Upstream)",
})
MtrInBytesS3 = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "s3_in_bytes",
@@ -257,11 +252,6 @@ func CacheLabels(name, component string) prometheus.Labels {
LabelComponent: component,
}
}
func ItttLabels(orig string) prometheus.Labels {
return prometheus.Labels{
LabelOrigin: orig,
}
}
func TrackError(direction string, e error) (shouldLog bool) { // shouldLog is a hack, but whatever
if e == nil {
@@ -306,20 +296,10 @@ func TrackError(direction string, e error) (shouldLog bool) { // shouldLog is a
} else if strings.Contains(err.Error(), "blob must be at most") {
//log.Warnln("blob must be at most X bytes is not the same as ErrBlobTooBig")
errType = errBlobTooBig
} else if strings.Contains(err.Error(), "invalid json request") {
errType = errInvalidPeerJSON
} else if strings.Contains(err.Error(), "Invalid data") {
errType = errInvalidPeerData
} else if strings.Contains(err.Error(), "request is too large") {
errType = errRequestTooLarge
} else if strings.Contains(err.Error(), "Invalid blob hash length") {
errType = errInvalidBlobHash
} else if strings.Contains(err.Error(), "hash of received blob data does not match hash from send request") {
errType = errHashMismatch
} else if strings.Contains(err.Error(), "blob not found") {
errType = errBlobNotFound
} else if strings.Contains(err.Error(), "requested blob is protected") {
errType = errProtectedBlob
} else if strings.Contains(err.Error(), "0-byte blob received") {
errType = errZeroByteBlob
} else if strings.Contains(err.Error(), "PROTOCOL_VIOLATION: tried to retire connection") {
+11 -10
View File
@@ -8,23 +8,24 @@ import (
qt "github.com/lbryio/lbry.go/v2/extras/query"
"github.com/go-sql-driver/mysql"
_ "github.com/go-sql-driver/mysql" // blank import for db driver ensures its imported even if its not used
log "github.com/sirupsen/logrus"
"github.com/volatiletech/null/v8"
"github.com/volatiletech/null"
)
// SdBlob is a special blob that contains information on the rest of the blobs in the stream
type SdBlob struct {
StreamName string `json:"stream_name"`
StreamName string `json:"stream_name"`
Blobs []struct {
Length int `json:"length"`
BlobNum int `json:"blob_num"`
BlobHash string `json:"blob_hash,omitempty"`
IV string `json:"iv"`
} `json:"blobs"`
StreamType string `json:"stream_type"`
Key string `json:"key"`
SuggestedFileName string `json:"suggested_file_name"`
StreamHash string `json:"stream_hash"`
Blobs []struct {
BlobHash string `json:"blob_hash,omitempty"`
IV string `json:"iv"`
Length int `json:"length"`
BlobNum int `json:"blob_num"`
} `json:"blobs"`
}
// SQL implements the DB interface
@@ -110,7 +111,7 @@ func (s *SQL) HasBlob(hash string) (bool, error) {
// HasBlobs checks if the database contains the set of blobs and returns a bool map.
func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
exists, streamsNeedingTouch, err := s.hasBlobs(hashes)
_ = s.touch(streamsNeedingTouch)
s.touch(streamsNeedingTouch)
return exists, err
}
@@ -181,7 +182,7 @@ WHERE is_stored = ? and hash IN (` + qt.Qs(len(batch)) + `)`
defer closeRows(rows)
for rows.Next() {
err = rows.Scan(&hash, &blobID, &lastAccessedAt)
err := rows.Scan(&hash, &blobID, &lastAccessedAt)
if err != nil {
return errors.Err(err)
}
+4
View File
@@ -1,6 +1,9 @@
package main
import (
"math/rand"
"time"
"github.com/lbryio/reflector.go/cmd"
"github.com/google/gops/agent"
@@ -11,5 +14,6 @@ func main() {
if err := agent.Listen(agent.Options{}); err != nil {
log.Fatal(err)
}
rand.Seed(time.Now().UnixNano())
cmd.Execute()
}
+8 -30
View File
@@ -6,36 +6,9 @@ import (
"time"
)
var (
name = "prism-bin"
version = "unknown"
commit = "unknown"
commitLong = "unknown"
branch = "unknown"
Time = "unknown"
BuildTime time.Time
)
// Name returns main application name
func Name() string {
return name
}
// Version returns current application version
func Version() string {
return version
}
// FullName returns current app version, commit and build time
func FullName() string {
return fmt.Sprintf(
`Name: %v
Version: %v
branch: %v
commit: %v
commit long: %v
build date: %v`, Name(), Version(), branch, commit, commitLong, BuildTime.String())
}
var Version = ""
var Time = ""
var BuildTime time.Time
func init() {
if Time != "" {
@@ -47,6 +20,11 @@ func init() {
}
func VersionString() string {
version := Version
if version == "" {
version = "<unset>"
}
var buildTime string
if BuildTime.IsZero() {
buildTime = "<now>"
+16 -13
View File
@@ -2,7 +2,6 @@ package prism
import (
"context"
"fmt"
"strconv"
"sync"
@@ -21,16 +20,20 @@ import (
)
type Config struct {
Blobs store.BlobStore
// limit the range of hashes to announce. useful for testing
HashRange *bits.Range
DB *db.SQL
DhtAddress string
ClusterSeedAddr string
DhtSeedNodes []string
PeerPort int
ReflectorPort int
PeerPort int
ReflectorPort int
DhtAddress string
DhtSeedNodes []string
ClusterPort int
ClusterSeedAddr string
// limit the range of hashes to announce. useful for testing
HashRange *bits.Range
DB *db.SQL
Blobs store.BlobStore
}
// DefaultConf returns a default config
@@ -75,8 +78,8 @@ func New(conf *Config) *Prism {
db: conf.DB,
dht: d,
cluster: c,
peer: peer.NewServer(conf.Blobs, fmt.Sprintf(":%d", conf.PeerPort)),
reflector: reflector.NewIngestionServer(conf.Blobs),
peer: peer.NewServer(conf.Blobs),
reflector: reflector.NewServer(conf.Blobs, conf.Blobs),
grp: stop.New(),
}
@@ -104,7 +107,7 @@ func (p *Prism) Start() error {
return errors.Err("blobs required in conf")
}
err = p.peer.Start()
err = p.peer.Start(":" + strconv.Itoa(p.conf.PeerPort))
if err != nil {
return err
}
+3 -2
View File
@@ -115,7 +115,7 @@ func Publish(client *lbrycrd.Client, path, name, address string, details Details
return signedTx, txid, nil
}
// TODO: lots of assumptions. hardcoded values need to be passed in or calculated
//TODO: lots of assumptions. hardcoded values need to be passed in or calculated
func baseTx(client *lbrycrd.Client, amount float64, changeAddress btcutil.Address) (*wire.MsgTx, error) {
txFee := 0.0002 // TODO: estimate this better?
@@ -222,7 +222,8 @@ func makeStream(path string) (stream.Stream, *pb.Stream, error) {
if err != nil {
return nil, nil, errors.Err(err)
}
defer func() { _ = file.Close() }()
defer file.Close()
enc := stream.NewEncoder(file)
s, err := enc.Stream()
+11 -11
View File
@@ -13,21 +13,21 @@ func LoadWallet(r io.Reader) (WalletFile, error) {
type WalletFile struct {
Name string `json:"name"`
Accounts []Account `json:"accounts"`
Preferences WalletPrefs `json:"preferences"`
Version int `json:"version"`
Preferences WalletPrefs `json:"preferences"`
Accounts []Account `json:"accounts"`
}
type Account struct {
AddressGenerator AddressGenerator `json:"address_generator"`
Certificates map[string]string `json:"certificates"`
Encrypted bool `json:"encrypted"`
Ledger string `json:"ledger"`
ModifiedOn float64 `json:"modified_on"`
Name string `json:"name"`
PrivateKey string `json:"private_key"`
PublicKey string `json:"public_key"`
Seed string `json:"seed"`
AddressGenerator AddressGenerator `json:"address_generator"`
ModifiedOn float64 `json:"modified_on"`
Encrypted bool `json:"encrypted"`
}
type AddressGenerator struct {
@@ -43,17 +43,17 @@ type AddressGenParams struct {
type WalletPrefs struct {
Shared struct {
Ts float64 `json:"ts"`
Value struct {
Type string `json:"type"`
Version string `json:"version"`
Value struct {
Type string `json:"type"`
Value struct {
AppWelcomeVersion int `json:"app_welcome_version"`
Blocked []interface{} `json:"blocked"`
Sharing3P bool `json:"sharing_3P"`
Subscriptions []string `json:"subscriptions"`
Tags []string `json:"tags"`
AppWelcomeVersion int `json:"app_welcome_version"`
Sharing3P bool `json:"sharing_3P"`
} `json:"value"`
Version string `json:"version"`
} `json:"value"`
Ts float64 `json:"ts"`
} `json:"shared"`
}
+103 -168
View File
@@ -1,189 +1,124 @@
# Reflector
Production-ready blob reflection, distribution, and caching for Odysee.
Reflector is a central piece of software that providers LBRY with the following features:
- Blobs reflection: when something is published, we capture the data and store it on our servers for quicker retrieval
- Blobs distribution: when a piece of content is requested and the LBRY network doesn't have it, reflector will retrieve it from its storage and distribute it
- Blobs caching: reflectors can be chained together in multiple regions or servers to form a chain of cached content. We call those "blobcaches". They are layered so that content distribution is favorable in all the regions we deploy it to
This repository provides the components used in production:
- Reflector ingestion server (command name: `reflector`)
- Blob cache/edge server (`blobcache`)
- Uploader to object storage (`upload`)
There are a few other features embedded in reflector.go including publishing streams from Go, downloading or upload blobs, resolving content and more unfinished tools.
Other commands exist in the tree for historical/legacy reasons and are not supported.
This code includes a Go implementations of the LBRY peer protocol, reflector protocol, and DHT.
## How it works (at a glance)
- Ingestion (reflector): accepts uploaded blobs, persists them to object storage (e.g., S3/Wasabi) and tracks state in MySQL.
- Distribution: serves blobs over HTTP/HTTP3/Peer. Blobcaches can be deployed in front of the origin to reduce latency and egress.
- Caching (blobcache): layered disk caches backed by HTTP(S) origins (e.g., S3 endpoints), with optional local DB metadata for capacity/eviction.
## Installation
All services are started by the `prism` binary and are configured via YAML files loaded from a configuration directory.
- Install mysql 8 (5.7 might work too)
- add a reflector user and database with password `reflector` with localhost access only
- Create the tables as described [here](https://github.com/lbryio/reflector.go/blob/ittt/db/db.go#L735) (the link might not update as the code does so just look for the schema in that file)
## Supported commands
The following are the only supported commands for production use:
- Reflector ingestion: `prism reflector`
- Flags: `--receiver-port` (default 5566), `--metrics-port` (default 2112), `--disable-blocklist`
- Loads `reflector.yaml` from the config directory.
- Blob cache: `prism blobcache`
- Flags: `--metrics-port` (default 2112), `--disable-blocklist`
- Loads `blobcache.yaml` from the config directory.
- Uploader: `prism upload PATH`
- Flags: `--workers`, `--skipExistsCheck`, `--deleteBlobsAfterUpload`
- Loads `upload.yaml` from the config directory.
Global flag for all commands:
- `--conf-dir` (default `./`): directory containing YAML config files.
## Configuration
Configuration is per-command. The loader reads `<command>.yaml` from `--conf-dir`.
Common sections:
- `servers`: enables HTTP/HTTP3/Peer servers. Keys: `http`, `http3`, `peer`. Each accepts:
- `port` (int)
- `max_concurrent_requests` (int, http/http3)
- `edge_token` (string, http)
- `address` (string, optional; bind address, omit for all interfaces)
- `store`: defines the storage topology using composable stores. Frequently used:
- `proxied-s3`: production pattern with a `writer` (DB-backed -> S3/multiwriter) and a `reader` (caching -> disk + HTTP origins).
- `caching`: layered cache with a `cache` (often `db_backed` -> `disk`) and an `origin` chain (`http`, `http3`, or `ittt` fan-in).
- `s3`, `disk`, `multiwriter`, `db_backed`, `http`, `http3`, `peer`, `upstream` are also available building blocks.
### Minimal examples
Reflector – conf-dir contains `reflector.yaml`:
```yaml
servers:
http:
port: 5569
max_concurrent_requests: 200
http3:
port: 5568
max_concurrent_requests: 200
peer:
port: 5567
store:
proxied-s3:
name: s3_read_proxy
writer:
db_backed:
user: reflector
password: reflector
database: reflector
host: localhost
port: 3306
access_tracking: 1
soft_deletes: true
store:
s3:
name: primary
aws_id: YOUR_KEY
aws_secret: YOUR_SECRET
region: us-east-1
bucket: blobs-bucket
endpoint: https://s3.yourendpoint.tv
reader:
caching:
cache:
disk:
name: local_cache
mount_point: /mnt/reflector/cache
sharding_size: 2
origin:
http:
endpoint: https://s3.yourendpoint.tv/blobs-bucket/
sharding_size: 4
```
Blobcache – conf-dir contains `blobcache.yaml`:
```yaml
servers:
http:
port: 5569
max_concurrent_requests: 200
http3:
port: 5568
max_concurrent_requests: 200
peer:
port: 5567
store:
caching:
cache:
db_backed:
user: reflector
password: reflector
database: reflector
host: localhost
port: 3306
has_cap: true
max_size: 500GB
store:
disk:
name: blobcache
mount_point: /mnt/blobcache/cache
sharding_size: 2
origin:
http:
endpoint: https://s3.yourendpoint.tv/blobs-bucket/
sharding_size: 4
```
Uploader – conf-dir contains `upload.yaml` (points to the same writer/backend as reflector):
```yaml
database:
user: reflector
password: reflector
database: reflector
host: localhost
port: 3306
store:
proxied-s3:
writer:
db_backed:
user: reflector
password: reflector
database: reflector
host: localhost
port: 3306
store:
s3:
aws_id: YOUR_KEY
aws_secret: YOUR_SECRET
region: us-east-1
bucket: blobs-bucket
endpoint: https://s3.yourendpoint.tv
```
## Quick start
1) Build
- Requires Go 1.25+
- `make` (binaries in `dist/<platform>/prism-bin`)
2) Run a local blobcache
#### We do not support running reflector.go as a blob receiver, however if you want to run it as a private blobcache you may compile it yourself and run it as following:
```bash
./dist/linux_amd64/prism-bin --conf-dir=./ blobcache
./prism-bin reflector \
--conf="none" \
--disable-uploads=true \
--use-db=false \
--upstream-reflector="reflector.lbry.com" \
--upstream-protocol="http" \
--request-queue-size=200 \
--disk-cache="2GB:/path/to/your/storage/:localdb" \
```
Place your `blobcache.yaml` in the `--conf-dir` directory.
3) Run reflector ingestion
Create a systemd script if you want to run it automatically on startup or as a service.
## Usage
Usage as reflector/blobcache:
```bash
./dist/linux_amd64/prism-bin --conf-dir=./ reflector --receiver-port=5566 --metrics-port=2112
Run reflector server
Usage:
prism reflector [flags]
Flags:
--disable-blocklist Disable blocklist watching/updating
--disable-uploads Disable uploads to this reflector server
--disk-cache string Where to cache blobs on the file system. format is 'sizeGB:CACHE_PATH:cachemanager' (cachemanagers: localdb/lfuda/lru) (default "100GB:/tmp/downloaded_blobs:localdb")
-h, --help help for reflector
--http-peer-port int The port reflector will distribute content from over HTTP protocol (default 5569)
--http3-peer-port int The port reflector will distribute content from over HTTP3 protocol (default 5568)
--mem-cache int enable in-memory cache with a max size of this many blobs
--metrics-port int The port reflector will use for prometheus metrics (default 2112)
--optional-disk-cache string Optional secondary file system cache for blobs. format is 'sizeGB:CACHE_PATH:cachemanager' (cachemanagers: localdb/lfuda/lru) (this would get hit before the one specified in disk-cache)
--origin-endpoint string HTTP edge endpoint for standard HTTP retrieval
--origin-endpoint-fallback string HTTP edge endpoint for standard HTTP retrieval if first origin fails
--receiver-port int The port reflector will receive content from (default 5566)
--request-queue-size int How many concurrent requests from downstream should be handled at once (the rest will wait) (default 200)
--tcp-peer-port int The port reflector will distribute content from for the TCP (LBRY) protocol (default 5567)
--upstream-protocol string protocol used to fetch blobs from another upstream reflector server (tcp/http3/http) (default "http")
--upstream-reflector string host:port of a reflector server where blobs are fetched from
--use-db Whether to connect to the reflector db or not (default true)
Global Flags:
--conf string Path to config. Use 'none' to disable (default "config.json")
-v, --verbose strings Verbose logging for specific components
```
4) Upload blobs
Other uses:
```bash
./dist/linux_amd64/prism-bin --conf-dir=./ upload /path/to/blobs \
--workers=4 --skipExistsCheck
Prism is a single entry point application with multiple sub modules which can be leveraged individually or together
Usage:
prism [command]
Available Commands:
check-integrity check blobs integrity for a given path
cluster Start(join) to or Start a new cluster
decode Decode a claim value
dht Run dht node
getstream Get a stream from a reflector server
help Help about any command
peer Run peer server
populate-db populate local database with blobs from a disk storage
publish Publish a file
reflector Run reflector server
resolve Resolve a URL
send Send a file to a reflector
sendblob Send a random blob to a reflector server
start Runs full prism application with cluster, dht, peer server, and reflector server.
test Test things
upload Upload blobs to S3
version Print the version
Flags:
--conf string Path to config. Use 'none' to disable (default "config.json")
-h, --help help for prism
-v, --verbose strings Verbose logging for specific components
```
## Running from Source
This project requires [Go v1.16](https://golang.org/doc/install).
On Ubuntu you can install it with `sudo snap install go --classic`
```
git clone git@github.com:lbryio/reflector.go.git
cd reflector.go
make
./bin/prism-bin
```
## Notes
- Only reflector, blobcache, and upload are supported. All other commands are legacy and may be removed in the future.
- Metrics are exposed on the configured `--metrics-port` at `/metrics` (Prometheus format).
- MySQL is required when using DB-backed stores (e.g., ingestion writer, capacity-aware caches).
## Contributing
## Security
If you discover a security issue, please email security@lbry.com. Our PGP key is available at https://lbry.com/faq/pgp-key.
coming soon
## License
MIT License. See LICENSE.
This project is MIT licensed.
## Security
We take security seriously. Please contact security@lbry.com regarding any security issues.
Our PGP key is [here](https://keybase.io/lbry/key.asc) if you need it.
## Contact
The primary contact for this project is [@Nikooo777](https://github.com/Nikooo777)
The primary contact for this project is [@Nikooo777](https://github.com/Nikooo777) (niko-at-lbry.com)
+11 -11
View File
@@ -22,10 +22,10 @@ const blocklistURL = "https://api.lbry.com/file/list_blocked"
func (s *Server) enableBlocklist(b store.Blocklister) {
walletServers := []string{
"a-hub1.odysee.com:50001",
"b-hub1.odysee.com:50001",
"c-hub1.odysee.com:50001",
"s-hub1.odysee.com:50001",
"spv25.lbry.com:50001",
"spv26.lbry.com:50001",
"spv19.lbry.com:50001",
"spv14.lbry.com:50001",
}
updateBlocklist(b, walletServers, s.grp.Ch())
@@ -69,18 +69,18 @@ func blockedSdHashes(walletServers []string, stopper stop.Chan) (map[string]valO
return nil, errors.Err(err)
}
defer func() {
closeErr := resp.Body.Close()
if closeErr != nil {
log.Errorln(errors.Err(closeErr))
err := resp.Body.Close()
if err != nil {
log.Errorln(errors.Err(err))
}
}()
var r struct {
Error string `json:"error"`
Data struct {
Success bool `json:"success"`
Error string `json:"error"`
Data struct {
Outpoints []string `json:"outpoints"`
} `json:"data"`
Success bool `json:"success"`
}
if err = json.NewDecoder(resp.Body).Decode(&r); err != nil {
@@ -95,8 +95,8 @@ func blockedSdHashes(walletServers []string, stopper stop.Chan) (map[string]valO
}
type valOrErr struct {
Err error
Value string
Err error
}
// sdHashesForOutpoints queries wallet server for the sd hashes in a given outpoints
-85
View File
@@ -1,85 +0,0 @@
package reflector
import (
"encoding/json"
"net/http"
"time"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/bluele/gcache"
"golang.org/x/sync/singleflight"
)
const protectedListURL = "https://direct.api.odysee.com/file/list_protected"
type ProtectedContent struct {
SDHash string `json:"sd_hash"`
ClaimID string `json:"claim_id"`
}
var protectedCache = gcache.New(10).Expiration(2 * time.Minute).Build()
func GetProtectedContent() (interface{}, error) {
cachedVal, err := protectedCache.Get("protected")
if err == nil && cachedVal != nil {
return cachedVal.(map[string]bool), nil
}
method := "GET"
var r struct {
Error string `json:"error"`
Data []ProtectedContent `json:"data"`
Success bool `json:"success"`
}
// Bound the request to avoid hanging the entire request path.
// Without a timeout, a slow or unreachable endpoint can block
// singleflight callers indefinitely and stall HTTP handlers.
client := &http.Client{Timeout: 5 * time.Second}
req, err := http.NewRequest(method, protectedListURL, nil)
if err != nil {
return nil, errors.Err(err)
}
res, err := client.Do(req)
if err != nil {
return nil, errors.Err(err)
}
defer func() { _ = res.Body.Close() }()
if res.StatusCode != http.StatusOK {
return nil, errors.Err("unexpected status code %d", res.StatusCode)
}
if err = json.NewDecoder(res.Body).Decode(&r); err != nil {
return nil, errors.Err(err)
}
if !r.Success {
return nil, errors.Prefix("file/list_protected API call", r.Error)
}
protectedMap := make(map[string]bool, len(r.Data))
for _, pc := range r.Data {
protectedMap[pc.SDHash] = true
}
err = protectedCache.Set("protected", protectedMap)
if err != nil {
return protectedMap, errors.Err(err)
}
return protectedMap, nil
}
var sf = singleflight.Group{}
func IsProtected(sdHash string) bool {
val, err, _ := sf.Do("protected", GetProtectedContent)
if err != nil {
return false
}
cachedMap, ok := val.(map[string]bool)
if !ok {
return false
}
return cachedMap[sdHash]
}
+28 -22
View File
@@ -6,6 +6,7 @@ import (
"encoding/hex"
"encoding/json"
"io"
"io/ioutil"
"net"
"time"
@@ -35,21 +36,22 @@ var ErrBlobTooBig = errors.Base("blob must be at most %d bytes", maxBlobSize)
// Server is and instance of the reflector server. It houses the blob store and listener.
type Server struct {
store store.BlobStore
grp *stop.Group
Timeout time.Duration // timeout to read or write next message
EnableBlocklist bool // if true, blocklist checking and blob deletion will be enabled
//underlyingStore store.BlobStore
//outerStore store.BlobStore
underlyingStore store.BlobStore
outerStore store.BlobStore
grp *stop.Group
}
func NewIngestionServer(store store.BlobStore) *Server {
// NewServer returns an initialized reflector server pointer.
func NewServer(underlying store.BlobStore, outer store.BlobStore) *Server {
return &Server{
Timeout: DefaultTimeout,
store: store,
grp: stop.New(),
Timeout: DefaultTimeout,
underlyingStore: underlying,
outerStore: outer,
grp: stop.New(),
}
}
@@ -60,7 +62,7 @@ func (s *Server) Shutdown() {
log.Println("reflector server stopped")
}
// Start starts the server to handle connections.
//Start starts the server to handle connections.
func (s *Server) Start(address string) error {
l, err := net.Listen(network, address)
if err != nil {
@@ -88,7 +90,7 @@ func (s *Server) Start(address string) error {
}()
if s.EnableBlocklist {
if b, ok := s.store.(store.Blocklister); ok {
if b, ok := s.underlyingStore.(store.Blocklister); ok {
s.grp.Add(1)
metrics.RoutinesQueue.WithLabelValues("reflector", "enableblocklist").Inc()
go func() {
@@ -151,7 +153,7 @@ func (s *Server) handleConn(conn net.Conn) {
if errors.Is(err, io.EOF) || s.quitting() {
return
}
err = s.doError(conn, err)
err := s.doError(conn, err)
if err != nil {
log.Error(errors.Prefix("sending handshake error", err))
}
@@ -164,7 +166,7 @@ func (s *Server) handleConn(conn net.Conn) {
if errors.Is(err, io.EOF) || s.quitting() {
return
}
err = s.doError(conn, err)
err := s.doError(conn, err)
if err != nil {
log.Error(errors.Prefix("sending blob receive error", err))
}
@@ -199,14 +201,13 @@ func (s *Server) receiveBlob(conn net.Conn) error {
}
var wantsBlob bool
if bl, ok := s.store.(store.Blocklister); ok {
if bl, ok := s.underlyingStore.(store.Blocklister); ok {
wantsBlob, err = bl.Wants(blobHash)
if err != nil {
return err
}
} else {
var blobExists bool
blobExists, err = s.store.Has(blobHash)
blobExists, err := s.underlyingStore.Has(blobHash)
if err != nil {
return err
}
@@ -216,14 +217,14 @@ func (s *Server) receiveBlob(conn net.Conn) error {
var neededBlobs []string
if isSdBlob && !wantsBlob {
if nbc, ok := s.store.(store.NeededBlobChecker); ok {
if nbc, ok := s.underlyingStore.(neededBlobChecker); ok {
neededBlobs, err = nbc.MissingBlobsForKnownStream(blobHash)
if err != nil {
return err
}
} else {
// if we can't check for blobs in a stream, we have to say that the sd blob is
// missing. if we say we have the sd blob, they won't try to send any content blobs
// missing. if we say we have the sd blob, they wont try to send any content blobs
wantsBlob = true
}
}
@@ -259,9 +260,9 @@ func (s *Server) receiveBlob(conn net.Conn) error {
log.Debugln("Got blob " + blobHash[:8])
if isSdBlob {
err = s.store.PutSD(blobHash, blob)
err = s.outerStore.PutSD(blobHash, blob)
} else {
err = s.store.Put(blobHash, blob)
err = s.outerStore.Put(blobHash, blob)
}
if err != nil {
return err
@@ -366,7 +367,7 @@ func (s *Server) read(conn net.Conn, v interface{}) error {
dec := json.NewDecoder(conn)
err = dec.Decode(v)
if err != nil {
data, _ := io.ReadAll(dec.Buffered())
data, _ := ioutil.ReadAll(dec.Buffered())
if len(data) > 0 {
return errors.Err("%s. Data: %s", err.Error(), hex.EncodeToString(data))
}
@@ -429,8 +430,8 @@ type handshakeRequestResponse struct {
type sendBlobRequest struct {
BlobHash string `json:"blob_hash,omitempty"`
SdBlobHash string `json:"sd_blob_hash,omitempty"`
BlobSize int `json:"blob_size,omitempty"`
SdBlobHash string `json:"sd_blob_hash,omitempty"`
SdBlobSize int `json:"sd_blob_size,omitempty"`
}
@@ -439,8 +440,8 @@ type sendBlobResponse struct {
}
type sendSdBlobResponse struct {
NeededBlobs []string `json:"needed_blobs,omitempty"`
SendSdBlob bool `json:"send_sd_blob"`
NeededBlobs []string `json:"needed_blobs,omitempty"`
}
type blobTransferResponse struct {
@@ -450,3 +451,8 @@ type blobTransferResponse struct {
type sdBlobTransferResponse struct {
ReceivedSdBlob bool `json:"received_sd_blob"`
}
// neededBlobChecker can check which blobs from a known stream are not uploaded yet
type neededBlobChecker interface {
MissingBlobsForKnownStream(string) ([]string, error)
}
+5 -5
View File
@@ -23,7 +23,7 @@ func startServerOnRandomPort(t *testing.T) (*Server, int) {
t.Fatal(err)
}
srv := NewIngestionServer(store.NewMemStore(store.MemParams{Name: "test"}))
srv := NewServer(store.NewMemStore(), store.NewMemStore())
err = srv.Start("127.0.0.1:" + strconv.Itoa(port))
if err != nil {
t.Fatal(err)
@@ -120,7 +120,7 @@ func TestServer_Timeout(t *testing.T) {
t.Fatal(err)
}
srv := NewIngestionServer(store.NewMemStore(store.MemParams{Name: "test"}))
srv := NewServer(store.NewMemStore(), store.NewMemStore())
srv.Timeout = testTimeout
err = srv.Start("127.0.0.1:" + strconv.Itoa(port))
if err != nil {
@@ -182,8 +182,8 @@ func TestServer_PartialUpload(t *testing.T) {
missing[i] = bits.Rand().String()
}
st := store.BlobStore(&mockPartialStore{MemStore: store.NewMemStore(store.MemParams{Name: "test"}), missing: missing})
if _, ok := st.(store.NeededBlobChecker); !ok {
st := store.BlobStore(&mockPartialStore{MemStore: store.NewMemStore(), missing: missing})
if _, ok := st.(neededBlobChecker); !ok {
t.Fatal("mock does not implement the relevant interface")
}
err = st.Put(sdHash, randBlob(10))
@@ -191,7 +191,7 @@ func TestServer_PartialUpload(t *testing.T) {
t.Fatal(err)
}
srv := NewIngestionServer(st)
srv := NewServer(st, st)
err = srv.Start("127.0.0.1:" + strconv.Itoa(port))
if err != nil {
t.Fatal(err)
+10 -8
View File
@@ -1,6 +1,7 @@
package reflector
import (
"io/ioutil"
"os"
"path"
"sync"
@@ -30,16 +31,17 @@ type Summary struct {
type Uploader struct {
db *db.SQL
store store.BlobStore
stopper *stop.Group
countChan chan increment
count Summary
store *store.DBBackedStore // could just be store.BlobStore interface
workers int
skipExistsCheck bool
deleteBlobsAfterUpload bool
stopper *stop.Group
countChan chan increment
count Summary
}
func NewUploader(db *db.SQL, store store.BlobStore, workers int, skipExistsCheck, deleteBlobsAfterUpload bool) *Uploader {
func NewUploader(db *db.SQL, store *store.DBBackedStore, workers int, skipExistsCheck, deleteBlobsAfterUpload bool) *Uploader {
return &Uploader{
db: db,
store: store,
@@ -131,7 +133,7 @@ Upload:
return nil
}
// worker reads paths from a channel, uploads them, and optionally deletes them
// worker reads paths from a channel, uploads them, and optionally deletes them
func (u *Uploader) worker(pathChan chan string) {
for {
select {
@@ -163,7 +165,7 @@ func (u *Uploader) uploadBlob(filepath string) (err error) {
}
}()
blob, err := os.ReadFile(filepath)
blob, err := ioutil.ReadFile(filepath)
if err != nil {
return errors.Err(err)
}
@@ -175,7 +177,7 @@ func (u *Uploader) uploadBlob(filepath string) (err error) {
if IsValidJSON(blob) {
log.Debugf("uploading SD blob %s", hash)
err = u.store.PutSD(hash, blob)
err := u.store.PutSD(hash, blob)
if err != nil {
return errors.Prefix("uploading SD blob "+hash, err)
}
-16
View File
@@ -1,16 +0,0 @@
#!/bin/bash
set -euo pipefail
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
cd "$SCRIPT_DIR/.."
if ! command -v golangci-lint &> /dev/null; then
echo "Installing golangci-lint..."
go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
else
echo "Ensuring golangci-lint is up to date..."
go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
fi
echo "Running golangci-lint..."
golangci-lint run --config .golangci.yml ./...
-15
View File
@@ -2,19 +2,16 @@ package http
import (
"net/http"
"strconv"
"sync"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/gin-gonic/gin"
log "github.com/sirupsen/logrus"
)
func (s *Server) getBlob(c *gin.Context) {
@@ -25,20 +22,9 @@ func (s *Server) getBlob(c *gin.Context) {
}
func (s *Server) HandleGetBlob(c *gin.Context) {
defer func() {
if r := recover(); r != nil {
log.Errorf("Recovered from panic: %v", r)
}
}()
start := time.Now()
hash := c.Query("hash")
edgeToken := c.Query("edge_token")
if reflector.IsProtected(hash) && edgeToken != s.edgeToken {
_ = c.Error(errors.Err("requested blob is protected"))
c.String(http.StatusForbidden, "requested blob is protected")
return
}
if s.missesCache.Has(hash) {
serialized, err := shared.NewBlobTrace(time.Since(start), "http").Serialize()
c.Header("Via", serialized)
@@ -80,7 +66,6 @@ func (s *Server) HandleGetBlob(c *gin.Context) {
metrics.HttpDownloadCount.Inc()
c.Header("Via", serialized)
c.Header("Content-Disposition", "filename="+hash)
c.Header("Content-Length", strconv.Itoa(len(blob)))
c.Data(http.StatusOK, "application/octet-stream", blob)
}
+8 -13
View File
@@ -19,21 +19,17 @@ import (
type Server struct {
store store.BlobStore
grp *stop.Group
missesCache gcache.Cache
edgeToken string
address string
concurrentRequests int
missesCache gcache.Cache
}
// NewServer returns an initialized Server pointer.
func NewServer(store store.BlobStore, requestQueueSize int, edgeToken string, address string) *Server {
func NewServer(store store.BlobStore, requestQueueSize int) *Server {
return &Server{
store: store,
grp: stop.New(),
concurrentRequests: requestQueueSize,
missesCache: gcache.New(2000).Expiration(5 * time.Minute).ARC().Build(),
edgeToken: edgeToken,
address: address,
}
}
@@ -45,16 +41,15 @@ func (s *Server) Shutdown() {
}
// Start starts the server listener to handle connections.
func (s *Server) Start() error {
func (s *Server) Start(address string) error {
gin.SetMode(gin.ReleaseMode)
router := gin.New()
router.Use(gin.Logger())
// Install nice.Recovery, passing the handler to call after recovery
router.Use(nice.Recovery(s.recoveryHandler))
router := gin.Default()
router.GET("/blob", s.getBlob)
router.HEAD("/blob", s.hasBlob)
// Install nice.Recovery, passing the handler to call after recovery
router.Use(nice.Recovery(s.recoveryHandler))
srv := &http.Server{
Addr: s.address,
Addr: address,
Handler: router,
}
go s.listenForShutdown(srv)
@@ -64,7 +59,7 @@ func (s *Server) Start() error {
s.grp.Add(1)
go func() {
defer s.grp.Done()
log.Println("HTTP server listening on " + s.address)
log.Println("HTTP server listening on " + address)
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
log.Fatalf("listen: %s\n", err)
}
+7 -8
View File
@@ -16,16 +16,16 @@ import (
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/quic-go/quic-go/http3"
"github.com/lucas-clemente/quic-go/http3"
log "github.com/sirupsen/logrus"
)
// Client is an instance of a client connected to a server.
type Client struct {
conn *http.Client
roundTripper *http3.Transport
ServerAddr string
Timeout time.Duration
conn *http.Client
roundTripper *http3.RoundTripper
ServerAddr string
}
// Close closes the connection with the client.
@@ -69,7 +69,7 @@ func (c *Client) HasBlob(hash string) (bool, error) {
if err != nil {
return false, errors.Err(err)
}
defer func() { _ = resp.Body.Close() }()
defer resp.Body.Close()
if resp.StatusCode == http.StatusOK {
return true, nil
}
@@ -86,7 +86,7 @@ func (c *Client) GetBlob(hash string) (stream.Blob, shared.BlobTrace, error) {
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err(err)
}
defer func() { _ = resp.Body.Close() }()
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound {
fmt.Printf("%s blob not found %d\n", hash, resp.StatusCode)
@@ -100,8 +100,7 @@ func (c *Client) GetBlob(hash string) (stream.Blob, shared.BlobTrace, error) {
serialized := resp.Header.Get("Via")
trace := shared.NewBlobTrace(time.Since(start), "http3")
if serialized != "" {
var parsedTrace *shared.BlobTrace
parsedTrace, err = shared.Deserialize(serialized)
parsedTrace, err := shared.Deserialize(serialized)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "http3"), err
}
+14 -20
View File
@@ -11,20 +11,18 @@ import (
"math/big"
"net/http"
"strconv"
"strings"
"sync"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/gorilla/mux"
"github.com/quic-go/quic-go"
"github.com/quic-go/quic-go/http3"
"github.com/lucas-clemente/quic-go"
"github.com/lucas-clemente/quic-go/http3"
log "github.com/sirupsen/logrus"
)
@@ -32,17 +30,15 @@ import (
type Server struct {
store store.BlobStore
grp *stop.Group
address string
concurrentRequests int
}
// NewServer returns an initialized Server pointer.
func NewServer(store store.BlobStore, requestQueueSize int, address string) *Server {
func NewServer(store store.BlobStore, requestQueueSize int) *Server {
return &Server{
store: store,
grp: stop.New(),
concurrentRequests: requestQueueSize,
address: address,
}
}
@@ -69,8 +65,8 @@ type availabilityResponse struct {
}
// Start starts the server listener to handle connections.
func (s *Server) Start() error {
log.Println("HTTP3 peer listening on " + s.address)
func (s *Server) Start(address string) error {
log.Println("HTTP3 peer listening on " + address)
window500M := 500 * 1 << 20
quicConf := &quic.Config{
@@ -116,10 +112,12 @@ func (s *Server) Start() error {
}
})
server := http3.Server{
Addr: s.address,
Handler: r,
TLSConfig: generateTLSConfig(),
QUICConfig: quicConf,
Server: &http.Server{
Handler: r,
Addr: address,
TLSConfig: generateTLSConfig(),
},
QuicConfig: quicConf,
}
go InitWorkers(s, s.concurrentRequests)
go s.listenForShutdown(&server)
@@ -158,7 +156,7 @@ func generateTLSConfig() *tls.Config {
func (s *Server) listenAndServe(server *http3.Server) {
err := server.ListenAndServe()
if err != nil && !strings.Contains(err.Error(), "Server closed") {
if err != nil && err.Error() != "server closed" {
log.Errorln(errors.FullTrace(err))
}
}
@@ -183,15 +181,11 @@ func (s *Server) HandleGetBlob(w http.ResponseWriter, r *http.Request) {
wantsTrace = false
}
}
if reflector.IsProtected(requestedBlob) {
http.Error(w, "requested blob is protected", http.StatusForbidden)
return
}
blob, trace, err := s.store.Get(requestedBlob)
if wantsTrace {
var serialized string
serialized, err = trace.Serialize()
serialized, err := trace.Serialize()
if err != nil {
http.Error(w, err.Error(), http.StatusNotFound)
return
+117
View File
@@ -0,0 +1,117 @@
package http3
import (
"crypto/tls"
"crypto/x509"
"net/http"
"strings"
"sync"
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/lucas-clemente/quic-go"
"github.com/lucas-clemente/quic-go/http3"
)
// Store is a blob store that gets blobs from a peer.
// It satisfies the store.BlobStore interface but cannot put or delete blobs.
type Store struct {
opts StoreOpts
NotFoundCache *sync.Map
}
// StoreOpts allows to set options for a new Store.
type StoreOpts struct {
Address string
Timeout time.Duration
}
// NewStore makes a new peer store.
func NewStore(opts StoreOpts) *Store {
return &Store{opts: opts, NotFoundCache: &sync.Map{}}
}
func (p *Store) getClient() (*Client, error) {
var qconf quic.Config
window500M := 500 * 1 << 20
qconf.MaxStreamReceiveWindow = uint64(window500M)
qconf.MaxConnectionReceiveWindow = uint64(window500M)
qconf.EnableDatagrams = true
qconf.HandshakeIdleTimeout = 4 * time.Second
qconf.MaxIdleTimeout = 20 * time.Second
pool, err := x509.SystemCertPool()
if err != nil {
return nil, err
}
roundTripper := &http3.RoundTripper{
TLSClientConfig: &tls.Config{
RootCAs: pool,
InsecureSkipVerify: true,
},
QuicConfig: &qconf,
}
connection := &http.Client{
Transport: roundTripper,
}
c := &Client{
conn: connection,
roundTripper: roundTripper,
ServerAddr: p.opts.Address,
}
return c, errors.Prefix("connection error", err)
}
func (p *Store) Name() string { return "http3" }
// Has asks the peer if they have a hash
func (p *Store) Has(hash string) (bool, error) {
c, err := p.getClient()
if err != nil {
return false, err
}
defer c.Close()
return c.HasBlob(hash)
}
// Get downloads the blob from the peer
func (p *Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
if lastChecked, ok := p.NotFoundCache.Load(hash); ok {
if lastChecked.(time.Time).After(time.Now().Add(-5 * time.Minute)) {
return nil, shared.NewBlobTrace(time.Since(start), p.Name()+"-notfoundcache"), store.ErrBlobNotFound
}
}
c, err := p.getClient()
if err != nil && strings.Contains(err.Error(), "blob not found") {
p.NotFoundCache.Store(hash, time.Now())
}
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), p.Name()), err
}
defer c.Close()
return c.GetBlob(hash)
}
// Put is not supported
func (p *Store) Put(hash string, blob stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// PutSD is not supported
func (p *Store) PutSD(hash string, blob stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// Delete is not supported
func (p *Store) Delete(hash string) error {
return errors.Err(shared.ErrNotImplemented)
}
// Shutdown is not supported
func (p *Store) Shutdown() {
}
+1 -1
View File
@@ -20,9 +20,9 @@ import (
// Client is an instance of a client connected to a server.
type Client struct {
Timeout time.Duration
conn net.Conn
buf *bufio.Reader
Timeout time.Duration
connected bool
}
+46 -64
View File
@@ -31,18 +31,19 @@ const (
// Server is an instance of a peer server that houses the listener and store.
type Server struct {
store store.BlobStore
grp *stop.Group
address string
closed bool
store store.BlobStore
disk *store.DiskStore
closed bool
grp *stop.Group
}
// NewServer returns an initialized Server pointer.
func NewServer(store store.BlobStore, address string) *Server {
func NewServer(blobstore store.BlobStore) *Server {
return &Server{
store: store,
grp: stop.New(),
address: address,
store: blobstore,
disk: store.NewDiskStore("/external/blobs", 2),
grp: stop.New(),
}
}
@@ -54,9 +55,9 @@ func (s *Server) Shutdown() {
}
// Start starts the server listener to handle connections.
func (s *Server) Start() error {
log.Println("peer listening on " + s.address)
l, err := net.Listen("tcp4", s.address)
func (s *Server) Start(address string) error {
log.Println("peer listening on " + address)
l, err := net.Listen("tcp4", address)
if err != nil {
return err
}
@@ -132,7 +133,7 @@ func (s *Server) handleConnection(conn net.Conn) {
log.Error(errors.FullTrace(err))
}
response, err = s.handleCompositeRequest(request)
response, blobHash, err := s.handleCompositeRequest(request)
if err != nil {
log.Error(errors.FullTrace(err))
return
@@ -144,6 +145,7 @@ func (s *Server) handleConnection(conn net.Conn) {
}
n, err := conn.Write(response)
s.disk.Sendfile(blobHash, conn)
if err != nil {
if !strings.Contains(err.Error(), "connection reset by peer") { // means the other side closed the connection using TCP reset
s.logError(err)
@@ -222,79 +224,70 @@ func (s *Server) handleAvailabilityRequest(data []byte) ([]byte, error) {
// return append(response, blob...), nil
//}
func (s *Server) handleCompositeRequest(data []byte) ([]byte, error) {
func (s *Server) handleCompositeRequest(data []byte) ([]byte, string, error) {
var request compositeRequest
err := json.Unmarshal(data, &request)
if err != nil {
var je *json.SyntaxError
if ee.As(err, &je) {
return nil, errors.Err("invalid json request: offset %d in data %s", je.Offset, hex.EncodeToString(data))
return nil, "", errors.Err("invalid json at offset %d in data %s", je.Offset, hex.EncodeToString(data))
}
return nil, errors.Err(err)
return nil, "", errors.Err(err)
}
response := compositeResponse{
LbrycrdAddress: LbrycrdAddress,
AvailableBlobs: []string{},
}
if len(request.RequestedBlobs) > 0 {
var availableBlobs []string
for _, blobHash := range request.RequestedBlobs {
if reflector.IsProtected(blobHash) {
return nil, errors.Err("requested blob is protected")
}
var exists bool
exists, err = s.store.Has(blobHash)
exists, err := s.store.Has(blobHash)
if err != nil {
return nil, err
return nil, "", err
}
if exists {
response.AvailableBlobs = append(response.AvailableBlobs, blobHash)
availableBlobs = append(availableBlobs, blobHash)
}
}
response.AvailableBlobs = availableBlobs
}
if request.BlobDataPaymentRate != nil {
response.BlobDataPaymentRate = paymentRateAccepted
if *request.BlobDataPaymentRate < 0 {
response.BlobDataPaymentRate = paymentRateTooLow
}
response.BlobDataPaymentRate = paymentRateAccepted
if request.BlobDataPaymentRate < 0 {
response.BlobDataPaymentRate = paymentRateTooLow
}
var blob []byte
var trace shared.BlobTrace
if request.RequestedBlob != "" {
if len(request.RequestedBlob) != stream.BlobHashHexLength {
return nil, errors.Err("Invalid blob hash length")
return nil, "", errors.Err("Invalid blob hash length")
}
log.Debugln("Sending blob " + request.RequestedBlob[:8])
blob, trace, err = s.store.Get(request.RequestedBlob)
log.Debug(trace.String())
if errors.Is(err, store.ErrBlobNotFound) {
response.IncomingBlob = &incomingBlob{
size, err := s.disk.Size(request.RequestedBlob)
if err != nil {
response.IncomingBlob = incomingBlob{
Error: err.Error(),
}
} else if err != nil {
return nil, err
} else {
response.IncomingBlob = &incomingBlob{
response.IncomingBlob = incomingBlob{
BlobHash: request.RequestedBlob,
Length: len(blob),
Length: int(size),
}
metrics.MtrOutBytesTcp.Add(float64(len(blob)))
metrics.BlobDownloadCount.Inc()
metrics.PeerDownloadCount.Inc()
}
metrics.MtrOutBytesTcp.Add(float64(len(blob)))
metrics.BlobDownloadCount.Inc()
metrics.PeerDownloadCount.Inc()
}
respData, err := json.Marshal(response)
if err != nil {
return nil, err
return nil, "", err
}
return append(respData, blob...), nil
return respData, request.RequestedBlob, nil
}
func (s *Server) logError(e error) {
@@ -308,15 +301,7 @@ func (s *Server) logError(e error) {
}
func readNextMessage(buf *bufio.Reader) ([]byte, error) {
firstByte, err := buf.ReadByte()
if err != nil {
return nil, err
}
if firstByte != '{' {
// every request starts with '{'. Checking here disconnects earlier, so we don't wait until timeout
return nil, errInvalidData
}
msg := []byte("{")
msg := make([]byte, 0)
eof := false
for {
@@ -337,8 +322,6 @@ func readNextMessage(buf *bufio.Reader) ([]byte, error) {
if len(msg) > maxRequestSize {
return msg, errRequestTooLarge
} else if len(msg) > 0 && msg[0] != '{' {
return msg, errInvalidData
}
// yes, this is how the peer protocol knows when the request finishes
@@ -373,11 +356,10 @@ const (
)
var errRequestTooLarge = errors.Base("request is too large")
var errInvalidData = errors.Base("Invalid data")
type availabilityRequest struct {
RequestedBlobs []string `json:"requested_blobs"`
LbrycrdAddress bool `json:"lbrycrd_address"`
RequestedBlobs []string `json:"requested_blobs"`
}
type availabilityResponse struct {
@@ -403,20 +385,20 @@ type incomingBlob struct {
Length int `json:"length"`
}
type blobResponse struct {
RequestTrace *shared.BlobTrace
IncomingBlob incomingBlob `json:"incoming_blob"`
RequestTrace *shared.BlobTrace
}
type compositeRequest struct {
BlobDataPaymentRate *float64 `json:"blob_data_payment_rate"`
RequestedBlob string `json:"requested_blob"`
RequestedBlobs []string `json:"requested_blobs"`
LbrycrdAddress bool `json:"lbrycrd_address"`
RequestedBlobs []string `json:"requested_blobs"`
BlobDataPaymentRate float64 `json:"blob_data_payment_rate"`
RequestedBlob string `json:"requested_blob"`
}
type compositeResponse struct {
IncomingBlob *incomingBlob `json:"incoming_blob,omitempty"`
LbrycrdAddress string `json:"lbrycrd_address,omitempty"`
BlobDataPaymentRate string `json:"blob_data_payment_rate,omitempty"`
AvailableBlobs []string `json:"available_blobs"`
LbrycrdAddress string `json:"lbrycrd_address,omitempty"`
AvailableBlobs []string `json:"available_blobs,omitempty"`
BlobDataPaymentRate string `json:"blob_data_payment_rate,omitempty"`
IncomingBlob incomingBlob `json:"incoming_blob,omitempty"`
}
+2 -66
View File
@@ -2,10 +2,7 @@ package peer
import (
"bytes"
"io"
"net"
"testing"
"time"
"github.com/lbryio/reflector.go/store"
)
@@ -37,9 +34,7 @@ var availabilityRequests = []pair{
}
func getServer(t *testing.T, withBlobs bool) *Server {
st := store.NewMemStore(store.MemParams{
Name: "test",
})
st := store.NewMemStore()
if withBlobs {
for k, v := range blobs {
err := st.Put(k, v)
@@ -48,7 +43,7 @@ func getServer(t *testing.T, withBlobs bool) *Server {
}
}
}
return NewServer(st, "127.0.0.1:50505")
return NewServer(st)
}
func TestAvailabilityRequest_NoBlobs(t *testing.T) {
@@ -80,62 +75,3 @@ func TestAvailabilityRequest_WithBlobs(t *testing.T) {
}
}
}
func TestRequestFromConnection(t *testing.T) {
s := getServer(t, true)
err := s.Start()
defer s.Shutdown()
if err != nil {
t.Error("error starting server", err)
}
for _, p := range availabilityRequests {
conn, err := net.Dial("tcp", "127.0.0.1:50505")
if err != nil {
t.Error("error opening connection", err)
}
defer func() { _ = conn.Close() }()
response := make([]byte, 8192)
_, err = conn.Write(p.request)
if err != nil {
t.Error("error writing", err)
}
_, err = conn.Read(response)
if err != nil {
t.Error("error reading", err)
}
if !bytes.Equal(response[:len(p.response)], p.response) {
t.Errorf("Response did not match expected response.\nExpected: %s\nGot: %s", string(p.response), string(response))
}
}
}
func TestInvalidData(t *testing.T) {
s := getServer(t, true)
err := s.Start()
defer s.Shutdown()
if err != nil {
t.Error("error starting server", err)
}
conn, err := net.Dial("tcp", "127.0.0.1:50505")
if err != nil {
t.Error("error opening connection", err)
}
defer func() { _ = conn.Close() }()
response := make([]byte, 8192)
_, err = conn.Write([]byte("hello dear server, I would like blobs. Please"))
if err != nil {
t.Error("error writing", err)
}
err = conn.SetReadDeadline(time.Now().Add(5 * time.Second))
if err != nil {
t.Error("error setting read deadline", err)
}
_, err = conn.Read(response)
if err != io.EOF {
t.Error("error reading", err)
}
println(response)
}
+82
View File
@@ -0,0 +1,82 @@
package peer
import (
"strings"
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
)
// Store is a blob store that gets blobs from a peer.
// It satisfies the store.BlobStore interface but cannot put or delete blobs.
type Store struct {
opts StoreOpts
}
// StoreOpts allows to set options for a new Store.
type StoreOpts struct {
Address string
Timeout time.Duration
}
// NewStore makes a new peer store.
func NewStore(opts StoreOpts) *Store {
return &Store{opts: opts}
}
func (p *Store) getClient() (*Client, error) {
c := &Client{Timeout: p.opts.Timeout}
err := c.Connect(p.opts.Address)
return c, errors.Prefix("connection error", err)
}
func (p *Store) Name() string { return "peer" }
// Has asks the peer if they have a hash
func (p *Store) Has(hash string) (bool, error) {
c, err := p.getClient()
if err != nil {
return false, err
}
defer c.Close()
return c.HasBlob(hash)
}
// Get downloads the blob from the peer
func (p *Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
c, err := p.getClient()
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), p.Name()), err
}
defer c.Close()
blob, trace, err := c.GetBlob(hash)
if err != nil && strings.Contains(err.Error(), "blob not found") {
return nil, trace, store.ErrBlobNotFound
}
return blob, trace, err
}
// Put is not supported
func (p *Store) Put(hash string, blob stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// PutSD is not supported
func (p *Store) PutSD(hash string, blob stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// Delete is not supported
func (p *Store) Delete(hash string) error {
return errors.Err(shared.ErrNotImplemented)
}
// Shutdown is not supported
func (p *Store) Shutdown() {
}
-14
View File
@@ -1,14 +0,0 @@
package server
// BlobServer defines the common interface for all blob server implementations
type BlobServer interface {
Start() error
Shutdown()
}
type BlobServerConfig struct {
Address string `mapstructure:"address"`
EdgeToken string `mapstructure:"edge_token"`
Port int `mapstructure:"port"`
MaxConcurrentRequests int `mapstructure:"max_concurrent_requests"`
}
+1 -1
View File
@@ -2,5 +2,5 @@ package shared
import "github.com/lbryio/lbry.go/v2/extras/errors"
// ErrNotImplemented is a standard error when a store that implements the store interface does not implement a method
//ErrNotImplemented is a standard error when a store that implements the store interface does not implement a method
var ErrNotImplemented = errors.Base("this store does not implement this method")
+1 -1
View File
@@ -10,9 +10,9 @@ import (
)
type BlobStack struct {
Timing time.Duration `json:"timing"`
OriginName string `json:"origin_name"`
HostName string `json:"host_name"`
Timing time.Duration `json:"timing"`
}
type BlobTrace struct {
Stacks []BlobStack `json:"stacks"`
+3 -11
View File
@@ -16,17 +16,9 @@ func TestBlobTrace_Serialize(t *testing.T) {
stack.Stack(30*time.Second, "test3")
serialized, err := stack.Serialize()
assert.NoError(t, err)
deserialized, err := Deserialize(serialized)
assert.NoError(t, err)
assert.Len(t, deserialized.Stacks, 3)
assert.Equal(t, 10*time.Second, deserialized.Stacks[0].Timing)
assert.Equal(t, "test", deserialized.Stacks[0].OriginName)
assert.Equal(t, "test_machine", deserialized.Stacks[0].HostName)
assert.Equal(t, 20*time.Second, deserialized.Stacks[1].Timing)
assert.Equal(t, "test2", deserialized.Stacks[1].OriginName)
assert.Equal(t, 30*time.Second, deserialized.Stacks[2].Timing)
assert.Equal(t, "test3", deserialized.Stacks[2].OriginName)
t.Log(serialized)
expected := "{\"stacks\":[{\"timing\":10000000000,\"origin_name\":\"test\",\"host_name\":\"test_machine\"},{\"timing\":20000000000,\"origin_name\":\"test2\",\"host_name\":\"test_machine\"},{\"timing\":30000000000,\"origin_name\":\"test3\",\"host_name\":\"test_machine\"}]}"
assert.Equal(t, expected, serialized)
}
func TestBlobTrace_Deserialize(t *testing.T) {
+1 -2
View File
@@ -1,4 +1,3 @@
//go:build linux
// +build linux
package store
@@ -10,7 +9,7 @@ import (
)
func timespecToTime(ts syscall.Timespec) time.Time {
return time.Unix(ts.Sec, ts.Nsec)
return time.Unix(int64(ts.Sec), int64(ts.Nsec))
}
func atime(fi os.FileInfo) time.Time {
-1
View File
@@ -1,4 +1,3 @@
//go:build !linux
// +build !linux
package store
+8 -67
View File
@@ -1,7 +1,6 @@
package store
import (
"strings"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
@@ -11,7 +10,6 @@ import (
"github.com/lbryio/lbry.go/v2/stream"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
)
// CachingStore combines two stores, typically a local and a remote store, to improve performance.
@@ -19,77 +17,20 @@ import (
// are retrieved from the origin and cached. Puts are cached and also forwarded to the origin.
type CachingStore struct {
origin, cache BlobStore
name string
}
type CachingParams struct {
Origin BlobStore `mapstructure:"origin"`
Cache BlobStore `mapstructure:"cache"`
Name string `mapstructure:"name"`
}
type CachingConfig struct {
Origin *viper.Viper
Cache *viper.Viper
Name string `mapstructure:"name"`
component string
}
// NewCachingStore makes a new caching disk store and returns a pointer to it.
func NewCachingStore(params CachingParams) *CachingStore {
func NewCachingStore(component string, origin, cache BlobStore) *CachingStore {
return &CachingStore{
name: params.Name,
origin: WithSingleFlight(params.Name, params.Origin),
cache: WithSingleFlight(params.Name, params.Cache),
component: component,
origin: WithSingleFlight(component, origin),
cache: WithSingleFlight(component, cache),
}
}
const nameCaching = "caching"
func CachingStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg CachingConfig
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
cfg.Cache = config.Sub("cache")
cfg.Origin = config.Sub("origin")
if cfg.Cache == nil || cfg.Origin == nil {
return nil, errors.Err("cache and origin missing")
}
originStoreType := strings.Split(cfg.Origin.AllKeys()[0], ".")[0]
originStoreConfig := cfg.Origin.Sub(originStoreType)
factory, ok := Factories[originStoreType]
if !ok {
return nil, errors.Err("unknown store type %s", originStoreType)
}
originStore, err := factory(originStoreConfig)
if err != nil {
return nil, errors.Err(err)
}
cacheStoreType := strings.Split(cfg.Cache.AllKeys()[0], ".")[0]
cacheStoreConfig := cfg.Cache.Sub(cacheStoreType)
factory, ok = Factories[cacheStoreType]
if !ok {
return nil, errors.Err("unknown store type %s", cacheStoreType)
}
cacheStore, err := factory(cacheStoreConfig)
if err != nil {
return nil, errors.Err(err)
}
return NewCachingStore(CachingParams{
Name: cfg.Name,
Origin: originStore,
Cache: cacheStore,
}), nil
}
func init() {
RegisterStore(nameCaching, CachingStoreFactory)
}
// Name is the cache type name
func (c *CachingStore) Name() string { return nameCaching }
@@ -108,17 +49,17 @@ func (c *CachingStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
blob, trace, err := c.cache.Get(hash)
if err == nil || !errors.Is(err, ErrBlobNotFound) {
metrics.CacheHitCount.With(metrics.CacheLabels(c.cache.Name(), c.name)).Inc()
metrics.CacheHitCount.With(metrics.CacheLabels(c.cache.Name(), c.component)).Inc()
rate := float64(len(blob)) / 1024 / 1024 / time.Since(start).Seconds()
metrics.CacheRetrievalSpeed.With(map[string]string{
metrics.LabelCacheType: c.cache.Name(),
metrics.LabelComponent: c.name,
metrics.LabelComponent: c.component,
metrics.LabelSource: "cache",
}).Set(rate)
return blob, trace.Stack(time.Since(start), c.Name()), err
}
metrics.CacheMissCount.With(metrics.CacheLabels(c.cache.Name(), c.name)).Inc()
metrics.CacheMissCount.With(metrics.CacheLabels(c.cache.Name(), c.component)).Inc()
blob, trace, err = c.origin.Get(hash)
if err != nil {
+13 -13
View File
@@ -9,14 +9,12 @@ import (
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/sirupsen/logrus"
)
func TestCachingStore_Put(t *testing.T) {
origin := NewMemStore(MemParams{Name: "test"})
cache := NewMemStore(MemParams{Name: "test"})
s := NewCachingStore(CachingParams{Name: "test", Origin: origin, Cache: cache})
origin := NewMemStore()
cache := NewMemStore()
s := NewCachingStore("test", origin, cache)
b := []byte("this is a blob of stuff")
hash := "hash"
@@ -44,9 +42,9 @@ func TestCachingStore_Put(t *testing.T) {
}
func TestCachingStore_CacheMiss(t *testing.T) {
origin := NewMemStore(MemParams{Name: "test"})
cache := NewMemStore(MemParams{Name: "test"})
s := NewCachingStore(CachingParams{Name: "test", Origin: origin, Cache: cache})
origin := NewMemStore()
cache := NewMemStore()
s := NewCachingStore("test", origin, cache)
b := []byte("this is a blob of stuff")
hash := "hash"
@@ -86,8 +84,8 @@ func TestCachingStore_CacheMiss(t *testing.T) {
func TestCachingStore_ThunderingHerd(t *testing.T) {
storeDelay := 100 * time.Millisecond
origin := NewSlowBlobStore(storeDelay)
cache := NewMemStore(MemParams{Name: "test"})
s := NewCachingStore(CachingParams{Name: "test", Origin: origin, Cache: cache})
cache := NewMemStore()
s := NewCachingStore("test", origin, cache)
b := []byte("this is a blob of stuff")
hash := "hash"
@@ -101,7 +99,7 @@ func TestCachingStore_ThunderingHerd(t *testing.T) {
getNoErr := func() {
res, _, err := s.Get(hash)
if err != nil {
logrus.Fatal(err)
t.Fatal(err)
}
if !bytes.Equal(b, res) {
t.Errorf("expected Get() to return %s, got %s", string(b), string(res))
@@ -142,7 +140,7 @@ type SlowBlobStore struct {
func NewSlowBlobStore(delay time.Duration) *SlowBlobStore {
return &SlowBlobStore{
mem: NewMemStore(MemParams{Name: "test"}),
mem: NewMemStore(),
delay: delay,
}
}
@@ -175,4 +173,6 @@ func (s *SlowBlobStore) Delete(hash string) error {
return s.mem.Delete(hash)
}
func (s *SlowBlobStore) Shutdown() {}
func (s *SlowBlobStore) Shutdown() {
return
}
+110
View File
@@ -0,0 +1,110 @@
package store
import (
"io"
"io/ioutil"
"net/http"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/meta"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
log "github.com/sirupsen/logrus"
)
// CloudFrontROStore reads from cloudfront. All writes panic.
type CloudFrontROStore struct {
endpoint string // cloudflare endpoint
}
// NewCloudFrontROStore returns an initialized CloudFrontROStore store pointer.
func NewCloudFrontROStore(endpoint string) *CloudFrontROStore {
return &CloudFrontROStore{endpoint: endpoint}
}
const nameCloudFrontRO = "cloudfront_ro"
// Name is the cache type name
func (c *CloudFrontROStore) Name() string { return nameCloudFrontRO }
// Has checks if the hash is in the store.
func (c *CloudFrontROStore) Has(hash string) (bool, error) {
status, body, err := c.cfRequest(http.MethodHead, hash)
if err != nil {
return false, err
}
defer body.Close()
switch status {
case http.StatusNotFound, http.StatusForbidden:
return false, nil
case http.StatusOK:
return true, nil
default:
return false, errors.Err("unexpected status %d", status)
}
}
// Get gets the blob from Cloudfront.
func (c *CloudFrontROStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
log.Debugf("Getting %s from S3", hash[:8])
start := time.Now()
defer func(t time.Time) {
log.Debugf("Getting %s from S3 took %s", hash[:8], time.Since(t).String())
}(start)
status, body, err := c.cfRequest(http.MethodGet, hash)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), c.Name()), err
}
defer body.Close()
switch status {
case http.StatusNotFound, http.StatusForbidden:
return nil, shared.NewBlobTrace(time.Since(start), c.Name()), errors.Err(ErrBlobNotFound)
case http.StatusOK:
b, err := ioutil.ReadAll(body)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), c.Name()), errors.Err(err)
}
metrics.MtrInBytesS3.Add(float64(len(b)))
return b, shared.NewBlobTrace(time.Since(start), c.Name()), nil
default:
return nil, shared.NewBlobTrace(time.Since(start), c.Name()), errors.Err("unexpected status %d", status)
}
}
func (c *CloudFrontROStore) cfRequest(method, hash string) (int, io.ReadCloser, error) {
url := c.endpoint + hash
req, err := http.NewRequest(method, url, nil)
if err != nil {
return 0, nil, errors.Err(err)
}
req.Header.Add("User-Agent", "reflector.go/"+meta.Version)
res, err := http.DefaultClient.Do(req)
if err != nil {
return 0, nil, errors.Err(err)
}
return res.StatusCode, res.Body, nil
}
func (c *CloudFrontROStore) Put(_ string, _ stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
func (c *CloudFrontROStore) PutSD(_ string, _ stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
func (c *CloudFrontROStore) Delete(_ string) error {
return errors.Err(shared.ErrNotImplemented)
}
// Shutdown shuts down the store gracefully
func (c *CloudFrontROStore) Shutdown() {
}
+62
View File
@@ -0,0 +1,62 @@
package store
import (
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/stream"
)
// CloudFrontRWStore combines a Cloudfront and an S3 store. Reads go to Cloudfront/Wasabi, writes go to S3.
type CloudFrontRWStore struct {
cf *ITTTStore
s3 *S3Store
}
// NewCloudFrontRWStore returns an initialized CloudFrontRWStore store pointer.
// NOTE: It panics if either argument is nil.
func NewCloudFrontRWStore(cf *ITTTStore, s3 *S3Store) *CloudFrontRWStore {
if cf == nil || s3 == nil {
panic("both stores must be set")
}
return &CloudFrontRWStore{cf: cf, s3: s3}
}
const nameCloudFrontRW = "cloudfront_rw"
// Name is the cache type name
func (c *CloudFrontRWStore) Name() string { return nameCloudFrontRW }
// Has checks if the hash is in the store.
func (c *CloudFrontRWStore) Has(hash string) (bool, error) {
return c.cf.Has(hash)
}
// Get gets the blob from Cloudfront.
func (c *CloudFrontRWStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
blob, trace, err := c.cf.Get(hash)
return blob, trace.Stack(time.Since(start), c.Name()), err
}
// Put stores the blob on S3
func (c *CloudFrontRWStore) Put(hash string, blob stream.Blob) error {
return c.s3.Put(hash, blob)
}
// PutSD stores the sd blob on S3
func (c *CloudFrontRWStore) PutSD(hash string, blob stream.Blob) error {
return c.s3.PutSD(hash, blob)
}
// Delete deletes the blob from S3
func (c *CloudFrontRWStore) Delete(hash string) error {
return c.s3.Delete(hash)
}
// Shutdown shuts down the store gracefully
func (c *CloudFrontRWStore) Shutdown() {
c.s3.Shutdown()
c.cf.Shutdown()
}
+24 -176
View File
@@ -2,8 +2,6 @@ package store
import (
"encoding/json"
"fmt"
"strings"
"sync"
"time"
@@ -11,70 +9,29 @@ import (
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/c2h5oh/datasize"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
)
// DBBackedStore is a store that's backed by a DB. The DB contains data about what's in the store.
type DBBackedStore struct {
blobs BlobStore
db *db.SQL
blocked map[string]bool
cleanerStop *stop.Group
name string
maxSize int
blockedMu sync.RWMutex
blocked map[string]bool
deleteOnMiss bool
}
type DBBackedParams struct {
Store BlobStore `mapstructure:"store"`
DB *db.SQL `mapstructure:"db"`
MaxSize *int `mapstructure:"max_size"`
Name string `mapstructure:"name"`
DeleteOnMiss bool `mapstructure:"delete_on_miss"`
}
type DBBackedConfig struct {
Store *viper.Viper
Database string `mapstructure:"database"`
User string `mapstructure:"user"`
Password string `mapstructure:"password"`
Host string `mapstructure:"host"`
Name string `mapstructure:"name"`
MaxSize string `mapstructure:"max_size"`
Port int `mapstructure:"port"`
AccessTracking int `mapstructure:"access_tracking"`
DeleteOnMiss bool `mapstructure:"delete_on_miss"`
SoftDeletes bool `mapstructure:"soft_deletes"`
LogQueries bool `mapstructure:"log_queries"`
HasCap bool `mapstructure:"has_cap"`
}
// NewDBBackedStore returns an initialized store pointer.
func NewDBBackedStore(params DBBackedParams) *DBBackedStore {
store := &DBBackedStore{
blobs: params.Store,
db: params.DB,
deleteOnMiss: params.DeleteOnMiss,
cleanerStop: stop.New(),
name: params.Name,
}
if params.MaxSize != nil {
store.maxSize = *params.MaxSize
go store.cleanOldestBlobs()
}
return store
func NewDBBackedStore(blobs BlobStore, db *db.SQL, deleteOnMiss bool) *DBBackedStore {
return &DBBackedStore{blobs: blobs, db: db, deleteOnMiss: deleteOnMiss}
}
const nameDBBacked = "db_backed"
const nameDBBacked = "db-backed"
// Name is the cache type name
func (d *DBBackedStore) Name() string { return nameDBBacked + "-" + d.name }
func (d *DBBackedStore) Name() string { return nameDBBacked }
// Has returns true if the blob is in the store
func (d *DBBackedStore) Has(hash string) (bool, error) {
@@ -144,18 +101,34 @@ func (d *DBBackedStore) Delete(hash string) error {
// Block deletes the blob and prevents it from being uploaded in the future
func (d *DBBackedStore) Block(hash string) error {
blocked, err := d.isBlocked(hash)
if blocked || err != nil {
if blocked, err := d.isBlocked(hash); blocked || err != nil {
return err
}
log.Debugf("blocking %s", hash)
err = d.db.Block(hash)
err := d.db.Block(hash)
if err != nil {
return err
}
has, err := d.db.HasBlob(hash, false)
if err != nil {
return err
}
if has {
err = d.blobs.Delete(hash)
if err != nil {
return err
}
err = d.db.Delete(hash)
if err != nil {
return err
}
}
return d.markBlocked(hash)
}
@@ -222,132 +195,7 @@ func (d *DBBackedStore) initBlocked() error {
return err
}
// cleanOldestBlobs periodically cleans up the oldest blobs if maxSize is set
func (d *DBBackedStore) cleanOldestBlobs() {
// Run on startup without waiting for 10 minutes
err := d.doClean()
if err != nil {
log.Error(errors.FullTrace(err))
}
const cleanupInterval = 10 * time.Minute
for {
select {
case <-d.cleanerStop.Ch():
log.Infoln("stopping self cleanup")
return
case <-time.After(cleanupInterval):
err := d.doClean()
if err != nil {
log.Error(errors.FullTrace(err))
}
}
}
}
// doClean removes the least recently accessed blobs if the store exceeds maxItems
func (d *DBBackedStore) doClean() error {
blobsCount, err := d.db.Count()
if err != nil {
return err
}
if blobsCount >= d.maxSize {
itemsToDelete := blobsCount / 10
blobs, err := d.db.LeastRecentlyAccessedHashes(itemsToDelete)
if err != nil {
return err
}
blobsChan := make(chan string, len(blobs))
wg := stop.New()
go func() {
for _, hash := range blobs {
select {
case <-d.cleanerStop.Ch():
return
default:
}
blobsChan <- hash
}
close(blobsChan)
}()
for i := 0; i < 3; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for h := range blobsChan {
select {
case <-d.cleanerStop.Ch():
return
default:
}
err = d.Delete(h)
if err != nil {
log.Errorf("error pruning %s: %s", h, errors.FullTrace(err))
continue
}
}
}()
}
wg.Wait()
}
return nil
}
// Shutdown shuts down the store gracefully
func (d *DBBackedStore) Shutdown() {
d.cleanerStop.Stop()
d.blobs.Shutdown()
}
func DBBackedStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg DBBackedConfig
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
cfg.Store = config.Sub("store")
storeType := strings.Split(cfg.Store.AllKeys()[0], ".")[0]
storeConfig := cfg.Store.Sub(storeType)
factory, ok := Factories[storeType]
if !ok {
return nil, errors.Err("unknown store type %s", storeType)
}
underlyingStore, err := factory(storeConfig)
if err != nil {
return nil, errors.Err(err)
}
parsedDb := &db.SQL{
TrackingLevel: db.AccessTrackingLevel(cfg.AccessTracking),
SoftDelete: cfg.SoftDeletes,
LogQueries: cfg.LogQueries || log.GetLevel() == log.DebugLevel,
}
err = parsedDb.Connect(fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", cfg.User, cfg.Password, cfg.Host, cfg.Port, cfg.Database))
if err != nil {
return nil, err
}
params := DBBackedParams{
Name: cfg.Name,
Store: underlyingStore,
DB: parsedDb,
DeleteOnMiss: cfg.DeleteOnMiss,
}
if cfg.HasCap {
var parsedSize datasize.ByteSize
err = parsedSize.UnmarshalText([]byte(cfg.MaxSize))
if err != nil {
return nil, errors.Err(err)
}
maxSize := int(float64(parsedSize) / float64(stream.MaxBlobSize))
params.MaxSize = &maxSize
}
return NewDBBackedStore(params), nil
}
func init() {
RegisterStore(nameDBBacked, DBBackedStoreFactory)
}
+100 -31
View File
@@ -1,6 +1,13 @@
package store
import (
"bytes"
"crypto/sha512"
"encoding/hex"
"fmt"
"io"
"io/ioutil"
"net"
"os"
"path"
"time"
@@ -11,49 +18,38 @@ import (
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
"github.com/brk0v/directio"
log "github.com/sirupsen/logrus"
"go.uber.org/atomic"
)
// DiskStore stores blobs on a local disk
type DiskStore struct {
blobDir string
name string
// the location of blobs on disk
blobDir string
// store files in subdirectories based on the first N chars in the filename. 0 = don't create subdirectories.
prefixLength int
initialized bool
// true if initOnce ran, false otherwise
initialized bool
concurrentChecks atomic.Int32
}
type DiskParams struct {
Name string `mapstructure:"name"`
MountPoint string `mapstructure:"mount_point"`
ShardingSize int `mapstructure:"sharding_size"`
}
const maxConcurrentChecks = 30
// NewDiskStore returns an initialized file disk store pointer.
func NewDiskStore(params DiskParams) *DiskStore {
func NewDiskStore(dir string, prefixLength int) *DiskStore {
return &DiskStore{
blobDir: params.MountPoint,
prefixLength: params.ShardingSize,
name: params.Name,
blobDir: dir,
prefixLength: prefixLength,
}
}
const nameDisk = "disk"
func DiskStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg DiskParams
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
return NewDiskStore(cfg), nil
}
func init() {
RegisterStore(nameDisk, DiskStoreFactory)
}
// Name is the cache type name
func (d *DiskStore) Name() string { return nameDisk + "-" + d.name }
func (d *DiskStore) Name() string { return nameDisk }
// Has returns T/F or Error if it the blob stored already. It will error with any IO disk error.
func (d *DiskStore) Has(hash string) (bool, error) {
@@ -72,6 +68,14 @@ func (d *DiskStore) Has(hash string) (bool, error) {
return true, nil
}
func (d *DiskStore) Size(hash string) (int64, error) {
stat, err := os.Stat(d.path(hash))
if err != nil {
return 0, err
}
return stat.Size(), nil
}
// Get returns the blob or an error if the blob doesn't exist.
func (d *DiskStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
@@ -80,16 +84,69 @@ func (d *DiskStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), err
}
blob, err := os.ReadFile(d.path(hash))
blob, err := ioutil.ReadFile(d.path(hash))
if err != nil {
if os.IsNotExist(err) {
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(ErrBlobNotFound)
}
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(err)
}
// this is a rather poor yet effective way of throttling how many blobs can be checked concurrently
// poor because there is a possible race condition between the check and the actual +1
if d.concurrentChecks.Load() < maxConcurrentChecks {
d.concurrentChecks.Add(1)
defer d.concurrentChecks.Sub(1)
hashBytes := sha512.Sum384(blob)
readHash := hex.EncodeToString(hashBytes[:])
if hash != readHash {
message := fmt.Sprintf("[%s] found a broken blob while reading from disk. Actual hash: %s", hash, readHash)
log.Errorf("%s", message)
err := d.Delete(hash)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), err
}
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(message)
}
}
return blob, shared.NewBlobTrace(time.Since(start), d.Name()), nil
}
// Put stores the blob on disk
func (d *DiskStore) Put(hash string, blob stream.Blob) error {
err := d.initOnce()
if err != nil {
return err
}
err = d.ensureDirExists(d.dir(hash))
if err != nil {
return err
}
// Open file with O_DIRECT
f, err := os.OpenFile(d.tmpPath(hash), openFileFlags, 0644)
if err != nil {
return errors.Err(err)
}
defer f.Close()
// Use directio writer
dio, err := directio.New(f)
if err != nil {
return errors.Err(err)
}
defer dio.Flush()
// Write the body to file
_, err = io.Copy(dio, bytes.NewReader(blob))
if err != nil {
return errors.Err(err)
}
err = os.Rename(d.tmpPath(hash), d.path(hash))
return errors.Err(err)
}
// PutSD stores the sd blob on the disk
func (d *DiskStore) PutSD(hash string, blob stream.Blob) error {
return d.Put(hash, blob)
@@ -130,14 +187,14 @@ func (d *DiskStore) dir(hash string) string {
}
return path.Join(d.blobDir, hash[:d.prefixLength])
}
func (d *DiskStore) tmpDir() string {
func (d *DiskStore) tmpDir(hash string) string {
return path.Join(d.blobDir, "tmp")
}
func (d *DiskStore) path(hash string) string {
return path.Join(d.dir(hash), hash)
}
func (d *DiskStore) tmpPath(hash string) string {
return path.Join(d.tmpDir(), hash)
return path.Join(d.tmpDir(hash), hash)
}
func (d *DiskStore) ensureDirExists(dir string) error {
return errors.Err(os.MkdirAll(dir, 0755))
@@ -152,7 +209,7 @@ func (d *DiskStore) initOnce() error {
if err != nil {
return err
}
err = d.ensureDirExists(d.tmpDir())
err = d.ensureDirExists(path.Join(d.blobDir, "tmp"))
if err != nil {
return err
}
@@ -163,3 +220,15 @@ func (d *DiskStore) initOnce() error {
// Shutdown shuts down the store gracefully
func (d *DiskStore) Shutdown() {
}
func (d *DiskStore) Sendfile(hash string, conn net.Conn) error {
f, err := os.Open(d.path(hash))
if err != nil {
return err
}
_, err = io.Copy(conn, f)
if err != nil {
return err
}
return nil
}
+8 -15
View File
@@ -1,6 +1,7 @@
package store
import (
"io/ioutil"
"os"
"path"
"path/filepath"
@@ -13,14 +14,10 @@ import (
)
func TestDiskStore_Get(t *testing.T) {
tmpDir, err := os.MkdirTemp("", "reflector_test_*")
tmpDir, err := ioutil.TempDir("", "reflector_test_*")
require.NoError(t, err)
defer func() { _ = os.RemoveAll(tmpDir) }()
d := NewDiskStore(DiskParams{
Name: "test",
MountPoint: tmpDir,
ShardingSize: 2,
})
defer os.RemoveAll(tmpDir)
d := NewDiskStore(tmpDir, 2)
hash := "f428b8265d65dad7f8ffa52922bba836404cbd62f3ecfe10adba6b444f8f658938e54f5981ac4de39644d5b93d89a94b"
data := []byte("oyuntyausntoyaunpdoyruoyduanrstjwfjyuwf")
@@ -28,7 +25,7 @@ func TestDiskStore_Get(t *testing.T) {
expectedPath := path.Join(tmpDir, hash[:2], hash)
err = os.MkdirAll(filepath.Dir(expectedPath), os.ModePerm)
require.NoError(t, err)
err = os.WriteFile(expectedPath, data, os.ModePerm)
err = ioutil.WriteFile(expectedPath, data, os.ModePerm)
require.NoError(t, err)
blob, _, err := d.Get(hash)
@@ -37,14 +34,10 @@ func TestDiskStore_Get(t *testing.T) {
}
func TestDiskStore_GetNonexistentBlob(t *testing.T) {
tmpDir, err := os.MkdirTemp("", "reflector_test_*")
tmpDir, err := ioutil.TempDir("", "reflector_test_*")
require.NoError(t, err)
defer func() { _ = os.RemoveAll(tmpDir) }()
d := NewDiskStore(DiskParams{
Name: "test",
MountPoint: tmpDir,
ShardingSize: 2,
})
defer os.RemoveAll(tmpDir)
d := NewDiskStore(tmpDir, 2)
blob, _, err := d.Get("nonexistent")
assert.Nil(t, blob)
-42
View File
@@ -1,42 +0,0 @@
//go:build darwin
// +build darwin
package store
import (
"bytes"
"io"
"os"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
)
var openFileFlags = os.O_WRONLY | os.O_CREATE
// Put stores the blob on disk
func (d *DiskStore) Put(hash string, blob stream.Blob) error {
err := d.initOnce()
if err != nil {
return err
}
err = d.ensureDirExists(d.dir(hash))
if err != nil {
return err
}
// Open file with O_DIRECT
f, err := os.OpenFile(d.tmpPath(hash), openFileFlags, 0644)
if err != nil {
return errors.Err(err)
}
defer f.Close()
_, err = io.Copy(f, bytes.NewReader(blob))
if err != nil {
return errors.Err(err)
}
err = os.Rename(d.tmpPath(hash), d.path(hash))
return errors.Err(err)
}
-49
View File
@@ -1,49 +0,0 @@
//go:build linux
// +build linux
package store
import (
"bytes"
"io"
"os"
"syscall"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/brk0v/directio"
)
var openFileFlags = os.O_WRONLY | os.O_CREATE | syscall.O_DIRECT
// Put stores the blob on disk
func (d *DiskStore) Put(hash string, blob stream.Blob) error {
err := d.initOnce()
if err != nil {
return err
}
err = d.ensureDirExists(d.dir(hash))
if err != nil {
return err
}
// Open file with O_DIRECT
f, err := os.OpenFile(d.tmpPath(hash), openFileFlags, 0644)
if err != nil {
return errors.Err(err)
}
defer func() { _ = f.Close() }()
dio, err := directio.New(f)
if err != nil {
return errors.Err(err)
}
defer func() { _ = dio.Flush() }()
_, err = io.Copy(dio, bytes.NewReader(blob))
if err != nil {
return errors.Err(err)
}
err = os.Rename(d.tmpPath(hash), d.path(hash))
return errors.Err(err)
}
-31
View File
@@ -1,31 +0,0 @@
//go:build !linux && !darwin
// +build !linux,!darwin
package store
import (
"os"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
)
// Put stores the blob on disk
func (d *DiskStore) Put(hash string, blob stream.Blob) error {
err := d.initOnce()
if err != nil {
return err
}
err = d.ensureDirExists(d.dir(hash))
if err != nil {
return err
}
err = os.WriteFile(d.tmpPath(hash), blob, 0644)
if err != nil {
return errors.Err(err)
}
err = os.Rename(d.tmpPath(hash), d.path(hash))
return errors.Err(err)
}
+9
View File
@@ -0,0 +1,9 @@
// +build darwin
package store
import (
"os"
)
var openFileFlags = os.O_WRONLY | os.O_CREATE
+10
View File
@@ -0,0 +1,10 @@
// +build linux
package store
import (
"os"
"syscall"
)
var openFileFlags = os.O_WRONLY | os.O_CREATE | syscall.O_DIRECT
+21 -73
View File
@@ -1,7 +1,6 @@
package store
import (
"strings"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
@@ -12,16 +11,15 @@ import (
"github.com/bluele/gcache"
"github.com/sirupsen/logrus"
"github.com/spf13/viper"
)
// GcacheStore adds a max cache size and Greedy-Dual-Size-Frequency cache eviction strategy to a BlobStore
type GcacheStore struct {
underlyingStore BlobStore
cache gcache.Cache
name string
// underlying store
store BlobStore
// cache implementation
cache gcache.Cache
}
type EvictionStrategy int
const (
@@ -35,30 +33,16 @@ const (
SIMPLE
)
type GcacheParams struct {
Store BlobStore `mapstructure:"store"`
Name string `mapstructure:"name"`
MaxSize int `mapstructure:"max_size"`
Strategy EvictionStrategy `mapstructure:"strategy"`
}
type GcacheConfig struct {
Store *viper.Viper
Name string `mapstructure:"name"`
MaxSize int `mapstructure:"max_size"`
Strategy EvictionStrategy `mapstructure:"strategy"`
}
// NewGcacheStore initialize a new LRUStore
func NewGcacheStore(params GcacheParams) *GcacheStore {
cacheBuilder := gcache.New(params.MaxSize)
func NewGcacheStore(component string, store BlobStore, maxSize int, strategy EvictionStrategy) *GcacheStore {
cacheBuilder := gcache.New(maxSize)
var cache gcache.Cache
evictFunc := func(key interface{}, value interface{}) {
logrus.Infof("evicting %s", key)
metrics.CacheLRUEvictCount.With(metrics.CacheLabels(params.Store.Name(), params.Name)).Inc()
_ = params.Store.Delete(key.(string)) // TODO: log this error. may happen if underlying entry is gone but cache entry still there
metrics.CacheLRUEvictCount.With(metrics.CacheLabels(store.Name(), component)).Inc()
_ = store.Delete(key.(string)) // TODO: log this error. may happen if underlying entry is gone but cache entry still there
}
switch params.Strategy {
switch strategy {
case LFU:
cache = cacheBuilder.LFU().EvictedFunc(evictFunc).Build()
case ARC:
@@ -67,15 +51,15 @@ func NewGcacheStore(params GcacheParams) *GcacheStore {
cache = cacheBuilder.LRU().EvictedFunc(evictFunc).Build()
case SIMPLE:
cache = cacheBuilder.Simple().EvictedFunc(evictFunc).Build()
}
l := &GcacheStore{
underlyingStore: params.Store,
cache: cache,
name: params.Name,
store: store,
cache: cache,
}
go func() {
if lstr, ok := params.Store.(lister); ok {
err := l.loadExisting(lstr, params.MaxSize)
if lstr, ok := store.(lister); ok {
err := l.loadExisting(lstr, int(maxSize))
if err != nil {
panic(err) // TODO: what should happen here? panic? return nil? just keep going?
}
@@ -87,40 +71,8 @@ func NewGcacheStore(params GcacheParams) *GcacheStore {
const nameGcache = "gcache"
func GcacheStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg GcacheConfig
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
cfg.Store = config.Sub("store")
storeType := strings.Split(cfg.Store.AllKeys()[0], ".")[0]
storeConfig := cfg.Store.Sub(storeType)
factory, ok := Factories[storeType]
if !ok {
return nil, errors.Err("unknown store type %s", storeType)
}
underlyingStore, err := factory(storeConfig)
if err != nil {
return nil, errors.Err(err)
}
return NewGcacheStore(GcacheParams{
Name: cfg.Name,
Store: underlyingStore,
MaxSize: cfg.MaxSize,
Strategy: cfg.Strategy,
}), nil
}
func init() {
RegisterStore(nameGcache, GcacheStoreFactory)
}
// Name is the cache type name
func (l *GcacheStore) Name() string { return nameGcache + "-" + l.name }
func (l *GcacheStore) Name() string { return nameGcache }
// Has returns whether the blob is in the store, without updating the recent-ness.
func (l *GcacheStore) Has(hash string) (bool, error) {
@@ -134,7 +86,7 @@ func (l *GcacheStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), l.Name()), errors.Err(ErrBlobNotFound)
}
blob, stack, err := l.underlyingStore.Get(hash)
blob, stack, err := l.store.Get(hash)
if errors.Is(err, ErrBlobNotFound) {
// Blob disappeared from underlying store
l.cache.Remove(hash)
@@ -147,7 +99,7 @@ func (l *GcacheStore) Put(hash string, blob stream.Blob) error {
_ = l.cache.Set(hash, true)
has, _ := l.Has(hash)
if has {
err := l.underlyingStore.Put(hash, blob)
err := l.store.Put(hash, blob)
if err != nil {
return err
}
@@ -160,7 +112,7 @@ func (l *GcacheStore) PutSD(hash string, blob stream.Blob) error {
_ = l.cache.Set(hash, true)
has, _ := l.Has(hash)
if has {
err := l.underlyingStore.PutSD(hash, blob)
err := l.store.PutSD(hash, blob)
if err != nil {
return err
}
@@ -170,7 +122,7 @@ func (l *GcacheStore) PutSD(hash string, blob stream.Blob) error {
// Delete deletes the blob from the store
func (l *GcacheStore) Delete(hash string) error {
err := l.underlyingStore.Delete(hash)
err := l.store.Delete(hash)
if err != nil {
return err
}
@@ -192,15 +144,11 @@ func (l *GcacheStore) loadExisting(store lister, maxItems int) error {
logrus.Infof("read %d files from underlying store", len(existing))
added := 0
for i, h := range existing {
for _, h := range existing {
_ = l.cache.Set(h, true)
added++
if maxItems > 0 && added >= maxItems { // underlying cache is bigger than the cache
err := l.Delete(h)
logrus.Infof("deleted overflowing blob: %s (%d/%d)", h, i, len(existing))
if err != nil {
logrus.Warnf("error while deleting a blob that's overflowing the cache: %s", err.Error())
}
break
}
}
return nil
+8 -11
View File
@@ -2,6 +2,7 @@ package store
import (
"fmt"
"io/ioutil"
"os"
"reflect"
"testing"
@@ -16,8 +17,8 @@ import (
const cacheMaxSize = 3
func getTestGcacheStore() (*GcacheStore, *MemStore) {
m := NewMemStore(MemParams{Name: "test"})
return NewGcacheStore(GcacheParams{Name: "test", Store: m, MaxSize: cacheMaxSize, Strategy: LFU}), m
m := NewMemStore()
return NewGcacheStore("test", m, cacheMaxSize, LFU), m
}
func TestGcacheStore_Eviction(t *testing.T) {
@@ -87,14 +88,10 @@ func TestGcacheStore_UnderlyingBlobMissing(t *testing.T) {
}
func TestGcacheStore_loadExisting(t *testing.T) {
tmpDir, err := os.MkdirTemp("", "reflector_test_*")
tmpDir, err := ioutil.TempDir("", "reflector_test_*")
require.NoError(t, err)
defer func() { _ = os.RemoveAll(tmpDir) }()
d := NewDiskStore(DiskParams{
Name: "test",
MountPoint: tmpDir,
ShardingSize: 2,
})
defer os.RemoveAll(tmpDir)
d := NewDiskStore(tmpDir, 2)
hash := "hash"
b := []byte("this is a blob of stuff")
@@ -106,8 +103,8 @@ func TestGcacheStore_loadExisting(t *testing.T) {
require.Equal(t, 1, len(existing), "blob should exist in cache")
assert.Equal(t, hash, existing[0])
lfu := NewGcacheStore(GcacheParams{Name: "test", Store: d, MaxSize: 3, Strategy: LFU}) // lru should load existing blobs when it's created
time.Sleep(100 * time.Millisecond) // async load so let's wait...
lfu := NewGcacheStore("test", d, 3, LFU) // lru should load existing blobs when it's created
time.Sleep(100 * time.Millisecond) // async load so let's wait...
has, err := lfu.Has(hash)
require.NoError(t, err)
assert.True(t, has, "hash should be loaded from disk store but it's not")
+101 -129
View File
@@ -1,188 +1,160 @@
package store
import (
"bytes"
"io"
"io/ioutil"
"net"
"net/http"
"path"
"strconv"
"sync"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/meta"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
)
// HttpStore reads from an HTTP endpoint that simply expects the hash to be appended to the endpoint
// HttpStore is a store that works on top of the HTTP protocol
type HttpStore struct {
httpClient *http.Client
endpoint string
name string
prefixLength int
upstream string
httpClient *http.Client
}
type HttpParams struct {
Name string `mapstructure:"name"`
Endpoint string `mapstructure:"endpoint"`
ShardingSize int `mapstructure:"sharding_size"`
}
// NewHttpStore returns an initialized HttpStore store pointer.
func NewHttpStore(params HttpParams) *HttpStore {
func NewHttpStore(upstream string) *HttpStore {
return &HttpStore{
endpoint: params.Endpoint,
httpClient: getClient(),
prefixLength: params.ShardingSize,
name: params.Name,
upstream: "http://" + upstream,
httpClient: getClient(),
}
}
const nameHttp = "http"
func HttpStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg HttpParams
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
return NewHttpStore(cfg), nil
}
func (n *HttpStore) Name() string { return nameHttp }
func (n *HttpStore) Has(hash string) (bool, error) {
url := n.upstream + "/blob?hash=" + hash
// Name is the cache type name
func (c *HttpStore) Name() string { return nameHttp + "-" + c.name }
// Has checks if the hash is in the store.
func (c *HttpStore) Has(hash string) (bool, error) {
status, body, err := c.cfRequest(http.MethodHead, hash)
req, err := http.NewRequest("HEAD", url, nil)
if err != nil {
return false, err
return false, errors.Err(err)
}
defer func() { _ = body.Close() }()
switch status {
case http.StatusNotFound, http.StatusForbidden:
res, err := n.httpClient.Do(req)
if err != nil {
return false, errors.Err(err)
}
defer res.Body.Close()
if res.StatusCode == http.StatusNotFound {
return false, nil
case http.StatusOK:
}
if res.StatusCode == http.StatusNoContent {
return true, nil
default:
return false, errors.Err("unexpected status %d", status)
}
var body []byte
if res.Body != nil {
body, _ = ioutil.ReadAll(res.Body)
}
return false, errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
}
// Get downloads the blob using the http client
func (c *HttpStore) Get(hash string) (b stream.Blob, trace shared.BlobTrace, err error) {
log.Debugf("Getting %s from HTTP(s) source", hash[:8])
func (n *HttpStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
url := n.upstream + "/blob?hash=" + hash
defer func(t time.Time) {
log.Debugf("Getting %s from HTTP(s) source took %s", hash[:8], time.Since(t).String())
trace = trace.Stack(time.Since(start), c.Name())
}(start)
url := c.endpoint + c.shardedPath(hash)
req, err := http.NewRequest(http.MethodGet, url, nil)
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, trace, errors.Err(err)
return nil, shared.NewBlobTrace(time.Since(start), n.Name()), errors.Err(err)
}
req.Header.Add("User-Agent", "reflector.go/"+meta.Version())
res, err := c.httpClient.Do(req)
res, err := n.httpClient.Do(req)
if err != nil {
return nil, trace, errors.Err(err)
return nil, shared.NewBlobTrace(time.Since(start), n.Name()), errors.Err(err)
}
defer func(Body io.ReadCloser) {
err := Body.Close()
defer res.Body.Close()
tmp := getBuffer()
defer putBuffer(tmp)
serialized := res.Header.Get("Via")
trace := shared.NewBlobTrace(time.Since(start), n.Name())
if serialized != "" {
parsedTrace, err := shared.Deserialize(serialized)
if err != nil {
log.Errorf("Error closing response body in HTTP-GET: %s", err.Error())
}
}(res.Body)
// Parse Via header if present
viaHeader := res.Header.Get("Via")
if viaHeader != "" {
parsedTrace, err := shared.Deserialize(viaHeader)
if err == nil {
trace = *parsedTrace
return nil, shared.NewBlobTrace(time.Since(start), n.Name()), err
}
trace = *parsedTrace
}
switch res.StatusCode {
case http.StatusNotFound:
return nil, trace, ErrBlobNotFound
case http.StatusOK:
contentLength := res.Header.Get("Content-Length")
if contentLength != "" {
size, err := strconv.Atoi(contentLength)
if err == nil && size > 0 && size <= stream.MaxBlobSize {
blob := make([]byte, size)
_, err = io.ReadFull(res.Body, blob)
if err == nil {
metrics.MtrInBytesHttp.Add(float64(size))
return blob, trace, nil
}
log.Warnf("Error reading body with known size: %s", err.Error())
}
if res.StatusCode == http.StatusNotFound {
return nil, trace.Stack(time.Since(start), n.Name()), ErrBlobNotFound
}
if res.StatusCode == http.StatusOK {
written, err := io.Copy(tmp, res.Body)
if err != nil {
return nil, trace.Stack(time.Since(start), n.Name()), errors.Err(err)
}
buffer := getBuffer()
defer putBuffer(buffer)
if _, err := io.Copy(buffer, res.Body); err != nil {
return nil, trace, errors.Err(err)
}
blob := make([]byte, buffer.Len())
copy(blob, buffer.Bytes())
blob := make([]byte, written)
copy(blob, tmp.Bytes())
metrics.MtrInBytesHttp.Add(float64(len(blob)))
return blob, trace, nil
default:
body, _ := io.ReadAll(res.Body)
log.Warnf("Got status code %d (%s)", res.StatusCode, string(body))
return nil, trace, errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
return blob, trace.Stack(time.Since(start), n.Name()), nil
}
}
func (c *HttpStore) cfRequest(method, hash string) (int, io.ReadCloser, error) {
url := c.endpoint + c.shardedPath(hash)
req, err := http.NewRequest(method, url, nil)
if err != nil {
return 0, nil, errors.Err(err)
}
req.Header.Add("User-Agent", "reflector.go/"+meta.Version())
res, err := c.httpClient.Do(req)
if err != nil {
return 0, nil, errors.Err(err)
var body []byte
if res.Body != nil {
body, _ = ioutil.ReadAll(res.Body)
}
return res.StatusCode, res.Body, nil
return nil, trace.Stack(time.Since(start), n.Name()), errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
}
func (c *HttpStore) Put(_ string, _ stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
func (n *HttpStore) Put(string, stream.Blob) error {
return shared.ErrNotImplemented
}
func (n *HttpStore) PutSD(string, stream.Blob) error {
return shared.ErrNotImplemented
}
func (n *HttpStore) Delete(string) error {
return shared.ErrNotImplemented
}
func (n *HttpStore) Shutdown() {}
// buffer pool to reduce GC
// https://www.captaincodeman.com/2017/06/02/golang-buffer-pool-gotcha
var buffers = sync.Pool{
// New is called when a new instance is needed
New: func() interface{} {
buf := make([]byte, 0, stream.MaxBlobSize)
return bytes.NewBuffer(buf)
},
}
func (c *HttpStore) PutSD(_ string, _ stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
// getBuffer fetches a buffer from the pool
func getBuffer() *bytes.Buffer {
return buffers.Get().(*bytes.Buffer)
}
func (c *HttpStore) Delete(_ string) error {
return errors.Err(shared.ErrNotImplemented)
// putBuffer returns a buffer to the pool
func putBuffer(buf *bytes.Buffer) {
buf.Reset()
buffers.Put(buf)
}
// Shutdown shuts down the store gracefully
func (c *HttpStore) Shutdown() {
}
func (c *HttpStore) shardedPath(hash string) string {
if c.prefixLength <= 0 || len(hash) < c.prefixLength {
return hash
// getClient gets an http client that's customized to be more performant when dealing with blobs of 2MB in size (most of our blobs)
func getClient() *http.Client {
// Customize the Transport to have larger connection pool
defaultTransport := &http.Transport{
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).DialContext,
ForceAttemptHTTP2: true,
MaxIdleConns: 100,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
DisableCompression: true,
MaxIdleConnsPerHost: 100,
ReadBufferSize: stream.MaxBlobSize + 1024*10, //add an extra few KBs to make sure it fits the extra information
}
return path.Join(hash[:c.prefixLength], hash)
}
func init() {
RegisterStore(nameHttp, HttpStoreFactory)
return &http.Client{Transport: defaultTransport}
}
-136
View File
@@ -1,136 +0,0 @@
package store
import (
"strings"
"sync"
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
)
// Http3Store is a blob store that gets blobs from a peer over HTTP3.
// It satisfies the BlobStore interface but cannot put or delete blobs.
type Http3Store struct {
NotFoundCache *sync.Map
client *Http3Client
name string
address string
timeout time.Duration
clientMu sync.RWMutex
}
// Http3Params allows to set options for a new Http3Store.
type Http3Params struct {
Name string `mapstructure:"name"`
Address string `mapstructure:"address"`
Timeout time.Duration `mapstructure:"timeout"`
}
// NewHttp3Store makes a new HTTP3 store.
func NewHttp3Store(params Http3Params) *Http3Store {
return &Http3Store{
name: params.Name,
NotFoundCache: &sync.Map{},
address: params.Address,
timeout: params.Timeout,
}
}
const nameHttp3 = "http3"
func Http3StoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg Http3Params
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
return NewHttp3Store(cfg), nil
}
func init() {
RegisterStore(nameHttp3, Http3StoreFactory)
}
func (h *Http3Store) Name() string { return nameHttp3 + "-" + h.name }
func (h *Http3Store) getClient() (*Http3Client, error) {
h.clientMu.RLock()
if h.client != nil {
client := h.client
h.clientMu.RUnlock()
return client, nil
}
h.clientMu.RUnlock()
h.clientMu.Lock()
defer h.clientMu.Unlock()
// Check again in case another goroutine created the client
if h.client != nil {
return h.client, nil
}
client, err := NewHttp3Client(h.address)
if err != nil {
return nil, err
}
h.client = client
return client, nil
}
// Has asks the peer if they have a hash
func (h *Http3Store) Has(hash string) (bool, error) {
c, err := h.getClient()
if err != nil {
return false, err
}
return c.HasBlob(hash)
}
// Get downloads the blob from the peer
func (h *Http3Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
if lastChecked, ok := h.NotFoundCache.Load(hash); ok {
if lastChecked.(time.Time).After(time.Now().Add(-5 * time.Minute)) {
return nil, shared.NewBlobTrace(time.Since(start), h.Name()+"-notfoundcache"), ErrBlobNotFound
}
}
c, err := h.getClient()
if err != nil && strings.Contains(err.Error(), "blob not found") {
h.NotFoundCache.Store(hash, time.Now())
}
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), h.Name()), err
}
return c.GetBlob(hash)
}
// Put is not supported
func (h *Http3Store) Put(hash string, blob stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// PutSD is not supported
func (h *Http3Store) PutSD(hash string, blob stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// Delete is not supported
func (h *Http3Store) Delete(hash string) error {
return errors.Err(shared.ErrNotImplemented)
}
// Shutdown shuts down the store gracefully
func (h *Http3Store) Shutdown() {
h.clientMu.Lock()
defer h.clientMu.Unlock()
if h.client != nil {
_ = h.client.Close()
h.client = nil
}
}
-137
View File
@@ -1,137 +0,0 @@
package store
import (
"crypto/tls"
"crypto/x509"
"io"
"net/http"
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/quic-go/quic-go"
"github.com/quic-go/quic-go/http3"
)
// Http3Client is a client for HTTP3 blob store
type Http3Client struct {
conn *http.Client
roundTripper *http3.Transport
ServerAddr string
}
// NewHttp3Client creates a new HTTP3 client
func NewHttp3Client(address string) (*Http3Client, error) {
var qconf quic.Config
window500M := 500 * 1 << 20
qconf.MaxStreamReceiveWindow = uint64(window500M)
qconf.MaxConnectionReceiveWindow = uint64(window500M)
qconf.EnableDatagrams = true
qconf.HandshakeIdleTimeout = 4 * time.Second
qconf.MaxIdleTimeout = 20 * time.Second
pool, err := x509.SystemCertPool()
if err != nil {
return nil, err
}
roundTripper := &http3.Transport{
TLSClientConfig: &tls.Config{
RootCAs: pool,
InsecureSkipVerify: true,
},
QUICConfig: &qconf,
}
connection := &http.Client{
Transport: roundTripper,
}
return &Http3Client{
conn: connection,
roundTripper: roundTripper,
ServerAddr: address,
}, nil
}
// Close closes the client
func (c *Http3Client) Close() error {
return nil
}
// HasBlob checks if the peer has a blob
func (c *Http3Client) HasBlob(hash string) (bool, error) {
url := c.ServerAddr + "/blob?hash=" + hash
req, err := http.NewRequest("HEAD", url, nil)
if err != nil {
return false, errors.Err(err)
}
res, err := c.conn.Do(req)
if err != nil {
return false, errors.Err(err)
}
defer func() { _ = res.Body.Close() }()
if res.StatusCode == http.StatusNotFound {
return false, nil
}
if res.StatusCode == http.StatusNoContent {
return true, nil
}
var body []byte
if res.Body != nil {
body, _ = io.ReadAll(res.Body)
}
return false, errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
}
// GetBlob gets a blob from the peer
func (c *Http3Client) GetBlob(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
url := c.ServerAddr + "/blob?hash=" + hash
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err(err)
}
res, err := c.conn.Do(req)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err(err)
}
defer func() { _ = res.Body.Close() }()
viaHeader := res.Header.Get("Via")
var trace shared.BlobTrace
if viaHeader != "" {
parsedTrace, err := shared.Deserialize(viaHeader)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "http3"), err
}
trace = *parsedTrace
} else {
trace = shared.NewBlobTrace(0, "http3")
}
switch res.StatusCode {
case http.StatusNotFound:
return nil, trace.Stack(time.Since(start), "http3"), ErrBlobNotFound
case http.StatusOK:
buffer := getBuffer()
defer putBuffer(buffer)
if _, err := io.Copy(buffer, res.Body); err != nil {
return nil, trace.Stack(time.Since(start), "http3"), errors.Err(err)
}
blob := make([]byte, buffer.Len())
copy(blob, buffer.Bytes())
return blob, trace.Stack(time.Since(start), "http3"), nil
default:
body, _ := io.ReadAll(res.Body)
return nil, trace.Stack(time.Since(start), "http3"),
errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
}
}
+7 -70
View File
@@ -1,7 +1,6 @@
package store
import (
"strings"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
@@ -9,84 +8,25 @@ import (
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
)
// ITTTStore performs an operation on this storage, if this fails, it attempts to run it on that
type ITTTStore struct {
this, that BlobStore
name string
}
type ITTTParams struct {
This BlobStore `mapstructure:"this"`
That BlobStore `mapstructure:"that"`
Name string `mapstructure:"name"`
}
type ITTTConfig struct {
This *viper.Viper
That *viper.Viper
Name string `mapstructure:"name"`
}
// NewITTTStore returns a new instance of the IF THIS THAN THAT store
func NewITTTStore(params ITTTParams) *ITTTStore {
func NewITTTStore(this, that BlobStore) *ITTTStore {
return &ITTTStore{
this: params.This,
that: params.That,
name: params.Name,
this: this,
that: that,
}
}
const nameIttt = "ittt"
func ITTTStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg ITTTConfig
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
cfg.This = config.Sub("this")
cfg.That = config.Sub("that")
thisStoreType := strings.Split(cfg.This.AllKeys()[0], ".")[0]
thisStoreConfig := cfg.This.Sub(thisStoreType)
factory, ok := Factories[thisStoreType]
if !ok {
return nil, errors.Err("unknown store type %s", thisStoreType)
}
thisStore, err := factory(thisStoreConfig)
if err != nil {
return nil, errors.Err(err)
}
thatStoreType := strings.Split(cfg.That.AllKeys()[0], ".")[0]
thatStoreConfig := cfg.That.Sub(thatStoreType)
factory, ok = Factories[thatStoreType]
if !ok {
return nil, errors.Err("unknown store type %s", thatStoreType)
}
thatStore, err := factory(thatStoreConfig)
if err != nil {
return nil, errors.Err(err)
}
return NewITTTStore(ITTTParams{
Name: cfg.Name,
This: thisStore,
That: thatStore,
}), nil
}
func init() {
RegisterStore(nameIttt, ITTTStoreFactory)
}
// Name is the cache type name
func (c *ITTTStore) Name() string { return nameIttt + "-" + c.name }
func (c *ITTTStore) Name() string { return nameIttt }
// Has checks in this for a hash, if it fails it checks in that. It returns true if either store has it.
func (c *ITTTStore) Has(hash string) (bool, error) {
@@ -102,7 +42,7 @@ func (c *ITTTStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
blob, trace, err := c.this.Get(hash)
if err == nil {
metrics.ItttHitCount.With(metrics.ItttLabels(c.this.Name())).Inc()
metrics.ThisHitCount.Inc()
return blob, trace.Stack(time.Since(start), c.Name()), err
}
@@ -110,7 +50,7 @@ func (c *ITTTStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
if err != nil {
return nil, trace.Stack(time.Since(start), c.Name()), err
}
metrics.ItttHitCount.With(metrics.ItttLabels(c.that.Name())).Inc()
metrics.ThatHitCount.Inc()
return blob, trace.Stack(time.Since(start), c.Name()), nil
}
@@ -130,7 +70,4 @@ func (c *ITTTStore) Delete(hash string) error {
}
// Shutdown shuts down the store gracefully
func (c *ITTTStore) Shutdown() {
c.this.Shutdown()
c.that.Shutdown()
}
func (c *ITTTStore) Shutdown() {}
+2 -23
View File
@@ -8,46 +8,25 @@ import (
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
)
// MemStore is an in memory only blob store with no persistence.
type MemStore struct {
blobs map[string]stream.Blob
mu *sync.RWMutex
name string
}
type MemParams struct {
Name string `mapstructure:"name"`
}
func NewMemStore(params MemParams) *MemStore {
func NewMemStore() *MemStore {
return &MemStore{
blobs: make(map[string]stream.Blob),
mu: &sync.RWMutex{},
name: params.Name,
}
}
const nameMem = "mem"
func MemStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg MemParams
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
return NewMemStore(cfg), nil
}
func init() {
RegisterStore(nameMem, MemStoreFactory)
}
// Name is the cache type name
func (m *MemStore) Name() string { return nameMem + "-" + m.name }
func (m *MemStore) Name() string { return nameMem }
// Has returns T/F if the blob is currently stored. It will never error.
func (m *MemStore) Has(hash string) (bool, error) {
+2 -2
View File
@@ -8,7 +8,7 @@ import (
)
func TestMemStore_Put(t *testing.T) {
s := NewMemStore(MemParams{Name: "test"})
s := NewMemStore()
blob := []byte("abcdefg")
err := s.Put("abc", blob)
if err != nil {
@@ -17,7 +17,7 @@ func TestMemStore_Put(t *testing.T) {
}
func TestMemStore_Get(t *testing.T) {
s := NewMemStore(MemParams{Name: "test"})
s := NewMemStore()
hash := "abc"
blob := []byte("abcdefg")
err := s.Put(hash, blob)
-206
View File
@@ -1,206 +0,0 @@
package store
import (
"strings"
"sync"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
)
// MultiWriterStore writes to multiple destination stores
type MultiWriterStore struct {
name string
destinations []BlobStore
}
type MultiWriterParams struct {
Name string
Destinations []BlobStore
}
type MultiWriterConfig struct {
Name string `mapstructure:"name"`
One viper.Viper
Two viper.Viper
Three viper.Viper
}
// NewMultiWriterStore returns a new instance of the MultiWriter store
func NewMultiWriterStore(params MultiWriterParams) *MultiWriterStore {
return &MultiWriterStore{
name: params.Name,
destinations: params.Destinations,
}
}
const nameMultiWriter = "multiwriter"
func MultiWriterStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg MultiWriterConfig
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
var destinations []BlobStore
one := config.Sub("one")
two := config.Sub("two")
//three := config.Sub("three")
storeTypeOne := strings.Split(one.AllKeys()[0], ".")[0]
storeTypeTwo := strings.Split(two.AllKeys()[0], ".")[0]
//storeTypeThree := strings.Split(three.AllKeys()[0], ".")[0]
storeCfgOne := one.Sub(storeTypeOne)
storeCfgTwo := two.Sub(storeTypeTwo)
//storeCfgThree := config.Sub(storeTypeThree)
factoryOne, ok := Factories[storeTypeOne]
if !ok {
return nil, errors.Err("unknown store type %s", storeTypeOne)
}
factoryTwo, ok := Factories[storeTypeTwo]
if !ok {
return nil, errors.Err("unknown store type %s", storeTypeTwo)
}
//factoryThree, ok := Factories[storeTypeThree]
//if !ok {
// return nil, errors.Err("unknown store type %s", storeTypeThree)
//}
store1, err := factoryOne(storeCfgOne)
if err != nil {
return nil, errors.Err(err)
}
store2, err := factoryTwo(storeCfgTwo)
if err != nil {
return nil, errors.Err(err)
}
//store3, err := factoryThree(storeCfgThree)
//if err != nil {
// return nil, errors.Err(err)
//}
destinations = append(destinations, store1, store2)
return NewMultiWriterStore(MultiWriterParams{
Name: cfg.Name,
Destinations: destinations,
}), nil
}
func init() {
RegisterStore(nameMultiWriter, MultiWriterStoreFactory)
}
// Name returns the store name
func (m *MultiWriterStore) Name() string { return nameMultiWriter + "-" + m.name }
// Has is not supported by MultiWriter
func (m *MultiWriterStore) Has(hash string) (bool, error) {
return false, errors.Err(shared.ErrNotImplemented)
}
// Get is not supported by MultiWriter
func (m *MultiWriterStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
return nil, shared.BlobTrace{}, errors.Err(shared.ErrNotImplemented)
}
// Put writes the blob to all destination stores
func (m *MultiWriterStore) Put(hash string, blob stream.Blob) error {
var wg sync.WaitGroup
errChan := make(chan error, len(m.destinations))
for _, dest := range m.destinations {
wg.Add(1)
go func(d BlobStore) {
defer wg.Done()
if err := d.Put(hash, blob); err != nil {
errChan <- errors.Err("failed to write to %s: %v", d.Name(), err)
}
}(dest)
}
wg.Wait()
close(errChan)
var errs []string
for err := range errChan {
errs = append(errs, err.Error())
}
if len(errs) > 0 {
return errors.Err("failed to write to some destinations: %s", strings.Join(errs, "; "))
}
return nil
}
// PutSD writes the SD blob to all destination stores
func (m *MultiWriterStore) PutSD(hash string, blob stream.Blob) error {
var wg sync.WaitGroup
errChan := make(chan error, len(m.destinations))
for _, dest := range m.destinations {
wg.Add(1)
go func(d BlobStore) {
defer wg.Done()
if err := d.PutSD(hash, blob); err != nil {
errChan <- errors.Err("failed to write SD to %s: %v", d.Name(), err)
}
}(dest)
}
wg.Wait()
close(errChan)
var errs []string
for err := range errChan {
errs = append(errs, err.Error())
}
if len(errs) > 0 {
return errors.Err("failed to write SD to some destinations: %s", strings.Join(errs, "; "))
}
return nil
}
// Delete deletes the blob from all destination stores
func (m *MultiWriterStore) Delete(hash string) error {
var wg sync.WaitGroup
errChan := make(chan error, len(m.destinations))
for _, dest := range m.destinations {
wg.Add(1)
go func(d BlobStore) {
defer wg.Done()
if err := d.Delete(hash); err != nil {
errChan <- errors.Err("failed to delete from %s: %v", d.Name(), err)
}
}(dest)
}
wg.Wait()
close(errChan)
var errs []string
for err := range errChan {
errs = append(errs, err.Error())
}
if len(errs) > 0 {
return errors.Err("failed to delete from some destinations: %s", strings.Join(errs, "; "))
}
return nil
}
// Shutdown shuts down all destination stores gracefully
func (m *MultiWriterStore) Shutdown() {
for _, dest := range m.destinations {
dest.Shutdown()
}
}
+3 -27
View File
@@ -5,39 +5,15 @@ import (
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
)
// NoopStore is a store that does nothing
type NoopStore struct {
name string
}
func NewNoopStore(name string) *NoopStore {
return &NoopStore{name: name}
}
type NoopStore struct{}
const nameNoop = "noop"
func NoopStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg struct {
Name string `mapstructure:"name"`
}
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
return NewNoopStore(cfg.Name), nil
}
func init() {
RegisterStore(nameNoop, NoopStoreFactory)
}
func (n *NoopStore) Name() string { return nameNoop + "-" + n.name }
func (n *NoopStore) Name() string { return nameNoop }
func (n *NoopStore) Has(_ string) (bool, error) { return false, nil }
func (n *NoopStore) Get(_ string) (stream.Blob, shared.BlobTrace, error) {
return nil, shared.NewBlobTrace(time.Since(time.Now()), n.Name()), nil
@@ -45,4 +21,4 @@ func (n *NoopStore) Get(_ string) (stream.Blob, shared.BlobTrace, error) {
func (n *NoopStore) Put(_ string, _ stream.Blob) error { return nil }
func (n *NoopStore) PutSD(_ string, _ stream.Blob) error { return nil }
func (n *NoopStore) Delete(_ string) error { return nil }
func (n *NoopStore) Shutdown() {}
func (n *NoopStore) Shutdown() { return }
-100
View File
@@ -1,100 +0,0 @@
package store
import (
"strings"
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
)
// PeerStore is a blob store that gets blobs from a peer.
// It satisfies the BlobStore interface but cannot put or delete blobs.
type PeerStore struct {
name string
opts PeerParams
}
// PeerParams allows to set options for a new PeerStore.
type PeerParams struct {
Name string `mapstructure:"name"`
Address string `mapstructure:"address"`
Timeout time.Duration `mapstructure:"timeout"`
}
// NewPeerStore makes a new peer store.
func NewPeerStore(params PeerParams) *PeerStore {
return &PeerStore{opts: params, name: params.Name}
}
const namePeer = "peer"
func PeerStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg PeerParams
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
return NewPeerStore(cfg), nil
}
func init() {
RegisterStore(namePeer, PeerStoreFactory)
}
func (p *PeerStore) Name() string { return namePeer + "-" + p.name }
func (p *PeerStore) getClient() (*PeerClient, error) {
c := &PeerClient{Timeout: p.opts.Timeout}
err := c.Connect(p.opts.Address)
return c, errors.Prefix("connection error", err)
}
// Has asks the peer if they have a hash
func (p *PeerStore) Has(hash string) (bool, error) {
c, err := p.getClient()
if err != nil {
return false, err
}
defer func() { _ = c.Close() }()
return c.HasBlob(hash)
}
// Get downloads the blob from the peer
func (p *PeerStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
c, err := p.getClient()
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), p.Name()), err
}
defer func() { _ = c.Close() }()
blob, trace, err := c.GetBlob(hash)
if err != nil && strings.Contains(err.Error(), "blob not found") {
return nil, trace, ErrBlobNotFound
}
return blob, trace, err
}
// Put is not supported
func (p *PeerStore) Put(hash string, blob stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// PutSD is not supported
func (p *PeerStore) PutSD(hash string, blob stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// Delete is not supported
func (p *PeerStore) Delete(hash string) error {
return errors.Err(shared.ErrNotImplemented)
}
// Shutdown is not supported
func (p *PeerStore) Shutdown() {
}
-95
View File
@@ -1,95 +0,0 @@
package store
import (
"bufio"
"bytes"
"encoding/binary"
"io"
"net"
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/stream"
)
// PeerClient is a client for peer blob store
type PeerClient struct {
conn net.Conn
Timeout time.Duration
}
// Connect connects to a peer
func (c *PeerClient) Connect(address string) error {
var err error
c.conn, err = net.DialTimeout("tcp", address, c.Timeout)
return err
}
// Close closes the connection
func (c *PeerClient) Close() error {
if c.conn != nil {
return c.conn.Close()
}
return nil
}
// HasBlob checks if the peer has a blob
func (c *PeerClient) HasBlob(hash string) (bool, error) {
err := c.writeRequest("has", hash)
if err != nil {
return false, err
}
response, err := c.readResponse()
if err != nil {
return false, err
}
return response == "yes", nil
}
// GetBlob gets a blob from the peer
func (c *PeerClient) GetBlob(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
err := c.writeRequest("get", hash)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "peer"), err
}
response, err := c.readResponse()
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "peer"), err
}
if response == "no" {
return nil, shared.NewBlobTrace(time.Since(start), "peer"), ErrBlobNotFound
}
size, err := binary.ReadVarint(bufio.NewReader(bytes.NewReader([]byte(response))))
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "peer"), err
}
blob := make([]byte, size)
_, err = io.ReadFull(c.conn, blob)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "peer"), err
}
return blob, shared.NewBlobTrace(time.Since(start), "peer"), nil
}
func (c *PeerClient) writeRequest(cmd, hash string) error {
_, err := c.conn.Write([]byte(cmd + " " + hash + "\n"))
return err
}
func (c *PeerClient) readResponse() (string, error) {
reader := bufio.NewReader(c.conn)
response, err := reader.ReadString('\n')
if err != nil {
return "", err
}
return response[:len(response)-1], nil
}
-147
View File
@@ -1,147 +0,0 @@
package store
import (
"strings"
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
)
// ProxiedS3Store writes to an S3 store and reads from any BlobStore (usually an ITTTStore of HttpStore endpoints).
type ProxiedS3Store struct {
readerStore BlobStore
writerStore BlobStore
name string
}
func (c *ProxiedS3Store) MissingBlobsForKnownStream(s string) ([]string, error) {
if bc, ok := c.writerStore.(NeededBlobChecker); ok {
return bc.MissingBlobsForKnownStream(s)
}
return nil, errors.Err("writer does not implement neededBlobChecker")
}
func (c *ProxiedS3Store) Block(hash string) error {
if bl, ok := c.writerStore.(Blocklister); ok {
return bl.Block(hash)
}
return errors.Err("writer does not implement Blocklister")
}
func (c *ProxiedS3Store) Wants(hash string) (bool, error) {
if bl, ok := c.writerStore.(Blocklister); ok {
return bl.Wants(hash)
}
return true, errors.Err("writer does not implement Blocklister")
}
type ProxiedS3Params struct {
Reader BlobStore `mapstructure:"reader"`
Writer BlobStore `mapstructure:"writer"`
Name string `mapstructure:"name"`
}
type ProxiedS3Config struct {
Reader *viper.Viper
Writer *viper.Viper
Name string `mapstructure:"name"`
}
// NewProxiedS3Store returns an initialized ProxiedS3Store store pointer.
// NOTE: It panics if either argument is nil.
func NewProxiedS3Store(params ProxiedS3Params) *ProxiedS3Store {
if params.Reader == nil || params.Writer == nil {
panic("both stores must be set")
}
return &ProxiedS3Store{
readerStore: params.Reader,
writerStore: params.Writer,
name: params.Name,
}
}
const nameProxiedS3 = "proxied-s3"
// Name is the cache type name
func (c *ProxiedS3Store) Name() string { return nameProxiedS3 + "-" + c.name }
// Has checks if the hash is in the store.
func (c *ProxiedS3Store) Has(hash string) (bool, error) {
return c.writerStore.Has(hash)
}
// Get gets the blob from Cloudfront.
func (c *ProxiedS3Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
blob, trace, err := c.readerStore.Get(hash)
return blob, trace.Stack(time.Since(start), c.Name()), err
}
// Put stores the blob on S3
func (c *ProxiedS3Store) Put(hash string, blob stream.Blob) error {
return c.writerStore.Put(hash, blob)
}
// PutSD stores the sd blob on S3
func (c *ProxiedS3Store) PutSD(hash string, blob stream.Blob) error {
return c.writerStore.PutSD(hash, blob)
}
// Delete deletes the blob from S3
func (c *ProxiedS3Store) Delete(hash string) error {
return c.writerStore.Delete(hash)
}
// Shutdown shuts down the store gracefully
func (c *ProxiedS3Store) Shutdown() {
c.writerStore.Shutdown()
c.readerStore.Shutdown()
}
func ProxiedS3StoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg ProxiedS3Config
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
cfg.Reader = config.Sub("reader")
cfg.Writer = config.Sub("writer")
readerStoreType := strings.Split(cfg.Reader.AllKeys()[0], ".")[0]
readerStoreConfig := cfg.Reader.Sub(readerStoreType)
factory, ok := Factories[readerStoreType]
if !ok {
return nil, errors.Err("unknown store type %s", readerStoreType)
}
readerStore, err := factory(readerStoreConfig)
if err != nil {
return nil, errors.Err(err)
}
writerStoreType := strings.Split(cfg.Writer.AllKeys()[0], ".")[0]
writerStoreConfig := cfg.Writer.Sub(writerStoreType)
factory, ok = Factories[writerStoreType]
if !ok {
return nil, errors.Err("unknown store type %s", writerStoreType)
}
writerStore, err := factory(writerStoreConfig)
if err != nil {
return nil, errors.Err(err)
}
return NewProxiedS3Store(ProxiedS3Params{
Name: cfg.Name,
Reader: readerStore,
Writer: writerStore,
}), nil
}
func init() {
RegisterStore(nameProxiedS3, ProxiedS3StoreFactory)
}
+56 -104
View File
@@ -2,9 +2,7 @@ package store
import (
"bytes"
"context"
stderrors "errors"
"path"
"net/http"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
@@ -13,88 +11,56 @@ import (
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/aws/aws-sdk-go-v2/aws"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
)
// S3Store is an S3 store
type S3Store struct {
client *s3.Client
awsID string
awsSecret string
region string
bucket string
endpoint string
name string
prefixLength int
}
awsID string
awsSecret string
region string
bucket string
type S3Params struct {
Name string `mapstructure:"name"`
AwsID string `mapstructure:"aws_id"`
AwsSecret string `mapstructure:"aws_secret"`
Region string `mapstructure:"region"`
Bucket string `mapstructure:"bucket"`
Endpoint string `mapstructure:"endpoint"`
ShardingSize int `mapstructure:"sharding_size"`
session *session.Session
}
// NewS3Store returns an initialized S3 store pointer.
func NewS3Store(params S3Params) *S3Store {
func NewS3Store(awsID, awsSecret, region, bucket string) *S3Store {
return &S3Store{
awsID: params.AwsID,
awsSecret: params.AwsSecret,
region: params.Region,
bucket: params.Bucket,
endpoint: params.Endpoint,
name: params.Name,
prefixLength: params.ShardingSize,
awsID: awsID,
awsSecret: awsSecret,
region: region,
bucket: bucket,
}
}
const nameS3 = "s3"
func S3StoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg S3Params
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
return NewS3Store(cfg), nil
}
func init() {
RegisterStore(nameS3, S3StoreFactory)
}
// Name is the cache type name
func (s *S3Store) Name() string { return nameS3 + "-" + s.name }
func (s *S3Store) Name() string { return nameS3 }
// Has returns T/F or Error (from S3) if the store contains the blob.
// Has returns T/F or Error ( from S3 ) if the store contains the blob.
func (s *S3Store) Has(hash string) (bool, error) {
err := s.initOnce()
if err != nil {
return false, err
}
ctx := context.Background()
_, err = s.client.HeadObject(ctx, &s3.HeadObjectInput{
_, err = s3.New(s.session).HeadObject(&s3.HeadObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(s.shardedPath(hash)),
Key: aws.String(hash),
})
if err != nil {
var notFound *types.NotFound
var noSuchKey *types.NoSuchKey
if stderrors.As(err, &notFound) || stderrors.As(err, &noSuchKey) {
if reqFail, ok := err.(s3.RequestFailure); ok && reqFail.StatusCode() == http.StatusNotFound {
return false, nil
}
return false, errors.Err(err)
return false, err
}
return true, nil
@@ -103,6 +69,7 @@ func (s *S3Store) Has(hash string) (bool, error) {
// Get returns the blob slice if present or errors on S3.
func (s *S3Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
//Todo-Need to handle error for blob doesn't exist for consistency.
err := s.initOnce()
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), s.Name()), err
@@ -110,26 +77,24 @@ func (s *S3Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
log.Debugf("Getting %s from S3", hash[:8])
defer func(t time.Time) {
log.Debugf("Getting %s from %s took %s", hash[:8], s.Name(), time.Since(t).String())
log.Debugf("Getting %s from S3 took %s", hash[:8], time.Since(t).String())
}(start)
ctx := context.Background()
buf := manager.NewWriteAtBuffer([]byte{})
_, err = manager.NewDownloader(s.client).Download(ctx, buf, &s3.GetObjectInput{
buf := &aws.WriteAtBuffer{}
_, err = s3manager.NewDownloader(s.session).Download(buf, &s3.GetObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(s.shardedPath(hash)),
Key: aws.String(hash),
})
if err != nil {
var noSuchBucket *types.NoSuchBucket
var noSuchKey *types.NoSuchKey
var notFound *types.NotFound
switch {
case stderrors.As(err, &noSuchBucket):
return nil, shared.NewBlobTrace(time.Since(start), s.Name()), errors.Err("bucket %s does not exist", s.bucket)
case stderrors.As(err, &noSuchKey), stderrors.As(err, &notFound):
return nil, shared.NewBlobTrace(time.Since(start), s.Name()), errors.Err(ErrBlobNotFound)
if aerr, ok := err.(awserr.Error); ok {
switch aerr.Code() {
case s3.ErrCodeNoSuchBucket:
return nil, shared.NewBlobTrace(time.Since(start), s.Name()), errors.Err("bucket %s does not exist", s.bucket)
case s3.ErrCodeNoSuchKey:
return nil, shared.NewBlobTrace(time.Since(start), s.Name()), errors.Err(ErrBlobNotFound)
}
}
return nil, shared.NewBlobTrace(time.Since(start), s.Name()), errors.Err(err)
return buf.Bytes(), shared.NewBlobTrace(time.Since(start), s.Name()), err
}
return buf.Bytes(), shared.NewBlobTrace(time.Since(start), s.Name()), nil
@@ -147,18 +112,16 @@ func (s *S3Store) Put(hash string, blob stream.Blob) error {
log.Debugf("Uploading %s took %s", hash[:8], time.Since(t).String())
}(time.Now())
ctx := context.Background()
_, err = manager.NewUploader(s.client).Upload(ctx, &s3.PutObjectInput{
_, err = s3manager.NewUploader(s.session).Upload(&s3manager.UploadInput{
Bucket: aws.String(s.bucket),
Key: aws.String(s.shardedPath(hash)),
Key: aws.String(hash),
Body: bytes.NewBuffer(blob),
ACL: types.ObjectCannedACLPublicRead,
ACL: aws.String("public-read"),
//StorageClass: aws.String(s3.StorageClassIntelligentTiering),
})
if err != nil {
return errors.Err(err)
}
metrics.MtrOutBytesReflector.Add(float64(blob.Size()))
return nil
return err
}
// PutSD stores the sd blob on S3 or errors if S3 connection errors.
@@ -175,44 +138,33 @@ func (s *S3Store) Delete(hash string) error {
log.Debugf("Deleting %s from S3", hash[:8])
ctx := context.Background()
_, err = s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
_, err = s3.New(s.session).DeleteObject(&s3.DeleteObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(s.shardedPath(hash)),
Key: aws.String(hash),
})
return errors.Err(err)
return err
}
func (s *S3Store) initOnce() error {
if s.client != nil {
if s.session != nil {
return nil
}
ctx := context.Background()
cfg, err := awsconfig.LoadDefaultConfig(ctx,
awsconfig.WithRegion(s.region),
awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(s.awsID, s.awsSecret, "")),
)
if err != nil {
return errors.Err(err)
}
s.client = s3.NewFromConfig(cfg, func(o *s3.Options) {
o.BaseEndpoint = aws.String(s.endpoint)
o.UsePathStyle = true
o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired
sess, err := session.NewSession(&aws.Config{
Credentials: credentials.NewStaticCredentials(s.awsID, s.awsSecret, ""),
Region: aws.String(s.region),
Endpoint: aws.String("https://s3.wasabisys.com"),
})
return nil
}
func (s *S3Store) shardedPath(hash string) string {
if s.prefixLength <= 0 || len(hash) < s.prefixLength {
return hash
if err != nil {
return err
}
return path.Join(hash[:s.prefixLength], hash)
s.session = sess
return nil
}
// Shutdown shuts down the store gracefully
func (s *S3Store) Shutdown() {
return
}
+3 -35
View File
@@ -1,7 +1,6 @@
package store
import (
"strings"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
@@ -10,7 +9,6 @@ import (
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
"golang.org/x/sync/singleflight"
)
@@ -24,40 +22,9 @@ func WithSingleFlight(component string, origin BlobStore) BlobStore {
type singleflightStore struct {
BlobStore
sf *singleflight.Group
component string
}
type SingleFlightConfig struct {
Store *viper.Viper
Component string `mapstructure:"component"`
}
func SingleFlightStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg SingleFlightConfig
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
cfg.Store = config.Sub("store")
storeType := strings.Split(cfg.Store.AllKeys()[0], ".")[0]
storeConfig := cfg.Store.Sub(storeType)
factory, ok := Factories[storeType]
if !ok {
return nil, errors.Err("unknown store type %s", storeType)
}
underlyingStore, err := factory(storeConfig)
if err != nil {
return nil, errors.Err(err)
}
return WithSingleFlight(cfg.Component, underlyingStore), nil
}
func init() {
RegisterStore("singleflight", SingleFlightStoreFactory)
sf *singleflight.Group
}
func (s *singleflightStore) Name() string {
@@ -157,4 +124,5 @@ func (s *singleflightStore) putter(hash string, blob stream.Blob) func() (interf
// Shutdown shuts down the store gracefully
func (s *singleflightStore) Shutdown() {
s.BlobStore.Shutdown()
return
}
+3 -17
View File
@@ -1,8 +1,7 @@
package speedwalk
import (
"io/fs"
"os"
"io/ioutil"
"path/filepath"
"runtime"
"sync"
@@ -18,20 +17,7 @@ import (
// AllFiles recursively lists every file in every subdirectory of a given directory
// If basename is true, return the basename of each file. Otherwise return the full path starting at startDir.
func AllFiles(startDir string, basename bool) ([]string, error) {
entries, err := os.ReadDir(startDir)
if err != nil {
return nil, err
}
items := make([]fs.FileInfo, 0, len(entries))
for _, entry := range entries {
var info fs.FileInfo
info, err = entry.Info()
if err != nil {
return nil, err
}
items = append(items, info)
}
items, err := ioutil.ReadDir(startDir)
if err != nil {
return nil, err
}
@@ -91,7 +77,7 @@ func AllFiles(startDir string, basename bool) ([]string, error) {
},
})
if err != nil {
logrus.Error(errors.FullTrace(err))
logrus.Errorf(errors.FullTrace(err))
}
}(item.Name())
}
+1 -15
View File
@@ -5,8 +5,6 @@ import (
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
)
// BlobStore is an interface for handling blob storage.
@@ -35,23 +33,11 @@ type Blocklister interface {
Wants(hash string) (bool, error)
}
// NeededBlobChecker can check which blobs from a known stream are not uploaded yet
type NeededBlobChecker interface {
MissingBlobsForKnownStream(string) ([]string, error)
}
// lister is a store that can list cached blobs. This is helpful when an overlay
// cache needs to track blob existence.
type lister interface {
list() ([]string, error)
}
// ErrBlobNotFound is a standard error when a blob is not found in the store.
//ErrBlobNotFound is a standard error when a blob is not found in the store.
var ErrBlobNotFound = errors.Base("blob not found")
var Factories = make(map[string]Factory)
func RegisterStore(name string, factory Factory) {
Factories[name] = factory
}
type Factory func(config *viper.Viper) (BlobStore, error)
-204
View File
@@ -1,204 +0,0 @@
package store
import (
"bytes"
"context"
"io"
"net"
"net/http"
"sync"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
)
// UpstreamStore is a store that works on top of the HTTP protocol
type UpstreamStore struct {
upstream string
httpClient *http.Client
edgeToken string
name string
}
type UpstreamParams struct {
Name string `mapstructure:"name"`
Upstream string `mapstructure:"upstream"`
EdgeToken string `mapstructure:"edge_token"`
}
func NewUpstreamStore(params UpstreamParams) *UpstreamStore {
return &UpstreamStore{
upstream: params.Upstream,
httpClient: getClient(),
edgeToken: params.EdgeToken,
name: params.Name,
}
}
const nameUpstream = "upstream"
func UpstreamStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg UpstreamParams
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
return NewUpstreamStore(cfg), nil
}
func init() {
RegisterStore(nameUpstream, UpstreamStoreFactory)
}
func (n *UpstreamStore) Name() string { return nameUpstream + "-" + n.name }
func (n *UpstreamStore) Has(hash string) (bool, error) {
url := n.upstream + "/blob?hash=" + hash
req, err := http.NewRequest("HEAD", url, nil)
if err != nil {
return false, errors.Err(err)
}
res, err := n.httpClient.Do(req)
if err != nil {
return false, errors.Err(err)
}
defer func() { _ = res.Body.Close() }()
if res.StatusCode == http.StatusNotFound {
return false, nil
}
if res.StatusCode == http.StatusNoContent {
return true, nil
}
var body []byte
if res.Body != nil {
body, _ = io.ReadAll(res.Body)
}
return false, errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
}
func (n *UpstreamStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
url := n.upstream + "/blob?hash=" + hash
if n.edgeToken != "" {
url += "&edge_token=" + n.edgeToken
}
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), n.Name()), errors.Err(err)
}
res, err := n.httpClient.Do(req)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), n.Name()), errors.Err(err)
}
defer func(Body io.ReadCloser) {
err := Body.Close()
if err != nil {
log.Errorf("Error closing response body in HTTP-GET: %s", err.Error())
}
}(res.Body)
viaHeader := res.Header.Get("Via")
var trace shared.BlobTrace
if viaHeader != "" {
parsedTrace, err := shared.Deserialize(viaHeader)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), n.Name()), err
}
trace = *parsedTrace
} else {
trace = shared.NewBlobTrace(0, n.Name())
}
switch res.StatusCode {
case http.StatusNotFound:
return nil, trace.Stack(time.Since(start), n.Name()), ErrBlobNotFound
case http.StatusOK:
buffer := getBuffer()
defer putBuffer(buffer)
if _, err := io.Copy(buffer, res.Body); err != nil {
return nil, trace.Stack(time.Since(start), n.Name()), errors.Err(err)
}
blob := make([]byte, buffer.Len())
copy(blob, buffer.Bytes())
metrics.MtrInBytesUpstream.Add(float64(len(blob)))
return blob, trace.Stack(time.Since(start), n.Name()), nil
default:
body, _ := io.ReadAll(res.Body)
log.Warnf("Got status code %d (%s)", res.StatusCode, string(body))
return nil, trace.Stack(time.Since(start), n.Name()),
errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
}
}
func (n *UpstreamStore) Put(string, stream.Blob) error {
return shared.ErrNotImplemented
}
func (n *UpstreamStore) PutSD(string, stream.Blob) error {
return shared.ErrNotImplemented
}
func (n *UpstreamStore) Delete(string) error {
return shared.ErrNotImplemented
}
func (n *UpstreamStore) Shutdown() {}
// buffer pool to reduce GC
// https://www.captaincodeman.com/2017/06/02/golang-buffer-pool-gotcha
var buffers = sync.Pool{
// New is called when a new instance is needed
New: func() interface{} {
buf := make([]byte, 0, stream.MaxBlobSize)
return bytes.NewBuffer(buf)
},
}
// getBuffer fetches a buffer from the pool
func getBuffer() *bytes.Buffer {
return buffers.Get().(*bytes.Buffer)
}
// putBuffer returns a buffer to the pool
func putBuffer(buf *bytes.Buffer) {
buf.Reset()
buffers.Put(buf)
}
func dialContext(ctx context.Context, network, address string) (net.Conn, error) {
dialer := &net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}
return dialer.DialContext(ctx, network, address)
}
// getClient gets an http client that's customized to be more performant when dealing with blobs of 2MB in size (most of our blobs)
func getClient() *http.Client {
// Customize the Transport to have larger connection pool
defaultTransport := &http.Transport{
DialContext: dialContext,
ForceAttemptHTTP2: true,
MaxIdleConns: 100,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
DisableCompression: true,
MaxIdleConnsPerHost: 100,
ReadBufferSize: stream.MaxBlobSize + 1024*10, //add an extra few KBs to make sure it fits the extra information
}
return &http.Client{Transport: defaultTransport}
}
+4 -4
View File
@@ -69,22 +69,22 @@ func (n *Node) Resolve(url string) (*types.Output, error) {
type GetClaimsInTxResp struct {
Jsonrpc string `json:"jsonrpc"`
ID int `json:"id"`
Result []struct {
Name string `json:"name"`
ClaimID string `json:"claim_id"`
Txid string `json:"txid"`
Value string `json:"value"`
Address string `json:"address"`
Supports []interface{} `json:"supports"` // TODO: finish me
Nout int `json:"nout"`
Amount int `json:"amount"`
Depth int `json:"depth"`
Height int `json:"height"`
Value string `json:"value"`
ClaimSequence int `json:"claim_sequence"`
Address string `json:"address"`
Supports []interface{} `json:"supports"` // TODO: finish me
EffectiveAmount int `json:"effective_amount"`
ValidAtHeight int `json:"valid_at_height"`
} `json:"result"`
ID int `json:"id"`
}
func (n *Node) GetClaimsInTx(txid string) (*GetClaimsInTxResp, error) {
+16 -13
View File
@@ -30,19 +30,22 @@ var (
)
type response struct {
err error
data []byte
err error
}
type Node struct {
transport *TCPTransport
grp *stop.Group
handlersMu *sync.RWMutex
handlers map[uint32]chan response
transport *TCPTransport
nextId atomic.Uint32
grp *stop.Group
handlersMu *sync.RWMutex
handlers map[uint32]chan response
pushHandlersMu *sync.RWMutex
pushHandlers map[string][]chan response
nextId atomic.Uint32
timeout time.Duration
timeout time.Duration
}
// NewNode creates a new node.
@@ -152,23 +155,23 @@ func (n *Node) listen() {
return
case bytes := <-n.transport.Responses():
msg := &struct {
Id uint32 `json:"id"`
Method string `json:"method"`
Error struct {
Message string `json:"message"`
Code int `json:"code"`
Message string `json:"message"`
} `json:"error"`
Id uint32 `json:"id"`
}{}
msg2 := &struct {
Id uint32 `json:"id"`
Method string `json:"method"`
Error struct {
Code int `json:"code"`
Message struct {
Message string `json:"message"`
Code int `json:"code"`
Message string `json:"message"`
} `json:"message"`
Code int `json:"code"`
} `json:"error"`
Id uint32 `json:"id"`
}{}
r := response{}
@@ -228,9 +231,9 @@ func (n *Node) listen() {
// request makes a request to the server and unmarshals the response into v.
func (n *Node) request(method string, params []string, v interface{}) error {
msg := struct {
Id uint32 `json:"id"`
Method string `json:"method"`
Params []string `json:"params"`
Id uint32 `json:"id"`
}{
Id: n.nextId.Load(),
Method: method,
+1 -1
View File
@@ -124,7 +124,7 @@ func (t *TCPTransport) test() error {
return errors.Err(err)
}
if response.Error.Message != "" {
return fmt.Errorf("%s", response.Error.Message)
return fmt.Errorf(response.Error.Message)
}
return nil
}