244 Commits
Author SHA1 Message Date
Niko Storni 0359d723c5 Add fallback DiskStore.Put for non-linux/darwin platforms
Go / build (push) Waiting to run
2026-01-03 04:47:29 +01:00
Niko Storni 0cfc336be5 Fix flaky BlobTrace serialization test 2026-01-03 04:36:48 +01:00
Niko Storni 05acdcb9e0 update go
rename cmd back to reflector
remove travis
update deps
2026-01-03 04:25:04 +01:00
Niko Storni 9e2cccc51f Migrate to AWS SDK v2 2026-01-03 04:08:40 +01:00
Niko Storni 1b79371f8b major linting
code cleanup
readme update
vulnerability fix
2025-10-11 04:30:30 +02:00
Niko Storni cbcc611196 deterministic imports 2025-08-29 16:27:14 +02:00
Niko 68c0fe3b97 Merge pull request #68 from lbryio/modular_reflector
Modular blobcache stores (huge changes)
2025-08-22 02:22:03 +02:00
Niko Storni 8bfe7e101d fix tests
fix build
2025-08-22 02:09:55 +02:00
Niko Storni 7b3cca45b4 update readme
remove dead code
2025-08-22 01:56:39 +02:00
Niko Storni ad710dfeea fix livelock
adjust API endpoint
2025-08-22 01:34:56 +02:00
Niko Storni ebc52245f3 fix viper for real 2025-05-14 01:17:13 +02:00
Niko Storni b098aa2dc8 stupid viper... 2025-05-14 00:58:37 +02:00
Niko Storni ddd23f9a36 fix config loading bug 2025-05-14 00:25:48 +02:00
Niko Storni 4928e430c0 fix bugs 2025-05-13 22:52:40 +02:00
Niko Storni d21b0867dd remove unused cmds
change conf param
fix upload cmd
2025-05-13 22:30:40 +02:00
Niko Storni 32a85a815e adjust s3 configuration
hardcode multiwriter for now
2025-05-13 18:01:28 +02:00
Niko Storni 9cdac902bd add multiwriter store 2025-05-13 14:47:19 +02:00
Niko Storni 8ec01bf1a7 fix tests 2025-05-13 14:37:51 +02:00
Niko Storni ba8d1ba67b add s3 sharding 2025-05-13 14:08:33 +02:00
Niko Storni edecf4f988 delegate cascading for blocker and missing blobs checker 2025-05-13 13:55:54 +02:00
Niko Storni 77836f9c83 prepare stores for reflector upload
create a new reflector cmd that will replace old reflector cmd
2025-05-13 05:54:46 +02:00
Niko Storni 9716e41d63 replace hub servers with valid ones
change ITTT metrics
fix tracing for http store
remove unused launch params
2025-04-25 20:04:50 +02:00
Niko Storni 7260388255 make servers modular
fix bugs
2025-04-25 17:39:52 +02:00
Niko Storni 071a7907c1 Refactors store implementations for config loading
Updates store implementations to load configurations from YAML files.

This change introduces a configuration loader that reads store parameters from a YAML file, allowing for more flexible and manageable store configurations.
2025-04-24 05:11:52 +02:00
Niko Storni 185900582d add support for sharding in http stores 2025-04-19 04:21:46 +02:00
Niko Storni 03445476ec rename stores to better reflect what they do
move cleanup code in db store
fix bugs
refactor code
update go
2025-04-19 03:49:54 +02:00
Niko Storni e5a9590cb6 make s3 endpoint configurable
update deps
2024-12-16 22:35:50 +01:00
Niko Storni de736b068d remove outdated linter
remove redundant imports
remove deprecated and implicit rand seed init
adjust test
2024-04-09 20:00:46 +02:00
Niko Storni 14fbd2ae27 update dependencies 2024-04-09 19:17:32 +02:00
Niko Storni 4d81a43a8f update dependencies 2023-10-11 21:01:37 +02:00
Niko Storni b3f0d63b4d fix bug 2023-10-11 20:52:13 +02:00
Niko Storni c880f0b80f fix bug rendering sf useless 2023-10-11 20:45:31 +02:00
Niko Storni 085490e92b upgrade go version
update dependencies
2023-07-07 15:34:22 +02:00
Niko 456fe53e01 Merge pull request #67 from lbryio/upgrades
upgrade dependencies & introduce github ci
2023-03-09 13:12:59 -05:00
Niko Storni 778fc17adf update readme 2023-03-09 18:59:21 +01:00
Niko Storni e93c097fd9 upgrade dependencies
replace deprecated function calls
refactor build process
2023-03-09 18:41:41 +01:00
Niko 0dfda70c70 Merge pull request #63 from lbryio/declare_err
Add undeclared errors from peer.go to metrics
2022-10-17 19:01:56 +02:00
Victor Shyba 6c082993cf metrics: declare request is too large 2022-10-14 15:21:06 -03:00
Victor Shyba 08ed3c9f13 metrics: declare invalid blob hash len 2022-10-14 15:18:46 -03:00
Victor Shyba 7f75602841 metrics: declare protected blob 2022-10-14 15:16:54 -03:00
Victor Shyba 4d168ddefc metrics: declare invalid json request 2022-10-14 15:15:32 -03:00
Niko b7abb77ea1 Merge pull request #62 from lbryio/declare_err
declare errInvalidPeerData on metrics
2022-10-14 19:45:40 +02:00
Victor Shyba 8c2a46752c declare errInvalidPeerData on metrics 2022-10-14 14:42:22 -03:00
Niko Storni 0c8da4abe5 Merge remote-tracking branch 'origin/peer_proto_tests' 2022-10-14 18:30:08 +02:00
Victor Shyba 87680d806c check for valid data earlier. reply composite as tests are expecting 2022-10-11 22:58:41 -03:00
Victor Shyba e1c59b9b63 test through the connection 2022-10-11 22:57:52 -03:00
Niko Storni c79634706b ignore server closed errors the right way 2022-09-23 14:46:55 +02:00
Niko Storni f5d30b1a6e update quic-go 2022-07-30 20:10:28 +02:00
Niko Storni 0177dd4ce0 fix query param 2022-07-29 05:40:50 +02:00
Niko Storni 5693529216 protect protected content 2022-07-29 04:59:15 +02:00
Niko a1c2e92ca3 Merge pull request #57 from andybeletsky/fix-store-macos
Do not use direction package on macos
2022-05-12 16:01:00 +02:00
Andrey Beletsky 9c0554ef05 Do not use direction package on macos 2022-05-08 15:39:34 +07:00
Niko Storni 4e80f91a57 fix memory leak 2022-05-04 19:14:20 +02:00
Niko Storni c211f83ba7 update dependencies 2022-05-02 23:32:36 +02:00
Niko Storni 29d1ccf68c add singleflight to web requests 2022-05-02 23:07:22 +02:00
Niko Storni 2f7d67794f remove debug leftover 2022-02-10 00:57:13 +01:00
Niko Storni 4d8e7739d7 Merge branch 'fix-ci' 2021-12-14 22:36:01 +01:00
Niko Storni 6fc0ceea2a adjust travis vars
fix build script


fix more scripts


adapt script to scale
2021-12-14 22:35:44 +01:00
Niko Storni ae0c7dd2bb upgrade quic-go for go1.17 2021-12-14 20:49:50 +01:00
Niko Storni 4af5c2f4c6 make slack channel configurable 2021-10-30 00:21:58 +02:00
Alex Grin def0a97f49 Update readme.md 2021-09-28 10:15:24 -04:00
Niko Storni 6dde793745 do not delete blobs for blocked content 2021-09-21 18:00:55 +02:00
Niko Storni 654cc44935 Merge branch 'hash_twice' 2021-09-21 16:21:25 +02:00
Victor Shyba 90d6d29452 remove all hashing on the download path 2021-09-21 16:19:49 +02:00
Niko Storni b2272fef3a delete overflowing blobs from underlying cache 2021-08-18 19:36:47 +02:00
Niko Storni 86f3e62aa8 fix a panic error
update gin-go
2021-08-05 17:47:20 +02:00
Niko e1b4f21e00 Merge pull request #52 from lbryio/ittt
Merge months of work including all sort of caches and O_DIRECT optimizations
2021-07-23 20:35:22 -04:00
Niko Storni b4913ecedf cleanup 2021-07-24 01:03:51 +02:00
Niko Storni 63a574ec2f unify caches
fix tests
2021-07-24 01:03:51 +02:00
Niko Storni b8af3408e0 move server packages 2021-07-24 01:03:51 +02:00
Niko Storni 847089d0d6 fix error propagation
update readme
2021-07-24 01:03:51 +02:00
Niko Storni 170dfef3a8 fix copy pasta mistake 2021-07-24 01:03:51 +02:00
Niko Storni 2b458a6bd0 fix params
more cleanups
2021-07-24 01:03:51 +02:00
Niko Storni febfc51cb0 refactor refactor refactor 2021-07-24 01:03:51 +02:00
Andrey Beletsky 72be487262 Fix broken import 2021-07-24 01:03:51 +02:00
Andrey Beletsky 94e7d81bd3 Fix OpenFile call flags for macos 2021-07-24 01:03:51 +02:00
Niko Storni c6c779da39 fix panic
fix counter leak
2021-07-24 01:03:51 +02:00
Niko Storni 2e101083e6 write blobs to tmp dir to avoid corruption 2021-07-24 01:03:51 +02:00
Niko Storni 63aacd8a69 use O_DIRECT to write to disk (fixes everything)
add queue back to serving blobs
improve a lot of things
upgrade modules
2021-07-24 01:03:51 +02:00
Niko Storni c03ae6487d fix unsafe dereference 2021-07-24 01:03:51 +02:00
Niko Storni 0c4f455f0c add metrics 2021-07-24 01:03:51 +02:00
Niko Storni af3e08c446 update lbry.go dep 2021-07-24 01:03:51 +02:00
Niko Storni 975bfe7fac upgrade singleflight
http store fix
2021-07-24 01:03:51 +02:00
Niko Storni b075d948bb remove locks causing deadlocks 2021-07-24 01:03:51 +02:00
Niko Storni 2651a64dbb add http server/client 2021-07-24 01:03:51 +02:00
Mark Beamer Jr fa7150cf2b Add queue to prevent writing too many files at once. 2021-07-24 01:03:51 +02:00
Mark Beamer Jr 6c4db980c9 Add queue to prevent writing too many files at once. 2021-07-24 01:03:51 +02:00
Mark Beamer Jr 7adaa510fd Add locks to disk store. 2021-07-24 01:03:51 +02:00
Niko Storni 64ed7304f6 add a lot of extra heavy debugging 2021-07-24 01:03:51 +02:00
Mark Beamer Jr 5aefaf061e Add single flight for cache not just origin 2021-07-24 01:03:51 +02:00
Mark Beamer Jr 724ee47c8b add metric calls for other packages 2021-07-24 01:03:51 +02:00
Mark Beamer Jr caaec6fcb1 add guage metrics for go routines in reflector package 2021-07-24 01:03:51 +02:00
Mark Beamer Jr 15984b8fd9 add gops to reflector server 2021-07-24 01:03:51 +02:00
Niko Storni 2be913b077 request queue size param 2021-07-24 01:03:51 +02:00
Niko Storni 34c11b0a0e increase window size 2021-07-24 01:03:51 +02:00
Niko Storni 64acdc29c3 improve disk cleanup
add index to is_stored
fix test
replace LRU cache
2021-07-24 01:03:51 +02:00
Niko Storni 598773c90d fix mess with lbry.go 2021-07-24 01:03:47 +02:00
Niko Storni 766238fd7e add if this than that store
switch to wasabi for uploads
2021-07-24 01:03:47 +02:00
Niko Storni ac5242f173 add integrity check cmd
throttle live integrity checks
bug fixes
2021-07-24 01:03:47 +02:00
Mark Beamer Jr 215103cb33 use wait group not stopper 2021-07-24 01:03:47 +02:00
Mark Beamer Jr ed3622d0a6 Wait for request to be handled before returning 2021-07-24 01:03:47 +02:00
Mark Beamer Jr 848fce5afa Add request queue for blob cache 2021-07-24 01:03:47 +02:00
Niko Storni e37eeba0c9 check blobs when reading them 2021-07-24 01:03:47 +02:00
Niko Storni 7da49a4ccb upgrade quic-go
add cache for blobs not found
2021-07-24 01:03:47 +02:00
Niko Storni 7b02ace5e2 fix issues caused by beamer's renaming 2021-07-24 01:03:47 +02:00
Niko Storni 5fb67b32db run go mod tidy 2021-07-24 01:03:47 +02:00
Niko Storni a0c9ed2ace make it simpler 2021-07-24 01:03:47 +02:00
Niko Storni 998b082a06 remove panics 2021-07-24 01:03:47 +02:00
Niko Storni 36d4156e2a add tracing to blobs 2021-07-24 01:03:47 +02:00
Niko Storni 74925ebba2 optimize batch insertions
reduce touch time to every 6 hours
2021-07-24 01:03:47 +02:00
Alex Grintsvayg 6f95b3395f avoid heavy interpolateparams call 2021-07-24 01:03:47 +02:00
Alex Grintsvayg dff00e2317 fix long query 2021-07-24 01:03:47 +02:00
Alex Grintsvayg 9a5d9d7ff5 only touch blobs when you get them 2021-07-24 01:03:47 +02:00
Niko Storni 5794c57898 save uploaded blobs and work around the blocklist issue 2021-07-24 01:03:47 +02:00
Niko Storni 35c713a26e add cmd to populate db
fix store init
try fixing unreasonable db bottleneck
2021-07-24 01:03:47 +02:00
Alex Grintsvayg 6fb0620091 something like this 2021-07-24 01:03:38 +02:00
Niko Storni 03df751bc7 add PoC for litedb to avoid all the overhead 2021-07-24 01:03:16 +02:00
Niko Storni c902858958 address some review comments 2021-07-24 01:03:16 +02:00
Niko Storni 84fabdd5f4 add option to run with RO-CF only as upstream
increase idle timeout to avoid errors downstream
add option to delete blobs from DB if storage doesn't have it (for future local tracking)
2021-07-24 01:03:16 +02:00
Niko Storni f5cad15f84 upgrade quic 2021-07-24 01:03:10 +02:00
Niko Storni dd3d0ae42c update lfuda library 2021-07-24 01:03:10 +02:00
Niko Storni 0b565852b8 only store the blobs in the underlying storage if LFUDA accepted them 2021-07-24 01:03:10 +02:00
Niko Storni ff13d7b2f7 fix cache size mess 2021-07-24 01:03:10 +02:00
Niko Storni 7f5a89fa5a fix buffer cache running out of space 2021-07-24 01:03:10 +02:00
Niko Storni 704e15f8c1 use LFUDA store
swap size to bytes
2021-07-24 01:03:10 +02:00
Niko Storni 5eb1f13b54 add LFUDA store
update quic
fix tests
2021-07-24 01:03:10 +02:00
Alex Grintsvayg 176e05714e rename cahces 2021-07-24 01:03:10 +02:00
Niko Storni eefd84b02d add buffer cache for nvme drive 2021-07-24 01:03:10 +02:00
Niko Storni af2742c34f update quic
don't wait for a blob to be written to disk before sending it downstream
don't wait for the disk store to be walked before starting everything up
2021-07-24 01:03:10 +02:00
Alex Grintsvayg 2cf4acdb59 add 'send' command to efficiently send a file to reflector 2021-04-02 14:30:36 -04:00
Alex Grintsvayg 044e2fe5d7 Merge branch 'fix_mem_leak'
* fix_mem_leak:
  dont over-allocate ram when reading blobs via QUIC
  dont overallocate ram when reading blobs from disk
2020-11-27 16:18:44 -05:00
Alex Grintsvayg fb77bf621e dont over-allocate ram when reading blobs via QUIC 2020-11-27 16:18:26 -05:00
Alex Grintsvayg e70b9af3e4 dont overallocate ram when reading blobs from disk
ReadFile checks the file size and allocates a bit more space than we
expect we'll need. ReadAll uses Go's standard resizing algo, which
doubles the underlying array each time you hit the end. So ReadAll
ends up allocating 4MB for a full blob, while ReadFile allocates
slightly over 2MB.
2020-11-27 16:18:26 -05:00
Alex Grintsvayg 5df05cf46f ensure autodeploy is off 2020-11-23 12:41:09 -05:00
Alex Grintsvayg 7bddcf01b8 Merge branch 'smarter_caches'
* smarter_caches:
  use speedwalk for faster file listing
  remove afero fs abstraction in prep for using speedwalk
  add faster file.Walk fn. meant for DiskStore.list()
  clarify Get() error requirement
  separate singleflight cache wrapper, component names for cache metrics
  apparently the normal check doesn't work
  split cloudfront into RO and RW stores
  add noop store that does nothing
  add test for DiskStore.list()
  add lru cache eviction metric
  rename the stores, add caching to reflector cmd
  separate disk and lru behavior
  LRU cache for disk store, abstract fs in disk store for testing
2020-11-04 16:04:16 -05:00
Alex Grintsvayg 659a6e73cc use speedwalk for faster file listing 2020-11-04 16:03:47 -05:00
Alex Grintsvayg aaae3ffa5b remove afero fs abstraction in prep for using speedwalk 2020-11-04 16:03:47 -05:00
Alex Grintsvayg 131fed28d2 add faster file.Walk fn. meant for DiskStore.list() 2020-11-04 16:03:47 -05:00
Alex Grintsvayg 72571236ab clarify Get() error requirement 2020-11-04 16:03:46 -05:00
Alex Grintsvayg 560e180e36 separate singleflight cache wrapper, component names for cache metrics 2020-11-04 16:03:46 -05:00
Alex Grintsvayg 070c378dfd apparently the normal check doesn't work 2020-11-04 16:03:46 -05:00
Alex Grintsvayg 124d4065c2 split cloudfront into RO and RW stores 2020-11-04 16:03:45 -05:00
Alex Grintsvayg f131c1f35b add noop store that does nothing 2020-11-04 16:03:45 -05:00
Alex Grintsvayg 7a3225434e add test for DiskStore.list() 2020-11-04 16:03:45 -05:00
Alex Grintsvayg 3608971f0b add lru cache eviction metric 2020-11-04 16:03:44 -05:00
Alex Grintsvayg c9fa04043c rename the stores, add caching to reflector cmd 2020-11-04 16:03:44 -05:00
Alex Grintsvayg c6b53792c8 separate disk and lru behavior 2020-11-04 16:03:44 -05:00
Alex Grintsvayg 69fa06420b LRU cache for disk store, abstract fs in disk store for testing 2020-11-04 16:03:43 -05:00
Niko Storni 5cb1365903 increase idle timeout 2020-10-19 23:42:40 +02:00
Niko Storni e430c2fd40 bump go version 2020-10-15 03:43:03 +02:00
Niko 66024716ac Merge pull request #45 from lbryio/thundering_herd
ensure only single origin Get request is in flight per hash. protects against thundering herd.
2020-10-15 03:16:34 +02:00
Niko Storni f043516a14 revert upload changes 2020-10-15 03:08:27 +02:00
Niko Storni c3db95a6c1 add more metrics
increase handshake timeout by 1 second
2020-10-15 02:59:12 +02:00
Niko Storni 08c93d44fd update quic lib 2020-10-15 00:13:16 +02:00
Alex Grintsvayg b02e80d472 ensure only single origin Get request is in flight per hash. protects against thundering herd. 2020-10-15 00:12:31 +02:00
Niko Storni f1875454cc store uploaded blobs 2020-10-15 00:12:18 +02:00
Alex Grintsvayg a80e0f5b0d change error msg 2020-10-06 10:17:50 -04:00
Alex Grintsvayg 2b3581a692 silence all those quic errors 2020-10-06 10:11:36 -04:00
Alex Grintsvayg a0f78028cc handle the case where last_accessed_at is null 2020-10-06 09:39:43 -04:00
Alex Grintsvayg a084330055 drop mediainfo requirement since travis cant install it 2020-10-05 18:28:41 -04:00
Alex Grintsvayg ea80ed6506 install mediainfo during build 2020-10-05 18:22:33 -04:00
Alex Grintsvayg 4b335ed692 bumping to 1.15 for the quic-go dependency 2020-10-05 18:15:02 -04:00
Alex Grintsvayg 6118dde36c Merge branch 'track_access'
* track_access:
  small changes for easier testing
  track approximate access time for blobs
2020-10-05 18:10:49 -04:00
Alex Grintsvayg 8364d3fc54 small changes for easier testing 2020-10-05 18:08:54 -04:00
Alex Grintsvayg de0ccd4da7 track approximate access time for blobs 2020-10-05 18:08:53 -04:00
Alex Grintsvayg 31f9346027 Merge branch 'publish_from_go'
* publish_from_go:
  add publish command
2020-10-05 18:08:22 -04:00
Alex Grintsvayg 5658fe4607 add publish command 2020-10-05 16:30:57 -04:00
Niko Storni fc5f5ff7d3 add cloudfront support 2020-09-09 00:18:07 +02:00
Niko Storni e8b98bc862 update dependencies 2020-09-01 20:15:34 +02:00
Niko Storni f458529c74 fix memory leak? 2020-07-14 00:20:58 +02:00
Niko Storni 150b1f6f1f fix travis 2020-07-10 15:24:48 +02:00
Niko Storni 47f28002ff close unclosed handle
reduce idle timeout for http3
update QUIC library
2020-07-10 15:19:57 +02:00
Niko beff466d18 Merge pull request #43 from lbryio/http3
Add/replace peer protocol with HTTP3/QUIC protocol
see previous commits for more details
2020-07-09 15:17:35 +02:00
Niko Storni 694bda105c add metrics 2020-07-09 15:02:32 +02:00
Niko Storni 34ca7847d0 return a better descriptive error when the blob isn't found 2020-07-09 15:02:32 +02:00
Niko Storni 5c91051b78 disable disk cleanup routine 2020-07-09 15:02:32 +02:00
Alex Grintsvayg df4f42db82 successfully shut down wallet server 2020-07-09 15:02:32 +02:00
Alex Grintsvayg 41d758ef5c test wallet server connection 2020-07-09 15:02:32 +02:00
Niko Storni 264390a2b2 add debug code 2020-07-09 15:02:32 +02:00
Niko Storni fdcc41829a handshake changes 2020-07-09 15:02:32 +02:00
Niko Storni 09c7718f30 refactor code 2020-07-09 15:02:32 +02:00
Niko Storni 8a5f57b14f remove QUIC protocol
add HTTP3/QUIC protocol
2020-07-09 15:02:32 +02:00
Niko Storni 90997b9918 use new connections for each blob 2020-07-09 15:02:32 +02:00
Niko Storni e0da2674a1 use new connections for each action
reduce timeout
2020-07-09 15:02:32 +02:00
Niko Storni a80599413c improve params description 2020-07-09 15:02:32 +02:00
Niko Storni 3ffe7a10c7 add other reflector store
add flags
improve disk cleanup
2020-07-09 15:02:32 +02:00
Andrey Beletsky 1bf3cb81b3 Use ModTime on systems that don't provide Atim file stat field 2020-07-09 15:02:32 +02:00
Mark Beamer Jr 4a9f127ecc Add gops to reflector 2020-07-09 15:02:32 +02:00
Mark Beamer Jr de1fb63a1c fix potentially missing client or stream for a quic store 2020-07-09 15:02:32 +02:00
Mark Beamer Jr e98794e125 Add close function for quic store so routines do not hang waiting for timeout. 2020-07-09 15:02:32 +02:00
Niko Storni fb0004bac4 increase packet size
add retro-compatibility
remove unused SPV servers
remove travis test failure
2020-07-09 15:02:32 +02:00
Andrey Beletsky 1f12b1d5d8 Add dockerfile 2020-05-06 00:36:35 +07:00
Andrey Beletsky bc95ca61d5 Merge pull request #40 from lbryio/delete-after-upload
add flag to delete blobs after upload
2020-05-05 13:36:41 +07:00
Alex Grintsvayg be69c2f05c add flag to delete blobs after upload 2020-05-02 14:31:10 -04:00
Alex Grintsvayg 4a902597df thats confusing 2020-03-30 16:05:08 -04:00
Alex Grintsvayg dd528f03f6 expose raw wallet server request fn 2020-03-30 15:57:53 -04:00
Alex Grintsvayg e6ba61fce2 more info on json errors 2020-03-20 10:15:41 -04:00
Niko Storni e03abad012 Merge branch 'disk-cache' 2020-03-06 11:24:33 -05:00
Alex Grintsvayg dde93a1fe6 make cache dir a cli flag 2020-02-27 14:53:33 -05:00
Niko Storni 4a5a148843 implement disk cleanup 2020-02-27 14:53:33 -05:00
Niko Storni cb6443fa1e update spv servers 2020-02-27 14:53:27 -05:00
Alex Grintsvayg 1a1f991e48 more error traces 2020-02-26 10:02:53 -05:00
Alex Grintsvayg d1063bd54e log full traces more 2020-02-25 18:22:22 -05:00
Alex Grintsvayg d291c063ec add traces to unmarshall errors 2020-02-25 15:49:51 -05:00
Thomas Zarebczan a939529d19 Merge pull request #35 from ykris45/patch-1
Update LICENSE
2020-02-03 17:11:25 -05:00
YULIUS KURNIAWAN KRISTIANTO cf34547b8a Update LICENSE 2020-02-03 05:58:24 +07:00
Alex Grintsvayg 6631ad325f more errors 2020-01-13 16:16:46 -05:00
Alex Grintsvayg df266f6194 track hash mismatches 2020-01-07 08:38:40 -05:00
Alex Grintsvayg b3b581c00e more granular errors 2020-01-03 14:02:00 -05:00
Alex Grintsvayg 18b15674d9 warn on untracked error 2020-01-03 12:28:01 -05:00
Alex Grintsvayg 780e899e90 more specific errors 2020-01-03 10:27:29 -05:00
Alex Grintsvayg 5c5643749e i/o timeout is not the same as context.DeadlineExceeded 2020-01-02 13:58:11 -05:00
Alex Grintsvayg 11e50a6022 remove unused stats code 2020-01-02 13:27:34 -05:00
Alex Grintsvayg 1603b3bb22 fix circular import 2020-01-02 13:22:51 -05:00
Alex Grintsvayg 5465527faf Merge branch 'metrics'
* metrics:
  use labels for different error types
  switch to prometheus for metrics
2020-01-02 13:13:29 -05:00
Alex Grintsvayg 5d8a2d697c use labels for different error types 2020-01-02 13:12:33 -05:00
Alex Grintsvayg 50089481fb switch to prometheus for metrics 2020-01-02 13:12:33 -05:00
Alex Grintsvayg d6cf9e9e63 new uber-go import path 2019-12-28 19:24:26 -05:00
Alex Grintsvayg dc6dd8d12b fix blocklist, log future blocklist errors, add resolve wallet server method 2019-12-28 19:17:52 -05:00
Alex Grintsvayg ad8d623863 work around shitty wallet server error response 2019-12-28 19:14:24 -05:00
sayplastic d89b58c52a Merge pull request #31 from lbryio/lbry_tv
Put back printf statement into the stats module
2019-12-17 00:36:56 +07:00
Andrey Beletsky 2aa40850b0 Put back printf statement into the stats module 2019-12-17 00:33:34 +07:00
sayplastic ca24c2be28 Merge pull request #30 from lbryio/lbry_tv
upgrade to lbry.go v2.4.0
2019-12-16 22:53:00 +07:00
Andrey Beletsky 52127eee7c Remove setting default client timeout in peer.NewStore 2019-12-16 22:47:35 +07:00
Andrey Beletsky 95eb94f5a7 Remove excessive INFO-level logging 2019-12-16 21:52:51 +07:00
Andrey Beletsky 834733b675 Add options for peer.NewStore to allow for setting TCP timeout 2019-11-22 18:56:20 +07:00
Mark Beamer Jr 86a553b876 upgrade to lbry.go v2.4.0 2019-11-13 19:11:35 -05:00
Alex Grin e5438713ce Merge pull request #27 from StrikerRUS/patch-2
bump year in license
2019-10-09 10:37:39 -04:00
Nikita Titov f6c4b7d36b bump year in license 2019-10-08 23:32:22 +03:00
Alex Grintsvayg 661c20a21d make db-backed store more generic (not specific to s3) 2019-10-03 16:58:17 -04:00
Alex Grintsvayg 69f1e0f4ca make MemoryStore consistent with the New...() pattern 2019-10-03 16:49:28 -04:00
Alex Grintsvayg 2ca83139df use stream.Blob for BlobStore interface 2019-10-03 16:34:57 -04:00
Alex Grintsvayg 0af6d65d40 rename FileStore -> DiskStore 2019-10-03 16:24:59 -04:00
Alex Grintsvayg a8230db802 rewrite getstream command using caching store 2019-10-03 16:13:08 -04:00
Alex Grintsvayg 24f885e268 store blobs on disk in prefix-based subdirectories to avoid too many files in one dir 2019-10-03 16:12:49 -04:00
Alex Grintsvayg acb9840871 add peer blob store, which gets blobs from a peer 2019-10-03 16:10:29 -04:00
Alex Grintsvayg a745bafc58 go 1.13 2019-10-03 14:12:42 -04:00
Alex Grintsvayg bf9e90f14c comment out github releasing for clarity 2019-10-03 14:02:00 -04:00
Alex Grintsvayg a98d1e1217 tidy go.mod 2019-10-03 13:41:02 -04:00
Alex Grintsvayg 36ee7e8d1f add caching blob store 2019-10-03 13:36:35 -04:00
Alex Grintsvayg c1e8e7481f fix getstream command to write as it downloads (or else it runs out of memory) 2019-09-27 14:38:49 -04:00
Alex Grintsvayg cb669eb1a7 add claim decode command 2019-09-11 14:35:25 -04:00
Alex Grintsvayg 08df3b167c add getstream command to download a stream from a peer 2019-09-11 13:28:58 -04:00
Alex Grintsvayg 1a6b862c96 error on invalid blob hash length 2019-09-10 17:18:44 -04:00
94 changed files with 8355 additions and 1414 deletions

No files matched your search

+37
View File
@@ -0,0 +1,37 @@
name: Go
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v4
with:
go-version: 1.25.x
- name: Build linux
run: make linux
- name: Build macos
run: make macos
- name: Test
run: make test
- name: Lint
run: make lint
- name: retrieve all tags
run: git fetch --prune --unshallow --tags
- name: Print changes since last version
run: git log $(git describe --tags --abbrev=0)..HEAD --no-merges --oneline
+62
View File
@@ -0,0 +1,62 @@
name: release
on:
push:
tags:
- "*.*.*"
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v4
with:
go-version: 1.25.x
- name: Build linux
run: make linux
- name: Build macos
run: make macos
- name: Test
run: make test
- name: Lint
run: make lint
- name: Zip macos
run: zip -r reflector_darwin_amd64.zip ./dist/darwin_amd64
- name: Zip linux
run: zip -r reflector_linux_amd64.zip ./dist/linux_amd64
- name: retrieve all tags
run: git fetch --prune --unshallow --tags
- name: Generate Changelog
run: git log $(git describe --tags --abbrev=0 @^)..@ --no-merges --oneline > ${{ github.workspace }}-CHANGELOG.txt
- name: upload to github releases
uses: softprops/action-gh-release@v1
with:
files: |
./reflector_linux_amd64.zip
./reflector_darwin_amd64.zip
body_path: ${{ github.workspace }}-CHANGELOG.txt
# - name: Login to DockerHub
# uses: docker/login-action@v2
# with:
# username: ${{ secrets.DOCKERHUB_USERNAME }}
# password: ${{ secrets.DOCKERHUB_TOKEN }}
# - name: Generate docker image
# run: make image
# - name: Docker push
# run: make publish_image
+9
View File
@@ -1,3 +1,12 @@
/vendor
/config.json*
/dist
/bin
blobcache.yaml
reflector.yaml
upload.yaml
remotereflector.yaml
+57
View File
@@ -0,0 +1,57 @@
run:
timeout: 5m
tests: false
modules-download-mode: readonly
linters-settings:
gci:
# Checks that no inline Comments are present.
# Default: false
no-inline-comments: false
# Checks that no prefix Comments(comment lines above an import) are present.
# Default: false
no-prefix-comments: false
# Skip generated files
skip-generated: true
# Use custom order
custom-order: true
sections:
- standard
- prefix(github.com/lbryio/reflector.go)
- prefix(github.com/OdyseeTeam,github.com/lbryio)
- default
gofmt:
simplify: true
govet:
enable-all: true
misspell:
locale: US
revive:
rules:
- name: exported
disabled: true
staticcheck:
checks: ["all"]
linters:
enable:
- errcheck
- gosimple
- govet
- ineffassign
- gci
- gofmt
- goimports
- misspell
- revive
- unconvert
- whitespace
- typecheck
disable:
- unused
- unparam
- staticcheck
-83
View File
@@ -1,83 +0,0 @@
os: linux
dist: trusty
language: go
env:
- GO111MODULE=on
go:
- 1.11.x
cache:
directories:
- $HOME/.cache/go-build
- $HOME/gopath/pkg/mod
notifications:
email: false
# Skip the install step. Don't `go get` dependencies. Only build with the code in vendor/
install: true
# Anything in before_script that returns a nonzero exit code will
# flunk the build and immediately stop. It's sorta like having
# set -e enabled in bash.
before_script:
# All the .go files, excluding vendor/ and model (auto generated)
- GO_FILES=$(find . -iname '*.go' ! -iname '*_test.go' -type f | grep -v /vendor/ )
- go get golang.org/x/tools/cmd/goimports # Used in build script for generated files
# - go get github.com/golang/lint/golint # Linter
# - go get honnef.co/go/tools/cmd/megacheck # Badass static analyzer/linter
- go get github.com/jgautheron/gocyclo # Check against high complexity
- go get github.com/mdempsky/unconvert # Identifies unnecessary type conversions
- go get github.com/kisielk/errcheck # Checks for unhandled errors
- go get github.com/opennota/check/cmd/varcheck # Checks for unused vars
- go get github.com/opennota/check/cmd/structcheck # Checks for unused fields in structs
# script always run to completion (set +e). All of these code checks are must haves
# in a modern Go project.
script:
# Fail if a .go file hasn't been formatted with gofmt
- test -z $(gofmt -s -l $GO_FILES)
# Run unit tests
- make test
# Checks for unused vars and fields on structs
- varcheck ./...
- structcheck ./...
# go vet is the official Go static analyzer
- go vet ./...
# forbid code with huge functions
#- gocyclo -ignore "_test.go" -avg -over 19 $GO_FILES
# checks for unhandled errors
# - errcheck ./... # COMMENTED OUT UNTIL https://github.com/kisielk/errcheck/issues/155 is fixed
# "go vet on steroids" + linter - ignore autogen code
# - megacheck -simple.exit-non-zero=true ./... # DISABLED UNTIL https://github.com/dominikh/go-tools/issues/328 is fixed
# check for unnecessary conversions - ignore autogen code
# - unconvert ./... # SEEMS TO BE BROKEN WITH GO MODULES
# one last linter - ignore autogen code
#- golint -set_exit_status $(go list ./... | grep -v /vendor/ )
# Finally, build the binary
- make
deploy:
- provider: s3
local_dir: ./bin
skip_cleanup: true
on:
repo: lbryio/reflector.go
access_key_id:
secure: "ABmoSTxTee1GubJmmi+MlyBcnqRT0ywEOtTL4hrH7T+Vj5UPcNBhjqGRiMXzO3MdrVWom19OnqMvsvAdehrB4uQJVlX/+zB26gTpEiiU5dWJgerNAYmfxDGNz/p9pyYWLvVY/cymSlJ4HkggJVg+dMTBTOP+gTNgQUhHL7pAd/Z5HMPmPW/rlNlQgDdczoY44OdiniuHny4Lr7mo8dcT4dcyJysPpHLCPGWp3kXcffIEbgO02aQZ14BbX9oU9xm1BacYnr4t8gRCU+uK0mt7TREHUzZ8bQBRXUymmpeeEQshXdifdwtk6sdRGIZiTi0NmQ7kYter6oh6623TH6rpNy/lkxKi823FbSPC1XZGH7CwcbnSIEwY++7e9PT0EA14VuDUF7+iXC/gpRkVQwUoMcoC9rHBjkXW+SiwHWoUeDKXrfVO9QGEJTi5bDRNwklwu2zkYWeRIPzfDaVayUBYwjunjT1eST997ygmJwJTJ6DeOWLx+WFlmwBJfQpBwbrlMrceiNjreMwGB6ffXEBzuM6S5inXGIChllGDpNyJeFVcH9zPpU0JsIpWoRjwLNFOiEhFsnzK5rvA2SKmlOOnSlgOCBQnFi2SmLDM790KoY8qjan2s984chQz2qsKnhYuVpJhG6LKgrI1QwE0KHr80spVFfewq11AU4Q+lfA9aAQ="
secret_access_key:
secure: "AxYRTy/GnjeTJKQdeJ/AEeAd+yXs783bFDKdyKNswtsHlU8sWPQgNcvTLpVqnSQMpiwkGDGi/70rvR5C+AT3SIWNw13RYrgBRpduQU0J+B2JS+3dN2DIePu25uvs++Wo22OfS8I+UjZ1mWY1SSHI2spPXvDCq5tb+Ih8nlYflEyAtxU9Oq2R3Kus2tkIlRnL25sP/2fY7RvuJFYIV63z8ZIJRzB5WzOeERqnXq2zfwos+hycAqyo/VevJnWAYTEDsvBuSODOpZF+QfKtIQ2rYSoqy8Lq1M6UOZimnC3Ulea4euBVf2ssBCnI7csGNG5UzkTiwrPDi2xIP8nM01rHW1yHJ7tQsJaghnUsfw2t6ui4ZofvbbOFTN/YCloHITifEi8Tc1/17isi3y+kX5yQ/Nk5UNry0Wbt91CP+nkL/ZmA5grkBXDL7VJMmB60TnO3ap24CtwBQartN3LoWs7h+4ov+LqbCt6IqpJVWQWlwJeb2MFPFByALtBpsqAyL1SxXlGNpPa94CuXxfQ6Bv436PtefA5FlTzR8uMmqsjWciv06bVnSvVlFEVovN2Fkplrjt7AASJ/8KJs4THDg4k61nfd8roAHx6ewQzl4wCWKCikQ0MuFd2mVHwdrbnCH1mIHuPRyvWMMIAK0ooc1/rmKiJlpgumjxoFYNE10MXtt+I="
bucket: "lbry-reflector-binary"
- provider: releases
file: ./bin/prism-bin
skip_cleanup: true
on:
repo: lbryio/reflector.go
tags: true
api_key:
secure: epAlhp3SUr8hhISarJ22n6tRw2TEa4s4oNFIvJUb5HGECVp1SYN7ao0ln5NoNLmfJS60pi911i/kMhhi21/uhZ0kCYlEhhIE2pc1zsiAxK9L9ENCssJ205HfVbe5grhwskLGzgjhU9OznO8WtmyOPWXr0it8M8RCTjx6rEC0A33Id3WMYyhP938Sj9CxEYeH4KS8wFvBXkgBVtrgaYwRTCIROFddHFXOb9jyNhqQ1RbfKtllsVtQhVk5WMlomheBNSS4vr6WMS4X4+2okFqnLtiSn1wrn5I/94UQbnrI1juVnQj0K+j32EyQbAOt4T2cLW3GtG0jhaYKyNMT9ycDCdVACPSDELlHWjeyoes9bnhUFftm6kDbQxwA1UsTF1yG8tMKXxBSmYyoT7qDloi6pBifZMrFXL61uTs6yhVB9LS/2oqg4sc0Ne87bRcn4OxsBeVCe3kbBHDTR/NTyF2gNPtRvgMAWULxTVcUm9VYdO0IWvAig5g4Row0DnFzEquD6CzezbRWD9WyZyV/AFyYHeeQ2PO7jTw0/3M7aDX33Fuhh34lehzmrC03cfgD/wZW+spxozIcQCYdiJqVw+u+/NvbNr0kkFzE9zW26JEmUFTyDvKxvnza1Kwtww3EgH6zaOL8r4yVbb54rePRvLw7pl93zlfJnEB2MCPqJOY5ZpU=
+9
View File
@@ -0,0 +1,9 @@
FROM alpine
EXPOSE 8080
RUN mkdir /app
WORKDIR /app
COPY dist/linux_amd64/prism-bin ./prism
RUN chmod +x prism
ENTRYPOINT [ "/app/prism" ]
+1 -1
View File
@@ -1,6 +1,6 @@
The MIT License (MIT)
Copyright (c) 2016-2018 LBRY Inc
Copyright (c) 2016-2020 LBRY Inc
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish,distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+30 -18
View File
@@ -1,25 +1,37 @@
version := $(shell git describe --dirty --always --long --abbrev=7)
commit := $(shell git rev-parse --short HEAD)
commit_long := $(shell git rev-parse HEAD)
branch := $(shell git rev-parse --abbrev-ref HEAD)
curTime := $(shell date +%s)
BINARY=prism-bin
DIR = $(shell cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)
BIN_DIR = ${DIR}/bin
IMPORT_PATH = github.com/lbryio/reflector.go
LDFLAGS="-X ${IMPORT_PATH}/meta.version=$(version) -X ${IMPORT_PATH}/meta.commit=$(commit) -X ${IMPORT_PATH}/meta.commitLong=$(commit_long) -X ${IMPORT_PATH}/meta.branch=$(branch) -X '${IMPORT_PATH}/meta.Time=$(curTime)'"
DIR = $(shell cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)
BIN_DIR = $(DIR)/dist
VERSION = $(shell git --git-dir=${DIR}/.git describe --dirty --always --long --abbrev=7)
LDFLAGS = -ldflags "-X ${IMPORT_PATH}/meta.Version=${VERSION} -X ${IMPORT_PATH}/meta.Time=$(shell date +%s)"
.PHONY: build clean test lint
.DEFAULT_GOAL: build
build:
mkdir -p ${BIN_DIR} && CGO_ENABLED=0 go build ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/${BINARY} main.go
clean:
if [ -f ${BIN_DIR}/${BINARY} ]; then rm ${BIN_DIR}/${BINARY}; fi
.DEFAULT_GOAL := linux
.PHONY: test
test:
go test ./... -v -cover
go test -cover -v ./...
.PHONY: lint
lint:
go get github.com/alecthomas/gometalinter && gometalinter --install && gometalinter ./...
./scripts/lint.sh
.PHONY: imports
imports:
gci write --skip-generated --custom-order -s standard -s "prefix(github.com/lbryio/reflector.go)" -s "prefix(github.com/OdyseeTeam,github.com/lbryio)" -s default .
.PHONY: linux
linux:
GOARCH=amd64 CGO_ENABLED=0 GOOS=linux go build -ldflags ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/linux_amd64/${BINARY}
.PHONY: macos
macos:
GOARCH=amd64 GOOS=darwin go build -ldflags ${LDFLAGS} -asmflags -trimpath=${DIR} -o ${BIN_DIR}/darwin_amd64/${BINARY}
.PHONY: image
image:
docker buildx build -t lbry/reflector:$(version) -t lbry/reflector:latest --platform linux/amd64 .
+9 -10
View File
@@ -1,14 +1,14 @@
package cluster
import (
"io/ioutil"
"io"
baselog "log"
"sort"
"time"
"github.com/lbryio/lbry.go/extras/crypto"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/stop"
"github.com/lbryio/lbry.go/v2/extras/crypto"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/hashicorp/serf/serf"
log "github.com/sirupsen/logrus"
@@ -23,13 +23,12 @@ const (
type Cluster struct {
OnMembershipChange func(n, total int)
s *serf.Serf
eventCh chan serf.Event
stop *stop.Group
name string
port int
seedAddr string
s *serf.Serf
eventCh chan serf.Event
stop *stop.Group
port int
}
// New returns a new Cluster instance that is not connected.
@@ -52,7 +51,7 @@ func (c *Cluster) Connect() error {
conf.MemberlistConfig.AdvertisePort = c.port
conf.NodeName = c.name
nullLogger := baselog.New(ioutil.Discard, "", 0)
nullLogger := baselog.New(io.Discard, "", 0)
conf.Logger = nullLogger
c.eventCh = make(chan serf.Event)
+55
View File
@@ -0,0 +1,55 @@
package cmd
import (
"os"
"os/signal"
"strconv"
"syscall"
"github.com/lbryio/reflector.go/config"
"github.com/lbryio/reflector.go/internal/metrics"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
func init() {
var cmd = &cobra.Command{
Use: "blobcache",
Short: "Run blobcache server",
Run: blobcacheCmd,
}
cmd.Flags().IntVar(&metricsPort, "metrics-port", 2112, "The port reflector will use for prometheus metrics")
cmd.Flags().BoolVar(&disableBlocklist, "disable-blocklist", false, "Disable blocklist watching/updating")
rootCmd.AddCommand(cmd)
}
func blobcacheCmd(cmd *cobra.Command, args []string) {
store, err := config.LoadStores(conf, "blobcache")
if err != nil {
log.Fatal(err)
}
defer store.Shutdown()
servers, err := config.LoadServers(store, conf, "blobcache")
if err != nil {
log.Fatal(err)
}
for _, s := range servers {
err = s.Start()
if err != nil {
log.Fatal(err)
}
defer s.Shutdown()
}
metricsServer := metrics.NewServer(":"+strconv.Itoa(metricsPort), "/metrics")
metricsServer.Start()
defer metricsServer.Shutdown()
interruptChan := make(chan os.Signal, 1)
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
<-interruptChan
}
+2 -1
View File
@@ -6,9 +6,10 @@ import (
"strconv"
"syscall"
"github.com/lbryio/lbry.go/extras/crypto"
"github.com/lbryio/reflector.go/cluster"
"github.com/lbryio/lbry.go/v2/extras/crypto"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
+55
View File
@@ -0,0 +1,55 @@
package cmd
import (
"encoding/hex"
"fmt"
"github.com/lbryio/lbry.go/v2/schema/stake"
"github.com/davecgh/go-spew/spew"
"github.com/gogo/protobuf/jsonpb"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
func init() {
var cmd = &cobra.Command{
Use: "decode VALUE",
Short: "Decode a claim value",
Args: cobra.ExactArgs(1),
Run: decodeCmd,
}
rootCmd.AddCommand(cmd)
}
func decodeCmd(cmd *cobra.Command, args []string) {
c, err := stake.DecodeClaimHex(args[0], "")
if err != nil {
log.Fatal(err)
}
m := jsonpb.Marshaler{Indent: " "}
if stream := c.Claim.GetStream(); stream != nil {
json, err := m.MarshalToString(stream)
if err != nil {
log.Fatal(err)
}
fmt.Println(json)
fmt.Printf("SD hash as hex: %s\n", hex.EncodeToString(stream.GetSource().GetSdHash()))
} else if channel := c.Claim.GetChannel(); channel != nil {
json, err := m.MarshalToString(channel)
if err != nil {
log.Fatal(err)
}
fmt.Println(json)
} else if repost := c.Claim.GetRepost(); repost != nil {
json, err := m.MarshalToString(repost)
if err != nil {
log.Fatal(err)
}
fmt.Println(json)
} else {
spew.Dump(c)
}
}
+2 -2
View File
@@ -8,8 +8,8 @@ import (
"syscall"
"time"
"github.com/lbryio/lbry.go/dht"
"github.com/lbryio/lbry.go/dht/bits"
"github.com/lbryio/lbry.go/v2/dht"
"github.com/lbryio/lbry.go/v2/dht/bits"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
+87
View File
@@ -0,0 +1,87 @@
package cmd
import (
"encoding/hex"
"os"
"time"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/stream"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
func init() {
var cmd = &cobra.Command{
Use: "getstream ADDRESS:PORT SDHASH",
Short: "Get a stream from a reflector server",
Args: cobra.ExactArgs(2),
Run: getStreamCmd,
}
rootCmd.AddCommand(cmd)
}
func getStreamCmd(cmd *cobra.Command, args []string) {
addr := args[0]
sdHash := args[1]
s := store.NewCachingStore(store.CachingParams{
Name: "getstream",
Cache: store.NewPeerStore(store.PeerParams{
Name: "getstream",
Address: addr,
Timeout: 30 * time.Second,
}),
Origin: store.NewDiskStore(store.DiskParams{
Name: "getstream",
MountPoint: "/tmp/lbry_downloaded_blobs",
ShardingSize: 2,
}),
})
wd, err := os.Getwd()
if err != nil {
log.Fatal(err)
}
var sd stream.SDBlob
sdb, _, err := s.Get(sdHash)
if err != nil {
log.Fatal(err)
}
err = sd.FromBlob(sdb)
if err != nil {
log.Fatal(err)
}
filename := sd.SuggestedFileName
if filename == "" {
filename = "stream_" + time.Now().Format("20060102_150405")
}
f, err := os.Create(wd + "/" + filename)
if err != nil {
log.Fatal(err)
}
for i := 0; i < len(sd.BlobInfos)-1; i++ {
b, _, err := s.Get(hex.EncodeToString(sd.BlobInfos[i].BlobHash))
if err != nil {
log.Fatal(err)
}
data, err := b.Plaintext(sd.Key, sd.BlobInfos[i].IV)
if err != nil {
log.Fatal(err)
}
_, err = f.Write(data)
if err != nil {
log.Fatal(err)
}
}
}
+93
View File
@@ -0,0 +1,93 @@
package cmd
import (
"crypto/sha512"
"encoding/hex"
"os"
"path"
"runtime"
"sync/atomic"
"time"
"github.com/lbryio/reflector.go/meta"
"github.com/lbryio/reflector.go/store/speedwalk"
"github.com/lbryio/lbry.go/v2/extras/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
var threads int
func init() {
var cmd = &cobra.Command{
Use: "check-integrity",
Short: "check blobs integrity for a given path",
Run: integrityCheckCmd,
}
cmd.Flags().StringVar(&diskStorePath, "store-path", "", "path of the store where all blobs are cached")
cmd.Flags().IntVar(&threads, "threads", runtime.NumCPU()-1, "number of concurrent threads to process blobs")
rootCmd.AddCommand(cmd)
}
func integrityCheckCmd(cmd *cobra.Command, args []string) {
log.Printf("reflector %s", meta.VersionString())
if diskStorePath == "" {
log.Fatal("store-path must be defined")
}
blobs, err := speedwalk.AllFiles(diskStorePath, true)
if err != nil {
log.Fatalf("error while reading blobs from disk %s", errors.FullTrace(err))
}
tasks := make(chan string, len(blobs))
done := make(chan bool)
processed := new(int32)
go produce(tasks, blobs)
cpus := runtime.NumCPU()
for i := 0; i < cpus-1; i++ {
go consume(i, tasks, done, len(blobs), processed)
}
<-done
}
func produce(tasks chan<- string, blobs []string) {
for _, b := range blobs {
tasks <- b
}
close(tasks)
}
func consume(worker int, tasks <-chan string, done chan<- bool, totalTasks int, processed *int32) {
start := time.Now()
for b := range tasks {
processedSoFar := atomic.AddInt32(processed, 1)
if worker == 0 {
remaining := int32(totalTasks) - processedSoFar
timePerBlob := time.Since(start).Microseconds() / int64(processedSoFar)
remainingTime := time.Duration(int64(remaining)*timePerBlob) * time.Microsecond
log.Infof("[T%d] %d/%d blobs processed so far. ETA: %s", worker, processedSoFar, totalTasks, remainingTime.String())
}
blobPath := path.Join(diskStorePath, b[:2], b)
blob, err := os.ReadFile(blobPath)
if err != nil {
if os.IsNotExist(err) {
continue
}
log.Errorf("[Worker %d] Error looking up blob %s: %s", worker, b, err.Error())
continue
}
hashBytes := sha512.Sum384(blob)
readHash := hex.EncodeToString(hashBytes[:])
if readHash != b {
log.Infof("[%s] found a broken blob while reading from disk. Actual hash: %s", b, readHash)
err := os.Remove(blobPath)
if err != nil {
log.Errorf("Error while deleting broken blob %s: %s", b, err.Error())
}
}
}
done <- true
}
-44
View File
@@ -1,44 +0,0 @@
package cmd
import (
"os"
"os/signal"
"strconv"
"syscall"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/peer"
"github.com/lbryio/reflector.go/store"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
func init() {
var cmd = &cobra.Command{
Use: "peer",
Short: "Run peer server",
Run: peerCmd,
}
rootCmd.AddCommand(cmd)
}
func peerCmd(cmd *cobra.Command, args []string) {
db := new(db.SQL)
err := db.Connect(globalConfig.DBConn)
checkErr(err)
s3 := store.NewS3BlobStore(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
combo := store.NewDBBackedS3Store(s3, db)
peerServer := peer.NewServer(combo)
err = peerServer.Start(":" + strconv.Itoa(peer.DefaultPort))
if err != nil {
log.Fatal(err)
}
interruptChan := make(chan os.Signal, 1)
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
<-interruptChan
peerServer.Shutdown()
}
+51
View File
@@ -0,0 +1,51 @@
package cmd
import (
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/meta"
"github.com/lbryio/reflector.go/store/speedwalk"
"github.com/lbryio/lbry.go/v2/extras/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
var (
diskStorePath string
)
func init() {
var cmd = &cobra.Command{
Use: "populate-db",
Short: "populate local database with blobs from a disk storage",
Run: populateDbCmd,
}
cmd.Flags().StringVar(&diskStorePath, "store-path", "",
"path of the store where all blobs are cached")
rootCmd.AddCommand(cmd)
}
func populateDbCmd(cmd *cobra.Command, args []string) {
log.Printf("reflector %s", meta.VersionString())
if diskStorePath == "" {
log.Fatal("store-path must be defined")
}
localDb := &db.SQL{
SoftDelete: true,
TrackingLevel: db.TrackAccessBlobs,
LogQueries: log.GetLevel() == log.DebugLevel,
}
err := localDb.Connect("reflector:reflector@tcp(localhost:3306)/reflector")
if err != nil {
log.Fatal(err)
}
blobs, err := speedwalk.AllFiles(diskStorePath, true)
if err != nil {
log.Fatal(err)
}
err = localDb.AddBlobs(blobs)
if err != nil {
log.Errorf("error while storing to db: %s", errors.FullTrace(err))
}
}
+65
View File
@@ -0,0 +1,65 @@
package cmd
import (
"fmt"
"github.com/lbryio/reflector.go/publish"
"github.com/lbryio/lbry.go/v2/lbrycrd"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
func init() {
var cmd = &cobra.Command{
Use: "publish FILE",
Short: "Publish a file",
Args: cobra.ExactArgs(1),
Run: publishCmd,
}
cmd.Flags().String("name", "", "Claim name")
cmd.Flags().String("title", "", "Title of the content")
cmd.Flags().String("description", "", "Description of the content")
cmd.Flags().String("author", "", "Content author")
cmd.Flags().String("tags", "", "Comma-separated list of tags")
cmd.Flags().Int64("release-time", 0, "original public release of content, seconds since UNIX epoch")
rootCmd.AddCommand(cmd)
}
func publishCmd(cmd *cobra.Command, args []string) {
var err error
claimName := mustGetFlagString(cmd, "name")
if claimName == "" {
log.Errorln("--name required")
return
}
path := args[0]
client, err := lbrycrd.NewWithDefaultURL(nil)
checkErr(err)
tx, txid, err := publish.Publish(
client,
path,
claimName,
"bSzpgkTnAoiT2YAhUShPpfpajPESfNXVTu",
publish.Details{
Title: mustGetFlagString(cmd, "title"),
Description: mustGetFlagString(cmd, "description"),
Author: mustGetFlagString(cmd, "author"),
Tags: nil,
ReleaseTime: mustGetFlagInt64(cmd, "release-time"),
},
"reflector.lbry.com:5566",
)
checkErr(err)
decoded, err := publish.Decode(client, tx)
checkErr(err)
fmt.Printf("TX: %s\n\n", decoded)
fmt.Printf("TXID: %s\n", txid.String())
}
+43 -51
View File
@@ -7,78 +7,70 @@ import (
"syscall"
"time"
"github.com/lbryio/reflector.go/config"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/meta"
"github.com/lbryio/reflector.go/peer"
"github.com/lbryio/reflector.go/store"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
var (
//port configuration
receiverPort int
metricsPort int
//flags configuration
disableBlocklist bool
)
func init() {
var cmd = &cobra.Command{
Use: "reflector",
Short: "Run reflector server",
Run: reflectorCmd,
}
cmd.Flags().IntVar(&metricsPort, "metrics-port", 2112, "The port reflector will use for prometheus metrics")
cmd.Flags().IntVar(&receiverPort, "receiver-port", 5566, "The port reflector will receive content from")
cmd.Flags().BoolVar(&disableBlocklist, "disable-blocklist", false, "Disable blocklist watching/updating")
rootCmd.AddCommand(cmd)
}
func reflectorCmd(cmd *cobra.Command, args []string) {
log.Printf("reflector version %s, built %s", meta.Version, meta.BuildTime.Format(time.RFC3339))
// flip this flag to false when doing db maintenance. uploads will not work (as reflector server wont be running)
// but downloads will still work straight from s3
useDB := true
s3 := store.NewS3BlobStore(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
var err error
var blobStore store.BlobStore = s3
var reflectorServer *reflector.Server
if useDB {
db := new(db.SQL)
err = db.Connect(globalConfig.DBConn)
if err != nil {
log.Fatal(err)
}
blobStore = store.NewDBBackedS3Store(s3, db)
reflectorServer = reflector.NewServer(blobStore)
reflectorServer.Timeout = 3 * time.Minute
if globalConfig.SlackHookURL != "" {
reflectorServer.StatLogger = log.StandardLogger()
reflectorServer.StatReportFrequency = 1 * time.Hour
}
reflectorServer.EnableBlocklist = true
err = reflectorServer.Start(":" + strconv.Itoa(reflector.DefaultPort))
if err != nil {
log.Fatal(err)
}
}
peerServer := peer.NewServer(blobStore)
if globalConfig.SlackHookURL != "" {
peerServer.StatLogger = log.StandardLogger()
peerServer.StatReportFrequency = 1 * time.Hour
}
err = peerServer.Start(":5567")
store, err := config.LoadStores(conf, "reflector")
if err != nil {
log.Fatal(err)
}
defer store.Shutdown()
servers, err := config.LoadServers(store, conf, "reflector")
if err != nil {
log.Fatal(err)
}
for _, s := range servers {
err = s.Start()
if err != nil {
log.Fatal(err)
}
defer s.Shutdown()
}
reflectorServer := reflector.NewIngestionServer(store)
reflectorServer.Timeout = 3 * time.Minute
reflectorServer.EnableBlocklist = !disableBlocklist
err = reflectorServer.Start(":" + strconv.Itoa(receiverPort))
if err != nil {
log.Fatal(err)
}
defer reflectorServer.Shutdown()
metricsServer := metrics.NewServer(":"+strconv.Itoa(metricsPort), "/metrics")
metricsServer.Start()
defer metricsServer.Shutdown()
interruptChan := make(chan os.Signal, 1)
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
<-interruptChan
peerServer.Shutdown()
if reflectorServer != nil {
reflectorServer.Shutdown()
}
}
+50
View File
@@ -0,0 +1,50 @@
package cmd
import (
"encoding/hex"
"encoding/json"
"fmt"
"github.com/lbryio/reflector.go/wallet"
"github.com/spf13/cobra"
)
func init() {
var cmd = &cobra.Command{
Use: "resolve ADDRESS:PORT URL",
Short: "Resolve a URL",
Args: cobra.ExactArgs(2),
Run: resolveCmd,
}
rootCmd.AddCommand(cmd)
}
func resolveCmd(cmd *cobra.Command, args []string) {
addr := args[0]
url := args[1]
node := wallet.NewNode()
defer node.Shutdown()
err := node.Connect([]string{addr}, nil)
checkErr(err)
output, err := node.Resolve(url)
checkErr(err)
claim, err := node.GetClaimInTx(hex.EncodeToString(rev(output.GetTxHash())), int(output.GetNout()))
checkErr(err)
jsonClaim, err := json.MarshalIndent(claim, "", " ")
checkErr(err)
fmt.Println(string(jsonClaim))
}
func rev(b []byte) []byte {
r := make([]byte, len(b))
for left, right := 0, len(b)-1; left < right; left, right = left+1, right-1 {
r[left], r[right] = b[right], b[left]
}
return r
}
+70 -47
View File
@@ -1,18 +1,15 @@
package cmd
import (
"encoding/json"
"io/ioutil"
"os"
"strings"
"github.com/lbryio/lbry.go/dht"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/util"
"github.com/lbryio/reflector.go/updater"
"github.com/lbryio/lbry.go/v2/dht"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/johntdyer/slackrus"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
// Config is the base configuration for Prism when no sub commands are used.
@@ -21,8 +18,10 @@ type Config struct {
AwsSecret string `json:"aws_secret"`
BucketRegion string `json:"bucket_region"`
BucketName string `json:"bucket_name"`
S3Endpoint string `json:"s3_endpoint"`
DBConn string `json:"db_conn"`
SlackHookURL string `json:"slack_hook_url"`
SlackChannel string `json:"slack_channel"`
UpdateBinURL string `json:"update_bin_url"`
UpdateCmd string `json:"update_cmd"`
}
@@ -36,7 +35,8 @@ const (
)
var conf string
var globalConfig Config
//var globalConfig Config
var rootCmd = &cobra.Command{
Use: "prism",
@@ -49,7 +49,7 @@ var rootCmd = &cobra.Command{
func init() {
rootCmd.PersistentFlags().StringSliceVarP(&verbose, "verbose", "v", []string{}, "Verbose logging for specific components")
rootCmd.PersistentFlags().StringVar(&conf, "conf", "config.json", "Path to config")
rootCmd.PersistentFlags().StringVar(&conf, "conf-dir", "./", "Path to config directory")
}
// Execute adds all child commands to the root command and sets flags appropriately.
@@ -68,8 +68,11 @@ func preRun(cmd *cobra.Command, args []string) {
debugLogger.SetOutput(os.Stderr)
if util.InSlice(verboseAll, verbose) {
logrus.Info("global verbose logging enabled")
logrus.SetLevel(logrus.DebugLevel)
verbose = []string{verboseDHT, verboseNodeFinder}
} else if len(verbose) > 0 {
logrus.Infof("verbose logging enabled for: %s", strings.Join(verbose, ", "))
}
for _, debugType := range verbose {
@@ -81,38 +84,40 @@ func preRun(cmd *cobra.Command, args []string) {
}
}
var err error
//var err error
if conf == "" {
logrus.Errorln("--conf flag required")
os.Exit(1)
} else if conf != "none" {
globalConfig, err = loadConfig(conf)
if err != nil {
logrus.Error(err)
os.Exit(1)
}
}
//else if conf != "none" {
// globalConfig, err = loadConfig(conf)
// if err != nil {
// logrus.Error(err)
// os.Exit(1)
// }
//}
viper.AddConfigPath(conf)
if globalConfig.SlackHookURL != "" {
hook := &slackrus.SlackrusHook{
HookURL: globalConfig.SlackHookURL,
AcceptedLevels: slackrus.LevelThreshold(logrus.InfoLevel),
Channel: "#reflector-logs",
//IconEmoji: ":ghost:",
//Username: "reflector.go",
}
//logrus.SetFormatter(&logrus.JSONFormatter{})
logrus.AddHook(hook)
debugLogger.AddHook(hook)
}
//if globalConfig.SlackHookURL != "" {
// hook := &slackrus.SlackrusHook{
// HookURL: globalConfig.SlackHookURL,
// AcceptedLevels: slackrus.LevelThreshold(logrus.InfoLevel),
// Channel: globalConfig.SlackChannel,
// //IconEmoji: ":ghost:",
// //Username: "reflector.go",
// }
// //logrus.SetFormatter(&logrus.JSONFormatter{})
// logrus.AddHook(hook)
// debugLogger.AddHook(hook)
//}
if globalConfig.UpdateBinURL != "" {
if globalConfig.UpdateCmd == "" {
logrus.Warnln("update_cmd is empty in conf file")
}
logrus.Println("starting update checker")
go updater.Run(globalConfig.UpdateBinURL, globalConfig.UpdateCmd)
}
//if globalConfig.UpdateBinURL != "" {
// if globalConfig.UpdateCmd == "" {
// logrus.Warnln("update_cmd is empty in conf file")
// }
// logrus.Println("starting update checker")
// go updater.Run(globalConfig.UpdateBinURL, globalConfig.UpdateCmd)
//}
}
func checkErr(err error) {
@@ -133,17 +138,35 @@ func argFuncs(funcs ...cobra.PositionalArgs) cobra.PositionalArgs {
}
}
func loadConfig(path string) (Config, error) {
var c Config
//func loadConfig(path string) (Config, error) {
// var c Config
//
// raw, err := os.ReadFile(path)
// if err != nil {
// if os.IsNotExist(err) {
// return c, errors.Err("config file not found")
// }
// return c, errors.Err(err)
// }
//
// err = json.Unmarshal(raw, &c)
// return c, errors.Err(err)
//}
raw, err := ioutil.ReadFile(path)
if err != nil {
if os.IsNotExist(err) {
return c, errors.Err("config file not found")
}
return c, err
}
err = json.Unmarshal(raw, &c)
return c, err
func mustGetFlagString(cmd *cobra.Command, name string) string {
v, err := cmd.Flags().GetString(name)
checkErr(err)
return v
}
func mustGetFlagInt64(cmd *cobra.Command, name string) int64 {
v, err := cmd.Flags().GetInt64(name)
checkErr(err)
return v
}
//func mustGetFlagBool(cmd *cobra.Command, name string) bool {
// v, err := cmd.Flags().GetBool(name)
// checkErr(err)
// return v
//}
+160
View File
@@ -0,0 +1,160 @@
package cmd
import (
"encoding/hex"
"encoding/json"
"fmt"
"io"
"os"
"os/signal"
"path"
"syscall"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/cobra"
)
func init() {
var cmd = &cobra.Command{
Use: "send ADDRESS:PORT PATH",
Short: "Send a file to a reflector",
Args: cobra.ExactArgs(2),
Run: sendCmd,
}
cmd.PersistentFlags().String("sd-cache", "", "path to dir where sd blobs will be cached")
rootCmd.AddCommand(cmd)
}
// todo: if retrying a large file is slow, we can add the ability to seek ahead in the file so we're not
// re-uploading blobs that already exist
var hackyReflector reflector.Client
func sendCmd(cmd *cobra.Command, args []string) {
reflectorAddress := args[0]
err := hackyReflector.Connect(reflectorAddress)
checkErr(err)
defer func() { _ = hackyReflector.Close() }()
filePath := args[1]
file, err := os.Open(filePath)
checkErr(err)
defer func() { _ = file.Close() }()
sdCachePath := ""
sdCacheDir := mustGetFlagString(cmd, "sd-cache")
if sdCacheDir != "" {
if _, statErr := os.Stat(sdCacheDir); os.IsNotExist(statErr) {
err = os.MkdirAll(sdCacheDir, 0777)
checkErr(err)
}
sdCachePath = path.Join(sdCacheDir, filePath+".sdblob")
}
var enc *stream.Encoder
if sdCachePath != "" {
if _, statErr := os.Stat(sdCachePath); !os.IsNotExist(statErr) {
var sdBlob []byte
sdBlob, err = os.ReadFile(sdCachePath)
checkErr(err)
cachedSDBlob := &stream.SDBlob{}
err = cachedSDBlob.FromBlob(sdBlob)
checkErr(err)
enc = stream.NewEncoderFromSD(file, cachedSDBlob)
}
}
if enc == nil {
enc = stream.NewEncoder(file)
}
exitCode := 0
var killed bool
interruptChan := make(chan os.Signal, 1)
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
go func() {
sig := <-interruptChan
fmt.Printf("caught %s, exiting...\n", sig.String())
killed = true
exitCode = 1
}()
for {
if killed {
break
}
var b stream.Blob
b, err = enc.Next()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
fmt.Printf("error reading next blob: %v\n", err)
exitCode = 1
break
}
err = hackyReflect(b, false)
if err != nil {
fmt.Printf("error reflecting blob %s: %v\n", b.HashHex()[:8], err)
exitCode = 1
break
}
}
sd := enc.SDBlob()
//sd.StreamName = filepath.Base(filePath)
//sd.SuggestedFileName = filepath.Base(filePath)
err = os.WriteFile(sdCachePath, sd.ToBlob(), 0666)
if err != nil {
fmt.Printf("error saving sd blob: %v\n", err)
fmt.Println(sd.ToJson())
exitCode = 1
}
if killed {
os.Exit(exitCode)
}
if reflectorAddress != "" {
err = hackyReflect(sd.ToBlob(), true)
if err != nil {
fmt.Printf("error reflecting sd blob %s: %v\n", sd.HashHex()[:8], err)
exitCode = 1
}
}
ret := struct {
SDHash string `json:"sd_hash"`
SourceHash string `json:"source_hash"`
}{
SDHash: sd.HashHex(),
SourceHash: hex.EncodeToString(enc.SourceHash()),
}
j, err := json.MarshalIndent(ret, "", " ")
checkErr(err)
fmt.Println(string(j))
os.Exit(exitCode)
}
func hackyReflect(b stream.Blob, sd bool) error {
var err error
if sd {
err = hackyReflector.SendSDBlob(b)
} else {
err = hackyReflector.SendBlob(b)
}
if errors.Is(err, reflector.ErrBlobExists) {
//fmt.Printf("%s already reflected\n", b.HashHex()[:8])
return nil
}
return err
}
+37 -10
View File
@@ -2,10 +2,11 @@ package cmd
import (
"crypto/rand"
"os"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/lbry.go/stream"
"github.com/lbryio/lbry.go/v2/stream"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
@@ -13,9 +14,9 @@ import (
func init() {
var cmd = &cobra.Command{
Use: "sendblob ADDRESS:PORT",
Use: "sendblob ADDRESS:PORT [PATH]",
Short: "Send a random blob to a reflector server",
Args: cobra.ExactArgs(1),
Args: cobra.RangeArgs(1, 2),
Run: sendBlobCmd,
}
rootCmd.AddCommand(cmd)
@@ -23,6 +24,10 @@ func init() {
func sendBlobCmd(cmd *cobra.Command, args []string) {
addr := args[0]
var path string
if len(args) >= 2 {
path = args[1]
}
c := reflector.Client{}
err := c.Connect(addr)
@@ -30,14 +35,36 @@ func sendBlobCmd(cmd *cobra.Command, args []string) {
log.Fatal("error connecting client to server: ", err)
}
blob := make(stream.Blob, 1024)
_, err = rand.Read(blob)
if err != nil {
log.Fatal("failed to make random blob: ", err)
if path == "" {
blob := make(stream.Blob, 1024)
_, err = rand.Read(blob)
if err != nil {
log.Fatal("failed to make random blob: ", err)
}
err = c.SendBlob(blob)
if err != nil {
log.Error(err)
}
return
}
err = c.SendBlob(blob)
if err != nil {
log.Error(err)
file, err := os.Open(path)
checkErr(err)
defer func() { _ = file.Close() }()
s, err := stream.New(file)
checkErr(err)
sdBlob := &stream.SDBlob{}
err = sdBlob.FromBlob(s[0])
checkErr(err)
for i, b := range s {
if i == 0 {
err = c.SendSDBlob(b)
} else {
err = c.SendBlob(b)
}
checkErr(err)
}
}
-99
View File
@@ -1,99 +0,0 @@
package cmd
import (
"os"
"os/signal"
"strconv"
"strings"
"syscall"
"github.com/lbryio/lbry.go/dht"
"github.com/lbryio/lbry.go/dht/bits"
"github.com/lbryio/reflector.go/cluster"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/peer"
"github.com/lbryio/reflector.go/prism"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/store"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
const (
startNewCluster = "new"
)
var (
startClusterPort int
startPeerPort int
startReflectorPort int
startDhtPort int
startDhtSeeds []string
startHashRange string
)
func init() {
var cmd = &cobra.Command{
Use: `start [cluster-address|"new"]`,
Short: "Runs full prism application with cluster, dht, peer server, and reflector server.",
Run: startCmd,
Args: cobra.ExactArgs(1),
}
cmd.PersistentFlags().IntVar(&startClusterPort, "cluster-port", cluster.DefaultPort, "Port that cluster listens on")
cmd.PersistentFlags().IntVar(&startPeerPort, "peer-port", peer.DefaultPort, "Port to start peer protocol on")
cmd.PersistentFlags().IntVar(&startReflectorPort, "reflector-port", reflector.DefaultPort, "Port to start reflector protocol on")
cmd.PersistentFlags().IntVar(&startDhtPort, "dht-port", dht.DefaultPort, "Port that dht will listen on")
cmd.PersistentFlags().StringSliceVar(&startDhtSeeds, "dht-seeds", []string{}, "Comma-separated list of dht seed nodes (addr:port,addr:port,...)")
cmd.PersistentFlags().StringVar(&startHashRange, "hash-range", "", "Limit on range of hashes to announce (start-end)")
rootCmd.AddCommand(cmd)
}
func startCmd(cmd *cobra.Command, args []string) {
db := new(db.SQL)
err := db.Connect(globalConfig.DBConn)
checkErr(err)
s3 := store.NewS3BlobStore(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
comboStore := store.NewDBBackedS3Store(s3, db)
conf := prism.DefaultConf()
// TODO: args we need:
// minNodes - minimum number of nodes before announcing starts. otherwise first node will try to announce all the blobs in the db
// or maybe we should do maxHashesPerNode?
// in either case, this should not kill the cluster, but should only limit announces (and notify when some hashes are being left unannounced)
if args[0] != startNewCluster {
conf.ClusterSeedAddr = args[0]
}
conf.DB = db
conf.Blobs = comboStore
conf.DhtAddress = "0.0.0.0:" + strconv.Itoa(startDhtPort)
conf.DhtSeedNodes = startDhtSeeds
conf.ClusterPort = startClusterPort
conf.PeerPort = startPeerPort
conf.ReflectorPort = startReflectorPort
if startHashRange != "" {
hashRange := strings.Split(startHashRange, "-")
if len(hashRange) != 2 {
log.Fatal("invalid hash range")
}
r := bits.Range{Start: bits.FromShortHexP(hashRange[0]), End: bits.FromShortHexP(hashRange[1])}
conf.HashRange = &r
}
p := prism.New(conf)
err = p.Start()
if err != nil {
log.Fatal(err)
}
interruptChan := make(chan os.Signal, 1)
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
<-interruptChan
p.Shutdown()
}
+6 -6
View File
@@ -9,8 +9,8 @@ import (
"time"
"github.com/lbryio/reflector.go/meta"
"github.com/lbryio/reflector.go/peer"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/server/peer"
"github.com/lbryio/reflector.go/store"
log "github.com/sirupsen/logrus"
@@ -27,11 +27,11 @@ func init() {
}
func testCmd(cmd *cobra.Command, args []string) {
log.Printf("reflector version %s", meta.Version)
log.Printf("reflector %s", meta.VersionString())
memStore := &store.MemoryBlobStore{}
memStore := store.NewMemStore(store.MemParams{Name: "test"})
reflectorServer := reflector.NewServer(memStore)
reflectorServer := reflector.NewIngestionServer(memStore)
reflectorServer.Timeout = 3 * time.Minute
err := reflectorServer.Start(":" + strconv.Itoa(reflector.DefaultPort))
@@ -39,8 +39,8 @@ func testCmd(cmd *cobra.Command, args []string) {
log.Fatal(err)
}
peerServer := peer.NewServer(memStore)
err = peerServer.Start(":" + strconv.Itoa(reflector.DefaultPort+1))
peerServer := peer.NewServer(memStore, fmt.Sprintf(":%d", reflector.DefaultPort+1))
err = peerServer.Start()
if err != nil {
log.Fatal(err)
}
+14 -9
View File
@@ -5,15 +5,16 @@ import (
"os/signal"
"syscall"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/config"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/store"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
var uploadWorkers int
var uploadSkipExistsCheck bool
var uploadDeleteBlobsAfterUpload bool
func init() {
var cmd = &cobra.Command{
@@ -24,19 +25,23 @@ func init() {
}
cmd.PersistentFlags().IntVar(&uploadWorkers, "workers", 1, "How many worker threads to run at once")
cmd.PersistentFlags().BoolVar(&uploadSkipExistsCheck, "skipExistsCheck", false, "Dont check if blobs exist before uploading")
cmd.PersistentFlags().BoolVar(&uploadDeleteBlobsAfterUpload, "deleteBlobsAfterUpload", false, "Delete blobs after uploading them")
rootCmd.AddCommand(cmd)
}
func uploadCmd(cmd *cobra.Command, args []string) {
db := new(db.SQL)
err := db.Connect(globalConfig.DBConn)
checkErr(err)
store, err := config.LoadStores(conf, "upload")
if err != nil {
log.Fatal(err)
}
defer store.Shutdown()
st := store.NewDBBackedS3Store(
store.NewS3BlobStore(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName),
db)
databaseConn, err := config.LoadDatabase(conf, "upload")
if err != nil {
log.Fatal(err)
}
uploader := reflector.NewUploader(db, st, uploadWorkers, uploadSkipExistsCheck)
uploader := reflector.NewUploader(databaseConn, store, uploadWorkers, uploadSkipExistsCheck, uploadDeleteBlobsAfterUpload)
interruptChan := make(chan os.Signal, 1)
signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
+2 -2
View File
@@ -2,9 +2,9 @@ package cmd
import (
"fmt"
"time"
"github.com/lbryio/reflector.go/meta"
"github.com/spf13/cobra"
)
@@ -18,5 +18,5 @@ func init() {
}
func versionCmd(cmd *cobra.Command, args []string) {
fmt.Printf("version %s (built %s)\n", meta.Version, meta.BuildTime.Format(time.RFC3339))
fmt.Println(meta.FullName())
}
+113
View File
@@ -0,0 +1,113 @@
package config
import (
"fmt"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/server"
"github.com/lbryio/reflector.go/server/http"
"github.com/lbryio/reflector.go/server/http3"
"github.com/lbryio/reflector.go/server/peer"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/extras/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
)
func LoadStores(path, file string) (store.BlobStore, error) {
v := viper.New()
v.SetConfigType("yaml")
v.AddConfigPath(path)
v.SetConfigName(file)
err := v.ReadInConfig()
if err != nil {
return nil, errors.Err(err)
}
storeViper := v.Sub("store")
for storeType := range storeViper.AllSettings() {
factory, exists := store.Factories[storeType]
if !exists {
return nil, errors.Err("unknown store type: %s", storeType)
}
storeConfig := storeViper.Sub(storeType)
s, err := factory(storeConfig)
if err != nil {
return nil, errors.Err(err)
}
//we only expect 1 store as the root, so let's return it
return s, nil
}
return nil, nil
}
func LoadServers(store store.BlobStore, path, file string) ([]server.BlobServer, error) {
v := viper.New()
v.SetConfigType("yaml")
v.AddConfigPath(path)
v.SetConfigName(file)
err := v.ReadInConfig()
if err != nil {
return nil, errors.Err(err)
}
servers := make([]server.BlobServer, 0)
serversViper := v.Sub("servers")
for serverType := range serversViper.AllSettings() {
var cfg server.BlobServerConfig
err := serversViper.Sub(serverType).Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
switch serverType {
case "http":
servers = append(servers, http.NewServer(store, cfg.MaxConcurrentRequests, cfg.EdgeToken, fmt.Sprintf("%s:%d", cfg.Address, cfg.Port)))
case "http3":
servers = append(servers, http3.NewServer(store, cfg.MaxConcurrentRequests, fmt.Sprintf("%s:%d", cfg.Address, cfg.Port)))
case "peer":
servers = append(servers, peer.NewServer(store, fmt.Sprintf("%s:%d", cfg.Address, cfg.Port)))
default:
return nil, errors.Err("unknown server type: %s", serverType)
}
}
return servers, nil
}
func LoadDatabase(path, file string) (*db.SQL, error) {
v := viper.New()
v.SetConfigType("yaml")
v.AddConfigPath(path)
v.SetConfigName(file)
err := v.ReadInConfig()
if err != nil {
return nil, errors.Err(err)
}
dbConfig := v.Sub("database")
if dbConfig == nil {
return nil, errors.Err("db config not found")
}
user := dbConfig.GetString("user")
password := dbConfig.GetString("password")
host := dbConfig.GetString("host")
port := dbConfig.GetInt("port")
database := dbConfig.GetString("database")
logQueries := dbConfig.GetBool("log_queries")
accessTracking := dbConfig.GetInt("access_tracking")
if user == "" || password == "" || host == "" || port == 0 || database == "" {
return nil, errors.Err("db config is missing required fields")
}
dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", user, password, host, port, database)
dbInstance := &db.SQL{
TrackingLevel: db.AccessTrackingLevel(accessTracking),
SoftDelete: true,
LogQueries: logQueries || log.GetLevel() == log.DebugLevel,
}
err = dbInstance.Connect(dsn)
return dbInstance, err
}
+322 -48
View File
@@ -3,43 +3,70 @@ package db
import (
"context"
"database/sql"
"fmt"
"runtime"
"strings"
"time"
"github.com/lbryio/lbry.go/dht/bits"
"github.com/lbryio/lbry.go/extras/errors"
qt "github.com/lbryio/lbry.go/extras/query"
"github.com/lbryio/lbry.go/v2/dht/bits"
"github.com/lbryio/lbry.go/v2/extras/errors"
qt "github.com/lbryio/lbry.go/v2/extras/query"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/go-sql-driver/mysql"
_ "github.com/go-sql-driver/mysql" // blank import for db driver ensures its imported even if its not used
log "github.com/sirupsen/logrus"
"github.com/volatiletech/null/v8"
"go.uber.org/atomic"
)
// SdBlob is a special blob that contains information on the rest of the blobs in the stream
type SdBlob struct {
StreamName string `json:"stream_name"`
Blobs []struct {
Length int `json:"length"`
BlobNum int `json:"blob_num"`
BlobHash string `json:"blob_hash,omitempty"`
IV string `json:"iv"`
} `json:"blobs"`
StreamName string `json:"stream_name"`
StreamType string `json:"stream_type"`
Key string `json:"key"`
SuggestedFileName string `json:"suggested_file_name"`
StreamHash string `json:"stream_hash"`
Blobs []struct {
BlobHash string `json:"blob_hash,omitempty"`
IV string `json:"iv"`
Length int `json:"length"`
BlobNum int `json:"blob_num"`
} `json:"blobs"`
}
type AccessTrackingLevel int
const (
TrackAccessNone AccessTrackingLevel = iota // Don't track accesses
TrackAccessStreams // Track accesses at the stream level
TrackAccessBlobs // Track accesses at the blob level
)
// SQL implements the DB interface
type SQL struct {
conn *sql.DB
// Track the approx last time a blob or stream was accessed
TrackingLevel AccessTrackingLevel
// Instead of deleting a blob, marked it as not stored in the db
SoftDelete bool
// Log executed queries. qt.InterpolateParams is cpu-heavy. This avoids that call if not needed.
LogQueries bool
}
func logQuery(query string, args ...interface{}) {
s, err := qt.InterpolateParams(query, args...)
func (s *SQL) logQuery(query string, args ...interface{}) {
if !s.LogQueries {
return
}
qStr, err := qt.InterpolateParams(query, args...)
if err != nil {
log.Errorln(err)
} else {
log.Debugln(s)
log.Debugln(qStr)
}
}
@@ -70,15 +97,97 @@ func (s *SQL) AddBlob(hash string, length int, isStored bool) error {
return err
}
// AddBlobs adds blobs to the database.
func (s *SQL) AddBlobs(hash []string) error {
if s.conn == nil {
return errors.Err("not connected")
}
batch := 10000
totalBlobs := int64(len(hash))
work := make(chan []string, 1000)
stopper := stop.New()
var totalInserted atomic.Int64
start := time.Now()
go func() {
for i := 0; i < len(hash); i += batch {
j := i + batch
if j > len(hash) {
j = len(hash)
}
work <- hash[i:j]
}
log.Infof("done loading %d hashes in the work queue", len(hash))
close(work)
}()
for i := 0; i < runtime.NumCPU(); i++ {
stopper.Add(1)
go func(worker int) {
log.Infof("starting worker %d", worker)
defer stopper.Done()
for hashes := range work {
inserted := totalInserted.Load()
remaining := totalBlobs - inserted
if inserted > 0 {
timePerBlob := time.Since(start).Microseconds() / inserted
remainingTime := time.Duration(remaining*timePerBlob) * time.Microsecond
log.Infof("[T%d] processing batch of %d items. ETA: %s", worker, len(hashes), remainingTime.String())
}
err := s.insertBlobs(hashes) // Process the batch.
if err != nil {
log.Errorf("error while inserting batch: %s", errors.FullTrace(err))
}
totalInserted.Add(int64(len(hashes)))
}
}(i)
}
stopper.Wait()
return nil
}
func (s *SQL) insertBlobs(hashes []string) error {
var (
q string
//args []interface{}
)
dayAgo := time.Now().AddDate(0, 0, -1).Format("2006-01-02 15:04:05")
q = "insert into blob_ (hash, is_stored, length, last_accessed_at) values "
for _, hash := range hashes {
// prepared statements slow everything down by a lot due to reflection
// for this specific instance we'll go ahead and hardcode the query to make it go faster
q += fmt.Sprintf("('%s',1,%d,'%s'),", hash, stream.MaxBlobSize, dayAgo)
//args = append(args, hash, true, stream.MaxBlobSize, dayAgo)
}
q = strings.TrimSuffix(q, ",")
_, err := s.exec(q)
if err != nil {
return err
}
return nil
}
func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error) {
if length <= 0 {
return 0, errors.Err("length must be positive")
}
blobID, err := s.exec(
"INSERT INTO blob_ (hash, is_stored, length) VALUES (?,?,?) ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored))",
hash, isStored, length,
var (
q string
args []interface{}
)
if s.TrackingLevel == TrackAccessBlobs {
args = []interface{}{hash, isStored, length, time.Now()}
q = "INSERT INTO blob_ (hash, is_stored, length, last_accessed_at) VALUES (" + qt.Qs(len(args)) + ") ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored)), last_accessed_at = VALUES(last_accessed_at)"
} else {
args = []interface{}{hash, isStored, length}
q = "INSERT INTO blob_ (hash, is_stored, length) VALUES (" + qt.Qs(len(args)) + ") ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored))"
}
blobID, err := s.exec(q, args...)
if err != nil {
return 0, err
}
@@ -91,16 +200,33 @@ func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error)
if blobID == 0 {
return 0, errors.Err("blob ID is 0 even after INSERTing and SELECTing")
}
if s.TrackingLevel == TrackAccessBlobs {
err := s.touchBlobs([]uint64{uint64(blobID)})
if err != nil {
return 0, errors.Err(err)
}
}
}
return blobID, nil
}
func (s *SQL) insertStream(hash string, sdBlobID int64) (int64, error) {
streamID, err := s.exec(
"INSERT IGNORE INTO stream (hash, sd_blob_id) VALUES (?,?)",
hash, sdBlobID,
var (
q string
args []interface{}
)
if s.TrackingLevel == TrackAccessStreams {
args = []interface{}{hash, sdBlobID, time.Now()}
q = "INSERT IGNORE INTO stream (hash, sd_blob_id, last_accessed_at) VALUES (" + qt.Qs(len(args)) + ")"
} else {
args = []interface{}{hash, sdBlobID}
q = "INSERT IGNORE INTO stream (hash, sd_blob_id) VALUES (" + qt.Qs(len(args)) + ")"
}
streamID, err := s.exec(q, args...)
if err != nil {
return 0, errors.Err(err)
}
@@ -113,13 +239,20 @@ func (s *SQL) insertStream(hash string, sdBlobID int64) (int64, error) {
if streamID == 0 {
return 0, errors.Err("stream ID is 0 even after INSERTing and SELECTing")
}
if s.TrackingLevel == TrackAccessStreams {
err := s.touchStreams([]uint64{uint64(streamID)})
if err != nil {
return 0, errors.Err(err)
}
}
}
return streamID, nil
}
// HasBlob checks if the database contains the blob information.
func (s *SQL) HasBlob(hash string) (bool, error) {
exists, err := s.HasBlobs([]string{hash})
func (s *SQL) HasBlob(hash string, touch bool) (bool, error) {
exists, err := s.HasBlobs([]string{hash}, touch)
if err != nil {
return false, err
}
@@ -127,13 +260,72 @@ func (s *SQL) HasBlob(hash string) (bool, error) {
}
// HasBlobs checks if the database contains the set of blobs and returns a bool map.
func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
if s.conn == nil {
return nil, errors.Err("not connected")
func (s *SQL) HasBlobs(hashes []string, touch bool) (map[string]bool, error) {
exists, idsNeedingTouch, err := s.hasBlobs(hashes)
if touch {
if s.TrackingLevel == TrackAccessBlobs {
_ = s.touchBlobs(idsNeedingTouch)
} else if s.TrackingLevel == TrackAccessStreams {
_ = s.touchStreams(idsNeedingTouch)
}
}
var hash string
return exists, err
}
func (s *SQL) touchBlobs(blobIDs []uint64) error {
if len(blobIDs) == 0 {
return nil
}
query := "UPDATE blob_ SET last_accessed_at = ? WHERE id IN (" + qt.Qs(len(blobIDs)) + ")"
args := make([]interface{}, len(blobIDs)+1)
args[0] = time.Now()
for i := range blobIDs {
args[i+1] = blobIDs[i]
}
startTime := time.Now()
_, err := s.exec(query, args...)
log.Debugf("touched %d blobs and took %s", len(blobIDs), time.Since(startTime))
return errors.Err(err)
}
func (s *SQL) touchStreams(streamIDs []uint64) error {
if len(streamIDs) == 0 {
return nil
}
query := "UPDATE stream SET last_accessed_at = ? WHERE id IN (" + qt.Qs(len(streamIDs)) + ")"
args := make([]interface{}, len(streamIDs)+1)
args[0] = time.Now()
for i := range streamIDs {
args[i+1] = streamIDs[i]
}
startTime := time.Now()
_, err := s.exec(query, args...)
log.Debugf("touched %d streams and took %s", len(streamIDs), time.Since(startTime))
return errors.Err(err)
}
func (s *SQL) hasBlobs(hashes []string) (map[string]bool, []uint64, error) {
if s.conn == nil {
return nil, nil, errors.Err("not connected")
}
var (
hash string
blobID uint64
streamID null.Uint64
lastAccessedAt null.Time
)
var needsTouch []uint64
exists := make(map[string]bool)
touchDeadline := time.Now().Add(-6 * time.Hour) // touch blob if last accessed before this time
maxBatchSize := 10000
doneIndex := 0
@@ -145,14 +337,29 @@ func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
log.Debugf("getting hashes[%d:%d] of %d", doneIndex, sliceEnd, len(hashes))
batch := hashes[doneIndex:sliceEnd]
query := "SELECT hash FROM blob_ WHERE is_stored = ? && hash IN (" + qt.Qs(len(batch)) + ")"
args := make([]interface{}, len(batch)+1)
args[0] = true
for i := range batch {
args[i+1] = batch[i]
var query string
if s.TrackingLevel == TrackAccessBlobs {
query = `SELECT b.hash, b.id, NULL, b.last_accessed_at
FROM blob_ b
WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)`
} else if s.TrackingLevel == TrackAccessStreams {
query = `SELECT b.hash, b.id, s.id, s.last_accessed_at
FROM blob_ b
LEFT JOIN stream_blob sb ON b.id = sb.blob_id
INNER JOIN stream s on (sb.stream_id = s.id or s.sd_blob_id = b.id)
WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)`
} else {
query = `SELECT b.hash, b.id, NULL, NULL
FROM blob_ b
WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)`
}
logQuery(query, args...)
args := make([]interface{}, len(batch))
for i := range batch {
args[i] = batch[i]
}
s.logQuery(query, args...)
err := func() error {
startTime := time.Now()
@@ -164,11 +371,18 @@ func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
defer closeRows(rows)
for rows.Next() {
err := rows.Scan(&hash)
err = rows.Scan(&hash, &blobID, &streamID, &lastAccessedAt)
if err != nil {
return errors.Err(err)
}
exists[hash] = true
if !lastAccessedAt.Valid || lastAccessedAt.Time.Before(touchDeadline) {
if s.TrackingLevel == TrackAccessBlobs {
needsTouch = append(needsTouch, blobID)
} else if s.TrackingLevel == TrackAccessStreams && !streamID.IsZero() {
needsTouch = append(needsTouch, streamID.Uint64)
}
}
}
err = rows.Err()
@@ -180,15 +394,21 @@ func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
return nil
}()
if err != nil {
return nil, err
return nil, nil, err
}
}
return exists, nil
return exists, needsTouch, nil
}
// Delete will remove the blob from the db
// Delete will remove (or soft-delete) the blob from the db
// NOTE: If SoftDelete is enabled, streams will never be deleted
func (s *SQL) Delete(hash string) error {
if s.SoftDelete {
_, err := s.exec("UPDATE blob_ SET is_stored = 0 WHERE hash = ?", hash)
return errors.Err(err)
}
_, err := s.exec("DELETE FROM stream WHERE sd_blob_id = (SELECT id FROM blob_ WHERE hash = ?)", hash)
if err != nil {
return errors.Err(err)
@@ -198,11 +418,59 @@ func (s *SQL) Delete(hash string) error {
return errors.Err(err)
}
// LeastRecentlyAccessedHashes gets the least recently accessed blobs
func (s *SQL) LeastRecentlyAccessedHashes(maxBlobs int) ([]string, error) {
if s.conn == nil {
return nil, errors.Err("not connected")
}
if s.TrackingLevel != TrackAccessBlobs {
return nil, errors.Err("blob access tracking is disabled")
}
query := "SELECT hash from blob_ where is_stored = 1 order by last_accessed_at limit ?"
s.logQuery(query, maxBlobs)
rows, err := s.conn.Query(query, maxBlobs)
if err != nil {
return nil, errors.Err(err)
}
defer closeRows(rows)
blobs := make([]string, 0, maxBlobs)
for rows.Next() {
var hash string
err := rows.Scan(&hash)
if err != nil {
return nil, errors.Err(err)
}
blobs = append(blobs, hash)
}
return blobs, nil
}
func (s *SQL) Count() (int, error) {
if s.conn == nil {
return 0, errors.Err("not connected")
}
query := "SELECT count(id) from blob_"
if s.SoftDelete {
query += " where is_stored = 1"
}
s.logQuery(query)
var count int
err := s.conn.QueryRow(query).Scan(&count)
return count, errors.Err(err)
}
// Block will mark a blob as blocked
func (s *SQL) Block(hash string) error {
query := "INSERT IGNORE INTO blocked SET hash = ?"
args := []interface{}{hash}
logQuery(query, args...)
s.logQuery(query, args...)
_, err := s.conn.Exec(query, args...)
return errors.Err(err)
}
@@ -210,7 +478,7 @@ func (s *SQL) Block(hash string) error {
// GetBlocked will return a list of blocked hashes
func (s *SQL) GetBlocked() (map[string]bool, error) {
query := "SELECT hash FROM blocked"
logQuery(query)
s.logQuery(query)
rows, err := s.conn.Query(query)
if err != nil {
return nil, errors.Err(err)
@@ -221,7 +489,7 @@ func (s *SQL) GetBlocked() (map[string]bool, error) {
var hash string
for rows.Next() {
err := rows.Scan(&hash)
err = rows.Scan(&hash)
if err != nil {
return nil, errors.Err(err)
}
@@ -253,7 +521,7 @@ func (s *SQL) MissingBlobsForKnownStream(sdHash string) ([]string, error) {
`
args := []interface{}{sdHash}
logQuery(query, args...)
s.logQuery(query, args...)
rows, err := s.conn.Query(query, args...)
if err != nil {
@@ -265,7 +533,7 @@ func (s *SQL) MissingBlobsForKnownStream(sdHash string) ([]string, error) {
var hash string
for rows.Next() {
err := rows.Scan(&hash)
err = rows.Scan(&hash)
if err != nil {
return nil, errors.Err(err)
}
@@ -309,9 +577,10 @@ func (s *SQL) AddSDBlob(sdHash string, sdBlobLength int, sdBlob SdBlob) error {
return err
}
args := []interface{}{streamID, blobID, contentBlob.BlobNum}
_, err = s.exec(
"INSERT IGNORE INTO stream_blob (stream_id, blob_id, num) VALUES (?,?,?)",
streamID, blobID, contentBlob.BlobNum,
"INSERT IGNORE INTO stream_blob (stream_id, blob_id, num) VALUES ("+qt.Qs(len(args))+")",
args...,
)
if err != nil {
return errors.Err(err)
@@ -331,7 +600,7 @@ func (s *SQL) GetHashRange() (string, string, error) {
query := "SELECT MIN(hash), MAX(hash) from blob_"
logQuery(query)
s.logQuery(query)
err := s.conn.QueryRow(query).Scan(&min, &max)
return min, max, err
@@ -355,7 +624,7 @@ func (s *SQL) GetStoredHashesInRange(ctx context.Context, start, end bits.Bitmap
query := "SELECT hash FROM blob_ WHERE hash >= ? AND hash <= ? AND is_stored = 1"
args := []interface{}{start.Hex(), end.Hex()}
logQuery(query, args...)
s.logQuery(query, args...)
rows, err := s.conn.Query(query, args...)
defer closeRows(rows)
@@ -367,7 +636,7 @@ func (s *SQL) GetStoredHashesInRange(ctx context.Context, start, end bits.Bitmap
var hash string
ScanLoop:
for rows.Next() {
err := rows.Scan(&hash)
err = rows.Scan(&hash)
if err != nil {
ech <- err
return
@@ -436,7 +705,7 @@ func closeRows(rows *sql.Rows) {
}
func (s *SQL) exec(query string, args ...interface{}) (int64, error) {
logQuery(query, args...)
s.logQuery(query, args...)
attempt, maxAttempts := 0, 3
Retry:
attempt++
@@ -474,17 +743,22 @@ CREATE TABLE blob_ (
hash char(96) NOT NULL,
is_stored TINYINT(1) NOT NULL DEFAULT 0,
length bigint(20) unsigned DEFAULT NULL,
last_accessed_at TIMESTAMP NULL DEFAULT NULL,
PRIMARY KEY (id),
UNIQUE KEY blob_hash_idx (hash)
UNIQUE KEY blob_hash_idx (hash),
KEY `blob_last_accessed_idx` (`last_accessed_at`),
KEY `is_stored_idx` (`is_stored`)
);
CREATE TABLE stream (
id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT UNIQUE,
hash char(96) NOT NULL,
sd_blob_id BIGINT UNSIGNED NOT NULL,
last_accessed_at TIMESTAMP NULL DEFAULT NULL,
PRIMARY KEY (id),
UNIQUE KEY stream_hash_idx (hash),
KEY stream_sd_blob_id_idx (sd_blob_id),
KEY last_accessed_at_idx (last_accessed_at),
FOREIGN KEY (sd_blob_id) REFERENCES blob_ (id) ON DELETE RESTRICT ON UPDATE CASCADE
);
+134 -26
View File
@@ -1,31 +1,139 @@
module github.com/lbryio/reflector.go
go 1.25
replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19
require (
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
github.com/aws/aws-sdk-go v1.16.11
github.com/davecgh/go-spew v1.1.1
github.com/go-sql-driver/mysql v0.0.0-20180719071942-99ff426eb706
github.com/golang/protobuf v1.3.1
github.com/gorilla/websocket v1.4.0 // indirect
github.com/hashicorp/go-msgpack v0.5.5 // indirect
github.com/hashicorp/memberlist v0.1.4 // indirect
github.com/hashicorp/serf v0.8.2
github.com/aws/aws-sdk-go-v2 v1.41.0
github.com/aws/aws-sdk-go-v2/config v1.32.6
github.com/aws/aws-sdk-go-v2/credentials v1.19.6
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.18
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0
github.com/bluele/gcache v0.0.2
github.com/brk0v/directio v0.0.0-20241105172640-ae9d82eb8fee
github.com/btcsuite/btcd v0.24.2
github.com/btcsuite/btcutil v1.0.2
github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
github.com/ekyoung/gin-nice-recovery v0.0.0-20160510022553-1654dca486db
github.com/gin-gonic/gin v1.11.0
github.com/go-sql-driver/mysql v1.9.3
github.com/gogo/protobuf v1.3.2
github.com/golang/protobuf v1.5.4
github.com/google/gops v0.3.28
github.com/gorilla/mux v1.8.1
github.com/hashicorp/serf v0.10.2
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22 // indirect
github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07
github.com/lbryio/lbry.go v0.0.0-20190828131228-f3a1fbdd5303
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c
github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6
github.com/sirupsen/logrus v1.3.0
github.com/spf13/cast v1.3.0 // indirect
github.com/spf13/cobra v0.0.0-20180722215644-7c4570c3ebeb
github.com/spf13/pflag v1.0.3 // indirect
github.com/uber-go/atomic v1.3.2
golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c // indirect
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c // indirect
golang.org/x/sync v0.0.0-20190423024810-112230192c58 // indirect
golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5 // indirect
golang.org/x/text v0.3.2 // indirect
google.golang.org/appengine v1.4.0 // indirect
github.com/karrick/godirwalk v1.17.0
github.com/lbryio/chainquery v1.9.1-0.20240927170248-48c092515dea
github.com/lbryio/lbry.go/v2 v2.7.2-0.20230307181431-a01aa6dc0629
github.com/lbryio/types v0.0.0-20220224142228-73610f6654a6
github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5
github.com/prometheus/client_golang v1.23.2
github.com/quic-go/quic-go v0.58.0
github.com/sirupsen/logrus v1.9.3
github.com/spf13/cast v1.10.0
github.com/spf13/cobra v1.10.2
github.com/spf13/viper v1.21.0
github.com/stretchr/testify v1.11.1
github.com/volatiletech/null/v8 v8.1.2
go.uber.org/atomic v1.11.0
golang.org/x/sync v0.19.0
)
require (
filippo.io/edwards25519 v1.1.0 // indirect
github.com/armon/go-metrics v0.4.1 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 // indirect
github.com/aws/smithy-go v1.24.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/btcsuite/btclog v1.0.0 // indirect
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd // indirect
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 // indirect
github.com/bytedance/sonic v1.14.0 // indirect
github.com/bytedance/sonic/loader v0.3.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/cloudwego/base64x v0.1.6 // indirect
github.com/friendsofgo/errors v0.9.2 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.8 // indirect
github.com/gin-contrib/sse v1.1.0 // indirect
github.com/go-errors/errors v1.5.1 // indirect
github.com/go-ini/ini v1.67.0 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.27.0 // indirect
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
github.com/goccy/go-json v0.10.5 // indirect
github.com/goccy/go-yaml v1.18.0 // indirect
github.com/gofrs/uuid v4.4.0+incompatible // indirect
github.com/google/btree v1.1.2 // indirect
github.com/gorilla/rpc v1.2.0 // indirect
github.com/gorilla/websocket v1.5.3 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
github.com/hashicorp/go-metrics v0.5.4 // indirect
github.com/hashicorp/go-msgpack/v2 v2.1.2 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-sockaddr v1.0.5 // indirect
github.com/hashicorp/golang-lru v1.0.2 // indirect
github.com/hashicorp/memberlist v0.5.2 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/miekg/dns v1.1.56 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.66.1 // indirect
github.com/prometheus/procfs v0.16.1 // indirect
github.com/quic-go/qpack v0.6.0 // indirect
github.com/sagikazarmark/locafero v0.11.0 // indirect
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
github.com/shopspring/decimal v1.4.0 // indirect
github.com/slack-go/slack v0.16.0 // indirect
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
github.com/spf13/afero v1.15.0 // indirect
github.com/spf13/pflag v1.0.10 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.3.0 // indirect
github.com/volatiletech/inflect v0.0.1 // indirect
github.com/volatiletech/randomize v0.0.1 // indirect
github.com/volatiletech/strmangle v0.0.8 // indirect
go.yaml.in/yaml/v2 v2.4.2 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/arch v0.20.0 // indirect
golang.org/x/crypto v0.41.0 // indirect
golang.org/x/mod v0.27.0 // indirect
golang.org/x/net v0.43.0 // indirect
golang.org/x/sys v0.35.0 // indirect
golang.org/x/text v0.28.0 // indirect
golang.org/x/time v0.8.0 // indirect
golang.org/x/tools v0.36.0 // indirect
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
google.golang.org/protobuf v1.36.9 // indirect
gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
+413 -159
View File
@@ -1,234 +1,488 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM=
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
github.com/aws/aws-sdk-go v1.16.11 h1:g/c7gJeVyHoXCxM2fddS85bPGVkBF8s2q8t3fyElegc=
github.com/aws/aws-sdk-go v1.16.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA=
github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
github.com/aws/aws-sdk-go-v2 v1.41.0 h1:tNvqh1s+v0vFYdA1xq0aOJH+Y5cRyZ5upu6roPgPKd4=
github.com/aws/aws-sdk-go-v2 v1.41.0/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
github.com/aws/aws-sdk-go-v2/config v1.32.6 h1:hFLBGUKjmLAekvi1evLi5hVvFQtSo3GYwi+Bx4lpJf8=
github.com/aws/aws-sdk-go-v2/config v1.32.6/go.mod h1:lcUL/gcd8WyjCrMnxez5OXkO3/rwcNmvfno62tnXNcI=
github.com/aws/aws-sdk-go-v2/credentials v1.19.6 h1:F9vWao2TwjV2MyiyVS+duza0NIRtAslgLUM0vTA1ZaE=
github.com/aws/aws-sdk-go-v2/credentials v1.19.6/go.mod h1:SgHzKjEVsdQr6Opor0ihgWtkWdfRAIwxYzSJ8O85VHY=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 h1:80+uETIWS1BqjnN9uJ0dBUaETh+P1XwFy5vwHwK5r9k=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16/go.mod h1:wOOsYuxYuB/7FlnVtzeBYRcjSRtQpAW0hCP7tIULMwo=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.18 h1:9vWXHtaepwoAl/UuKzxwgOoJDXPCC3hvgNMfcmdS2Tk=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.18/go.mod h1:sKuUZ+MwUTuJbYvZ8pK0x10LvgcJK3Y4rmh63YBekwk=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 h1:rgGwPzb82iBYSvHMHXc8h9mRoOUBZIGFgKb9qniaZZc=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16/go.mod h1:L/UxsGeKpGoIj6DxfhOWHWQ/kGKcd4I1VncE4++IyKA=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 h1:1jtGzuV7c82xnqOVfx2F0xmJcOw5374L7N6juGW6x6U=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16/go.mod h1:M2E5OQf+XLe+SZGmmpaI2yy+J326aFf6/+54PoxSANc=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 h1:CjMzUs78RDDv4ROu3JnJn/Ig1r6ZD7/T2DXLLRpejic=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16/go.mod h1:uVW4OLBqbJXSHJYA9svT9BluSvvwbzLQ2Crf6UPzR3c=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 h1:DIBqIrJ7hv+e4CmIk2z3pyKT+3B6qVMgRsawHiR3qso=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7/go.mod h1:vLm00xmBke75UmpNvOcZQ/Q30ZFjbczeLFqGx5urmGo=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 h1:oHjJHeUy0ImIV0bsrX0X91GkV5nJAyv1l1CC9lnO0TI=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16/go.mod h1:iRSNGgOYmiYwSCXxXaKb9HfOEj40+oTKn8pTxMlYkRM=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 h1:NSbvS17MlI2lurYgXnCOLvCFX38sBW4eiVER7+kkgsU=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16/go.mod h1:SwT8Tmqd4sA6G1qaGdzWCJN99bUmPGHfRwwq3G5Qb+A=
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0 h1:MIWra+MSq53CFaXXAywB2qg9YvVZifkk6vEGl/1Qor0=
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0/go.mod h1:79S2BdqCJpScXZA2y+cpZuocWsjGjJINyXnOsf5DTz8=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 h1:HpI7aMmJ+mm1wkSHIA2t5EaFFv5EFYXePW30p1EIrbQ=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.4/go.mod h1:C5RdGMYGlfM0gYq/tifqgn4EbyX99V15P2V3R+VHbQU=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 h1:aM/Q24rIlS3bRAhTyFurowU8A0SMyGDtEOY/l/s/1Uw=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.8/go.mod h1:+fWt2UHSb4kS7Pu8y+BMBvJF0EWx+4H0hzNwtDNRTrg=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 h1:AHDr0DaHIAo8c9t1emrzAlVDFp+iMMKnPdYy6XO4MCE=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12/go.mod h1:GQ73XawFFiWxyWXMHWfhiomvP3tXtdNar/fi8z18sx0=
github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 h1:SciGFVNZ4mHdm7gpD1dgZYnCuVdX1s+lFTg4+4DOy70=
github.com/aws/aws-sdk-go-v2/service/sts v1.41.5/go.mod h1:iW40X4QBmUxdP+fZNOpfmkdMZqsovezbAeO+Ubiv2pk=
github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk=
github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw=
github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0=
github.com/brk0v/directio v0.0.0-20241105172640-ae9d82eb8fee h1:gwVgfLo8dvQMauElELV1b1kwMuUh2ETzX6weEbdAKgA=
github.com/brk0v/directio v0.0.0-20241105172640-ae9d82eb8fee/go.mod h1:M/KA3XJG5PJaApPiv4gWNsgcSJquOQTqumZNLyYE0KM=
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803 h1:j3AgPKKZtZStM2nyhrDSLSYgT7YHrZKdSkq1OYeLjvM=
github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
github.com/btcsuite/btclog v1.0.0 h1:sEkpKJMmfGiyZjADwEIgB1NSwMyfdD1FB8v6+w1T0Ns=
github.com/btcsuite/btclog v1.0.0/go.mod h1:w7xnGOhwT3lmrS4H3b/D1XAXxvh+tbhUm8xeHN2y3TQ=
github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
github.com/btcsuite/btcutil v1.0.2 h1:9iZ1Terx9fMIOtq1VrwdqfsATL9MC2l8ZrUY6YZ2uts=
github.com/btcsuite/btcutil v1.0.2/go.mod h1:j9HUFwoQRsZL3V4n+qG+CUnEGHOarIxfC3Le2Yhbcts=
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd h1:R/opQEbFEy9JGkIguV40SvRY1uliPX8ifOvi6ICsFCw=
github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg=
github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY=
github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3SkEwmHoWBmX1DNXhXZqlTpq6s4tyJGc=
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
github.com/bytedance/sonic v1.14.0 h1:/OfKt8HFw0kh2rj8N0F6C/qPGRESq0BbaNZgcNXXzQQ=
github.com/bytedance/sonic v1.14.0/go.mod h1:WoEbx8WTcFJfzCe0hbmyTGrfjt8PzNEBdxlNUO24NhA=
github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA=
github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI=
github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 h1:6lhrsTEnloDPXyeZBvSYvQf8u86jbKehZPVDDlkgDl4=
github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M=
github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU=
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/ekyoung/gin-nice-recovery v0.0.0-20160510022553-1654dca486db h1:oZ4U9IqO8NS+61OmGTBi8vopzqTRxwQeogyBHdrhjbc=
github.com/ekyoung/gin-nice-recovery v0.0.0-20160510022553-1654dca486db/go.mod h1:Pk7/9x6tyChFTkahDvLBQMlvdsWvfC+yU8HTT5VD314=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/friendsofgo/errors v0.9.2 h1:X6NYxef4efCBdwI7BgS820zFaN7Cphrmb+Pljdzjtgk=
github.com/friendsofgo/errors v0.9.2/go.mod h1:yCvFW5AkDIL9qn7suHVLiI/gH228n7PC4Pn44IGoTOI=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
github.com/go-ini/ini v1.38.2/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
github.com/go-sql-driver/mysql v0.0.0-20180719071942-99ff426eb706 h1:P3NPKb7qq581SeMCB+dU1SuCX1kQh8VoQ/4HmT2ftQY=
github.com/go-sql-driver/mysql v0.0.0-20180719071942-99ff426eb706/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM=
github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8=
github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w=
github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM=
github.com/gin-gonic/gin v1.11.0 h1:OW/6PLjyusp2PPXtyxKHU0RbX6I/l28FTdDlae5ueWk=
github.com/gin-gonic/gin v1.11.0/go.mod h1:+iq/FyxlGzII0KHiBGjuNn4UNENUlKbGlNmc+W50Dls=
github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk=
github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
github.com/go-playground/validator/v10 v10.27.0 h1:w8+XrWVMhGkxOaaowyKH35gFydVHOvC0/uWoy2Fzwn4=
github.com/go-playground/validator/v10 v10.27.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo=
github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-test/deep v1.0.4 h1:u2CU3YKy9I2pmu9pX0eq50wCgjfGIt539SqR7FbHiho=
github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw=
github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA=
github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8=
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk=
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/rpc v1.1.0 h1:marKfvVP0Gpd/jHlVBKCQ8RAoUPdX7K1Nuh6l1BNh7A=
github.com/gorilla/rpc v1.1.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ=
github.com/gorilla/websocket v1.2.0 h1:VJtLvh6VQym50czpZzx07z/kw9EgAxI3x1ZB8taTMQQ=
github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU=
github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/gops v0.3.28 h1:2Xr57tqKAmQYRAfG12E+yLcoa2Y42UJo2lOrUFL9ark=
github.com/google/gops v0.3.28/go.mod h1:6f6+Nl8LcHrzJwi8+p0ii+vmBFSlB4f8cOOkTJ7sk4c=
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
github.com/gorilla/rpc v1.2.0 h1:WvvdC2lNeT1SP32zrIce5l0ECBfbAlmrmSBsuc57wfk=
github.com/gorilla/rpc v1.2.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0=
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI=
github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc=
github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
github.com/hashicorp/go-metrics v0.5.4 h1:8mmPiIJkTPPEbAiV97IxdAGNdRdaWwVap1BU6elejKY=
github.com/hashicorp/go-metrics v0.5.4/go.mod h1:CG5yz4NZ/AI/aQt9Ucm/vdBnbh7fvmv4lxZ350i+QQI=
github.com/hashicorp/go-msgpack/v2 v2.1.2 h1:4Ee8FTp834e+ewB71RDrQ0VKpyFdrKOjvYtnQ/ltVj0=
github.com/hashicorp/go-msgpack/v2 v2.1.2/go.mod h1:upybraOAblm4S7rx0+jeNy+CWWhzywQsSRV5033mMu4=
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs=
github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
github.com/hashicorp/go-sockaddr v1.0.5 h1:dvk7TIXCZpmfOlM+9mlcrWmWjw/wlKT+VDq2wMvfPJU=
github.com/hashicorp/go-sockaddr v1.0.5/go.mod h1:uoUUmtwU7n9Dv3O4SNLeFvg0SxQ3lyjsj6+CCykpaxI=
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE=
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
github.com/hashicorp/memberlist v0.1.4 h1:gkyML/r71w3FL8gUi74Vk76avkj/9lYAY9lvg0OcoGs=
github.com/hashicorp/memberlist v0.1.4/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
github.com/hashicorp/serf v0.8.2 h1:YZ7UKsJv+hKjqGVUUbtE3HNj79Eln2oQ75tniF6iPt0=
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c=
github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/hashicorp/memberlist v0.5.2 h1:rJoNPWZ0juJBgqn48gjy59K5H4rNgvUoM1kUD7bXiuI=
github.com/hashicorp/memberlist v0.5.2/go.mod h1:Ri9p/tRShbjYnpNf4FFPXG7wxEGY4Nrcn6E7jrVa//4=
github.com/hashicorp/serf v0.10.2 h1:m5IORhuNSjaxeljg5DeQVDlQyVkhRIjJDimbkCa8aAc=
github.com/hashicorp/serf v0.10.2/go.mod h1:T1CmSGfSeGfnfNy/w0odXQUR1rfECGd2Qdsp84DjOiY=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf h1:WfD7VjIE6z8dIvMsI4/s+1qr5EL+zoIGev1BQj1eoJ8=
github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h1:hyb9oH7vZsitZCiBt0ZvifOrB+qc8PS5IiilCIb87rg=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22 h1:jKUP9TQ0c7X3w6+IPyMit07RE42MtTWNd77sN2cHngQ=
github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22/go.mod h1:u0Jo4f2dNlTJeeOywkM6bLwxq6gC3pZ9rEFHn3AhTdk=
github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07 h1:+kBG/8rjCa6vxJZbUjAiE4MQmBEBYc8nLEb51frnvBY=
github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07/go.mod h1:j1kV/8f3jowErEq4XyeypkCdvg5EeHkf0YCKCcq5Ybo=
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwSWoI=
github.com/karrick/godirwalk v1.17.0/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c h1:BhdcWGsuKif/XoSZnqVGNqJ1iEmH0czWR5upj+AuR8M=
github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8=
github.com/lbryio/lbry.go v0.0.0-20190828131228-f3a1fbdd5303 h1:CyDDxUMREhAxPlgP+mgcArgkGJKtdXssj7CXk7o3h84=
github.com/lbryio/lbry.go v0.0.0-20190828131228-f3a1fbdd5303/go.mod h1:qR+Ui0hYhemIU4fXqM3d1P9eiaRFlof777VJgV7KJ8w=
github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4=
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c h1:m3O7561xBQ00lfUVayW4c6SnpVbUDQtPUwGcGYSUYQA=
github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5 h1:AsEBgzv3DhuYHI/GiQh2HxvTP71HCCE9E/tzGUzGdtU=
github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0=
github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6 h1:iOAVXzZyXtW408TMYejlUPo6BIn92HmOacWtIfNyYns=
github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg=
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/lbryio/chainquery v1.9.1-0.20240927170248-48c092515dea h1:Rr5hL3GibxiP221+3tBX7Ep9+13jszLXdqftM2oEPFc=
github.com/lbryio/chainquery v1.9.1-0.20240927170248-48c092515dea/go.mod h1:dzrrwADJ2CGlI66MyRoWKWX1NmewTiQ3II4fpaTJjZ4=
github.com/lbryio/lbry.go/v2 v2.7.2-0.20230307181431-a01aa6dc0629 h1:klpHPQ5iERUhczdITuKUpYuUZrWDGWb3zlAv3qYgc+o=
github.com/lbryio/lbry.go/v2 v2.7.2-0.20230307181431-a01aa6dc0629/go.mod h1:JTkXBAVK8iHNcYmffbLzQ7IFKd/+/oBQGIwiG53bbqw=
github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19 h1:/zWD8dVIl7bV1TdJWqPqy9tpqixzX2Qxgit48h3hQcY=
github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
github.com/lbryio/types v0.0.0-20220224142228-73610f6654a6 h1:IhL9D2QfDWhLNDQpZ3Uiiw0gZEUYeLBS6uDqOd59G5o=
github.com/lbryio/types v0.0.0-20220224142228-73610f6654a6/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 h1:mG83tLXWSRdcXMWfkoumVwhcCbf3jHF9QKv/m37BkM0=
github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCWffGOaDcjkw1iB7W9DVLp6GXmfcJY/7YZCWPA4=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v0.0.0-20180511142126-bb74f1db0675/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/nlopes/slack v0.5.0 h1:NbIae8Kd0NpqaEI3iUrsuS0KbcEDhzhc939jLW5fNm0=
github.com/nlopes/slack v0.5.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM=
github.com/miekg/dns v1.1.56 h1:5imZaSeoRNvpM9SzWNhEcP9QliKiz20/dA2QabIGVnE=
github.com/miekg/dns v1.1.56/go.mod h1:cRm6Oo2C8TY9ZS/TqsSrseAcncm74lfK5G+ikN2SWWY=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6 h1:2bae6N0SZjgzk+Zg8mzTsfmpwHXY9VBNp9UdjhaElA0=
github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 h1:Ii+DKncOVM8Cu1Hc+ETb5K+23HdAMvESYE3ZJ5b5cMI=
github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
github.com/quic-go/quic-go v0.58.0 h1:ggY2pvZaVdB9EyojxL1p+5mptkuHyX5MOSv4dgWF4Ug=
github.com/quic-go/quic-go v0.58.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/sebdah/goldie v0.0.0-20180424091453-8784dd1ab561 h1:IY+sDBJR/wRtsxq+626xJnt4Tw7/ROA9cDIR8MMhWyg=
github.com/sebdah/goldie v0.0.0-20180424091453-8784dd1ab561/go.mod h1:lvjGftC8oe7XPtyrOidaMi0rp5B9+XY/ZRUynGnuaxQ=
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/shopspring/decimal v0.0.0-20180607144847-19e3cb6c2930/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
github.com/sebdah/goldie v1.0.0 h1:9GNhIat69MSlz/ndaBg48vl9dF5fI+NBB6kfOxgfkMc=
github.com/sebdah/goldie v1.0.0/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4=
github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME=
github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
github.com/spf13/cast v1.2.0 h1:HHl1DSRbEQN2i8tJmtS6ViPyHx35+p51amrdsiTCrkg=
github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.0-20180722215644-7c4570c3ebeb h1:9EsYJzSlhhaP+nYmMOcptMF2VEUH52jxPzt/TX14KWM=
github.com/spf13/cobra v0.0.0-20180722215644-7c4570c3ebeb/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/slack-go/slack v0.16.0 h1:khp/WCFv+Hb/B/AJaAwvcxKun0hM6grN0bUZ8xG60P8=
github.com/slack-go/slack v0.16.0/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw=
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw=
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U=
github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
github.com/uber-go/atomic v1.3.2 h1:Azu9lPBWRNKzYXSIwRfgRuDuS0YKsK4NFhiQv98gkxo=
github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE=
go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
github.com/ugorji/go/codec v1.3.0 h1:Qd2W2sQawAfG8XSvzwhBeoGq71zXOC/Q1E9y/wUcsUA=
github.com/ugorji/go/codec v1.3.0/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4=
github.com/volatiletech/inflect v0.0.1 h1:2a6FcMQyhmPZcLa+uet3VJ8gLn/9svWhJxJYwvE8KsU=
github.com/volatiletech/inflect v0.0.1/go.mod h1:IBti31tG6phkHitLlr5j7shC5SOo//x0AjDzaJU1PLA=
github.com/volatiletech/null/v8 v8.1.2 h1:kiTiX1PpwvuugKwfvUNX/SU/5A2KGZMXfGD0DUHdKEI=
github.com/volatiletech/null/v8 v8.1.2/go.mod h1:98DbwNoKEpRrYtGjWFctievIfm4n4MxG0A6EBUcoS5g=
github.com/volatiletech/randomize v0.0.1 h1:eE5yajattWqTB2/eN8df4dw+8jwAzBtbdo5sbWC4nMk=
github.com/volatiletech/randomize v0.0.1/go.mod h1:GN3U0QYqfZ9FOJ67bzax1cqZ5q2xuj2mXrXBjWaRTlY=
github.com/volatiletech/strmangle v0.0.1/go.mod h1:F6RA6IkB5vq0yTG4GQ0UsbbRcl3ni9P76i+JrTBKFFg=
github.com/volatiletech/strmangle v0.0.8 h1:UZkTDFIjZcL1Lk4BXhGsxcyXxNcWuM5ZwdzZc0sJcWg=
github.com/volatiletech/strmangle v0.0.8/go.mod h1:ycDvbDkjDvhC0NUU8w3fWwl5JEMTV56vTKXzR3GeR+0=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko=
go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o=
go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
golang.org/x/arch v0.20.0 h1:dx1zTU0MAE98U+TQ8BLl7XsJbgze2WnNKF/8tGp/Q6c=
golang.org/x/arch v0.20.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk=
golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c h1:Vj5n4GlwjmQteupaxJ9+0FNOmBrHfq7vN4btdGoDZgI=
golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4=
golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ=
golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c h1:uOCk1iQW6Vc18bnC13MfzScl+wdKBmM9Y9kU7Z83/lw=
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE=
golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190520201301-c432e742b0af/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5 h1:sM3evRHxE/1RuMe1FYAL3j7C7fUfIjkbE+NiDAYUF8U=
golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg=
golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg=
golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY=
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.36.9 h1:w2gp2mA27hUeUzj9Ex9FBjsBm40zfaDtEWow293U7Iw=
google.golang.org/protobuf v1.36.9/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 h1:FpCr9V8wuOei4BAen+93HtVJ+XSi+KPbaPKm0Vj5R64=
gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79/go.mod h1:gWkaRU7CoXpezCBWfWjm3999QqS+1pYPXGbqQCTMzo8=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+344
View File
@@ -0,0 +1,344 @@
package metrics
import (
"context"
"encoding/json"
"errors"
"io"
"net/http"
"strings"
"syscall"
"time"
ee "github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/prometheus/client_golang/prometheus/promhttp"
log "github.com/sirupsen/logrus"
)
type Server struct {
srv *http.Server
stop *stop.Stopper
}
func NewServer(address string, path string) *Server {
h := http.NewServeMux()
h.Handle(path, promhttp.Handler())
return &Server{
srv: &http.Server{
Addr: address,
Handler: h,
//https://blog.cloudflare.com/the-complete-guide-to-golang-net-http-timeouts/
//https://blog.cloudflare.com/exposing-go-on-the-internet/
ReadTimeout: 5 * time.Second,
WriteTimeout: 10 * time.Second,
IdleTimeout: 120 * time.Second,
},
stop: stop.New(),
}
}
func (s *Server) Start() {
s.stop.Add(1)
go func() {
defer s.stop.Done()
err := s.srv.ListenAndServe()
if err != nil && !errors.Is(err, http.ErrServerClosed) {
log.Error(err)
}
}()
}
func (s *Server) Shutdown() {
_ = s.srv.Shutdown(context.Background())
s.stop.StopAndWait()
}
const (
ns = "reflector"
subsystemCache = "cache"
subsystemITTT = "ittt"
labelDirection = "direction"
labelErrorType = "error_type"
DirectionUpload = "upload" // to reflector
DirectionDownload = "download" // from reflector
LabelCacheType = "cache_type"
LabelOrigin = "origin"
LabelComponent = "component"
LabelSource = "source"
errConnReset = "conn_reset"
errReadConnReset = "read_conn_reset"
errWriteConnReset = "write_conn_reset"
errReadConnTimedOut = "read_conn_timed_out"
errNoNetworkActivity = "no_network_activity"
errWriteConnTimedOut = "write_conn_timed_out"
errWriteBrokenPipe = "write_broken_pipe"
errEPipe = "e_pipe"
errETimedout = "e_timedout"
errIOTimeout = "io_timeout"
errUnexpectedEOF = "unexpected_eof"
errUnexpectedEOFStr = "unexpected_eof_str"
errJSONSyntax = "json_syntax"
errBlobTooBig = "blob_too_big"
errInvalidPeerJSON = "invalid_peer_json"
errInvalidPeerData = "invalid_peer_data"
errRequestTooLarge = "request_too_large"
errDeadlineExceeded = "deadline_exceeded"
errHashMismatch = "hash_mismatch"
errProtectedBlob = "protected_blob"
errInvalidBlobHash = "invalid_blob_hash"
errZeroByteBlob = "zero_byte_blob"
errInvalidCharacter = "invalid_character"
errBlobNotFound = "blob_not_found"
errNoErr = "no_error"
errQuicProto = "quic_protocol_violation"
errOther = "other"
)
var (
ErrorCount = promauto.NewCounterVec(prometheus.CounterOpts{
Namespace: ns,
Name: "error_total",
Help: "Total number of errors",
}, []string{labelDirection, labelErrorType})
BlobDownloadCount = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "blob_download_total",
Help: "Total number of blobs downloaded from reflector",
})
PeerDownloadCount = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "peer_download_total",
Help: "Total number of blobs downloaded from reflector through tcp protocol",
})
Http3DownloadCount = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "http3_blob_download_total",
Help: "Total number of blobs downloaded from reflector through QUIC protocol",
})
HttpDownloadCount = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "http_blob_download_total",
Help: "Total number of blobs downloaded from reflector through HTTP protocol",
})
CacheHitCount = promauto.NewCounterVec(prometheus.CounterOpts{
Namespace: ns,
Subsystem: subsystemCache,
Name: "hit_total",
Help: "Total number of blobs retrieved from the cache storage",
}, []string{LabelCacheType, LabelComponent})
ItttHitCount = promauto.NewCounterVec(prometheus.CounterOpts{
Namespace: ns,
Subsystem: subsystemITTT,
Name: "hits_total",
Help: "Total number of blobs retrieved from the this/that storage",
}, []string{LabelOrigin})
CacheMissCount = promauto.NewCounterVec(prometheus.CounterOpts{
Namespace: ns,
Subsystem: subsystemCache,
Name: "miss_total",
Help: "Total number of blobs retrieved from origin rather than cache storage",
}, []string{LabelCacheType, LabelComponent})
CacheOriginRequestsCount = promauto.NewGaugeVec(prometheus.GaugeOpts{
Namespace: ns,
Subsystem: subsystemCache,
Name: "origin_requests_total",
Help: "How many Get requests are in flight from the cache to the origin",
}, []string{LabelCacheType, LabelComponent})
//during thundering-herd situations, the metric below should be a lot smaller than the metric above
CacheWaitingRequestsCount = promauto.NewGaugeVec(prometheus.GaugeOpts{
Namespace: ns,
Subsystem: subsystemCache,
Name: "waiting_requests_total",
Help: "How many cache requests are waiting for an in-flight origin request",
}, []string{LabelCacheType, LabelComponent})
CacheLRUEvictCount = promauto.NewCounterVec(prometheus.CounterOpts{
Namespace: ns,
Subsystem: subsystemCache,
Name: "evict_total",
Help: "Count of blobs evicted from cache",
}, []string{LabelCacheType, LabelComponent})
CacheRetrievalSpeed = promauto.NewGaugeVec(prometheus.GaugeOpts{
Namespace: ns,
Name: "speed_mbps",
Help: "Speed of blob retrieval from cache or from origin",
}, []string{LabelCacheType, LabelComponent, LabelSource})
BlobUploadCount = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "blob_upload_total",
Help: "Total number of blobs uploaded to reflector",
})
SDBlobUploadCount = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "sdblob_upload_total",
Help: "Total number of SD blobs (and therefore streams) uploaded to reflector",
})
MtrInBytesTcp = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "tcp_in_bytes",
Help: "Total number of bytes downloaded through TCP",
})
MtrOutBytesTcp = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "tcp_out_bytes",
Help: "Total number of bytes streamed out through TCP",
})
MtrInBytesUdp = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "udp_in_bytes",
Help: "Total number of bytes downloaded through UDP",
})
MtrInBytesHttp = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "http_in_bytes",
Help: "Total number of bytes downloaded through HTTP",
})
MtrOutBytesUdp = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "udp_out_bytes",
Help: "Total number of bytes streamed out through UDP",
})
MtrOutBytesHttp = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "http_out_bytes",
Help: "Total number of bytes streamed out through UDP",
})
MtrInBytesReflector = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "reflector_in_bytes",
Help: "Total number of incoming bytes (from users)",
})
MtrOutBytesReflector = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "s3_out_bytes",
Help: "Total number of outgoing bytes (to S3)",
})
MtrInBytesUpstream = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "upstream_in_bytes",
Help: "Total number of incoming bytes (from Upstream)",
})
MtrInBytesS3 = promauto.NewCounter(prometheus.CounterOpts{
Namespace: ns,
Name: "s3_in_bytes",
Help: "Total number of incoming bytes (from S3-CF)",
})
Http3BlobReqQueue = promauto.NewGauge(prometheus.GaugeOpts{
Namespace: ns,
Name: "http3_blob_request_queue_size",
Help: "Blob requests of https queue size",
})
HttpBlobReqQueue = promauto.NewGauge(prometheus.GaugeOpts{
Namespace: ns,
Name: "http_blob_request_queue_size",
Help: "Blob requests queue size of the HTTP protocol",
})
RoutinesQueue = promauto.NewGaugeVec(prometheus.GaugeOpts{
Namespace: ns,
Name: "routines",
Help: "routines running by type",
}, []string{"package", "kind"})
)
func CacheLabels(name, component string) prometheus.Labels {
return prometheus.Labels{
LabelCacheType: name,
LabelComponent: component,
}
}
func ItttLabels(orig string) prometheus.Labels {
return prometheus.Labels{
LabelOrigin: orig,
}
}
func TrackError(direction string, e error) (shouldLog bool) { // shouldLog is a hack, but whatever
if e == nil {
return
}
err := ee.Wrap(e, 0)
errType := errOther
if strings.Contains(err.Error(), "i/o timeout") {
errType = errIOTimeout
} else if errors.Is(e, syscall.ECONNRESET) {
// Looks like we're getting this when direction == "download", but read_conn_reset and
// write_conn_reset when its "upload"
errType = errConnReset
} else if errors.Is(e, context.DeadlineExceeded) {
errType = errDeadlineExceeded
} else if strings.Contains(err.Error(), "read: connection reset by peer") { // the other side closed the connection using TCP reset
errType = errReadConnReset
} else if strings.Contains(err.Error(), "write: connection reset by peer") { // the other side closed the connection using TCP reset
errType = errWriteConnReset
} else if errors.Is(e, syscall.ETIMEDOUT) {
errType = errETimedout
} else if strings.Contains(err.Error(), "read: connection timed out") { // the other side closed the connection using TCP reset
//log.Warnln("read conn timed out is not the same as ETIMEDOUT")
errType = errReadConnTimedOut
} else if strings.Contains(err.Error(), "NO_ERROR: No recent network activity") { // the other side closed the QUIC connection
//log.Warnln("read conn timed out is not the same as ETIMEDOUT")
errType = errNoNetworkActivity
} else if strings.Contains(err.Error(), "write: connection timed out") {
errType = errWriteConnTimedOut
} else if errors.Is(e, io.ErrUnexpectedEOF) {
errType = errUnexpectedEOF
} else if strings.Contains(err.Error(), "unexpected EOF") { // tried to read from closed pipe or socket
errType = errUnexpectedEOFStr
} else if errors.Is(e, syscall.EPIPE) {
errType = errEPipe
} else if strings.Contains(err.Error(), "write: broken pipe") { // tried to write to a pipe or socket that was closed by the peer
// I believe this is the same as EPipe when direction == "download", but not for upload
errType = errWriteBrokenPipe
//} else if errors.Is(e, reflector.ErrBlobTooBig) { # this creates a circular import
// errType = errBlobTooBig
} else if strings.Contains(err.Error(), "blob must be at most") {
//log.Warnln("blob must be at most X bytes is not the same as ErrBlobTooBig")
errType = errBlobTooBig
} else if strings.Contains(err.Error(), "invalid json request") {
errType = errInvalidPeerJSON
} else if strings.Contains(err.Error(), "Invalid data") {
errType = errInvalidPeerData
} else if strings.Contains(err.Error(), "request is too large") {
errType = errRequestTooLarge
} else if strings.Contains(err.Error(), "Invalid blob hash length") {
errType = errInvalidBlobHash
} else if strings.Contains(err.Error(), "hash of received blob data does not match hash from send request") {
errType = errHashMismatch
} else if strings.Contains(err.Error(), "blob not found") {
errType = errBlobNotFound
} else if strings.Contains(err.Error(), "requested blob is protected") {
errType = errProtectedBlob
} else if strings.Contains(err.Error(), "0-byte blob received") {
errType = errZeroByteBlob
} else if strings.Contains(err.Error(), "PROTOCOL_VIOLATION: tried to retire connection") {
errType = errQuicProto
} else if strings.Contains(err.Error(), "invalid character") {
errType = errInvalidCharacter
} else if _, ok := e.(*json.SyntaxError); ok {
errType = errJSONSyntax
} else if strings.Contains(err.Error(), "NO_ERROR") {
errType = errNoErr
} else {
log.Warnf("error '%s' for direction '%s' is not being tracked", err.TypeName(), direction)
shouldLog = true
}
ErrorCount.With(map[string]string{
labelDirection: direction,
labelErrorType: errType,
}).Inc()
return
}
+347
View File
@@ -0,0 +1,347 @@
package lite_db
import (
"database/sql"
"time"
"github.com/lbryio/lbry.go/v2/extras/errors"
qt "github.com/lbryio/lbry.go/v2/extras/query"
"github.com/go-sql-driver/mysql"
log "github.com/sirupsen/logrus"
"github.com/volatiletech/null/v8"
)
// SdBlob is a special blob that contains information on the rest of the blobs in the stream
type SdBlob struct {
StreamName string `json:"stream_name"`
StreamType string `json:"stream_type"`
Key string `json:"key"`
SuggestedFileName string `json:"suggested_file_name"`
StreamHash string `json:"stream_hash"`
Blobs []struct {
BlobHash string `json:"blob_hash,omitempty"`
IV string `json:"iv"`
Length int `json:"length"`
BlobNum int `json:"blob_num"`
} `json:"blobs"`
}
// SQL implements the DB interface
type SQL struct {
conn *sql.DB
TrackAccessTime bool
}
func logQuery(query string, args ...interface{}) {
s, err := qt.InterpolateParams(query, args...)
if err != nil {
log.Errorln(err)
} else {
log.Debugln(s)
}
}
// Connect will create a connection to the database
func (s *SQL) Connect(dsn string) error {
var err error
// interpolateParams is necessary. otherwise uploading a stream with thousands of blobs
// will hit MySQL's max_prepared_stmt_count limit because the prepared statements are all
// opened inside a transaction. closing them manually doesn't seem to help
dsn += "?parseTime=1&collation=utf8mb4_unicode_ci&interpolateParams=1"
s.conn, err = sql.Open("mysql", dsn)
if err != nil {
return errors.Err(err)
}
s.conn.SetMaxIdleConns(12)
return errors.Err(s.conn.Ping())
}
// AddBlob adds a blob to the database.
func (s *SQL) AddBlob(hash string, length int) error {
if s.conn == nil {
return errors.Err("not connected")
}
_, err := s.insertBlob(hash, length)
return err
}
func (s *SQL) insertBlob(hash string, length int) (int64, error) {
if length <= 0 {
return 0, errors.Err("length must be positive")
}
const isStored = true
now := time.Now()
args := []interface{}{hash, isStored, length, now}
blobID, err := s.exec(
"INSERT INTO blob_ (hash, is_stored, length, last_accessed_at) VALUES ("+qt.Qs(len(args))+") ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored)), last_accessed_at=VALUES(last_accessed_at)",
args...,
)
if err != nil {
return 0, err
}
if blobID == 0 {
err = s.conn.QueryRow("SELECT id FROM blob_ WHERE hash = ?", hash).Scan(&blobID)
if err != nil {
return 0, errors.Err(err)
}
if blobID == 0 {
return 0, errors.Err("blob ID is 0 even after INSERTing and SELECTing")
}
}
return blobID, nil
}
// HasBlob checks if the database contains the blob information.
func (s *SQL) HasBlob(hash string) (bool, error) {
exists, err := s.HasBlobs([]string{hash})
if err != nil {
return false, err
}
return exists[hash], nil
}
// HasBlobs checks if the database contains the set of blobs and returns a bool map.
func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
exists, streamsNeedingTouch, err := s.hasBlobs(hashes)
_ = s.touch(streamsNeedingTouch)
return exists, err
}
func (s *SQL) touch(blobIDs []uint64) error {
if len(blobIDs) == 0 {
return nil
}
query := "UPDATE blob_ SET last_accessed_at = ? WHERE id IN (" + qt.Qs(len(blobIDs)) + ")"
args := make([]interface{}, len(blobIDs)+1)
args[0] = time.Now()
for i := range blobIDs {
args[i+1] = blobIDs[i]
}
startTime := time.Now()
_, err := s.exec(query, args...)
log.Debugf("blobs access query touched %d blobs and took %s", len(blobIDs), time.Since(startTime))
return errors.Err(err)
}
func (s *SQL) hasBlobs(hashes []string) (map[string]bool, []uint64, error) {
if s.conn == nil {
return nil, nil, errors.Err("not connected")
}
var (
hash string
blobID uint64
lastAccessedAt null.Time
)
var needsTouch []uint64
exists := make(map[string]bool)
touchDeadline := time.Now().AddDate(0, 0, -1) // touch blob if last accessed before this time
maxBatchSize := 10000
doneIndex := 0
for len(hashes) > doneIndex {
sliceEnd := doneIndex + maxBatchSize
if sliceEnd > len(hashes) {
sliceEnd = len(hashes)
}
log.Debugf("getting hashes[%d:%d] of %d", doneIndex, sliceEnd, len(hashes))
batch := hashes[doneIndex:sliceEnd]
// TODO: this query doesn't work for SD blobs, which are not in the stream_blob table
query := `SELECT hash, id, last_accessed_at
FROM blob_
WHERE is_stored = ? and hash IN (` + qt.Qs(len(batch)) + `)`
args := make([]interface{}, len(batch)+1)
args[0] = true
for i := range batch {
args[i+1] = batch[i]
}
logQuery(query, args...)
err := func() error {
startTime := time.Now()
rows, err := s.conn.Query(query, args...)
log.Debugf("hashes query took %s", time.Since(startTime))
if err != nil {
return errors.Err(err)
}
defer closeRows(rows)
for rows.Next() {
err = rows.Scan(&hash, &blobID, &lastAccessedAt)
if err != nil {
return errors.Err(err)
}
exists[hash] = true
if s.TrackAccessTime && (!lastAccessedAt.Valid || lastAccessedAt.Time.Before(touchDeadline)) {
needsTouch = append(needsTouch, blobID)
}
}
err = rows.Err()
if err != nil {
return errors.Err(err)
}
doneIndex += len(batch)
return nil
}()
if err != nil {
return nil, nil, err
}
}
return exists, needsTouch, nil
}
// Delete will remove the blob from the db
func (s *SQL) Delete(hash string) error {
_, err := s.exec("UPDATE blob_ set is_stored = ? WHERE hash = ?", 0, hash)
return errors.Err(err)
}
// AddSDBlob insert the SD blob and all the content blobs. The content blobs are marked as "not stored",
// but they are tracked so reflector knows what it is missing.
func (s *SQL) AddSDBlob(sdHash string, sdBlobLength int) error {
if s.conn == nil {
return errors.Err("not connected")
}
_, err := s.insertBlob(sdHash, sdBlobLength)
return err
}
// GetHashRange gets the smallest and biggest hashes in the db
func (s *SQL) GetLRUBlobs(maxBlobs int) ([]string, error) {
if s.conn == nil {
return nil, errors.Err("not connected")
}
query := "SELECT hash from blob_ where is_stored = ? order by last_accessed_at limit ?"
const isStored = true
logQuery(query, isStored, maxBlobs)
rows, err := s.conn.Query(query, isStored, maxBlobs)
if err != nil {
return nil, errors.Err(err)
}
defer closeRows(rows)
blobs := make([]string, 0, maxBlobs)
for rows.Next() {
var hash string
err := rows.Scan(&hash)
if err != nil {
return nil, errors.Err(err)
}
blobs = append(blobs, hash)
}
return blobs, nil
}
func (s *SQL) AllBlobs() ([]string, error) {
if s.conn == nil {
return nil, errors.Err("not connected")
}
query := "SELECT hash from blob_ where is_stored = ?" //TODO: maybe sorting them makes more sense?
const isStored = true
logQuery(query, isStored)
rows, err := s.conn.Query(query, isStored)
if err != nil {
return nil, errors.Err(err)
}
defer closeRows(rows)
totalBlobs, err := s.BlobsCount()
if err != nil {
return nil, err
}
blobs := make([]string, 0, totalBlobs)
for rows.Next() {
var hash string
err := rows.Scan(&hash)
if err != nil {
return nil, errors.Err(err)
}
blobs = append(blobs, hash)
}
return blobs, nil
}
func (s *SQL) BlobsCount() (int, error) {
if s.conn == nil {
return 0, errors.Err("not connected")
}
query := "SELECT count(id) from blob_ where is_stored = ?" //TODO: maybe sorting them makes more sense?
const isStored = true
logQuery(query, isStored)
var count int
err := s.conn.QueryRow(query, isStored).Scan(&count)
return count, errors.Err(err)
}
func closeRows(rows *sql.Rows) {
if rows != nil {
err := rows.Close()
if err != nil {
log.Error("error closing rows: ", err)
}
}
}
func (s *SQL) exec(query string, args ...interface{}) (int64, error) {
logQuery(query, args...)
attempt, maxAttempts := 0, 3
Retry:
attempt++
result, err := s.conn.Exec(query, args...)
if isLockTimeoutError(err) {
if attempt <= maxAttempts {
//Error 1205: Lock wait timeout exceeded; try restarting transaction
goto Retry
}
err = errors.Prefix("Lock timeout for query "+query, err)
}
if err != nil {
return 0, errors.Err(err)
}
lastID, err := result.LastInsertId()
return lastID, errors.Err(err)
}
func isLockTimeoutError(err error) bool {
e, ok := err.(*mysql.MySQLError)
return ok && e != nil && e.Number == 1205
}
/* SQL schema
in prod make sure you use latin1 or utf8 charset, NOT utf8mb4. that's a waste of space.
CREATE TABLE `blob_` (
`id` bigint unsigned NOT NULL AUTO_INCREMENT,
`hash` char(96) NOT NULL,
`is_stored` tinyint(1) NOT NULL DEFAULT '0',
`length` bigint unsigned DEFAULT NULL,
`last_accessed_at` datetime DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
UNIQUE KEY `id` (`id`),
UNIQUE KEY `blob_hash_idx` (`hash`),
KEY `blob_last_accessed_idx` (`last_accessed_at`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1
*/
+6 -4
View File
@@ -1,13 +1,15 @@
package main
import (
"math/rand"
"time"
"github.com/lbryio/reflector.go/cmd"
"github.com/google/gops/agent"
log "github.com/sirupsen/logrus"
)
func main() {
rand.Seed(time.Now().UnixNano())
if err := agent.Listen(agent.Options{}); err != nil {
log.Fatal(err)
}
cmd.Execute()
}
+44 -6
View File
@@ -1,20 +1,58 @@
package meta
import (
"fmt"
"strconv"
"time"
)
var Version = ""
var Time = ""
var BuildTime time.Time
var (
name = "prism-bin"
version = "unknown"
commit = "unknown"
commitLong = "unknown"
branch = "unknown"
Time = "unknown"
BuildTime time.Time
)
// Name returns main application name
func Name() string {
return name
}
// Version returns current application version
func Version() string {
return version
}
// FullName returns current app version, commit and build time
func FullName() string {
return fmt.Sprintf(
`Name: %v
Version: %v
branch: %v
commit: %v
commit long: %v
build date: %v`, Name(), Version(), branch, commit, commitLong, BuildTime.String())
}
func init() {
if Time != "" {
t, err := strconv.Atoi(Time)
if err != nil {
return
if err == nil {
BuildTime = time.Unix(int64(t), 0).UTC()
}
BuildTime = time.Unix(int64(t), 0).UTC()
}
}
func VersionString() string {
var buildTime string
if BuildTime.IsZero() {
buildTime = "<now>"
} else {
buildTime = BuildTime.Format(time.RFC3339)
}
return fmt.Sprintf("version %s, built %s", version, buildTime)
}
+18 -21
View File
@@ -2,38 +2,35 @@ package prism
import (
"context"
"fmt"
"strconv"
"sync"
"github.com/lbryio/lbry.go/dht"
"github.com/lbryio/lbry.go/dht/bits"
"github.com/lbryio/reflector.go/cluster"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/peer"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/server/peer"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/stop"
"github.com/lbryio/lbry.go/v2/dht"
"github.com/lbryio/lbry.go/v2/dht/bits"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
log "github.com/sirupsen/logrus"
)
type Config struct {
PeerPort int
ReflectorPort int
DhtAddress string
DhtSeedNodes []string
ClusterPort int
ClusterSeedAddr string
// limit the range of hashes to announce. useful for testing
HashRange *bits.Range
DB *db.SQL
Blobs store.BlobStore
// limit the range of hashes to announce. useful for testing
HashRange *bits.Range
DB *db.SQL
DhtAddress string
ClusterSeedAddr string
DhtSeedNodes []string
PeerPort int
ReflectorPort int
ClusterPort int
}
// DefaultConf returns a default config
@@ -78,8 +75,8 @@ func New(conf *Config) *Prism {
db: conf.DB,
dht: d,
cluster: c,
peer: peer.NewServer(conf.Blobs),
reflector: reflector.NewServer(conf.Blobs),
peer: peer.NewServer(conf.Blobs, fmt.Sprintf(":%d", conf.PeerPort)),
reflector: reflector.NewIngestionServer(conf.Blobs),
grp: stop.New(),
}
@@ -107,7 +104,7 @@ func (p *Prism) Start() error {
return errors.Err("blobs required in conf")
}
err = p.peer.Start(":" + strconv.Itoa(p.conf.PeerPort))
err = p.peer.Start()
if err != nil {
return err
}
+2 -1
View File
@@ -4,8 +4,9 @@ import (
"math/big"
"testing"
"github.com/lbryio/lbry.go/v2/dht/bits"
"github.com/davecgh/go-spew/spew"
"github.com/lbryio/lbry.go/dht/bits"
)
func TestAnnounceRange(t *testing.T) {
+174
View File
@@ -0,0 +1,174 @@
package publish
import "strings"
func guessMimeType(ext string) (string, string) {
if ext == "" {
return "application/octet-stream", "binary"
}
ext = strings.ToLower(strings.TrimLeft(strings.TrimSpace(ext), "."))
types := map[string]struct{ mime, t string }{
"a": {"application/octet-stream", "binary"},
"ai": {"application/postscript", "image"},
"aif": {"audio/x-aiff", "audio"},
"aifc": {"audio/x-aiff", "audio"},
"aiff": {"audio/x-aiff", "audio"},
"au": {"audio/basic", "audio"},
"avi": {"video/x-msvideo", "video"},
"bat": {"text/plain", "document"},
"bcpio": {"application/x-bcpio", "binary"},
"bin": {"application/octet-stream", "binary"},
"bmp": {"image/bmp", "image"},
"c": {"text/plain", "document"},
"cdf": {"application/x-netcdf", "binary"},
"cpio": {"application/x-cpio", "binary"},
"csh": {"application/x-csh", "binary"},
"css": {"text/css", "document"},
"csv": {"text/csv", "document"},
"dll": {"application/octet-stream", "binary"},
"doc": {"application/msword", "document"},
"dot": {"application/msword", "document"},
"dvi": {"application/x-dvi", "binary"},
"eml": {"message/rfc822", "document"},
"eps": {"application/postscript", "document"},
"epub": {"application/epub+zip", "document"},
"etx": {"text/x-setext", "document"},
"exe": {"application/octet-stream", "binary"},
"gif": {"image/gif", "image"},
"gtar": {"application/x-gtar", "binary"},
"h": {"text/plain", "document"},
"hdf": {"application/x-hdf", "binary"},
"htm": {"text/html", "document"},
"html": {"text/html", "document"},
"ico": {"image/vnd.microsoft.icon", "image"},
"ief": {"image/ief", "image"},
"iges": {"model/iges", "model"},
"jpe": {"image/jpeg", "image"},
"jpeg": {"image/jpeg", "image"},
"jpg": {"image/jpeg", "image"},
"js": {"application/javascript", "document"},
"json": {"application/json", "document"},
"ksh": {"text/plain", "document"},
"latex": {"application/x-latex", "binary"},
"m1v": {"video/mpeg", "video"},
"m3u": {"application/vnd.apple.mpegurl", "audio"},
"m3u8": {"application/vnd.apple.mpegurl", "audio"},
"man": {"application/x-troff-man", "document"},
"markdown": {"text/markdown", "document"},
"md": {"text/markdown", "document"},
"me": {"application/x-troff-me", "binary"},
"mht": {"message/rfc822", "document"},
"mhtml": {"message/rfc822", "document"},
"mif": {"application/x-mif", "binary"},
"mov": {"video/quicktime", "video"},
"movie": {"video/x-sgi-movie", "video"},
"mp2": {"audio/mpeg", "audio"},
"mp3": {"audio/mpeg", "audio"},
"mp4": {"video/mp4", "video"},
"mpa": {"video/mpeg", "video"},
"mpe": {"video/mpeg", "video"},
"mpeg": {"video/mpeg", "video"},
"mpg": {"video/mpeg", "video"},
"ms": {"application/x-troff-ms", "binary"},
"nc": {"application/x-netcdf", "binary"},
"nws": {"message/rfc822", "document"},
"o": {"application/octet-stream", "binary"},
"obj": {"application/octet-stream", "model"},
"oda": {"application/oda", "binary"},
"p12": {"application/x-pkcs12", "binary"},
"p7c": {"application/pkcs7-mime", "binary"},
"pbm": {"image/x-portable-bitmap", "image"},
"pdf": {"application/pdf", "document"},
"pfx": {"application/x-pkcs12", "binary"},
"pgm": {"image/x-portable-graymap", "image"},
"pl": {"text/plain", "document"},
"png": {"image/png", "image"},
"pnm": {"image/x-portable-anymap", "image"},
"pot": {"application/vnd.ms-powerpoint", "document"},
"ppa": {"application/vnd.ms-powerpoint", "document"},
"ppm": {"image/x-portable-pixmap", "image"},
"pps": {"application/vnd.ms-powerpoint", "document"},
"ppt": {"application/vnd.ms-powerpoint", "document"},
"ps": {"application/postscript", "document"},
"pwz": {"application/vnd.ms-powerpoint", "document"},
"py": {"text/x-python", "document"},
"pyc": {"application/x-python-code", "binary"},
"pyo": {"application/x-python-code", "binary"},
"qt": {"video/quicktime", "video"},
"ra": {"audio/x-pn-realaudio", "audio"},
"ram": {"application/x-pn-realaudio", "audio"},
"ras": {"image/x-cmu-raster", "image"},
"rdf": {"application/xml", "binary"},
"rgb": {"image/x-rgb", "image"},
"roff": {"application/x-troff", "binary"},
"rtx": {"text/richtext", "document"},
"sgm": {"text/x-sgml", "document"},
"sgml": {"text/x-sgml", "document"},
"sh": {"application/x-sh", "document"},
"shar": {"application/x-shar", "binary"},
"snd": {"audio/basic", "audio"},
"so": {"application/octet-stream", "binary"},
"src": {"application/x-wais-source", "binary"},
"stl": {"model/stl", "model"},
"sv4cpio": {"application/x-sv4cpio", "binary"},
"sv4crc": {"application/x-sv4crc", "binary"},
"svg": {"image/svg+xml", "image"},
"swf": {"application/x-shockwave-flash", "binary"},
"t": {"application/x-troff", "binary"},
"tar": {"application/x-tar", "binary"},
"tcl": {"application/x-tcl", "binary"},
"tex": {"application/x-tex", "binary"},
"texi": {"application/x-texinfo", "binary"},
"texinfo": {"application/x-texinfo", "binary"},
"tif": {"image/tiff", "image"},
"tiff": {"image/tiff", "image"},
"tr": {"application/x-troff", "binary"},
"tsv": {"text/tab-separated-values", "document"},
"txt": {"text/plain", "document"},
"ustar": {"application/x-ustar", "binary"},
"vcf": {"text/x-vcard", "document"},
"wav": {"audio/x-wav", "audio"},
"webm": {"video/webm", "video"},
"wiz": {"application/msword", "document"},
"wsdl": {"application/xml", "document"},
"xbm": {"image/x-xbitmap", "image"},
"xlb": {"application/vnd.ms-excel", "document"},
"xls": {"application/vnd.ms-excel", "document"},
"xml": {"text/xml", "document"},
"xpdl": {"application/xml", "document"},
"xpm": {"image/x-xpixmap", "image"},
"xsl": {"application/xml", "document"},
"xwd": {"image/x-xwindowdump", "image"},
"zip": {"application/zip", "binary"},
// These are non-standard types, commonly found in the wild.
"cbr": {"application/vnd.comicbook-rar", "document"},
"cbz": {"application/vnd.comicbook+zip", "document"},
"flac": {"audio/flac", "audio"},
"lbry": {"application/x-ext-lbry", "document"},
"m4v": {"video/m4v", "video"},
"mid": {"audio/midi", "audio"},
"midi": {"audio/midi", "audio"},
"mkv": {"video/x-matroska", "video"},
"mobi": {"application/x-mobipocket-ebook", "document"},
"oga": {"audio/ogg", "audio"},
"ogv": {"video/ogg", "video"},
"pct": {"image/pict", "image"},
"pic": {"image/pict", "image"},
"pict": {"image/pict", "image"},
"prc": {"application/x-mobipocket-ebook", "document"},
"rtf": {"application/rtf", "document"},
"xul": {"text/xul", "document"},
// microsoft is special and has its own "standard"
// https://docs.microsoft.com/en-us/windows/desktop/wmp/file-name-extensions
"wmv": {"video/x-ms-wmv", "video"},
}
if data, ok := types[ext]; ok {
return data.mime, data.t
}
return "application/x-ext-" + ext, "binary"
}
+291
View File
@@ -0,0 +1,291 @@
package publish
import (
"bytes"
"encoding/json"
"os"
"path/filepath"
"sort"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/lbrycrd"
"github.com/lbryio/lbry.go/v2/stream"
pb "github.com/lbryio/types/v2/go"
"github.com/btcsuite/btcd/btcjson"
"github.com/btcsuite/btcd/chaincfg/chainhash"
"github.com/btcsuite/btcd/txscript"
"github.com/btcsuite/btcd/wire"
"github.com/btcsuite/btcutil"
"github.com/golang/protobuf/proto"
)
/* TODO:
import cert from wallet
get all utxos from chainquery
create transaction
sign it with the channel
track state of utxos across publishes from this channel so that we can just do one query to get utxos
prioritize only confirmed utxos
Handling all the issues we handle currently with lbrynet:
"Couldn't find private key for id",
"You already have a stream claim published under the name",
"Cannot publish using channel",
"txn-mempool-conflict",
"too-long-mempool-chain",
"Missing inputs",
"Not enough funds to cover this transaction",
*/
type Details struct {
Title string
Description string
Author string
Tags []string
ReleaseTime int64
}
func Publish(client *lbrycrd.Client, path, name, address string, details Details, reflectorAddress string) (*wire.MsgTx, *chainhash.Hash, error) {
if name == "" {
return nil, nil, errors.Err("name required")
}
//TODO: sign claim if publishing into channel
addr, err := btcutil.DecodeAddress(address, &lbrycrd.MainNetParams)
if errors.Is(err, btcutil.ErrUnknownAddressType) {
return nil, nil, errors.Err(`unknown address type. here's what you need to make this work:
- deprecatedrpc=validateaddress" and "deprecatedrpc=signrawtransaction" in your lbrycrd.conf
- github.com/btcsuite/btcd pinned to hash 306aecffea32
- github.com/btcsuite/btcutil pinned to 4c204d697803
- github.com/lbryio/lbry.go/v2 (make sure you have v2 at the end)`)
}
if err != nil {
return nil, nil, err
}
amount := 0.01
changeAddr := addr // TODO: fix this? or maybe its fine?
tx, err := baseTx(client, amount, changeAddr)
if err != nil {
return nil, nil, err
}
st, stPB, err := makeStream(path)
if err != nil {
return nil, nil, err
}
stPB.Author = details.Author
stPB.ReleaseTime = details.ReleaseTime
claim := &pb.Claim{
Title: details.Title,
Description: details.Description,
Type: &pb.Claim_Stream{Stream: stPB},
}
err = addClaimToTx(tx, claim, name, amount, addr)
if err != nil {
return nil, nil, err
}
// sign and send
signedTx, allInputsSigned, err := client.SignRawTransaction(tx)
if err != nil {
return nil, nil, err
}
if !allInputsSigned {
return nil, nil, errors.Err("not all inputs for the tx could be signed")
}
err = reflect(st, reflectorAddress)
if err != nil {
return nil, nil, err
}
txid, err := client.SendRawTransaction(signedTx, false)
if err != nil {
return nil, nil, err
}
return signedTx, txid, nil
}
// TODO: lots of assumptions. hardcoded values need to be passed in or calculated
func baseTx(client *lbrycrd.Client, amount float64, changeAddress btcutil.Address) (*wire.MsgTx, error) {
txFee := 0.0002 // TODO: estimate this better?
inputs, total, err := coinChooser(client, amount+txFee)
if err != nil {
return nil, err
}
change := total - amount - txFee
// create base raw tx
addresses := make(map[btcutil.Address]btcutil.Amount)
//changeAddr, err := client.GetNewAddress("")
changeAmount, err := btcutil.NewAmount(change)
if err != nil {
return nil, err
}
addresses[changeAddress] = changeAmount
lockTime := int64(0)
return client.CreateRawTransaction(inputs, addresses, &lockTime)
}
func coinChooser(client *lbrycrd.Client, amount float64) ([]btcjson.TransactionInput, float64, error) {
utxos, err := client.ListUnspentMin(1)
if err != nil {
return nil, 0, err
}
sort.Slice(utxos, func(i, j int) bool { return utxos[i].Amount < utxos[j].Amount })
var utxo btcjson.ListUnspentResult
for _, u := range utxos {
if u.Spendable && u.Amount >= amount {
utxo = u
break
}
}
if utxo.TxID == "" {
return nil, 0, errors.Err("not enough utxos to create tx")
}
return []btcjson.TransactionInput{{Txid: utxo.TxID, Vout: utxo.Vout}}, utxo.Amount, nil
}
func addClaimToTx(tx *wire.MsgTx, claim *pb.Claim, name string, amount float64, claimAddress btcutil.Address) error {
claimBytes, err := proto.Marshal(claim)
if err != nil {
return err
}
claimBytes = append([]byte{0}, claimBytes...) // version 0 = no channel sig
amt, err := btcutil.NewAmount(amount)
if err != nil {
return err
}
script, err := getClaimPayoutScript(name, claimBytes, claimAddress)
if err != nil {
return err
}
tx.AddTxOut(wire.NewTxOut(int64(amt), script))
return nil
}
func Decode(client *lbrycrd.Client, tx *wire.MsgTx) (string, error) {
buf := bytes.NewBuffer(make([]byte, 0, tx.SerializeSize()))
if err := tx.Serialize(buf); err != nil {
return "", errors.Err(err)
}
//txHex := hex.EncodeToString(buf.Bytes())
//spew.Dump(txHex)
decoded, err := client.DecodeRawTransaction(buf.Bytes())
if err != nil {
return "", err
}
data, err := json.MarshalIndent(decoded, "", " ")
return string(data), err
}
func reflect(st stream.Stream, reflectorAddress string) error {
// upload blobs to reflector
c := reflector.Client{}
err := c.Connect(reflectorAddress)
if err != nil {
return errors.Err(err)
}
for i, b := range st {
if i == 0 {
err = c.SendSDBlob(b)
} else {
err = c.SendBlob(b)
}
if err != nil {
return errors.Err(err)
}
}
return nil
}
func makeStream(path string) (stream.Stream, *pb.Stream, error) {
file, err := os.Open(path)
if err != nil {
return nil, nil, errors.Err(err)
}
defer func() { _ = file.Close() }()
enc := stream.NewEncoder(file)
s, err := enc.Stream()
if err != nil {
return nil, nil, errors.Err(err)
}
streamProto := &pb.Stream{
Source: &pb.Source{
SdHash: enc.SDBlob().Hash(),
Name: filepath.Base(file.Name()),
Size: uint64(enc.SourceLen()),
Hash: enc.SourceHash(),
},
}
mimeType, category := guessMimeType(filepath.Ext(file.Name()))
streamProto.Source.MediaType = mimeType
switch category {
case "video":
//t, err := streamVideoMetadata(path)
//if err != nil {
// return nil, nil, err
//}
streamProto.Type = &pb.Stream_Video{}
case "audio":
streamProto.Type = &pb.Stream_Audio{}
case "image":
streamProto.Type = &pb.Stream_Image{}
}
return s, streamProto, nil
}
func getClaimPayoutScript(name string, value []byte, address btcutil.Address) ([]byte, error) {
//OP_CLAIM_NAME <name> <value> OP_2DROP OP_DROP OP_DUP OP_HASH160 <address> OP_EQUALVERIFY OP_CHECKSIG
pkscript, err := txscript.PayToAddrScript(address)
if err != nil {
return nil, errors.Err(err)
}
return txscript.NewScriptBuilder().
AddOp(txscript.OP_NOP6). //OP_CLAIM_NAME
AddData([]byte(name)). //<name>
AddData(value). //<value>
AddOp(txscript.OP_2DROP). //OP_2DROP
AddOp(txscript.OP_DROP). //OP_DROP
AddOps(pkscript). //OP_DUP OP_HASH160 <address> OP_EQUALVERIFY OP_CHECKSIG
Script()
}
//func streamVideoMetadata(path string) (*pb.Stream_Video, error) {
// mi, err := mediainfo.GetMediaInfo(path)
// if err != nil {
// return nil, err
// }
// return &pb.Stream_Video{
// Video: &pb.Video{
// Duration: uint32(mi.General.Duration / 1000),
// Height: uint32(mi.Video.Height),
// Width: uint32(mi.Video.Width),
// },
// }, nil
//}
+59
View File
@@ -0,0 +1,59 @@
package publish
import (
"encoding/json"
"io"
)
func LoadWallet(r io.Reader) (WalletFile, error) {
var w WalletFile
err := json.NewDecoder(r).Decode(&w)
return w, err
}
type WalletFile struct {
Name string `json:"name"`
Accounts []Account `json:"accounts"`
Preferences WalletPrefs `json:"preferences"`
Version int `json:"version"`
}
type Account struct {
Certificates map[string]string `json:"certificates"`
Ledger string `json:"ledger"`
Name string `json:"name"`
PrivateKey string `json:"private_key"`
PublicKey string `json:"public_key"`
Seed string `json:"seed"`
AddressGenerator AddressGenerator `json:"address_generator"`
ModifiedOn float64 `json:"modified_on"`
Encrypted bool `json:"encrypted"`
}
type AddressGenerator struct {
Name string `json:"name"`
Change AddressGenParams `json:"change"` // should "change" and "receiving" be replaced with a map[string]AddressGenParams?
Receiving AddressGenParams `json:"receiving"`
}
type AddressGenParams struct {
Gap int `json:"gap"`
MaximumUsesPerAddress int `json:"maximum_uses_per_address"`
}
type WalletPrefs struct {
Shared struct {
Value struct {
Type string `json:"type"`
Version string `json:"version"`
Value struct {
Blocked []interface{} `json:"blocked"`
Subscriptions []string `json:"subscriptions"`
Tags []string `json:"tags"`
AppWelcomeVersion int `json:"app_welcome_version"`
Sharing3P bool `json:"sharing_3P"`
} `json:"value"`
} `json:"value"`
Ts float64 `json:"ts"`
} `json:"shared"`
}
+170 -21
View File
@@ -1,40 +1,189 @@
# Reflector
A reflector cluster to accept LBRY content for hosting en masse, rehost the content, and make money on data fees (TODO).
This code includes Go implementations of the LBRY peer protocol, reflector protocol, and DHT.
Production-ready blob reflection, distribution, and caching for Odysee.
## Installation
This repository provides the components used in production:
- Reflector ingestion server (command name: `reflector`)
- Blob cache/edge server (`blobcache`)
- Uploader to object storage (`upload`)
coming soon
Other commands exist in the tree for historical/legacy reasons and are not supported.
## Usage
## How it works (at a glance)
- Ingestion (reflector): accepts uploaded blobs, persists them to object storage (e.g., S3/Wasabi) and tracks state in MySQL.
- Distribution: serves blobs over HTTP/HTTP3/Peer. Blobcaches can be deployed in front of the origin to reduce latency and egress.
- Caching (blobcache): layered disk caches backed by HTTP(S) origins (e.g., S3 endpoints), with optional local DB metadata for capacity/eviction.
coming soon
All services are started by the `prism` binary and are configured via YAML files loaded from a configuration directory.
## Running from Source
## Supported commands
The following are the only supported commands for production use:
This project requires [Go v1.10](https://golang.org/doc/install) or higher.
- Reflector ingestion: `prism reflector`
- Flags: `--receiver-port` (default 5566), `--metrics-port` (default 2112), `--disable-blocklist`
- Loads `reflector.yaml` from the config directory.
```
go get -u github.com/lbryio/reflector.go
cd "$(go env GOPATH)/src/github.com/lbryio/reflector.go"
make
./bin/prism-bin
- Blob cache: `prism blobcache`
- Flags: `--metrics-port` (default 2112), `--disable-blocklist`
- Loads `blobcache.yaml` from the config directory.
- Uploader: `prism upload PATH`
- Flags: `--workers`, `--skipExistsCheck`, `--deleteBlobsAfterUpload`
- Loads `upload.yaml` from the config directory.
Global flag for all commands:
- `--conf-dir` (default `./`): directory containing YAML config files.
## Configuration
Configuration is per-command. The loader reads `<command>.yaml` from `--conf-dir`.
Common sections:
- `servers`: enables HTTP/HTTP3/Peer servers. Keys: `http`, `http3`, `peer`. Each accepts:
- `port` (int)
- `max_concurrent_requests` (int, http/http3)
- `edge_token` (string, http)
- `address` (string, optional; bind address, omit for all interfaces)
- `store`: defines the storage topology using composable stores. Frequently used:
- `proxied-s3`: production pattern with a `writer` (DB-backed -> S3/multiwriter) and a `reader` (caching -> disk + HTTP origins).
- `caching`: layered cache with a `cache` (often `db_backed` -> `disk`) and an `origin` chain (`http`, `http3`, or `ittt` fan-in).
- `s3`, `disk`, `multiwriter`, `db_backed`, `http`, `http3`, `peer`, `upstream` are also available building blocks.
### Minimal examples
Reflector – conf-dir contains `reflector.yaml`:
```yaml
servers:
http:
port: 5569
max_concurrent_requests: 200
http3:
port: 5568
max_concurrent_requests: 200
peer:
port: 5567
store:
proxied-s3:
name: s3_read_proxy
writer:
db_backed:
user: reflector
password: reflector
database: reflector
host: localhost
port: 3306
access_tracking: 1
soft_deletes: true
store:
s3:
name: primary
aws_id: YOUR_KEY
aws_secret: YOUR_SECRET
region: us-east-1
bucket: blobs-bucket
endpoint: https://s3.yourendpoint.tv
reader:
caching:
cache:
disk:
name: local_cache
mount_point: /mnt/reflector/cache
sharding_size: 2
origin:
http:
endpoint: https://s3.yourendpoint.tv/blobs-bucket/
sharding_size: 4
```
## Contributing
Blobcache – conf-dir contains `blobcache.yaml`:
```yaml
servers:
http:
port: 5569
max_concurrent_requests: 200
http3:
port: 5568
max_concurrent_requests: 200
peer:
port: 5567
store:
caching:
cache:
db_backed:
user: reflector
password: reflector
database: reflector
host: localhost
port: 3306
has_cap: true
max_size: 500GB
store:
disk:
name: blobcache
mount_point: /mnt/blobcache/cache
sharding_size: 2
origin:
http:
endpoint: https://s3.yourendpoint.tv/blobs-bucket/
sharding_size: 4
```
coming soon
Uploader – conf-dir contains `upload.yaml` (points to the same writer/backend as reflector):
```yaml
database:
user: reflector
password: reflector
database: reflector
host: localhost
port: 3306
store:
proxied-s3:
writer:
db_backed:
user: reflector
password: reflector
database: reflector
host: localhost
port: 3306
store:
s3:
aws_id: YOUR_KEY
aws_secret: YOUR_SECRET
region: us-east-1
bucket: blobs-bucket
endpoint: https://s3.yourendpoint.tv
```
## License
## Quick start
1) Build
- Requires Go 1.25+
- `make` (binaries in `dist/<platform>/prism-bin`)
This project is MIT licensed.
2) Run a local blobcache
```bash
./dist/linux_amd64/prism-bin --conf-dir=./ blobcache
```
Place your `blobcache.yaml` in the `--conf-dir` directory.
3) Run reflector ingestion
```bash
./dist/linux_amd64/prism-bin --conf-dir=./ reflector --receiver-port=5566 --metrics-port=2112
```
4) Upload blobs
```bash
./dist/linux_amd64/prism-bin --conf-dir=./ upload /path/to/blobs \
--workers=4 --skipExistsCheck
```
## Notes
- Only reflector, blobcache, and upload are supported. All other commands are legacy and may be removed in the future.
- Metrics are exposed on the configured `--metrics-port` at `/metrics` (Prometheus format).
- MySQL is required when using DB-backed stores (e.g., ingestion writer, capacity-aware caches).
## Security
If you discover a security issue, please email security@lbry.com. Our PGP key is available at https://lbry.com/faq/pgp-key.
We take security seriously. Please contact security@lbry.com regarding any security issues.
Our PGP key is [here](https://keybase.io/lbry/key.asc) if you need it.
## License
MIT License. See LICENSE.
## Contact
The primary contact for this project is [@lyoshenka](https://github.com/lyoshenka) (grin@lbry.com)
The primary contact for this project is [@Nikooo777](https://github.com/Nikooo777)
+57 -74
View File
@@ -8,42 +8,49 @@ import (
"strings"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/reflector.go/wallet"
"github.com/lbryio/lbry.go/extras/errors"
types1 "github.com/lbryio/types/v1/go"
types2 "github.com/lbryio/types/v2/go"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/golang/protobuf/proto"
log "github.com/sirupsen/logrus"
)
const blocklistURL = "https://api.lbry.com/file/list_blocked"
func (s *Server) enableBlocklist(b store.Blocklister) {
// TODO: updateBlocklist should be killed when server is shutting down
updateBlocklist(b)
walletServers := []string{
"a-hub1.odysee.com:50001",
"b-hub1.odysee.com:50001",
"c-hub1.odysee.com:50001",
"s-hub1.odysee.com:50001",
}
updateBlocklist(b, walletServers, s.grp.Ch())
t := time.NewTicker(12 * time.Hour)
for {
select {
case <-s.grp.Ch():
return
case <-t.C:
updateBlocklist(b)
updateBlocklist(b, walletServers, s.grp.Ch())
}
}
}
func updateBlocklist(b store.Blocklister) {
values, err := blockedSdHashes()
func updateBlocklist(b store.Blocklister, walletServers []string, stopper stop.Chan) {
log.Debugf("blocklist update starting")
values, err := blockedSdHashes(walletServers, stopper)
if err != nil {
log.Error(err)
return
}
for _, v := range values {
for name, v := range values {
if v.Err != nil {
log.Error(errors.FullTrace(errors.Err("blocklist: %s: %s", name, v.Err)))
continue
}
@@ -52,26 +59,28 @@ func updateBlocklist(b store.Blocklister) {
log.Error(err)
}
}
log.Debugf("blocklist update done")
}
func blockedSdHashes() (map[string]valOrErr, error) {
resp, err := http.Get(blocklistURL)
func blockedSdHashes(walletServers []string, stopper stop.Chan) (map[string]valOrErr, error) {
client := http.Client{Timeout: 1 * time.Second}
resp, err := client.Get(blocklistURL)
if err != nil {
return nil, errors.Err(err)
}
defer func() {
err := resp.Body.Close()
if err != nil {
log.Errorln(err)
closeErr := resp.Body.Close()
if closeErr != nil {
log.Errorln(errors.Err(closeErr))
}
}()
var r struct {
Success bool `json:"success"`
Error string `json:"error"`
Data struct {
Error string `json:"error"`
Data struct {
Outpoints []string `json:"outpoints"`
} `json:"data"`
Success bool `json:"success"`
}
if err = json.NewDecoder(resp.Body).Decode(&r); err != nil {
@@ -82,32 +91,43 @@ func blockedSdHashes() (map[string]valOrErr, error) {
return nil, errors.Prefix("list_blocked API call", r.Error)
}
return sdHashesForOutpoints(r.Data.Outpoints)
return sdHashesForOutpoints(walletServers, r.Data.Outpoints, stopper)
}
type valOrErr struct {
Value string
Err error
Value string
}
// sdHashesForOutpoints queries wallet server for the sd hashes in a given outpoints
func sdHashesForOutpoints(outpoints []string) (map[string]valOrErr, error) {
func sdHashesForOutpoints(walletServers, outpoints []string, stopper stop.Chan) (map[string]valOrErr, error) {
values := make(map[string]valOrErr)
node := wallet.NewNode()
defer node.Shutdown()
err := node.Connect([]string{
"spv1.lbry.com:50001",
"spv2.lbry.com:50001",
"spv3.lbry.com:50001",
"spv4.lbry.com:50001",
"spv5.lbry.com:50001",
}, nil)
err := node.Connect(walletServers, nil)
if err != nil {
return nil, err
return nil, errors.Err(err)
}
done := make(chan bool)
metrics.RoutinesQueue.WithLabelValues("reflector", "sdhashesforoutput").Inc()
go func() {
defer metrics.RoutinesQueue.WithLabelValues("reflector", "sdhashesforoutput").Dec()
select {
case <-done:
case <-stopper:
}
node.Shutdown()
}()
OutpointLoop:
for _, outpoint := range outpoints {
select {
case <-stopper:
break OutpointLoop
default:
}
parts := strings.Split(outpoint, ":")
if len(parts) != 2 {
values[outpoint] = valOrErr{Err: errors.Err("invalid outpoint format")}
@@ -120,57 +140,20 @@ func sdHashesForOutpoints(outpoints []string) (map[string]valOrErr, error) {
continue
}
resp, err := node.GetClaimsInTx(parts[0])
claim, err := node.GetClaimInTx(parts[0], nout)
if err != nil {
values[outpoint] = valOrErr{Err: err}
continue
}
var value []byte
for _, tx := range resp.Result {
if tx.Nout != nout {
continue
}
hash := hex.EncodeToString(claim.GetStream().GetSource().GetSdHash())
values[outpoint] = valOrErr{Value: hash, Err: nil}
}
value, err = hex.DecodeString(tx.Value)
break
}
if err != nil {
values[outpoint] = valOrErr{Err: err}
continue
} else if value == nil {
values[outpoint] = valOrErr{Err: errors.Err("outpoint not found")}
continue
}
hash, err := hashFromClaim(value)
values[outpoint] = valOrErr{Value: hash, Err: err}
select {
case done <- true:
default: // in case of race where stopper got stopped right after loop finished
}
return values, nil
}
func hashFromClaim(value []byte) (string, error) {
claim := &types1.Claim{}
err := proto.Unmarshal(value, claim)
if err != nil {
return "", err
}
if claim.GetStream().GetSource().GetSourceType() == types1.Source_lbry_sd_hash && claim.GetStream().GetSource().GetSource() != nil {
return hex.EncodeToString(claim.GetStream().GetSource().GetSource()), nil
}
claim2 := &types2.Claim{}
err = proto.Unmarshal(value, claim2)
if err != nil {
return "", err
}
stream, ok := claim2.GetType().(*types2.Claim_Stream)
if !ok || stream == nil {
return "", errors.Err("not a stream claim")
}
return hex.EncodeToString(claim2.GetStream().GetSource().GetSdHash()), nil
}
+61 -26
View File
@@ -2,12 +2,11 @@ package reflector
import (
"encoding/json"
"log"
"net"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/stream"
log "github.com/sirupsen/logrus"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
)
// ErrBlobExists is a default error for when a blob already exists on the reflector server.
@@ -36,8 +35,18 @@ func (c *Client) Close() error {
return c.conn.Close()
}
// SendBlob sends a send blob request to the client.
// SendBlob sends a blob to the server.
func (c *Client) SendBlob(blob stream.Blob) error {
return c.sendBlob(blob, false)
}
// SendSDBlob sends an SD blob request to the server.
func (c *Client) SendSDBlob(blob stream.Blob) error {
return c.sendBlob(blob, true)
}
// sendBlob does the actual blob sending
func (c *Client) sendBlob(blob stream.Blob, isSDBlob bool) error {
if !c.connected {
return errors.Err("not connected")
}
@@ -47,10 +56,15 @@ func (c *Client) SendBlob(blob stream.Blob) error {
}
blobHash := blob.HashHex()
sendRequest, err := json.Marshal(sendBlobRequest{
BlobSize: blob.Size(),
BlobHash: blobHash,
})
var req sendBlobRequest
if isSDBlob {
req.SdBlobSize = blob.Size()
req.SdBlobHash = blobHash
} else {
req.BlobSize = blob.Size()
req.BlobHash = blobHash
}
sendRequest, err := json.Marshal(req)
if err != nil {
return err
}
@@ -62,30 +76,51 @@ func (c *Client) SendBlob(blob stream.Blob) error {
dec := json.NewDecoder(c.conn)
var sendResp sendBlobResponse
err = dec.Decode(&sendResp)
if err != nil {
return err
if isSDBlob {
var sendResp sendSdBlobResponse
err = dec.Decode(&sendResp)
if err != nil {
return err
}
if !sendResp.SendSdBlob {
return errors.Prefix(blobHash[:8], ErrBlobExists)
}
log.Println("Sending SD blob " + blobHash[:8])
} else {
var sendResp sendBlobResponse
err = dec.Decode(&sendResp)
if err != nil {
return err
}
if !sendResp.SendBlob {
return errors.Prefix(blobHash[:8], ErrBlobExists)
}
log.Println("Sending blob " + blobHash[:8])
}
if !sendResp.SendBlob {
return errors.Prefix(blobHash[:8], ErrBlobExists)
}
log.Println("Sending blob " + blobHash[:8])
_, err = c.conn.Write(blob)
if err != nil {
return err
}
var transferResp blobTransferResponse
err = dec.Decode(&transferResp)
if err != nil {
return err
}
if !transferResp.ReceivedBlob {
return errors.Err("server did not received blob")
if isSDBlob {
var transferResp sdBlobTransferResponse
err = dec.Decode(&transferResp)
if err != nil {
return err
}
if !transferResp.ReceivedSdBlob {
return errors.Err("server did not received SD blob")
}
} else {
var transferResp blobTransferResponse
err = dec.Decode(&transferResp)
if err != nil {
return err
}
if !transferResp.ReceivedBlob {
return errors.Err("server did not received blob")
}
}
return nil
+85
View File
@@ -0,0 +1,85 @@
package reflector
import (
"encoding/json"
"net/http"
"time"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/bluele/gcache"
"golang.org/x/sync/singleflight"
)
const protectedListURL = "https://direct.api.odysee.com/file/list_protected"
type ProtectedContent struct {
SDHash string `json:"sd_hash"`
ClaimID string `json:"claim_id"`
}
var protectedCache = gcache.New(10).Expiration(2 * time.Minute).Build()
func GetProtectedContent() (interface{}, error) {
cachedVal, err := protectedCache.Get("protected")
if err == nil && cachedVal != nil {
return cachedVal.(map[string]bool), nil
}
method := "GET"
var r struct {
Error string `json:"error"`
Data []ProtectedContent `json:"data"`
Success bool `json:"success"`
}
// Bound the request to avoid hanging the entire request path.
// Without a timeout, a slow or unreachable endpoint can block
// singleflight callers indefinitely and stall HTTP handlers.
client := &http.Client{Timeout: 5 * time.Second}
req, err := http.NewRequest(method, protectedListURL, nil)
if err != nil {
return nil, errors.Err(err)
}
res, err := client.Do(req)
if err != nil {
return nil, errors.Err(err)
}
defer func() { _ = res.Body.Close() }()
if res.StatusCode != http.StatusOK {
return nil, errors.Err("unexpected status code %d", res.StatusCode)
}
if err = json.NewDecoder(res.Body).Decode(&r); err != nil {
return nil, errors.Err(err)
}
if !r.Success {
return nil, errors.Prefix("file/list_protected API call", r.Error)
}
protectedMap := make(map[string]bool, len(r.Data))
for _, pc := range r.Data {
protectedMap[pc.SDHash] = true
}
err = protectedCache.Set("protected", protectedMap)
if err != nil {
return protectedMap, errors.Err(err)
}
return protectedMap, nil
}
var sf = singleflight.Group{}
func IsProtected(sdHash string) bool {
val, err, _ := sf.Do("protected", GetProtectedContent)
if err != nil {
return false
}
cachedMap, ok := val.(map[string]bool)
if !ok {
return false
}
return cachedMap[sdHash]
}
+39 -39
View File
@@ -6,16 +6,15 @@ import (
"encoding/hex"
"encoding/json"
"io"
"io/ioutil"
"net"
"strconv"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/stop"
"github.com/lbryio/lbry.go/stream"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/lbry.go/v2/stream"
log "github.com/sirupsen/logrus"
)
@@ -32,22 +31,21 @@ const (
maxBlobSize = stream.MaxBlobSize
)
var ErrBlobTooBig = errors.Base("blob must be at most %d bytes", maxBlobSize)
// Server is and instance of the reflector server. It houses the blob store and listener.
type Server struct {
store store.BlobStore
grp *stop.Group
Timeout time.Duration // timeout to read or write next message
StatLogger *log.Logger // logger to log stats
StatReportFrequency time.Duration // how often to log stats
EnableBlocklist bool // if true, blocklist checking and blob deletion will be enabled
store store.BlobStore
grp *stop.Group
stats *Stats
//underlyingStore store.BlobStore
//outerStore store.BlobStore
}
// NewServer returns an initialized reflector server pointer.
func NewServer(store store.BlobStore) *Server {
func NewIngestionServer(store store.BlobStore) *Server {
return &Server{
Timeout: DefaultTimeout,
store: store,
@@ -58,21 +56,21 @@ func NewServer(store store.BlobStore) *Server {
// Shutdown shuts down the reflector server gracefully.
func (s *Server) Shutdown() {
log.Println("shutting down reflector server...")
s.stats.Shutdown()
s.grp.StopAndWait()
log.Println("reflector server stopped")
}
//Start starts the server to handle connections.
// Start starts the server to handle connections.
func (s *Server) Start(address string) error {
l, err := net.Listen(network, address)
if err != nil {
return errors.Err(err)
}
log.Println("reflector listening on " + address)
s.grp.Add(1)
metrics.RoutinesQueue.WithLabelValues("reflector", "listener").Inc()
go func() {
defer metrics.RoutinesQueue.WithLabelValues("reflector", "listener").Dec()
<-s.grp.Ch()
err := l.Close()
if err != nil {
@@ -82,20 +80,19 @@ func (s *Server) Start(address string) error {
}()
s.grp.Add(1)
metrics.RoutinesQueue.WithLabelValues("reflector", "start").Inc()
go func() {
defer metrics.RoutinesQueue.WithLabelValues("reflector", "start").Dec()
s.listenAndServe(l)
s.grp.Done()
}()
s.stats = NewStatLogger("UPLOAD", s.StatLogger, s.StatReportFrequency, s.grp.Child())
if s.StatLogger != nil && s.StatReportFrequency > 0 {
s.stats.Start()
}
if s.EnableBlocklist {
if b, ok := s.store.(store.Blocklister); ok {
s.grp.Add(1)
metrics.RoutinesQueue.WithLabelValues("reflector", "enableblocklist").Inc()
go func() {
defer metrics.RoutinesQueue.WithLabelValues("reflector", "enableblocklist").Dec()
s.enableBlocklist(b)
s.grp.Done()
}()
@@ -118,7 +115,9 @@ func (s *Server) listenAndServe(listener net.Listener) {
log.Error(err)
} else {
s.grp.Add(1)
metrics.RoutinesQueue.WithLabelValues("reflector", "server-listenandserve").Inc()
go func() {
defer metrics.RoutinesQueue.WithLabelValues("reflector", "server-listenandserve").Dec()
s.handleConn(conn)
s.grp.Done()
}()
@@ -133,7 +132,9 @@ func (s *Server) handleConn(conn net.Conn) {
close(connNeedsClosing)
}()
s.grp.Add(1)
metrics.RoutinesQueue.WithLabelValues("reflector", "server-handleconn").Inc()
go func() {
defer metrics.RoutinesQueue.WithLabelValues("reflector", "server-handleconn").Dec()
defer s.grp.Done()
select {
case <-connNeedsClosing:
@@ -150,7 +151,7 @@ func (s *Server) handleConn(conn net.Conn) {
if errors.Is(err, io.EOF) || s.quitting() {
return
}
err := s.doError(conn, err)
err = s.doError(conn, err)
if err != nil {
log.Error(errors.Prefix("sending handshake error", err))
}
@@ -163,7 +164,7 @@ func (s *Server) handleConn(conn net.Conn) {
if errors.Is(err, io.EOF) || s.quitting() {
return
}
err := s.doError(conn, err)
err = s.doError(conn, err)
if err != nil {
log.Error(errors.Prefix("sending blob receive error", err))
}
@@ -173,7 +174,10 @@ func (s *Server) handleConn(conn net.Conn) {
}
func (s *Server) doError(conn net.Conn, err error) error {
shouldLog := s.stats.AddError(err)
if err == nil {
return nil
}
shouldLog := metrics.TrackError(metrics.DirectionUpload, err)
if shouldLog {
log.Errorln(errors.FullTrace(err))
}
@@ -201,7 +205,8 @@ func (s *Server) receiveBlob(conn net.Conn) error {
return err
}
} else {
blobExists, err := s.store.Has(blobHash)
var blobExists bool
blobExists, err = s.store.Has(blobHash)
if err != nil {
return err
}
@@ -211,14 +216,14 @@ func (s *Server) receiveBlob(conn net.Conn) error {
var neededBlobs []string
if isSdBlob && !wantsBlob {
if nbc, ok := s.store.(neededBlobChecker); ok {
if nbc, ok := s.store.(store.NeededBlobChecker); ok {
neededBlobs, err = nbc.MissingBlobsForKnownStream(blobHash)
if err != nil {
return err
}
} else {
// if we can't check for blobs in a stream, we have to say that the sd blob is
// missing. if we say we have the sd blob, they wont try to send any content blobs
// missing. if we say we have the sd blob, they won't try to send any content blobs
wantsBlob = true
}
}
@@ -261,10 +266,10 @@ func (s *Server) receiveBlob(conn net.Conn) error {
if err != nil {
return err
}
s.stats.AddBlob()
metrics.MtrInBytesReflector.Add(float64(len(blob)))
metrics.BlobUploadCount.Inc()
if isSdBlob {
s.stats.AddStream()
metrics.SDBlobUploadCount.Inc()
}
return s.sendTransferResponse(conn, true, isSdBlob)
}
@@ -311,7 +316,7 @@ func (s *Server) readBlobRequest(conn net.Conn) (int, string, bool, error) {
return blobSize, blobHash, isSdBlob, errors.Err("blob hash is empty")
}
if blobSize > maxBlobSize {
return blobSize, blobHash, isSdBlob, errors.Err("blob must be at most " + strconv.Itoa(maxBlobSize) + " bytes")
return blobSize, blobHash, isSdBlob, errors.Err(ErrBlobTooBig)
}
if blobSize == 0 {
return blobSize, blobHash, isSdBlob, errors.Err("0-byte blob received")
@@ -361,7 +366,7 @@ func (s *Server) read(conn net.Conn, v interface{}) error {
dec := json.NewDecoder(conn)
err = dec.Decode(v)
if err != nil {
data, _ := ioutil.ReadAll(dec.Buffered())
data, _ := io.ReadAll(dec.Buffered())
if len(data) > 0 {
return errors.Err("%s. Data: %s", err.Error(), hex.EncodeToString(data))
}
@@ -424,8 +429,8 @@ type handshakeRequestResponse struct {
type sendBlobRequest struct {
BlobHash string `json:"blob_hash,omitempty"`
BlobSize int `json:"blob_size,omitempty"`
SdBlobHash string `json:"sd_blob_hash,omitempty"`
BlobSize int `json:"blob_size,omitempty"`
SdBlobSize int `json:"sd_blob_size,omitempty"`
}
@@ -434,8 +439,8 @@ type sendBlobResponse struct {
}
type sendSdBlobResponse struct {
SendSdBlob bool `json:"send_sd_blob"`
NeededBlobs []string `json:"needed_blobs,omitempty"`
SendSdBlob bool `json:"send_sd_blob"`
}
type blobTransferResponse struct {
@@ -445,8 +450,3 @@ type blobTransferResponse struct {
type sdBlobTransferResponse struct {
ReceivedSdBlob bool `json:"received_sd_blob"`
}
// neededBlobChecker can check which blobs from a known stream are not uploaded yet
type neededBlobChecker interface {
MissingBlobsForKnownStream(string) ([]string, error)
}
+8 -7
View File
@@ -9,9 +9,10 @@ import (
"testing"
"time"
"github.com/lbryio/lbry.go/dht/bits"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/dht/bits"
"github.com/davecgh/go-spew/spew"
"github.com/phayes/freeport"
)
@@ -22,7 +23,7 @@ func startServerOnRandomPort(t *testing.T) (*Server, int) {
t.Fatal(err)
}
srv := NewServer(&store.MemoryBlobStore{})
srv := NewIngestionServer(store.NewMemStore(store.MemParams{Name: "test"}))
err = srv.Start("127.0.0.1:" + strconv.Itoa(port))
if err != nil {
t.Fatal(err)
@@ -119,7 +120,7 @@ func TestServer_Timeout(t *testing.T) {
t.Fatal(err)
}
srv := NewServer(&store.MemoryBlobStore{})
srv := NewIngestionServer(store.NewMemStore(store.MemParams{Name: "test"}))
srv.Timeout = testTimeout
err = srv.Start("127.0.0.1:" + strconv.Itoa(port))
if err != nil {
@@ -161,7 +162,7 @@ func TestServer_Timeout(t *testing.T) {
//}
type mockPartialStore struct {
store.MemoryBlobStore
*store.MemStore
missing []string
}
@@ -181,8 +182,8 @@ func TestServer_PartialUpload(t *testing.T) {
missing[i] = bits.Rand().String()
}
st := store.BlobStore(&mockPartialStore{missing: missing})
if _, ok := st.(neededBlobChecker); !ok {
st := store.BlobStore(&mockPartialStore{MemStore: store.NewMemStore(store.MemParams{Name: "test"}), missing: missing})
if _, ok := st.(store.NeededBlobChecker); !ok {
t.Fatal("mock does not implement the relevant interface")
}
err = st.Put(sdHash, randBlob(10))
@@ -190,7 +191,7 @@ func TestServer_PartialUpload(t *testing.T) {
t.Fatal(err)
}
srv := NewServer(st)
srv := NewIngestionServer(st)
err = srv.Start("127.0.0.1:" + strconv.Itoa(port))
if err != nil {
t.Fatal(err)
-122
View File
@@ -1,122 +0,0 @@
package reflector
import (
"fmt"
"strings"
"sync"
"time"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/stop"
log "github.com/sirupsen/logrus"
)
// TODO: store daily stats too. and maybe other intervals
type Stats struct {
mu *sync.Mutex
blobs int
streams int
errors map[string]int
started bool
name string
logger *log.Logger
logFreq time.Duration
grp *stop.Group
}
func NewStatLogger(name string, logger *log.Logger, logFreq time.Duration, parentGrp *stop.Group) *Stats {
return &Stats{
mu: &sync.Mutex{},
grp: stop.New(parentGrp),
logger: logger,
logFreq: logFreq,
errors: make(map[string]int),
name: name,
}
}
func (s *Stats) Start() {
s.started = true
s.grp.Add(1)
go func() {
defer s.grp.Done()
s.runSlackLogger()
}()
}
func (s *Stats) Shutdown() {
if !s.started {
return
}
s.log()
s.grp.StopAndWait()
s.started = false
}
func (s *Stats) AddBlob() {
s.mu.Lock()
defer s.mu.Unlock()
s.blobs++
}
func (s *Stats) AddStream() {
s.mu.Lock()
defer s.mu.Unlock()
s.streams++
}
func (s *Stats) AddError(e error) (shouldLog bool) { // shouldLog is a hack, but whatever
if e == nil {
return
}
err := errors.Wrap(e, 0)
name := err.TypeName()
if strings.Contains(err.Error(), "i/o timeout") { // hit a read or write deadline
name = "i/o timeout"
} else if strings.Contains(err.Error(), "read: connection reset by peer") { // the other side closed the connection using TCP reset
name = "read conn reset"
} else if strings.Contains(err.Error(), "unexpected EOF") { // tried to read from closed pipe or socket
name = "unexpected EOF"
} else if strings.Contains(err.Error(), "write: broken pipe") { // tried to write to a pipe or socket that was closed by the peer
name = "write broken pipe"
} else {
shouldLog = true
}
s.mu.Lock()
defer s.mu.Unlock()
s.errors[name]++
return
}
func (s *Stats) runSlackLogger() {
t := time.NewTicker(s.logFreq)
for {
select {
case <-s.grp.Ch():
return
case <-t.C:
s.log()
}
}
}
func (s *Stats) log() {
s.mu.Lock()
blobs, streams := s.blobs, s.streams
s.blobs, s.streams = 0, 0
errStr := ""
for name, count := range s.errors {
errStr += fmt.Sprintf("%d %s, ", count, name)
delete(s.errors, name)
}
s.mu.Unlock()
if len(errStr) > 2 {
errStr = errStr[:len(errStr)-2] // trim last comma and space
}
s.logger.Printf("%s stats: %d blobs, %d streams, errors: %s", s.name, blobs, streams, errStr)
}
+44 -37
View File
@@ -1,17 +1,17 @@
package reflector
import (
"io/ioutil"
"os"
"path"
"sync"
"time"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/stop"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
log "github.com/sirupsen/logrus"
)
@@ -29,24 +29,25 @@ type Summary struct {
}
type Uploader struct {
db *db.SQL
store *store.DBBackedS3Store // could just be store.BlobStore interface
workers int
skipExistsCheck bool
stopper *stop.Group
countChan chan increment
count Summary
db *db.SQL
store store.BlobStore
stopper *stop.Group
countChan chan increment
count Summary
workers int
skipExistsCheck bool
deleteBlobsAfterUpload bool
}
func NewUploader(db *db.SQL, store *store.DBBackedS3Store, workers int, skipExistsCheck bool) *Uploader {
func NewUploader(db *db.SQL, store store.BlobStore, workers int, skipExistsCheck, deleteBlobsAfterUpload bool) *Uploader {
return &Uploader{
db: db,
store: store,
workers: workers,
skipExistsCheck: skipExistsCheck,
stopper: stop.New(),
countChan: make(chan increment),
db: db,
store: store,
workers: workers,
skipExistsCheck: skipExistsCheck,
deleteBlobsAfterUpload: deleteBlobsAfterUpload,
stopper: stop.New(),
countChan: make(chan increment),
}
}
@@ -68,25 +69,27 @@ func (u *Uploader) Upload(dirOrFilePath string) error {
hashes[i] = path.Base(p)
}
log.Infoln("checking for existing blobs")
log.Debug("checking for existing blobs")
var exists map[string]bool
if !u.skipExistsCheck {
exists, err = u.db.HasBlobs(hashes)
exists, err = u.db.HasBlobs(hashes, false)
if err != nil {
return err
}
u.count.AlreadyStored = len(exists)
}
log.Infof("%d new blobs to upload", u.count.Total-u.count.AlreadyStored)
log.Debugf("%d new blobs to upload", u.count.Total-u.count.AlreadyStored)
workerWG := sync.WaitGroup{}
pathChan := make(chan string)
for i := 0; i < u.workers; i++ {
workerWG.Add(1)
metrics.RoutinesQueue.WithLabelValues("reflector", "upload").Inc()
go func(i int) {
defer metrics.RoutinesQueue.WithLabelValues("reflector", "upload").Dec()
defer workerWG.Done()
defer func(i int) { log.Debugf("worker %d quitting", i) }(i)
u.worker(pathChan)
@@ -95,7 +98,9 @@ func (u *Uploader) Upload(dirOrFilePath string) error {
countWG := sync.WaitGroup{}
countWG.Add(1)
metrics.RoutinesQueue.WithLabelValues("reflector", "uploader").Inc()
go func() {
defer metrics.RoutinesQueue.WithLabelValues("reflector", "uploader").Dec()
defer countWG.Done()
u.counter()
}()
@@ -119,17 +124,14 @@ Upload:
countWG.Wait()
u.stopper.Stop()
log.Infoln("SUMMARY")
log.Infof("%d blobs total", u.count.Total)
log.Infof("%d blobs already stored", u.count.AlreadyStored)
log.Infof("%d SD blobs uploaded", u.count.Sd)
log.Infof("%d content blobs uploaded", u.count.Blob)
log.Infof("%d errors encountered", u.count.Err)
log.Debugf(
"upload stats: %d blobs total, %d already stored, %d SD blobs uploaded, %d content blobs uploaded, %d errors",
u.count.Total, u.count.AlreadyStored, u.count.Sd, u.count.Blob, u.count.Err,
)
return nil
}
// worker reads paths from a channel and uploads them
// worker reads paths from a channel, uploads them, and optionally deletes them
func (u *Uploader) worker(pathChan chan string) {
for {
select {
@@ -143,6 +145,11 @@ func (u *Uploader) worker(pathChan chan string) {
err := u.uploadBlob(filepath)
if err != nil {
log.Errorln(err)
} else if u.deleteBlobsAfterUpload {
err = os.Remove(filepath)
if err != nil {
log.Errorln(errors.Prefix("deleting blob", err))
}
}
}
}
@@ -156,9 +163,9 @@ func (u *Uploader) uploadBlob(filepath string) (err error) {
}
}()
blob, err := ioutil.ReadFile(filepath)
blob, err := os.ReadFile(filepath)
if err != nil {
return err
return errors.Err(err)
}
hash := BlobHash(blob)
@@ -167,17 +174,17 @@ func (u *Uploader) uploadBlob(filepath string) (err error) {
}
if IsValidJSON(blob) {
log.Debugf("Uploading SD blob %s", hash)
err := u.store.PutSD(hash, blob)
log.Debugf("uploading SD blob %s", hash)
err = u.store.PutSD(hash, blob)
if err != nil {
return errors.Prefix("Uploading SD blob "+hash, err)
return errors.Prefix("uploading SD blob "+hash, err)
}
u.inc(sdInc)
} else {
log.Debugf("Uploading blob %s", hash)
log.Debugf("uploading blob %s", hash)
err = u.store.Put(hash, blob)
if err != nil {
return errors.Prefix("Uploading blob "+hash, err)
return errors.Prefix("uploading blob "+hash, err)
}
u.inc(blobInc)
}
@@ -208,7 +215,7 @@ func (u *Uploader) counter() {
}
}
if (u.count.Sd+u.count.Blob)%50 == 0 {
log.Infof("%d of %d done (%s elapsed, %.3fs per blob)", u.count.Sd+u.count.Blob, u.count.Total-u.count.AlreadyStored, time.Since(start).String(), time.Since(start).Seconds()/float64(u.count.Sd+u.count.Blob))
log.Debugf("%d of %d done (%s elapsed, %.3fs per blob)", u.count.Sd+u.count.Blob, u.count.Total-u.count.AlreadyStored, time.Since(start).String(), time.Since(start).Seconds()/float64(u.count.Sd+u.count.Blob))
}
}
}
+16
View File
@@ -0,0 +1,16 @@
#!/bin/bash
set -euo pipefail
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
cd "$SCRIPT_DIR/.."
if ! command -v golangci-lint &> /dev/null; then
echo "Installing golangci-lint..."
go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
else
echo "Ensuring golangci-lint is up to date..."
go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
fi
echo "Running golangci-lint..."
golangci-lint run --config .golangci.yml ./...
+107
View File
@@ -0,0 +1,107 @@
package http
import (
"net/http"
"strconv"
"sync"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/gin-gonic/gin"
log "github.com/sirupsen/logrus"
)
func (s *Server) getBlob(c *gin.Context) {
waiter := &sync.WaitGroup{}
waiter.Add(1)
enqueue(&blobRequest{c: c, finished: waiter})
waiter.Wait()
}
func (s *Server) HandleGetBlob(c *gin.Context) {
defer func() {
if r := recover(); r != nil {
log.Errorf("Recovered from panic: %v", r)
}
}()
start := time.Now()
hash := c.Query("hash")
edgeToken := c.Query("edge_token")
if reflector.IsProtected(hash) && edgeToken != s.edgeToken {
_ = c.Error(errors.Err("requested blob is protected"))
c.String(http.StatusForbidden, "requested blob is protected")
return
}
if s.missesCache.Has(hash) {
serialized, err := shared.NewBlobTrace(time.Since(start), "http").Serialize()
c.Header("Via", serialized)
if err != nil {
_ = c.Error(errors.Err(err))
c.String(http.StatusInternalServerError, err.Error())
return
}
c.AbortWithStatus(http.StatusNotFound)
return
}
blob, trace, err := s.store.Get(hash)
if err != nil {
serialized, serializeErr := trace.Serialize()
if serializeErr != nil {
_ = c.Error(errors.Prefix(serializeErr.Error(), err))
c.String(http.StatusInternalServerError, errors.Prefix(serializeErr.Error(), err).Error())
return
}
c.Header("Via", serialized)
if errors.Is(err, store.ErrBlobNotFound) {
_ = s.missesCache.Set(hash, true)
c.AbortWithStatus(http.StatusNotFound)
return
}
_ = c.Error(err)
c.String(http.StatusInternalServerError, err.Error())
return
}
serialized, err := trace.Serialize()
if err != nil {
_ = c.Error(err)
c.String(http.StatusInternalServerError, err.Error())
return
}
metrics.MtrOutBytesHttp.Add(float64(len(blob)))
metrics.BlobDownloadCount.Inc()
metrics.HttpDownloadCount.Inc()
c.Header("Via", serialized)
c.Header("Content-Disposition", "filename="+hash)
c.Header("Content-Length", strconv.Itoa(len(blob)))
c.Data(http.StatusOK, "application/octet-stream", blob)
}
func (s *Server) hasBlob(c *gin.Context) {
hash := c.Query("hash")
has, err := s.store.Has(hash)
if err != nil {
_ = c.Error(err)
c.String(http.StatusInternalServerError, err.Error())
return
}
if has {
c.Status(http.StatusNoContent)
return
}
c.Status(http.StatusNotFound)
}
func (s *Server) recoveryHandler(c *gin.Context, err interface{}) {
c.JSON(500, gin.H{
"title": "Error",
"err": err,
})
}
+84
View File
@@ -0,0 +1,84 @@
package http
import (
"context"
"net/http"
"time"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/bluele/gcache"
nice "github.com/ekyoung/gin-nice-recovery"
"github.com/gin-gonic/gin"
log "github.com/sirupsen/logrus"
)
// Server is an instance of a peer server that houses the listener and store.
type Server struct {
store store.BlobStore
grp *stop.Group
missesCache gcache.Cache
edgeToken string
address string
concurrentRequests int
}
// NewServer returns an initialized Server pointer.
func NewServer(store store.BlobStore, requestQueueSize int, edgeToken string, address string) *Server {
return &Server{
store: store,
grp: stop.New(),
concurrentRequests: requestQueueSize,
missesCache: gcache.New(2000).Expiration(5 * time.Minute).ARC().Build(),
edgeToken: edgeToken,
address: address,
}
}
// Shutdown gracefully shuts down the peer server.
func (s *Server) Shutdown() {
log.Debug("shutting down HTTP server")
s.grp.StopAndWait()
log.Debug("HTTP server stopped")
}
// Start starts the server listener to handle connections.
func (s *Server) Start() error {
gin.SetMode(gin.ReleaseMode)
router := gin.New()
router.Use(gin.Logger())
// Install nice.Recovery, passing the handler to call after recovery
router.Use(nice.Recovery(s.recoveryHandler))
router.GET("/blob", s.getBlob)
router.HEAD("/blob", s.hasBlob)
srv := &http.Server{
Addr: s.address,
Handler: router,
}
go s.listenForShutdown(srv)
go InitWorkers(s, s.concurrentRequests)
// Initializing the server in a goroutine so that
// it won't block the graceful shutdown handling below
s.grp.Add(1)
go func() {
defer s.grp.Done()
log.Println("HTTP server listening on " + s.address)
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
log.Fatalf("listen: %s\n", err)
}
}()
return nil
}
func (s *Server) listenForShutdown(listener *http.Server) {
<-s.grp.Ch()
// The context is used to inform the server it has 5 seconds to finish
// the request it is currently handling
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := listener.Shutdown(ctx); err != nil {
log.Fatal("Server forced to shutdown:", err)
}
}
+46
View File
@@ -0,0 +1,46 @@
package http
import (
"sync"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/gin-gonic/gin"
)
type blobRequest struct {
c *gin.Context
finished *sync.WaitGroup
}
var getReqCh = make(chan *blobRequest, 20000)
func InitWorkers(server *Server, workers int) {
stopper := stop.New(server.grp)
for i := 0; i < workers; i++ {
metrics.RoutinesQueue.WithLabelValues("http", "worker").Inc()
go func(worker int) {
defer metrics.RoutinesQueue.WithLabelValues("http", "worker").Dec()
for {
select {
case <-stopper.Ch():
case r := <-getReqCh:
process(server, r)
metrics.HttpBlobReqQueue.Dec()
}
}
}(i)
}
}
func enqueue(b *blobRequest) {
metrics.HttpBlobReqQueue.Inc()
getReqCh <- b
}
func process(server *Server, r *blobRequest) {
server.HandleGetBlob(r.c)
r.finished.Done()
}
+142
View File
@@ -0,0 +1,142 @@
package http3
import (
"bytes"
"encoding/hex"
"fmt"
"io"
"net/http"
"sync"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/quic-go/quic-go/http3"
log "github.com/sirupsen/logrus"
)
// Client is an instance of a client connected to a server.
type Client struct {
conn *http.Client
roundTripper *http3.Transport
ServerAddr string
Timeout time.Duration
}
// Close closes the connection with the client.
func (c *Client) Close() error {
c.conn.CloseIdleConnections()
return c.roundTripper.Close()
}
// GetStream gets a stream
func (c *Client) GetStream(sdHash string, blobCache store.BlobStore) (stream.Stream, error) {
var sd stream.SDBlob
b, _, err := c.GetBlob(sdHash)
if err != nil {
return nil, err
}
err = sd.FromBlob(b)
if err != nil {
return nil, err
}
s := make(stream.Stream, len(sd.BlobInfos)+1-1) // +1 for sd blob, -1 for last null blob
s[0] = b
for i := 0; i < len(sd.BlobInfos)-1; i++ {
var trace shared.BlobTrace
s[i+1], trace, err = c.GetBlob(hex.EncodeToString(sd.BlobInfos[i].BlobHash))
if err != nil {
return nil, err
}
log.Debug(trace.String())
}
return s, nil
}
// HasBlob checks if the blob is available
func (c *Client) HasBlob(hash string) (bool, error) {
resp, err := c.conn.Get(fmt.Sprintf("https://%s/has/%s", c.ServerAddr, hash))
if err != nil {
return false, errors.Err(err)
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode == http.StatusOK {
return true, nil
}
if resp.StatusCode == http.StatusNotFound {
return false, nil
}
return false, errors.Err("non 200 status code returned: %d", resp.StatusCode)
}
// GetBlob gets a blob
func (c *Client) GetBlob(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
resp, err := c.conn.Get(fmt.Sprintf("https://%s/get/%s?trace=true", c.ServerAddr, hash))
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err(err)
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode == http.StatusNotFound {
fmt.Printf("%s blob not found %d\n", hash, resp.StatusCode)
return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err(store.ErrBlobNotFound)
} else if resp.StatusCode != http.StatusOK {
return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err("non 200 status code returned: %d", resp.StatusCode)
}
tmp := getBuffer()
defer putBuffer(tmp)
serialized := resp.Header.Get("Via")
trace := shared.NewBlobTrace(time.Since(start), "http3")
if serialized != "" {
var parsedTrace *shared.BlobTrace
parsedTrace, err = shared.Deserialize(serialized)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "http3"), err
}
trace = *parsedTrace
}
written, err := io.Copy(tmp, resp.Body)
if err != nil {
return nil, trace.Stack(time.Since(start), "http3"), errors.Err(err)
}
blob := make([]byte, written)
copy(blob, tmp.Bytes())
metrics.MtrInBytesUdp.Add(float64(len(blob)))
return blob, trace.Stack(time.Since(start), "http3"), nil
}
// buffer pool to reduce GC
// https://www.captaincodeman.com/2017/06/02/golang-buffer-pool-gotcha
var buffers = sync.Pool{
// New is called when a new instance is needed
New: func() interface{} {
buf := make([]byte, 0, stream.MaxBlobSize)
return bytes.NewBuffer(buf)
},
}
// getBuffer fetches a buffer from the pool
func getBuffer() *bytes.Buffer {
return buffers.Get().(*bytes.Buffer)
}
// putBuffer returns a buffer to the pool
func putBuffer(buf *bytes.Buffer) {
buf.Reset()
buffers.Put(buf)
}
+220
View File
@@ -0,0 +1,220 @@
package http3
import (
"crypto/rand"
"crypto/rsa"
"crypto/tls"
"crypto/x509"
"encoding/json"
"encoding/pem"
"fmt"
"math/big"
"net/http"
"strconv"
"strings"
"sync"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/gorilla/mux"
"github.com/quic-go/quic-go"
"github.com/quic-go/quic-go/http3"
log "github.com/sirupsen/logrus"
)
// Server is an instance of a peer server that houses the listener and store.
type Server struct {
store store.BlobStore
grp *stop.Group
address string
concurrentRequests int
}
// NewServer returns an initialized Server pointer.
func NewServer(store store.BlobStore, requestQueueSize int, address string) *Server {
return &Server{
store: store,
grp: stop.New(),
concurrentRequests: requestQueueSize,
address: address,
}
}
// Shutdown gracefully shuts down the peer server.
func (s *Server) Shutdown() {
log.Debug("shutting down http3 peer server")
s.grp.StopAndWait()
log.Debug("http3 peer server stopped")
}
func (s *Server) logError(e error) {
if e == nil {
return
}
shouldLog := metrics.TrackError(metrics.DirectionDownload, e)
if shouldLog {
log.Errorln(errors.FullTrace(e))
}
}
type availabilityResponse struct {
LbrycrdAddress string `json:"lbrycrd_address"`
IsAvailable bool `json:"is_available"`
}
// Start starts the server listener to handle connections.
func (s *Server) Start() error {
log.Println("HTTP3 peer listening on " + s.address)
window500M := 500 * 1 << 20
quicConf := &quic.Config{
MaxStreamReceiveWindow: uint64(window500M),
MaxConnectionReceiveWindow: uint64(window500M),
EnableDatagrams: true,
HandshakeIdleTimeout: 4 * time.Second,
MaxIdleTimeout: 20 * time.Second,
}
r := mux.NewRouter()
r.HandleFunc("/get/{hash}", func(w http.ResponseWriter, r *http.Request) {
waiter := &sync.WaitGroup{}
waiter.Add(1)
enqueue(&blobRequest{request: r, reply: w, finished: waiter})
waiter.Wait()
})
r.HandleFunc("/has/{hash}", func(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
requestedBlob := vars["hash"]
blobExists, err := s.store.Has(requestedBlob)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
s.logError(err)
return
}
if !blobExists {
w.WriteHeader(http.StatusNotFound)
}
// LbrycrdAddress to be used when paying for data. Not implemented yet.
const LbrycrdAddress = "bJxKvpD96kaJLriqVajZ7SaQTsWWyrGQct"
resp, err := json.Marshal(availabilityResponse{
LbrycrdAddress: LbrycrdAddress,
IsAvailable: blobExists,
})
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
s.logError(err)
return
}
_, err = w.Write(resp)
if err != nil {
s.logError(err)
}
})
server := http3.Server{
Addr: s.address,
Handler: r,
TLSConfig: generateTLSConfig(),
QUICConfig: quicConf,
}
go InitWorkers(s, s.concurrentRequests)
go s.listenForShutdown(&server)
s.grp.Add(1)
go func() {
s.listenAndServe(&server)
s.grp.Done()
}()
return nil
}
// Setup a bare-bones TLS config for the server
func generateTLSConfig() *tls.Config {
key, err := rsa.GenerateKey(rand.Reader, 1024)
if err != nil {
panic(err)
}
template := x509.Certificate{SerialNumber: big.NewInt(1)}
certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key)
if err != nil {
panic(err)
}
keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})
certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
tlsCert, err := tls.X509KeyPair(certPEM, keyPEM)
if err != nil {
panic(err)
}
return &tls.Config{
Certificates: []tls.Certificate{tlsCert},
NextProtos: []string{"http3-reflector-server"},
}
}
func (s *Server) listenAndServe(server *http3.Server) {
err := server.ListenAndServe()
if err != nil && !strings.Contains(err.Error(), "Server closed") {
log.Errorln(errors.FullTrace(err))
}
}
func (s *Server) listenForShutdown(listener *http3.Server) {
<-s.grp.Ch()
err := listener.Close()
if err != nil {
log.Error("error closing listener for peer server - ", err)
}
}
func (s *Server) HandleGetBlob(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
requestedBlob := vars["hash"]
traceParam := r.URL.Query().Get("trace")
var err error
wantsTrace := false
if traceParam != "" {
wantsTrace, err = strconv.ParseBool(traceParam)
if err != nil {
wantsTrace = false
}
}
if reflector.IsProtected(requestedBlob) {
http.Error(w, "requested blob is protected", http.StatusForbidden)
return
}
blob, trace, err := s.store.Get(requestedBlob)
if wantsTrace {
var serialized string
serialized, err = trace.Serialize()
if err != nil {
http.Error(w, err.Error(), http.StatusNotFound)
return
}
w.Header().Add("Via", serialized)
log.Debug(trace.String())
}
if err != nil {
if errors.Is(err, store.ErrBlobNotFound) {
http.Error(w, err.Error(), http.StatusNotFound)
return
}
fmt.Printf("%s: %s", requestedBlob, errors.FullTrace(err))
s.logError(err)
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
_, err = w.Write(blob)
if err != nil {
s.logError(err)
}
metrics.MtrOutBytesUdp.Add(float64(len(blob)))
metrics.BlobDownloadCount.Inc()
metrics.Http3DownloadCount.Inc()
}
+46
View File
@@ -0,0 +1,46 @@
package http3
import (
"net/http"
"sync"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/lbry.go/v2/extras/stop"
)
type blobRequest struct {
request *http.Request
reply http.ResponseWriter
finished *sync.WaitGroup
}
var getReqCh = make(chan *blobRequest, 20000)
func InitWorkers(server *Server, workers int) {
stopper := stop.New(server.grp)
for i := 0; i < workers; i++ {
metrics.RoutinesQueue.WithLabelValues("http3", "worker").Inc()
go func(worker int) {
defer metrics.RoutinesQueue.WithLabelValues("http3", "worker").Dec()
for {
select {
case <-stopper.Ch():
case r := <-getReqCh:
metrics.Http3BlobReqQueue.Dec()
process(server, r)
}
}
}(i)
}
}
func enqueue(b *blobRequest) {
metrics.Http3BlobReqQueue.Inc()
getReqCh <- b
}
func process(server *Server, r *blobRequest) {
server.HandleGetBlob(r.reply, r.request)
r.finished.Done()
}
+207
View File
@@ -0,0 +1,207 @@
package peer
import (
"bufio"
"encoding/hex"
"encoding/json"
"io"
"net"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
log "github.com/sirupsen/logrus"
)
// Client is an instance of a client connected to a server.
type Client struct {
conn net.Conn
buf *bufio.Reader
Timeout time.Duration
connected bool
}
// Connect connects to a specific clients and errors if it cannot be contacted.
func (c *Client) Connect(address string) error {
var err error
if c.Timeout == 0 {
c.Timeout = 5 * time.Second
}
c.conn, err = net.Dial("tcp4", address)
if err != nil {
return err
}
c.connected = true
c.buf = bufio.NewReader(c.conn)
return nil
}
// Close closes the connection with the client.
func (c *Client) Close() error {
c.connected = false
return c.conn.Close()
}
// GetStream gets a stream
func (c *Client) GetStream(sdHash string, blobCache store.BlobStore) (stream.Stream, error) {
if !c.connected {
return nil, errors.Err("not connected")
}
var sd stream.SDBlob
b, trace, err := c.GetBlob(sdHash)
if err != nil {
return nil, err
}
log.Debug(trace.String())
err = sd.FromBlob(b)
if err != nil {
return nil, err
}
s := make(stream.Stream, len(sd.BlobInfos)+1-1) // +1 for sd blob, -1 for last null blob
s[0] = b
for i := 0; i < len(sd.BlobInfos)-1; i++ {
s[i+1], trace, err = c.GetBlob(hex.EncodeToString(sd.BlobInfos[i].BlobHash))
if err != nil {
return nil, err
}
log.Debug(trace.String())
}
return s, nil
}
// HasBlob checks if the blob is available
func (c *Client) HasBlob(hash string) (bool, error) {
if !c.connected {
return false, errors.Err("not connected")
}
sendRequest, err := json.Marshal(availabilityRequest{
RequestedBlobs: []string{hash},
})
if err != nil {
return false, err
}
err = c.write(sendRequest)
if err != nil {
return false, err
}
var resp availabilityResponse
err = c.read(&resp)
if err != nil {
return false, err
}
for _, h := range resp.AvailableBlobs {
if h == hash {
return true, nil
}
}
return false, nil
}
// GetBlob gets a blob
func (c *Client) GetBlob(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
if !c.connected {
return nil, shared.NewBlobTrace(time.Since(start), "tcp"), errors.Err("not connected")
}
sendRequest, err := json.Marshal(blobRequest{
RequestedBlob: hash,
})
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "tcp"), err
}
err = c.write(sendRequest)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "tcp"), err
}
var resp blobResponse
err = c.read(&resp)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "tcp"), err
}
trace := shared.NewBlobTrace(time.Since(start), "tcp")
if resp.RequestTrace != nil {
trace = *resp.RequestTrace
}
if resp.IncomingBlob.Error != "" {
return nil, trace, errors.Prefix(hash[:8], resp.IncomingBlob.Error)
}
if resp.IncomingBlob.BlobHash != hash {
return nil, trace.Stack(time.Since(start), "tcp"), errors.Prefix(hash[:8], "blob hash in response does not match requested hash")
}
if resp.IncomingBlob.Length <= 0 {
return nil, trace, errors.Prefix(hash[:8], "length reported as <= 0")
}
log.Debugf("receiving blob %s from %s", hash[:8], c.conn.RemoteAddr())
blob, err := c.readRawBlob(resp.IncomingBlob.Length)
if err != nil {
return nil, (*resp.RequestTrace).Stack(time.Since(start), "tcp"), err
}
metrics.MtrInBytesTcp.Add(float64(len(blob)))
return blob, trace.Stack(time.Since(start), "tcp"), nil
}
func (c *Client) read(v interface{}) error {
err := c.conn.SetReadDeadline(time.Now().Add(c.Timeout))
if err != nil {
return errors.Err(err)
}
m, err := readNextMessage(c.buf)
if err != nil {
return err
}
log.Debugf("read %d bytes from %s", len(m), c.conn.RemoteAddr())
err = json.Unmarshal(m, v)
return errors.Err(err)
}
func (c *Client) readRawBlob(blobSize int) ([]byte, error) {
err := c.conn.SetReadDeadline(time.Now().Add(c.Timeout))
if err != nil {
return nil, errors.Err(err)
}
blob := make([]byte, blobSize)
n, err := io.ReadFull(c.buf, blob)
log.Debugf("read %d bytes from %s", n, c.conn.RemoteAddr())
return blob, errors.Err(err)
}
func (c *Client) write(b []byte) error {
err := c.conn.SetWriteDeadline(time.Now().Add(c.Timeout))
if err != nil {
return errors.Err(err)
}
log.Debugf("writing %d bytes to %s", len(b), c.conn.RemoteAddr())
n, err := c.conn.Write(b)
if err == nil && n != len(b) {
err = io.ErrShortWrite
}
return errors.Err(err)
}
+129 -119
View File
@@ -2,17 +2,22 @@ package peer
import (
"bufio"
"encoding/hex"
"encoding/json"
ee "errors"
"io"
"net"
"strings"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/stop"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/lbry.go/v2/stream"
log "github.com/sirupsen/logrus"
)
@@ -26,36 +31,32 @@ const (
// Server is an instance of a peer server that houses the listener and store.
type Server struct {
StatLogger *log.Logger // logger to log stats
StatReportFrequency time.Duration // how often to log stats
store store.BlobStore
closed bool
grp *stop.Group
stats *reflector.Stats
store store.BlobStore
grp *stop.Group
address string
closed bool
}
// NewServer returns an initialized Server pointer.
func NewServer(store store.BlobStore) *Server {
func NewServer(store store.BlobStore, address string) *Server {
return &Server{
store: store,
grp: stop.New(),
store: store,
grp: stop.New(),
address: address,
}
}
// Shutdown gracefully shuts down the peer server.
func (s *Server) Shutdown() {
log.Debug("shutting down peer server...")
s.stats.Shutdown()
log.Debug("shutting down peer server")
s.grp.StopAndWait()
log.Debug("peer server stopped")
}
// Start starts the server listener to handle connections.
func (s *Server) Start(address string) error {
log.Println("peer listening on " + address)
l, err := net.Listen("tcp4", address)
func (s *Server) Start() error {
log.Println("peer listening on " + s.address)
l, err := net.Listen("tcp4", s.address)
if err != nil {
return err
}
@@ -67,11 +68,6 @@ func (s *Server) Start(address string) error {
s.grp.Done()
}()
s.stats = reflector.NewStatLogger("DOWNLOAD", s.StatLogger, s.StatReportFrequency, s.grp.Child())
if s.StatLogger != nil && s.StatReportFrequency > 0 {
s.stats.Start()
}
return nil
}
@@ -94,7 +90,9 @@ func (s *Server) listenAndServe(listener net.Listener) {
log.Error(errors.Prefix("accepting conn", err))
} else {
s.grp.Add(1)
metrics.RoutinesQueue.WithLabelValues("peer", "server-handleconn").Inc()
go func() {
defer metrics.RoutinesQueue.WithLabelValues("peer", "server-handleconn").Dec()
s.handleConnection(conn)
s.grp.Done()
}()
@@ -110,6 +108,7 @@ func (s *Server) handleConnection(conn net.Conn) {
}()
timeoutDuration := 1 * time.Minute
buf := bufio.NewReader(conn)
for {
var request []byte
@@ -120,7 +119,7 @@ func (s *Server) handleConnection(conn net.Conn) {
log.Error(errors.FullTrace(err))
}
request, err = readNextRequest(conn)
request, err = readNextMessage(buf)
if err != nil {
if err != io.EOF {
s.logError(err)
@@ -135,7 +134,7 @@ func (s *Server) handleConnection(conn net.Conn) {
response, err = s.handleCompositeRequest(request)
if err != nil {
log.Error(err)
log.Error(errors.FullTrace(err))
return
}
@@ -166,14 +165,14 @@ func (s *Server) handleAvailabilityRequest(data []byte) ([]byte, error) {
var request availabilityRequest
err := json.Unmarshal(data, &request)
if err != nil {
return []byte{}, err
return nil, errors.Err(err)
}
availableBlobs := []string{}
for _, blobHash := range request.RequestedBlobs {
exists, err := s.store.Has(blobHash)
if err != nil {
return []byte{}, err
return nil, err
}
if exists {
availableBlobs = append(availableBlobs, blobHash)
@@ -183,99 +182,116 @@ func (s *Server) handleAvailabilityRequest(data []byte) ([]byte, error) {
return json.Marshal(availabilityResponse{LbrycrdAddress: LbrycrdAddress, AvailableBlobs: availableBlobs})
}
func (s *Server) handlePaymentRateNegotiation(data []byte) ([]byte, error) {
var request paymentRateRequest
err := json.Unmarshal(data, &request)
if err != nil {
return []byte{}, err
}
offerReply := paymentRateAccepted
if request.BlobDataPaymentRate < 0 {
offerReply = paymentRateTooLow
}
return json.Marshal(paymentRateResponse{BlobDataPaymentRate: offerReply})
}
func (s *Server) handleBlobRequest(data []byte) ([]byte, error) {
var request blobRequest
err := json.Unmarshal(data, &request)
if err != nil {
return []byte{}, err
}
log.Debugln("Sending blob " + request.RequestedBlob[:8])
blob, err := s.store.Get(request.RequestedBlob)
if err != nil {
return []byte{}, err
}
response, err := json.Marshal(blobResponse{IncomingBlob: incomingBlob{
BlobHash: reflector.BlobHash(blob),
Length: len(blob),
}})
if err != nil {
return []byte{}, err
}
return append(response, blob...), nil
}
//func (s *Server) handlePaymentRateNegotiation(data []byte) ([]byte, error) {
// var request paymentRateRequest
// err := json.Unmarshal(data, &request)
// if err != nil {
// return nil, err
// }
//
// offerReply := paymentRateAccepted
// if request.BlobDataPaymentRate < 0 {
// offerReply = paymentRateTooLow
// }
//
// return json.Marshal(paymentRateResponse{BlobDataPaymentRate: offerReply})
//}
//
//func (s *Server) handleBlobRequest(data []byte) ([]byte, error) {
// var request blobRequest
// err := json.Unmarshal(data, &request)
// if err != nil {
// return nil, err
// }
//
// log.Debugln("Sending blob " + request.RequestedBlob[:8])
//
// blob, err := s.store.Get(request.RequestedBlob)
// if err != nil {
// return nil, err
// }
//
// response, err := json.Marshal(blobResponse{IncomingBlob: incomingBlob{
// BlobHash: reflector.BlobHash(blob),
// Length: len(blob),
// }})
// if err != nil {
// return nil, err
// }
//
// return append(response, blob...), nil
//}
func (s *Server) handleCompositeRequest(data []byte) ([]byte, error) {
var request compositeRequest
err := json.Unmarshal(data, &request)
if err != nil {
return []byte{}, err
var je *json.SyntaxError
if ee.As(err, &je) {
return nil, errors.Err("invalid json request: offset %d in data %s", je.Offset, hex.EncodeToString(data))
}
return nil, errors.Err(err)
}
response := compositeResponse{
LbrycrdAddress: LbrycrdAddress,
AvailableBlobs: []string{},
}
if len(request.RequestedBlobs) > 0 {
var availableBlobs []string
for _, blobHash := range request.RequestedBlobs {
exists, err := s.store.Has(blobHash)
if reflector.IsProtected(blobHash) {
return nil, errors.Err("requested blob is protected")
}
var exists bool
exists, err = s.store.Has(blobHash)
if err != nil {
return []byte{}, err
return nil, err
}
if exists {
availableBlobs = append(availableBlobs, blobHash)
response.AvailableBlobs = append(response.AvailableBlobs, blobHash)
}
}
response.AvailableBlobs = availableBlobs
}
response.BlobDataPaymentRate = paymentRateAccepted
if request.BlobDataPaymentRate < 0 {
response.BlobDataPaymentRate = paymentRateTooLow
if request.BlobDataPaymentRate != nil {
response.BlobDataPaymentRate = paymentRateAccepted
if *request.BlobDataPaymentRate < 0 {
response.BlobDataPaymentRate = paymentRateTooLow
}
}
var blob []byte
var trace shared.BlobTrace
if request.RequestedBlob != "" {
if len(request.RequestedBlob) != stream.BlobHashHexLength {
return nil, errors.Err("Invalid blob hash length")
}
log.Debugln("Sending blob " + request.RequestedBlob[:8])
blob, err = s.store.Get(request.RequestedBlob)
blob, trace, err = s.store.Get(request.RequestedBlob)
log.Debug(trace.String())
if errors.Is(err, store.ErrBlobNotFound) {
response.IncomingBlob = incomingBlob{
response.IncomingBlob = &incomingBlob{
Error: err.Error(),
}
} else if err != nil {
return []byte{}, err
return nil, err
} else {
response.IncomingBlob = incomingBlob{
BlobHash: reflector.BlobHash(blob),
response.IncomingBlob = &incomingBlob{
BlobHash: request.RequestedBlob,
Length: len(blob),
}
s.stats.AddBlob()
metrics.MtrOutBytesTcp.Add(float64(len(blob)))
metrics.BlobDownloadCount.Inc()
metrics.PeerDownloadCount.Inc()
}
}
respData, err := json.Marshal(response)
if err != nil {
return []byte{}, err
return nil, err
}
return append(respData, blob...), nil
@@ -285,40 +301,30 @@ func (s *Server) logError(e error) {
if e == nil {
return
}
shouldLog := s.stats.AddError(e)
shouldLog := metrics.TrackError(metrics.DirectionDownload, e)
if shouldLog {
log.Errorln(errors.FullTrace(e))
}
return
// old stuff below. its here for posterity, because we're gonna have to deal with these errors someday for real
//err := errors.Wrap(e, 0)
// these happen because the peer protocol does not have a way to cancel blob downloads
// so the client will just close the connection if its in the middle of downloading a blob
// but receives the blob from a different peer first or simply goes offline (timeout)
//if strings.Contains(err.Error(), "connection reset by peer") ||
// strings.Contains(err.Error(), "i/o timeout") ||
// strings.Contains(err.Error(), "broken pipe") {
// return
//}
//
//log.Error(errors.FullTrace(e))
}
func readNextRequest(conn net.Conn) ([]byte, error) {
request := make([]byte, 0)
func readNextMessage(buf *bufio.Reader) ([]byte, error) {
firstByte, err := buf.ReadByte()
if err != nil {
return nil, err
}
if firstByte != '{' {
// every request starts with '{'. Checking here disconnects earlier, so we don't wait until timeout
return nil, errInvalidData
}
msg := []byte("{")
eof := false
buf := bufio.NewReader(conn)
for {
chunk, err := buf.ReadBytes('}')
if err != nil {
if err != io.EOF {
//log.Errorln("readBytes error:", err) // logged by caller
return request, err
return msg, err
}
eof = true
}
@@ -327,14 +333,16 @@ func readNextRequest(conn net.Conn) ([]byte, error) {
//spew.Dump(chunk)
if len(chunk) > 0 {
request = append(request, chunk...)
msg = append(msg, chunk...)
if len(request) > maxRequestSize {
return request, errRequestTooLarge
if len(msg) > maxRequestSize {
return msg, errRequestTooLarge
} else if len(msg) > 0 && msg[0] != '{' {
return msg, errInvalidData
}
// yes, this is how the peer protocol knows when the request finishes
if reflector.IsValidJSON(request) {
if reflector.IsValidJSON(msg) {
break
}
}
@@ -349,11 +357,11 @@ func readNextRequest(conn net.Conn) ([]byte, error) {
// spew.Dump(request)
//}
if len(request) == 0 && eof {
return []byte{}, io.EOF
if len(msg) == 0 && eof {
return nil, io.EOF
}
return request, nil
return msg, nil
}
const (
@@ -365,10 +373,11 @@ const (
)
var errRequestTooLarge = errors.Base("request is too large")
var errInvalidData = errors.Base("Invalid data")
type availabilityRequest struct {
LbrycrdAddress bool `json:"lbrycrd_address"`
RequestedBlobs []string `json:"requested_blobs"`
LbrycrdAddress bool `json:"lbrycrd_address"`
}
type availabilityResponse struct {
@@ -394,19 +403,20 @@ type incomingBlob struct {
Length int `json:"length"`
}
type blobResponse struct {
RequestTrace *shared.BlobTrace
IncomingBlob incomingBlob `json:"incoming_blob"`
}
type compositeRequest struct {
LbrycrdAddress bool `json:"lbrycrd_address"`
RequestedBlobs []string `json:"requested_blobs"`
BlobDataPaymentRate float64 `json:"blob_data_payment_rate"`
BlobDataPaymentRate *float64 `json:"blob_data_payment_rate"`
RequestedBlob string `json:"requested_blob"`
RequestedBlobs []string `json:"requested_blobs"`
LbrycrdAddress bool `json:"lbrycrd_address"`
}
type compositeResponse struct {
LbrycrdAddress string `json:"lbrycrd_address,omitempty"`
AvailableBlobs []string `json:"available_blobs,omitempty"`
BlobDataPaymentRate string `json:"blob_data_payment_rate,omitempty"`
IncomingBlob incomingBlob `json:"incoming_blob,omitempty"`
IncomingBlob *incomingBlob `json:"incoming_blob,omitempty"`
LbrycrdAddress string `json:"lbrycrd_address,omitempty"`
BlobDataPaymentRate string `json:"blob_data_payment_rate,omitempty"`
AvailableBlobs []string `json:"available_blobs"`
}
@@ -2,7 +2,10 @@ package peer
import (
"bytes"
"io"
"net"
"testing"
"time"
"github.com/lbryio/reflector.go/store"
)
@@ -34,7 +37,9 @@ var availabilityRequests = []pair{
}
func getServer(t *testing.T, withBlobs bool) *Server {
st := store.MemoryBlobStore{}
st := store.NewMemStore(store.MemParams{
Name: "test",
})
if withBlobs {
for k, v := range blobs {
err := st.Put(k, v)
@@ -43,7 +48,7 @@ func getServer(t *testing.T, withBlobs bool) *Server {
}
}
}
return NewServer(&st)
return NewServer(st, "127.0.0.1:50505")
}
func TestAvailabilityRequest_NoBlobs(t *testing.T) {
@@ -75,3 +80,62 @@ func TestAvailabilityRequest_WithBlobs(t *testing.T) {
}
}
}
func TestRequestFromConnection(t *testing.T) {
s := getServer(t, true)
err := s.Start()
defer s.Shutdown()
if err != nil {
t.Error("error starting server", err)
}
for _, p := range availabilityRequests {
conn, err := net.Dial("tcp", "127.0.0.1:50505")
if err != nil {
t.Error("error opening connection", err)
}
defer func() { _ = conn.Close() }()
response := make([]byte, 8192)
_, err = conn.Write(p.request)
if err != nil {
t.Error("error writing", err)
}
_, err = conn.Read(response)
if err != nil {
t.Error("error reading", err)
}
if !bytes.Equal(response[:len(p.response)], p.response) {
t.Errorf("Response did not match expected response.\nExpected: %s\nGot: %s", string(p.response), string(response))
}
}
}
func TestInvalidData(t *testing.T) {
s := getServer(t, true)
err := s.Start()
defer s.Shutdown()
if err != nil {
t.Error("error starting server", err)
}
conn, err := net.Dial("tcp", "127.0.0.1:50505")
if err != nil {
t.Error("error opening connection", err)
}
defer func() { _ = conn.Close() }()
response := make([]byte, 8192)
_, err = conn.Write([]byte("hello dear server, I would like blobs. Please"))
if err != nil {
t.Error("error writing", err)
}
err = conn.SetReadDeadline(time.Now().Add(5 * time.Second))
if err != nil {
t.Error("error setting read deadline", err)
}
_, err = conn.Read(response)
if err != io.EOF {
t.Error("error reading", err)
}
println(response)
}
+14
View File
@@ -0,0 +1,14 @@
package server
// BlobServer defines the common interface for all blob server implementations
type BlobServer interface {
Start() error
Shutdown()
}
type BlobServerConfig struct {
Address string `mapstructure:"address"`
EdgeToken string `mapstructure:"edge_token"`
Port int `mapstructure:"port"`
MaxConcurrentRequests int `mapstructure:"max_concurrent_requests"`
}
+6
View File
@@ -0,0 +1,6 @@
package shared
import "github.com/lbryio/lbry.go/v2/extras/errors"
// ErrNotImplemented is a standard error when a store that implements the store interface does not implement a method
var ErrNotImplemented = errors.Base("this store does not implement this method")
+82
View File
@@ -0,0 +1,82 @@
package shared
import (
"encoding/json"
"fmt"
"os"
"time"
"github.com/lbryio/lbry.go/v2/extras/errors"
)
type BlobStack struct {
OriginName string `json:"origin_name"`
HostName string `json:"host_name"`
Timing time.Duration `json:"timing"`
}
type BlobTrace struct {
Stacks []BlobStack `json:"stacks"`
}
var hostName *string
func getHostName() string {
if hostName == nil {
hn, err := os.Hostname()
if err != nil {
hn = "unknown"
}
hostName = &hn
}
return *hostName
}
func (b *BlobTrace) Stack(timing time.Duration, originName string) BlobTrace {
b.Stacks = append(b.Stacks, BlobStack{
Timing: timing,
OriginName: originName,
HostName: getHostName(),
})
return *b
}
func (b *BlobTrace) Merge(otherTrance BlobTrace) BlobTrace {
b.Stacks = append(b.Stacks, otherTrance.Stacks...)
return *b
}
func NewBlobTrace(timing time.Duration, originName string) BlobTrace {
b := BlobTrace{}
b.Stacks = append(b.Stacks, BlobStack{
Timing: timing,
OriginName: originName,
HostName: getHostName(),
})
return b
}
func (b BlobTrace) String() string {
var fullTrace string
for i, stack := range b.Stacks {
delta := time.Duration(0)
if i > 0 {
delta = stack.Timing - b.Stacks[i-1].Timing
}
fullTrace += fmt.Sprintf("[%d](%s) origin: %s - timing: %s - delta: %s\n", i, stack.HostName, stack.OriginName, stack.Timing.String(), delta.String())
}
return fullTrace
}
func (b BlobTrace) Serialize() (string, error) {
t, err := json.Marshal(b)
if err != nil {
return "", errors.Err(err)
}
return string(t), nil
}
func Deserialize(serializedData string) (*BlobTrace, error) {
var trace BlobTrace
err := json.Unmarshal([]byte(serializedData), &trace)
if err != nil {
return nil, errors.Err(err)
}
return &trace, nil
}
+44
View File
@@ -0,0 +1,44 @@
package shared
import (
"testing"
"time"
"github.com/lbryio/lbry.go/v2/extras/util"
"github.com/stretchr/testify/assert"
)
func TestBlobTrace_Serialize(t *testing.T) {
hostName = util.PtrToString("test_machine")
stack := NewBlobTrace(10*time.Second, "test")
stack.Stack(20*time.Second, "test2")
stack.Stack(30*time.Second, "test3")
serialized, err := stack.Serialize()
assert.NoError(t, err)
deserialized, err := Deserialize(serialized)
assert.NoError(t, err)
assert.Len(t, deserialized.Stacks, 3)
assert.Equal(t, 10*time.Second, deserialized.Stacks[0].Timing)
assert.Equal(t, "test", deserialized.Stacks[0].OriginName)
assert.Equal(t, "test_machine", deserialized.Stacks[0].HostName)
assert.Equal(t, 20*time.Second, deserialized.Stacks[1].Timing)
assert.Equal(t, "test2", deserialized.Stacks[1].OriginName)
assert.Equal(t, 30*time.Second, deserialized.Stacks[2].Timing)
assert.Equal(t, "test3", deserialized.Stacks[2].OriginName)
}
func TestBlobTrace_Deserialize(t *testing.T) {
hostName = util.PtrToString("test_machine")
serialized := "{\"stacks\":[{\"timing\":10000000000,\"origin_name\":\"test\"},{\"timing\":20000000000,\"origin_name\":\"test2\"},{\"timing\":30000000000,\"origin_name\":\"test3\"}]}"
stack, err := Deserialize(serialized)
assert.NoError(t, err)
assert.Len(t, stack.Stacks, 3)
assert.Equal(t, stack.Stacks[0].Timing, 10*time.Second)
assert.Equal(t, stack.Stacks[1].Timing, 20*time.Second)
assert.Equal(t, stack.Stacks[2].Timing, 30*time.Second)
assert.Equal(t, stack.Stacks[0].OriginName, "test")
assert.Equal(t, stack.Stacks[1].OriginName, "test2")
assert.Equal(t, stack.Stacks[2].OriginName, "test3")
}
+18
View File
@@ -0,0 +1,18 @@
//go:build linux
// +build linux
package store
import (
"os"
"syscall"
"time"
)
func timespecToTime(ts syscall.Timespec) time.Time {
return time.Unix(ts.Sec, ts.Nsec)
}
func atime(fi os.FileInfo) time.Time {
return timespecToTime(fi.Sys().(*syscall.Stat_t).Atim)
}
+13
View File
@@ -0,0 +1,13 @@
//go:build !linux
// +build !linux
package store
import (
"os"
"time"
)
func atime(fi os.FileInfo) time.Time {
return fi.ModTime()
}
+166
View File
@@ -0,0 +1,166 @@
package store
import (
"strings"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
)
// CachingStore combines two stores, typically a local and a remote store, to improve performance.
// Accessed blobs are stored in and retrieved from the cache. If they are not in the cache, they
// are retrieved from the origin and cached. Puts are cached and also forwarded to the origin.
type CachingStore struct {
origin, cache BlobStore
name string
}
type CachingParams struct {
Origin BlobStore `mapstructure:"origin"`
Cache BlobStore `mapstructure:"cache"`
Name string `mapstructure:"name"`
}
type CachingConfig struct {
Origin *viper.Viper
Cache *viper.Viper
Name string `mapstructure:"name"`
}
// NewCachingStore makes a new caching disk store and returns a pointer to it.
func NewCachingStore(params CachingParams) *CachingStore {
return &CachingStore{
name: params.Name,
origin: WithSingleFlight(params.Name, params.Origin),
cache: WithSingleFlight(params.Name, params.Cache),
}
}
const nameCaching = "caching"
func CachingStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg CachingConfig
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
cfg.Cache = config.Sub("cache")
cfg.Origin = config.Sub("origin")
if cfg.Cache == nil || cfg.Origin == nil {
return nil, errors.Err("cache and origin missing")
}
originStoreType := strings.Split(cfg.Origin.AllKeys()[0], ".")[0]
originStoreConfig := cfg.Origin.Sub(originStoreType)
factory, ok := Factories[originStoreType]
if !ok {
return nil, errors.Err("unknown store type %s", originStoreType)
}
originStore, err := factory(originStoreConfig)
if err != nil {
return nil, errors.Err(err)
}
cacheStoreType := strings.Split(cfg.Cache.AllKeys()[0], ".")[0]
cacheStoreConfig := cfg.Cache.Sub(cacheStoreType)
factory, ok = Factories[cacheStoreType]
if !ok {
return nil, errors.Err("unknown store type %s", cacheStoreType)
}
cacheStore, err := factory(cacheStoreConfig)
if err != nil {
return nil, errors.Err(err)
}
return NewCachingStore(CachingParams{
Name: cfg.Name,
Origin: originStore,
Cache: cacheStore,
}), nil
}
func init() {
RegisterStore(nameCaching, CachingStoreFactory)
}
// Name is the cache type name
func (c *CachingStore) Name() string { return nameCaching }
// Has checks the cache and then the origin for a hash. It returns true if either store has it.
func (c *CachingStore) Has(hash string) (bool, error) {
has, err := c.cache.Has(hash)
if has || err != nil {
return has, err
}
return c.origin.Has(hash)
}
// Get tries to get the blob from the cache first, falling back to the origin. If the blob comes
// from the origin, it is also stored in the cache.
func (c *CachingStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
blob, trace, err := c.cache.Get(hash)
if err == nil || !errors.Is(err, ErrBlobNotFound) {
metrics.CacheHitCount.With(metrics.CacheLabels(c.cache.Name(), c.name)).Inc()
rate := float64(len(blob)) / 1024 / 1024 / time.Since(start).Seconds()
metrics.CacheRetrievalSpeed.With(map[string]string{
metrics.LabelCacheType: c.cache.Name(),
metrics.LabelComponent: c.name,
metrics.LabelSource: "cache",
}).Set(rate)
return blob, trace.Stack(time.Since(start), c.Name()), err
}
metrics.CacheMissCount.With(metrics.CacheLabels(c.cache.Name(), c.name)).Inc()
blob, trace, err = c.origin.Get(hash)
if err != nil {
return nil, trace.Stack(time.Since(start), c.Name()), err
}
// do not do this async unless you're prepared to deal with mayhem
err = c.cache.Put(hash, blob)
if err != nil {
log.Errorf("error saving blob to underlying cache: %s", errors.FullTrace(err))
}
return blob, trace.Stack(time.Since(start), c.Name()), nil
}
// Put stores the blob in the origin and the cache
func (c *CachingStore) Put(hash string, blob stream.Blob) error {
err := c.origin.Put(hash, blob)
if err != nil {
return err
}
return c.cache.Put(hash, blob)
}
// PutSD stores the sd blob in the origin and the cache
func (c *CachingStore) PutSD(hash string, blob stream.Blob) error {
err := c.origin.PutSD(hash, blob)
if err != nil {
return err
}
return c.cache.PutSD(hash, blob)
}
// Delete deletes the blob from the origin and the cache
func (c *CachingStore) Delete(hash string) error {
err := c.origin.Delete(hash)
if err != nil {
return err
}
return c.cache.Delete(hash)
}
// Shutdown shuts down the store gracefully
func (c *CachingStore) Shutdown() {
c.origin.Shutdown()
c.cache.Shutdown()
}
+178
View File
@@ -0,0 +1,178 @@
package store
import (
"bytes"
"sync"
"testing"
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/sirupsen/logrus"
)
func TestCachingStore_Put(t *testing.T) {
origin := NewMemStore(MemParams{Name: "test"})
cache := NewMemStore(MemParams{Name: "test"})
s := NewCachingStore(CachingParams{Name: "test", Origin: origin, Cache: cache})
b := []byte("this is a blob of stuff")
hash := "hash"
err := s.Put(hash, b)
if err != nil {
t.Fatal(err)
}
has, err := origin.Has(hash)
if err != nil {
t.Fatal(err)
}
if !has {
t.Errorf("failed to store blob in origin")
}
has, err = cache.Has(hash)
if err != nil {
t.Fatal(err)
}
if !has {
t.Errorf("failed to store blob in cache")
}
}
func TestCachingStore_CacheMiss(t *testing.T) {
origin := NewMemStore(MemParams{Name: "test"})
cache := NewMemStore(MemParams{Name: "test"})
s := NewCachingStore(CachingParams{Name: "test", Origin: origin, Cache: cache})
b := []byte("this is a blob of stuff")
hash := "hash"
err := origin.Put(hash, b)
if err != nil {
t.Fatal(err)
}
res, stack, err := s.Get(hash)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(b, res) {
t.Errorf("expected Get() to return %s, got %s", string(b), string(res))
}
time.Sleep(10 * time.Millisecond) //storing to cache is done async so let's give it some time
has, err := cache.Has(hash)
if err != nil {
t.Fatal(err)
}
if !has {
t.Errorf("Get() did not copy blob to cache")
}
t.Logf("stack: %s", stack.String())
res, stack, err = cache.Get(hash)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(b, res) {
t.Errorf("expected cached Get() to return %s, got %s", string(b), string(res))
}
t.Logf("stack: %s", stack.String())
}
func TestCachingStore_ThunderingHerd(t *testing.T) {
storeDelay := 100 * time.Millisecond
origin := NewSlowBlobStore(storeDelay)
cache := NewMemStore(MemParams{Name: "test"})
s := NewCachingStore(CachingParams{Name: "test", Origin: origin, Cache: cache})
b := []byte("this is a blob of stuff")
hash := "hash"
err := origin.Put(hash, b)
if err != nil {
t.Fatal(err)
}
wg := &sync.WaitGroup{}
getNoErr := func() {
res, _, err := s.Get(hash)
if err != nil {
logrus.Fatal(err)
}
if !bytes.Equal(b, res) {
t.Errorf("expected Get() to return %s, got %s", string(b), string(res))
}
wg.Done()
}
start := time.Now()
wg.Add(4)
go func() {
go getNoErr()
time.Sleep(10 * time.Millisecond)
go getNoErr()
time.Sleep(10 * time.Millisecond)
go getNoErr()
time.Sleep(10 * time.Millisecond)
go getNoErr()
}()
wg.Wait()
duration := time.Since(start)
// only the first getNoErr() should hit the origin. the rest should wait for the first request to return
// once the first returns, the others should return immediately
// therefore, if the delay much longer than 100ms, it means subsequent requests also went to the origin
expectedMaxDelay := storeDelay + 5*time.Millisecond // a bit of extra time to let requests finish
if duration > expectedMaxDelay {
t.Errorf("Expected delay of at most %s, got %s", expectedMaxDelay, duration)
}
}
// SlowBlobStore adds a delay to each request
type SlowBlobStore struct {
mem *MemStore
delay time.Duration
}
func NewSlowBlobStore(delay time.Duration) *SlowBlobStore {
return &SlowBlobStore{
mem: NewMemStore(MemParams{Name: "test"}),
delay: delay,
}
}
func (s *SlowBlobStore) Name() string {
return "slow"
}
func (s *SlowBlobStore) Has(hash string) (bool, error) {
time.Sleep(s.delay)
return s.mem.Has(hash)
}
func (s *SlowBlobStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
time.Sleep(s.delay)
return s.mem.Get(hash)
}
func (s *SlowBlobStore) Put(hash string, blob stream.Blob) error {
time.Sleep(s.delay)
return s.mem.Put(hash, blob)
}
func (s *SlowBlobStore) PutSD(hash string, blob stream.Blob) error {
time.Sleep(s.delay)
return s.mem.PutSD(hash, blob)
}
func (s *SlowBlobStore) Delete(hash string) error {
time.Sleep(s.delay)
return s.mem.Delete(hash)
}
func (s *SlowBlobStore) Shutdown() {}
+230 -46
View File
@@ -2,40 +2,110 @@ package store
import (
"encoding/json"
"fmt"
"strings"
"sync"
"time"
"github.com/lbryio/reflector.go/db"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/c2h5oh/datasize"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
)
// DBBackedS3Store is an instance of an S3 Store that is backed by a DB for what is stored.
type DBBackedS3Store struct {
s3 *S3BlobStore
db *db.SQL
blockedMu sync.RWMutex
blocked map[string]bool
// DBBackedStore is a store that's backed by a DB. The DB contains data about what's in the store.
type DBBackedStore struct {
blobs BlobStore
db *db.SQL
blocked map[string]bool
cleanerStop *stop.Group
name string
maxSize int
blockedMu sync.RWMutex
deleteOnMiss bool
}
// NewDBBackedS3Store returns an initialized store pointer.
func NewDBBackedS3Store(s3 *S3BlobStore, db *db.SQL) *DBBackedS3Store {
return &DBBackedS3Store{s3: s3, db: db}
type DBBackedParams struct {
Store BlobStore `mapstructure:"store"`
DB *db.SQL `mapstructure:"db"`
MaxSize *int `mapstructure:"max_size"`
Name string `mapstructure:"name"`
DeleteOnMiss bool `mapstructure:"delete_on_miss"`
}
type DBBackedConfig struct {
Store *viper.Viper
Database string `mapstructure:"database"`
User string `mapstructure:"user"`
Password string `mapstructure:"password"`
Host string `mapstructure:"host"`
Name string `mapstructure:"name"`
MaxSize string `mapstructure:"max_size"`
Port int `mapstructure:"port"`
AccessTracking int `mapstructure:"access_tracking"`
DeleteOnMiss bool `mapstructure:"delete_on_miss"`
SoftDeletes bool `mapstructure:"soft_deletes"`
LogQueries bool `mapstructure:"log_queries"`
HasCap bool `mapstructure:"has_cap"`
}
// NewDBBackedStore returns an initialized store pointer.
func NewDBBackedStore(params DBBackedParams) *DBBackedStore {
store := &DBBackedStore{
blobs: params.Store,
db: params.DB,
deleteOnMiss: params.DeleteOnMiss,
cleanerStop: stop.New(),
name: params.Name,
}
if params.MaxSize != nil {
store.maxSize = *params.MaxSize
go store.cleanOldestBlobs()
}
return store
}
const nameDBBacked = "db_backed"
// Name is the cache type name
func (d *DBBackedStore) Name() string { return nameDBBacked + "-" + d.name }
// Has returns true if the blob is in the store
func (d *DBBackedS3Store) Has(hash string) (bool, error) {
return d.db.HasBlob(hash)
func (d *DBBackedStore) Has(hash string) (bool, error) {
return d.db.HasBlob(hash, false)
}
// Get gets the blob
func (d *DBBackedS3Store) Get(hash string) ([]byte, error) {
return d.s3.Get(hash)
func (d *DBBackedStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
has, err := d.db.HasBlob(hash, true)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), err
}
if !has {
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), ErrBlobNotFound
}
b, stack, err := d.blobs.Get(hash)
if d.deleteOnMiss && errors.Is(err, ErrBlobNotFound) {
e2 := d.Delete(hash)
if e2 != nil {
log.Errorf("error while deleting blob from db: %s", errors.FullTrace(err))
}
}
return b, stack.Stack(time.Since(start), d.Name()), err
}
// Put stores the blob in the S3 store and stores the blob information in the DB.
func (d *DBBackedS3Store) Put(hash string, blob []byte) error {
err := d.s3.Put(hash, blob)
func (d *DBBackedStore) Put(hash string, blob stream.Blob) error {
err := d.blobs.Put(hash, blob)
if err != nil {
return err
}
@@ -45,17 +115,17 @@ func (d *DBBackedS3Store) Put(hash string, blob []byte) error {
// PutSD stores the SDBlob in the S3 store. It will return an error if the sd blob is missing the stream hash or if
// there is an error storing the blob information in the DB.
func (d *DBBackedS3Store) PutSD(hash string, blob []byte) error {
func (d *DBBackedStore) PutSD(hash string, blob stream.Blob) error {
var blobContents db.SdBlob
err := json.Unmarshal(blob, &blobContents)
if err != nil {
return err
return errors.Err(err)
}
if blobContents.StreamHash == "" {
return errors.Err("sd blob is missing stream hash")
}
err = d.s3.PutSD(hash, blob)
err = d.blobs.PutSD(hash, blob)
if err != nil {
return err
}
@@ -63,8 +133,8 @@ func (d *DBBackedS3Store) PutSD(hash string, blob []byte) error {
return d.db.AddSDBlob(hash, len(blob), blobContents)
}
func (d *DBBackedS3Store) Delete(hash string) error {
err := d.s3.Delete(hash)
func (d *DBBackedStore) Delete(hash string) error {
err := d.blobs.Delete(hash)
if err != nil {
return err
}
@@ -73,40 +143,24 @@ func (d *DBBackedS3Store) Delete(hash string) error {
}
// Block deletes the blob and prevents it from being uploaded in the future
func (d *DBBackedS3Store) Block(hash string) error {
if blocked, err := d.isBlocked(hash); blocked || err != nil {
func (d *DBBackedStore) Block(hash string) error {
blocked, err := d.isBlocked(hash)
if blocked || err != nil {
return err
}
log.Debugf("blocking %s", hash)
err := d.db.Block(hash)
err = d.db.Block(hash)
if err != nil {
return err
}
has, err := d.db.HasBlob(hash)
if err != nil {
return err
}
if has {
err = d.s3.Delete(hash)
if err != nil {
return err
}
err = d.db.Delete(hash)
if err != nil {
return err
}
}
return d.markBlocked(hash)
}
// Wants returns false if the hash exists or is blocked, true otherwise
func (d *DBBackedS3Store) Wants(hash string) (bool, error) {
func (d *DBBackedStore) Wants(hash string) (bool, error) {
blocked, err := d.isBlocked(hash)
if blocked || err != nil {
return false, err
@@ -119,11 +173,11 @@ func (d *DBBackedS3Store) Wants(hash string) (bool, error) {
// MissingBlobsForKnownStream returns missing blobs for an existing stream
// WARNING: if the stream does NOT exist, no blob hashes will be returned, which looks
// like no blobs are missing
func (d *DBBackedS3Store) MissingBlobsForKnownStream(sdHash string) ([]string, error) {
func (d *DBBackedStore) MissingBlobsForKnownStream(sdHash string) ([]string, error) {
return d.db.MissingBlobsForKnownStream(sdHash)
}
func (d *DBBackedS3Store) markBlocked(hash string) error {
func (d *DBBackedStore) markBlocked(hash string) error {
err := d.initBlocked()
if err != nil {
return err
@@ -136,7 +190,7 @@ func (d *DBBackedS3Store) markBlocked(hash string) error {
return nil
}
func (d *DBBackedS3Store) isBlocked(hash string) (bool, error) {
func (d *DBBackedStore) isBlocked(hash string) (bool, error) {
err := d.initBlocked()
if err != nil {
return false, err
@@ -148,7 +202,7 @@ func (d *DBBackedS3Store) isBlocked(hash string) (bool, error) {
return d.blocked[hash], nil
}
func (d *DBBackedS3Store) initBlocked() error {
func (d *DBBackedStore) initBlocked() error {
// first check without blocking since this is the most likely scenario
if d.blocked != nil {
return nil
@@ -167,3 +221,133 @@ func (d *DBBackedS3Store) initBlocked() error {
return err
}
// cleanOldestBlobs periodically cleans up the oldest blobs if maxSize is set
func (d *DBBackedStore) cleanOldestBlobs() {
// Run on startup without waiting for 10 minutes
err := d.doClean()
if err != nil {
log.Error(errors.FullTrace(err))
}
const cleanupInterval = 10 * time.Minute
for {
select {
case <-d.cleanerStop.Ch():
log.Infoln("stopping self cleanup")
return
case <-time.After(cleanupInterval):
err := d.doClean()
if err != nil {
log.Error(errors.FullTrace(err))
}
}
}
}
// doClean removes the least recently accessed blobs if the store exceeds maxItems
func (d *DBBackedStore) doClean() error {
blobsCount, err := d.db.Count()
if err != nil {
return err
}
if blobsCount >= d.maxSize {
itemsToDelete := blobsCount / 10
blobs, err := d.db.LeastRecentlyAccessedHashes(itemsToDelete)
if err != nil {
return err
}
blobsChan := make(chan string, len(blobs))
wg := stop.New()
go func() {
for _, hash := range blobs {
select {
case <-d.cleanerStop.Ch():
return
default:
}
blobsChan <- hash
}
close(blobsChan)
}()
for i := 0; i < 3; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for h := range blobsChan {
select {
case <-d.cleanerStop.Ch():
return
default:
}
err = d.Delete(h)
if err != nil {
log.Errorf("error pruning %s: %s", h, errors.FullTrace(err))
continue
}
}
}()
}
wg.Wait()
}
return nil
}
// Shutdown shuts down the store gracefully
func (d *DBBackedStore) Shutdown() {
d.cleanerStop.Stop()
d.blobs.Shutdown()
}
func DBBackedStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg DBBackedConfig
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
cfg.Store = config.Sub("store")
storeType := strings.Split(cfg.Store.AllKeys()[0], ".")[0]
storeConfig := cfg.Store.Sub(storeType)
factory, ok := Factories[storeType]
if !ok {
return nil, errors.Err("unknown store type %s", storeType)
}
underlyingStore, err := factory(storeConfig)
if err != nil {
return nil, errors.Err(err)
}
parsedDb := &db.SQL{
TrackingLevel: db.AccessTrackingLevel(cfg.AccessTracking),
SoftDelete: cfg.SoftDeletes,
LogQueries: cfg.LogQueries || log.GetLevel() == log.DebugLevel,
}
err = parsedDb.Connect(fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", cfg.User, cfg.Password, cfg.Host, cfg.Port, cfg.Database))
if err != nil {
return nil, err
}
params := DBBackedParams{
Name: cfg.Name,
Store: underlyingStore,
DB: parsedDb,
DeleteOnMiss: cfg.DeleteOnMiss,
}
if cfg.HasCap {
var parsedSize datasize.ByteSize
err = parsedSize.UnmarshalText([]byte(cfg.MaxSize))
if err != nil {
return nil, errors.Err(err)
}
maxSize := int(float64(parsedSize) / float64(stream.MaxBlobSize))
params.MaxSize = &maxSize
}
return NewDBBackedStore(params), nil
}
func init() {
RegisterStore(nameDBBacked, DBBackedStoreFactory)
}
+165
View File
@@ -0,0 +1,165 @@
package store
import (
"os"
"path"
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/reflector.go/store/speedwalk"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
)
// DiskStore stores blobs on a local disk
type DiskStore struct {
blobDir string
name string
prefixLength int
initialized bool
}
type DiskParams struct {
Name string `mapstructure:"name"`
MountPoint string `mapstructure:"mount_point"`
ShardingSize int `mapstructure:"sharding_size"`
}
// NewDiskStore returns an initialized file disk store pointer.
func NewDiskStore(params DiskParams) *DiskStore {
return &DiskStore{
blobDir: params.MountPoint,
prefixLength: params.ShardingSize,
name: params.Name,
}
}
const nameDisk = "disk"
func DiskStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg DiskParams
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
return NewDiskStore(cfg), nil
}
func init() {
RegisterStore(nameDisk, DiskStoreFactory)
}
// Name is the cache type name
func (d *DiskStore) Name() string { return nameDisk + "-" + d.name }
// Has returns T/F or Error if it the blob stored already. It will error with any IO disk error.
func (d *DiskStore) Has(hash string) (bool, error) {
err := d.initOnce()
if err != nil {
return false, err
}
_, err = os.Stat(d.path(hash))
if err != nil {
if os.IsNotExist(err) {
return false, nil
}
return false, errors.Err(err)
}
return true, nil
}
// Get returns the blob or an error if the blob doesn't exist.
func (d *DiskStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
err := d.initOnce()
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), err
}
blob, err := os.ReadFile(d.path(hash))
if err != nil {
if os.IsNotExist(err) {
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(ErrBlobNotFound)
}
return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(err)
}
return blob, shared.NewBlobTrace(time.Since(start), d.Name()), nil
}
// PutSD stores the sd blob on the disk
func (d *DiskStore) PutSD(hash string, blob stream.Blob) error {
return d.Put(hash, blob)
}
// Delete deletes the blob from the store
func (d *DiskStore) Delete(hash string) error {
err := d.initOnce()
if err != nil {
return err
}
has, err := d.Has(hash)
if err != nil {
return err
}
if !has {
return nil
}
err = os.Remove(d.path(hash))
return errors.Err(err)
}
// list returns the hashes of blobs that already exist in the blobDir
func (d *DiskStore) list() ([]string, error) {
err := d.initOnce()
if err != nil {
return nil, err
}
return speedwalk.AllFiles(d.blobDir, true)
}
func (d *DiskStore) dir(hash string) string {
if d.prefixLength <= 0 || len(hash) < d.prefixLength {
return d.blobDir
}
return path.Join(d.blobDir, hash[:d.prefixLength])
}
func (d *DiskStore) tmpDir() string {
return path.Join(d.blobDir, "tmp")
}
func (d *DiskStore) path(hash string) string {
return path.Join(d.dir(hash), hash)
}
func (d *DiskStore) tmpPath(hash string) string {
return path.Join(d.tmpDir(), hash)
}
func (d *DiskStore) ensureDirExists(dir string) error {
return errors.Err(os.MkdirAll(dir, 0755))
}
func (d *DiskStore) initOnce() error {
if d.initialized {
return nil
}
err := d.ensureDirExists(d.blobDir)
if err != nil {
return err
}
err = d.ensureDirExists(d.tmpDir())
if err != nil {
return err
}
d.initialized = true
return nil
}
// Shutdown shuts down the store gracefully
func (d *DiskStore) Shutdown() {
}
+52
View File
@@ -0,0 +1,52 @@
package store
import (
"os"
"path"
"path/filepath"
"testing"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestDiskStore_Get(t *testing.T) {
tmpDir, err := os.MkdirTemp("", "reflector_test_*")
require.NoError(t, err)
defer func() { _ = os.RemoveAll(tmpDir) }()
d := NewDiskStore(DiskParams{
Name: "test",
MountPoint: tmpDir,
ShardingSize: 2,
})
hash := "f428b8265d65dad7f8ffa52922bba836404cbd62f3ecfe10adba6b444f8f658938e54f5981ac4de39644d5b93d89a94b"
data := []byte("oyuntyausntoyaunpdoyruoyduanrstjwfjyuwf")
expectedPath := path.Join(tmpDir, hash[:2], hash)
err = os.MkdirAll(filepath.Dir(expectedPath), os.ModePerm)
require.NoError(t, err)
err = os.WriteFile(expectedPath, data, os.ModePerm)
require.NoError(t, err)
blob, _, err := d.Get(hash)
assert.NoError(t, err)
assert.EqualValues(t, data, blob)
}
func TestDiskStore_GetNonexistentBlob(t *testing.T) {
tmpDir, err := os.MkdirTemp("", "reflector_test_*")
require.NoError(t, err)
defer func() { _ = os.RemoveAll(tmpDir) }()
d := NewDiskStore(DiskParams{
Name: "test",
MountPoint: tmpDir,
ShardingSize: 2,
})
blob, _, err := d.Get("nonexistent")
assert.Nil(t, blob)
assert.True(t, errors.Is(err, ErrBlobNotFound))
}
+42
View File
@@ -0,0 +1,42 @@
//go:build darwin
// +build darwin
package store
import (
"bytes"
"io"
"os"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
)
var openFileFlags = os.O_WRONLY | os.O_CREATE
// Put stores the blob on disk
func (d *DiskStore) Put(hash string, blob stream.Blob) error {
err := d.initOnce()
if err != nil {
return err
}
err = d.ensureDirExists(d.dir(hash))
if err != nil {
return err
}
// Open file with O_DIRECT
f, err := os.OpenFile(d.tmpPath(hash), openFileFlags, 0644)
if err != nil {
return errors.Err(err)
}
defer f.Close()
_, err = io.Copy(f, bytes.NewReader(blob))
if err != nil {
return errors.Err(err)
}
err = os.Rename(d.tmpPath(hash), d.path(hash))
return errors.Err(err)
}
+49
View File
@@ -0,0 +1,49 @@
//go:build linux
// +build linux
package store
import (
"bytes"
"io"
"os"
"syscall"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/brk0v/directio"
)
var openFileFlags = os.O_WRONLY | os.O_CREATE | syscall.O_DIRECT
// Put stores the blob on disk
func (d *DiskStore) Put(hash string, blob stream.Blob) error {
err := d.initOnce()
if err != nil {
return err
}
err = d.ensureDirExists(d.dir(hash))
if err != nil {
return err
}
// Open file with O_DIRECT
f, err := os.OpenFile(d.tmpPath(hash), openFileFlags, 0644)
if err != nil {
return errors.Err(err)
}
defer func() { _ = f.Close() }()
dio, err := directio.New(f)
if err != nil {
return errors.Err(err)
}
defer func() { _ = dio.Flush() }()
_, err = io.Copy(dio, bytes.NewReader(blob))
if err != nil {
return errors.Err(err)
}
err = os.Rename(d.tmpPath(hash), d.path(hash))
return errors.Err(err)
}
+31
View File
@@ -0,0 +1,31 @@
//go:build !linux && !darwin
// +build !linux,!darwin
package store
import (
"os"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
)
// Put stores the blob on disk
func (d *DiskStore) Put(hash string, blob stream.Blob) error {
err := d.initOnce()
if err != nil {
return err
}
err = d.ensureDirExists(d.dir(hash))
if err != nil {
return err
}
err = os.WriteFile(d.tmpPath(hash), blob, 0644)
if err != nil {
return errors.Err(err)
}
err = os.Rename(d.tmpPath(hash), d.path(hash))
return errors.Err(err)
}
-115
View File
@@ -1,115 +0,0 @@
package store
import (
"io/ioutil"
"os"
"path"
"github.com/lbryio/lbry.go/extras/errors"
)
// FileBlobStore is a local disk store.
type FileBlobStore struct {
dir string
initialized bool
}
// NewFileBlobStore returns an initialized file disk store pointer.
func NewFileBlobStore(dir string) *FileBlobStore {
return &FileBlobStore{dir: dir}
}
func (f *FileBlobStore) path(hash string) string {
return path.Join(f.dir, hash)
}
func (f *FileBlobStore) initOnce() error {
if f.initialized {
return nil
}
if stat, err := os.Stat(f.dir); err != nil {
if os.IsNotExist(err) {
err2 := os.Mkdir(f.dir, 0755)
if err2 != nil {
return err2
}
} else {
return err
}
} else if !stat.IsDir() {
return errors.Err("blob dir exists but is not a dir")
}
f.initialized = true
return nil
}
// Has returns T/F or Error if it the blob stored already. It will error with any IO disk error.
func (f *FileBlobStore) Has(hash string) (bool, error) {
err := f.initOnce()
if err != nil {
return false, err
}
_, err = os.Stat(f.path(hash))
if err != nil {
if os.IsNotExist(err) {
return false, nil
}
return false, err
}
return true, nil
}
// Get returns the byte slice of the blob stored or will error if the blob doesn't exist.
func (f *FileBlobStore) Get(hash string) ([]byte, error) {
err := f.initOnce()
if err != nil {
return []byte{}, err
}
file, err := os.Open(f.path(hash))
if err != nil {
if os.IsNotExist(err) {
return []byte{}, errors.Err(ErrBlobNotFound)
}
return []byte{}, err
}
return ioutil.ReadAll(file)
}
// Put stores the blob on disk
func (f *FileBlobStore) Put(hash string, blob []byte) error {
err := f.initOnce()
if err != nil {
return err
}
return ioutil.WriteFile(f.path(hash), blob, 0644)
}
// PutSD stores the sd blob on the disk
func (f *FileBlobStore) PutSD(hash string, blob []byte) error {
return f.Put(hash, blob)
}
// Delete deletes the blob from the store
func (f *FileBlobStore) Delete(hash string) error {
err := f.initOnce()
if err != nil {
return err
}
has, err := f.Has(hash)
if err != nil {
return err
}
if !has {
return nil
}
return os.Remove(f.path(hash))
}
+211
View File
@@ -0,0 +1,211 @@
package store
import (
"strings"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/bluele/gcache"
"github.com/sirupsen/logrus"
"github.com/spf13/viper"
)
// GcacheStore adds a max cache size and Greedy-Dual-Size-Frequency cache eviction strategy to a BlobStore
type GcacheStore struct {
underlyingStore BlobStore
cache gcache.Cache
name string
}
type EvictionStrategy int
const (
//LFU Discards the least frequently used items first.
LFU EvictionStrategy = iota
//ARC Constantly balances between LRU and LFU, to improve the combined result.
ARC
//LRU Discards the least recently used items first.
LRU
//SIMPLE has no clear priority for evict cache. It depends on key-value map order.
SIMPLE
)
type GcacheParams struct {
Store BlobStore `mapstructure:"store"`
Name string `mapstructure:"name"`
MaxSize int `mapstructure:"max_size"`
Strategy EvictionStrategy `mapstructure:"strategy"`
}
type GcacheConfig struct {
Store *viper.Viper
Name string `mapstructure:"name"`
MaxSize int `mapstructure:"max_size"`
Strategy EvictionStrategy `mapstructure:"strategy"`
}
// NewGcacheStore initialize a new LRUStore
func NewGcacheStore(params GcacheParams) *GcacheStore {
cacheBuilder := gcache.New(params.MaxSize)
var cache gcache.Cache
evictFunc := func(key interface{}, value interface{}) {
logrus.Infof("evicting %s", key)
metrics.CacheLRUEvictCount.With(metrics.CacheLabels(params.Store.Name(), params.Name)).Inc()
_ = params.Store.Delete(key.(string)) // TODO: log this error. may happen if underlying entry is gone but cache entry still there
}
switch params.Strategy {
case LFU:
cache = cacheBuilder.LFU().EvictedFunc(evictFunc).Build()
case ARC:
cache = cacheBuilder.ARC().EvictedFunc(evictFunc).Build()
case LRU:
cache = cacheBuilder.LRU().EvictedFunc(evictFunc).Build()
case SIMPLE:
cache = cacheBuilder.Simple().EvictedFunc(evictFunc).Build()
}
l := &GcacheStore{
underlyingStore: params.Store,
cache: cache,
name: params.Name,
}
go func() {
if lstr, ok := params.Store.(lister); ok {
err := l.loadExisting(lstr, params.MaxSize)
if err != nil {
panic(err) // TODO: what should happen here? panic? return nil? just keep going?
}
}
}()
return l
}
const nameGcache = "gcache"
func GcacheStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg GcacheConfig
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
cfg.Store = config.Sub("store")
storeType := strings.Split(cfg.Store.AllKeys()[0], ".")[0]
storeConfig := cfg.Store.Sub(storeType)
factory, ok := Factories[storeType]
if !ok {
return nil, errors.Err("unknown store type %s", storeType)
}
underlyingStore, err := factory(storeConfig)
if err != nil {
return nil, errors.Err(err)
}
return NewGcacheStore(GcacheParams{
Name: cfg.Name,
Store: underlyingStore,
MaxSize: cfg.MaxSize,
Strategy: cfg.Strategy,
}), nil
}
func init() {
RegisterStore(nameGcache, GcacheStoreFactory)
}
// Name is the cache type name
func (l *GcacheStore) Name() string { return nameGcache + "-" + l.name }
// Has returns whether the blob is in the store, without updating the recent-ness.
func (l *GcacheStore) Has(hash string) (bool, error) {
return l.cache.Has(hash), nil
}
// Get returns the blob or an error if the blob doesn't exist.
func (l *GcacheStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
_, err := l.cache.Get(hash)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), l.Name()), errors.Err(ErrBlobNotFound)
}
blob, stack, err := l.underlyingStore.Get(hash)
if errors.Is(err, ErrBlobNotFound) {
// Blob disappeared from underlying store
l.cache.Remove(hash)
}
return blob, stack.Stack(time.Since(start), l.Name()), err
}
// Put stores the blob. Following LFUDA rules it's not guaranteed that a SET will store the value!!!
func (l *GcacheStore) Put(hash string, blob stream.Blob) error {
_ = l.cache.Set(hash, true)
has, _ := l.Has(hash)
if has {
err := l.underlyingStore.Put(hash, blob)
if err != nil {
return err
}
}
return nil
}
// PutSD stores the sd blob. Following LFUDA rules it's not guaranteed that a SET will store the value!!!
func (l *GcacheStore) PutSD(hash string, blob stream.Blob) error {
_ = l.cache.Set(hash, true)
has, _ := l.Has(hash)
if has {
err := l.underlyingStore.PutSD(hash, blob)
if err != nil {
return err
}
}
return nil
}
// Delete deletes the blob from the store
func (l *GcacheStore) Delete(hash string) error {
err := l.underlyingStore.Delete(hash)
if err != nil {
return err
}
// This must come after store.Delete()
// Remove triggers onEvict function, which also tries to delete blob from store
// We need to delete it manually first so any errors can be propagated up
l.cache.Remove(hash)
return nil
}
// loadExisting imports existing blobs from the underlying store into the LRU cache
func (l *GcacheStore) loadExisting(store lister, maxItems int) error {
logrus.Infof("loading at most %d items", maxItems)
existing, err := store.list()
if err != nil {
return err
}
logrus.Infof("read %d files from underlying store", len(existing))
added := 0
for i, h := range existing {
_ = l.cache.Set(h, true)
added++
if maxItems > 0 && added >= maxItems { // underlying cache is bigger than the cache
err := l.Delete(h)
logrus.Infof("deleted overflowing blob: %s (%d/%d)", h, i, len(existing))
if err != nil {
logrus.Warnf("error while deleting a blob that's overflowing the cache: %s", err.Error())
}
}
}
return nil
}
// Shutdown shuts down the store gracefully
func (l *GcacheStore) Shutdown() {
}
+114
View File
@@ -0,0 +1,114 @@
package store
import (
"fmt"
"os"
"reflect"
"testing"
"time"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
const cacheMaxSize = 3
func getTestGcacheStore() (*GcacheStore, *MemStore) {
m := NewMemStore(MemParams{Name: "test"})
return NewGcacheStore(GcacheParams{Name: "test", Store: m, MaxSize: cacheMaxSize, Strategy: LFU}), m
}
func TestGcacheStore_Eviction(t *testing.T) {
lfu, mem := getTestGcacheStore()
b := []byte("x")
for i := 0; i < 3; i++ {
err := lfu.Put(fmt.Sprintf("%d", i), b)
require.NoError(t, err)
for j := 0; j < 3-i; j++ {
_, _, err = lfu.Get(fmt.Sprintf("%d", i))
require.NoError(t, err)
}
}
for k, v := range map[string]bool{
"0": true,
"1": true,
"2": true,
} {
has, err := lfu.Has(k)
assert.NoError(t, err)
assert.Equal(t, v, has)
}
err := lfu.Put("3", b)
require.NoError(t, err)
for k, v := range map[string]bool{
"0": true,
"1": true,
"2": false,
"3": true,
} {
has, err := lfu.Has(k)
assert.NoError(t, err)
assert.Equal(t, v, has)
}
assert.Equal(t, cacheMaxSize, len(mem.Debug()))
err = lfu.Delete("0")
assert.NoError(t, err)
err = lfu.Delete("1")
assert.NoError(t, err)
err = lfu.Delete("3")
assert.NoError(t, err)
assert.Equal(t, 0, len(mem.Debug()))
}
func TestGcacheStore_UnderlyingBlobMissing(t *testing.T) {
lfu, mem := getTestGcacheStore()
hash := "hash"
b := []byte("this is a blob of stuff")
err := lfu.Put(hash, b)
require.NoError(t, err)
err = mem.Delete(hash)
require.NoError(t, err)
// hash still exists in lru
assert.True(t, lfu.cache.Has(hash))
blob, _, err := lfu.Get(hash)
assert.Nil(t, blob)
assert.True(t, errors.Is(err, ErrBlobNotFound), "expected (%s) %s, got (%s) %s",
reflect.TypeOf(ErrBlobNotFound).String(), ErrBlobNotFound.Error(),
reflect.TypeOf(err).String(), err.Error())
// lru.Get() removes hash if underlying store doesn't have it
assert.False(t, lfu.cache.Has(hash))
}
func TestGcacheStore_loadExisting(t *testing.T) {
tmpDir, err := os.MkdirTemp("", "reflector_test_*")
require.NoError(t, err)
defer func() { _ = os.RemoveAll(tmpDir) }()
d := NewDiskStore(DiskParams{
Name: "test",
MountPoint: tmpDir,
ShardingSize: 2,
})
hash := "hash"
b := []byte("this is a blob of stuff")
err = d.Put(hash, b)
require.NoError(t, err)
existing, err := d.list()
require.NoError(t, err)
require.Equal(t, 1, len(existing), "blob should exist in cache")
assert.Equal(t, hash, existing[0])
lfu := NewGcacheStore(GcacheParams{Name: "test", Store: d, MaxSize: 3, Strategy: LFU}) // lru should load existing blobs when it's created
time.Sleep(100 * time.Millisecond) // async load so let's wait...
has, err := lfu.Has(hash)
require.NoError(t, err)
assert.True(t, has, "hash should be loaded from disk store but it's not")
}
+188
View File
@@ -0,0 +1,188 @@
package store
import (
"io"
"net/http"
"path"
"strconv"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/meta"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
)
// HttpStore reads from an HTTP endpoint that simply expects the hash to be appended to the endpoint
type HttpStore struct {
httpClient *http.Client
endpoint string
name string
prefixLength int
}
type HttpParams struct {
Name string `mapstructure:"name"`
Endpoint string `mapstructure:"endpoint"`
ShardingSize int `mapstructure:"sharding_size"`
}
// NewHttpStore returns an initialized HttpStore store pointer.
func NewHttpStore(params HttpParams) *HttpStore {
return &HttpStore{
endpoint: params.Endpoint,
httpClient: getClient(),
prefixLength: params.ShardingSize,
name: params.Name,
}
}
const nameHttp = "http"
func HttpStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg HttpParams
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
return NewHttpStore(cfg), nil
}
// Name is the cache type name
func (c *HttpStore) Name() string { return nameHttp + "-" + c.name }
// Has checks if the hash is in the store.
func (c *HttpStore) Has(hash string) (bool, error) {
status, body, err := c.cfRequest(http.MethodHead, hash)
if err != nil {
return false, err
}
defer func() { _ = body.Close() }()
switch status {
case http.StatusNotFound, http.StatusForbidden:
return false, nil
case http.StatusOK:
return true, nil
default:
return false, errors.Err("unexpected status %d", status)
}
}
// Get downloads the blob using the http client
func (c *HttpStore) Get(hash string) (b stream.Blob, trace shared.BlobTrace, err error) {
log.Debugf("Getting %s from HTTP(s) source", hash[:8])
start := time.Now()
defer func(t time.Time) {
log.Debugf("Getting %s from HTTP(s) source took %s", hash[:8], time.Since(t).String())
trace = trace.Stack(time.Since(start), c.Name())
}(start)
url := c.endpoint + c.shardedPath(hash)
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, trace, errors.Err(err)
}
req.Header.Add("User-Agent", "reflector.go/"+meta.Version())
res, err := c.httpClient.Do(req)
if err != nil {
return nil, trace, errors.Err(err)
}
defer func(Body io.ReadCloser) {
err := Body.Close()
if err != nil {
log.Errorf("Error closing response body in HTTP-GET: %s", err.Error())
}
}(res.Body)
// Parse Via header if present
viaHeader := res.Header.Get("Via")
if viaHeader != "" {
parsedTrace, err := shared.Deserialize(viaHeader)
if err == nil {
trace = *parsedTrace
}
}
switch res.StatusCode {
case http.StatusNotFound:
return nil, trace, ErrBlobNotFound
case http.StatusOK:
contentLength := res.Header.Get("Content-Length")
if contentLength != "" {
size, err := strconv.Atoi(contentLength)
if err == nil && size > 0 && size <= stream.MaxBlobSize {
blob := make([]byte, size)
_, err = io.ReadFull(res.Body, blob)
if err == nil {
metrics.MtrInBytesHttp.Add(float64(size))
return blob, trace, nil
}
log.Warnf("Error reading body with known size: %s", err.Error())
}
}
buffer := getBuffer()
defer putBuffer(buffer)
if _, err := io.Copy(buffer, res.Body); err != nil {
return nil, trace, errors.Err(err)
}
blob := make([]byte, buffer.Len())
copy(blob, buffer.Bytes())
metrics.MtrInBytesHttp.Add(float64(len(blob)))
return blob, trace, nil
default:
body, _ := io.ReadAll(res.Body)
log.Warnf("Got status code %d (%s)", res.StatusCode, string(body))
return nil, trace, errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
}
}
func (c *HttpStore) cfRequest(method, hash string) (int, io.ReadCloser, error) {
url := c.endpoint + c.shardedPath(hash)
req, err := http.NewRequest(method, url, nil)
if err != nil {
return 0, nil, errors.Err(err)
}
req.Header.Add("User-Agent", "reflector.go/"+meta.Version())
res, err := c.httpClient.Do(req)
if err != nil {
return 0, nil, errors.Err(err)
}
return res.StatusCode, res.Body, nil
}
func (c *HttpStore) Put(_ string, _ stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
func (c *HttpStore) PutSD(_ string, _ stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
func (c *HttpStore) Delete(_ string) error {
return errors.Err(shared.ErrNotImplemented)
}
// Shutdown shuts down the store gracefully
func (c *HttpStore) Shutdown() {
}
func (c *HttpStore) shardedPath(hash string) string {
if c.prefixLength <= 0 || len(hash) < c.prefixLength {
return hash
}
return path.Join(hash[:c.prefixLength], hash)
}
func init() {
RegisterStore(nameHttp, HttpStoreFactory)
}
+136
View File
@@ -0,0 +1,136 @@
package store
import (
"strings"
"sync"
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
)
// Http3Store is a blob store that gets blobs from a peer over HTTP3.
// It satisfies the BlobStore interface but cannot put or delete blobs.
type Http3Store struct {
NotFoundCache *sync.Map
client *Http3Client
name string
address string
timeout time.Duration
clientMu sync.RWMutex
}
// Http3Params allows to set options for a new Http3Store.
type Http3Params struct {
Name string `mapstructure:"name"`
Address string `mapstructure:"address"`
Timeout time.Duration `mapstructure:"timeout"`
}
// NewHttp3Store makes a new HTTP3 store.
func NewHttp3Store(params Http3Params) *Http3Store {
return &Http3Store{
name: params.Name,
NotFoundCache: &sync.Map{},
address: params.Address,
timeout: params.Timeout,
}
}
const nameHttp3 = "http3"
func Http3StoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg Http3Params
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
return NewHttp3Store(cfg), nil
}
func init() {
RegisterStore(nameHttp3, Http3StoreFactory)
}
func (h *Http3Store) Name() string { return nameHttp3 + "-" + h.name }
func (h *Http3Store) getClient() (*Http3Client, error) {
h.clientMu.RLock()
if h.client != nil {
client := h.client
h.clientMu.RUnlock()
return client, nil
}
h.clientMu.RUnlock()
h.clientMu.Lock()
defer h.clientMu.Unlock()
// Check again in case another goroutine created the client
if h.client != nil {
return h.client, nil
}
client, err := NewHttp3Client(h.address)
if err != nil {
return nil, err
}
h.client = client
return client, nil
}
// Has asks the peer if they have a hash
func (h *Http3Store) Has(hash string) (bool, error) {
c, err := h.getClient()
if err != nil {
return false, err
}
return c.HasBlob(hash)
}
// Get downloads the blob from the peer
func (h *Http3Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
if lastChecked, ok := h.NotFoundCache.Load(hash); ok {
if lastChecked.(time.Time).After(time.Now().Add(-5 * time.Minute)) {
return nil, shared.NewBlobTrace(time.Since(start), h.Name()+"-notfoundcache"), ErrBlobNotFound
}
}
c, err := h.getClient()
if err != nil && strings.Contains(err.Error(), "blob not found") {
h.NotFoundCache.Store(hash, time.Now())
}
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), h.Name()), err
}
return c.GetBlob(hash)
}
// Put is not supported
func (h *Http3Store) Put(hash string, blob stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// PutSD is not supported
func (h *Http3Store) PutSD(hash string, blob stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// Delete is not supported
func (h *Http3Store) Delete(hash string) error {
return errors.Err(shared.ErrNotImplemented)
}
// Shutdown shuts down the store gracefully
func (h *Http3Store) Shutdown() {
h.clientMu.Lock()
defer h.clientMu.Unlock()
if h.client != nil {
_ = h.client.Close()
h.client = nil
}
}
+137
View File
@@ -0,0 +1,137 @@
package store
import (
"crypto/tls"
"crypto/x509"
"io"
"net/http"
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/quic-go/quic-go"
"github.com/quic-go/quic-go/http3"
)
// Http3Client is a client for HTTP3 blob store
type Http3Client struct {
conn *http.Client
roundTripper *http3.Transport
ServerAddr string
}
// NewHttp3Client creates a new HTTP3 client
func NewHttp3Client(address string) (*Http3Client, error) {
var qconf quic.Config
window500M := 500 * 1 << 20
qconf.MaxStreamReceiveWindow = uint64(window500M)
qconf.MaxConnectionReceiveWindow = uint64(window500M)
qconf.EnableDatagrams = true
qconf.HandshakeIdleTimeout = 4 * time.Second
qconf.MaxIdleTimeout = 20 * time.Second
pool, err := x509.SystemCertPool()
if err != nil {
return nil, err
}
roundTripper := &http3.Transport{
TLSClientConfig: &tls.Config{
RootCAs: pool,
InsecureSkipVerify: true,
},
QUICConfig: &qconf,
}
connection := &http.Client{
Transport: roundTripper,
}
return &Http3Client{
conn: connection,
roundTripper: roundTripper,
ServerAddr: address,
}, nil
}
// Close closes the client
func (c *Http3Client) Close() error {
return nil
}
// HasBlob checks if the peer has a blob
func (c *Http3Client) HasBlob(hash string) (bool, error) {
url := c.ServerAddr + "/blob?hash=" + hash
req, err := http.NewRequest("HEAD", url, nil)
if err != nil {
return false, errors.Err(err)
}
res, err := c.conn.Do(req)
if err != nil {
return false, errors.Err(err)
}
defer func() { _ = res.Body.Close() }()
if res.StatusCode == http.StatusNotFound {
return false, nil
}
if res.StatusCode == http.StatusNoContent {
return true, nil
}
var body []byte
if res.Body != nil {
body, _ = io.ReadAll(res.Body)
}
return false, errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
}
// GetBlob gets a blob from the peer
func (c *Http3Client) GetBlob(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
url := c.ServerAddr + "/blob?hash=" + hash
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err(err)
}
res, err := c.conn.Do(req)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err(err)
}
defer func() { _ = res.Body.Close() }()
viaHeader := res.Header.Get("Via")
var trace shared.BlobTrace
if viaHeader != "" {
parsedTrace, err := shared.Deserialize(viaHeader)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "http3"), err
}
trace = *parsedTrace
} else {
trace = shared.NewBlobTrace(0, "http3")
}
switch res.StatusCode {
case http.StatusNotFound:
return nil, trace.Stack(time.Since(start), "http3"), ErrBlobNotFound
case http.StatusOK:
buffer := getBuffer()
defer putBuffer(buffer)
if _, err := io.Copy(buffer, res.Body); err != nil {
return nil, trace.Stack(time.Since(start), "http3"), errors.Err(err)
}
blob := make([]byte, buffer.Len())
copy(blob, buffer.Bytes())
return blob, trace.Stack(time.Since(start), "http3"), nil
default:
body, _ := io.ReadAll(res.Body)
return nil, trace.Stack(time.Since(start), "http3"),
errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
}
}
+136
View File
@@ -0,0 +1,136 @@
package store
import (
"strings"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
)
// ITTTStore performs an operation on this storage, if this fails, it attempts to run it on that
type ITTTStore struct {
this, that BlobStore
name string
}
type ITTTParams struct {
This BlobStore `mapstructure:"this"`
That BlobStore `mapstructure:"that"`
Name string `mapstructure:"name"`
}
type ITTTConfig struct {
This *viper.Viper
That *viper.Viper
Name string `mapstructure:"name"`
}
// NewITTTStore returns a new instance of the IF THIS THAN THAT store
func NewITTTStore(params ITTTParams) *ITTTStore {
return &ITTTStore{
this: params.This,
that: params.That,
name: params.Name,
}
}
const nameIttt = "ittt"
func ITTTStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg ITTTConfig
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
cfg.This = config.Sub("this")
cfg.That = config.Sub("that")
thisStoreType := strings.Split(cfg.This.AllKeys()[0], ".")[0]
thisStoreConfig := cfg.This.Sub(thisStoreType)
factory, ok := Factories[thisStoreType]
if !ok {
return nil, errors.Err("unknown store type %s", thisStoreType)
}
thisStore, err := factory(thisStoreConfig)
if err != nil {
return nil, errors.Err(err)
}
thatStoreType := strings.Split(cfg.That.AllKeys()[0], ".")[0]
thatStoreConfig := cfg.That.Sub(thatStoreType)
factory, ok = Factories[thatStoreType]
if !ok {
return nil, errors.Err("unknown store type %s", thatStoreType)
}
thatStore, err := factory(thatStoreConfig)
if err != nil {
return nil, errors.Err(err)
}
return NewITTTStore(ITTTParams{
Name: cfg.Name,
This: thisStore,
That: thatStore,
}), nil
}
func init() {
RegisterStore(nameIttt, ITTTStoreFactory)
}
// Name is the cache type name
func (c *ITTTStore) Name() string { return nameIttt + "-" + c.name }
// Has checks in this for a hash, if it fails it checks in that. It returns true if either store has it.
func (c *ITTTStore) Has(hash string) (bool, error) {
has, err := c.this.Has(hash)
if err != nil || !has {
has, err = c.that.Has(hash)
}
return has, err
}
// Get tries to get the blob from this first, falling back to that.
func (c *ITTTStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
blob, trace, err := c.this.Get(hash)
if err == nil {
metrics.ItttHitCount.With(metrics.ItttLabels(c.this.Name())).Inc()
return blob, trace.Stack(time.Since(start), c.Name()), err
}
blob, trace, err = c.that.Get(hash)
if err != nil {
return nil, trace.Stack(time.Since(start), c.Name()), err
}
metrics.ItttHitCount.With(metrics.ItttLabels(c.that.Name())).Inc()
return blob, trace.Stack(time.Since(start), c.Name()), nil
}
// Put not implemented
func (c *ITTTStore) Put(hash string, blob stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// PutSD not implemented
func (c *ITTTStore) PutSD(hash string, blob stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// Delete not implemented
func (c *ITTTStore) Delete(hash string) error {
return errors.Err(shared.ErrNotImplemented)
}
// Shutdown shuts down the store gracefully
func (c *ITTTStore) Shutdown() {
c.this.Shutdown()
c.that.Shutdown()
}
+68 -21
View File
@@ -1,54 +1,101 @@
package store
import "github.com/lbryio/lbry.go/extras/errors"
import (
"sync"
"time"
// MemoryBlobStore is an in memory only blob store with no persistence.
type MemoryBlobStore struct {
blobs map[string][]byte
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
)
// MemStore is an in memory only blob store with no persistence.
type MemStore struct {
blobs map[string]stream.Blob
mu *sync.RWMutex
name string
}
// Has returns T/F if the blob is currently stored. It will never error.
func (m *MemoryBlobStore) Has(hash string) (bool, error) {
if m.blobs == nil {
m.blobs = make(map[string][]byte)
type MemParams struct {
Name string `mapstructure:"name"`
}
func NewMemStore(params MemParams) *MemStore {
return &MemStore{
blobs: make(map[string]stream.Blob),
mu: &sync.RWMutex{},
name: params.Name,
}
}
const nameMem = "mem"
func MemStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg MemParams
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
return NewMemStore(cfg), nil
}
func init() {
RegisterStore(nameMem, MemStoreFactory)
}
// Name is the cache type name
func (m *MemStore) Name() string { return nameMem + "-" + m.name }
// Has returns T/F if the blob is currently stored. It will never error.
func (m *MemStore) Has(hash string) (bool, error) {
m.mu.RLock()
defer m.mu.RUnlock()
_, ok := m.blobs[hash]
return ok, nil
}
// Get returns the blob byte slice if present and errors if the blob is not found.
func (m *MemoryBlobStore) Get(hash string) ([]byte, error) {
if m.blobs == nil {
m.blobs = make(map[string][]byte)
}
func (m *MemStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
m.mu.RLock()
defer m.mu.RUnlock()
blob, ok := m.blobs[hash]
if !ok {
return []byte{}, errors.Err(ErrBlobNotFound)
return nil, shared.NewBlobTrace(time.Since(start), m.Name()), errors.Err(ErrBlobNotFound)
}
return blob, nil
return blob, shared.NewBlobTrace(time.Since(start), m.Name()), nil
}
// Put stores the blob in memory
func (m *MemoryBlobStore) Put(hash string, blob []byte) error {
if m.blobs == nil {
m.blobs = make(map[string][]byte)
}
func (m *MemStore) Put(hash string, blob stream.Blob) error {
m.mu.Lock()
defer m.mu.Unlock()
m.blobs[hash] = blob
return nil
}
// PutSD stores the sd blob in memory
func (m *MemoryBlobStore) PutSD(hash string, blob []byte) error {
func (m *MemStore) PutSD(hash string, blob stream.Blob) error {
return m.Put(hash, blob)
}
// Delete deletes the blob from the store
func (m *MemoryBlobStore) Delete(hash string) error {
func (m *MemStore) Delete(hash string) error {
m.mu.Lock()
defer m.mu.Unlock()
delete(m.blobs, hash)
return nil
}
// Debug returns the blobs in memory. It's useful for testing and debugging.
func (m *MemoryBlobStore) Debug() map[string][]byte {
func (m *MemStore) Debug() map[string]stream.Blob {
m.mu.RLock()
defer m.mu.RUnlock()
return m.blobs
}
// Shutdown shuts down the store gracefully
func (m *MemStore) Shutdown() {}
+7 -7
View File
@@ -4,11 +4,11 @@ import (
"bytes"
"testing"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/errors"
)
func TestMemoryBlobStore_Put(t *testing.T) {
s := MemoryBlobStore{}
func TestMemStore_Put(t *testing.T) {
s := NewMemStore(MemParams{Name: "test"})
blob := []byte("abcdefg")
err := s.Put("abc", blob)
if err != nil {
@@ -16,8 +16,8 @@ func TestMemoryBlobStore_Put(t *testing.T) {
}
}
func TestMemoryBlobStore_Get(t *testing.T) {
s := MemoryBlobStore{}
func TestMemStore_Get(t *testing.T) {
s := NewMemStore(MemParams{Name: "test"})
hash := "abc"
blob := []byte("abcdefg")
err := s.Put(hash, blob)
@@ -25,7 +25,7 @@ func TestMemoryBlobStore_Get(t *testing.T) {
t.Error("error getting memory blob - ", err)
}
gotBlob, err := s.Get(hash)
gotBlob, _, err := s.Get(hash)
if err != nil {
t.Errorf("Expected no error, got %v", err)
}
@@ -33,7 +33,7 @@ func TestMemoryBlobStore_Get(t *testing.T) {
t.Error("Got blob that is different from expected blob")
}
missingBlob, err := s.Get("nonexistent hash")
missingBlob, _, err := s.Get("nonexistent hash")
if err == nil {
t.Errorf("Expected ErrBlobNotFound, got nil")
}
+206
View File
@@ -0,0 +1,206 @@
package store
import (
"strings"
"sync"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
)
// MultiWriterStore writes to multiple destination stores
type MultiWriterStore struct {
name string
destinations []BlobStore
}
type MultiWriterParams struct {
Name string
Destinations []BlobStore
}
type MultiWriterConfig struct {
Name string `mapstructure:"name"`
One viper.Viper
Two viper.Viper
Three viper.Viper
}
// NewMultiWriterStore returns a new instance of the MultiWriter store
func NewMultiWriterStore(params MultiWriterParams) *MultiWriterStore {
return &MultiWriterStore{
name: params.Name,
destinations: params.Destinations,
}
}
const nameMultiWriter = "multiwriter"
func MultiWriterStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg MultiWriterConfig
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
var destinations []BlobStore
one := config.Sub("one")
two := config.Sub("two")
//three := config.Sub("three")
storeTypeOne := strings.Split(one.AllKeys()[0], ".")[0]
storeTypeTwo := strings.Split(two.AllKeys()[0], ".")[0]
//storeTypeThree := strings.Split(three.AllKeys()[0], ".")[0]
storeCfgOne := one.Sub(storeTypeOne)
storeCfgTwo := two.Sub(storeTypeTwo)
//storeCfgThree := config.Sub(storeTypeThree)
factoryOne, ok := Factories[storeTypeOne]
if !ok {
return nil, errors.Err("unknown store type %s", storeTypeOne)
}
factoryTwo, ok := Factories[storeTypeTwo]
if !ok {
return nil, errors.Err("unknown store type %s", storeTypeTwo)
}
//factoryThree, ok := Factories[storeTypeThree]
//if !ok {
// return nil, errors.Err("unknown store type %s", storeTypeThree)
//}
store1, err := factoryOne(storeCfgOne)
if err != nil {
return nil, errors.Err(err)
}
store2, err := factoryTwo(storeCfgTwo)
if err != nil {
return nil, errors.Err(err)
}
//store3, err := factoryThree(storeCfgThree)
//if err != nil {
// return nil, errors.Err(err)
//}
destinations = append(destinations, store1, store2)
return NewMultiWriterStore(MultiWriterParams{
Name: cfg.Name,
Destinations: destinations,
}), nil
}
func init() {
RegisterStore(nameMultiWriter, MultiWriterStoreFactory)
}
// Name returns the store name
func (m *MultiWriterStore) Name() string { return nameMultiWriter + "-" + m.name }
// Has is not supported by MultiWriter
func (m *MultiWriterStore) Has(hash string) (bool, error) {
return false, errors.Err(shared.ErrNotImplemented)
}
// Get is not supported by MultiWriter
func (m *MultiWriterStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
return nil, shared.BlobTrace{}, errors.Err(shared.ErrNotImplemented)
}
// Put writes the blob to all destination stores
func (m *MultiWriterStore) Put(hash string, blob stream.Blob) error {
var wg sync.WaitGroup
errChan := make(chan error, len(m.destinations))
for _, dest := range m.destinations {
wg.Add(1)
go func(d BlobStore) {
defer wg.Done()
if err := d.Put(hash, blob); err != nil {
errChan <- errors.Err("failed to write to %s: %v", d.Name(), err)
}
}(dest)
}
wg.Wait()
close(errChan)
var errs []string
for err := range errChan {
errs = append(errs, err.Error())
}
if len(errs) > 0 {
return errors.Err("failed to write to some destinations: %s", strings.Join(errs, "; "))
}
return nil
}
// PutSD writes the SD blob to all destination stores
func (m *MultiWriterStore) PutSD(hash string, blob stream.Blob) error {
var wg sync.WaitGroup
errChan := make(chan error, len(m.destinations))
for _, dest := range m.destinations {
wg.Add(1)
go func(d BlobStore) {
defer wg.Done()
if err := d.PutSD(hash, blob); err != nil {
errChan <- errors.Err("failed to write SD to %s: %v", d.Name(), err)
}
}(dest)
}
wg.Wait()
close(errChan)
var errs []string
for err := range errChan {
errs = append(errs, err.Error())
}
if len(errs) > 0 {
return errors.Err("failed to write SD to some destinations: %s", strings.Join(errs, "; "))
}
return nil
}
// Delete deletes the blob from all destination stores
func (m *MultiWriterStore) Delete(hash string) error {
var wg sync.WaitGroup
errChan := make(chan error, len(m.destinations))
for _, dest := range m.destinations {
wg.Add(1)
go func(d BlobStore) {
defer wg.Done()
if err := d.Delete(hash); err != nil {
errChan <- errors.Err("failed to delete from %s: %v", d.Name(), err)
}
}(dest)
}
wg.Wait()
close(errChan)
var errs []string
for err := range errChan {
errs = append(errs, err.Error())
}
if len(errs) > 0 {
return errors.Err("failed to delete from some destinations: %s", strings.Join(errs, "; "))
}
return nil
}
// Shutdown shuts down all destination stores gracefully
func (m *MultiWriterStore) Shutdown() {
for _, dest := range m.destinations {
dest.Shutdown()
}
}
+48
View File
@@ -0,0 +1,48 @@
package store
import (
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
)
// NoopStore is a store that does nothing
type NoopStore struct {
name string
}
func NewNoopStore(name string) *NoopStore {
return &NoopStore{name: name}
}
const nameNoop = "noop"
func NoopStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg struct {
Name string `mapstructure:"name"`
}
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
return NewNoopStore(cfg.Name), nil
}
func init() {
RegisterStore(nameNoop, NoopStoreFactory)
}
func (n *NoopStore) Name() string { return nameNoop + "-" + n.name }
func (n *NoopStore) Has(_ string) (bool, error) { return false, nil }
func (n *NoopStore) Get(_ string) (stream.Blob, shared.BlobTrace, error) {
return nil, shared.NewBlobTrace(time.Since(time.Now()), n.Name()), nil
}
func (n *NoopStore) Put(_ string, _ stream.Blob) error { return nil }
func (n *NoopStore) PutSD(_ string, _ stream.Blob) error { return nil }
func (n *NoopStore) Delete(_ string) error { return nil }
func (n *NoopStore) Shutdown() {}
+100
View File
@@ -0,0 +1,100 @@
package store
import (
"strings"
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
)
// PeerStore is a blob store that gets blobs from a peer.
// It satisfies the BlobStore interface but cannot put or delete blobs.
type PeerStore struct {
name string
opts PeerParams
}
// PeerParams allows to set options for a new PeerStore.
type PeerParams struct {
Name string `mapstructure:"name"`
Address string `mapstructure:"address"`
Timeout time.Duration `mapstructure:"timeout"`
}
// NewPeerStore makes a new peer store.
func NewPeerStore(params PeerParams) *PeerStore {
return &PeerStore{opts: params, name: params.Name}
}
const namePeer = "peer"
func PeerStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg PeerParams
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
return NewPeerStore(cfg), nil
}
func init() {
RegisterStore(namePeer, PeerStoreFactory)
}
func (p *PeerStore) Name() string { return namePeer + "-" + p.name }
func (p *PeerStore) getClient() (*PeerClient, error) {
c := &PeerClient{Timeout: p.opts.Timeout}
err := c.Connect(p.opts.Address)
return c, errors.Prefix("connection error", err)
}
// Has asks the peer if they have a hash
func (p *PeerStore) Has(hash string) (bool, error) {
c, err := p.getClient()
if err != nil {
return false, err
}
defer func() { _ = c.Close() }()
return c.HasBlob(hash)
}
// Get downloads the blob from the peer
func (p *PeerStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
c, err := p.getClient()
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), p.Name()), err
}
defer func() { _ = c.Close() }()
blob, trace, err := c.GetBlob(hash)
if err != nil && strings.Contains(err.Error(), "blob not found") {
return nil, trace, ErrBlobNotFound
}
return blob, trace, err
}
// Put is not supported
func (p *PeerStore) Put(hash string, blob stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// PutSD is not supported
func (p *PeerStore) PutSD(hash string, blob stream.Blob) error {
return errors.Err(shared.ErrNotImplemented)
}
// Delete is not supported
func (p *PeerStore) Delete(hash string) error {
return errors.Err(shared.ErrNotImplemented)
}
// Shutdown is not supported
func (p *PeerStore) Shutdown() {
}
+95
View File
@@ -0,0 +1,95 @@
package store
import (
"bufio"
"bytes"
"encoding/binary"
"io"
"net"
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/stream"
)
// PeerClient is a client for peer blob store
type PeerClient struct {
conn net.Conn
Timeout time.Duration
}
// Connect connects to a peer
func (c *PeerClient) Connect(address string) error {
var err error
c.conn, err = net.DialTimeout("tcp", address, c.Timeout)
return err
}
// Close closes the connection
func (c *PeerClient) Close() error {
if c.conn != nil {
return c.conn.Close()
}
return nil
}
// HasBlob checks if the peer has a blob
func (c *PeerClient) HasBlob(hash string) (bool, error) {
err := c.writeRequest("has", hash)
if err != nil {
return false, err
}
response, err := c.readResponse()
if err != nil {
return false, err
}
return response == "yes", nil
}
// GetBlob gets a blob from the peer
func (c *PeerClient) GetBlob(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
err := c.writeRequest("get", hash)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "peer"), err
}
response, err := c.readResponse()
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "peer"), err
}
if response == "no" {
return nil, shared.NewBlobTrace(time.Since(start), "peer"), ErrBlobNotFound
}
size, err := binary.ReadVarint(bufio.NewReader(bytes.NewReader([]byte(response))))
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "peer"), err
}
blob := make([]byte, size)
_, err = io.ReadFull(c.conn, blob)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), "peer"), err
}
return blob, shared.NewBlobTrace(time.Since(start), "peer"), nil
}
func (c *PeerClient) writeRequest(cmd, hash string) error {
_, err := c.conn.Write([]byte(cmd + " " + hash + "\n"))
return err
}
func (c *PeerClient) readResponse() (string, error) {
reader := bufio.NewReader(c.conn)
response, err := reader.ReadString('\n')
if err != nil {
return "", err
}
return response[:len(response)-1], nil
}
+147
View File
@@ -0,0 +1,147 @@
package store
import (
"strings"
"time"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
)
// ProxiedS3Store writes to an S3 store and reads from any BlobStore (usually an ITTTStore of HttpStore endpoints).
type ProxiedS3Store struct {
readerStore BlobStore
writerStore BlobStore
name string
}
func (c *ProxiedS3Store) MissingBlobsForKnownStream(s string) ([]string, error) {
if bc, ok := c.writerStore.(NeededBlobChecker); ok {
return bc.MissingBlobsForKnownStream(s)
}
return nil, errors.Err("writer does not implement neededBlobChecker")
}
func (c *ProxiedS3Store) Block(hash string) error {
if bl, ok := c.writerStore.(Blocklister); ok {
return bl.Block(hash)
}
return errors.Err("writer does not implement Blocklister")
}
func (c *ProxiedS3Store) Wants(hash string) (bool, error) {
if bl, ok := c.writerStore.(Blocklister); ok {
return bl.Wants(hash)
}
return true, errors.Err("writer does not implement Blocklister")
}
type ProxiedS3Params struct {
Reader BlobStore `mapstructure:"reader"`
Writer BlobStore `mapstructure:"writer"`
Name string `mapstructure:"name"`
}
type ProxiedS3Config struct {
Reader *viper.Viper
Writer *viper.Viper
Name string `mapstructure:"name"`
}
// NewProxiedS3Store returns an initialized ProxiedS3Store store pointer.
// NOTE: It panics if either argument is nil.
func NewProxiedS3Store(params ProxiedS3Params) *ProxiedS3Store {
if params.Reader == nil || params.Writer == nil {
panic("both stores must be set")
}
return &ProxiedS3Store{
readerStore: params.Reader,
writerStore: params.Writer,
name: params.Name,
}
}
const nameProxiedS3 = "proxied-s3"
// Name is the cache type name
func (c *ProxiedS3Store) Name() string { return nameProxiedS3 + "-" + c.name }
// Has checks if the hash is in the store.
func (c *ProxiedS3Store) Has(hash string) (bool, error) {
return c.writerStore.Has(hash)
}
// Get gets the blob from Cloudfront.
func (c *ProxiedS3Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
blob, trace, err := c.readerStore.Get(hash)
return blob, trace.Stack(time.Since(start), c.Name()), err
}
// Put stores the blob on S3
func (c *ProxiedS3Store) Put(hash string, blob stream.Blob) error {
return c.writerStore.Put(hash, blob)
}
// PutSD stores the sd blob on S3
func (c *ProxiedS3Store) PutSD(hash string, blob stream.Blob) error {
return c.writerStore.PutSD(hash, blob)
}
// Delete deletes the blob from S3
func (c *ProxiedS3Store) Delete(hash string) error {
return c.writerStore.Delete(hash)
}
// Shutdown shuts down the store gracefully
func (c *ProxiedS3Store) Shutdown() {
c.writerStore.Shutdown()
c.readerStore.Shutdown()
}
func ProxiedS3StoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg ProxiedS3Config
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
cfg.Reader = config.Sub("reader")
cfg.Writer = config.Sub("writer")
readerStoreType := strings.Split(cfg.Reader.AllKeys()[0], ".")[0]
readerStoreConfig := cfg.Reader.Sub(readerStoreType)
factory, ok := Factories[readerStoreType]
if !ok {
return nil, errors.Err("unknown store type %s", readerStoreType)
}
readerStore, err := factory(readerStoreConfig)
if err != nil {
return nil, errors.Err(err)
}
writerStoreType := strings.Split(cfg.Writer.AllKeys()[0], ".")[0]
writerStoreConfig := cfg.Writer.Sub(writerStoreType)
factory, ok = Factories[writerStoreType]
if !ok {
return nil, errors.Err("unknown store type %s", writerStoreType)
}
writerStore, err := factory(writerStoreConfig)
if err != nil {
return nil, errors.Err(err)
}
return NewProxiedS3Store(ProxiedS3Params{
Name: cfg.Name,
Reader: readerStore,
Writer: writerStore,
}), nil
}
func init() {
RegisterStore(nameProxiedS3, ProxiedS3StoreFactory)
}
+137 -71
View File
@@ -2,113 +2,141 @@ package store
import (
"bytes"
"net/http"
"context"
stderrors "errors"
"path"
"time"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/shared"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/aws/aws-sdk-go-v2/aws"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
)
// S3BlobStore is an S3 store
type S3BlobStore struct {
awsID string
awsSecret string
region string
bucket string
session *session.Session
// S3Store is an S3 store
type S3Store struct {
client *s3.Client
awsID string
awsSecret string
region string
bucket string
endpoint string
name string
prefixLength int
}
// NewS3BlobStore returns an initialized S3 store pointer.
func NewS3BlobStore(awsID, awsSecret, region, bucket string) *S3BlobStore {
return &S3BlobStore{
awsID: awsID,
awsSecret: awsSecret,
region: region,
bucket: bucket,
type S3Params struct {
Name string `mapstructure:"name"`
AwsID string `mapstructure:"aws_id"`
AwsSecret string `mapstructure:"aws_secret"`
Region string `mapstructure:"region"`
Bucket string `mapstructure:"bucket"`
Endpoint string `mapstructure:"endpoint"`
ShardingSize int `mapstructure:"sharding_size"`
}
// NewS3Store returns an initialized S3 store pointer.
func NewS3Store(params S3Params) *S3Store {
return &S3Store{
awsID: params.AwsID,
awsSecret: params.AwsSecret,
region: params.Region,
bucket: params.Bucket,
endpoint: params.Endpoint,
name: params.Name,
prefixLength: params.ShardingSize,
}
}
func (s *S3BlobStore) initOnce() error {
if s.session != nil {
return nil
}
const nameS3 = "s3"
sess, err := session.NewSession(&aws.Config{
Credentials: credentials.NewStaticCredentials(s.awsID, s.awsSecret, ""),
Region: aws.String(s.region),
})
func S3StoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg S3Params
err := config.Unmarshal(&cfg)
if err != nil {
return err
return nil, errors.Err(err)
}
s.session = sess
return nil
return NewS3Store(cfg), nil
}
// Has returns T/F or Error ( from S3 ) if the store contains the blob.
func (s *S3BlobStore) Has(hash string) (bool, error) {
func init() {
RegisterStore(nameS3, S3StoreFactory)
}
// Name is the cache type name
func (s *S3Store) Name() string { return nameS3 + "-" + s.name }
// Has returns T/F or Error (from S3) if the store contains the blob.
func (s *S3Store) Has(hash string) (bool, error) {
err := s.initOnce()
if err != nil {
return false, err
}
_, err = s3.New(s.session).HeadObject(&s3.HeadObjectInput{
ctx := context.Background()
_, err = s.client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(hash),
Key: aws.String(s.shardedPath(hash)),
})
if err != nil {
if reqFail, ok := err.(s3.RequestFailure); ok && reqFail.StatusCode() == http.StatusNotFound {
var notFound *types.NotFound
var noSuchKey *types.NoSuchKey
if stderrors.As(err, &notFound) || stderrors.As(err, &noSuchKey) {
return false, nil
}
return false, err
return false, errors.Err(err)
}
return true, nil
}
// Get returns the blob slice if present or errors on S3.
func (s *S3BlobStore) Get(hash string) ([]byte, error) {
//Todo-Need to handle error for blob doesn't exist for consistency.
func (s *S3Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
err := s.initOnce()
if err != nil {
return []byte{}, err
return nil, shared.NewBlobTrace(time.Since(start), s.Name()), err
}
log.Debugf("Getting %s from S3", hash[:8])
defer func(t time.Time) {
log.Debugf("Getting %s from S3 took %s", hash[:8], time.Since(t).String())
}(time.Now())
log.Debugf("Getting %s from %s took %s", hash[:8], s.Name(), time.Since(t).String())
}(start)
buf := &aws.WriteAtBuffer{}
_, err = s3manager.NewDownloader(s.session).Download(buf, &s3.GetObjectInput{
ctx := context.Background()
buf := manager.NewWriteAtBuffer([]byte{})
_, err = manager.NewDownloader(s.client).Download(ctx, buf, &s3.GetObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(hash),
Key: aws.String(s.shardedPath(hash)),
})
if err != nil {
if aerr, ok := err.(awserr.Error); ok {
switch aerr.Code() {
case s3.ErrCodeNoSuchBucket:
return []byte{}, errors.Err("bucket %s does not exist", s.bucket)
case s3.ErrCodeNoSuchKey:
return []byte{}, errors.Err(ErrBlobNotFound)
}
var noSuchBucket *types.NoSuchBucket
var noSuchKey *types.NoSuchKey
var notFound *types.NotFound
switch {
case stderrors.As(err, &noSuchBucket):
return nil, shared.NewBlobTrace(time.Since(start), s.Name()), errors.Err("bucket %s does not exist", s.bucket)
case stderrors.As(err, &noSuchKey), stderrors.As(err, &notFound):
return nil, shared.NewBlobTrace(time.Since(start), s.Name()), errors.Err(ErrBlobNotFound)
}
return buf.Bytes(), err
return nil, shared.NewBlobTrace(time.Since(start), s.Name()), errors.Err(err)
}
return buf.Bytes(), nil
return buf.Bytes(), shared.NewBlobTrace(time.Since(start), s.Name()), nil
}
// Put stores the blob on S3 or errors if S3 connection errors.
func (s *S3BlobStore) Put(hash string, blob []byte) error {
func (s *S3Store) Put(hash string, blob stream.Blob) error {
err := s.initOnce()
if err != nil {
return err
@@ -119,23 +147,27 @@ func (s *S3BlobStore) Put(hash string, blob []byte) error {
log.Debugf("Uploading %s took %s", hash[:8], time.Since(t).String())
}(time.Now())
_, err = s3manager.NewUploader(s.session).Upload(&s3manager.UploadInput{
Bucket: aws.String(s.bucket),
Key: aws.String(hash),
Body: bytes.NewBuffer(blob),
StorageClass: aws.String(s3.StorageClassIntelligentTiering),
ctx := context.Background()
_, err = manager.NewUploader(s.client).Upload(ctx, &s3.PutObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(s.shardedPath(hash)),
Body: bytes.NewBuffer(blob),
ACL: types.ObjectCannedACLPublicRead,
})
return err
if err != nil {
return errors.Err(err)
}
metrics.MtrOutBytesReflector.Add(float64(blob.Size()))
return nil
}
// PutSD stores the sd blob on S3 or errors if S3 connection errors.
func (s *S3BlobStore) PutSD(hash string, blob []byte) error {
func (s *S3Store) PutSD(hash string, blob stream.Blob) error {
//Todo - handle missing stream for consistency
return s.Put(hash, blob)
}
func (s *S3BlobStore) Delete(hash string) error {
func (s *S3Store) Delete(hash string) error {
err := s.initOnce()
if err != nil {
return err
@@ -143,10 +175,44 @@ func (s *S3BlobStore) Delete(hash string) error {
log.Debugf("Deleting %s from S3", hash[:8])
_, err = s3.New(s.session).DeleteObject(&s3.DeleteObjectInput{
ctx := context.Background()
_, err = s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(hash),
Key: aws.String(s.shardedPath(hash)),
})
return err
return errors.Err(err)
}
func (s *S3Store) initOnce() error {
if s.client != nil {
return nil
}
ctx := context.Background()
cfg, err := awsconfig.LoadDefaultConfig(ctx,
awsconfig.WithRegion(s.region),
awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(s.awsID, s.awsSecret, "")),
)
if err != nil {
return errors.Err(err)
}
s.client = s3.NewFromConfig(cfg, func(o *s3.Options) {
o.BaseEndpoint = aws.String(s.endpoint)
o.UsePathStyle = true
o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired
})
return nil
}
func (s *S3Store) shardedPath(hash string) string {
if s.prefixLength <= 0 || len(hash) < s.prefixLength {
return hash
}
return path.Join(hash[:s.prefixLength], hash)
}
// Shutdown shuts down the store gracefully
func (s *S3Store) Shutdown() {
}
+160
View File
@@ -0,0 +1,160 @@
package store
import (
"strings"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
"golang.org/x/sync/singleflight"
)
func WithSingleFlight(component string, origin BlobStore) BlobStore {
return &singleflightStore{
BlobStore: origin,
component: component,
sf: new(singleflight.Group),
}
}
type singleflightStore struct {
BlobStore
sf *singleflight.Group
component string
}
type SingleFlightConfig struct {
Store *viper.Viper
Component string `mapstructure:"component"`
}
func SingleFlightStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg SingleFlightConfig
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
cfg.Store = config.Sub("store")
storeType := strings.Split(cfg.Store.AllKeys()[0], ".")[0]
storeConfig := cfg.Store.Sub(storeType)
factory, ok := Factories[storeType]
if !ok {
return nil, errors.Err("unknown store type %s", storeType)
}
underlyingStore, err := factory(storeConfig)
if err != nil {
return nil, errors.Err(err)
}
return WithSingleFlight(cfg.Component, underlyingStore), nil
}
func init() {
RegisterStore("singleflight", SingleFlightStoreFactory)
}
func (s *singleflightStore) Name() string {
return "sf_" + s.BlobStore.Name()
}
type getterResponse struct {
blob stream.Blob
stack shared.BlobTrace
}
// Get ensures that only one request per hash is sent to the origin at a time,
// thereby protecting against https://en.wikipedia.org/wiki/Thundering_herd_problem
func (s *singleflightStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
metrics.CacheWaitingRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Inc()
defer metrics.CacheWaitingRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Dec()
gr, err, _ := s.sf.Do(hash, s.getter(hash))
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), s.Name()), err
}
if gr == nil {
return nil, shared.NewBlobTrace(time.Since(start), s.Name()), errors.Err("getter response is nil")
}
rsp := gr.(getterResponse)
return rsp.blob, rsp.stack, nil
}
// getter returns a function that gets a blob from the origin
// only one getter per hash will be executing at a time
func (s *singleflightStore) getter(hash string) func() (interface{}, error) {
return func() (interface{}, error) {
metrics.CacheOriginRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Inc()
defer metrics.CacheOriginRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Dec()
start := time.Now()
blob, stack, err := s.BlobStore.Get(hash)
if err != nil {
return getterResponse{
blob: nil,
stack: stack.Stack(time.Since(start), s.Name()),
}, err
}
rate := float64(len(blob)) / 1024 / 1024 / time.Since(start).Seconds()
metrics.CacheRetrievalSpeed.With(map[string]string{
metrics.LabelCacheType: s.Name(),
metrics.LabelComponent: s.component,
metrics.LabelSource: "origin",
}).Set(rate)
return getterResponse{
blob: blob,
stack: stack.Stack(time.Since(start), s.Name()),
}, nil
}
}
// Put ensures that only one request per hash is sent to the origin at a time,
// thereby protecting against https://en.wikipedia.org/wiki/Thundering_herd_problem
func (s *singleflightStore) Put(hash string, blob stream.Blob) error {
metrics.CacheWaitingRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Inc()
defer metrics.CacheWaitingRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Dec()
_, err, _ := s.sf.Do(hash, s.putter(hash, blob))
if err != nil {
return err
}
return nil
}
// putter returns a function that puts a blob from the origin
// only one putter per hash will be executing at a time
func (s *singleflightStore) putter(hash string, blob stream.Blob) func() (interface{}, error) {
return func() (interface{}, error) {
metrics.CacheOriginRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Inc()
defer metrics.CacheOriginRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Dec()
start := time.Now()
err := s.BlobStore.Put(hash, blob)
if err != nil {
return nil, err
}
rate := float64(len(blob)) / 1024 / 1024 / time.Since(start).Seconds()
metrics.CacheRetrievalSpeed.With(map[string]string{
metrics.LabelCacheType: s.Name(),
metrics.LabelComponent: s.component,
metrics.LabelSource: "origin",
}).Set(rate)
return nil, nil
}
}
// Shutdown shuts down the store gracefully
func (s *singleflightStore) Shutdown() {
s.BlobStore.Shutdown()
}
+104
View File
@@ -0,0 +1,104 @@
package speedwalk
import (
"io/fs"
"os"
"path/filepath"
"runtime"
"sync"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/karrick/godirwalk"
"github.com/sirupsen/logrus"
)
// AllFiles recursively lists every file in every subdirectory of a given directory
// If basename is true, return the basename of each file. Otherwise return the full path starting at startDir.
func AllFiles(startDir string, basename bool) ([]string, error) {
entries, err := os.ReadDir(startDir)
if err != nil {
return nil, err
}
items := make([]fs.FileInfo, 0, len(entries))
for _, entry := range entries {
var info fs.FileInfo
info, err = entry.Info()
if err != nil {
return nil, err
}
items = append(items, info)
}
if err != nil {
return nil, err
}
pathChan := make(chan string)
paths := make([]string, 0, 1000)
pathWG := &sync.WaitGroup{}
pathWG.Add(1)
metrics.RoutinesQueue.WithLabelValues("speedwalk", "worker").Inc()
go func() {
defer pathWG.Done()
for {
path, ok := <-pathChan
if !ok {
return
}
paths = append(paths, path)
}
}()
maxThreads := runtime.NumCPU() - 1
goroutineLimiter := make(chan struct{}, maxThreads)
for i := 0; i < maxThreads; i++ {
goroutineLimiter <- struct{}{}
}
walkerWG := &sync.WaitGroup{}
for _, item := range items {
if !item.IsDir() {
if basename {
pathChan <- item.Name()
} else {
pathChan <- filepath.Join(startDir, item.Name())
}
continue
}
<-goroutineLimiter
walkerWG.Add(1)
go func(dir string) {
defer func() {
walkerWG.Done()
goroutineLimiter <- struct{}{}
}()
err = godirwalk.Walk(filepath.Join(startDir, dir), &godirwalk.Options{
Unsorted: true, // faster this way
Callback: func(osPathname string, de *godirwalk.Dirent) error {
if de.IsRegular() {
if basename {
pathChan <- de.Name()
} else {
pathChan <- osPathname
}
}
return nil
},
})
if err != nil {
logrus.Error(errors.FullTrace(err))
}
}(item.Name())
}
walkerWG.Wait()
close(pathChan)
pathWG.Wait()
return paths, nil
}
+42 -12
View File
@@ -1,27 +1,57 @@
package store
import "github.com/lbryio/lbry.go/extras/errors"
import (
"github.com/lbryio/reflector.go/shared"
// BlobStore is an interface with methods for consistently handling blob storage.
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
"github.com/spf13/viper"
)
// BlobStore is an interface for handling blob storage.
type BlobStore interface {
// Does blob exist in the store
// Name of blob store (useful for metrics)
Name() string
// Has Does blob exist in the store.
Has(hash string) (bool, error)
// Get the blob from the store
Get(hash string) ([]byte, error)
// Put the blob into the store
Put(hash string, blob []byte) error
// Put an SD blob into the store
PutSD(hash string, blob []byte) error
// Delete the blob from the store
// Get the blob from the store. Must return ErrBlobNotFound if blob is not in store.
Get(hash string) (stream.Blob, shared.BlobTrace, error)
// Put the blob into the store.
Put(hash string, blob stream.Blob) error
// PutSD an SD blob into the store.
PutSD(hash string, blob stream.Blob) error
// Delete the blob from the store.
Delete(hash string) error
// Shutdown the store gracefully
Shutdown()
}
// Blocklister is a store that supports blocking blobs to prevent their inclusion in the store.
type Blocklister interface {
// Block deletes the blob and prevents it from being uploaded in the future
Block(hash string) error
// Wants returns false if the hash exists or is blocked, true otherwise
// Wants returns false if the hash exists in store or is blocked, true otherwise
Wants(hash string) (bool, error)
}
//ErrBlobNotFound is a standard error when a blob is not found in the store.
// NeededBlobChecker can check which blobs from a known stream are not uploaded yet
type NeededBlobChecker interface {
MissingBlobsForKnownStream(string) ([]string, error)
}
// lister is a store that can list cached blobs. This is helpful when an overlay
// cache needs to track blob existence.
type lister interface {
list() ([]string, error)
}
// ErrBlobNotFound is a standard error when a blob is not found in the store.
var ErrBlobNotFound = errors.Base("blob not found")
var Factories = make(map[string]Factory)
func RegisterStore(name string, factory Factory) {
Factories[name] = factory
}
type Factory func(config *viper.Viper) (BlobStore, error)
+204
View File
@@ -0,0 +1,204 @@
package store
import (
"bytes"
"context"
"io"
"net"
"net/http"
"sync"
"time"
"github.com/lbryio/reflector.go/internal/metrics"
"github.com/lbryio/reflector.go/shared"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/stream"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
)
// UpstreamStore is a store that works on top of the HTTP protocol
type UpstreamStore struct {
upstream string
httpClient *http.Client
edgeToken string
name string
}
type UpstreamParams struct {
Name string `mapstructure:"name"`
Upstream string `mapstructure:"upstream"`
EdgeToken string `mapstructure:"edge_token"`
}
func NewUpstreamStore(params UpstreamParams) *UpstreamStore {
return &UpstreamStore{
upstream: params.Upstream,
httpClient: getClient(),
edgeToken: params.EdgeToken,
name: params.Name,
}
}
const nameUpstream = "upstream"
func UpstreamStoreFactory(config *viper.Viper) (BlobStore, error) {
var cfg UpstreamParams
err := config.Unmarshal(&cfg)
if err != nil {
return nil, errors.Err(err)
}
return NewUpstreamStore(cfg), nil
}
func init() {
RegisterStore(nameUpstream, UpstreamStoreFactory)
}
func (n *UpstreamStore) Name() string { return nameUpstream + "-" + n.name }
func (n *UpstreamStore) Has(hash string) (bool, error) {
url := n.upstream + "/blob?hash=" + hash
req, err := http.NewRequest("HEAD", url, nil)
if err != nil {
return false, errors.Err(err)
}
res, err := n.httpClient.Do(req)
if err != nil {
return false, errors.Err(err)
}
defer func() { _ = res.Body.Close() }()
if res.StatusCode == http.StatusNotFound {
return false, nil
}
if res.StatusCode == http.StatusNoContent {
return true, nil
}
var body []byte
if res.Body != nil {
body, _ = io.ReadAll(res.Body)
}
return false, errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
}
func (n *UpstreamStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
start := time.Now()
url := n.upstream + "/blob?hash=" + hash
if n.edgeToken != "" {
url += "&edge_token=" + n.edgeToken
}
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), n.Name()), errors.Err(err)
}
res, err := n.httpClient.Do(req)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), n.Name()), errors.Err(err)
}
defer func(Body io.ReadCloser) {
err := Body.Close()
if err != nil {
log.Errorf("Error closing response body in HTTP-GET: %s", err.Error())
}
}(res.Body)
viaHeader := res.Header.Get("Via")
var trace shared.BlobTrace
if viaHeader != "" {
parsedTrace, err := shared.Deserialize(viaHeader)
if err != nil {
return nil, shared.NewBlobTrace(time.Since(start), n.Name()), err
}
trace = *parsedTrace
} else {
trace = shared.NewBlobTrace(0, n.Name())
}
switch res.StatusCode {
case http.StatusNotFound:
return nil, trace.Stack(time.Since(start), n.Name()), ErrBlobNotFound
case http.StatusOK:
buffer := getBuffer()
defer putBuffer(buffer)
if _, err := io.Copy(buffer, res.Body); err != nil {
return nil, trace.Stack(time.Since(start), n.Name()), errors.Err(err)
}
blob := make([]byte, buffer.Len())
copy(blob, buffer.Bytes())
metrics.MtrInBytesUpstream.Add(float64(len(blob)))
return blob, trace.Stack(time.Since(start), n.Name()), nil
default:
body, _ := io.ReadAll(res.Body)
log.Warnf("Got status code %d (%s)", res.StatusCode, string(body))
return nil, trace.Stack(time.Since(start), n.Name()),
errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
}
}
func (n *UpstreamStore) Put(string, stream.Blob) error {
return shared.ErrNotImplemented
}
func (n *UpstreamStore) PutSD(string, stream.Blob) error {
return shared.ErrNotImplemented
}
func (n *UpstreamStore) Delete(string) error {
return shared.ErrNotImplemented
}
func (n *UpstreamStore) Shutdown() {}
// buffer pool to reduce GC
// https://www.captaincodeman.com/2017/06/02/golang-buffer-pool-gotcha
var buffers = sync.Pool{
// New is called when a new instance is needed
New: func() interface{} {
buf := make([]byte, 0, stream.MaxBlobSize)
return bytes.NewBuffer(buf)
},
}
// getBuffer fetches a buffer from the pool
func getBuffer() *bytes.Buffer {
return buffers.Get().(*bytes.Buffer)
}
// putBuffer returns a buffer to the pool
func putBuffer(buf *bytes.Buffer) {
buf.Reset()
buffers.Put(buf)
}
func dialContext(ctx context.Context, network, address string) (net.Conn, error) {
dialer := &net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}
return dialer.DialContext(ctx, network, address)
}
// getClient gets an http client that's customized to be more performant when dealing with blobs of 2MB in size (most of our blobs)
func getClient() *http.Client {
// Customize the Transport to have larger connection pool
defaultTransport := &http.Transport{
DialContext: dialContext,
ForceAttemptHTTP2: true,
MaxIdleConns: 100,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
DisableCompression: true,
MaxIdleConnsPerHost: 100,
ReadBufferSize: stream.MaxBlobSize + 1024*10, //add an extra few KBs to make sure it fits the extra information
}
return &http.Client{Transport: defaultTransport}
}
+110 -4
View File
@@ -1,5 +1,24 @@
package wallet
import (
"encoding/base64"
"encoding/hex"
"github.com/lbryio/chainquery/lbrycrd"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/schema/stake"
types "github.com/lbryio/types/v2/go"
"github.com/btcsuite/btcutil"
"github.com/golang/protobuf/proto"
"github.com/spf13/cast"
)
// Raw makes a raw wallet server request
func (n *Node) Raw(method string, params []string, v interface{}) error {
return n.request(method, params, v)
}
// ServerVersion returns the server's version.
// https://electrumx.readthedocs.io/en/latest/protocol-methods.html#server-version
func (n *Node) ServerVersion() (string, error) {
@@ -16,24 +35,56 @@ func (n *Node) ServerVersion() (string, error) {
return v, err
}
func (n *Node) Resolve(url string) (*types.Output, error) {
outputs := &types.Outputs{}
resp := &struct {
Result string `json:"result"`
}{}
err := n.request("blockchain.claimtrie.resolve", []string{url}, resp)
if err != nil {
return nil, err
}
b, err := base64.StdEncoding.DecodeString(resp.Result)
if err != nil {
return nil, errors.Err(err)
}
err = proto.Unmarshal(b, outputs)
if err != nil {
return nil, errors.Err(err)
}
if len(outputs.GetTxos()) != 1 {
return nil, errors.Err("expected 1 output, got " + cast.ToString(len(outputs.GetTxos())))
}
if e := outputs.GetTxos()[0].GetError(); e != nil {
return nil, errors.Err("%s: %s", e.GetCode(), e.GetText())
}
return outputs.GetTxos()[0], nil
}
type GetClaimsInTxResp struct {
Jsonrpc string `json:"jsonrpc"`
ID int `json:"id"`
Result []struct {
Name string `json:"name"`
ClaimID string `json:"claim_id"`
Txid string `json:"txid"`
Value string `json:"value"`
Address string `json:"address"`
Supports []interface{} `json:"supports"` // TODO: finish me
Nout int `json:"nout"`
Amount int `json:"amount"`
Depth int `json:"depth"`
Height int `json:"height"`
Value string `json:"value"`
ClaimSequence int `json:"claim_sequence"`
Address string `json:"address"`
Supports []interface{} `json:"supports"` // TODO: finish me
EffectiveAmount int `json:"effective_amount"`
ValidAtHeight int `json:"valid_at_height"`
} `json:"result"`
ID int `json:"id"`
}
func (n *Node) GetClaimsInTx(txid string) (*GetClaimsInTxResp, error) {
@@ -41,3 +92,58 @@ func (n *Node) GetClaimsInTx(txid string) (*GetClaimsInTxResp, error) {
err := n.request("blockchain.claimtrie.getclaimsintx", []string{txid}, &resp)
return &resp, err
}
func (n *Node) GetTx(txid string) (string, error) {
resp := &struct {
Result string `json:"result"`
}{}
err := n.request("blockchain.transaction.get", []string{txid}, resp)
if err != nil {
return "", err
}
return resp.Result, nil
}
func (n *Node) GetClaimInTx(txid string, nout int) (*types.Claim, error) {
hexTx, err := n.GetTx(txid)
if err != nil {
return nil, errors.Err(err)
}
rawTx, err := hex.DecodeString(hexTx)
if err != nil {
return nil, errors.Err(err)
}
tx, err := btcutil.NewTxFromBytes(rawTx)
if err != nil {
return nil, errors.Err(err)
}
if len(tx.MsgTx().TxOut) <= nout {
return nil, errors.Err("nout not found")
}
script := tx.MsgTx().TxOut[nout].PkScript
var value []byte
if lbrycrd.IsClaimNameScript(script) {
_, value, _, err = lbrycrd.ParseClaimNameScript(script)
} else if lbrycrd.IsClaimUpdateScript(script) {
_, _, value, _, err = lbrycrd.ParseClaimUpdateScript(script)
} else {
err = errors.Err("no claim found in output")
}
if err != nil {
return nil, errors.Err(err)
}
ch, err := stake.DecodeClaimBytes(value, "")
if err != nil {
return nil, errors.Err(err)
}
return ch.Claim, nil
}
+67 -31
View File
@@ -10,11 +10,11 @@ import (
"sync"
"time"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/extras/stop"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
log "github.com/sirupsen/logrus"
"github.com/uber-go/atomic"
"go.uber.org/atomic"
)
const (
@@ -30,22 +30,19 @@ var (
)
type response struct {
data []byte
err error
data []byte
}
type Node struct {
transport *TCPTransport
nextId atomic.Uint32
grp *stop.Group
handlersMu *sync.RWMutex
handlers map[uint32]chan response
transport *TCPTransport
grp *stop.Group
handlersMu *sync.RWMutex
handlers map[uint32]chan response
pushHandlersMu *sync.RWMutex
pushHandlers map[string][]chan response
timeout time.Duration
nextId atomic.Uint32
timeout time.Duration
}
// NewNode creates a new node.
@@ -63,7 +60,7 @@ func NewNode() *Node {
// Connect creates a new connection to the specified address.
func (n *Node) Connect(addrs []string, config *tls.Config) error {
if n.transport != nil {
return ErrNodeConnected
return errors.Err(ErrNodeConnected)
}
// shuffle addresses for load balancing
@@ -76,15 +73,18 @@ func (n *Node) Connect(addrs []string, config *tls.Config) error {
if err == nil {
break
}
if errors.Is(err, ErrTimeout) {
continue
}
if e, ok := err.(*net.OpError); ok && e.Err.Error() == "no such host" {
// net.errNoSuchHost is not exported, so we have to string-match
continue
}
return err
return errors.Err(err)
}
if n.transport == nil {
return ErrConnectFailed
return errors.Err(ErrConnectFailed)
}
log.Debugf("wallet connected to %s", n.transport.conn.RemoteAddr())
@@ -112,7 +112,13 @@ func (n *Node) Connect(addrs []string, config *tls.Config) error {
}
func (n *Node) Shutdown() {
var addr net.Addr
if n.transport != nil {
addr = n.transport.conn.RemoteAddr()
}
log.Debugf("shutting down wallet %s", addr)
n.grp.StopAndWait()
log.Debugf("wallet stopped")
}
func (n *Node) handleErrors() {
@@ -121,7 +127,7 @@ func (n *Node) handleErrors() {
case <-n.grp.Ch():
return
case err := <-n.transport.Errors():
n.err(err)
n.err(errors.Err(err))
}
}
}
@@ -129,32 +135,60 @@ func (n *Node) handleErrors() {
// err handles errors produced by the foreign node.
func (n *Node) err(err error) {
// TODO: Better error handling.
log.Error(err)
log.Error(errors.FullTrace(err))
}
// listen processes messages from the server.
func (n *Node) listen() {
for {
select {
case <-n.grp.Ch():
return
default:
}
select {
case <-n.grp.Ch():
return
case bytes := <-n.transport.Responses():
msg := &struct {
Id uint32 `json:"id"`
Method string `json:"method"`
Error struct {
Code int `json:"code"`
Message string `json:"message"`
Code int `json:"code"`
} `json:"error"`
Id uint32 `json:"id"`
}{}
if err := json.Unmarshal(bytes, msg); err != nil {
n.err(err)
continue
msg2 := &struct {
Method string `json:"method"`
Error struct {
Message struct {
Message string `json:"message"`
Code int `json:"code"`
} `json:"message"`
Code int `json:"code"`
} `json:"error"`
Id uint32 `json:"id"`
}{}
r := response{}
err := json.Unmarshal(bytes, msg)
if err != nil {
// try msg2, a hack around the weird error-in-error response we sometimes get from wallet server
// maybe that happens because the wallet server passes a lbrycrd error through to us?
if err2 := json.Unmarshal(bytes, msg2); err2 == nil {
err = nil
msg.Id = msg2.Id
msg.Method = msg2.Method
msg.Error = msg2.Error.Message
}
}
r := response{}
if len(msg.Error.Message) > 0 {
r.err = errors.Base("%d: %s", msg.Error.Code, msg.Error.Message)
if err != nil {
r.err = errors.Err(err)
n.err(r.err)
} else if len(msg.Error.Message) > 0 {
r.err = errors.Err("%d: %s", msg.Error.Code, msg.Error.Message)
} else {
r.data = bytes
}
@@ -194,9 +228,9 @@ func (n *Node) listen() {
// request makes a request to the server and unmarshals the response into v.
func (n *Node) request(method string, params []string, v interface{}) error {
msg := struct {
Id uint32 `json:"id"`
Method string `json:"method"`
Params []string `json:"params"`
Id uint32 `json:"id"`
}{
Id: n.nextId.Load(),
Method: method,
@@ -206,7 +240,7 @@ func (n *Node) request(method string, params []string, v interface{}) error {
bytes, err := json.Marshal(msg)
if err != nil {
return err
return errors.Err(err)
}
bytes = append(bytes, delimiter)
@@ -218,11 +252,13 @@ func (n *Node) request(method string, params []string, v interface{}) error {
err = n.transport.Send(bytes)
if err != nil {
return err
return errors.Err(err)
}
var r response
select {
case <-n.grp.Ch():
return nil
case r = <-c:
case <-time.After(n.timeout):
r = response{err: errors.Err(ErrTimeout)}
@@ -233,8 +269,8 @@ func (n *Node) request(method string, params []string, v interface{}) error {
n.handlersMu.Unlock()
if r.err != nil {
return r.err
return errors.Err(r.err)
}
return json.Unmarshal(r.data, v)
return errors.Err(json.Unmarshal(r.data, v))
}
+48 -12
View File
@@ -5,10 +5,14 @@ package wallet
import (
"bufio"
"crypto/tls"
"encoding/json"
"fmt"
"net"
"time"
"github.com/lbryio/lbry.go/extras/stop"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/v2/extras/stop"
log "github.com/sirupsen/logrus"
)
@@ -49,15 +53,31 @@ func NewTransport(addr string, config *tls.Config) (*TCPTransport, error) {
t.grp.Add(1)
go func() {
t.grp.Done()
defer t.grp.Done()
t.listen()
}()
err = t.test()
if err != nil {
t.grp.StopAndWait()
return nil, errors.Prefix(addr, err)
}
return t, nil
}
const delimiter = byte('\n')
func (t *TCPTransport) Send(body []byte) error {
log.Debugf("%s <- %s", t.conn.RemoteAddr(), body)
_, err := t.conn.Write(body)
return err
}
func (t *TCPTransport) Responses() <-chan []byte { return t.responses }
func (t *TCPTransport) Errors() <-chan error { return t.errors }
func (t *TCPTransport) Shutdown() { t.grp.StopAndWait() }
func (t *TCPTransport) listen() {
reader := bufio.NewReader(t.conn)
for {
@@ -73,12 +93,6 @@ func (t *TCPTransport) listen() {
}
}
func (t *TCPTransport) Send(body []byte) error {
log.Debugf("%s <- %s", t.conn.RemoteAddr(), body)
_, err := t.conn.Write(body)
return err
}
func (t *TCPTransport) error(err error) {
select {
case t.errors <- err:
@@ -86,11 +100,33 @@ func (t *TCPTransport) error(err error) {
}
}
func (t *TCPTransport) Responses() <-chan []byte { return t.responses }
func (t *TCPTransport) Errors() <-chan error { return t.errors }
func (t *TCPTransport) test() error {
err := t.Send([]byte(`{"id":1,"method":"server.version"}` + "\n"))
if err != nil {
return errors.Err(err)
}
func (t *TCPTransport) Shutdown() {
t.grp.StopAndWait()
var data []byte
select {
case data = <-t.Responses():
case <-time.Tick(1 * time.Second):
return errors.Err(ErrTimeout)
}
var response struct {
Error struct {
Message string `json:"message"`
} `json:"error"`
}
err = json.Unmarshal(data, &response)
if err != nil {
return errors.Err(err)
}
if response.Error.Message != "" {
return fmt.Errorf("%s", response.Error.Message)
}
return nil
}
func (t *TCPTransport) close() {